From 3e00979590bfbb7738f63284233e7c491053ffd3 Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Thu, 20 Aug 2026 11:42:59 +0200 Subject: [PATCH 01/22] feat(aggregation): address a grouping entity's resources through one model A grouping entity - a merged Area, a merged Function, a Component with members - has no owner of its own, and its resources come from members that do. Nothing resolved those members, so its collections were assembled from whatever the local walk happened to find. Members are now resolved through one walk of the declared tree, and every listed item names the members that provide it. An item more than one member provides is addressed `:`; the bare form is refused rather than run against whichever member was walked first. An item a single member provides keeps its bare id, which is what every current client sends. The specification drives two gateways over HTTP rather than reasoning about one. --- docs/api/rest.rst | 107 +++- src/ros2_medkit_gateway/README.md | 51 ++ .../core/http/member_qualified_id.hpp | 114 ++++ .../core/models/thread_safe_entity_cache.hpp | 50 +- .../include/ros2_medkit_gateway/dto/data.hpp | 22 +- .../ros2_medkit_gateway/dto/operations.hpp | 12 +- .../core/models/thread_safe_entity_cache.cpp | 164 ++++- .../src/core/openapi/route_registry.cpp | 8 +- .../src/http/handlers/data_handlers.cpp | 137 +++- .../src/http/handlers/operation_handlers.cpp | 242 +++++-- .../src/openapi/capability_generator.cpp | 55 +- .../test/test_entity_resource_model.cpp | 124 ++++ .../test/test_operation_handlers.cpp | 104 +++- .../CMakeLists.txt | 1 + .../test_grouping_entity_aggregation.test.py | 589 ++++++++++++++++++ 15 files changed, 1695 insertions(+), 85 deletions(-) create mode 100644 src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/http/member_qualified_id.hpp create mode 100644 src/ros2_medkit_integration_tests/test/features/test_grouping_entity_aggregation.test.py diff --git a/docs/api/rest.rst b/docs/api/rest.rst index 56c37ca1a..899219b57 100644 --- a/docs/api/rest.rst +++ b/docs/api/rest.rst @@ -608,10 +608,100 @@ Functions supported, and the vendor resource ``/x-medkit-graph`` exposes a function-scoped graph snapshot. See :ref:`sovd-compliance` for details. +.. _member-qualified-ids: + +Item Ids and Their Providers +---------------------------- + +An entity that draws its items from members - an Area, a Function, or a +Component with hosted apps - lists what its members provide. Each listed item +carries the members that contribute it in ``x-medkit.member_ids``, so a caller +can always see where an item came from. + +**Ids stay bare.** An item id is the ROS name the member itself uses: the topic +path for ``/data``, the service or action short name for ``/operations``. This +is the ordinary case and it does not change with aggregation - in runtime +discovery every App hangs off the single host Component, so almost every entity +draws from members. + +**Except when the id is ambiguous.** When more than one item in the merged +collection carries the same id, each of those copies is addressed with the +member that owns it:: + + : + +The split is on the first colon: an entity id is restricted to alphanumerics, +underscore and hyphen and so never contains one, while an item name can. + +Ambiguity is decided on the merged collection, after peer fan-out, because that +is where it becomes visible - two gateways each holding one ``calibrate`` both +consider it unique. In practice: + +- ``/operations`` - two members exposing one short name at different ROS paths + are two items with one id, so both are qualified + (``primary_calibration:calibrate``, ``peer_calibration:calibrate``). +- ``/data`` - a topic path names one topic however many members publish and + subscribe to it; those merge into a single item, so the bare path is kept and + every contributor is named in ``member_ids``. A path is qualified only when + two gateways each contribute an item under it. + +What this means for a request: + +- A bare id that names one item works, on every route. Every client that sends + the ROS short name keeps working, and it is what the generated OpenAPI + document describes. +- ``POST /{entity}/operations/{id}/executions`` with a bare id that more than + one member provides is refused with ``400 invalid-request``, naming the + qualified form and listing the members in ``parameters.member_ids``. Running + whichever member was walked first without saying which one ran is the defect + this removes. +- A qualified id is accepted on the single-item routes. A member half that + names no member of the entity is ``404``, and so is an item half that member + does not provide - which is what tells an absent item apart from an item that + exists and currently carries no data. +- Reads are permissive: ``GET`` of a bare id returns the first match rather + than refusing, which is the behaviour every existing client depends on. + +Ambiguity is a property of the declared tree, not of who is reachable right +now. A peer's declared operations are held locally, so the same request gets +the same answer whether or not that peer is currently answering, and deciding +it costs no network call. It cannot be changed by anything a client sends. + +.. _retained-entities: + +Entities of a Silent Peer +~~~~~~~~~~~~~~~~~~~~~~~~~ + +When a peer stops answering, the entities it **declared in its manifest** are +retained and marked unavailable; the ones it merely discovered from its live +ROS graph disappear, because nothing can observe that graph any more. A +retained entity: + +- stays listed and stays addressable, so the tree does not change shape when a + link drops; +- reports ``x-medkit.available: false`` and ``x-medkit.is_online: false``; +- answers any request addressed to it with ``504`` and the SOVD standard code + ``not-responding``, naming the member - rather than being forwarded to the + silent peer and surfacing as a ``502``, or falling through to a local read + that returns ``200`` with an empty body. + +``/health`` is unchanged: the peer itself is listed there with +``status: "offline"``. Availability of an entity and health of a peer are +separate questions and are reported separately. + +.. note:: + + ``/configurations`` predates this rule and keeps its own: on an entity whose + parameters come from more than one node, **every** parameter id is + ``:``, and a bare id is refused on write. Its items carry + ``x-medkit.source`` (a single app id), not ``member_ids``. See + :ref:`configuration-endpoints`. + Data Endpoints -------------- -Read and publish data from ROS 2 topics. +Read and publish data from ROS 2 topics. Item ids follow +:ref:`member-qualified-ids`. ``GET /api/v1/components/{id}/data`` Read all topic data from an entity. @@ -664,7 +754,9 @@ Read and publish data from ROS 2 topics. Operations Endpoints -------------------- -Execute ROS 2 services and actions. +Execute ROS 2 services and actions. Operation ids follow +:ref:`member-qualified-ids`: a short name that only one member exposes is used +bare, and one that several expose is addressed ``:``. List Operations ~~~~~~~~~~~~~~~ @@ -904,11 +996,22 @@ Request Transition curl -X PUT http://localhost:8080/api/v1/apps/temp_sensor/status/restart +.. _configuration-endpoints: + Configurations Endpoints ------------------------ Manage ROS 2 node parameters. +.. note:: + + Parameter ids do not follow :ref:`member-qualified-ids`. On an entity backed + by more than one node every parameter id is ``:``, + whether or not that name is ambiguous, and a bare id is refused on ``PUT`` + and ``DELETE`` with ``400 invalid-request``. ``GET`` accepts the bare form + and returns the first node that answers. Items carry the owning app in + ``x-medkit.source``. + ``GET /api/v1/components/{id}/configurations`` List all parameters for an entity. diff --git a/src/ros2_medkit_gateway/README.md b/src/ros2_medkit_gateway/README.md index e5be493a9..ecc1b7588 100644 --- a/src/ros2_medkit_gateway/README.md +++ b/src/ros2_medkit_gateway/README.md @@ -257,6 +257,57 @@ curl http://localhost:8080/api/v1/areas/nonexistent/components - Hierarchical navigation (select area → view its components) - Area-specific health checks +### Item Ids and Their Providers + +An entity that draws items from members - an Area, a Function, or a Component +with hosted apps - names the contributing members of every listed item in +`x-medkit.member_ids`. + +Ids stay bare. An item id is the ROS name its member uses: the topic path for +`/data`, the service or action short name for `/operations`. That does not +change with aggregation, which is the ordinary case - in runtime discovery +every App hangs off the single host Component. + +An id is qualified only when it is ambiguous, meaning more than one item in the +merged collection carries it: + +``` +: +``` + +Ambiguity is decided after the peer fan-out, since neither gateway can see the +collision alone. Two members exposing the operation short name `calibrate` at +different ROS paths are two items with one id, so both are qualified. A topic +path names one topic however many members publish and subscribe to it, so it +stays bare and lists its contributors in `member_ids`; it is qualified only if +two gateways each contribute an item under that path. + +- A bare id that names one item works on every route, which is what the web UI, + the Foxglove panel, the MCP tools and the generated OpenAPI document all send. +- `POST /{entity}/operations/{id}/executions` with a bare id several members + provide is `400 invalid-request`, naming the qualified form and the members. +- A qualified id is accepted on single-item routes; an unknown member half, or + an item half that member does not provide, is `404` - which is what tells an + absent item apart from one that exists and carries no data. +- `GET` of a bare id stays permissive and returns the first match. + +Ambiguity is decided from the declared tree, which includes a peer's declared +operations held locally. The answer therefore does not change with who is +reachable, costs no network call, and cannot be altered by a client-supplied +header. + +When a peer stops answering, the entities it declared in its manifest are +retained and marked unavailable (`x-medkit.available: false`, +`x-medkit.is_online: false`); the ones it only discovered at runtime disappear. +A request addressed to a retained entity answers `504 not-responding` naming +the member, instead of being forwarded to the silent peer as a `502`. `/health` +still reports the peer itself as `offline` - entity availability and peer +health are separate questions. + +`/configurations` predates this rule and keeps its own: on a multi-node entity +every parameter id is `:`, a bare id is refused on write, +and items carry `x-medkit.source` rather than `member_ids`. + ### Component Data Read Endpoints #### GET /api/v1/components/{component_id}/data diff --git a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/http/member_qualified_id.hpp b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/http/member_qualified_id.hpp new file mode 100644 index 000000000..ecb1762a1 --- /dev/null +++ b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/http/member_qualified_id.hpp @@ -0,0 +1,114 @@ +// Copyright 2026 bburda +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include + +namespace ros2_medkit_gateway { +namespace http { + +/** + * @brief Addressing for an item whose wire id more than one member carries. + * + * Qualification follows AMBIGUITY, not aggregation. An entity that draws items + * from members is the ordinary case, not the exception - in runtime discovery + * every App hangs off the single host Component and namespace Functions are on + * by default - so qualifying every id there would rename the items of the most + * used entity in the product and refuse requests every current client sends. + * + * An id is ambiguous when more than one item in the merged collection carries + * it: two members exposing the operation short name `calibrate` at different + * ROS paths are two items with one id, and a caller holding that id cannot say + * which one it meant. Those copies are addressed `:`. An id only + * one item carries names one thing already and is left alone. + * + * A topic path is not ambiguous merely because several members publish and + * subscribe to it: that is still one topic, merged into one item, and the bare + * path addresses it exactly. It becomes ambiguous only if two gateways each + * contribute an item under the same path. + */ +struct MemberQualifiedId { + std::string member_id; ///< Owning member; empty when the id carries no member half. + std::string item_id; ///< The item as its owning member names it. + bool has_member{false}; ///< Whether the id carried a member half. +}; + +/** + * @brief Split `id` into member and item halves at the FIRST colon. + * + * The first colon is the separator because an entity id is restricted to + * alphanumerics, underscore and hyphen and so can never contain one, while an + * item name can - a ROS 2 parameter name, for instance. + * + * Splitting only happens where a member half can mean something. On an entity + * with no members a colon is an ordinary character of the item name, and + * treating it as a separator would make the item addressable under a name + * nothing exposes. + */ +inline MemberQualifiedId parse_member_qualified_id(const std::string & id, bool member_half_possible) { + MemberQualifiedId parsed; + parsed.item_id = id; + + const auto colon_pos = id.find(':'); + if (colon_pos != std::string::npos && member_half_possible) { + parsed.member_id = id.substr(0, colon_pos); + parsed.item_id = id.substr(colon_pos + 1); + parsed.has_member = true; + } + return parsed; +} + +/// The id that addresses `item_id` as the copy owned by `member_id`. +inline std::string make_member_qualified_id(const std::string & member_id, const std::string & item_id) { + return member_id + ":" + item_id; +} + +/** + * @brief Rewrite the id of every item whose id another item in `items` shares. + * + * Runs on the merged collection - local items plus whatever the peer fan-out + * contributed - because that is the first point at which a duplicate is + * visible. Neither gateway can see the collision on its own: each holds one + * `calibrate` and considers it unique. + * + * `member_ids_of` returns the item's contributing members, or nullptr when it + * names none. An item that names no member, or names several, is left bare: a + * qualifier picked from an ambiguous set would be a guess, and a guess here + * reproduces the defect the qualified form exists to remove. + */ +template +void qualify_ambiguous_ids(std::vector & items, MemberIdsOf member_ids_of) { + std::unordered_map id_counts; + for (const auto & item : items) { + ++id_counts[item.id]; + } + + for (auto & item : items) { + auto count = id_counts.find(item.id); + if (count == id_counts.end() || count->second < 2) { + continue; + } + const std::vector * members = member_ids_of(item); + if (members == nullptr || members->size() != 1) { + continue; + } + item.id = make_member_qualified_id(members->front(), item.id); + } +} + +} // namespace http +} // namespace ros2_medkit_gateway diff --git a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/models/thread_safe_entity_cache.hpp b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/models/thread_safe_entity_cache.hpp index 9bf53b2d3..04d801939 100644 --- a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/models/thread_safe_entity_cache.hpp +++ b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/models/thread_safe_entity_cache.hpp @@ -59,8 +59,20 @@ struct TopicData { struct AggregatedData { std::vector topics; std::vector source_ids; ///< Entity IDs that contributed - std::string aggregation_level; ///< "app" | "component" | "area" | "function" - bool is_aggregated{false}; ///< true if collected from sub-entities + /// Contributing members per topic name, for the items in `topics`. + /// + /// `source_ids` says which members contributed something; it does not say + /// which member contributed WHICH item, and a caller holding one item id + /// cannot recover the owner from it. A grouping that lists an item it cannot + /// then address is the defect this exists to close. + /// + /// A list rather than one id, because a topic legitimately has more than one + /// contributor: a topic published by one member and subscribed by another is + /// merged into a single item with direction "both", so recording only the + /// first would name a publisher for an item a subscriber also owns. + std::unordered_map> owners_by_topic; + std::string aggregation_level; ///< "app" | "component" | "area" | "function" + bool is_aggregated{false}; ///< true if collected from sub-entities bool empty() const { return topics.empty(); @@ -77,8 +89,17 @@ struct AggregatedOperations { std::vector services; std::vector actions; std::vector source_ids; ///< Entity IDs that contributed - std::string aggregation_level; ///< "app" | "component" | "area" | "function" - bool is_aggregated{false}; ///< true if collected from sub-entities + /// Owning member per full ROS path, for the items in `services`/`actions`. + /// + /// See AggregatedData::owner_by_topic for why the collection-level + /// `source_ids` is not enough. Keyed by full path, matching the existing + /// deduplication: two members exposing the same full path are one item with + /// one owner, while two members exposing the same SHORT name at different + /// paths stay two items with an owner each - which is what lets a caller see + /// that the short name, the id the HTTP API uses, is ambiguous. + std::unordered_map owner_by_path; + std::string aggregation_level; ///< "app" | "component" | "area" | "function" + bool is_aggregated{false}; ///< true if collected from sub-entities bool empty() const { return services.empty() && actions.empty(); @@ -329,6 +350,25 @@ class ThreadSafeEntityCache { */ std::vector get_subareas(const std::string & area_id) const; + /// Entity ids that contribute the resources of `entity_id`. + /// + /// A grouping entity - an Area, a Function, or a Component with children - + /// has no resources of its own beyond what its members provide, so resolving + /// an item to its owner and listing the grouping's items must walk the same + /// graph. This is that walk, and it is the only one: the per-collection walks + /// that predate it disagree about subareas, about Function hosts that are + /// Components, and about whether a parent Component has children at all. + /// + /// A Component is included in its own member list because, unlike an Area or + /// a Function, it can expose services and actions directly. + /// + /// An App is a leaf and returns just itself, so a caller does not have to + /// special-case the non-grouping kinds. + /// + /// Cycles in the parent graph terminate: parents are only checked for + /// existence when a manifest is validated, not for acyclicity. + std::vector get_members(SovdEntityType type, const std::string & entity_id) const; + // ========================================================================= // Aggregation methods (uses relationship indexes) // ========================================================================= @@ -522,9 +562,11 @@ class ThreadSafeEntityCache { // Relationship indexes (parent ID -> child slot ids) FlatHashMap> component_to_apps_; + FlatHashMap> component_to_subcomponents_; FlatHashMap> area_to_components_; FlatHashMap> area_to_subareas_; FlatHashMap> function_to_apps_; + FlatHashMap> function_to_components_; // Operation index (operation full_path -> owning entity) FlatHashMap operation_index_; diff --git a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/dto/data.hpp b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/dto/data.hpp index 7a0275e06..511f0e8e0 100644 --- a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/dto/data.hpp +++ b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/dto/data.hpp @@ -69,16 +69,24 @@ struct XMedkitDataItem { std::optional timestamp; // sample timestamp in ns (read responses) std::optional publisher_count; // publisher count at sample time (read responses) std::optional subscriber_count; // subscriber count at sample time (read responses) + /// Members of the grouping that contribute this item. + /// + /// Absent on a leaf entity, where the entity is the only contributor. + /// A list rather than one id because an item can genuinely have several: + /// a topic published by one member and subscribed by another is merged + /// into a single item, and two members can expose the same operation + /// short name. More than one entry is what makes the bare item id + /// ambiguous for addressing. + std::optional> member_ids; }; template <> -inline constexpr auto dto_fields = - std::make_tuple(field("ros2", &XMedkitDataItem::ros2), field("type_info", &XMedkitDataItem::type_info), - field("entity_id", &XMedkitDataItem::entity_id), field("status", &XMedkitDataItem::status), - field("publisher_created", &XMedkitDataItem::publisher_created), - field("timestamp", &XMedkitDataItem::timestamp), - field("publisher_count", &XMedkitDataItem::publisher_count), - field("subscriber_count", &XMedkitDataItem::subscriber_count)); +inline constexpr auto dto_fields = std::make_tuple( + field("ros2", &XMedkitDataItem::ros2), field("type_info", &XMedkitDataItem::type_info), + field("entity_id", &XMedkitDataItem::entity_id), field("status", &XMedkitDataItem::status), + field("publisher_created", &XMedkitDataItem::publisher_created), field("timestamp", &XMedkitDataItem::timestamp), + field("publisher_count", &XMedkitDataItem::publisher_count), + field("subscriber_count", &XMedkitDataItem::subscriber_count), field("member_ids", &XMedkitDataItem::member_ids)); template <> inline constexpr std::string_view dto_name = "XMedkitDataItem"; diff --git a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/dto/operations.hpp b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/dto/operations.hpp index fe6d8fe93..a49583b44 100644 --- a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/dto/operations.hpp +++ b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/dto/operations.hpp @@ -54,13 +54,23 @@ struct XMedkitOperationItem { std::optional entity_id; std::optional source; std::optional type_info; // free-form: dynamic ROS IDL schemas + /// Members of the grouping that contribute this item. + /// + /// Absent on a leaf entity, where the entity is the only contributor. + /// A list rather than one id because an item can genuinely have several: + /// a topic published by one member and subscribed by another is merged + /// into a single item, and two members can expose the same operation + /// short name. More than one entry is what makes the bare item id + /// ambiguous for addressing. + std::optional> member_ids; }; template <> inline constexpr auto dto_fields = std::make_tuple(field("ros2", &XMedkitOperationItem::ros2), field("entity_id", &XMedkitOperationItem::entity_id), field("source", &XMedkitOperationItem::source), - field("type_info", &XMedkitOperationItem::type_info)); + field("type_info", &XMedkitOperationItem::type_info), + field("member_ids", &XMedkitOperationItem::member_ids)); template <> inline constexpr std::string_view dto_name = "XMedkitOperationItem"; diff --git a/src/ros2_medkit_gateway/src/core/models/thread_safe_entity_cache.cpp b/src/ros2_medkit_gateway/src/core/models/thread_safe_entity_cache.cpp index d30bd5959..39c58e94c 100644 --- a/src/ros2_medkit_gateway/src/core/models/thread_safe_entity_cache.cpp +++ b/src/ros2_medkit_gateway/src/core/models/thread_safe_entity_cache.cpp @@ -15,8 +15,10 @@ #include "ros2_medkit_gateway/core/models/thread_safe_entity_cache.hpp" #include +#include #include #include +#include #include namespace ros2_medkit_gateway { @@ -118,9 +120,9 @@ bool ThreadSafeEntityCache::patch_map(FlatHashMap & ma void ThreadSafeEntityCache::refresh_grew() { const bool any_grew = areas_.grew() || components_.grew() || apps_.grew() || functions_.grew() || area_index_.grew() || component_index_.grew() || app_index_.grew() || function_index_.grew() || - component_to_apps_.grew() || area_to_components_.grew() || area_to_subareas_.grew() || - function_to_apps_.grew() || operation_index_.grew() || topic_type_cache_.grew() || - node_to_app_.grew(); + component_to_apps_.grew() || component_to_subcomponents_.grew() || area_to_components_.grew() || + area_to_subareas_.grew() || function_to_apps_.grew() || function_to_components_.grew() || + operation_index_.grew() || topic_type_cache_.grew() || node_to_app_.grew(); if (any_grew) { grew_ = true; ++overflow_count_; @@ -135,9 +137,11 @@ void ThreadSafeEntityCache::refresh_grew() { app_index_.clear_grew(); function_index_.clear_grew(); component_to_apps_.clear_grew(); + component_to_subcomponents_.clear_grew(); area_to_components_.clear_grew(); area_to_subareas_.clear_grew(); function_to_apps_.clear_grew(); + function_to_components_.clear_grew(); operation_index_.clear_grew(); topic_type_cache_.clear_grew(); node_to_app_.clear_grew(); @@ -172,9 +176,11 @@ void ThreadSafeEntityCache::reserve(size_t capacity) { function_index_.reserve(capacity); component_to_apps_.reserve(capacity); + component_to_subcomponents_.reserve(capacity); area_to_components_.reserve(capacity); area_to_subareas_.reserve(capacity); function_to_apps_.reserve(capacity); + function_to_components_.reserve(capacity); operation_index_.reserve(capacity); topic_type_cache_.reserve(capacity); @@ -958,9 +964,11 @@ uint64_t ThreadSafeEntityCache::generation() const { void ThreadSafeEntityCache::rebuild_relationship_indexes() { component_to_apps_.reset(); + component_to_subcomponents_.reset(); area_to_components_.reset(); area_to_subareas_.reset(); function_to_apps_.reset(); + function_to_components_.reset(); // Build component_to_apps from apps' component_id apps_.for_each_live([&](uint32_t slot, const App & app) { @@ -976,6 +984,19 @@ void ThreadSafeEntityCache::rebuild_relationship_indexes() { } }); + // Build component_to_subcomponents from components' parent_component_id. + // + // Nothing indexed this direction before, so a hierarchical parent Component + // could not have its children enumerated from the cache at all: the only + // child lookups were linear scans in the discovery layer. Member resolution + // needs it, because a parent Component is a grouping whose resources come + // from its children. + components_.for_each_live([&](uint32_t slot, const Component & comp) { + if (!comp.parent_component_id.empty()) { + component_to_subcomponents_.get_or_create(comp.parent_component_id).push_back(slot); + } + }); + // Build area_to_subareas from areas' parent_area_id areas_.for_each_live([&](uint32_t slot, const Area & area) { if (!area.parent_area_id.empty()) { @@ -983,19 +1004,130 @@ void ThreadSafeEntityCache::rebuild_relationship_indexes() { } }); - // Build function_to_apps (functions have a hosts field which is a vector of app IDs). - // Function.hosts may also name Components - host ids that do not resolve to an - // app are silently dropped (preserving the original behaviour). + // Build function_to_apps and function_to_components from Function::hosts. + // + // `hosts` may name either an App or a Component. Only the App half was + // indexed, and a host id that resolved to a Component was dropped without a + // word, so a Function hosting a Component reported none of that Component's + // resources. Both halves are indexed now; a host id that resolves to neither + // is still skipped, because there is nothing to point at. functions_.for_each_live([&](uint32_t, const Function & func) { - for (const auto & app_id : func.hosts) { - const uint32_t * app_slot = app_index_.find(app_id); + for (const auto & host_id : func.hosts) { + const uint32_t * app_slot = app_index_.find(host_id); if (app_slot && apps_.is_live(*app_slot)) { function_to_apps_.get_or_create(func.id).push_back(*app_slot); + continue; + } + const uint32_t * comp_slot = component_index_.find(host_id); + if (comp_slot && components_.is_live(*comp_slot)) { + function_to_components_.get_or_create(func.id).push_back(*comp_slot); } } }); } +std::vector ThreadSafeEntityCache::get_members(SovdEntityType type, const std::string & entity_id) const { + std::shared_lock lock(mutex_); + std::vector members; + std::unordered_set seen; + + // One traversal for every grouping kind. Before this there were several, and + // they disagreed: Area operations did not descend into subareas, Function + // walks dropped Component hosts, and a parent Component had no walk at all. + // A caller that resolves an item to a member and a caller that lists the + // grouping's items have to agree, or the aggregator advertises what it will + // not serve. + // + // `seen` is not defensive bookkeeping. Component-parent cycles are known to + // occur - the aggregation classifier handles them explicitly - and the + // manifest validator only checks that a parent exists, not that the graph is + // acyclic. + const auto add = [&](const std::string & id) { + if (!id.empty() && seen.insert(id).second) { + members.push_back(id); + } + }; + + // Declared before the recursive lambdas that call each other. + std::function collect_component; + std::function collect_area; + + collect_component = [&](const std::string & component_id) { + add(component_id); + if (const std::vector * app_slots = component_to_apps_.find(component_id)) { + for (uint32_t slot : *app_slots) { + if (apps_.is_live(slot)) { + add(apps_[slot].id); + } + } + } + if (const std::vector * child_slots = component_to_subcomponents_.find(component_id)) { + for (uint32_t slot : *child_slots) { + if (components_.is_live(slot) && !seen.count(components_[slot].id)) { + collect_component(components_[slot].id); + } + } + } + }; + + collect_area = [&](const std::string & area_id) { + if (const std::vector * comp_slots = area_to_components_.find(area_id)) { + for (uint32_t slot : *comp_slots) { + if (components_.is_live(slot)) { + collect_component(components_[slot].id); + } + } + } + if (const std::vector * sub_slots = area_to_subareas_.find(area_id)) { + for (uint32_t slot : *sub_slots) { + if (areas_.is_live(slot) && seen.insert("area:" + areas_[slot].id).second) { + collect_area(areas_[slot].id); + } + } + } + }; + + switch (type) { + case SovdEntityType::AREA: + seen.insert("area:" + entity_id); + collect_area(entity_id); + break; + case SovdEntityType::COMPONENT: + // The Component itself is a member: unlike an Area or a Function it can + // own services and actions of its own, and the operations aggregation + // already counts those before its Apps. + collect_component(entity_id); + break; + case SovdEntityType::FUNCTION: + if (const std::vector * app_slots = function_to_apps_.find(entity_id)) { + for (uint32_t slot : *app_slots) { + if (apps_.is_live(slot)) { + add(apps_[slot].id); + } + } + } + if (const std::vector * comp_slots = function_to_components_.find(entity_id)) { + for (uint32_t slot : *comp_slots) { + if (components_.is_live(slot)) { + collect_component(components_[slot].id); + } + } + } + break; + case SovdEntityType::APP: + case SovdEntityType::SERVER: + case SovdEntityType::UNKNOWN: + // A leaf is its own only member, so a caller that resolves an item + // against one needs no traversal and no special case of its own. Listed + // rather than defaulted because the build treats an unhandled enumerator + // as an error, which is what makes a new entity type surface here. + add(entity_id); + break; + } + + return members; +} + void ThreadSafeEntityCache::rebuild_operation_index() { operation_index_.reset(); @@ -1037,11 +1169,13 @@ void ThreadSafeEntityCache::collect_operations_from_apps(const std::vector read_topic_id(const http::TypedRequest & re return tl::make_unexpected(make_error(400, ERR_INVALID_REQUEST, "Invalid request")); } +/// A data item is addressed by its full ROS topic path; the leading slash is +/// optional on the wire because the route captures a percent-decoded segment. +std::string to_full_topic_path(const std::string & topic_name) { + if (topic_name.empty() || topic_name.front() == '/') { + return topic_name; + } + return "/" + topic_name; +} + +/// What one addressed data item resolves to: the ROS topic to act on, and the +/// id to echo back to the caller. +struct AddressedDataItem { + std::string full_topic_path; + std::string item_id; +}; + +/// Resolve the id in the route against the entity, for reads and writes alike. +/// +/// A qualified id is answered exactly, because the member set and what each +/// member contributes are both known here: an id naming an unknown member, or +/// an item that member does not provide, is a miss. Without that check the +/// gateway samples the local graph, finds nothing, and returns 200 with an +/// empty body and status `metadata_only` - a typo reported as success. +/// +/// A ROS topic name cannot contain a colon, so one in the id can only be the +/// member separator. Building the member set is not free, so the cache is only +/// consulted for an id that carries one; a bare id addresses a topic path, +/// which names one topic on its own and keeps its existing behaviour. +tl::expected +address_data_item(const ThreadSafeEntityCache & cache, const std::string & entity_id, const std::string & topic_name) { + AddressedDataItem addressed; + addressed.full_topic_path = to_full_topic_path(topic_name); + addressed.item_id = addressed.full_topic_path; + + if (topic_name.find(':') == std::string::npos) { + return addressed; + } + + auto aggregated = cache.get_entity_data(entity_id); + auto parsed = http::parse_member_qualified_id(topic_name, aggregated.is_aggregated); + if (!parsed.has_member) { + return addressed; + } + + if (std::find(aggregated.source_ids.begin(), aggregated.source_ids.end(), parsed.member_id) == + aggregated.source_ids.end()) { + return tl::make_unexpected( + make_error(404, ERR_RESOURCE_NOT_FOUND, "Member not found in entity", + json{{"entity_id", entity_id}, {"id", topic_name}, {"member_id", parsed.member_id}})); + } + + // A member retained while its gateway is silent stays addressable and says + // why it cannot answer, rather than falling through to a sample of the local + // graph that comes back empty and reads as success. + if (auto app = cache.get_app(parsed.member_id); app && !app->available) { + return tl::make_unexpected( + make_error(504, ERR_NOT_RESPONDING, "Member '" + parsed.member_id + "' is not available", + json{{"details", + "The gateway contributing this member is not answering; it is retained from its " + "last known declaration"}, + {"entity_id", entity_id}, + {"id", topic_name}, + {"member_id", parsed.member_id}})); + } + + addressed.full_topic_path = to_full_topic_path(parsed.item_id); + auto owners = aggregated.owners_by_topic.find(addressed.full_topic_path); + if (owners == aggregated.owners_by_topic.end() || + std::find(owners->second.begin(), owners->second.end(), parsed.member_id) == owners->second.end()) { + return tl::make_unexpected(make_error(404, ERR_RESOURCE_NOT_FOUND, "Data item not provided by member", + json{{"entity_id", entity_id}, + {"id", topic_name}, + {"member_id", parsed.member_id}, + {"topic_name", addressed.full_topic_path}})); + } + + addressed.item_id = http::make_member_qualified_id(parsed.member_id, addressed.full_topic_path); + return addressed; +} + /// Build the typed x-medkit per-item payload for the list endpoint. dto::XMedkitDataItem build_list_item_xmedkit(const std::string & topic_name, const std::string & direction, const std::string & topic_type, @@ -127,11 +208,15 @@ void apply_fan_out_observability(dto::DataListXMedkit & xm, const FanOutResult DataHandlers::list_data(const http::TypedReque di.category = "currentData"; const std::string topic_type = cache.get_topic_type(topic.name); di.x_medkit = build_list_item_xmedkit(topic.name, topic.direction, topic_type, type_introspection); + // Attribute the item to the members that contribute it. Only a grouping + // has members to name; on a leaf the entity is the sole contributor and + // the field stays absent rather than repeating the entity id. + if (auto owners = aggregated.owners_by_topic.find(topic.name); + owners != aggregated.owners_by_topic.end() && aggregated.is_aggregated) { + di.x_medkit->member_ids = owners->second; + } response.items.push_back(std::move(di)); } @@ -307,6 +400,15 @@ http::Result DataHandlers::list_data(const http::TypedReque response.items.push_back(std::move(item)); } + // A topic path names one topic however many members publish and subscribe + // to it - those merge into a single item, whose contributors are already + // named in member_ids - so nothing here is qualified in the ordinary case. + // Two gateways each contributing an item under one path is the case that + // is genuinely ambiguous, and it is the case this catches. + http::qualify_ambiguous_ids(response.items, [](const dto::DataItem & item) { + return item.x_medkit.has_value() && item.x_medkit->member_ids.has_value() ? &*item.x_medkit->member_ids : nullptr; + }); + dto::DataListXMedkit xm; xm.entity_id = entity_id; if (aggregated.is_aggregated) { @@ -383,13 +485,11 @@ http::Result DataHandlers::get_data_item(const http::TypedReques } try { - // Determine the full ROS topic path. - std::string full_topic_path; - if (topic_name.empty() || topic_name[0] == '/') { - full_topic_path = topic_name; - } else { - full_topic_path = "/" + topic_name; + auto addressed = address_data_item(ctx_.node()->get_thread_safe_cache(), entity_id, topic_name); + if (!addressed) { + return tl::make_unexpected(addressed.error()); } + const std::string & full_topic_path = addressed->full_topic_path; // Sampling goes through the pool-backed TopicDataProvider (issue #375 race // fix). The provider is configured in main() before serving traffic. @@ -417,7 +517,7 @@ http::Result DataHandlers::get_data_item(const http::TypedReques } auto type_introspection = data_access_mgr->get_type_introspection(); - return build_read_response(full_topic_path, *r, entity_id, type_introspection); + return build_read_response(addressed->item_id, full_topic_path, *r, entity_id, type_introspection); } catch (const TopicNotAvailableException & e) { RCLCPP_DEBUG(HandlerContext::logger(), "Topic not available for entity '%s', topic '%s': %s", entity_id.c_str(), topic_name.c_str(), e.what()); @@ -560,16 +660,17 @@ http::Result DataHandlers::put_data_item(const http::TypedReques json{{"details", "Message type should be in format: package/msg/Type"}, {"type", msg_type}})); } - // Build full topic path (mirror GET logic: only prefix '/' when needed). - std::string full_topic_path = topic_name; - if (!full_topic_path.empty() && full_topic_path.front() != '/') { - full_topic_path = "/" + full_topic_path; + // A write addresses the same item a read does, so it resolves the same way. + auto addressed = address_data_item(ctx_.node()->get_thread_safe_cache(), entity_id, topic_name); + if (!addressed) { + return tl::make_unexpected(addressed.error()); } + const std::string & full_topic_path = addressed->full_topic_path; // Publish data using DataAccessManager. auto data_access_mgr = ctx_.node()->get_data_access_manager(); json publish_result = data_access_mgr->publish_to_topic(full_topic_path, msg_type, data); - return build_write_response(full_topic_path, msg_type, entity_id, data, publish_result); + return build_write_response(addressed->item_id, full_topic_path, msg_type, entity_id, data, publish_result); } catch (const std::exception & e) { RCLCPP_ERROR(HandlerContext::logger(), "Error in put_data_item for entity '%s', topic '%s': %s", entity_id.c_str(), topic_name.c_str(), e.what()); diff --git a/src/ros2_medkit_gateway/src/http/handlers/operation_handlers.cpp b/src/ros2_medkit_gateway/src/http/handlers/operation_handlers.cpp index 23ce7c0af..05df3768a 100644 --- a/src/ros2_medkit_gateway/src/http/handlers/operation_handlers.cpp +++ b/src/ros2_medkit_gateway/src/http/handlers/operation_handlers.cpp @@ -14,9 +14,11 @@ #include "ros2_medkit_gateway/core/http/handlers/operation_handlers.hpp" +#include #include #include #include +#include #include #include #include @@ -28,6 +30,7 @@ #include "ros2_medkit_gateway/core/http/error_codes.hpp" #include "ros2_medkit_gateway/core/http/fan_out_helpers.hpp" #include "ros2_medkit_gateway/core/http/http_utils.hpp" +#include "ros2_medkit_gateway/core/http/member_qualified_id.hpp" #include "ros2_medkit_gateway/core/managers/operation_manager.hpp" #include "ros2_medkit_gateway/core/plugins/plugin_manager.hpp" #include "ros2_medkit_gateway/core/providers/operation_provider.hpp" @@ -123,6 +126,107 @@ tl::expected resolve_entity_operations(const ThreadS } } +/// True when `member_id` is one of the members that contributed to `ops`. +/// +/// `source_ids` also carries the aggregating entity's own id, which is +/// harmless here: an id naming the entity itself resolves against exactly the +/// operations the entity contributed under its own name. +bool names_a_member(const AggregatedOperations & ops, const std::string & member_id) { + return std::find(ops.source_ids.begin(), ops.source_ids.end(), member_id) != ops.source_ids.end(); +} + +/// The operation an item id resolves to inside one entity's operations. +struct ResolvedOperation { + std::optional service; + std::optional action; + + bool found() const { + return service.has_value() || action.has_value(); + } +}; + +/// Resolve `parsed` against the entity's operations. +/// +/// A member half selects among same-named operations using the owner recorded +/// per full ROS path. A bare id keeps the first match, which is the only thing +/// it can mean when it is unique and the only thing this gateway did before. +ResolvedOperation resolve_operation(const AggregatedOperations & ops, const http::MemberQualifiedId & parsed) { + const auto owned_by_target = [&ops, &parsed](const std::string & full_path) { + if (!parsed.has_member) { + return true; + } + auto owner = ops.owner_by_path.find(full_path); + return owner != ops.owner_by_path.end() && owner->second == parsed.member_id; + }; + + ResolvedOperation resolved; + for (const auto & svc : ops.services) { + if (svc.name == parsed.item_id && owned_by_target(svc.full_path)) { + resolved.service = svc; + return resolved; + } + } + for (const auto & act : ops.actions) { + if (act.name == parsed.item_id && owned_by_target(act.full_path)) { + resolved.action = act; + return resolved; + } + } + return resolved; +} + +/// Members of `ops` that expose `short_name`, in collection order. +/// +/// More than one means the bare id names more than one operation. Keyed on the +/// short name because that is the wire id; the full ROS paths differ, which is +/// exactly why the short name stops identifying one of them. +std::vector local_providers_of(const AggregatedOperations & ops, const std::string & short_name) { + std::vector providers; + const auto record = [&ops, &providers](const std::string & full_path) { + auto owner = ops.owner_by_path.find(full_path); + providers.push_back(owner != ops.owner_by_path.end() ? owner->second : std::string{}); + }; + for (const auto & svc : ops.services) { + if (svc.name == short_name) { + record(svc.full_path); + } + } + for (const auto & act : ops.actions) { + if (act.name == short_name) { + record(act.full_path); + } + } + return providers; +} + +/// The error for a member that is in the tree but whose gateway is silent. +/// +/// A retained member is kept precisely so that the answer to a request does not +/// change when a link drops: the item is still addressable, and asking for it +/// says why it cannot be served right now. The alternatives are what this +/// replaces - quietly running a different member's operation, or a 200 with +/// nothing in it. `not-responding` is the SOVD code for "no response from the +/// underlying entity", which is exactly the situation. +std::optional member_unavailable_error(const ThreadSafeEntityCache & cache, const std::string & entity_id, + const std::string & member_id, const std::string & operation_id) { + bool unavailable = false; + if (auto app = cache.get_app(member_id)) { + unavailable = !app->available; + } else if (auto component = cache.get_component(member_id)) { + unavailable = !component->available; + } + if (!unavailable) { + return std::nullopt; + } + return make_error(504, ERR_NOT_RESPONDING, "Member '" + member_id + "' is not available", + json{{"details", + "The gateway contributing this member is not answering; it is retained from its " + "last known declaration"}, + {"entity_id", entity_id}, + {"operation_id", operation_id}, + {"member_id", member_id}}); +} + /// Convert a ROS 2 action goal status into the SOVD `ExecutionStatus` enum /// the gateway emits on the wire. Identical mapping to the legacy helper. std::string sovd_status_from_ros2(ActionGoalStatus status) { @@ -374,22 +478,53 @@ http::Result> OperationHandlers::list_operat auto data_access_mgr = ctx_.node()->get_data_access_manager(); auto type_introspection = data_access_mgr->get_type_introspection(); + // A peer's declared operations live in this cache so that ambiguity can be + // decided without asking anyone at request time. They are not listed from + // here: the gateway that owns an operation is the one that reports it, and + // this walk runs even when the caller asked for no fan-out at all. + const auto contributed_by_peer = [&cache](const std::string & member_id) { + static constexpr std::string_view kPeerPrefix = "peer:"; + if (auto app = cache.get_app(member_id)) { + return app->source.rfind(kPeerPrefix, 0) == 0; + } + if (auto component = cache.get_component(member_id)) { + return component->source.rfind(kPeerPrefix, 0) == 0; + } + return false; + }; + const auto owner_is_remote = [&ops, &contributed_by_peer](const std::string & full_path) { + auto owner = ops.owner_by_path.find(full_path); + return owner != ops.owner_by_path.end() && contributed_by_peer(owner->second); + }; + for (const auto & svc : ops.services) { + if (owner_is_remote(svc.full_path)) { + continue; + } dto::OperationItem item; item.id = svc.name; item.name = svc.name; item.proximity_proof_required = false; item.asynchronous_execution = false; item.x_medkit = build_service_xmedkit(svc, entity_id, type_introspection); + if (auto owner = ops.owner_by_path.find(svc.full_path); owner != ops.owner_by_path.end() && ops.is_aggregated) { + item.x_medkit->member_ids = std::vector{owner->second}; + } collection.items.push_back(std::move(item)); } for (const auto & act : ops.actions) { + if (owner_is_remote(act.full_path)) { + continue; + } dto::OperationItem item; item.id = act.name; item.name = act.name; item.proximity_proof_required = false; item.asynchronous_execution = true; item.x_medkit = build_action_xmedkit(act, entity_id, type_introspection); + if (auto owner = ops.owner_by_path.find(act.full_path); owner != ops.owner_by_path.end() && ops.is_aggregated) { + item.x_medkit->member_ids = std::vector{owner->second}; + } collection.items.push_back(std::move(item)); } @@ -408,6 +543,16 @@ http::Result> OperationHandlers::list_operat for (auto & item : fan_out.items) { collection.items.push_back(std::move(item)); } + + // Two members exposing one short name are two items with one id, and the + // merged collection is the first place that is visible: each gateway holds + // one `calibrate` and considers it unique. An id only one item carries is + // left alone - it already names one thing, and rewriting it would break + // every client that sends the bare name. + http::qualify_ambiguous_ids(collection.items, [](const dto::OperationItem & item) { + return item.x_medkit.has_value() && item.x_medkit->member_ids.has_value() ? &*item.x_medkit->member_ids : nullptr; + }); + if (fan_out.partial || !fan_out.dropped_items.empty()) { dto::XMedkitCollection xm; if (fan_out.partial) { @@ -479,23 +624,18 @@ http::Result OperationHandlers::get_operation(const http:: } const auto & ops = lookup->ops; - std::optional service_info; - std::optional action_info; - for (const auto & svc : ops.services) { - if (svc.name == operation_id) { - service_info = svc; - break; - } - } - if (!service_info.has_value()) { - for (const auto & act : ops.actions) { - if (act.name == operation_id) { - action_info = act; - break; - } - } + // A qualified id is accepted wherever the entity has members to name. A bare + // id keeps resolving to the first match, so every client that sends the + // short name - and the OpenAPI document this gateway generates - still work. + auto parsed = http::parse_member_qualified_id(operation_id, ops.is_aggregated); + if (parsed.has_member && !names_a_member(ops, parsed.member_id)) { + return tl::make_unexpected( + make_error(404, ERR_RESOURCE_NOT_FOUND, "Member not found in entity", + json{{"entity_id", entity_id}, {"operation_id", operation_id}, {"member_id", parsed.member_id}})); } - if (!service_info.has_value() && !action_info.has_value()) { + + auto resolved = resolve_operation(ops, parsed); + if (!resolved.found()) { return tl::make_unexpected(make_error(404, ERR_OPERATION_NOT_FOUND, "Operation not found", json{{"entity_id", entity_id}, {"operation_id", operation_id}})); } @@ -504,18 +644,19 @@ http::Result OperationHandlers::get_operation(const http:: auto type_introspection = data_access_mgr->get_type_introspection(); dto::OperationDetail detail; - if (service_info.has_value()) { - detail.item.id = service_info->name; - detail.item.name = service_info->name; + // The id echoes what was requested, so a caller that took a qualified id out + // of the collection sees the same id come back and can keep using it. + detail.item.id = operation_id; + if (resolved.service.has_value()) { + detail.item.name = resolved.service->name; detail.item.proximity_proof_required = false; detail.item.asynchronous_execution = false; - detail.item.x_medkit = build_service_xmedkit(*service_info, entity_id, type_introspection); + detail.item.x_medkit = build_service_xmedkit(*resolved.service, entity_id, type_introspection); } else { - detail.item.id = action_info->name; - detail.item.name = action_info->name; + detail.item.name = resolved.action->name; detail.item.proximity_proof_required = false; detail.item.asynchronous_execution = true; - detail.item.x_medkit = build_action_xmedkit(*action_info, entity_id, type_introspection); + detail.item.x_medkit = build_action_xmedkit(*resolved.action, entity_id, type_introspection); } return detail; } @@ -609,26 +750,51 @@ OperationHandlers::create_execution(const http::TypedRequest & req, dto::Executi const auto & ops = lookup->ops; const std::string id_field = (lookup->entity_type == "app") ? "app_id" : "component_id"; - std::optional service_info; - std::optional action_info; - for (const auto & svc : ops.services) { - if (svc.name == operation_id) { - service_info = svc; - break; + auto parsed = http::parse_member_qualified_id(operation_id, ops.is_aggregated); + if (parsed.has_member && !names_a_member(ops, parsed.member_id)) { + return tl::make_unexpected( + make_error(404, ERR_RESOURCE_NOT_FOUND, "Member not found in entity", + json{{"entity_id", entity_id}, {"operation_id", operation_id}, {"member_id", parsed.member_id}})); + } + + auto resolved = resolve_operation(ops, parsed); + if (!resolved.found()) { + return tl::make_unexpected(make_error(404, ERR_OPERATION_NOT_FOUND, "Operation not found", + json{{"entity_id", entity_id}, {"operation_id", operation_id}})); + } + + // A bare id that names more than one operation runs whichever member was + // walked first, and the caller never learns which. That is the one case the + // bare form cannot carry, so it is the one case that is refused - an id that + // names a single operation still executes, which is what every current + // client sends. An id that names nothing was already answered as not found + // above: telling a caller to qualify a typo would not help them. + if (!parsed.has_member) { + const std::vector providers = local_providers_of(ops, operation_id); + if (providers.size() > 1) { + return tl::make_unexpected( + make_error(400, ERR_INVALID_REQUEST, "Ambiguous operation id: more than one member provides it", + json{{"details", "Use format 'member_id:operation_id' to name the member that runs it"}, + {"entity_id", entity_id}, + {"operation_id", operation_id}, + {"member_ids", providers}})); } } - if (!service_info.has_value()) { - for (const auto & act : ops.actions) { - if (act.name == operation_id) { - action_info = act; - break; + + // Whoever ends up owning the resolved operation must actually be reachable. + { + const std::string & full_path = + resolved.service.has_value() ? resolved.service->full_path : resolved.action->full_path; + auto owner = ops.owner_by_path.find(full_path); + if (owner != ops.owner_by_path.end()) { + if (auto err = member_unavailable_error(cache, entity_id, owner->second, operation_id)) { + return tl::make_unexpected(*err); } } } - if (!service_info.has_value() && !action_info.has_value()) { - return tl::make_unexpected(make_error(404, ERR_OPERATION_NOT_FOUND, "Operation not found", - json{{"entity_id", entity_id}, {"operation_id", operation_id}})); - } + + const std::optional & service_info = resolved.service; + const std::optional & action_info = resolved.action; auto * operation_mgr = ctx_.node()->get_operation_manager(); diff --git a/src/ros2_medkit_gateway/src/openapi/capability_generator.cpp b/src/ros2_medkit_gateway/src/openapi/capability_generator.cpp index 8c94a7175..730fa7bcc 100644 --- a/src/ros2_medkit_gateway/src/openapi/capability_generator.cpp +++ b/src/ros2_medkit_gateway/src/openapi/capability_generator.cpp @@ -16,12 +16,14 @@ #include #include +#include #include #include "openapi_spec_builder.hpp" #include "path_builder.hpp" #include "ros2_medkit_gateway/core/http/http_utils.hpp" +#include "ros2_medkit_gateway/core/http/member_qualified_id.hpp" #include "ros2_medkit_gateway/core/models/entity_capabilities.hpp" #include "ros2_medkit_gateway/core/models/entity_types.hpp" #include "ros2_medkit_gateway/core/plugins/plugin_manager.hpp" @@ -238,6 +240,20 @@ nlohmann::json CapabilityGenerator::generate_resource_collection(const ResolvedP std::string collection_path = entity_path + "/" + resolved.resource_collection; const auto & cache = node_.get_thread_safe_cache(); + // The documented id of an item has to be the id the collection emits, or the + // spec describes a request the gateway then refuses. Both use the same rule: + // an id that more than one item carries is qualified with its owning member, + // an id that names one thing is left bare. What the generator cannot see is + // a peer contributing a second item under the same id - it reads the local + // cache only, and there is no peer view here to consult. + const auto qualified_item_id = [](const std::string & item_id, size_t provider_count, + const std::string & owner) -> std::string { + if (provider_count < 2 || owner.empty()) { + return item_id; + } + return http::make_member_qualified_id(owner, item_id); + }; + if (resolved.resource_collection == "data") { auto data = cache.get_entity_data(resolved.entity_id); paths[collection_path] = path_builder.build_data_collection(entity_path, data.topics); @@ -258,12 +274,30 @@ nlohmann::json CapabilityGenerator::generate_resource_collection(const ResolvedP ops = cache.get_function_operations(resolved.entity_id); } paths[collection_path] = path_builder.build_operations_collection(entity_path, ops); + + // Short names are not unique across members, and two of them produced the + // same key here, so one item's documentation silently replaced the other's. + std::unordered_map short_name_counts; + for (const auto & svc : ops.services) { + ++short_name_counts[svc.name]; + } + for (const auto & action : ops.actions) { + ++short_name_counts[action.name]; + } + const auto owner_of = [&ops](const std::string & full_path) -> std::string { + auto owner = ops.owner_by_path.find(full_path); + return owner != ops.owner_by_path.end() ? owner->second : std::string{}; + }; + for (const auto & svc : ops.services) { - std::string item_path = collection_path + "/" + svc.name; + std::string item_path = + collection_path + "/" + qualified_item_id(svc.name, short_name_counts[svc.name], owner_of(svc.full_path)); paths[item_path] = path_builder.build_operation_item(entity_path, svc); } for (const auto & action : ops.actions) { - std::string item_path = collection_path + "/" + action.name; + std::string item_path = + collection_path + "/" + + qualified_item_id(action.name, short_name_counts[action.name], owner_of(action.full_path)); paths[item_path] = path_builder.build_operation_item(entity_path, action); } } else if (resolved.resource_collection == "configurations") { @@ -351,9 +385,22 @@ nlohmann::json CapabilityGenerator::generate_specific_resource(const ResolvedPat ops = cache.get_function_operations(resolved.entity_id); } + // The requested id may name its owning member, which is how the collection + // addresses an operation short name that more than one member exposes. + // Matching the bare half alone would document one member's operation under + // the other member's id. + auto requested = http::parse_member_qualified_id(resolved.resource_id, ops.is_aggregated); + const auto owned_by_requested = [&ops, &requested](const std::string & full_path) { + if (!requested.has_member) { + return true; + } + auto owner = ops.owner_by_path.find(full_path); + return owner != ops.owner_by_path.end() && owner->second == requested.member_id; + }; + bool found = false; for (const auto & svc : ops.services) { - if (svc.name == resolved.resource_id) { + if (svc.name == requested.item_id && owned_by_requested(svc.full_path)) { paths[resource_path] = path_builder.build_operation_item(entity_path, svc); found = true; break; @@ -361,7 +408,7 @@ nlohmann::json CapabilityGenerator::generate_specific_resource(const ResolvedPat } if (!found) { for (const auto & action : ops.actions) { - if (action.name == resolved.resource_id) { + if (action.name == requested.item_id && owned_by_requested(action.full_path)) { paths[resource_path] = path_builder.build_operation_item(entity_path, action); found = true; break; diff --git a/src/ros2_medkit_gateway/test/test_entity_resource_model.cpp b/src/ros2_medkit_gateway/test/test_entity_resource_model.cpp index 73bb67546..471cb877d 100644 --- a/src/ros2_medkit_gateway/test/test_entity_resource_model.cpp +++ b/src/ros2_medkit_gateway/test/test_entity_resource_model.cpp @@ -14,8 +14,10 @@ #include +#include #include #include +#include #include #include @@ -745,6 +747,128 @@ TEST_F(DataAggregationTest, UnknownEntityReturnsEmptyData) { EXPECT_TRUE(result.aggregation_level.empty()); } +// ============================================================================= +// Member resolution +// +// A grouping entity - an Area, a Function, or a Component with children - owns +// no resources beyond what its members provide. Resolving an item to its owner +// and listing the grouping's items therefore have to walk the same graph, or +// the gateway advertises what it will not serve. These cases pin that one walk. +// ============================================================================= + +class MemberResolutionTest : public ::testing::Test { + protected: + /// Members compared as a set: the walk fixes no order and no caller needs one. + static std::vector sorted(std::vector v) { + std::sort(v.begin(), v.end()); + return v; + } + + ThreadSafeEntityCache cache_; +}; + +TEST_F(MemberResolutionTest, AreaMembersAreItsComponentsAndTheirApps) { + std::vector areas{make_area("vehicle", "Vehicle")}; + std::vector components{ + make_component("primary-ecu", "Primary", "vehicle"), + make_component("secondary-ecu", "Secondary", "vehicle"), + }; + std::vector apps{ + make_app("temp_sensor", "Temp", "primary-ecu"), + make_app("pressure_sensor", "Pressure", "secondary-ecu"), + }; + cache_.update_all(areas, components, apps, {}); + + EXPECT_EQ(sorted(cache_.get_members(SovdEntityType::AREA, "vehicle")), + sorted({"primary-ecu", "secondary-ecu", "temp_sensor", "pressure_sensor"})); +} + +TEST_F(MemberResolutionTest, AreaMembersDescendIntoSubareas) { + // The operations walk that predates this one stops at the area's own + // components, so a subarea's apps were invisible to any caller that asked the + // parent area what it contains. + Area parent = make_area("vehicle", "Vehicle"); + Area child = make_area("powertrain", "Powertrain"); + child.parent_area_id = "vehicle"; + + std::vector components{make_component("engine-ecu", "Engine", "powertrain")}; + std::vector apps{make_app("rpm_sensor", "RPM", "engine-ecu")}; + cache_.update_all({parent, child}, components, apps, {}); + + EXPECT_EQ(sorted(cache_.get_members(SovdEntityType::AREA, "vehicle")), sorted({"engine-ecu", "rpm_sensor"})); +} + +TEST_F(MemberResolutionTest, FunctionMembersIncludeHostsThatAreComponents) { + // A Function host id may name an App or a Component. Only the App half was + // indexed, and a Component host was dropped without a word, so a Function + // hosting a Component reported none of that Component's resources. + std::vector areas{make_area("vehicle", "Vehicle")}; + std::vector components{make_component("brake-ecu", "Brake", "vehicle")}; + std::vector apps{ + make_app("actuator", "Actuator", "brake-ecu"), + make_app("temp_sensor", "Temp", "brake-ecu"), + }; + + Function func; + func.id = "vehicle_health"; + func.hosts = {"temp_sensor", "brake-ecu"}; + cache_.update_all(areas, components, apps, {func}); + + const auto members = sorted(cache_.get_members(SovdEntityType::FUNCTION, "vehicle_health")); + for (const char * expected : {"actuator", "brake-ecu", "temp_sensor"}) { + EXPECT_NE(std::find(members.begin(), members.end(), expected), members.end()) + << "function member missing: " << expected; + } +} + +TEST_F(MemberResolutionTest, ComponentIsAMemberOfItselfAndCarriesItsChildren) { + // Two things this pins. A Component is its own member because, unlike an Area + // or a Function, it can expose services and actions directly - the operations + // aggregation already counts those before its apps. And a parent Component's + // children were not reachable from the cache at all before: nothing indexed + // that direction. + std::vector areas{make_area("vehicle", "Vehicle")}; + Component parent = make_component("vehicle-ecu", "Vehicle ECU", "vehicle"); + parent.services = {make_service("reset", "/vehicle/reset")}; + Component child = make_component("brake-ecu", "Brake ECU", "vehicle"); + child.parent_component_id = "vehicle-ecu"; + + std::vector apps{make_app("actuator", "Actuator", "brake-ecu")}; + cache_.update_all(areas, {parent, child}, apps, {}); + + EXPECT_EQ(sorted(cache_.get_members(SovdEntityType::COMPONENT, "vehicle-ecu")), + sorted({"vehicle-ecu", "brake-ecu", "actuator"})); +} + +TEST_F(MemberResolutionTest, ComponentParentCycleTerminates) { + // Parent-component cycles are representable: the manifest validator checks + // only that a parent exists, not that the graph is acyclic, and the + // aggregation classifier handles cycles explicitly because they occur. A walk + // without a visited set would not return. + std::vector areas{make_area("vehicle", "Vehicle")}; + Component first = make_component("ecu-a", "A", "vehicle"); + first.parent_component_id = "ecu-b"; + Component second = make_component("ecu-b", "B", "vehicle"); + second.parent_component_id = "ecu-a"; + cache_.update_all(areas, {first, second}, {}, {}); + + EXPECT_EQ(sorted(cache_.get_members(SovdEntityType::COMPONENT, "ecu-a")), sorted({"ecu-a", "ecu-b"})); +} + +TEST_F(MemberResolutionTest, LeafReturnsOnlyItself) { + std::vector areas{make_area("vehicle", "Vehicle")}; + std::vector components{make_component("primary-ecu", "Primary", "vehicle")}; + std::vector apps{make_app("temp_sensor", "Temp", "primary-ecu")}; + cache_.update_all(areas, components, apps, {}); + + EXPECT_EQ(cache_.get_members(SovdEntityType::APP, "temp_sensor"), std::vector{"temp_sensor"}); +} + +TEST_F(MemberResolutionTest, UnknownGroupingHasNoMembers) { + cache_.update_all({}, {}, {}, {}); + EXPECT_TRUE(cache_.get_members(SovdEntityType::AREA, "nope").empty()); +} + int main(int argc, char ** argv) { testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); diff --git a/src/ros2_medkit_gateway/test/test_operation_handlers.cpp b/src/ros2_medkit_gateway/test/test_operation_handlers.cpp index 27590fc7f..5f245ed7e 100644 --- a/src/ros2_medkit_gateway/test/test_operation_handlers.cpp +++ b/src/ros2_medkit_gateway/test/test_operation_handlers.cpp @@ -385,6 +385,21 @@ class OperationHandlersFixtureTest : public ::testing::Test { component.actions = {ActionInfo{"long_calibration", "/powertrain/engine/long_calibration", "example_interfaces/action/Fibonacci", std::nullopt}}; + // A second member of the same area exposing `calibrate` at a different ROS + // path. Deduplication keys on the full path, so both survive the area-level + // walk and the short name - which is the wire id - stops naming one of + // them. That is the ambiguity the qualified form exists for, and it needs + // two members to exist at all. + Component gearbox; + gearbox.id = "gearbox"; + gearbox.name = "Gearbox"; + gearbox.namespace_path = "/powertrain/gearbox"; + gearbox.fqn = "/powertrain/gearbox"; + gearbox.area = "powertrain"; + gearbox.source = "manifest"; + gearbox.services = { + ServiceInfo{"calibrate", "/powertrain/gearbox/calibrate", "std_srvs/srv/Trigger", std::nullopt}}; + // The area the component sits in. The operation routes are registered for // all four entity types and create_execution rejects a collection / // entity-type mismatch, so exercising a non-component collection needs a @@ -396,7 +411,7 @@ class OperationHandlersFixtureTest : public ::testing::Test { area.source = "manifest"; auto & cache = const_cast(gateway_node_->get_thread_safe_cache()); - cache.update_all({area}, {component}, {}, {}); + cache.update_all({area}, {component, gearbox}, {}, {}); } /// Drive `create_execution` and assert the typed response carries the async @@ -600,6 +615,93 @@ TEST_F(OperationHandlersFixtureTest, CreateExecutionLocationExtendsTheRequestedC EXPECT_EQ(location->second, requested_path + "/" + async_ptr->id); } +// Qualification follows ambiguity, so one collection must show both halves at +// once: the id two members share is qualified, the id only one member has is +// not. Asserting only the qualified half would pass a rule that renamed +// everything, which is what breaks every client sending the bare short name. +TEST_F(OperationHandlersFixtureTest, OnlyAnAmbiguousOperationIdIsQualified) { + auto raw_req = make_request_with_match("/api/v1/areas/powertrain/operations", R"(/api/v1/areas/([^/]+)/operations)"); + http::TypedRequest typed(raw_req); + + auto result = handlers_->list_operations(typed); + ASSERT_TRUE(result.has_value()); + + std::multiset ids; + for (const auto & item : result->items) { + ids.insert(item.id); + } + EXPECT_EQ(ids.count("engine:calibrate"), 1u); + EXPECT_EQ(ids.count("gearbox:calibrate"), 1u); + EXPECT_EQ(ids.count("calibrate"), 0u) << "a bare id survived alongside the qualified ones"; + EXPECT_EQ(ids.count("long_calibration"), 1u) << "a single-provider id was qualified"; + EXPECT_EQ(ids.count("engine:long_calibration"), 0u); +} + +// The bare id names two operations, so executing it would run whichever member +// was walked first and never say which. +TEST_F(OperationHandlersFixtureTest, CreateExecutionRefusesAnAmbiguousBareId) { + auto raw_req = make_request_with_match("/api/v1/areas/powertrain/operations/calibrate/executions", + R"(/api/v1/areas/([^/]+)/operations/([^/]+)/executions)"); + http::TypedRequest typed(raw_req); + dto::ExecutionCreateRequest body; + body.parameters = json::object(); + + auto result = handlers_->create_execution(typed, body); + + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(result.error().http_status, 400); + EXPECT_EQ(result.error().code, ros2_medkit_gateway::ERR_INVALID_REQUEST); + EXPECT_NE(result.error().message.find("member"), std::string::npos); + ASSERT_TRUE(result.error().params.contains("member_ids")); + EXPECT_EQ(result.error().params["member_ids"].size(), 2u); +} + +// An id that names nothing is a miss, not an invitation to qualify a typo. +TEST_F(OperationHandlersFixtureTest, CreateExecutionUnknownBareIdIsNotFound) { + auto raw_req = make_request_with_match("/api/v1/areas/powertrain/operations/does_not_exist/executions", + R"(/api/v1/areas/([^/]+)/operations/([^/]+)/executions)"); + http::TypedRequest typed(raw_req); + dto::ExecutionCreateRequest body; + body.parameters = json::object(); + + auto result = handlers_->create_execution(typed, body); + + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(result.error().http_status, 404); + EXPECT_EQ(result.error().code, ros2_medkit_gateway::ERR_OPERATION_NOT_FOUND); +} + +// The member half has to name a member of THIS entity; a leaf that exists +// elsewhere in the tree is a miss, not a fallback to the first match. +TEST_F(OperationHandlersFixtureTest, GetOperationRejectsAnIdNamingAForeignMember) { + auto raw_req = make_request_with_match("/api/v1/areas/powertrain/operations/chassis:calibrate", + R"(/api/v1/areas/([^/]+)/operations/([^/]+))"); + http::TypedRequest typed(raw_req); + + auto result = handlers_->get_operation(typed); + + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(result.error().http_status, 404); + EXPECT_EQ(result.error().code, ros2_medkit_gateway::ERR_RESOURCE_NOT_FOUND); +} + +// A qualified id selects among the same-named copies, and the response echoes +// the id that was asked for so a caller can keep using it. +TEST_F(OperationHandlersFixtureTest, GetOperationResolvesAQualifiedIdToItsMember) { + auto raw_req = make_request_with_match("/api/v1/areas/powertrain/operations/gearbox:calibrate", + R"(/api/v1/areas/([^/]+)/operations/([^/]+))"); + http::TypedRequest typed(raw_req); + + auto result = handlers_->get_operation(typed); + + ASSERT_TRUE(result.has_value()) << result.error().code << ": " << result.error().message; + EXPECT_EQ(result->item.id, "gearbox:calibrate"); + EXPECT_EQ(result->item.name, "calibrate"); + ASSERT_TRUE(result->item.x_medkit.has_value()); + ASSERT_TRUE(result->item.x_medkit->ros2.has_value()); + EXPECT_EQ(result->item.x_medkit->ros2->service, "/powertrain/gearbox/calibrate"); +} + TEST_F(OperationHandlersFixtureTest, UpdateExecutionStopReturnsAcceptedAndLocation) { const auto execution_id = create_action_execution(20); ASSERT_FALSE(execution_id.empty()); diff --git a/src/ros2_medkit_integration_tests/CMakeLists.txt b/src/ros2_medkit_integration_tests/CMakeLists.txt index ce57c6baf..b02e8c999 100644 --- a/src/ros2_medkit_integration_tests/CMakeLists.txt +++ b/src/ros2_medkit_integration_tests/CMakeLists.txt @@ -232,6 +232,7 @@ if(BUILD_TESTING) test_peer_aggregation test_cross_ecu_fanout test_daisy_chain_aggregation + test_grouping_entity_aggregation test_leaf_collision_aggregation test_startup_param_clamp_warnings) set(_MULTI_GATEWAY_DOMAINS 4) diff --git a/src/ros2_medkit_integration_tests/test/features/test_grouping_entity_aggregation.test.py b/src/ros2_medkit_integration_tests/test/features/test_grouping_entity_aggregation.test.py new file mode 100644 index 000000000..13a239909 --- /dev/null +++ b/src/ros2_medkit_integration_tests/test/features/test_grouping_entity_aggregation.test.py @@ -0,0 +1,589 @@ +# Copyright 2026 bburda +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +End-to-end specification for one resource-addressing model across any entity. + +An AGGREGATING entity draws its resources from members rather than owning them: +an Area, a Function (cross-component and cross-area by definition), or a +Component with subcomponents - and a subcomponent can live on its own gateway, +so a parent on the host with a child on another ECU is a normal deployment. + +The model has two halves and they have to be built in this order. + +IDENTITY FIRST. An item can only be addressed back to whatever serves it if +that thing has a name that is unique in the merged tree. Today Apps are +renamed on collision (`__`) but Components are not - two peers +announcing one Component id resolve last-writer-wins - and member ids arriving +from a peer are re-emitted verbatim, so they can name something the aggregator +does not have. Until leaf identity is unique, no addressing scheme works, +because the qualifier itself is ambiguous. + +ADDRESSING SECOND, and it has exactly two cases, decided per collection rather +than per handler: + + a LEAF-OWNED item - a topic, an operation, a configuration key - belongs to + one leaf, and is addressed ":". + an AGGREGATE item - a fault, which has one code and a SET of reporting + sources - belongs to the aggregate itself, and is + addressed by its own id while naming its contributors. + +Faults, logs and bulk-data are not exceptions to the model; they are the second +case. Which case a collection is, is declared once. + +WHAT THIS SUITE EXISTS TO PREVENT, all three measured on this topology: + + * A grouping lists an item, then answers a read of it from the LOCAL graph + only. An unknown topic returns 200 with status "metadata_only", so the + client is told the read succeeded and handed an empty body. + * Two members exposing one operation short name: the list shows both, and + execution resolves against the local cache alone. + * The fan-out concatenates peer items with no key, so one id can appear twice + with nothing on the wire telling the copies apart. + +THE RULES + +R1 A leaf contributed by more than one gateway is addressable as more than one + entity. Identity is unique after the merge. +R2 Every listed item names the leaf or leaves that contribute it. +R3 A leaf-owned item provided by MORE THAN ONE leaf is addressed + ":". An item with a single provider keeps its bare id. + Qualification follows ambiguity, not aggregation: in runtime discovery every + App hangs off the one host Component, so "aggregating" is the ordinary + entity, and qualifying everything there would change the ids of the most + used entity in the product and refuse requests every current client sends. +R4 A bare id is refused only when it is ambiguous, and the refusal names the + form to use. An unambiguous bare id keeps working. +R5 A compound id reaches its leaf, local or on a peer, and returns that leaf's + data. Asserting the status alone cannot show this: the failure mode is a + 200 with an empty body. +R6 An item no leaf provides is refused, and is distinguishable from an item + that exists and is empty. +R7 A Component both sides contribute to aggregates. Routing it wholesale to + one peer would discard the other half, which is what happens today. +R8 A lock on a leaf is honoured by a request dispatched through an aggregate. +R9 Peered gateways terminate. +""" + +import os +import tempfile +import time +import unittest +from urllib.parse import quote + +from launch import LaunchDescription +from launch.actions import SetEnvironmentVariable, TimerAction +import launch_ros.actions +import launch_testing.actions +import requests +from ros2_medkit_test_utils.constants import ( + API_BASE_PATH, + get_test_domain_id, + get_test_port, +) +from ros2_medkit_test_utils.launch_helpers import ( + create_demo_nodes, + create_fault_manager_node, + create_gateway_node, +) + +PRIMARY_PORT = get_test_port(0) +PEER_PORT = get_test_port(1) +PRIMARY_URL = f'http://localhost:{PRIMARY_PORT}{API_BASE_PATH}' +PEER_URL = f'http://localhost:{PEER_PORT}{API_BASE_PATH}' + +PRIMARY_DOMAIN_ID = get_test_domain_id(0) +PEER_DOMAIN_ID = get_test_domain_id(1) + +# `calibration` runs on BOTH domains deliberately. Each gateway binds it to an +# App of its own, so one operation short name - the id the HTTP API addresses - +# is exposed by a member on each side. That is R3's case and it is not +# hypothetical: deduplication keys on the full ROS path, the wire id is the +# short name, so both survive. +# The peer's calibration node runs in a DIFFERENT namespace on purpose. Both +# gateways expose an operation whose short name - the id the HTTP API addresses +# - is `calibrate`, but the full ROS paths differ. That is the case R3 and R4 +# exist for. With the same namespace on both sides the full paths are identical +# and the local walk simply deduplicates the peer's copy away, which builds a +# dedup collapse rather than the ambiguity. +PRIMARY_NODES = ['temp_sensor', 'calibration', 'rpm_sensor'] +PEER_NODES = ['pressure_sensor', 'actuator'] +PEER_CALIBRATION_NAMESPACE = '/chassis/brakes' + +# Declared with the SAME id on both gateways. Apps are renamed on collision, +# Components are not, so this is the case that shows whether leaf identity +# survives the merge - R1, the precondition for every addressing rule. +COLLIDING_LEAF = 'shared_sensor' + +MERGED_AREA = 'vehicle' +MERGED_FUNCTION = 'vehicle_health' +PARENT_COMPONENT = 'vehicle-ecu' +PEER_SUBCOMPONENT = 'brake-ecu' + +# Both gateways contribute to one Area id and one Function id, and the peer's +# Component declares the primary's Component as its parent - so all three +# aggregating kinds span the pair. +PRIMARY_MANIFEST = f"""\ +manifest_version: "1.0" +metadata: + name: "Primary ECU" + version: "1.0.0" +config: + unmanifested_nodes: ignore +areas: + - id: {MERGED_AREA} + name: "Vehicle" +components: + - id: {PARENT_COMPONENT} + name: "Vehicle ECU" + area: {MERGED_AREA} +apps: + - id: temp_sensor + name: "Engine Temperature Sensor" + is_located_on: {PARENT_COMPONENT} + ros_binding: + node_name: temp_sensor + namespace: /powertrain/engine + - id: primary_calibration + name: "Primary Calibration Service" + is_located_on: {PARENT_COMPONENT} + ros_binding: + node_name: calibration + namespace: /powertrain/engine + - id: {COLLIDING_LEAF} + name: "Shared Sensor (primary)" + is_located_on: {PARENT_COMPONENT} + ros_binding: + node_name: rpm_sensor + namespace: /powertrain/engine +functions: + - id: {MERGED_FUNCTION} + name: "Vehicle Health Monitoring" + category: monitoring + hosted_by: + - temp_sensor + - primary_calibration + - {COLLIDING_LEAF} +""" + +PEER_MANIFEST = f"""\ +manifest_version: "1.0" +metadata: + name: "Secondary ECU" + version: "1.0.0" +config: + unmanifested_nodes: ignore +areas: + - id: {MERGED_AREA} + name: "Vehicle" +components: + # The parent is declared on BOTH gateways on purpose. A subcomponent may not + # name a parent that is absent from its own manifest: rule R006 rejects it as + # an ERROR, and an errored manifest is not loaded at all, so the peer would + # contribute nothing. Declaring the parent on both sides is how a hierarchy + # crosses a gateway boundary - the parent then merges, and the child stays + # owned by the gateway that runs it. + - id: {PARENT_COMPONENT} + name: "Vehicle ECU" + area: {MERGED_AREA} + - id: {PEER_SUBCOMPONENT} + name: "Brake ECU" + area: {MERGED_AREA} + parent_component_id: {PARENT_COMPONENT} +apps: + - id: pressure_sensor + name: "Brake Pressure Sensor" + is_located_on: {PEER_SUBCOMPONENT} + ros_binding: + node_name: pressure_sensor + namespace: /chassis/brakes + - id: peer_calibration + name: "Peer Calibration Service" + is_located_on: {PEER_SUBCOMPONENT} + ros_binding: + node_name: calibration + namespace: {PEER_CALIBRATION_NAMESPACE} + - id: {COLLIDING_LEAF} + name: "Shared Sensor (peer)" + is_located_on: {PEER_SUBCOMPONENT} + ros_binding: + node_name: actuator + namespace: /chassis/brakes +functions: + - id: {MERGED_FUNCTION} + name: "Vehicle Health Monitoring" + category: monitoring + hosted_by: + - pressure_sensor + - peer_calibration + - {COLLIDING_LEAF} +""" + + +def _write_manifest(content): + """Write manifest YAML to a temporary file and return its path.""" + fd, path = tempfile.mkstemp(suffix='.yaml', prefix='test_grouping_manifest_') + with os.fdopen(fd, 'w') as handle: + handle.write(content) + return path + + +def generate_test_description(): + primary_manifest_path = _write_manifest(PRIMARY_MANIFEST) + peer_manifest_path = _write_manifest(PEER_MANIFEST) + + peer_domain_env = {'ROS_DOMAIN_ID': str(PEER_DOMAIN_ID)} + + primary_gateway = create_gateway_node( + port=PRIMARY_PORT, + extra_params={ + 'discovery.mode': 'hybrid', + 'discovery.manifest_path': primary_manifest_path, + 'discovery.manifest_strict_validation': False, + 'aggregation.enabled': True, + 'aggregation.timeout_ms': 5000, + 'aggregation.announce': False, + 'aggregation.discover': False, + 'aggregation.peer_urls': [f'http://localhost:{PEER_PORT}'], + 'aggregation.peer_names': ['secondary_gateway'], + }, + ) + + peer_gateway = create_gateway_node( + name='secondary_gateway_node', + port=PEER_PORT, + extra_params={ + 'discovery.mode': 'hybrid', + 'discovery.manifest_path': peer_manifest_path, + 'discovery.manifest_strict_validation': False, + }, + extra_env=peer_domain_env, + ) + + delayed = TimerAction( + period=2.0, + actions=( + create_demo_nodes(PRIMARY_NODES, lidar_faulty=False) + + create_demo_nodes(PEER_NODES, lidar_faulty=False, extra_env=peer_domain_env) + # Built inline because create_demo_nodes takes its namespace from a + # fixed registry and cannot place a node elsewhere. + + [launch_ros.actions.Node( + package='ros2_medkit_integration_tests', + executable='demo_calibration_service', + name='calibration', + namespace=PEER_CALIBRATION_NAMESPACE, + output='screen', + additional_env=peer_domain_env, + )] + + [ + create_fault_manager_node(rosbag_enabled=False), + create_fault_manager_node(rosbag_enabled=False, extra_env=peer_domain_env), + ] + ), + ) + + launch_description = LaunchDescription([ + SetEnvironmentVariable('ROS_DOMAIN_ID', str(PRIMARY_DOMAIN_ID)), + primary_gateway, + peer_gateway, + delayed, + launch_testing.actions.ReadyToTest(), + ]) + + return ( + launch_description, + {'gateway_node': primary_gateway, 'peer_gateway': peer_gateway}, + ) + + +class GroupingAggregationTest(unittest.TestCase): + """Drives the aggregating gateway; the peer is only ever used to verify.""" + + @classmethod + def setUpClass(cls): + # Order matters. The entity merge is driven by HTTP from the peer and + # completes while the local ROS graph is still binding Apps to nodes, so + # a collection read between those two moments is legitimately empty and + # would fail every rule below for a reason unrelated to the rule. + cls._wait_for_apps(PRIMARY_URL, {'temp_sensor', 'primary_calibration'}, 'primary') + cls._wait_for_apps(PEER_URL, {'pressure_sensor', 'peer_calibration'}, 'peer') + cls._wait_until_merged() + + @classmethod + def _wait_for_apps(cls, base_url, required, label): + """Block until `required` Apps are present AND bound to a live node. + + Presence is not enough: a manifest App exists before its node does, and + an App with no live binding contributes no resources. + """ + deadline = time.monotonic() + 60.0 + while time.monotonic() < deadline: + try: + response = requests.get(f'{base_url}/apps', timeout=5) + if response.status_code == 200: + online = { + item.get('id') + for item in response.json().get('items', []) + if item.get('x-medkit', {}).get('is_online') + } + if required <= online: + return + except requests.RequestException: + pass + time.sleep(1.0) + raise AssertionError(f'{label}: {required} not online within 60s') + + @classmethod + def _wait_until_merged(cls): + """Block until the primary has merged the peer's half of the Function.""" + deadline = time.monotonic() + 60.0 + while time.monotonic() < deadline: + try: + response = requests.get( + f'{PRIMARY_URL}/functions/{MERGED_FUNCTION}', timeout=5, + ) + if response.status_code == 200: + contributors = ( + response.json().get('x-medkit', {}).get('contributors', []) + ) + if 'local' in contributors and any( + c.startswith('peer:') for c in contributors + ): + return + except requests.RequestException: + pass + time.sleep(0.5) + raise AssertionError(f'{MERGED_FUNCTION} did not merge both contributors in 60s') + + # ------------------------------------------------------------------ helpers + + def _items(self, entity_path, collection): + response = requests.get(f'{PRIMARY_URL}/{entity_path}/{collection}', timeout=10) + self.assertEqual(response.status_code, 200, response.text) + return response.json().get('items', []) + + # ---------------------------------------------------------------------- R1 + + def test_a_leaf_contributed_by_both_gateways_stays_two_addressable_leaves(self): + """R1, the precondition everything else rests on. + + Both gateways declare an App with the id `shared_sensor` bound to + different nodes. They are two different things and the merged tree has + to be able to name both, because every addressing rule below uses the + leaf id as its qualifier. If the merge collapses them, a compound id + built from that qualifier names two entities at once and the ambiguity + the compound form exists to remove has simply moved up one level. + """ + response = requests.get(f'{PRIMARY_URL}/apps', timeout=10) + self.assertEqual(response.status_code, 200) + ids = [item.get('id') for item in response.json().get('items', [])] + + matching = [i for i in ids if i == COLLIDING_LEAF or i.endswith(f'__{COLLIDING_LEAF}')] + self.assertEqual( + len(matching), 2, + f'a leaf declared on both gateways is not addressable twice: {ids}', + ) + # And each must actually resolve, not merely appear in the list. + for leaf_id in matching: + detail = requests.get(f'{PRIMARY_URL}/apps/{leaf_id}', timeout=10) + self.assertEqual(detail.status_code, 200, f'{leaf_id} is listed but not addressable') + + # ---------------------------------------------------------------------- R7 + + def test_a_component_both_sides_contribute_to_aggregates(self): + """R7: a shared Component must fan in, not be handed to one peer. + + A Component whose id appears on both sides is put in the routing table + today, so the whole request is forwarded and the local half is dropped. + That is the same "local half vanishes" failure the design argues against + for Areas and Functions, reached by a different route. + """ + items = self._items(f'components/{PARENT_COMPONENT}', 'operations') + members = set() + for item in items: + members.update(item.get('x-medkit', {}).get('member_ids') or []) + self.assertIn( + 'primary_calibration', members, + f'the local half of the shared Component was discarded: {members}', + ) + + # ---------------------------------------------------------------------- R2 + + def test_every_item_of_an_aggregating_entity_names_its_member(self): + """R1, for all three aggregating kinds. + + Without attribution nothing downstream can address an item, which is why + every attempt to fix this by guessing has failed. The parent Component + case matters most: its subcomponent is on the other gateway, so its + members are not even reachable from the local graph. + """ + for entity_path in ( + f'areas/{MERGED_AREA}', + f'functions/{MERGED_FUNCTION}', + f'components/{PARENT_COMPONENT}', + ): + with self.subTest(entity=entity_path): + items = self._items(entity_path, 'operations') + self.assertTrue(items, f'{entity_path} exposed no operations') + for item in items: + self.assertTrue( + item.get('x-medkit', {}).get('member_ids'), + f"{entity_path}: {item.get('id')!r} names no member", + ) + + def test_a_functions_members_span_components_and_gateways(self): + """A Function is cross-component and cross-area by definition. + + This pins that the merged Function really does reach both sides, so the + rules below are not being checked against a local-only view. + """ + members = set() + for item in self._items(f'functions/{MERGED_FUNCTION}', 'operations'): + members.update(item.get('x-medkit', {}).get('member_ids') or []) + self.assertIn('primary_calibration', members) + self.assertIn('peer_calibration', members) + + # ---------------------------------------------------------------------- R2 + + def test_only_an_ambiguous_item_is_qualified(self): + """R3, both halves. + + Both members expose `calibrate`, so that id is ambiguous and each copy + must carry its provider. `pressure` comes from one member only, so it + keeps its bare id - qualifying it would change an id that was never + ambiguous, on the entity type a default deployment uses most. + """ + items = self._items(f'functions/{MERGED_FUNCTION}', 'operations') + ids = [item.get('id') for item in items] + + # Counted, not set-ified. A set of ids collapses two items that share a + # bare id into one entry, so a fix that emitted one compound id and + # dropped the other member's would satisfy a membership assertion. Both + # members expose `calibrate`, so both must survive as separate items. + self.assertEqual( + ids.count('primary_calibration:calibrate'), 1, f'ids were {ids}', + ) + self.assertEqual( + ids.count('peer_calibration:calibrate'), 1, f'ids were {ids}', + ) + self.assertEqual( + ids.count('calibrate'), 0, + f'a bare id survived alongside the compound ones: {ids}', + ) + + # ---------------------------------------------------------------------- R3 + + def test_an_unambiguous_bare_id_still_works(self): + """R4, the half that protects every client that exists today. + + A single-provider item stays addressable by its bare id. Refusing it + would break the web UI, the Foxglove panel and the MCP tools, which all + send the bare name, and would contradict the OpenAPI document this + gateway generates for itself. + """ + items = self._items(f'functions/{MERGED_FUNCTION}', 'data') + single = [ + item for item in items + if len(item.get('x-medkit', {}).get('member_ids') or []) == 1 + ] + self.assertTrue(single, 'no single-provider item to check') + item_id = single[0]['id'] + self.assertNotIn(':', item_id, f'a single-provider item was qualified: {item_id}') + url = f'{PRIMARY_URL}/functions/{MERGED_FUNCTION}/data/{quote(item_id, safe="")}' + self.assertEqual(requests.get(url, timeout=15).status_code, 200) + + def test_an_ambiguous_bare_id_is_refused(self): + """R4: refuse only where the bare form cannot mean one thing. + + Both members expose `calibrate`, so the bare form names two operations. + Today it returns 200 and runs one of them without saying which. + """ + response = requests.post( + f'{PRIMARY_URL}/functions/{MERGED_FUNCTION}/operations/calibrate/executions', + json={}, + timeout=15, + ) + self.assertEqual(response.status_code, 400, response.text) + body = response.json() + message = ( + body.get('message', '') + ' ' + str(body.get('parameters', {})) + ).lower() + self.assertIn('member', message, f'refusal does not name the qualified form: {body}') + + # ---------------------------------------------------------------------- R4 + + def test_a_compound_id_reaches_a_peer_owned_member(self): + """R4, and the reason it asserts the body rather than the status. + + A read that misses samples the local ROS graph, finds nothing, and + returns 200 with an empty body and status "metadata_only". Asserting + only the status code passes on that false success, which is exactly how + an earlier version of this suite failed to catch anything. + """ + item_id = f'pressure_sensor:{"/chassis/brakes/pressure".lstrip("/")}' + url = f'{PRIMARY_URL}/functions/{MERGED_FUNCTION}/data/{quote(item_id, safe="")}' + response = requests.get(url, timeout=15) + self.assertEqual(response.status_code, 200, response.text) + body = response.json() + self.assertEqual( + body.get('x-medkit', {}).get('status'), 'data', + f'read reported success with no data from the peer member: {body}', + ) + self.assertTrue(body.get('data'), 'peer member returned an empty payload') + + def test_a_compound_id_reaches_a_local_member(self): + """R4 in the other direction: resolving members must not lose the local half.""" + item_id = f'temp_sensor:{"/powertrain/engine/temperature".lstrip("/")}' + url = f'{PRIMARY_URL}/functions/{MERGED_FUNCTION}/data/{quote(item_id, safe="")}' + response = requests.get(url, timeout=15) + self.assertEqual(response.status_code, 200, response.text) + self.assertEqual(response.json().get('x-medkit', {}).get('status'), 'data') + + # ---------------------------------------------------------------------- R5 + + def test_an_item_no_member_provides_is_refused(self): + """R5: absent must be distinguishable from present-but-empty. + + Today an unknown topic answers 200 metadata-only on every entity, so a + client cannot tell a typo from a silent sensor. On an aggregating entity + the member set is known, so the answer can be exact. + """ + response = requests.get( + f'{PRIMARY_URL}/functions/{MERGED_FUNCTION}/data/' + f'{quote("temp_sensor:no/such/topic", safe="")}', + timeout=10, + ) + self.assertEqual(response.status_code, 404, response.text) + + # ---------------------------------------------------------------------- R7 + + def test_loop_suppression_is_carried_on_every_hop(self): + """R7: without it a bidirectionally peered pair bounces a request. + + Asserted by the header's effect rather than by waiting for a hang, which + would cost the full budget on every run. + """ + response = requests.get( + f'{PRIMARY_URL}/functions/{MERGED_FUNCTION}/operations', + headers={'X-Medkit-No-Fan-Out': '1'}, + timeout=10, + ) + self.assertEqual(response.status_code, 200) + members = set() + for item in response.json().get('items', []): + members.update(item.get('x-medkit', {}).get('member_ids') or []) + self.assertNotIn( + 'peer_calibration', members, + 'suppression header ignored, so a peered pair would not terminate', + ) From 2ecaa4bb065c4e9d0e14bcc96ced9290b7dd40ea Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Thu, 20 Aug 2026 11:43:00 +0200 Subject: [PATCH 02/22] feat(aggregation): retain a silent peer's declarations as unreachable When a peer stops answering, what it declared in its manifest does not stop being true. Those entities stay listed, keep the items they last reported, and answer `504 not-responding`. What the peer only discovered at runtime disappears, because nothing can observe that graph any more. Reachability is reported separately from ambiguity. It belongs to the entities a request can be addressed to, which are Apps and Components; an Area or a Function groups members and has none of its own. A retained item still counts towards ambiguity, so an id two members provide stays qualified whether or not either of them is answering. --- docs/api/rest.rst | 6 + src/ros2_medkit_gateway/README.md | 8 +- .../aggregation/aggregation_manager.hpp | 21 +- .../core/discovery/models/app.hpp | 14 +- .../core/discovery/models/area.hpp | 10 +- .../core/discovery/models/component.hpp | 23 +- .../core/discovery/models/function.hpp | 9 +- .../ros2_medkit_gateway/dto/operations.hpp | 13 +- .../ros2_medkit_gateway/dto/x_medkit.hpp | 10 +- .../http/handlers/handler_context.hpp | 4 + .../src/aggregation/aggregation_manager.cpp | 96 +++++ .../src/core/aggregation/peer_client.cpp | 80 ++++ .../src/http/handlers/discovery_handlers.cpp | 36 ++ .../src/http/handlers/handler_context.cpp | 31 ++ .../src/http/handlers/operation_handlers.cpp | 118 ++++-- .../test/test_aggregation_manager.cpp | 110 ++++++ .../test_grouping_entity_aggregation.test.py | 362 +++++++++++++++++- 17 files changed, 900 insertions(+), 51 deletions(-) diff --git a/docs/api/rest.rst b/docs/api/rest.rst index 899219b57..bdab9ac9f 100644 --- a/docs/api/rest.rst +++ b/docs/api/rest.rst @@ -680,6 +680,12 @@ retained entity: - stays listed and stays addressable, so the tree does not change shape when a link drops; - reports ``x-medkit.available: false`` and ``x-medkit.is_online: false``; +- keeps the operations it last reported. They stay listed on the aggregating + entity, each marked ``x-medkit.available: false``, and they still count + towards ambiguity - so an id that two members provide stays qualified and its + bare form stays refused whether or not either member is answering. A response + that suppressed fan-out (``X-Medkit-No-Fan-Out``) omits peer-owned items, + because the peers were never asked, but still qualifies what it does list; - answers any request addressed to it with ``504`` and the SOVD standard code ``not-responding``, naming the member - rather than being forwarded to the silent peer and surfacing as a ``502``, or falling through to a local read diff --git a/src/ros2_medkit_gateway/README.md b/src/ros2_medkit_gateway/README.md index ecc1b7588..a0c967ea3 100644 --- a/src/ros2_medkit_gateway/README.md +++ b/src/ros2_medkit_gateway/README.md @@ -299,8 +299,12 @@ header. When a peer stops answering, the entities it declared in its manifest are retained and marked unavailable (`x-medkit.available: false`, `x-medkit.is_online: false`); the ones it only discovered at runtime disappear. -A request addressed to a retained entity answers `504 not-responding` naming -the member, instead of being forwarded to the silent peer as a `502`. `/health` +A retained member keeps the operations it last reported: they stay listed on the +aggregating entity marked `x-medkit.available: false`, and still count towards +ambiguity, so a qualified id never degrades back to a bare one that execution +would refuse. A request addressed to a retained entity answers +`504 not-responding` naming the member, instead of being forwarded to the silent +peer as a `502`. `/health` still reports the peer itself as `offline` - entity availability and peer health are separate questions. diff --git a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/aggregation/aggregation_manager.hpp b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/aggregation/aggregation_manager.hpp index 4d7a9e272..aedbc7383 100644 --- a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/aggregation/aggregation_manager.hpp +++ b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/aggregation/aggregation_manager.hpp @@ -300,10 +300,29 @@ class AggregationManager { private: AggregationConfig config_; rclcpp::Logger logger_; - size_t static_peer_count_{0}; ///< Number of statically configured peers (not subject to max_discovered_peers) + size_t static_peer_count_{0}; ///< Number of statically configured peers (not subject to max_discovered_peers) + /// Record what `peer_name` declared, so it can be replayed while that peer is + /// silent. Stored already marked unavailable - it is only ever read back on + /// the path where the peer did not answer. + void remember_declaration(const std::string & peer_name, const PeerEntities & entities); + + /// The retained declaration for `peer_name`, empty if none was ever recorded. + PeerEntities replay_declaration(const std::string & peer_name) const; + mutable std::shared_mutex mutex_; // Declared before data it protects (destruction order) std::vector> peers_; std::unordered_map routing_table_; + /// Last known manifest-declared entities per peer name, replayed while that + /// peer is not answering. + /// + /// A tree that changes shape because a link went down cannot be reasoned + /// about: the same request gets a different answer depending on who happens + /// to be reachable. What a peer DECLARED is a property of its configuration + /// and does not stop being true when it stops replying, so it is retained and + /// marked unavailable. What a peer merely DISCOVERED at runtime is a property + /// of a live graph this gateway can no longer observe, so it is allowed to + /// disappear. + std::unordered_map retained_peer_entities_; std::unordered_map> peer_contributors_by_entity_; std::vector leaf_warnings_; diff --git a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/discovery/models/app.hpp b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/discovery/models/app.hpp index 995497c12..cf1f5d2dd 100644 --- a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/discovery/models/app.hpp +++ b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/discovery/models/app.hpp @@ -133,6 +133,17 @@ struct App { std::string source = "manifest"; ///< "manifest" or "runtime" std::string original_id; ///< Pre-rename ID when collision-prefixed by aggregation std::vector contributors; ///< Aggregation provenance: "local" and/or "peer:" + /// What the contributing gateway itself called this entity: "manifest", + /// "runtime", "node" or "topic". `source` is overwritten with `peer:` + /// on arrival, because the identity-merge precedence keys on it, so the + /// origin would otherwise be lost - and the origin is what decides whether + /// the entity is retained when its peer stops answering. Empty for entities + /// this gateway discovered itself. + std::string declared_source; + /// False while the peer contributing this entity is not answering and the + /// entity is being retained from its last known declaration. A retained + /// entity stays addressable and reports why it cannot be reached. + bool available{true}; // === Serialization methods === @@ -159,7 +170,8 @@ inline bool operator==(const App & a, const App & b) { a.tags == b.tags && a.component_id == b.component_id && a.depends_on == b.depends_on && a.ros_binding == b.ros_binding && a.bound_fqn == b.bound_fqn && a.is_online == b.is_online && a.external == b.external && a.topics == b.topics && a.services == b.services && a.actions == b.actions && - a.source == b.source && a.original_id == b.original_id && a.contributors == b.contributors; + a.source == b.source && a.original_id == b.original_id && a.contributors == b.contributors && + a.declared_source == b.declared_source && a.available == b.available; } inline bool operator!=(const App & a, const App & b) { return !(a == b); diff --git a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/discovery/models/area.hpp b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/discovery/models/area.hpp index bd6b39a3b..33b27c368 100644 --- a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/discovery/models/area.hpp +++ b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/discovery/models/area.hpp @@ -41,6 +41,13 @@ struct Area { std::string parent_area_id; ///< Parent area ID for sub-areas std::string source; ///< Origin of this area (e.g., "manifest", "heuristic") std::vector contributors; ///< Aggregation provenance: "local" and/or "peer:" + /// What the contributing gateway itself called this entity: "manifest", + /// "runtime", "node" or "topic". `source` is overwritten with `peer:` + /// on arrival, because the identity-merge precedence keys on it, so the + /// origin would otherwise be lost - and the origin is what decides whether + /// the entity is retained when its peer stops answering. Empty for entities + /// this gateway discovered itself. + std::string declared_source; /** * @brief Convert to JSON representation @@ -132,7 +139,8 @@ struct Area { inline bool operator==(const Area & a, const Area & b) { return a.id == b.id && a.name == b.name && a.namespace_path == b.namespace_path && a.type == b.type && a.translation_id == b.translation_id && a.description == b.description && a.tags == b.tags && - a.parent_area_id == b.parent_area_id && a.source == b.source && a.contributors == b.contributors; + a.parent_area_id == b.parent_area_id && a.source == b.source && a.contributors == b.contributors && + a.declared_source == b.declared_source; } inline bool operator!=(const Area & a, const Area & b) { return !(a == b); diff --git a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/discovery/models/component.hpp b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/discovery/models/component.hpp index 276e2a80e..9e6556db1 100644 --- a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/discovery/models/component.hpp +++ b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/discovery/models/component.hpp @@ -47,11 +47,22 @@ struct Component { std::string parent_component_id; ///< Parent component ID for sub-components std::vector depends_on; ///< Component IDs this component depends on std::vector contributors; ///< Aggregation provenance: "local" and/or "peer:" - std::vector services; ///< Services exposed by this component - std::vector actions; ///< Actions exposed by this component - ComponentTopics topics; ///< Topics this component publishes/subscribes - std::optional host_metadata; ///< Host system metadata (for runtime default component) - AssetIdentity identity; ///< Asset-identity nameplate (merged across sources, per-field provenance) + /// What the contributing gateway itself called this entity: "manifest", + /// "runtime", "node" or "topic". `source` is overwritten with `peer:` + /// on arrival, because the identity-merge precedence keys on it, so the + /// origin would otherwise be lost - and the origin is what decides whether + /// the entity is retained when its peer stops answering. Empty for entities + /// this gateway discovered itself. + std::string declared_source; + /// False while the peer contributing this entity is not answering and the + /// entity is being retained from its last known declaration. A retained + /// entity stays addressable and reports why it cannot be reached. + bool available{true}; + std::vector services; ///< Services exposed by this component + std::vector actions; ///< Actions exposed by this component + ComponentTopics topics; ///< Topics this component publishes/subscribes + std::optional host_metadata; ///< Host system metadata (for runtime default component) + AssetIdentity identity; ///< Asset-identity nameplate (merged across sources, per-field provenance) /// Tri-state: nullopt = no layer classified this component, true = non-ROS /// external asset (PLC/fieldbus/device), false = explicitly a ROS component. @@ -195,7 +206,7 @@ inline bool operator==(const Component & a, const Component & b) { a.parent_component_id == b.parent_component_id && a.depends_on == b.depends_on && a.contributors == b.contributors && a.services == b.services && a.actions == b.actions && a.topics == b.topics && a.host_metadata == b.host_metadata && a.identity == b.identity && - a.external == b.external; + a.external == b.external && a.declared_source == b.declared_source && a.available == b.available; } inline bool operator!=(const Component & a, const Component & b) { return !(a == b); diff --git a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/discovery/models/function.hpp b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/discovery/models/function.hpp index c5acdaac4..b2caf7005 100644 --- a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/discovery/models/function.hpp +++ b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/discovery/models/function.hpp @@ -50,6 +50,13 @@ struct Function { // === Discovery metadata === std::string source = "manifest"; ///< Discovery source: manifest or runtime std::vector contributors; ///< Aggregation provenance: "local" and/or "peer:" + /// What the contributing gateway itself called this entity: "manifest", + /// "runtime", "node" or "topic". `source` is overwritten with `peer:` + /// on arrival, because the identity-merge precedence keys on it, so the + /// origin would otherwise be lost - and the origin is what decides whether + /// the entity is retained when its peer stops answering. Empty for entities + /// this gateway discovered itself. + std::string declared_source; // === Serialization methods === @@ -74,7 +81,7 @@ struct Function { inline bool operator==(const Function & a, const Function & b) { return a.id == b.id && a.name == b.name && a.translation_id == b.translation_id && a.description == b.description && a.tags == b.tags && a.hosts == b.hosts && a.depends_on == b.depends_on && a.source == b.source && - a.contributors == b.contributors; + a.contributors == b.contributors && a.declared_source == b.declared_source; } inline bool operator!=(const Function & a, const Function & b) { return !(a == b); diff --git a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/dto/operations.hpp b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/dto/operations.hpp index a49583b44..04f65b449 100644 --- a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/dto/operations.hpp +++ b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/dto/operations.hpp @@ -63,14 +63,17 @@ struct XMedkitOperationItem { /// short name. More than one entry is what makes the bare item id /// ambiguous for addressing. std::optional> member_ids; + /// Absent while the item can be served. False marks an item listed from a + /// retained declaration because the member that owns it is not answering - + /// the item is still part of the tree, and still counts towards ambiguity. + std::optional available; }; template <> -inline constexpr auto dto_fields = - std::make_tuple(field("ros2", &XMedkitOperationItem::ros2), field("entity_id", &XMedkitOperationItem::entity_id), - field("source", &XMedkitOperationItem::source), - field("type_info", &XMedkitOperationItem::type_info), - field("member_ids", &XMedkitOperationItem::member_ids)); +inline constexpr auto dto_fields = std::make_tuple( + field("ros2", &XMedkitOperationItem::ros2), field("entity_id", &XMedkitOperationItem::entity_id), + field("source", &XMedkitOperationItem::source), field("type_info", &XMedkitOperationItem::type_info), + field("member_ids", &XMedkitOperationItem::member_ids), field("available", &XMedkitOperationItem::available)); template <> inline constexpr std::string_view dto_name = "XMedkitOperationItem"; diff --git a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/dto/x_medkit.hpp b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/dto/x_medkit.hpp index caacbb74d..4d043900d 100644 --- a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/dto/x_medkit.hpp +++ b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/dto/x_medkit.hpp @@ -126,6 +126,8 @@ struct XMedkitComponent { std::optional identity; std::optional missing; // broken reference sentinel std::optional external; // non-ROS external asset classification (#516) + // Emitted only when false; an absent field means reachable. + std::optional available; }; template <> @@ -136,7 +138,7 @@ inline constexpr auto dto_fields = std::make_tuple( field("variant", &XMedkitComponent::variant), field("description", &XMedkitComponent::description), field("contributors", &XMedkitComponent::contributors), field("capabilities", &XMedkitComponent::capabilities), field("identity", &XMedkitComponent::identity), field("missing", &XMedkitComponent::missing), - field("external", &XMedkitComponent::external)); + field("external", &XMedkitComponent::external), field("available", &XMedkitComponent::available)); template <> inline constexpr std::string_view dto_name = "XMedkitComponent"; @@ -164,6 +166,10 @@ struct XMedkitApp { std::optional> contributors; std::optional missing; // broken reference sentinel std::optional external; // non-ROS external asset classification (#516/#517) + /// Absent while the entity is reachable. False marks an entity retained from + /// a peer's last known declaration while that peer is not answering, so a + /// client can tell "declared but unreachable" from "gone". + std::optional available; }; template <> @@ -171,7 +177,7 @@ inline constexpr auto dto_fields = std::make_tuple(field("ros2", &XMedkitApp::ros2), field("source", &XMedkitApp::source), field("is_online", &XMedkitApp::is_online), field("component_id", &XMedkitApp::component_id), field("contributors", &XMedkitApp::contributors), field("missing", &XMedkitApp::missing), - field("external", &XMedkitApp::external)); + field("external", &XMedkitApp::external), field("available", &XMedkitApp::available)); template <> inline constexpr std::string_view dto_name = "XMedkitApp"; diff --git a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/http/handlers/handler_context.hpp b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/http/handlers/handler_context.hpp index adb4e14aa..5d5c631fe 100644 --- a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/http/handlers/handler_context.hpp +++ b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/http/handlers/handler_context.hpp @@ -290,6 +290,10 @@ class HandlerContext { * @param res HTTP response * @param origin Origin header value */ + /// False when the entity is only present because a peer's declaration is + /// being retained while that peer is silent. + bool is_entity_available(const std::string & entity_id) const; + void set_cors_headers(httplib::Response & res, const std::string & origin) const; /** diff --git a/src/ros2_medkit_gateway/src/aggregation/aggregation_manager.cpp b/src/ros2_medkit_gateway/src/aggregation/aggregation_manager.cpp index b4715a38a..7c99381a6 100644 --- a/src/ros2_medkit_gateway/src/aggregation/aggregation_manager.cpp +++ b/src/ros2_medkit_gateway/src/aggregation/aggregation_manager.cpp @@ -355,6 +355,72 @@ PeerEntities AggregationManager::fetch_all_peer_entities() { return merged; } +namespace { + +/// True for an entity a peer said it had DECLARED, rather than discovered from +/// its live ROS graph. Only a declaration outlives the link that reported it: +/// runtime discovery describes a graph this gateway can no longer observe once +/// the peer stops answering, so it is not ours to keep asserting. +template +bool is_declared(const Entity & entity) { + return entity.declared_source == "manifest"; +} + +/// Availability belongs to an entity a request can be addressed to. A grouping +/// entity has none of its own: it is a view over its members, and the members +/// carry the flag, so retaining one says nothing about what can be reached. +void mark_unreachable(Area &) { +} +void mark_unreachable(Function &) { +} +void mark_unreachable(App & app) { + app.available = false; +} +void mark_unreachable(Component & component) { + component.available = false; +} + +/// The declared entities of `src`, with the addressable ones marked unreachable. +template +std::vector declared_only(const std::vector & src) { + std::vector kept; + for (const auto & entity : src) { + if (!is_declared(entity)) { + continue; + } + kept.push_back(entity); + mark_unreachable(kept.back()); + } + return kept; +} + +} // namespace + +void AggregationManager::remember_declaration(const std::string & peer_name, const PeerEntities & entities) { + PeerEntities declared; + declared.areas = declared_only(entities.areas); + declared.components = declared_only(entities.components); + declared.apps = declared_only(entities.apps); + declared.functions = declared_only(entities.functions); + + // A retained App is not observably running: the graph it was bound to is on + // the other side of a link that is down. `is_online` is the field every + // consumer already reads for that, so it carries the news rather than a + // second flag they would each have to learn. + for (auto & app : declared.apps) { + app.is_online = false; + } + + std::unique_lock lock(mutex_); + retained_peer_entities_[peer_name] = std::move(declared); +} + +PeerEntities AggregationManager::replay_declaration(const std::string & peer_name) const { + std::shared_lock lock(mutex_); + auto it = retained_peer_entities_.find(peer_name); + return it == retained_peer_entities_.end() ? PeerEntities{} : it->second; +} + AggregationManager::MergedPeerResult AggregationManager::fetch_and_merge_peer_entities( const std::vector & local_areas, const std::vector & local_components, const std::vector & local_apps, const std::vector & local_functions, size_t max_entities_per_peer, @@ -384,11 +450,14 @@ AggregationManager::MergedPeerResult AggregationManager::fetch_and_merge_peer_en // shared_ptr copies keep PeerClients alive even if remove_discovered_peer() // erases them from peers_ concurrently. std::vector> snapshot; + std::vector silent_peers; { std::shared_lock lock(mutex_); for (const auto & peer : peers_) { if (peer->is_healthy()) { snapshot.push_back(peer); + } else { + silent_peers.push_back(peer->name()); } } } @@ -437,15 +506,42 @@ AggregationManager::MergedPeerResult AggregationManager::fetch_and_merge_peer_en std::vector peer_component_claims; // Collect results and merge sequentially (merge order must be deterministic) + std::vector to_merge; + to_merge.reserve(futures.size() + silent_peers.size()); for (auto & f : futures) { auto pfr = f.get(); if (!pfr.success) { if (logger) { RCLCPP_WARN(*logger, "Failed to fetch entities from peer '%s': %s", pfr.peer_name.c_str(), pfr.error.c_str()); } + silent_peers.push_back(pfr.peer_name); continue; } + remember_declaration(pfr.peer_name, pfr.entities); + to_merge.push_back(std::move(pfr)); + } + + // A peer that did not answer still contributes what it declared, marked + // unavailable. Merged after the peers that did answer, so a live declaration + // always wins over a retained one carrying the same id. + for (const auto & peer_name : silent_peers) { + auto retained = replay_declaration(peer_name); + if (retained.areas.empty() && retained.components.empty() && retained.apps.empty() && retained.functions.empty()) { + continue; + } + if (logger) { + RCLCPP_INFO(*logger, "Peer '%s' not answering; retaining %zu declared entities as unavailable", peer_name.c_str(), + retained.areas.size() + retained.components.size() + retained.apps.size() + + retained.functions.size()); + } + PeerFetchResult replayed; + replayed.peer_name = peer_name; + replayed.success = true; + replayed.entities = std::move(retained); + to_merge.push_back(std::move(replayed)); + } + for (auto & pfr : to_merge) { PeerClaim claim; claim.peer_name = pfr.peer_name; for (const auto & c : pfr.entities.components) { diff --git a/src/ros2_medkit_gateway/src/core/aggregation/peer_client.cpp b/src/ros2_medkit_gateway/src/core/aggregation/peer_client.cpp index 1204eba85..eb9147c92 100644 --- a/src/ros2_medkit_gateway/src/core/aggregation/peer_client.cpp +++ b/src/ros2_medkit_gateway/src/core/aggregation/peer_client.cpp @@ -225,6 +225,57 @@ std::string component_id_from_located_on(const std::string & uri) { return is_valid_entity_id(candidate) ? candidate : std::string{}; } +/** + * @brief Read a peer's `/operations` collection into an App's service/action lists. + * + * The wire id is not used as the name: on the peer it is already qualified when + * that peer saw a collision among its own members, and the qualifier is the + * peer's business, not ours. `name` is always the bare short name, which is what + * ambiguity is keyed on, and the full ROS path comes from x-medkit.ros2 so two + * same-named operations stay distinguishable. + */ +void parse_operations_into(const nlohmann::json & j, App & app) { + if (!j.contains("items") || !j["items"].is_array()) { + return; + } + for (const auto & item : j["items"]) { + if (!item.is_object()) { + continue; + } + const std::string name = item.value("name", ""); + if (name.empty()) { + continue; + } + std::string full_path; + std::string type; + bool is_action = item.value("asynchronous_execution", false); + if (item.contains("x-medkit") && item["x-medkit"].is_object()) { + const auto & xm = item["x-medkit"]; + if (xm.contains("ros2") && xm["ros2"].is_object()) { + const auto & ros2 = xm["ros2"]; + type = ros2.value("type", ""); + full_path = ros2.value("service", ""); + if (full_path.empty()) { + full_path = ros2.value("action", ""); + is_action = is_action || !full_path.empty(); + } + const std::string kind = ros2.value("kind", ""); + if (!kind.empty()) { + is_action = kind == "action"; + } + } + } + if (full_path.empty()) { + continue; + } + if (is_action) { + app.actions.push_back(ActionInfo{name, full_path, type, std::nullopt}); + } else { + app.services.push_back(ServiceInfo{name, full_path, type, std::nullopt}); + } + } +} + /** * @brief Parse an App from JSON. * @@ -405,6 +456,7 @@ tl::expected PeerClient::fetch_entities() { " areas (max " + std::to_string(MAX_ENTITIES_PER_COLLECTION) + ")"); } for (auto & area : entities.areas) { + area.declared_source = area.source; area.source = peer_source; } @@ -422,6 +474,7 @@ tl::expected PeerClient::fetch_entities() { if (!is_valid_entity_id(sub.id)) { continue; } + sub.declared_source = sub.source; sub.source = peer_source; all_subareas.push_back(std::move(sub)); } @@ -469,6 +522,7 @@ tl::expected PeerClient::fetch_entities() { comp = parse_component(detail_json); } } + comp.declared_source = comp.source; comp.source = peer_source; } // Fetch subcomponents for each top-level component (list endpoint filters them out). @@ -493,6 +547,7 @@ tl::expected PeerClient::fetch_entities() { sub = parse_component(detail_json); } } + sub.declared_source = sub.source; sub.source = peer_source; all_subcomps.push_back(std::move(sub)); } @@ -534,6 +589,7 @@ tl::expected PeerClient::fetch_entities() { " apps (max " + std::to_string(MAX_ENTITIES_PER_COLLECTION) + ")"); } for (auto & app : entities.apps) { + app.declared_source = app.source; app.source = peer_source; } // Filter ROS 2 internal nodes (underscore prefix convention) at source. @@ -544,6 +600,29 @@ tl::expected PeerClient::fetch_entities() { return !app.id.empty() && app.id[0] == '_'; }), entities.apps.end()); + + // Fetch each app's operations. An operation is never declared in a + // manifest - it is discovered from the ROS graph - so the only record of + // what a peer's app exposes is what the peer reports. Without it the + // aggregator cannot tell that two members share an operation short name + // except by asking at request time, and an answer that depends on who is + // reachable is not an answer a client can rely on. + // + // `X-Medkit-No-Fan-Out` keeps the peer from re-asking ITS peers: each + // gateway reports what it holds, and the hop that owns the entity is the + // hop that answers for it. It is also what makes this terminate. + for (auto & app : entities.apps) { + httplib::Headers no_fan_out{{"X-Medkit-No-Fan-Out", "1"}}; + auto ops_result = cli.Get(std::string(API_PREFIX) + "/apps/" + app.id + "/operations", no_fan_out); + if (!ops_result || ops_result->status != 200 || ops_result->body.size() > MAX_PEER_RESPONSE_SIZE) { + continue; + } + auto ops_json = nlohmann::json::parse(ops_result->body, nullptr, false); + if (ops_json.is_discarded()) { + continue; + } + parse_operations_into(ops_json, app); + } } // Fetch functions (list then detail per entity for hosts data) @@ -583,6 +662,7 @@ tl::expected PeerClient::fetch_entities() { func = parse_function(detail_json); } } + func.declared_source = func.source; func.source = peer_source; } entities.functions = std::move(func_list); diff --git a/src/ros2_medkit_gateway/src/http/handlers/discovery_handlers.cpp b/src/ros2_medkit_gateway/src/http/handlers/discovery_handlers.cpp index 41f4079e8..bdb4db90c 100644 --- a/src/ros2_medkit_gateway/src/http/handlers/discovery_handlers.cpp +++ b/src/ros2_medkit_gateway/src/http/handlers/discovery_handlers.cpp @@ -51,6 +51,16 @@ void set_x_medkit_external(dto::XMedkitApp & x_medkit, const std::optional } } +/// Emit the x-medkit `available` flag only when the entity cannot be reached, +/// so an absent field means reachable. Carried by the entities a request can be +/// addressed to; a grouping entity is a view over members and has no +/// availability of its own. +void set_x_medkit_available(dto::XMedkitComponent & x_medkit, bool available) { + if (!available) { + x_medkit.available = false; + } +} + /// Check if a capability name is already present in the capabilities array bool has_capability(const json & capabilities, const std::string & name) { for (const auto & cap : capabilities) { @@ -371,6 +381,7 @@ DiscoveryHandlers::get_area_components(const http::TypedRequest & req) { } dto::XMedkitComponent x_medkit_comp; + set_x_medkit_available(x_medkit_comp, component.available); if (!component.source.empty()) { x_medkit_comp.source = component.source; } @@ -526,6 +537,7 @@ DiscoveryHandlers::get_area_contains(const http::TypedRequest & req) { item.type = "component"; dto::XMedkitComponent x_medkit_comp; + set_x_medkit_available(x_medkit_comp, comp.available); if (!comp.source.empty()) { x_medkit_comp.source = comp.source; } @@ -588,6 +600,7 @@ DiscoveryHandlers::get_components(const http::TypedRequest & req) { } dto::XMedkitComponent x_medkit_comp; + set_x_medkit_available(x_medkit_comp, component.available); if (!component.source.empty()) { x_medkit_comp.source = component.source; } @@ -698,6 +711,7 @@ http::Result DiscoveryHandlers::get_component(const http:: detail.links = links.build(); dto::XMedkitComponent x_medkit_comp; + set_x_medkit_available(x_medkit_comp, comp.available); if (!comp.source.empty()) { x_medkit_comp.source = comp.source; } @@ -810,6 +824,7 @@ DiscoveryHandlers::get_subcomponents(const http::TypedRequest & req) { item.type = "component"; dto::XMedkitComponent x_medkit_comp; + set_x_medkit_available(x_medkit_comp, sub.available); if (!sub.source.empty()) { x_medkit_comp.source = sub.source; } @@ -885,6 +900,10 @@ http::Result> DiscoveryHandlers::get_component dto::XMedkitApp x_medkit_app; x_medkit_app.is_online = app.is_online; + // Emitted only when false: an absent field means reachable. + if (!app.available) { + x_medkit_app.available = false; + } if (!app.source.empty()) { x_medkit_app.source = app.source; } @@ -955,6 +974,7 @@ DiscoveryHandlers::get_component_depends_on(const http::TypedRequest & req) { item.name = dep_opt->name.empty() ? dep_id : dep_opt->name; dto::XMedkitComponent x_medkit_comp; + set_x_medkit_available(x_medkit_comp, dep_opt->available); if (!dep_opt->source.empty()) { x_medkit_comp.source = dep_opt->source; } @@ -1018,6 +1038,10 @@ http::Result> DiscoveryHandlers::get_apps(cons x_medkit_app.source = app.source; } x_medkit_app.is_online = app.is_online; + // Emitted only when false: an absent field means reachable. + if (!app.available) { + x_medkit_app.available = false; + } if (!app.component_id.empty()) { x_medkit_app.component_id = app.component_id; } @@ -1155,6 +1179,10 @@ http::Result DiscoveryHandlers::get_app(const http::TypedRequest x_medkit_app.source = app.source; } x_medkit_app.is_online = app.is_online; + // Emitted only when false: an absent field means reachable. + if (!app.available) { + x_medkit_app.available = false; + } if (app.bound_fqn) { dto::XMedkitRos2 ros2; ros2.node = *app.bound_fqn; @@ -1226,6 +1254,10 @@ http::Result> DiscoveryHandlers::get_app_depen x_medkit_app.source = dep_opt->source; } x_medkit_app.is_online = dep_opt->is_online; + // Emitted only when false: an absent field means reachable. + if (!dep_opt->available) { + x_medkit_app.available = false; + } set_x_medkit_external(x_medkit_app, dep_opt->external); item.x_medkit = x_medkit_app; } else { @@ -1607,6 +1639,10 @@ http::Result> DiscoveryHandlers::get_function_ dto::XMedkitApp x_medkit_app; x_medkit_app.is_online = app_opt->is_online; + // Emitted only when false: an absent field means reachable. + if (!app_opt->available) { + x_medkit_app.available = false; + } if (!app_opt->source.empty()) { x_medkit_app.source = app_opt->source; } diff --git a/src/ros2_medkit_gateway/src/http/handlers/handler_context.cpp b/src/ros2_medkit_gateway/src/http/handlers/handler_context.cpp index 58beaf582..e037bba82 100644 --- a/src/ros2_medkit_gateway/src/http/handlers/handler_context.cpp +++ b/src/ros2_medkit_gateway/src/http/handlers/handler_context.cpp @@ -313,6 +313,23 @@ http::ValidatorResult HandlerContext::validate_entity_for_route(cons // aggregation) the forwarding path returns Forwarded without mutating any // wire - the framework guarantees a sink whenever aggregation is active. if (entity_info.is_remote && aggregation_mgr_) { + // A retained entity is one its peer declared and is no longer answering + // for. Forwarding to that peer produces a connection failure dressed as a + // 502, which says the gateway had a problem rather than that the thing + // asked for is not reachable. The retained declaration is exactly the + // information needed to answer properly, so answer here. + if (!is_entity_available(entity_id)) { + ErrorInfo err; + err.code = ERR_NOT_RESPONDING; + err.message = "Member '" + entity_id + "' is not available"; + err.http_status = 504; + err.params = json{{"entity_id", entity_id}, + {"peer", entity_info.peer_name}, + {"details", + "The gateway contributing this entity is not answering; it is retained from its " + "last known declaration"}}; + return tl::unexpected(ErrorVariant{std::move(err)}); + } if (tl_forward_response != nullptr) { aggregation_mgr_->forward_request(entity_info.peer_name, raw_req, *tl_forward_response); } @@ -322,6 +339,20 @@ http::ValidatorResult HandlerContext::validate_entity_for_route(cons return entity_info; } +/// False when the entity is present only because its peer's declaration is +/// being retained. An entity this gateway has never heard of is not "not +/// available" - it is absent, and the caller already got a 404 for it. +bool HandlerContext::is_entity_available(const std::string & entity_id) const { + const auto & cache = node_->get_thread_safe_cache(); + if (auto app = cache.get_app(entity_id)) { + return app->available; + } + if (auto component = cache.get_component(entity_id)) { + return component->available; + } + return true; +} + void HandlerContext::set_cors_headers(httplib::Response & res, const std::string & origin) const { res.set_header("Access-Control-Allow-Origin", origin); diff --git a/src/ros2_medkit_gateway/src/http/handlers/operation_handlers.cpp b/src/ros2_medkit_gateway/src/http/handlers/operation_handlers.cpp index 05df3768a..a36859ffd 100644 --- a/src/ros2_medkit_gateway/src/http/handlers/operation_handlers.cpp +++ b/src/ros2_medkit_gateway/src/http/handlers/operation_handlers.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -478,10 +479,18 @@ http::Result> OperationHandlers::list_operat auto data_access_mgr = ctx_.node()->get_data_access_manager(); auto type_introspection = data_access_mgr->get_type_introspection(); - // A peer's declared operations live in this cache so that ambiguity can be - // decided without asking anyone at request time. They are not listed from - // here: the gateway that owns an operation is the one that reports it, and - // this walk runs even when the caller asked for no fan-out at all. + // How many members the DECLARED tree says provide each short name. This is + // the same count `create_execution` refuses on, and it is read here so the + // listing and the execution cannot disagree: an id the tree calls ambiguous + // is never offered bare, wherever the copy came from. + std::unordered_map declared_providers; + for (const auto & svc : ops.services) { + ++declared_providers[svc.name]; + } + for (const auto & act : ops.actions) { + ++declared_providers[act.name]; + } + const auto contributed_by_peer = [&cache](const std::string & member_id) { static constexpr std::string_view kPeerPrefix = "peer:"; if (auto app = cache.get_app(member_id)) { @@ -497,35 +506,56 @@ http::Result> OperationHandlers::list_operat return owner != ops.owner_by_path.end() && contributed_by_peer(owner->second); }; - for (const auto & svc : ops.services) { - if (owner_is_remote(svc.full_path)) { - continue; + // Qualify from the declared tree rather than by counting copies in this + // response. A response can be short a copy - the caller suppressed fan-out, + // or a peer did not answer - and counting copies would then hand back a bare + // id that the execution refuses. + const auto qualify_from_declared_tree = [&declared_providers](dto::OperationItem & item) { + if (item.id != item.name) { + return; // already qualified, by us or by the peer that sent it } - dto::OperationItem item; - item.id = svc.name; - item.name = svc.name; - item.proximity_proof_required = false; - item.asynchronous_execution = false; - item.x_medkit = build_service_xmedkit(svc, entity_id, type_introspection); - if (auto owner = ops.owner_by_path.find(svc.full_path); owner != ops.owner_by_path.end() && ops.is_aggregated) { - item.x_medkit->member_ids = std::vector{owner->second}; + auto count = declared_providers.find(item.name); + if (count == declared_providers.end() || count->second < 2) { + return; } - collection.items.push_back(std::move(item)); - } - for (const auto & act : ops.actions) { - if (owner_is_remote(act.full_path)) { - continue; + if (!item.x_medkit.has_value() || !item.x_medkit->member_ids.has_value() || + item.x_medkit->member_ids->size() != 1) { + return; } + item.id = http::make_member_qualified_id(item.x_medkit->member_ids->front(), item.name); + }; + + const auto build_item = [&](const auto & op, bool asynchronous) { dto::OperationItem item; - item.id = act.name; - item.name = act.name; + item.id = op.name; + item.name = op.name; item.proximity_proof_required = false; - item.asynchronous_execution = true; - item.x_medkit = build_action_xmedkit(act, entity_id, type_introspection); - if (auto owner = ops.owner_by_path.find(act.full_path); owner != ops.owner_by_path.end() && ops.is_aggregated) { + item.asynchronous_execution = asynchronous; + if constexpr (std::is_same_v, ServiceInfo>) { + item.x_medkit = build_service_xmedkit(op, entity_id, type_introspection); + } else { + item.x_medkit = build_action_xmedkit(op, entity_id, type_introspection); + } + if (auto owner = ops.owner_by_path.find(op.full_path); owner != ops.owner_by_path.end() && ops.is_aggregated) { item.x_medkit->member_ids = std::vector{owner->second}; } - collection.items.push_back(std::move(item)); + qualify_from_declared_tree(item); + return item; + }; + + // A peer's operations are held here so ambiguity can be decided without + // asking anyone. They are not reported from this walk while the peer is + // reachable - the gateway that owns an operation is the one that reports it - + // so they are set aside and only fall back into the list below, when the + // fan-out that should have carried them did not. + std::vector retained_from_peers; + for (const auto & svc : ops.services) { + auto item = build_item(svc, false); + (owner_is_remote(svc.full_path) ? retained_from_peers : collection.items).push_back(std::move(item)); + } + for (const auto & act : ops.actions) { + auto item = build_item(act, true); + (owner_is_remote(act.full_path) ? retained_from_peers : collection.items).push_back(std::move(item)); } // Typed fan-out for the operations list. Replacement for the legacy raw-JSON @@ -539,16 +569,42 @@ http::Result> OperationHandlers::list_operat #pragma GCC diagnostic ignored "-Wdeprecated-declarations" const auto & raw_req = req.raw_for_framework(); #pragma GCC diagnostic pop + // Two different reasons a peer's copy can be missing, and they are not the + // same answer. The caller asking for no fan-out means the peers were never + // consulted, and reporting their items anyway is what turns a bidirectionally + // peered pair into a bounce. A fan-out that ran and came back without them + // means the peer is not answering, and the tree still knows what it declared. + const bool fan_out_suppressed = raw_req.has_header("X-Medkit-No-Fan-Out"); auto fan_out = fan_out_collection(ctx_.aggregation_manager(), raw_req); + std::unordered_set paths_from_peers; for (auto & item : fan_out.items) { + if (item.x_medkit.has_value() && item.x_medkit->ros2.has_value()) { + const auto & ros2 = *item.x_medkit->ros2; + auto path = ros2.service.value_or(ros2.action.value_or(std::string{})); + if (!path.empty()) { + paths_from_peers.insert(std::move(path)); + } + } + qualify_from_declared_tree(item); collection.items.push_back(std::move(item)); } - // Two members exposing one short name are two items with one id, and the - // merged collection is the first place that is visible: each gateway holds - // one `calibrate` and considers it unique. An id only one item carries is - // left alone - it already names one thing, and rewriting it would break - // every client that sends the bare name. + if (!fan_out_suppressed) { + for (auto & item : retained_from_peers) { + const auto & ros2 = *item.x_medkit->ros2; + auto path = ros2.service.value_or(ros2.action.value_or(std::string{})); + if (!path.empty() && paths_from_peers.count(path) > 0u) { + continue; // the owner answered for itself, which is the better copy + } + item.x_medkit->available = false; + collection.items.push_back(std::move(item)); + } + } + + // Catches ambiguity the declared tree has not seen: a copy that only reached + // this gateway through the fan-out, from a member whose operations are not in + // the local cache yet. An id only one item carries is left alone - it already + // names one thing, and rewriting it would break every client sending it. http::qualify_ambiguous_ids(collection.items, [](const dto::OperationItem & item) { return item.x_medkit.has_value() && item.x_medkit->member_ids.has_value() ? &*item.x_medkit->member_ids : nullptr; }); diff --git a/src/ros2_medkit_gateway/test/test_aggregation_manager.cpp b/src/ros2_medkit_gateway/test/test_aggregation_manager.cpp index b93c6078d..cb120f594 100644 --- a/src/ros2_medkit_gateway/test/test_aggregation_manager.cpp +++ b/src/ros2_medkit_gateway/test/test_aggregation_manager.cpp @@ -1609,3 +1609,113 @@ TEST(AggregationManager, forward_accepts_api_v1_path) { // Should get 502 (peer unreachable), not 400 (path rejected) EXPECT_EQ(res.status, 502); } + +// A tree that changes shape when a link drops cannot be reasoned about: the +// same request gets a different answer depending on who happens to be +// reachable. What a peer DECLARED stays true while it is silent; what it merely +// discovered from a live graph does not, because nothing can observe that graph +// any more. These two tests pin both halves of that split. +namespace { + +/// Mock peer serving one manifest-declared app and one runtime-discovered app. +void install_two_origin_apps(httplib::Server & svr) { + svr.Get("/api/v1/health", [](const httplib::Request &, httplib::Response & res) { + res.set_content(R"({"status":"healthy"})", "application/json"); + }); + svr.Get("/api/v1/areas", [](const httplib::Request &, httplib::Response & res) { + res.set_content(R"({"items":[]})", "application/json"); + }); + svr.Get(R"(/api/v1/areas/([^/]+)/subareas)", [](const httplib::Request &, httplib::Response & res) { + res.set_content(R"({"items":[]})", "application/json"); + }); + svr.Get("/api/v1/components", [](const httplib::Request &, httplib::Response & res) { + res.set_content(R"({"items":[]})", "application/json"); + }); + svr.Get("/api/v1/functions", [](const httplib::Request &, httplib::Response & res) { + res.set_content(R"({"items":[]})", "application/json"); + }); + svr.Get("/api/v1/apps", [](const httplib::Request &, httplib::Response & res) { + nlohmann::json items = nlohmann::json::array(); + items.push_back({{"id", "declared_app"}, {"name", "Declared"}, {"x-medkit", {{"source", "manifest"}}}}); + items.push_back({{"id", "discovered_app"}, {"name", "Discovered"}, {"x-medkit", {{"source", "runtime"}}}}); + res.set_content(nlohmann::json({{"items", items}}).dump(), "application/json"); + }); + svr.Get(R"(/api/v1/apps/([^/]+)/operations)", [](const httplib::Request &, httplib::Response & res) { + res.set_content(R"({"items":[]})", "application/json"); + }); +} + +const App * find_app(const std::vector & apps, const std::string & id) { + for (const auto & app : apps) { + if (app.id == id) { + return &app; + } + } + return nullptr; +} + +} // namespace + +TEST(AggregationManager, retains_declared_peer_entities_when_the_peer_goes_silent) { + AggregationConfig config; + config.enabled = true; + config.timeout_ms = 1000; + AggregationConfig::PeerConfig peer; + peer.name = "silent_peer"; + + int port = 0; + { + MockPeerServer mock; + install_two_origin_apps(mock.server()); + port = mock.start(); + peer.url = "http://127.0.0.1:" + std::to_string(port); + config.peers.push_back(peer); + + AggregationManager manager(config); + manager.check_all_health(); + ASSERT_EQ(manager.healthy_peer_count(), 1u); + + auto live = manager.fetch_and_merge_peer_entities({}, {}, {}, {}); + const App * declared = find_app(live.apps, "declared_app"); + const App * discovered = find_app(live.apps, "discovered_app"); + ASSERT_NE(declared, nullptr); + ASSERT_NE(discovered, nullptr); + EXPECT_EQ(declared->declared_source, "manifest") << "the peer's own origin was overwritten by peer:"; + EXPECT_TRUE(declared->available); + EXPECT_TRUE(discovered->available); + + // The peer stops answering: health goes false and the fetch fails. + mock.server().stop(); + manager.check_all_health(); + ASSERT_EQ(manager.healthy_peer_count(), 0u); + + auto silent = manager.fetch_and_merge_peer_entities({}, {}, {}, {}); + const App * retained = find_app(silent.apps, "declared_app"); + ASSERT_NE(retained, nullptr) << "a declared entity vanished when its peer went quiet"; + EXPECT_FALSE(retained->available) << "a retained entity must say it cannot be reached"; + EXPECT_FALSE(retained->is_online) << "a retained app is not observably running"; + EXPECT_EQ(find_app(silent.apps, "discovered_app"), nullptr) + << "a runtime-discovered entity describes a graph this gateway can no longer observe"; + } +} + +TEST(AggregationManager, retains_nothing_for_a_peer_that_never_answered) { + // Never reachable means nothing was ever declared to us, so there is nothing + // to keep asserting - the entity is absent, not unavailable. + AggregationConfig config; + config.enabled = true; + config.timeout_ms = 200; + AggregationConfig::PeerConfig peer; + peer.url = "http://127.0.0.1:1"; // nothing listens here + peer.name = "never_up"; + config.peers.push_back(peer); + + AggregationManager manager(config); + manager.check_all_health(); + ASSERT_EQ(manager.healthy_peer_count(), 0u); + + auto result = manager.fetch_and_merge_peer_entities({}, {}, {}, {}); + EXPECT_TRUE(result.apps.empty()); + EXPECT_TRUE(result.components.empty()); + EXPECT_TRUE(result.routing_table.empty()); +} diff --git a/src/ros2_medkit_integration_tests/test/features/test_grouping_entity_aggregation.test.py b/src/ros2_medkit_integration_tests/test/features/test_grouping_entity_aggregation.test.py index 13a239909..09234558c 100644 --- a/src/ros2_medkit_integration_tests/test/features/test_grouping_entity_aggregation.test.py +++ b/src/ros2_medkit_integration_tests/test/features/test_grouping_entity_aggregation.test.py @@ -74,9 +74,16 @@ one peer would discard the other half, which is what happens today. R8 A lock on a leaf is honoured by a request dispatched through an aggregate. R9 Peered gateways terminate. +R10 A manifest-declared entity outlives the link that reported it: it stays in + the tree, keeps the items it last reported, and says it cannot be reached. + A runtime-discovered one vanishes, because it describes a graph this + gateway can no longer observe. Availability belongs to what a request can + be addressed to - an App or a Component. A grouping entity is a view over + members and carries none of its own; a client asks the members. """ import os +import signal import tempfile import time import unittest @@ -183,7 +190,10 @@ name: "Secondary ECU" version: "1.0.0" config: - unmanifested_nodes: ignore + # The peer exposes what it did not declare, so that its half of the tree has + # both origins in it. Retention keeps the declared ones and drops these, and + # a topology where everything is declared cannot show the difference. + unmanifested_nodes: warn areas: - id: {MERGED_AREA} name: "Vehicle" @@ -587,3 +597,353 @@ def test_loop_suppression_is_carried_on_every_hop(self): 'peer_calibration', members, 'suppression header ignored, so a peered pair would not terminate', ) + + # ------------------------------------------------------------------ R10 + # A PEER THAT STOPS ANSWERING. + # + # Everything above runs against a live pair. These run after it, because + # unittest orders methods alphabetically within a class and these are the + # only ones prefixed `test_z`. The peer is killed exactly once, by the + # first of them, and the rest read the aggregator afterwards. + # + # The suite has never taken a peer down before, which is why "the entities + # simply vanish" survived so long: every case measured a healthy pair, and + # a tree that changes shape when a link drops looks identical to a tree + # that never had the entity. + + #: Seconds the aggregator took to notice, filled in by the first case. + _noticed_after_s = None + #: Ids the PEER called runtime-discovered, captured before it was killed. + _peer_runtime_ids = None + #: Ids the PEER called manifest-declared, captured before it was killed. + _peer_declared_ids = None + + @staticmethod + def _peer_ids_by_declared_source(): + """Ask the PEER which of its entities are manifest- and which runtime-declared. + + The aggregator overwrites `source` with `peer:` on arrival, so + the peer's own answer is the only place the distinction is visible. + Both collections are read: this topology manifests every app, so the + runtime half of the split lives among the components. + """ + by_source = {} + + def record(collection, item): + source = item.get('x-medkit', {}).get('source', '') + by_source.setdefault(source, []).append((collection, item.get('id'))) + + for collection in ('apps', 'components'): + response = requests.get(f'{PEER_URL}/{collection}', timeout=10) + response.raise_for_status() + for item in response.json().get('items', []): + record(collection, item) + if collection != 'components': + continue + # A subcomponent is not in the flat list, and this topology puts + # the peer's leaf Component exactly there. + nested = requests.get( + f"{PEER_URL}/components/{item.get('id')}/subcomponents", timeout=10) + if nested.status_code != 200: + continue + for sub in nested.json().get('items', []): + record('components', sub) + return by_source + + @staticmethod + def _primary_entity(collection, entity_id): + """One entity as the PRIMARY currently sees it, or None.""" + response = requests.get(f'{PRIMARY_URL}/{collection}', timeout=10) + if response.status_code != 200: + return None + for item in response.json().get('items', []): + if item.get('id') == entity_id: + return item + return None + + @classmethod + def _primary_app(cls, app_id): + """One app as the PRIMARY currently sees it, or None.""" + return cls._primary_entity('apps', app_id) + + @staticmethod + def _primary_subcomponent(parent_id, subcomponent_id): + """One subcomponent as the PRIMARY currently sees it, or None.""" + response = requests.get( + f'{PRIMARY_URL}/components/{parent_id}/subcomponents', timeout=10) + if response.status_code != 200: + return None + for item in response.json().get('items', []): + if item.get('id') == subcomponent_id: + return item + return None + + def test_z1_a_declared_entity_survives_its_peer_going_silent(self, peer_gateway): + """R10: what a peer DECLARED does not stop being true when it goes quiet. + + Retention is what makes every later rule stable. Without it the merged + set is rebuilt from healthy peers only, so an entity - and the + ambiguity it takes part in - disappears the moment a link drops. + """ + cls = type(self) + by_source = self._peer_ids_by_declared_source() + cls._peer_declared_ids = by_source.get('manifest', []) + # Anything the peer did not call "manifest" it worked out for itself. + cls._peer_runtime_ids = [ + entry + for source, entries in by_source.items() if source != 'manifest' + for entry in entries + ] + self.assertIn( + ('apps', 'peer_calibration'), cls._peer_declared_ids, + f'the peer does not consider peer_calibration manifest-declared: {by_source}', + ) + + before = self._primary_app('peer_calibration') + self.assertIsNotNone(before, 'peer_calibration was not merged while the peer was up') + self.assertNotEqual( + before.get('x-medkit', {}).get('available'), False, + 'peer_calibration was already marked unavailable before the peer was killed', + ) + + pid = peer_gateway.process_details['pid'] + os.kill(pid, signal.SIGKILL) + + deadline = time.monotonic() + 60.0 + started = time.monotonic() + observed = None + while time.monotonic() < deadline: + observed = self._primary_app('peer_calibration') + if observed is not None and observed.get('x-medkit', {}).get('available') is False: + cls._noticed_after_s = time.monotonic() - started + break + time.sleep(0.25) + + self.assertIsNotNone(observed, 'peer_calibration vanished when its peer went silent') + x_medkit = observed.get('x-medkit', {}) + self.assertIs( + x_medkit.get('available'), False, + f'a declared entity did not report itself unavailable within 60s: {observed}', + ) + self.assertIs( + x_medkit.get('is_online'), False, + f'a retained app still claims to be running: {observed}', + ) + # Detection is bounded by one discovery refresh (refresh_interval_ms, + # 1000 ms for a test gateway) plus the failed fetch, because the health + # check runs inside the refresh. Reported so a regression that pushes it + # towards the 30 s production default is visible rather than merely slow. + print(f'[retention] aggregator noticed the peer was gone in {cls._noticed_after_s:.1f}s') + self.assertLess( + cls._noticed_after_s, 30.0, + f'took {cls._noticed_after_s:.1f}s to notice a dead peer', + ) + + def test_z2_a_runtime_discovered_peer_entity_is_not_retained(self): + """R10, the other half: a live graph nobody can observe is not retained. + + Keeping a runtime-discovered entity would assert something this + gateway can no longer see. + """ + cls = type(self) + self.assertTrue( + cls._peer_runtime_ids, + 'the peer declared everything it exposes, so this case would prove nothing', + ) + for collection, entity_id in cls._peer_runtime_ids: + self.assertIsNone( + self._primary_entity(collection, entity_id), + f'{collection}/{entity_id} was runtime-discovered on the peer ' + 'and must not be retained', + ) + + def test_z2a_availability_is_carried_by_what_a_request_can_reach(self): + """R10: a Component answers for its reachability; a grouping has none. + + The peer's subcomponent is a real thing behind a link that is down, so + it reports itself unreachable. The parent Component is declared on both + sides and this gateway still serves its half, so it stays reachable. An + Area and a Function are views over members: there is nothing to reach, + so they carry no availability at all and a client must ask the members. + """ + cls = type(self) + self.assertIn( + ('components', PEER_SUBCOMPONENT), cls._peer_declared_ids, + f'the peer does not consider {PEER_SUBCOMPONENT} manifest-declared: ' + f'{cls._peer_declared_ids}', + ) + + retained = self._primary_subcomponent(PARENT_COMPONENT, PEER_SUBCOMPONENT) + self.assertIsNotNone( + retained, f'{PEER_SUBCOMPONENT} vanished when its peer went silent') + self.assertIs( + retained.get('x-medkit', {}).get('available'), False, + f'a retained Component does not report itself unreachable: {retained}', + ) + + parent = self._primary_entity('components', PARENT_COMPONENT) + self.assertIsNotNone( + parent, f'{PARENT_COMPONENT} is declared here too and must remain') + self.assertNotEqual( + parent.get('x-medkit', {}).get('available'), False, + f'a Component this gateway still serves was marked unreachable: {parent}', + ) + + for collection, entity_id in ( + ('areas', MERGED_AREA), + ('functions', MERGED_FUNCTION), + ): + with self.subTest(entity=f'{collection}/{entity_id}'): + grouping = self._primary_entity(collection, entity_id) + self.assertIsNotNone(grouping, f'{collection}/{entity_id} vanished') + # Read the block rather than defaulting it: "no available key" + # is only evidence if there is a block that could have held one. + self.assertIn( + 'x-medkit', grouping, + f'{collection}/{entity_id} carries no x-medkit block, so this ' + f'case would prove nothing: {grouping}', + ) + self.assertNotIn( + 'available', grouping['x-medkit'], + f'{collection}/{entity_id} groups members and has no ' + f'availability of its own: {grouping}', + ) + + def test_z3_a_request_to_a_retained_entity_says_it_is_unavailable(self): + """R10: an unreachable member is answered, not proxied into a 502. + + The body is asserted, not just the status: the failure this branch + exists to remove is a success shape with nothing in it, and a bare + status check passes straight over that. + """ + response = requests.get(f'{PRIMARY_URL}/apps/peer_calibration', timeout=15) + self.assertEqual( + response.status_code, 504, + f'expected 504 for a retained member, got {response.status_code}: {response.text}', + ) + body = response.json() + self.assertEqual(body.get('error_code'), 'not-responding', body) + self.assertIn('peer_calibration', body.get('message', ''), body) + self.assertEqual(body.get('parameters', {}).get('entity_id'), 'peer_calibration', body) + + def test_z4_a_retained_member_keeps_the_operations_it_reported(self): + """R10: unreachable, not amnesiac. + + What the member exposed is part of what it declared. Forgetting it + would make the tree change shape on a link drop all over again, one + level down. + """ + items = self._items(f'functions/{MERGED_FUNCTION}', 'operations') + ids = [item.get('id') for item in items] + + # Counted, not merely present. A list that shrinks to one qualified + # entry satisfies "the id is still qualified" and "the bare form is + # still refused" while the client's view has quietly lost a member, + # which is the failure this whole rule exists to make impossible. + self.assertEqual( + ids.count('primary_calibration:calibrate'), 1, f'ids were {ids}', + ) + self.assertEqual( + ids.count('peer_calibration:calibrate'), 1, + f'a retained member forgot the operation it last reported: {ids}', + ) + self.assertEqual( + ids.count('calibrate'), 0, + f'a bare id reappeared once a member became unreachable: {ids}', + ) + + # And the retained copy says it cannot be served, so a client can tell + # "declared, unreachable" from "ready to run". + by_id = {item.get('id'): item for item in items} + self.assertIs( + by_id['peer_calibration:calibrate'].get('x-medkit', {}).get('available'), False, + 'the retained operation does not report itself unavailable', + ) + self.assertNotEqual( + by_id['primary_calibration:calibrate'].get('x-medkit', {}).get('available'), False, + 'the local operation was marked unavailable', + ) + + def test_z5_ambiguity_does_not_move_when_a_peer_goes_silent(self): + """R10, and the reason retention matters at all. + + Ambiguity is a property of the declared tree. If it tracked + reachability instead, this same request would be refused while the + peer answers and would quietly run the local member once it stopped - + the same request, two different answers, decided by a link. + """ + response = requests.post( + f'{PRIMARY_URL}/functions/{MERGED_FUNCTION}/operations/calibrate/executions', + json={}, + timeout=15, + ) + self.assertEqual( + response.status_code, 400, + f'a bare ambiguous id stopped being refused once the peer went quiet: {response.text}', + ) + body = response.json() + message = (body.get('message', '') + ' ' + str(body.get('parameters', {}))).lower() + self.assertIn('member', message, body) + + def test_z6_every_id_the_list_offers_is_executable(self): + """The forward half of list/execution agreement. + + z4 and z5 together cover the other half - a refused id is not offered. + This walks what the list actually offers and drives each one, because + an agreement checked in one direction only is how the list came to + advertise a bare id that execution refused. + + `not 400` is the property, not `200`: a member whose gateway is silent + answers 504, which reports reachability rather than rejecting the id. + """ + items = self._items(f'functions/{MERGED_FUNCTION}', 'operations') + offered = [item.get('id') for item in items if item.get('id', '').endswith('calibrate')] + self.assertTrue(offered, 'the list offered no calibrate operation to check') + + for operation_id in offered: + with self.subTest(operation=operation_id): + response = requests.post( + f'{PRIMARY_URL}/functions/{MERGED_FUNCTION}/operations/' + f'{quote(operation_id, safe="")}/executions', + json={}, + timeout=15, + ) + self.assertNotEqual( + response.status_code, 400, + f'the list offers {operation_id!r} but executing it is ' + f'refused: {response.text}', + ) + + def test_z7_suppression_omits_the_peer_without_losing_ambiguity(self): + """The loop-suppression guard, checked for the reason it was written. + + Suppression means the peers were never asked, so their items are not + reported - that is what stops a bidirectionally peered pair bouncing a + request. It does not mean the tree forgot they exist: the id stays + qualified, so this response never offers a bare id that execution + refuses either. + """ + response = requests.get( + f'{PRIMARY_URL}/functions/{MERGED_FUNCTION}/operations', + headers={'X-Medkit-No-Fan-Out': '1'}, + timeout=10, + ) + self.assertEqual(response.status_code, 200) + items = response.json().get('items', []) + ids = [item.get('id') for item in items] + + members = set() + for item in items: + members.update(item.get('x-medkit', {}).get('member_ids') or []) + self.assertNotIn( + 'peer_calibration', members, + f'a suppressed response reported a peer-owned member: {ids}', + ) + self.assertEqual( + ids.count('primary_calibration:calibrate'), 1, + f'suppression dropped the qualification along with the peer: {ids}', + ) + self.assertEqual( + ids.count('calibrate'), 0, + f'a suppressed response offered a bare id that execution refuses: {ids}', + ) From f4a3cbc69e853ed2186ea9c1270bdafc913e4b55 Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Thu, 20 Aug 2026 11:43:02 +0200 Subject: [PATCH 03/22] fix(aggregation): a refresh that could not read a peer is not a picture of it A Function's members, a Component's relationships and an Area's subareas are carried only by the routes that describe them one at a time. Those requests were allowed to fail quietly, so a fetch that lost one still reported success and published an entity stripped of the part it could not read. Retention made that permanent, because the declaration a peer is retained from is replaced whole on every successful fetch. An unreadable sub-response now fails the fetch, so the last complete declaration stands. A 404 on a nested collection is a peer too old to offer that route, and a peer whose health check still passes is not called unreachable for missing one read. --- docs/config/aggregation.rst | 33 ++ src/ros2_medkit_gateway/README.md | 19 + .../design/aggregation.rst | 18 + .../aggregation/aggregation_manager.hpp | 29 +- .../core/aggregation/peer_client.hpp | 29 +- .../src/aggregation/aggregation_manager.cpp | 133 ++++-- .../src/core/aggregation/peer_client.cpp | 317 ++++++++----- .../test/test_aggregation_manager.cpp | 435 ++++++++++++++++++ .../test/test_peer_client.cpp | 15 +- 9 files changed, 868 insertions(+), 160 deletions(-) diff --git a/docs/config/aggregation.rst b/docs/config/aggregation.rst index ed553f962..50579b10c 100644 --- a/docs/config/aggregation.rst +++ b/docs/config/aggregation.rst @@ -389,6 +389,39 @@ action. Individual entity requests for remote entities (e.g., ``GET /api/v1/apps/{id}``) return ``502 Bad Gateway`` if the owning peer is unreachable. +Peer Refresh Completeness +~~~~~~~~~~~~~~~~~~~~~~~~~ + +A cache refresh reads a peer over several requests: the four entity lists, the +nested ``subareas`` and ``subcomponents`` collections, the per-entity detail +that carries a Component's relationships and a Function's hosts, and each app's +``operations``. If any of them cannot be read - connection failure, a status +the route has no other meaning for, an oversized body, unparsable JSON - the +refresh for that peer is discarded whole. A partial picture is never published +as a complete one, and the peer's last complete declaration is left in place. + +What clients see then depends on the peer's health check: + +- Health check fails: the retained declaration is served with + ``x-medkit.available: false`` (and ``x-medkit.is_online: false`` for Apps), + because a request addressed there cannot arrive. +- Health check passes: the retained declaration is served unchanged and the + incomplete refresh is logged at ``WARN``. Availability is untouched - the + peer can still be reached; this gateway merely failed to read all of it. + +Two statuses are read rather than treated as failures: + +- ``404`` on a nested collection route means the peer runs a gateway version + that does not expose the route. Those members are omitted, the rest of the + peer merges normally, and the absent routes are logged once per refresh at + ``WARN``. +- ``504`` with error code ``not-responding`` on a Component's detail means the + peer holds that id and the gateway contributing it has gone quiet - the + answer an aggregating peer gives for a declaration it is retaining. In a + chain topology this is how the far end reports a dead leaf, so the Component + is kept as the peer's list named it and marked + ``x-medkit.available: false``. + .. _aggregation-breaking-changes: Breaking Changes (Entity Model Simplification) diff --git a/src/ros2_medkit_gateway/README.md b/src/ros2_medkit_gateway/README.md index a0c967ea3..9ebcfb5ef 100644 --- a/src/ros2_medkit_gateway/README.md +++ b/src/ros2_medkit_gateway/README.md @@ -296,9 +296,28 @@ operations held locally. The answer therefore does not change with who is reachable, costs no network call, and cannot be altered by a client-supplied header. +A refresh describes a peer or it does not. Reading one takes several requests - +the four entity lists, the nested `subareas` and `subcomponents` collections, +the per-entity detail that carries a Component's relationships and a Function's +hosts, and each app's `operations` - and if any of them cannot be read, the +whole refresh is dropped instead of published with the missing branch silently +absent. The peer's last complete declaration stands for another cycle. Two statuses +carry a meaning of their own and are read instead: a `404` on a nested +collection route means the peer runs a gateway that predates that route, so +those members are omitted, the rest of the peer merges normally and the absent +routes are logged once per refresh; a `504 not-responding` on a Component's +detail is the peer saying it holds that id and whoever contributes it has gone +quiet, which is what an aggregating peer answers for a declaration it is +retaining, so the Component is kept as its list named it and marked +`x-medkit.available: false`. + When a peer stops answering, the entities it declared in its manifest are retained and marked unavailable (`x-medkit.available: false`, `x-medkit.is_online: false`); the ones it only discovered at runtime disappear. +`available: false` answers "can a request get there", so it is earned by a +failed health check alone - a peer that still answers `/health` keeps its +entities as they were last read even when a refresh against it came back +incomplete. A retained member keeps the operations it last reported: they stay listed on the aggregating entity marked `x-medkit.available: false`, and still count towards ambiguity, so a qualified id never degrades back to a bare one that execution diff --git a/src/ros2_medkit_gateway/design/aggregation.rst b/src/ros2_medkit_gateway/design/aggregation.rst index 388a1229a..89eb052d9 100644 --- a/src/ros2_medkit_gateway/design/aggregation.rst +++ b/src/ros2_medkit_gateway/design/aggregation.rst @@ -513,6 +513,24 @@ marked unhealthy and excluded from fan-out queries and entity fetching. When a peer recovers (health check succeeds again), it is automatically re-included. +``PeerClient::fetch_entities()`` reads a peer over several requests and either +describes it whole or reports failure: a dead connection, a status a route has +no other meaning for, an oversized body or unparsable JSON on any of them fails +the fetch, because a picture missing a branch is indistinguishable on the wire +from a peer that does not have that branch. Two statuses carry a meaning of +their own: a ``404`` on a nested collection route (``/subareas``, +``/subcomponents``, an app's ``/operations``) identifies a peer running a +gateway that predates the route and is reported in +``PeerEntities::absent_routes`` for the caller to log; a ``504`` with error code +``not-responding`` on a Component's detail is the peer reporting that the +gateway contributing that Component has gone quiet - what a middle gateway in a +chain answers for a declaration it is retaining - so the Component is kept as +the list named it and marked unavailable. +``AggregationManager`` never records a failed fetch as the peer's declaration, +so the last complete one survives; it re-checks that peer's health to decide +whether to replay it marked unavailable (health check failed) or exactly as it +was last read (health check still passes). + The aggregator also publishes its own ``/health`` response with two additional fields when aggregation is enabled (x-medkit extensions on our own endpoint, outside the SOVD core contract): diff --git a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/aggregation/aggregation_manager.hpp b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/aggregation/aggregation_manager.hpp index aedbc7383..1116ade63 100644 --- a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/aggregation/aggregation_manager.hpp +++ b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/aggregation/aggregation_manager.hpp @@ -301,13 +301,29 @@ class AggregationManager { AggregationConfig config_; rclcpp::Logger logger_; size_t static_peer_count_{0}; ///< Number of statically configured peers (not subject to max_discovered_peers) - /// Record what `peer_name` declared, so it can be replayed while that peer is - /// silent. Stored already marked unavailable - it is only ever read back on - /// the path where the peer did not answer. + /// Whether a peer whose picture could not be refreshed can still be reached. + /// + /// `available:false` is the answer to "can a request get there", so it is + /// earned by a failed health check alone. A refresh that came back + /// incomplete from a peer that is still answering says nothing about + /// reachability - it says this gateway could not read the whole picture. + enum class Reachability { + kReachable, + kUnreachable, + }; + + /// Record what `peer_name` declared, so it can be replayed while that peer + /// cannot be described. Stored as the peer reported it; the unavailable + /// marking belongs to the replay, which is where reachability is known. + /// + /// Only ever called for a fetch that described the whole peer: a partial + /// picture recorded here would overwrite the last complete one and outlive + /// the cycle that failed to read it. void remember_declaration(const std::string & peer_name, const PeerEntities & entities); /// The retained declaration for `peer_name`, empty if none was ever recorded. - PeerEntities replay_declaration(const std::string & peer_name) const; + /// Marked unavailable when `reachability` says the peer cannot be reached. + PeerEntities replay_declaration(const std::string & peer_name, Reachability reachability) const; mutable std::shared_mutex mutex_; // Declared before data it protects (destruction order) std::vector> peers_; @@ -318,8 +334,9 @@ class AggregationManager { /// A tree that changes shape because a link went down cannot be reasoned /// about: the same request gets a different answer depending on who happens /// to be reachable. What a peer DECLARED is a property of its configuration - /// and does not stop being true when it stops replying, so it is retained and - /// marked unavailable. What a peer merely DISCOVERED at runtime is a property + /// and does not stop being true when it stops replying, so it is retained - + /// marked unavailable while the peer cannot be reached, and exactly as last + /// read while it can. What a peer merely DISCOVERED at runtime is a property /// of a live graph this gateway can no longer observe, so it is allowed to /// disappear. std::unordered_map retained_peer_entities_; diff --git a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/aggregation/peer_client.hpp b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/aggregation/peer_client.hpp index 90267f3a6..8fc0cdf04 100644 --- a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/aggregation/peer_client.hpp +++ b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/aggregation/peer_client.hpp @@ -40,6 +40,15 @@ struct PeerEntities { std::vector components; std::vector apps; std::vector functions; + + /// Nested collection routes this peer does not offer, as route templates + /// (e.g. ``/components/{id}/subcomponents``), each recorded once. + /// + /// A peer running an older gateway answers 404 for a route that did not + /// exist yet. That is a version boundary, not a read failure, so the fetch + /// carries on without those members and names the routes here for the + /// caller to report. + std::vector absent_routes; }; /** @@ -84,7 +93,25 @@ class PeerClient { * @brief Fetch all entity collections from the peer * * GETs /api/v1/areas, /api/v1/components, /api/v1/apps, /api/v1/functions - * and parses the items[] arrays. Each entity's source is set to "peer:". + * and parses the items[] arrays, then the nested routes that carry structure + * the lists omit: subareas, subcomponents, per-entity detail (the only source + * of a Component's relationships and a Function's hosts) and per-app + * operations. Each entity's source is set to "peer:". + * + * The result describes the peer or it does not. Any sub-request that could + * not be read - a dead connection, a non-200 the route has no other meaning + * for, an oversized body, unparsable JSON - fails the whole fetch, because a + * picture missing a branch is indistinguishable on the wire from a peer that + * does not have that branch. Two statuses do carry a meaning of their own and + * are read rather than failed: + * + * - 404 on a nested collection route: the peer predates the route. Those + * members are omitted and the route is named in + * PeerEntities::absent_routes. + * - `504 not-responding` on a Component detail: the peer holds that id and + * says whoever contributes it has gone quiet, which is what an aggregating + * peer answers for a declaration it is retaining. The Component is kept as + * its list named it, marked unavailable. * * @return PeerEntities on success, error message on failure */ diff --git a/src/ros2_medkit_gateway/src/aggregation/aggregation_manager.cpp b/src/ros2_medkit_gateway/src/aggregation/aggregation_manager.cpp index 7c99381a6..4be73ebd5 100644 --- a/src/ros2_medkit_gateway/src/aggregation/aggregation_manager.cpp +++ b/src/ros2_medkit_gateway/src/aggregation/aggregation_manager.cpp @@ -369,31 +369,42 @@ bool is_declared(const Entity & entity) { /// Availability belongs to an entity a request can be addressed to. A grouping /// entity has none of its own: it is a view over its members, and the members /// carry the flag, so retaining one says nothing about what can be reached. -void mark_unreachable(Area &) { +void mark_unreachable(Area & /*area*/) { } -void mark_unreachable(Function &) { +void mark_unreachable(Function & /*function*/) { } void mark_unreachable(App & app) { app.available = false; + // A retained App is not observably running: the graph it was bound to is on + // the other side of a link that is down. `is_online` is the field every + // consumer already reads for that, so it carries the news rather than a + // second flag they would each have to learn. + app.is_online = false; } void mark_unreachable(Component & component) { component.available = false; } -/// The declared entities of `src`, with the addressable ones marked unreachable. +/// The declared entities of `src`, as the peer reported them. template std::vector declared_only(const std::vector & src) { std::vector kept; for (const auto & entity : src) { - if (!is_declared(entity)) { - continue; + if (is_declared(entity)) { + kept.push_back(entity); } - kept.push_back(entity); - mark_unreachable(kept.back()); } return kept; } +/// Mark every addressable entity of a replayed declaration unreachable. +template +void mark_all_unreachable(std::vector & entities) { + for (auto & entity : entities) { + mark_unreachable(entity); + } +} + } // namespace void AggregationManager::remember_declaration(const std::string & peer_name, const PeerEntities & entities) { @@ -403,22 +414,28 @@ void AggregationManager::remember_declaration(const std::string & peer_name, con declared.apps = declared_only(entities.apps); declared.functions = declared_only(entities.functions); - // A retained App is not observably running: the graph it was bound to is on - // the other side of a link that is down. `is_online` is the field every - // consumer already reads for that, so it carries the news rather than a - // second flag they would each have to learn. - for (auto & app : declared.apps) { - app.is_online = false; - } - std::unique_lock lock(mutex_); retained_peer_entities_[peer_name] = std::move(declared); } -PeerEntities AggregationManager::replay_declaration(const std::string & peer_name) const { - std::shared_lock lock(mutex_); - auto it = retained_peer_entities_.find(peer_name); - return it == retained_peer_entities_.end() ? PeerEntities{} : it->second; +PeerEntities AggregationManager::replay_declaration(const std::string & peer_name, Reachability reachability) const { + PeerEntities replayed; + { + std::shared_lock lock(mutex_); + auto it = retained_peer_entities_.find(peer_name); + if (it == retained_peer_entities_.end()) { + return replayed; + } + replayed = it->second; + } + + if (reachability == Reachability::kUnreachable) { + mark_all_unreachable(replayed.areas); + mark_all_unreachable(replayed.components); + mark_all_unreachable(replayed.apps); + mark_all_unreachable(replayed.functions); + } + return replayed; } AggregationManager::MergedPeerResult AggregationManager::fetch_and_merge_peer_entities( @@ -450,14 +467,21 @@ AggregationManager::MergedPeerResult AggregationManager::fetch_and_merge_peer_en // shared_ptr copies keep PeerClients alive even if remove_discovered_peer() // erases them from peers_ concurrently. std::vector> snapshot; - std::vector silent_peers; + + // A peer this cycle could not describe, and whether it can still be reached. + struct UnreadPeer { + std::string name; + Reachability reachability{Reachability::kUnreachable}; + std::string reason; + }; + std::vector unread_peers; { std::shared_lock lock(mutex_); for (const auto & peer : peers_) { if (peer->is_healthy()) { snapshot.push_back(peer); } else { - silent_peers.push_back(peer->name()); + unread_peers.push_back({peer->name(), Reachability::kUnreachable, "not answering"}); } } } @@ -467,6 +491,10 @@ AggregationManager::MergedPeerResult AggregationManager::fetch_and_merge_peer_en std::string peer_name; bool success{false}; std::string error; + /// Reachability as of the health check taken after a failed fetch, which is + /// what separates a peer that died mid-refresh from one that is answering + /// but could not be read in full. + Reachability reachability{Reachability::kReachable}; PeerEntities entities; }; @@ -479,19 +507,27 @@ AggregationManager::MergedPeerResult AggregationManager::fetch_and_merge_peer_en PeerFetchResult pfr; pfr.peer_name = peer->name(); - auto result = peer->fetch_entities(); - if (!result.has_value()) { + // Re-checking health only on the failure path answers the one question the + // caller needs and costs nothing on the path that succeeded: the flag from + // before the fetch cannot tell a peer that has since died from one that is + // still there and merely could not be read. + auto fetch_failed = [&pfr, &peer](std::string reason) { pfr.success = false; - pfr.error = result.error(); + pfr.error = std::move(reason); + peer->check_health(); + pfr.reachability = peer->is_healthy() ? Reachability::kReachable : Reachability::kUnreachable; return pfr; + }; + + auto result = peer->fetch_entities(); + if (!result.has_value()) { + return fetch_failed(result.error()); } size_t total = result->areas.size() + result->components.size() + result->apps.size() + result->functions.size(); if (total > max_entities_per_peer) { - pfr.success = false; - pfr.error = "returned " + std::to_string(total) + " entities (max " + std::to_string(max_entities_per_peer) + - "), skipping"; - return pfr; + return fetch_failed("returned " + std::to_string(total) + " entities (max " + + std::to_string(max_entities_per_peer) + "), skipping"); } pfr.success = true; @@ -507,35 +543,54 @@ AggregationManager::MergedPeerResult AggregationManager::fetch_and_merge_peer_en // Collect results and merge sequentially (merge order must be deterministic) std::vector to_merge; - to_merge.reserve(futures.size() + silent_peers.size()); + to_merge.reserve(futures.size() + unread_peers.size()); for (auto & f : futures) { auto pfr = f.get(); if (!pfr.success) { + // The last complete declaration is not replaced by a picture with holes + // in it: `remember_declaration` overwrites, so recording a partial fetch + // would outlive the cycle that failed and be replayed as the peer's own + // account of itself. if (logger) { RCLCPP_WARN(*logger, "Failed to fetch entities from peer '%s': %s", pfr.peer_name.c_str(), pfr.error.c_str()); } - silent_peers.push_back(pfr.peer_name); + unread_peers.push_back({pfr.peer_name, pfr.reachability, + pfr.reachability == Reachability::kReachable ? "refresh incomplete" : "not answering"}); continue; } + if (logger && !pfr.entities.absent_routes.empty()) { + std::string routes; + for (const auto & route : pfr.entities.absent_routes) { + routes += routes.empty() ? route : ", " + route; + } + RCLCPP_WARN(*logger, "Peer '%s' does not expose %s; aggregating without the members those routes carry", + pfr.peer_name.c_str(), routes.c_str()); + } remember_declaration(pfr.peer_name, pfr.entities); to_merge.push_back(std::move(pfr)); } - // A peer that did not answer still contributes what it declared, marked - // unavailable. Merged after the peers that did answer, so a live declaration - // always wins over a retained one carrying the same id. - for (const auto & peer_name : silent_peers) { - auto retained = replay_declaration(peer_name); + // A peer this cycle could not describe still contributes what it declared. + // Merged after the peers that did answer, so a live declaration always wins + // over a retained one carrying the same id. + for (const auto & peer : unread_peers) { + auto retained = replay_declaration(peer.name, peer.reachability); if (retained.areas.empty() && retained.components.empty() && retained.apps.empty() && retained.functions.empty()) { continue; } if (logger) { - RCLCPP_INFO(*logger, "Peer '%s' not answering; retaining %zu declared entities as unavailable", peer_name.c_str(), - retained.areas.size() + retained.components.size() + retained.apps.size() + - retained.functions.size()); + const size_t count = + retained.areas.size() + retained.components.size() + retained.apps.size() + retained.functions.size(); + if (peer.reachability == Reachability::kUnreachable) { + RCLCPP_INFO(*logger, "Peer '%s' %s; retaining %zu declared entities as unavailable", peer.name.c_str(), + peer.reason.c_str(), count); + } else { + RCLCPP_WARN(*logger, "Peer '%s' %s; it is still reachable, so its %zu declared entities stand as last read", + peer.name.c_str(), peer.reason.c_str(), count); + } } PeerFetchResult replayed; - replayed.peer_name = peer_name; + replayed.peer_name = peer.name; replayed.success = true; replayed.entities = std::move(retained); to_merge.push_back(std::move(replayed)); diff --git a/src/ros2_medkit_gateway/src/core/aggregation/peer_client.cpp b/src/ros2_medkit_gateway/src/core/aggregation/peer_client.cpp index eb9147c92..9243bd39e 100644 --- a/src/ros2_medkit_gateway/src/core/aggregation/peer_client.cpp +++ b/src/ros2_medkit_gateway/src/core/aggregation/peer_client.cpp @@ -358,6 +358,98 @@ Function parse_function(const nlohmann::json & j) { return func; } +/** + * @brief What a non-200 on a sub-request means, which depends on the route. + */ +enum class RouteKind { + /// A top-level collection: ``/areas``, ``/components``, ``/apps``, + /// ``/functions``. Every peer serves these, so nothing but 200 describes one. + kCollection, + /// A nested collection: ``/areas/{id}/subareas``, ``/components/{id}/subcomponents``, + /// ``/apps/{id}/operations``. A gateway old enough not to have the route answers + /// 404, and aggregation has to keep working across that version boundary, so a + /// 404 here means "not offered" rather than "could not be read". + kNestedCollection, + /// The detail of an entity that carries availability of its own (a Component). + /// ``504 not-responding`` is the peer describing that entity as unreachable - + /// a statement about the entity, which an aggregating peer makes whenever it + /// is itself holding a declaration for a gateway that went quiet. It is + /// carried, not read as a hole in the picture. + kAddressableDetail, + /// The detail of a grouping entity (a Function). It has no availability of its + /// own to carry, and its members are named nowhere else, so nothing but 200 + /// describes it. + kGroupingDetail, +}; + +/** + * @brief What one sub-request issued while describing a peer produced. + */ +struct SubResponse { + enum class Kind { + kBody, ///< `body` holds the parsed response + kRouteAbsent, ///< the peer does not offer this route; carry on without it + kEntityUnreachable, ///< the peer named this entity and says it cannot be reached + kIncomplete, ///< part of the picture could not be read; `error` says why + }; + + Kind kind{Kind::kIncomplete}; + nlohmann::json body; + std::string error; +}; + +/** + * @brief True for a SOVD error body carrying the ``not-responding`` code. + */ +bool says_not_responding(const std::string & body) { + if (body.size() > MAX_PEER_RESPONSE_SIZE) { + return false; + } + auto parsed = nlohmann::json::parse(body, nullptr, false); + return !parsed.is_discarded() && parsed.is_object() && parsed.value("error_code", "") == ERR_NOT_RESPONDING; +} + +/** + * @brief Classify one sub-request of a peer fetch. + * + * @param result httplib result for the call + * @param peer_name Peer the call was made against, for the error text + * @param path Route that was called, for the error text + * @param kind What a non-200 means on this route + */ +SubResponse read_sub_response(const httplib::Result & result, const std::string & peer_name, const std::string & path, + RouteKind kind) { + SubResponse out; + if (!result) { + out.error = "Failed to connect to peer '" + peer_name + "' for " + path; + return out; + } + if (result->status == 404 && kind == RouteKind::kNestedCollection) { + out.kind = SubResponse::Kind::kRouteAbsent; + return out; + } + if (result->status == 504 && kind == RouteKind::kAddressableDetail && says_not_responding(result->body)) { + out.kind = SubResponse::Kind::kEntityUnreachable; + return out; + } + if (result->status != 200) { + out.error = "Peer '" + peer_name + "' returned status " + std::to_string(result->status) + " for " + path; + return out; + } + if (result->body.size() > MAX_PEER_RESPONSE_SIZE) { + out.error = "Response from peer '" + peer_name + "' for " + path + " exceeds size limit"; + return out; + } + auto parsed = nlohmann::json::parse(result->body, nullptr, false); + if (parsed.is_discarded()) { + out.error = "Invalid JSON from peer '" + peer_name + "' for " + path; + return out; + } + out.kind = SubResponse::Kind::kBody; + out.body = std::move(parsed); + return out; +} + } // namespace PeerClient::PeerClient(const std::string & url, const std::string & name, int timeout_ms, bool forward_auth) @@ -427,24 +519,24 @@ tl::expected PeerClient::fetch_entities() { PeerEntities entities; const std::string peer_source = "peer:" + name_; + // A route the peer does not offer is a property of the peer, not of the + // entity that happened to hit it first, so it is recorded once however many + // entities ask for it. + auto note_absent_route = [&entities](const std::string & route) { + if (std::find(entities.absent_routes.begin(), entities.absent_routes.end(), route) == + entities.absent_routes.end()) { + entities.absent_routes.push_back(route); + } + }; + // Fetch areas { - auto result = cli.Get(std::string(API_PREFIX) + "/areas"); - if (!result) { - return tl::unexpected("Failed to connect to peer '" + name_ + "' at " + url_); - } - if (result->status != 200) { - return tl::unexpected("Peer '" + name_ + "' returned status " + std::to_string(result->status) + - " for /areas"); + auto response = + read_sub_response(cli.Get(std::string(API_PREFIX) + "/areas"), name_, "/areas", RouteKind::kCollection); + if (response.kind != SubResponse::Kind::kBody) { + return tl::unexpected(response.error); } - if (result->body.size() > MAX_PEER_RESPONSE_SIZE) { - return tl::unexpected("Response from peer '" + name_ + "' for /areas exceeds size limit"); - } - auto response_json = nlohmann::json::parse(result->body, nullptr, false); - if (response_json.is_discarded()) { - return tl::unexpected("Invalid JSON from peer '" + name_ + "' for /areas"); - } - entities.areas = parse_collection(response_json, parse_area); + entities.areas = parse_collection(response.body, parse_area); // Validate entity IDs and enforce per-collection limit entities.areas.erase(std::remove_if(entities.areas.begin(), entities.areas.end(), [](const Area & a) { @@ -465,20 +557,24 @@ tl::expected PeerClient::fetch_entities() { // (which can invalidate references if the vector reallocates). std::vector all_subareas; for (const auto & area : entities.areas) { - auto sub_result = cli.Get(std::string(API_PREFIX) + "/areas/" + area.id + "/subareas"); - if (sub_result && sub_result->status == 200 && sub_result->body.size() <= MAX_PEER_RESPONSE_SIZE) { - auto sub_json = nlohmann::json::parse(sub_result->body, nullptr, false); - if (!sub_json.is_discarded()) { - auto subareas = parse_collection(sub_json, parse_area); - for (auto & sub : subareas) { - if (!is_valid_entity_id(sub.id)) { - continue; - } - sub.declared_source = sub.source; - sub.source = peer_source; - all_subareas.push_back(std::move(sub)); - } + const std::string route = "/areas/" + area.id + "/subareas"; + auto sub = + read_sub_response(cli.Get(std::string(API_PREFIX) + route), name_, route, RouteKind::kNestedCollection); + if (sub.kind == SubResponse::Kind::kIncomplete) { + return tl::unexpected(sub.error); + } + if (sub.kind == SubResponse::Kind::kRouteAbsent) { + note_absent_route("/areas/{id}/subareas"); + continue; + } + auto subareas = parse_collection(sub.body, parse_area); + for (auto & subarea : subareas) { + if (!is_valid_entity_id(subarea.id)) { + continue; } + subarea.declared_source = subarea.source; + subarea.source = peer_source; + all_subareas.push_back(std::move(subarea)); } } entities.areas.insert(entities.areas.end(), std::make_move_iterator(all_subareas.begin()), @@ -487,23 +583,13 @@ tl::expected PeerClient::fetch_entities() { // Fetch components (list then detail per entity for full relationship data) { - auto result = cli.Get(std::string(API_PREFIX) + "/components"); - if (!result) { - return tl::unexpected("Failed to connect to peer '" + name_ + "' at " + url_); - } - if (result->status != 200) { - return tl::unexpected("Peer '" + name_ + "' returned status " + std::to_string(result->status) + - " for /components"); - } - if (result->body.size() > MAX_PEER_RESPONSE_SIZE) { - return tl::unexpected("Response from peer '" + name_ + "' for /components exceeds size limit"); - } - auto response_json = nlohmann::json::parse(result->body, nullptr, false); - if (response_json.is_discarded()) { - return tl::unexpected("Invalid JSON from peer '" + name_ + "' for /components"); + auto response = read_sub_response(cli.Get(std::string(API_PREFIX) + "/components"), name_, "/components", + RouteKind::kCollection); + if (response.kind != SubResponse::Kind::kBody) { + return tl::unexpected(response.error); } // Parse IDs from list, then fetch detail per entity for relationships - auto comp_list = parse_collection(response_json, parse_component); + auto comp_list = parse_collection(response.body, parse_component); // Validate entity IDs and enforce per-collection limit comp_list.erase(std::remove_if(comp_list.begin(), comp_list.end(), [](const Component & c) { @@ -514,13 +600,23 @@ tl::expected PeerClient::fetch_entities() { return tl::unexpected("Peer '" + name_ + "' returned " + std::to_string(comp_list.size()) + " components (max " + std::to_string(MAX_ENTITIES_PER_COLLECTION) + ")"); } + // The detail response carries the relationships (parent, dependencies, + // identity) the list omits, so a Component built from the list alone is a + // Component asserted to have none. for (auto & comp : comp_list) { - auto detail = cli.Get(std::string(API_PREFIX) + "/components/" + comp.id); - if (detail && detail->status == 200) { - auto detail_json = nlohmann::json::parse(detail->body, nullptr, false); - if (!detail_json.is_discarded()) { - comp = parse_component(detail_json); - } + const std::string route = "/components/" + comp.id; + auto detail = + read_sub_response(cli.Get(std::string(API_PREFIX) + route), name_, route, RouteKind::kAddressableDetail); + if (detail.kind == SubResponse::Kind::kIncomplete) { + return tl::unexpected(detail.error); + } + if (detail.kind == SubResponse::Kind::kBody) { + comp = parse_component(detail.body); + } else { + // The peer holds this Component's id but cannot describe it: whoever + // contributes it has gone quiet. What the list gave is everything the + // peer can still say, and the availability is what it is saying. + comp.available = false; } comp.declared_source = comp.source; comp.source = peer_source; @@ -530,28 +626,40 @@ tl::expected PeerClient::fetch_entities() { // (which can invalidate references if the vector reallocates). std::vector all_subcomps; for (const auto & comp : comp_list) { - auto sub_result = cli.Get(std::string(API_PREFIX) + "/components/" + comp.id + "/subcomponents"); - if (sub_result && sub_result->status == 200 && sub_result->body.size() <= MAX_PEER_RESPONSE_SIZE) { - auto sub_json = nlohmann::json::parse(sub_result->body, nullptr, false); - if (!sub_json.is_discarded()) { - auto subcomps = parse_collection(sub_json, parse_component); - for (auto & sub : subcomps) { - if (!is_valid_entity_id(sub.id)) { - continue; - } - // Fetch detail for each subcomponent to get full relationships - auto detail = cli.Get(std::string(API_PREFIX) + "/components/" + sub.id); - if (detail && detail->status == 200) { - auto detail_json = nlohmann::json::parse(detail->body, nullptr, false); - if (!detail_json.is_discarded()) { - sub = parse_component(detail_json); - } - } - sub.declared_source = sub.source; - sub.source = peer_source; - all_subcomps.push_back(std::move(sub)); - } + const std::string route = "/components/" + comp.id + "/subcomponents"; + auto sub = + read_sub_response(cli.Get(std::string(API_PREFIX) + route), name_, route, RouteKind::kNestedCollection); + if (sub.kind == SubResponse::Kind::kIncomplete) { + return tl::unexpected(sub.error); + } + if (sub.kind == SubResponse::Kind::kRouteAbsent) { + note_absent_route("/components/{id}/subcomponents"); + continue; + } + auto subcomps = parse_collection(sub.body, parse_component); + for (auto & subcomp : subcomps) { + if (!is_valid_entity_id(subcomp.id)) { + continue; } + // Fetch detail for each subcomponent to get full relationships + const std::string detail_route = "/components/" + subcomp.id; + auto detail = read_sub_response(cli.Get(std::string(API_PREFIX) + detail_route), name_, detail_route, + RouteKind::kAddressableDetail); + if (detail.kind == SubResponse::Kind::kIncomplete) { + return tl::unexpected(detail.error); + } + if (detail.kind == SubResponse::Kind::kBody) { + subcomp = parse_component(detail.body); + } else { + subcomp.available = false; + // The route this id came from is itself the statement that `comp` is + // its parent, so the tree keeps its shape even though the entity's + // own description is out of reach. + subcomp.parent_component_id = comp.id; + } + subcomp.declared_source = subcomp.source; + subcomp.source = peer_source; + all_subcomps.push_back(std::move(subcomp)); } } comp_list.insert(comp_list.end(), std::make_move_iterator(all_subcomps.begin()), @@ -562,22 +670,12 @@ tl::expected PeerClient::fetch_entities() { // Fetch apps { - auto result = cli.Get(std::string(API_PREFIX) + "/apps"); - if (!result) { - return tl::unexpected("Failed to connect to peer '" + name_ + "' at " + url_); - } - if (result->status != 200) { - return tl::unexpected("Peer '" + name_ + "' returned status " + std::to_string(result->status) + - " for /apps"); - } - if (result->body.size() > MAX_PEER_RESPONSE_SIZE) { - return tl::unexpected("Response from peer '" + name_ + "' for /apps exceeds size limit"); - } - auto response_json = nlohmann::json::parse(result->body, nullptr, false); - if (response_json.is_discarded()) { - return tl::unexpected("Invalid JSON from peer '" + name_ + "' for /apps"); + auto response = + read_sub_response(cli.Get(std::string(API_PREFIX) + "/apps"), name_, "/apps", RouteKind::kCollection); + if (response.kind != SubResponse::Kind::kBody) { + return tl::unexpected(response.error); } - entities.apps = parse_collection(response_json, parse_app); + entities.apps = parse_collection(response.body, parse_app); // Validate entity IDs and enforce per-collection limit entities.apps.erase(std::remove_if(entities.apps.begin(), entities.apps.end(), [](const App & a) { @@ -613,37 +711,29 @@ tl::expected PeerClient::fetch_entities() { // hop that answers for it. It is also what makes this terminate. for (auto & app : entities.apps) { httplib::Headers no_fan_out{{"X-Medkit-No-Fan-Out", "1"}}; - auto ops_result = cli.Get(std::string(API_PREFIX) + "/apps/" + app.id + "/operations", no_fan_out); - if (!ops_result || ops_result->status != 200 || ops_result->body.size() > MAX_PEER_RESPONSE_SIZE) { - continue; + const std::string route = "/apps/" + app.id + "/operations"; + auto ops = read_sub_response(cli.Get(std::string(API_PREFIX) + route, no_fan_out), name_, route, + RouteKind::kNestedCollection); + if (ops.kind == SubResponse::Kind::kIncomplete) { + return tl::unexpected(ops.error); } - auto ops_json = nlohmann::json::parse(ops_result->body, nullptr, false); - if (ops_json.is_discarded()) { + if (ops.kind == SubResponse::Kind::kRouteAbsent) { + note_absent_route("/apps/{id}/operations"); continue; } - parse_operations_into(ops_json, app); + parse_operations_into(ops.body, app); } } // Fetch functions (list then detail per entity for hosts data) { - auto result = cli.Get(std::string(API_PREFIX) + "/functions"); - if (!result) { - return tl::unexpected("Failed to connect to peer '" + name_ + "' at " + url_); - } - if (result->status != 200) { - return tl::unexpected("Peer '" + name_ + "' returned status " + std::to_string(result->status) + - " for /functions"); - } - if (result->body.size() > MAX_PEER_RESPONSE_SIZE) { - return tl::unexpected("Response from peer '" + name_ + "' for /functions exceeds size limit"); - } - auto response_json = nlohmann::json::parse(result->body, nullptr, false); - if (response_json.is_discarded()) { - return tl::unexpected("Invalid JSON from peer '" + name_ + "' for /functions"); + auto response = + read_sub_response(cli.Get(std::string(API_PREFIX) + "/functions"), name_, "/functions", RouteKind::kCollection); + if (response.kind != SubResponse::Kind::kBody) { + return tl::unexpected(response.error); } // Parse IDs from list, then fetch detail per entity for hosts - auto func_list = parse_collection(response_json, parse_function); + auto func_list = parse_collection(response.body, parse_function); // Validate entity IDs and enforce per-collection limit func_list.erase(std::remove_if(func_list.begin(), func_list.end(), [](const Function & f) { @@ -654,14 +744,17 @@ tl::expected PeerClient::fetch_entities() { return tl::unexpected("Peer '" + name_ + "' returned " + std::to_string(func_list.size()) + " functions (max " + std::to_string(MAX_ENTITIES_PER_COLLECTION) + ")"); } + // A Function's hosts live only in its detail response: the list carries + // none. A Function built from the list alone is a Function asserted to + // group nothing, which is a statement the peer never made. for (auto & func : func_list) { - auto detail = cli.Get(std::string(API_PREFIX) + "/functions/" + func.id); - if (detail && detail->status == 200) { - auto detail_json = nlohmann::json::parse(detail->body, nullptr, false); - if (!detail_json.is_discarded()) { - func = parse_function(detail_json); - } + const std::string route = "/functions/" + func.id; + auto detail = + read_sub_response(cli.Get(std::string(API_PREFIX) + route), name_, route, RouteKind::kGroupingDetail); + if (detail.kind != SubResponse::Kind::kBody) { + return tl::unexpected(detail.error); } + func = parse_function(detail.body); func.declared_source = func.source; func.source = peer_source; } diff --git a/src/ros2_medkit_gateway/test/test_aggregation_manager.cpp b/src/ros2_medkit_gateway/test/test_aggregation_manager.cpp index cb120f594..e7eb0884d 100644 --- a/src/ros2_medkit_gateway/test/test_aggregation_manager.cpp +++ b/src/ros2_medkit_gateway/test/test_aggregation_manager.cpp @@ -16,6 +16,7 @@ #include #include +#include #include #include #include @@ -732,6 +733,8 @@ class MockPeerServer { // Non-copyable, non-movable MockPeerServer(const MockPeerServer &) = delete; MockPeerServer & operator=(const MockPeerServer &) = delete; + MockPeerServer(MockPeerServer &&) = delete; + MockPeerServer & operator=(MockPeerServer &&) = delete; httplib::Server & server() { if (!server_) { @@ -1042,6 +1045,438 @@ TEST(AggregationManager, fetch_and_merge_remaps_function_hosts_after_app_collisi EXPECT_TRUE(has_lidar) << "Function hosts should still contain 'lidar_driver'"; } +TEST(AggregationManager, a_function_whose_members_could_not_be_read_is_not_published_memberless) { + // A Function's `hosts` live only in its detail response: the list endpoint + // carries none. So a detail request that does not answer leaves the fetch + // holding a Function it cannot describe. Publishing it anyway asserts the + // Function has no members, which is a statement the peer never made, and + // retention then keeps that statement after the peer is gone. + MockPeerServer mock; + + mock.server().Get("/api/v1/health", [](const httplib::Request &, httplib::Response & res) { + res.set_content(R"({"status":"healthy"})", "application/json"); + }); + mock.server().Get("/api/v1/areas", [](const httplib::Request &, httplib::Response & res) { + res.set_content(R"({"items":[]})", "application/json"); + }); + mock.server().Get("/api/v1/components", [](const httplib::Request &, httplib::Response & res) { + res.set_content(R"({"items":[]})", "application/json"); + }); + mock.server().Get(R"(/api/v1/components/([^/]+)/subcomponents)", + [](const httplib::Request &, httplib::Response & res) { + res.set_content(R"({"items":[]})", "application/json"); + }); + mock.server().Get("/api/v1/apps", [](const httplib::Request &, httplib::Response & res) { + nlohmann::json items = nlohmann::json::array(); + items.push_back({{"id", "brake_monitor"}, {"name", "Brake Monitor"}}); + res.set_content(nlohmann::json({{"items", items}}).dump(), "application/json"); + }); + + // The list names the Function, exactly as a real gateway does: no hosts. + mock.server().Get("/api/v1/functions", [](const httplib::Request &, httplib::Response & res) { + nlohmann::json items = nlohmann::json::array(); + items.push_back({{"id", "vehicle_health"}, {"name", "Vehicle Health"}}); + res.set_content(nlohmann::json({{"items", items}}).dump(), "application/json"); + }); + // The detail, which is the only source of members, does not answer. + mock.server().Get(R"(/api/v1/functions/([^/]+))", [](const httplib::Request &, httplib::Response & res) { + res.status = 500; + }); + + AggregationConfig config; + config.enabled = true; + AggregationConfig::PeerConfig peer; + peer.name = "brake_ecu"; + const int port = mock.start(); + peer.url = "http://127.0.0.1:" + std::to_string(port); + config.peers.push_back(peer); + + AggregationManager manager(config); + manager.check_all_health(); + auto result = manager.fetch_and_merge_peer_entities({}, {}, {}, {}); + + auto it = std::find_if(result.functions.begin(), result.functions.end(), [](const Function & f) { + return f.id == "vehicle_health"; + }); + if (it != result.functions.end()) { + EXPECT_FALSE(it->hosts.empty()) + << "a Function was published with no members after the only request that reports them failed"; + } +} + +// ============================================================================= +// A refresh describes the peer or it does not +// ============================================================================= + +namespace { + +/// Switches that stop one of a peer's routes from answering, so a test can flip +/// exactly one thing between two refreshes of the same peer. +struct PeerFaults { + std::atomic subareas_fail{false}; + std::atomic subcomponent_detail_fails{false}; + std::atomic function_detail_fails{false}; +}; + +/// A peer whose picture is complete, with the structure its list endpoints omit +/// living where a real gateway puts it: subareas and subcomponents behind their +/// nested routes, a Component's relationships and a Function's hosts behind the +/// per-entity detail. Everything is manifest-declared, so it is retained when +/// the peer stops being readable. +void install_complete_peer(httplib::Server & svr, PeerFaults & faults) { + svr.Get("/api/v1/health", [](const httplib::Request &, httplib::Response & res) { + res.set_content(R"({"status":"healthy"})", "application/json"); + }); + + svr.Get("/api/v1/areas", [](const httplib::Request &, httplib::Response & res) { + res.set_content(R"({"items":[{"id":"vehicle","name":"Vehicle","x-medkit":{"source":"manifest"}}]})", + "application/json"); + }); + svr.Get(R"(/api/v1/areas/([^/]+)/subareas)", [&faults](const httplib::Request & req, httplib::Response & res) { + if (faults.subareas_fail.load()) { + res.status = 500; + return; + } + if (req.matches[1].str() == "vehicle") { + res.set_content(R"({"items":[{"id":"sensors","name":"Sensors","x-medkit":{"source":"manifest"}}]})", + "application/json"); + return; + } + res.set_content(R"({"items":[]})", "application/json"); + }); + + svr.Get("/api/v1/components", [](const httplib::Request &, httplib::Response & res) { + res.set_content(R"({"items":[{"id":"robot_alpha","name":"Robot Alpha","x-medkit":{"source":"manifest"}}]})", + "application/json"); + }); + // The nested list carries ids and names, as the gateway's own collection + // responses do; the relationships are only in the detail below. + svr.Get(R"(/api/v1/components/([^/]+)/subcomponents)", [](const httplib::Request & req, httplib::Response & res) { + if (req.matches[1].str() == "robot_alpha") { + res.set_content(R"({"items":[{"id":"perception_ecu","name":"Perception ECU","x-medkit":{"source":"manifest"}}]})", + "application/json"); + return; + } + res.set_content(R"({"items":[]})", "application/json"); + }); + svr.Get(R"(/api/v1/components/([^/]+))", [&faults](const httplib::Request & req, httplib::Response & res) { + if (req.matches[1].str() != "perception_ecu") { + res.set_content(R"({"id":"robot_alpha","name":"Robot Alpha","x-medkit":{"source":"manifest"}})", + "application/json"); + return; + } + if (faults.subcomponent_detail_fails.load()) { + res.status = 500; + return; + } + res.set_content(R"({"id":"perception_ecu","name":"Perception ECU","x-medkit":{"source":"manifest",)" + R"("parentComponentId":"robot_alpha","dependsOn":["compute_unit"]}})", + "application/json"); + }); + + svr.Get("/api/v1/apps", [](const httplib::Request &, httplib::Response & res) { + res.set_content(R"({"items":[{"id":"brake_monitor","name":"Brake Monitor",)" + R"("x-medkit":{"source":"manifest","is_online":true}}]})", + "application/json"); + }); + svr.Get(R"(/api/v1/apps/([^/]+)/operations)", [](const httplib::Request &, httplib::Response & res) { + res.set_content(R"({"items":[]})", "application/json"); + }); + + svr.Get("/api/v1/functions", [](const httplib::Request &, httplib::Response & res) { + res.set_content(R"({"items":[{"id":"vehicle_health","name":"Vehicle Health"}]})", "application/json"); + }); + svr.Get(R"(/api/v1/functions/([^/]+))", [&faults](const httplib::Request &, httplib::Response & res) { + if (faults.function_detail_fails.load()) { + res.status = 500; + return; + } + res.set_content(R"({"id":"vehicle_health","name":"Vehicle Health",)" + R"("x-medkit":{"source":"manifest","hosts":["brake_monitor"]}})", + "application/json"); + }); +} + +/// The entity carrying `id`, or nullptr. Order within a merged collection is +/// not part of any promise, so tests look entities up rather than index them. +template +const Entity * find_entity(const std::vector & entities, const std::string & id) { + for (const auto & entity : entities) { + if (entity.id == id) { + return &entity; + } + } + return nullptr; +} + +AggregationConfig config_for(const std::string & peer_name, int port) { + AggregationConfig config; + config.enabled = true; + config.timeout_ms = 2000; + AggregationConfig::PeerConfig peer; + peer.name = peer_name; + peer.url = "http://127.0.0.1:" + std::to_string(port); + config.peers.push_back(peer); + return config; +} + +} // namespace + +TEST(AggregationManager, a_subarea_branch_that_could_not_be_read_is_not_dropped_silently) { + // The list endpoint filters subareas out, so the nested route is the only + // place they are named. A refresh that cannot read it holds an area tree one + // branch short of what the peer described. + PeerFaults faults; + MockPeerServer mock; + install_complete_peer(mock.server(), faults); + const int port = mock.start(); + + AggregationManager manager(config_for("zone_peer", port)); + manager.check_all_health(); + ASSERT_EQ(manager.healthy_peer_count(), 1u); + + auto complete = manager.fetch_and_merge_peer_entities({}, {}, {}, {}); + ASSERT_NE(find_entity(complete.areas, "vehicle"), nullptr); + ASSERT_NE(find_entity(complete.areas, "sensors"), nullptr) << "the mock never served the subarea branch"; + + faults.subareas_fail.store(true); + auto partial = manager.fetch_and_merge_peer_entities({}, {}, {}, {}); + + EXPECT_EQ(find_entity(partial.areas, "vehicle") != nullptr, find_entity(partial.areas, "sensors") != nullptr) + << "the area tree kept a parent and lost its branch: a client sees a shape the peer never described"; +} + +TEST(AggregationManager, a_subcomponent_whose_detail_could_not_be_read_is_not_published_without_relationships) { + // A Component's parent and dependencies live in its detail response. Built + // from the nested list alone it is a Component asserted to have neither. + PeerFaults faults; + MockPeerServer mock; + install_complete_peer(mock.server(), faults); + const int port = mock.start(); + + AggregationManager manager(config_for("ecu_peer", port)); + manager.check_all_health(); + ASSERT_EQ(manager.healthy_peer_count(), 1u); + + auto complete = manager.fetch_and_merge_peer_entities({}, {}, {}, {}); + const Component * described = find_entity(complete.components, "perception_ecu"); + ASSERT_NE(described, nullptr); + ASSERT_EQ(described->parent_component_id, "robot_alpha") << "the mock never served the relationships"; + ASSERT_EQ(described->depends_on.size(), 1u); + + faults.subcomponent_detail_fails.store(true); + auto partial = manager.fetch_and_merge_peer_entities({}, {}, {}, {}); + + const Component * kept = find_entity(partial.components, "perception_ecu"); + ASSERT_NE(kept, nullptr) << "a declared subcomponent vanished while its peer still declares it"; + EXPECT_EQ(kept->parent_component_id, "robot_alpha") + << "a subcomponent was published with its relationships stripped after the only request that reports them " + "failed"; + EXPECT_EQ(kept->depends_on.size(), 1u); +} + +TEST(AggregationManager, a_peer_that_does_not_offer_the_nested_routes_still_aggregates) { + // A gateway older than the nested collection routes answers 404 for them. + // That is a version boundary, not a failed read: everything else the peer + // says still stands, and aggregation across the boundary keeps working. + MockPeerServer mock; + mock.server().Get("/api/v1/health", [](const httplib::Request &, httplib::Response & res) { + res.set_content(R"({"status":"healthy"})", "application/json"); + }); + mock.server().Get("/api/v1/areas", [](const httplib::Request &, httplib::Response & res) { + res.set_content(R"({"items":[{"id":"zone_a","name":"Zone A"}]})", "application/json"); + }); + mock.server().Get("/api/v1/components", [](const httplib::Request &, httplib::Response & res) { + res.set_content(R"({"items":[{"id":"ecu_1","name":"ECU 1"}]})", "application/json"); + }); + mock.server().Get(R"(/api/v1/components/([^/]+))", [](const httplib::Request & req, httplib::Response & res) { + res.set_content(nlohmann::json({{"id", req.matches[1].str()}, {"name", "ECU 1"}}).dump(), "application/json"); + }); + mock.server().Get("/api/v1/apps", [](const httplib::Request &, httplib::Response & res) { + res.set_content(R"({"items":[{"id":"nav","name":"Navigation"}]})", "application/json"); + }); + mock.server().Get("/api/v1/functions", [](const httplib::Request &, httplib::Response & res) { + res.set_content(R"({"items":[{"id":"drive","name":"Drive"}]})", "application/json"); + }); + mock.server().Get(R"(/api/v1/functions/([^/]+))", [](const httplib::Request &, httplib::Response & res) { + res.set_content(R"({"id":"drive","name":"Drive","x-medkit":{"hosts":["nav"]}})", "application/json"); + }); + // /areas/{id}/subareas, /components/{id}/subcomponents and + // /apps/{id}/operations are deliberately absent: this peer predates them. + const int port = mock.start(); + + AggregationManager manager(config_for("old_peer", port)); + manager.check_all_health(); + ASSERT_EQ(manager.healthy_peer_count(), 1u); + + auto result = manager.fetch_and_merge_peer_entities({}, {}, {}, {}); + + EXPECT_NE(find_entity(result.areas, "zone_a"), nullptr) << "an absent nested route swallowed the whole peer"; + EXPECT_NE(find_entity(result.components, "ecu_1"), nullptr); + EXPECT_NE(find_entity(result.apps, "nav"), nullptr); + const Function * function = find_entity(result.functions, "drive"); + ASSERT_NE(function, nullptr); + EXPECT_EQ(function->hosts.size(), 1u); +} + +TEST(AggregationManager, a_peer_reporting_one_entity_as_not_responding_still_describes_the_rest_of_itself) { + // What a middle gateway in a chain answers once its own leaf goes quiet: it + // still lists the leaf's declared Component but answers `504 not-responding` + // for the detail. That names the entity and reports it unreachable, so it is + // read as such - the middle gateway is healthy and everything else it says + // still stands. + MockPeerServer mock; + mock.server().Get("/api/v1/health", [](const httplib::Request &, httplib::Response & res) { + res.set_content(R"({"status":"healthy"})", "application/json"); + }); + mock.server().Get("/api/v1/areas", [](const httplib::Request &, httplib::Response & res) { + res.set_content(R"({"items":[]})", "application/json"); + }); + mock.server().Get("/api/v1/components", [](const httplib::Request &, httplib::Response & res) { + res.set_content(R"({"items":[{"id":"ecu_b","name":"ECU B","x-medkit":{"source":"manifest"}},)" + R"({"id":"ecu_c","name":"ECU C","x-medkit":{"source":"manifest"}}]})", + "application/json"); + }); + mock.server().Get(R"(/api/v1/components/([^/]+)/subcomponents)", + [](const httplib::Request &, httplib::Response & res) { + res.set_content(R"({"items":[]})", "application/json"); + }); + mock.server().Get(R"(/api/v1/components/([^/]+))", [](const httplib::Request & req, httplib::Response & res) { + if (req.matches[1].str() == "ecu_c") { + res.status = 504; + res.set_content(R"({"error_code":"not-responding","message":"Member 'ecu_c' is not available",)" + R"("parameters":{"entity_id":"ecu_c"}})", + "application/json"); + return; + } + res.set_content(R"({"id":"ecu_b","name":"ECU B","x-medkit":{"source":"manifest","dependsOn":["compute_unit"]}})", + "application/json"); + }); + mock.server().Get("/api/v1/apps", [](const httplib::Request &, httplib::Response & res) { + res.set_content(R"({"items":[{"id":"brake_monitor","name":"Brake Monitor"}]})", "application/json"); + }); + mock.server().Get(R"(/api/v1/apps/([^/]+)/operations)", [](const httplib::Request &, httplib::Response & res) { + res.set_content(R"({"items":[]})", "application/json"); + }); + mock.server().Get("/api/v1/functions", [](const httplib::Request &, httplib::Response & res) { + res.set_content(R"({"items":[]})", "application/json"); + }); + const int port = mock.start(); + + AggregationManager manager(config_for("middle_gateway", port)); + manager.check_all_health(); + ASSERT_EQ(manager.healthy_peer_count(), 1u); + + auto result = manager.fetch_and_merge_peer_entities({}, {}, {}, {}); + + const Component * own = find_entity(result.components, "ecu_b"); + ASSERT_NE(own, nullptr) << "one entity the peer could not describe swallowed everything else it said"; + EXPECT_EQ(own->depends_on.size(), 1u); + EXPECT_TRUE(own->available); + EXPECT_NE(find_entity(result.apps, "brake_monitor"), nullptr); + + const Component * far = find_entity(result.components, "ecu_c"); + ASSERT_NE(far, nullptr) << "the peer still names this Component, so it is not absent"; + EXPECT_FALSE(far->available) << "the peer said this Component cannot be reached and that is not carried"; +} + +TEST(AggregationManager, an_incomplete_refresh_does_not_replace_the_declaration_a_silent_peer_is_remembered_by) { + // The user-visible bug: one failed detail request overwrites the last good + // declaration with a memberless one, and retention then serves that forever. + PeerFaults faults; + { + MockPeerServer mock; + install_complete_peer(mock.server(), faults); + const int port = mock.start(); + + AggregationManager manager(config_for("brake_ecu", port)); + manager.check_all_health(); + ASSERT_EQ(manager.healthy_peer_count(), 1u); + + auto complete = manager.fetch_and_merge_peer_entities({}, {}, {}, {}); + const Function * described = find_entity(complete.functions, "vehicle_health"); + ASSERT_NE(described, nullptr); + ASSERT_EQ(described->hosts.size(), 1u) << "the mock never served the Function's members"; + + // One refresh in the middle cannot read the members. + faults.function_detail_fails.store(true); + manager.fetch_and_merge_peer_entities({}, {}, {}, {}); + + // Then the peer goes away for good and only the retained picture is left. + mock.server().stop(); + manager.check_all_health(); + ASSERT_EQ(manager.healthy_peer_count(), 0u); + + auto silent = manager.fetch_and_merge_peer_entities({}, {}, {}, {}); + const Function * retained = find_entity(silent.functions, "vehicle_health"); + ASSERT_NE(retained, nullptr) << "a declared Function vanished when its peer went quiet"; + ASSERT_EQ(retained->hosts.size(), 1u) + << "the retained Function lost its members: an incomplete refresh became the record of what the peer " + "declared"; + EXPECT_EQ(retained->hosts[0], "brake_monitor"); + } +} + +TEST(AggregationManager, an_incomplete_refresh_from_a_reachable_peer_does_not_report_its_entities_unavailable) { + // `available:false` answers "can a request get there". A peer whose health + // check passes can be reached; failing to read one of its routes says + // something about this refresh, not about the peer. + PeerFaults faults; + MockPeerServer mock; + install_complete_peer(mock.server(), faults); + const int port = mock.start(); + + AggregationManager manager(config_for("live_peer", port)); + manager.check_all_health(); + ASSERT_EQ(manager.healthy_peer_count(), 1u); + + auto complete = manager.fetch_and_merge_peer_entities({}, {}, {}, {}); + const App * live = find_entity(complete.apps, "brake_monitor"); + ASSERT_NE(live, nullptr); + ASSERT_TRUE(live->available); + ASSERT_TRUE(live->is_online); + + faults.function_detail_fails.store(true); + auto incomplete = manager.fetch_and_merge_peer_entities({}, {}, {}, {}); + + const App * kept = find_entity(incomplete.apps, "brake_monitor"); + ASSERT_NE(kept, nullptr); + EXPECT_TRUE(kept->available) + << "a peer that answers its health check was reported unreachable because one of its routes could not be read"; + EXPECT_TRUE(kept->is_online) << "a running app was reported offline by a refresh that never asked about it"; + EXPECT_EQ(manager.healthy_peer_count(), 1u) << "the peer was still answering; it must not be counted as silent"; +} + +TEST(AggregationManager, a_peer_that_dies_mid_refresh_reports_its_entities_unavailable) { + // The other half of the same rule, taken where it is actually decided: the + // health check that gates a refresh is taken before it, so a peer that dies + // after it passes is inside the refresh and the failed fetch is the first + // this gateway hears of it. Availability has to follow the peer as it is + // now, not the flag from before. + PeerFaults faults; + { + MockPeerServer mock; + install_complete_peer(mock.server(), faults); + const int port = mock.start(); + + AggregationManager manager(config_for("dying_peer", port)); + manager.check_all_health(); + ASSERT_EQ(manager.healthy_peer_count(), 1u); + + auto complete = manager.fetch_and_merge_peer_entities({}, {}, {}, {}); + ASSERT_NE(find_entity(complete.apps, "brake_monitor"), nullptr); + + mock.server().stop(); + ASSERT_EQ(manager.healthy_peer_count(), 1u) << "the refresh must start out believing the peer is there"; + + auto silent = manager.fetch_and_merge_peer_entities({}, {}, {}, {}); + const App * retained = find_entity(silent.apps, "brake_monitor"); + ASSERT_NE(retained, nullptr) << "a declared entity vanished when its peer went quiet"; + EXPECT_FALSE(retained->available) << "a peer that stopped answering during the refresh was reported reachable"; + EXPECT_FALSE(retained->is_online) << "a retained app is not observably running"; + } +} + // ============================================================================= // fan_out_get happy-path tests with mock server // ============================================================================= diff --git a/src/ros2_medkit_gateway/test/test_peer_client.cpp b/src/ros2_medkit_gateway/test/test_peer_client.cpp index 048fa750d..a5dd99be1 100644 --- a/src/ros2_medkit_gateway/test/test_peer_client.cpp +++ b/src/ros2_medkit_gateway/test/test_peer_client.cpp @@ -187,6 +187,10 @@ TEST(PeerClientHappyPath, fetch_entities_parses_collections) { svr.Get("/api/v1/components", [](const httplib::Request &, httplib::Response & res) { res.set_content(R"({"items":[{"id":"ecu_1","name":"ECU 1"}]})", "application/json"); }); + // A peer that names a Component in its list describes it on the detail route. + svr.Get(R"(/api/v1/components/ecu_1)", [](const httplib::Request &, httplib::Response & res) { + res.set_content(R"({"id":"ecu_1","name":"ECU 1"})", "application/json"); + }); svr.Get("/api/v1/apps", [](const httplib::Request &, httplib::Response & res) { res.set_content(R"({"items":[{"id":"nav","name":"Navigation"}]})", "application/json"); }); @@ -290,11 +294,18 @@ TEST(PeerClientHappyPath, fetch_entities_parses_relationship_fields) { svr.Get("/api/v1/functions", [](const httplib::Request &, httplib::Response & res) { res.set_content( R"({"items":[ - {"id":"autonomous-navigation","name":"Autonomous Navigation", - "x-medkit":{"hosts":["lidar-driver","path-planner"],"source":"manifest"}} + {"id":"autonomous-navigation","name":"Autonomous Navigation","x-medkit":{"source":"manifest"}} ]})", "application/json"); }); + // hosts are carried by the detail route only, which is where a real gateway + // puts them. + svr.Get(R"(/api/v1/functions/autonomous-navigation)", [](const httplib::Request &, httplib::Response & res) { + res.set_content( + R"({"id":"autonomous-navigation","name":"Autonomous Navigation", + "x-medkit":{"hosts":["lidar-driver","path-planner"],"source":"manifest"}})", + "application/json"); + }); int port = svr.bind_to_any_port("127.0.0.1"); std::thread t([&]() { From 1308fc15fa825924b5acff5360231fa302390b29 Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Thu, 20 Aug 2026 11:43:03 +0200 Subject: [PATCH 04/22] fix(operations): refuse an id that names more than one operation The ambiguity check ran only on the bare form and counted members. A qualified id whose member exposes the same short name at two ROS paths skipped it entirely and ran whichever copy was walked first. On an entity that exposes its own operations the count came back as empty strings, so the refusal named no member and offered a form that entity cannot parse. The check now counts what the id actually resolves to, using the predicate the resolver walks. When the copies belong to different members the remedy is to name one; when they belong to a single member the answer says so and reports the ROS paths that collided. --- .../core/http/member_qualified_id.hpp | 7 + .../ros2_medkit_gateway/dto/operations.hpp | 8 +- .../src/http/handlers/operation_handlers.cpp | 124 +++++++++++++----- .../test_grouping_entity_aggregation.test.py | 34 ++++- 4 files changed, 135 insertions(+), 38 deletions(-) diff --git a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/http/member_qualified_id.hpp b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/http/member_qualified_id.hpp index ecb1762a1..87fb44aec 100644 --- a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/http/member_qualified_id.hpp +++ b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/http/member_qualified_id.hpp @@ -106,6 +106,13 @@ void qualify_ambiguous_ids(std::vector & items, MemberIdsOf member_ids_of) if (members == nullptr || members->size() != 1) { continue; } + // An id already addressed to this member is left alone. Prefixing it again + // yields a form whose first colon splits off the member twice, which names + // nothing - and it happens whenever one member exposes the same short name + // more than once, because both copies then carry the same qualified id. + if (item.id.rfind(members->front() + ":", 0) == 0) { + continue; + } item.id = make_member_qualified_id(members->front(), item.id); } } diff --git a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/dto/operations.hpp b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/dto/operations.hpp index 04f65b449..6c5f04f51 100644 --- a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/dto/operations.hpp +++ b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/dto/operations.hpp @@ -48,6 +48,10 @@ namespace dto { // services; goal/result/feedback for actions); kept as // nlohmann::json because the structure is runtime-determined // by type introspection and cannot be statically typed +// member_ids - the members of an aggregating entity that provide this +// operation; absent on an entity that exposes its own +// available - present only as false, marking an item whose provider is not +// answering; absent means it can be served // ============================================================================= struct XMedkitOperationItem { std::optional ros2; @@ -112,7 +116,9 @@ inline constexpr std::string_view dto_name = "XMedkit // proximity_proof_required - bool (required, always false for ROS 2) // asynchronous_execution - bool (required; false for services, true for actions) // x-medkit - typed vendor extension; carries ros2.{service|action, -// type,kind}, entity_id, source, and optional type_info +// type,kind}, entity_id, source, optional type_info, +// and, on an aggregating entity, member_ids plus +// available // ============================================================================= struct OperationItem { std::string id; diff --git a/src/ros2_medkit_gateway/src/http/handlers/operation_handlers.cpp b/src/ros2_medkit_gateway/src/http/handlers/operation_handlers.cpp index a36859ffd..9acc067e5 100644 --- a/src/ros2_medkit_gateway/src/http/handlers/operation_handlers.cpp +++ b/src/ros2_medkit_gateway/src/http/handlers/operation_handlers.cpp @@ -176,28 +176,55 @@ ResolvedOperation resolve_operation(const AggregatedOperations & ops, const http return resolved; } -/// Members of `ops` that expose `short_name`, in collection order. -/// -/// More than one means the bare id names more than one operation. Keyed on the -/// short name because that is the wire id; the full ROS paths differ, which is -/// exactly why the short name stops identifying one of them. -std::vector local_providers_of(const AggregatedOperations & ops, const std::string & short_name) { - std::vector providers; - const auto record = [&ops, &providers](const std::string & full_path) { +/// One operation `parsed` names: its ROS path, and the member that owns it if +/// the entity has members at all. +struct OperationMatch { + std::string full_path; + std::string member_id; ///< empty when the entity exposes its operations directly +}; + +/// Everything `parsed` names, in collection order, using the same predicate +/// `resolve_operation` walks - so what is counted here is exactly what would +/// have been run. More than one match means the id does not identify an +/// operation, whether the extra copies belong to different members or to one +/// member that uses the same short name at two ROS paths. +std::vector matching_operations(const AggregatedOperations & ops, + const http::MemberQualifiedId & parsed) { + std::vector matches; + const auto record = [&ops, &parsed, &matches](const std::string & full_path) { auto owner = ops.owner_by_path.find(full_path); - providers.push_back(owner != ops.owner_by_path.end() ? owner->second : std::string{}); + const std::string member = owner != ops.owner_by_path.end() ? owner->second : std::string{}; + if (parsed.has_member && member != parsed.member_id) { + return; + } + matches.push_back(OperationMatch{full_path, member}); }; for (const auto & svc : ops.services) { - if (svc.name == short_name) { + if (svc.name == parsed.item_id) { record(svc.full_path); } } for (const auto & act : ops.actions) { - if (act.name == short_name) { + if (act.name == parsed.item_id) { record(act.full_path); } } - return providers; + return matches; +} + +/// The distinct members among `matches`, dropping the empty owner an entity +/// that exposes its own operations reports. +std::vector distinct_members(const std::vector & matches) { + std::vector members; + for (const auto & match : matches) { + if (match.member_id.empty()) { + continue; + } + if (std::find(members.begin(), members.end(), match.member_id) == members.end()) { + members.push_back(match.member_id); + } + } + return members; } /// The error for a member that is in the tree but whose gateway is silent. @@ -208,15 +235,19 @@ std::vector local_providers_of(const AggregatedOperations & ops, co /// replaces - quietly running a different member's operation, or a 200 with /// nothing in it. `not-responding` is the SOVD code for "no response from the /// underlying entity", which is exactly the situation. -std::optional member_unavailable_error(const ThreadSafeEntityCache & cache, const std::string & entity_id, - const std::string & member_id, const std::string & operation_id) { - bool unavailable = false; +bool member_is_unreachable(const ThreadSafeEntityCache & cache, const std::string & member_id) { if (auto app = cache.get_app(member_id)) { - unavailable = !app->available; - } else if (auto component = cache.get_component(member_id)) { - unavailable = !component->available; + return !app->available; + } + if (auto component = cache.get_component(member_id)) { + return !component->available; } - if (!unavailable) { + return false; +} + +std::optional member_unavailable_error(const ThreadSafeEntityCache & cache, const std::string & entity_id, + const std::string & member_id, const std::string & operation_id) { + if (!member_is_unreachable(cache, member_id)) { return std::nullopt; } return make_error(504, ERR_NOT_RESPONDING, "Member '" + member_id + "' is not available", @@ -714,6 +745,17 @@ http::Result OperationHandlers::get_operation(const http:: detail.item.asynchronous_execution = true; detail.item.x_medkit = build_action_xmedkit(*resolved.action, entity_id, type_introspection); } + + // A retained member's operation is still described, because the description + // is what was retained - but it says it cannot be served, so a client reading + // the item on its own learns what the collection already told it, instead of + // an absent field that means the opposite. + const std::string & full_path = + resolved.service.has_value() ? resolved.service->full_path : resolved.action->full_path; + if (auto owner = ops.owner_by_path.find(full_path); + owner != ops.owner_by_path.end() && member_is_unreachable(cache, owner->second)) { + detail.item.x_medkit->available = false; + } return detail; } @@ -819,22 +861,38 @@ OperationHandlers::create_execution(const http::TypedRequest & req, dto::Executi json{{"entity_id", entity_id}, {"operation_id", operation_id}})); } - // A bare id that names more than one operation runs whichever member was - // walked first, and the caller never learns which. That is the one case the - // bare form cannot carry, so it is the one case that is refused - an id that - // names a single operation still executes, which is what every current - // client sends. An id that names nothing was already answered as not found - // above: telling a caller to qualify a typo would not help them. - if (!parsed.has_member) { - const std::vector providers = local_providers_of(ops, operation_id); - if (providers.size() > 1) { + // An id that names more than one operation would run whichever was walked + // first, and the caller would never learn which. An id that names a single + // operation still executes, which is what every current client sends, and an + // id that names nothing was already answered as not found above. + // + // The qualified form is checked too. Naming the member narrows the set, but + // one member that uses the same short name at two ROS paths is still not + // identified by it - and for those two the member half has nothing left to + // add, so the caller is told what collided rather than handed a remedy that + // cannot work. + const std::vector matches = matching_operations(ops, parsed); + if (matches.size() > 1) { + std::vector paths; + paths.reserve(matches.size()); + for (const auto & match : matches) { + paths.push_back(match.full_path); + } + const std::vector members = distinct_members(matches); + json params{{"entity_id", entity_id}, {"operation_id", operation_id}, {"ros2_paths", paths}}; + if (!members.empty()) { + params["member_ids"] = members; + } + if (members.size() > 1) { + params["details"] = "Use format 'member_id:operation_id' to name the member that runs it"; return tl::make_unexpected( - make_error(400, ERR_INVALID_REQUEST, "Ambiguous operation id: more than one member provides it", - json{{"details", "Use format 'member_id:operation_id' to name the member that runs it"}, - {"entity_id", entity_id}, - {"operation_id", operation_id}, - {"member_ids", providers}})); + make_error(400, ERR_INVALID_REQUEST, "Ambiguous operation id: more than one member provides it", params)); } + params["details"] = + "One provider exposes this short name at more than one ROS path, so naming the member " + "cannot separate them"; + return tl::make_unexpected( + make_error(400, ERR_INVALID_REQUEST, "Ambiguous operation id: it names more than one operation", params)); } // Whoever ends up owning the resolved operation must actually be reachable. diff --git a/src/ros2_medkit_integration_tests/test/features/test_grouping_entity_aggregation.test.py b/src/ros2_medkit_integration_tests/test/features/test_grouping_entity_aggregation.test.py index 09234558c..7a870888c 100644 --- a/src/ros2_medkit_integration_tests/test/features/test_grouping_entity_aggregation.test.py +++ b/src/ros2_medkit_integration_tests/test/features/test_grouping_entity_aggregation.test.py @@ -678,6 +678,32 @@ def _primary_subcomponent(parent_id, subcomponent_id): return item return None + def test_y_every_id_the_list_offers_runs_while_every_member_answers(self): + """The same agreement as z6, but with nothing broken. + + z6 walks the list after the peer has been killed, so a peer-owned id is + allowed to answer 504 and the case cannot tell a working dispatch from a + missing one. Here every member is reachable, so the only honest answer + is that the operation ran. + """ + items = self._items(f'functions/{MERGED_FUNCTION}', 'operations') + offered = [item.get('id') for item in items if item.get('id', '').endswith('calibrate')] + self.assertTrue(offered, 'the list offered no calibrate operation to check') + + for operation_id in offered: + with self.subTest(operation=operation_id): + response = requests.post( + f'{PRIMARY_URL}/functions/{MERGED_FUNCTION}/operations/' + f'{quote(operation_id, safe="")}/executions', + json={}, + timeout=15, + ) + self.assertIn( + response.status_code, (200, 202), + f'every member answers, yet executing the offered id ' + f'{operation_id!r} returned {response.status_code}: {response.text}', + ) + def test_z1_a_declared_entity_survives_its_peer_going_silent(self, peer_gateway): """R10: what a peer DECLARED does not stop being true when it goes quiet. @@ -908,10 +934,10 @@ def test_z6_every_id_the_list_offers_is_executable(self): json={}, timeout=15, ) - self.assertNotEqual( - response.status_code, 400, - f'the list offers {operation_id!r} but executing it is ' - f'refused: {response.text}', + self.assertIn( + response.status_code, (200, 202, 504), + f'the list offers {operation_id!r} but executing it answered ' + f'{response.status_code}: {response.text}', ) def test_z7_suppression_omits_the_peer_without_losing_ambiguity(self): From b83ac3d4559f6cfe27badbde21383e72ca4ffda4 Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Thu, 20 Aug 2026 11:43:04 +0200 Subject: [PATCH 05/22] feat(aggregation): serve a member-qualified request on the gateway that owns the member An aggregating entity listed resources belonging to members on other gateways and then served every request for them locally, looking for a peer's topic or service on a graph where it does not exist. A read answered "not provided by member" and an execution answered "service not available" while the member and its gateway were healthy. Ownership is settled once, after the id is resolved. An unreachable member is answered before anything is forwarded, and a member this gateway owns is served here without a hop. Data, operations and configurations share the one dispatch point. The forward addresses the member's own route rather than replaying the incoming path, so the SSRF guard applies to the path actually sent. --- docs/api/rest.rst | 61 +- docs/config/aggregation.rst | 35 ++ src/ros2_medkit_gateway/README.md | 50 +- .../design/aggregation.rst | 74 +++ .../aggregation/aggregation_manager.hpp | 21 + .../http/handlers/handler_context.hpp | 48 ++ .../src/aggregation/aggregation_manager.cpp | 13 +- .../src/http/handlers/config_handlers.cpp | 42 ++ .../src/http/handlers/data_handlers.cpp | 103 +++- .../src/http/handlers/handler_context.cpp | 57 ++ .../src/http/handlers/operation_handlers.cpp | 33 +- .../test/test_aggregation_manager.cpp | 133 +++++ .../CMakeLists.txt | 2 +- .../demo_nodes/calibration_service.cpp | 11 + .../test_grouping_entity_aggregation.test.py | 539 +++++++++++++++++- 15 files changed, 1164 insertions(+), 58 deletions(-) diff --git a/docs/api/rest.rst b/docs/api/rest.rst index bdab9ac9f..f22c3f259 100644 --- a/docs/api/rest.rst +++ b/docs/api/rest.rst @@ -658,10 +658,64 @@ What this means for a request: - A qualified id is accepted on the single-item routes. A member half that names no member of the entity is ``404``, and so is an item half that member does not provide - which is what tells an absent item apart from an item that - exists and currently carries no data. + exists and currently carries no data. A member half followed by nothing names + no item and is ``404`` as well. - Reads are permissive: ``GET`` of a bare id returns the first match rather than refusing, which is the behaviour every existing client depends on. +Where a Member-Qualified Request is Served +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +An aggregating entity holds no resources of its own, and its members can belong +to different gateways. A request naming one member is therefore served by the +gateway that owns **that member**, on the member's own entity route: + +.. code-block:: text + + POST /api/v1/functions/vehicle_health/operations/peer_calibration:calibrate/executions + +is answered, when ``peer_calibration`` belongs to a peer, by + +.. code-block:: text + + POST /api/v1/apps/peer_calibration/operations/calibrate/executions + +on that peer, and the peer's response is what the client receives. The ROS +service or topic behind the id only exists on the owner's graph, so no other +gateway can answer. A member this gateway owns is served locally exactly as +before. This applies to ``GET`` and ``PUT`` of a single ``/data`` item, to +``POST`` of an ``/operations`` execution, and to ``GET``, ``PUT`` and +``DELETE`` of a single ``/configurations`` item. + +``/configurations`` keeps its own id scheme, ``:``, and the +member half is the app id. Because nothing on the owning gateway is aggregating, +the parameter is addressed there by its bare name: + +.. code-block:: text + + PUT /api/v1/functions/vehicle_health/configurations/peer_calibration:calibration_offset + +is answered, when ``peer_calibration`` belongs to a peer, by + +.. code-block:: text + + PUT /api/v1/apps/peer_calibration/configurations/calibration_offset + +so the value comes from - and the write lands on - the ROS node that actually +declares the parameter. ``GET /{entity}/configurations`` is unaffected: peer +parameters reach that listing through the collection fan-out, and the ids it +offers are the ids the single-item routes accept. + +Reachability is decided before anything is forwarded. A member retained while +its gateway is silent answers ``504 not-responding`` naming the member (see +:ref:`retained-entities`) rather than a ``502`` from a connection that could not +be made. + +``X-Medkit-No-Fan-Out`` does not change this. The header bounds the collection +fan-out that merges peer items into a listing; a request naming one member +already names its owner, so it is one hop and is answered by that owner whether +or not the header is present. + Ambiguity is a property of the declared tree, not of who is reachable right now. A peer's declared operations are held locally, so the same request gets the same answer whether or not that peer is currently answering, and deciding @@ -1018,6 +1072,11 @@ Manage ROS 2 node parameters. and returns the first node that answers. Items carry the owning app in ``x-medkit.source``. + The ```` half is a member id, so ``GET``, ``PUT`` and ``DELETE`` of a + qualified id are served by the gateway that owns that app, on its own + ``/apps/{app_id}/configurations/{param_name}`` route. See + :ref:`member-qualified-ids` for the dispatch and its ``504`` case. + ``GET /api/v1/components/{id}/configurations`` List all parameters for an entity. diff --git a/docs/config/aggregation.rst b/docs/config/aggregation.rst index 50579b10c..6897270ff 100644 --- a/docs/config/aggregation.rst +++ b/docs/config/aggregation.rst @@ -355,6 +355,41 @@ When aggregation is enabled, entities from peers are merged with local entities: Requests for remote entities are transparently forwarded to the owning peer. The routing table maps entity IDs to peer names. +An entity that draws its resources from members - an Area, a merged Function, a +hierarchical parent Component - is deliberately absent from that table, because +its members can sit on different gateways and routing it whole would discard +every member the other contributors hold. A request that names one member is +routed instead: it is re-addressed to that member's own entity route on the +gateway that owns it, so + +.. code-block:: text + + POST /api/v1/functions/vehicle_health/operations/peer_calibration:calibrate/executions + +becomes, on the peer that runs ``peer_calibration``, + +.. code-block:: text + + POST /api/v1/apps/peer_calibration/operations/calibrate/executions + +The same routing applies to a single ``/data`` item and to a single +``/configurations`` parameter. A configuration id is ``:``, +and on the owning gateway the parameter is addressed by its bare name, so + +.. code-block:: text + + PUT /api/v1/functions/vehicle_health/configurations/peer_calibration:calibration_offset + +becomes + +.. code-block:: text + + PUT /api/v1/apps/peer_calibration/configurations/calibration_offset + +Reachability is answered before anything is forwarded, so a member whose gateway +is silent gets ``504 not-responding`` rather than a ``502`` from a failed +connection. A member this gateway owns is served here, unchanged. + See :doc:`../design/ros2_medkit_gateway/aggregation` for detailed merge logic and architecture diagrams. diff --git a/src/ros2_medkit_gateway/README.md b/src/ros2_medkit_gateway/README.md index 9ebcfb5ef..b62aa9574 100644 --- a/src/ros2_medkit_gateway/README.md +++ b/src/ros2_medkit_gateway/README.md @@ -286,11 +286,55 @@ two gateways each contribute an item under that path. the Foxglove panel, the MCP tools and the generated OpenAPI document all send. - `POST /{entity}/operations/{id}/executions` with a bare id several members provide is `400 invalid-request`, naming the qualified form and the members. -- A qualified id is accepted on single-item routes; an unknown member half, or - an item half that member does not provide, is `404` - which is what tells an - absent item apart from one that exists and carries no data. +- A qualified id is accepted on single-item routes; an unknown member half, an + item half that member does not provide, or a member half followed by nothing, + is `404` - which is what tells an absent item apart from one that exists and + carries no data. - `GET` of a bare id stays permissive and returns the first match. +A member-qualified request is served by the gateway that owns that member, on +the member's own entity route. An aggregating entity holds nothing itself and +its members can belong to different gateways, so + +``` +POST /api/v1/functions/vehicle_health/operations/peer_calibration:calibrate/executions +``` + +becomes, when `peer_calibration` belongs to a peer, + +``` +POST /api/v1/apps/peer_calibration/operations/calibrate/executions +``` + +on that peer, and the peer's answer is what the client gets. The ROS service or +topic behind the id lives on the owner's graph and nowhere else. A locally owned +member is served here as before. This covers `GET` and `PUT` of one `/data` item, +`POST` of an `/operations` execution, and `GET`, `PUT` and `DELETE` of one +`/configurations` item. + +`/configurations` keeps its own id scheme, `:`, whose member +half is the app id. Nothing on the owning gateway is aggregating, so the +parameter is addressed there by its bare name: + +``` +PUT /api/v1/functions/vehicle_health/configurations/peer_calibration:calibration_offset +``` + +becomes + +``` +PUT /api/v1/apps/peer_calibration/configurations/calibration_offset +``` + +and the write lands on the ROS node that declares the parameter. The +`GET /{entity}/configurations` listing is unchanged - peer parameters reach it +through the collection fan-out, and the ids it offers are the ids the +single-item routes accept. Reachability is settled first, so a +member whose gateway has gone silent answers `504 not-responding` instead of a +`502` from a connection that could not be made. `X-Medkit-No-Fan-Out` bounds the +collection fan-out and does not change where a member-qualified request is +served - it already names its owner and is one hop. + Ambiguity is decided from the declared tree, which includes a peer's declared operations held locally. The answer therefore does not change with who is reachable, costs no network call, and cannot be altered by a client-supplied diff --git a/src/ros2_medkit_gateway/design/aggregation.rst b/src/ros2_medkit_gateway/design/aggregation.rst index 89eb052d9..ef0bdfa5e 100644 --- a/src/ros2_medkit_gateway/design/aggregation.rst +++ b/src/ros2_medkit_gateway/design/aggregation.rst @@ -409,6 +409,80 @@ maps to a peer, the request is forwarded transparently: @enduml +Member Dispatch +--------------- + +The routing table answers "who owns this entity", and for an entity that draws +its resources from members that question has no single answer: an Area, a +merged Function and a hierarchical parent Component all have members on both +sides of the link, which is exactly why they are deliberately absent from the +table. Routing one of them whole would hand the request to one peer and discard +every member the other contributors hold. + +A request naming one member is a different question, and it does have a single +answer. ``HandlerContext::dispatch_to_member`` re-addresses such a request to +the member's own entity route - ``/apps/{member}/...`` or +``/components/{member}/...`` - on the gateway the routing table names for that +member: + +.. code-block:: text + + POST /api/v1/functions/vehicle_health/operations/peer_calibration:calibrate/executions + -> POST /api/v1/apps/peer_calibration/operations/calibrate/executions (on the peer) + + GET /api/v1/functions/vehicle_health/data/pressure_sensor:chassis/brakes/pressure + -> GET /api/v1/apps/pressure_sensor/data/chassis/brakes/pressure (on the peer) + + PUT /api/v1/functions/vehicle_health/configurations/peer_calibration:calibration_offset + -> PUT /api/v1/apps/peer_calibration/configurations/calibration_offset (on the peer) + +Each collection keeps its own id scheme and each hands the same two halves to +the dispatch. ``/data`` and ``/operations`` qualify only an ambiguous id and +carry ``x-medkit.member_ids``; ``/configurations`` qualifies every id on a +multi-node entity as ``:`` and carries +``x-medkit.source``. What the two schemes agree on is what the dispatch needs: +the member half is an entity id, and the item half is the id the member's own +route uses. Nothing on the owning gateway is aggregating, so the item half is +sent bare - a parameter as its plain name, a topic as its plain path. + +The member's own gateway is the only one that can answer: the ROS service, the +topic and the parameter behind the id exist on its graph and nowhere else. What +this gateway holds for a peer-owned member is a declaration, which is why the +local walk's record of "does this member provide this item" is consulted only +once the member is known to be served here - on a peer-owned member it holds no +topics at all, and no node FQN to ask for a parameter, so every one of them +would be a miss. + +The order inside ``dispatch_to_member`` is load-bearing: + +1. **Reachability, before anything is sent.** A member retained while its + gateway is silent is answered from what it declared: ``504`` with the SOVD + code ``not-responding``, naming the member. Forwarding to a dead peer instead + produces a socket failure dressed as ``502``, which reports this gateway as + broken rather than the link as down. +2. **Ownership.** No routing entry means this gateway owns the member and the + handler carries on. +3. **Addressing.** The member is looked up in the cache to decide whether it is + an App or a Component, because that decides which collection its route lives + under. +4. **Forward.** ``AggregationManager::forward_request`` is called with the + built path. The overload taking an explicit target applies the same + ``/api/v1/`` SSRF guard and the same ``__`` prefix rewrite to that path + as the two-argument form applies to the incoming one - the target is + assembled from client-supplied ids and is exactly as untrusted. + +The wire is committed by the forward, so the handler returns +``HandlerContext::forwarded_sentinel_error()``: the typed router recognises the +``x-medkit-internal-forwarded`` code and renders nothing, the same channel the +remote-entity branch of ``validate_entity_for_route`` uses. + +Termination does not depend on a hop count. The target is the member's own route +on the gateway that owns it, and there that entity is local, so the receiving +gateway serves it rather than forwarding again. A ``X-Medkit-No-Fan-Out`` header +on the incoming request is propagated but does not suppress the dispatch: +suppression bounds collection fan-out, while a member-qualified request names +its owner and is one hop by construction. + **Entity collection endpoints** (``GET /api/v1/areas``, ``/components``, ``/apps``, ``/functions``) serve from the local entity cache, which is populated during periodic cache refresh cycles that fetch entities from all diff --git a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/aggregation/aggregation_manager.hpp b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/aggregation/aggregation_manager.hpp index 1116ade63..65417adc6 100644 --- a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/aggregation/aggregation_manager.hpp +++ b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/aggregation/aggregation_manager.hpp @@ -273,6 +273,27 @@ class AggregationManager { */ void forward_request(const std::string & peer_name, const httplib::Request & req, httplib::Response & res); + /** + * @brief Forward an HTTP request to a peer, addressing a path of the caller's + * choosing instead of the one the client sent. + * + * A request addressed to an entity that draws its resources from members is + * not the request the owning gateway can answer: that gateway has no such + * entity, only the member. The target path names the member's own route + * there, so the peer answers a request about something it actually holds. + * + * `target_path` goes through the same SSRF guard and peer-prefix rewrite as + * the incoming path, because it is built from client-supplied ids. Query + * parameters and body come from `req` unchanged. + * + * @param peer_name Name of the target peer + * @param req Incoming HTTP request supplying method, headers, body and query + * @param res Outgoing HTTP response to populate + * @param target_path Absolute path to request on the peer, `/api/v1/...` + */ + void forward_request(const std::string & peer_name, const httplib::Request & req, httplib::Response & res, + const std::string & target_path); + /** * @brief Fan-out a GET request to healthy peers in parallel. * diff --git a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/http/handlers/handler_context.hpp b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/http/handlers/handler_context.hpp index 5d5c631fe..030b24c90 100644 --- a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/http/handlers/handler_context.hpp +++ b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/http/handlers/handler_context.hpp @@ -108,6 +108,22 @@ class AggregationManager; namespace handlers { +/** + * @brief Where a request addressed to one member of an aggregating entity is + * served. + * + * An aggregating entity - a merged Function, an Area, a Component with members - + * has no resources of its own; it lists what its members provide, under ids that + * name the member. Such an entity is deliberately absent from the routing table, + * because routing it whole to one peer would discard every member the other + * contributors hold. The member, however, has exactly one owner, and that owner + * is the only gateway whose ROS graph carries the topic or service behind the id. + */ +enum class MemberDispatch { + kServeLocally, ///< This gateway owns the member; the handler continues. + kForwarded, ///< The owning peer answered; the wire is committed. +}; + /** * @brief Shared context for all HTTP handlers * @@ -265,6 +281,38 @@ class HandlerContext { http::ValidatorResult validate_entity_for_route(const http::TypedRequest & req, const std::string & entity_id) const; + /** + * @brief Serve a member-qualified request on the gateway that owns the member. + * + * `validate_entity_for_route` routes a whole entity, which is the wrong unit + * for an aggregating one: the entity is a view whose members can sit on + * different gateways, so the routing table deliberately holds no entry for it + * and every request lands here. The member is the unit that has an owner, so + * the request is re-addressed to the member's own entity route - `/apps/{id}/...` + * or `/components/{id}/...` - on that owner. Anything the local walk knows about + * a member another gateway runs is at best a declaration; the topic and the + * service themselves only exist over there. + * + * Reachability is answered before anything is sent. A member retained while + * its gateway is silent is still in the tree and still addressable, and the + * honest answer is that it cannot be reached - forwarding to a dead peer would + * dress that up as a 502, which blames this gateway for a link that is down. + * + * @param req Typed request supplying method, headers, body and query. + * @param member_id Member the id named. Empty means the id named none. + * @param member_resource_path Path below the member entity, without a leading + * slash (e.g. `data/chassis/brakes/pressure`, + * `operations/calibrate/executions`). + * @param error_params Parameters to attach to the `not-responding` body so a + * caller learns which request could not be served. + * @return kServeLocally when the handler must carry on, kForwarded when the + * peer's response is already on the wire, or a 504 ErrorInfo when the + * member's gateway is silent. + */ + http::Result dispatch_to_member(const http::TypedRequest & req, const std::string & member_id, + const std::string & member_resource_path, + nlohmann::json error_params) const; + /** * @brief Build the framework-internal sentinel error that typed handlers * return after the validator's Forwarded path already committed the diff --git a/src/ros2_medkit_gateway/src/aggregation/aggregation_manager.cpp b/src/ros2_medkit_gateway/src/aggregation/aggregation_manager.cpp index 4be73ebd5..859cf8b3e 100644 --- a/src/ros2_medkit_gateway/src/aggregation/aggregation_manager.cpp +++ b/src/ros2_medkit_gateway/src/aggregation/aggregation_manager.cpp @@ -772,6 +772,11 @@ std::string AggregationManager::get_peer_url(const std::string & peer_name) cons void AggregationManager::forward_request(const std::string & peer_name, const httplib::Request & req, httplib::Response & res) { + forward_request(peer_name, req, res, req.path); +} + +void AggregationManager::forward_request(const std::string & peer_name, const httplib::Request & req, + httplib::Response & res, const std::string & target_path) { // Find peer under lock, take shared_ptr copy for lifetime safety, then release // before network I/O. The shared_ptr keeps the PeerClient alive even if // remove_discovered_peer() erases it from peers_ concurrently. @@ -792,8 +797,10 @@ void AggregationManager::forward_request(const std::string & peer_name, const ht } // Validate forwarded path - only allow SOVD API paths to prevent SSRF - // to internal peer endpoints (e.g., /metrics, /debug, /admin). - if (req.path.rfind("/api/v1/", 0) != 0) { + // to internal peer endpoints (e.g., /metrics, /debug, /admin). The path is + // checked whether it came from the client or was built by a caller from + // client-supplied ids; both are equally untrusted. + if (target_path.rfind("/api/v1/", 0) != 0) { res.status = 400; nlohmann::json error_body; error_body["error_code"] = ERR_INVALID_REQUEST; @@ -808,7 +815,7 @@ void AggregationManager::forward_request(const std::string & peer_name, const ht // the prefix before forwarding. // Anchor to path segment boundary: the prefix must appear right after '/' to avoid // false matches inside other path segments (e.g., "v1" matching "/api/v1/"). - std::string forwarded_path = req.path; + std::string forwarded_path = target_path; std::string prefix = peer_name + EntityMerger::SEPARATOR; auto prefix_pos = forwarded_path.find(prefix); if (prefix_pos != std::string::npos && prefix_pos > 0 && forwarded_path[prefix_pos - 1] == '/') { diff --git a/src/ros2_medkit_gateway/src/http/handlers/config_handlers.cpp b/src/ros2_medkit_gateway/src/http/handlers/config_handlers.cpp index c90467ce6..dc2ba0052 100644 --- a/src/ros2_medkit_gateway/src/http/handlers/config_handlers.cpp +++ b/src/ros2_medkit_gateway/src/http/handlers/config_handlers.cpp @@ -115,6 +115,36 @@ const NodeConfigInfo * find_node_for_app(const std::vector & nod return nullptr; } +/// Settle where a member-qualified configuration id is served, and hand the +/// request over when that is another gateway. Reads, writes and resets share it +/// because a parameter has exactly one owning node whatever the method is. +/// +/// A parameter is owned by a ROS node, and the node behind a member another +/// gateway runs is on a graph this one cannot see: the local walk records no +/// FQN for it, so every method resolved here refuses an id that names it. The +/// member, however, is an entity with an owner, and on that owner the parameter +/// is addressed by its bare name under the member's own App or Component route - +/// nothing over there is aggregating, so nothing over there qualifies the id. +/// +/// Returns the answer the handler must return - including the sentinel that says +/// the owning peer has already committed the wire - or nullopt when this gateway +/// serves the parameter itself, which is every unprefixed id and every member +/// this gateway owns. An unprefixed id carries no member half, and an empty +/// member id is the case `dispatch_to_member` answers with kServeLocally. +std::optional dispatch_configuration(const HandlerContext & ctx, const http::TypedRequest & req, + const std::string & entity_id, const std::string & param_id, + const ParsedParamId & parsed) { + auto dispatch = ctx.dispatch_to_member(req, parsed.app_id, "configurations/" + parsed.param_name, + json{{"entity_id", entity_id}, {"id", param_id}}); + if (!dispatch) { + return dispatch.error(); + } + if (*dispatch == MemberDispatch::kForwarded) { + return HandlerContext::forwarded_sentinel_error(); + } + return std::nullopt; +} + /// Build a typed `ErrorInfo` for a failed parameter operation. Mirrors the /// legacy `send_parameter_error` helper's wire shape (params: details + /// entity_id + id, message: "Failed to parameter"). @@ -470,6 +500,10 @@ http::Result ConfigHandlers::get_configuration(cons auto * config_mgr = ctx_.node()->get_configuration_manager(); auto parsed = parse_aggregated_param_id(param_id, agg_configs.is_aggregated); + if (auto answered = dispatch_configuration(ctx_, req, entity_id, param_id, parsed)) { + return tl::unexpected(*answered); + } + // If targeting a specific app in an aggregated entity, dispatch to that // app's node directly. if (parsed.has_prefix) { @@ -592,6 +626,10 @@ http::Result ConfigHandlers::set_configuration(cons return make_read_value(entity_id, node_fqn, param_id, source_app, param_data); }; + if (auto answered = dispatch_configuration(ctx_, req, entity_id, param_id, parsed)) { + return tl::unexpected(*answered); + } + if (parsed.has_prefix) { const auto * node_info = find_node_for_app(agg_configs.nodes, parsed.app_id); if (node_info == nullptr) { @@ -664,6 +702,10 @@ http::Result ConfigHandlers::delete_configuration(const http::T auto * config_mgr = ctx_.node()->get_configuration_manager(); auto parsed = parse_aggregated_param_id(param_id, agg_configs.is_aggregated); + if (auto answered = dispatch_configuration(ctx_, req, entity_id, param_id, parsed)) { + return tl::unexpected(*answered); + } + if (parsed.has_prefix) { const auto * node_info = find_node_for_app(agg_configs.nodes, parsed.app_id); if (node_info == nullptr) { diff --git a/src/ros2_medkit_gateway/src/http/handlers/data_handlers.cpp b/src/ros2_medkit_gateway/src/http/handlers/data_handlers.cpp index 6c623c28b..6deabfb51 100644 --- a/src/ros2_medkit_gateway/src/http/handlers/data_handlers.cpp +++ b/src/ros2_medkit_gateway/src/http/handlers/data_handlers.cpp @@ -97,20 +97,27 @@ std::string to_full_topic_path(const std::string & topic_name) { return "/" + topic_name; } -/// What one addressed data item resolves to: the ROS topic to act on, and the -/// id to echo back to the caller. +/// What one addressed data item resolves to: the ROS topic to act on, the id to +/// echo back to the caller, the member the id named, and whether this gateway's +/// own walk records that member as a provider of the topic. struct AddressedDataItem { std::string full_topic_path; std::string item_id; + std::string member_id; ///< Empty when the id carried no member half. + bool provided_by_named_member{false}; ///< Meaningful only when member_id is set. }; /// Resolve the id in the route against the entity, for reads and writes alike. /// -/// A qualified id is answered exactly, because the member set and what each -/// member contributes are both known here: an id naming an unknown member, or -/// an item that member does not provide, is a miss. Without that check the -/// gateway samples the local graph, finds nothing, and returns 200 with an -/// empty body and status `metadata_only` - a typo reported as success. +/// A qualified id is answered exactly, because the member set is known here: an +/// id naming a member the entity does not have is a miss. Without that check the +/// gateway samples the local graph, finds nothing, and returns 200 with an empty +/// body and status `metadata_only` - a typo reported as success. +/// +/// Whether the named member provides the item is REPORTED, not decided: the +/// answer comes from this gateway's own walk, which holds no topics for a member +/// another gateway runs, so acting on it here would turn every peer-owned item +/// into a miss. The caller resolves ownership first and only then reads the flag. /// /// A ROS topic name cannot contain a colon, so one in the id can only be the /// member separator. Building the member set is not free, so the cache is only @@ -139,35 +146,65 @@ address_data_item(const ThreadSafeEntityCache & cache, const std::string & entit json{{"entity_id", entity_id}, {"id", topic_name}, {"member_id", parsed.member_id}})); } - // A member retained while its gateway is silent stays addressable and says - // why it cannot answer, rather than falling through to a sample of the local - // graph that comes back empty and reads as success. - if (auto app = cache.get_app(parsed.member_id); app && !app->available) { - return tl::make_unexpected( - make_error(504, ERR_NOT_RESPONDING, "Member '" + parsed.member_id + "' is not available", - json{{"details", - "The gateway contributing this member is not answering; it is retained from its " - "last known declaration"}, - {"entity_id", entity_id}, - {"id", topic_name}, - {"member_id", parsed.member_id}})); + // An id whose member half is followed by nothing names no item. Carried + // further it would address the member's data COLLECTION - the route that + // answers a request with one trailing slash fewer - and hand back a list to a + // caller that asked for a single value. + if (parsed.item_id.empty()) { + return tl::make_unexpected(make_error( + 404, ERR_RESOURCE_NOT_FOUND, "Data item not provided by member", + json{{"entity_id", entity_id}, {"id", topic_name}, {"member_id", parsed.member_id}, {"topic_name", ""}})); } + addressed.member_id = parsed.member_id; addressed.full_topic_path = to_full_topic_path(parsed.item_id); auto owners = aggregated.owners_by_topic.find(addressed.full_topic_path); - if (owners == aggregated.owners_by_topic.end() || - std::find(owners->second.begin(), owners->second.end(), parsed.member_id) == owners->second.end()) { - return tl::make_unexpected(make_error(404, ERR_RESOURCE_NOT_FOUND, "Data item not provided by member", - json{{"entity_id", entity_id}, - {"id", topic_name}, - {"member_id", parsed.member_id}, - {"topic_name", addressed.full_topic_path}})); - } + addressed.provided_by_named_member = + owners != aggregated.owners_by_topic.end() && + std::find(owners->second.begin(), owners->second.end(), parsed.member_id) != owners->second.end(); addressed.item_id = http::make_member_qualified_id(parsed.member_id, addressed.full_topic_path); return addressed; } +/// The member's own data route for the topic an id addressed. A qualified id +/// with an empty item half is refused before this point, so `full_topic_path` +/// always opens with the slash `to_full_topic_path` guarantees and the +/// concatenation carries exactly one separator. +std::string member_data_resource_path(const std::string & full_topic_path) { + return "data" + full_topic_path; +} + +/// Settle where an addressed data item is served, and hand the request over when +/// that is another gateway. Reads and writes share it because they address the +/// item identically and the owner of the topic is the same either way. +/// +/// Returns the answer the handler must return - including the sentinel that says +/// the owning peer has already committed the wire - or nullopt when this gateway +/// serves the item itself. The "member does not provide it" refusal is decided +/// here rather than while addressing, because it rests on the local walk, which +/// says nothing about a member another gateway runs. +std::optional dispatch_data_item(const HandlerContext & ctx, const http::TypedRequest & req, + const std::string & entity_id, const std::string & topic_name, + const AddressedDataItem & addressed) { + auto dispatch = ctx.dispatch_to_member(req, addressed.member_id, member_data_resource_path(addressed.full_topic_path), + json{{"entity_id", entity_id}, {"id", topic_name}}); + if (!dispatch) { + return dispatch.error(); + } + if (*dispatch == MemberDispatch::kForwarded) { + return HandlerContext::forwarded_sentinel_error(); + } + if (!addressed.member_id.empty() && !addressed.provided_by_named_member) { + return make_error(404, ERR_RESOURCE_NOT_FOUND, "Data item not provided by member", + json{{"entity_id", entity_id}, + {"id", topic_name}, + {"member_id", addressed.member_id}, + {"topic_name", addressed.full_topic_path}}); + } + return std::nullopt; +} + /// Build the typed x-medkit per-item payload for the list endpoint. dto::XMedkitDataItem build_list_item_xmedkit(const std::string & topic_name, const std::string & direction, const std::string & topic_type, @@ -489,6 +526,10 @@ http::Result DataHandlers::get_data_item(const http::TypedReques if (!addressed) { return tl::make_unexpected(addressed.error()); } + + if (auto answered = dispatch_data_item(ctx_, req, entity_id, topic_name, *addressed)) { + return tl::make_unexpected(*answered); + } const std::string & full_topic_path = addressed->full_topic_path; // Sampling goes through the pool-backed TopicDataProvider (issue #375 race @@ -660,11 +701,17 @@ http::Result DataHandlers::put_data_item(const http::TypedReques json{{"details", "Message type should be in format: package/msg/Type"}, {"type", msg_type}})); } - // A write addresses the same item a read does, so it resolves the same way. + // A write addresses the same item a read does, so it resolves the same way - + // including which gateway publishes it. Publishing here for a member another + // gateway runs would create a publisher on a graph that member is not on. auto addressed = address_data_item(ctx_.node()->get_thread_safe_cache(), entity_id, topic_name); if (!addressed) { return tl::make_unexpected(addressed.error()); } + + if (auto answered = dispatch_data_item(ctx_, req, entity_id, topic_name, *addressed)) { + return tl::make_unexpected(*answered); + } const std::string & full_topic_path = addressed->full_topic_path; // Publish data using DataAccessManager. diff --git a/src/ros2_medkit_gateway/src/http/handlers/handler_context.cpp b/src/ros2_medkit_gateway/src/http/handlers/handler_context.cpp index e037bba82..24413b9f2 100644 --- a/src/ros2_medkit_gateway/src/http/handlers/handler_context.cpp +++ b/src/ros2_medkit_gateway/src/http/handlers/handler_context.cpp @@ -339,6 +339,63 @@ http::ValidatorResult HandlerContext::validate_entity_for_route(cons return entity_info; } +http::Result HandlerContext::dispatch_to_member(const http::TypedRequest & req, + const std::string & member_id, + const std::string & member_resource_path, + json error_params) const { + if (member_id.empty()) { + return MemberDispatch::kServeLocally; + } + + // Reachability first, and unconditionally: a retained member is answered from + // what it declared, never by asking the gateway that stopped answering. + if (!is_entity_available(member_id)) { + ErrorInfo err; + err.code = ERR_NOT_RESPONDING; + err.message = "Member '" + member_id + "' is not available"; + err.http_status = 504; + error_params["details"] = + "The gateway contributing this member is not answering; it is retained from its last known declaration"; + error_params["member_id"] = member_id; + err.params = std::move(error_params); + return tl::unexpected(std::move(err)); + } + + if (aggregation_mgr_ == nullptr) { + return MemberDispatch::kServeLocally; + } + auto peer = aggregation_mgr_->find_peer_for_entity(member_id); + if (!peer) { + return MemberDispatch::kServeLocally; + } + + // Only an App or a Component can be addressed on the peer; those are the two + // collections a resource route exists under. A member that is neither names + // nothing this gateway can re-address, so the local path answers for it. + const auto & cache = node_->get_thread_safe_cache(); + std::string collection; + if (cache.get_app(member_id)) { + collection = "apps"; + } else if (cache.get_component(member_id)) { + collection = "components"; + } else { + return MemberDispatch::kServeLocally; + } + + // Same escape hatch, and same justification, as the remote-entity branch of + // validate_entity_for_route: proxying commits the wire, and the sink is the + // framework's, not the handler's. +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" + const auto & raw_req = req.raw_for_framework(); +#pragma GCC diagnostic pop + if (tl_forward_response != nullptr) { + aggregation_mgr_->forward_request(*peer, raw_req, *tl_forward_response, + api_path("/" + collection + "/" + member_id + "/" + member_resource_path)); + } + return MemberDispatch::kForwarded; +} + /// False when the entity is present only because its peer's declaration is /// being retained. An entity this gateway has never heard of is not "not /// available" - it is absent, and the caller already got a 404 for it. diff --git a/src/ros2_medkit_gateway/src/http/handlers/operation_handlers.cpp b/src/ros2_medkit_gateway/src/http/handlers/operation_handlers.cpp index 9acc067e5..71d7bed44 100644 --- a/src/ros2_medkit_gateway/src/http/handlers/operation_handlers.cpp +++ b/src/ros2_medkit_gateway/src/http/handlers/operation_handlers.cpp @@ -227,14 +227,13 @@ std::vector distinct_members(const std::vector & ma return members; } -/// The error for a member that is in the tree but whose gateway is silent. +/// True for a member that is in the tree but whose gateway is silent. /// /// A retained member is kept precisely so that the answer to a request does not /// change when a link drops: the item is still addressable, and asking for it /// says why it cannot be served right now. The alternatives are what this /// replaces - quietly running a different member's operation, or a 200 with -/// nothing in it. `not-responding` is the SOVD code for "no response from the -/// underlying entity", which is exactly the situation. +/// nothing in it. bool member_is_unreachable(const ThreadSafeEntityCache & cache, const std::string & member_id) { if (auto app = cache.get_app(member_id)) { return !app->available; @@ -245,20 +244,6 @@ bool member_is_unreachable(const ThreadSafeEntityCache & cache, const std::strin return false; } -std::optional member_unavailable_error(const ThreadSafeEntityCache & cache, const std::string & entity_id, - const std::string & member_id, const std::string & operation_id) { - if (!member_is_unreachable(cache, member_id)) { - return std::nullopt; - } - return make_error(504, ERR_NOT_RESPONDING, "Member '" + member_id + "' is not available", - json{{"details", - "The gateway contributing this member is not answering; it is retained from its " - "last known declaration"}, - {"entity_id", entity_id}, - {"operation_id", operation_id}, - {"member_id", member_id}}); -} - /// Convert a ROS 2 action goal status into the SOVD `ExecutionStatus` enum /// the gateway emits on the wire. Identical mapping to the legacy helper. std::string sovd_status_from_ros2(ActionGoalStatus status) { @@ -895,14 +880,22 @@ OperationHandlers::create_execution(const http::TypedRequest & req, dto::Executi make_error(400, ERR_INVALID_REQUEST, "Ambiguous operation id: it names more than one operation", params)); } - // Whoever ends up owning the resolved operation must actually be reachable. + // Whoever ends up owning the resolved operation must be reachable, and must + // run it themselves when they are another gateway. The service behind this id + // lives on the owner's ROS graph; calling it from here finds nothing and + // reports the member's own operation as unavailable. { const std::string & full_path = resolved.service.has_value() ? resolved.service->full_path : resolved.action->full_path; auto owner = ops.owner_by_path.find(full_path); if (owner != ops.owner_by_path.end()) { - if (auto err = member_unavailable_error(cache, entity_id, owner->second, operation_id)) { - return tl::make_unexpected(*err); + auto dispatch = ctx_.dispatch_to_member(req, owner->second, "operations/" + parsed.item_id + "/executions", + json{{"entity_id", entity_id}, {"operation_id", operation_id}}); + if (!dispatch) { + return tl::make_unexpected(dispatch.error()); + } + if (*dispatch == MemberDispatch::kForwarded) { + return tl::make_unexpected(HandlerContext::forwarded_sentinel_error()); } } } diff --git a/src/ros2_medkit_gateway/test/test_aggregation_manager.cpp b/src/ros2_medkit_gateway/test/test_aggregation_manager.cpp index e7eb0884d..388952f67 100644 --- a/src/ros2_medkit_gateway/test/test_aggregation_manager.cpp +++ b/src/ros2_medkit_gateway/test/test_aggregation_manager.cpp @@ -845,6 +845,139 @@ static void install_entity_endpoints(httplib::Server & svr, size_t num_areas, si }); } +// ============================================================================= +// Explicitly addressed forwarding (forward_request with a target path) +// ============================================================================= + +TEST(AggregationManager, forward_to_target_path_addresses_the_path_the_caller_chose) { + // An aggregating entity has no counterpart on the peer, so the request the + // client sent cannot be replayed there. Measured on the peer's own view of + // the request line, because a forward that quietly kept the incoming path + // would still answer 200 from whatever the peer happens to serve. + MockPeerServer mock; + std::string seen_path; + mock.server().Get(R"(/api/v1/(.+))", [&seen_path](const httplib::Request & req, httplib::Response & res) { + seen_path = req.path; + res.set_content(R"({"id":"served"})", "application/json"); + }); + int port = mock.start(); + + AggregationConfig config; + config.enabled = true; + config.timeout_ms = 5000; + AggregationConfig::PeerConfig peer; + peer.url = "http://127.0.0.1:" + std::to_string(port); + peer.name = "peer_0"; + config.peers.push_back(peer); + AggregationManager manager(config); + + httplib::Request req; + req.method = "GET"; + req.path = "/api/v1/functions/vehicle_health/data/pressure_sensor:chassis/brakes/pressure"; + httplib::Response res; + + manager.forward_request("peer_0", req, res, "/api/v1/apps/pressure_sensor/data/chassis/brakes/pressure"); + + EXPECT_EQ(res.status, 200); + EXPECT_EQ(seen_path, "/api/v1/apps/pressure_sensor/data/chassis/brakes/pressure"); +} + +TEST(AggregationManager, forward_to_target_path_strips_the_peer_prefix_from_the_target) { + // A member renamed on collision is known to its own gateway by the original + // id, and the target path is built from the merged id, so the rewrite has to + // reach the path the caller supplied rather than the one the client sent. + MockPeerServer mock; + std::string seen_path; + mock.server().Get(R"(/api/v1/(.+))", [&seen_path](const httplib::Request & req, httplib::Response & res) { + seen_path = req.path; + res.set_content(R"({"id":"served"})", "application/json"); + }); + int port = mock.start(); + + AggregationConfig config; + config.enabled = true; + config.timeout_ms = 5000; + AggregationConfig::PeerConfig peer; + peer.url = "http://127.0.0.1:" + std::to_string(port); + peer.name = "peer_0"; + config.peers.push_back(peer); + AggregationManager manager(config); + + httplib::Request req; + req.method = "GET"; + req.path = "/api/v1/functions/vehicle_health/data/peer_0__shared_sensor:chassis/brakes/pressure"; + httplib::Response res; + + manager.forward_request("peer_0", req, res, "/api/v1/apps/peer_0__shared_sensor/data/chassis/brakes/pressure"); + + EXPECT_EQ(res.status, 200); + EXPECT_EQ(seen_path, "/api/v1/apps/shared_sensor/data/chassis/brakes/pressure"); +} + +TEST(AggregationManager, forward_to_target_path_refuses_a_target_outside_the_api) { + // The target is assembled from client-supplied ids, so it is exactly as + // untrusted as the incoming path and gets the same SSRF guard. A guard that + // checked the incoming path instead would pass this, because the incoming + // path is a perfectly ordinary API path. + MockPeerServer mock; + std::atomic reached{false}; + mock.server().Get(R"(/(.*))", [&reached](const httplib::Request &, httplib::Response & res) { + reached = true; + res.set_content("{}", "application/json"); + }); + int port = mock.start(); + + AggregationConfig config; + config.enabled = true; + config.timeout_ms = 5000; + AggregationConfig::PeerConfig peer; + peer.url = "http://127.0.0.1:" + std::to_string(port); + peer.name = "peer_0"; + config.peers.push_back(peer); + AggregationManager manager(config); + + httplib::Request req; + req.method = "GET"; + req.path = "/api/v1/functions/vehicle_health/data/admin:x"; + httplib::Response res; + + manager.forward_request("peer_0", req, res, "/admin/shutdown"); + + EXPECT_EQ(res.status, 400); + EXPECT_FALSE(reached.load()); +} + +TEST(AggregationManager, forward_without_a_target_path_still_addresses_the_incoming_one) { + // The two-argument form is the whole-entity forward and must keep replaying + // the client's own path. + MockPeerServer mock; + std::string seen_path; + mock.server().Get(R"(/api/v1/(.+))", [&seen_path](const httplib::Request & req, httplib::Response & res) { + seen_path = req.path; + res.set_content(R"({"id":"served"})", "application/json"); + }); + int port = mock.start(); + + AggregationConfig config; + config.enabled = true; + config.timeout_ms = 5000; + AggregationConfig::PeerConfig peer; + peer.url = "http://127.0.0.1:" + std::to_string(port); + peer.name = "peer_0"; + config.peers.push_back(peer); + AggregationManager manager(config); + + httplib::Request req; + req.method = "GET"; + req.path = "/api/v1/apps/camera_driver/data"; + httplib::Response res; + + manager.forward_request("peer_0", req, res); + + EXPECT_EQ(res.status, 200); + EXPECT_EQ(seen_path, "/api/v1/apps/camera_driver/data"); +} + // ============================================================================= // max_entities_per_peer safety limit test // ============================================================================= diff --git a/src/ros2_medkit_integration_tests/CMakeLists.txt b/src/ros2_medkit_integration_tests/CMakeLists.txt index b02e8c999..9c5aecc35 100644 --- a/src/ros2_medkit_integration_tests/CMakeLists.txt +++ b/src/ros2_medkit_integration_tests/CMakeLists.txt @@ -73,7 +73,7 @@ medkit_target_dependencies(demo_light_controller rclcpp std_msgs) add_executable(demo_calibration_service demo_nodes/calibration_service.cpp) target_include_directories(demo_calibration_service PRIVATE ${_demo_include_dir}) -medkit_target_dependencies(demo_calibration_service rclcpp std_srvs) +medkit_target_dependencies(demo_calibration_service rclcpp rcl_interfaces std_srvs) add_executable(demo_long_calibration_action demo_nodes/long_calibration_action.cpp) target_include_directories(demo_long_calibration_action PRIVATE ${_demo_include_dir}) diff --git a/src/ros2_medkit_integration_tests/demo_nodes/calibration_service.cpp b/src/ros2_medkit_integration_tests/demo_nodes/calibration_service.cpp index 07f75e17e..baecafae4 100644 --- a/src/ros2_medkit_integration_tests/demo_nodes/calibration_service.cpp +++ b/src/ros2_medkit_integration_tests/demo_nodes/calibration_service.cpp @@ -20,8 +20,12 @@ * - Exposes /powertrain/engine/calibrate service (Trigger type) * - Returns success with calibration message * - Used to test POST /services/{service} endpoint + * - Declares a writable `calibration_offset` parameter, so the node is + * addressable through the configurations collection as well as the + * operations one */ +#include #include #include @@ -35,6 +39,13 @@ class CalibrationService : public rclcpp::Node { "calibrate", std::bind(&CalibrationService::calibrate_callback, this, std::placeholders::_1, std::placeholders::_2)); + // Writable, and the same name on every instance of this node: two gateways + // each running one is what makes a member-qualified configuration id + // resolvable to exactly one of them. + rcl_interfaces::msg::ParameterDescriptor offset_desc; + offset_desc.description = "Calibration zero-point offset applied to the next calibration run"; + this->declare_parameter("calibration_offset", 0.0, offset_desc); + RCLCPP_INFO(this->get_logger(), "Calibration service started"); } diff --git a/src/ros2_medkit_integration_tests/test/features/test_grouping_entity_aggregation.test.py b/src/ros2_medkit_integration_tests/test/features/test_grouping_entity_aggregation.test.py index 7a870888c..96c9323ce 100644 --- a/src/ros2_medkit_integration_tests/test/features/test_grouping_entity_aggregation.test.py +++ b/src/ros2_medkit_integration_tests/test/features/test_grouping_entity_aggregation.test.py @@ -133,6 +133,12 @@ # survives the merge - R1, the precondition for every addressing rule. COLLIDING_LEAF = 'shared_sensor' +# Declared by the calibration demo node, so BOTH `primary_calibration` and +# `peer_calibration` expose it under one name. A member-qualified id is the only +# thing that separates the two copies, which is what makes this the parameter +# the configuration cases below address. +CALIBRATION_PARAM = 'calibration_offset' + MERGED_AREA = 'vehicle' MERGED_FUNCTION = 'vehicle_health' PARENT_COMPONENT = 'vehicle-ecu' @@ -383,6 +389,38 @@ def _items(self, entity_path, collection): self.assertEqual(response.status_code, 200, response.text) return response.json().get('items', []) + @staticmethod + def _set_offset(base_url, app_id, value): + """Set `calibration_offset` on one App through ITS OWN gateway. + + The member's own route on the gateway that runs it, so the value the + aggregate is then asked for was put there by a request that never went + through the aggregate. + """ + response = requests.put( + f'{base_url}/apps/{app_id}/configurations/{CALIBRATION_PARAM}', + json={'data': value}, + timeout=15, + ) + if response.status_code != 200: + raise AssertionError( + f'could not seed {app_id}.{CALIBRATION_PARAM} on {base_url}: ' + f'{response.status_code} {response.text}' + ) + return response + + @staticmethod + def _offset_of(base_url, app_id): + """Read `calibration_offset` from one App through ITS OWN gateway.""" + response = requests.get( + f'{base_url}/apps/{app_id}/configurations/{CALIBRATION_PARAM}', timeout=15) + if response.status_code != 200: + raise AssertionError( + f'could not read {app_id}.{CALIBRATION_PARAM} on {base_url}: ' + f'{response.status_code} {response.text}' + ) + return response.json().get('data') + # ---------------------------------------------------------------------- R1 def test_a_leaf_contributed_by_both_gateways_stays_two_addressable_leaves(self): @@ -552,13 +590,386 @@ def test_a_compound_id_reaches_a_peer_owned_member(self): ) self.assertTrue(body.get('data'), 'peer member returned an empty payload') + # And it is THAT member's item, not merely some 200. The comparison is + # against the peer's own answer for the same member and topic, so a read + # that silently fell back to a local member - the other half of this + # Function publishes a different message type on a different topic - + # cannot satisfy it, and neither can a hand-built empty envelope. + direct = requests.get( + f'{PEER_URL}/apps/pressure_sensor/data/chassis/brakes/pressure', timeout=15) + self.assertEqual(direct.status_code, 200, direct.text) + direct_body = direct.json() + self.assertEqual( + body.get('x-medkit', {}).get('ros2', {}).get('topic'), + '/chassis/brakes/pressure', + f'the answer names a topic the member does not publish: {body}', + ) + self.assertEqual( + body.get('x-medkit', {}).get('ros2', {}).get('type'), + direct_body.get('x-medkit', {}).get('ros2', {}).get('type'), + f'the answer is not the message the member publishes: {body}', + ) + self.assertEqual( + sorted(body['data'].keys()), sorted(direct_body['data'].keys()), + f"the payload is not shaped like the member's own: {body}", + ) + # The answer says which entity produced it, and for a peer-owned member + # that is the member itself - the request was served on the member's own + # route, over there. Served here it would name the aggregating entity, + # which is the shape a local sample of a topic this gateway cannot see + # would also carry. + self.assertEqual( + body.get('x-medkit', {}).get('entity_id'), 'pressure_sensor', + f'the aggregating entity answered for a member it does not run: {body}', + ) + def test_a_compound_id_reaches_a_local_member(self): - """R4 in the other direction: resolving members must not lose the local half.""" + """R4 in the other direction: resolving members must not lose the local half. + + The payload is asserted, not the status, for the same reason as the peer + case - and against the member's own App route on THIS gateway, so a + dispatch that forwarded a local member to a peer that has never heard of + it would fail here rather than pass as a 404 nobody looked at. + """ item_id = f'temp_sensor:{"/powertrain/engine/temperature".lstrip("/")}' url = f'{PRIMARY_URL}/functions/{MERGED_FUNCTION}/data/{quote(item_id, safe="")}' response = requests.get(url, timeout=15) self.assertEqual(response.status_code, 200, response.text) - self.assertEqual(response.json().get('x-medkit', {}).get('status'), 'data') + body = response.json() + self.assertEqual(body.get('x-medkit', {}).get('status'), 'data') + self.assertTrue(body.get('data'), 'local member returned an empty payload') + + direct = requests.get( + f'{PRIMARY_URL}/apps/temp_sensor/data/powertrain/engine/temperature', timeout=15) + self.assertEqual(direct.status_code, 200, direct.text) + direct_body = direct.json() + self.assertEqual( + body.get('x-medkit', {}).get('ros2', {}).get('topic'), + '/powertrain/engine/temperature', + f'the answer names a topic the local member does not publish: {body}', + ) + self.assertEqual( + body.get('x-medkit', {}).get('ros2', {}).get('type'), + direct_body.get('x-medkit', {}).get('ros2', {}).get('type'), + f'the answer is not the message the local member publishes: {body}', + ) + self.assertEqual( + sorted(body['data'].keys()), sorted(direct_body['data'].keys()), + f"the payload is not shaped like the local member's own: {body}", + ) + # And this gateway answered: the entity named is the one the request + # addressed. A dispatch that forwarded a locally owned member would come + # back naming the member instead - or, more likely, not come back at all. + self.assertEqual( + body.get('x-medkit', {}).get('entity_id'), MERGED_FUNCTION, + f'a locally owned member was not served here: {body}', + ) + + def test_a_compound_operation_id_runs_on_the_members_gateway(self): + """R4 for an operation, asserted on the result rather than the status. + + The peer's calibration service is on another ROS domain, so this gateway + cannot call it: served locally the request answers 500 service-unavailable. + A 200 alone would still not say the service RAN, so the response payload + is read - a service that answered is the only thing that can fill it. + """ + response = requests.post( + f'{PRIMARY_URL}/functions/{MERGED_FUNCTION}/operations/' + f'{quote("peer_calibration:calibrate", safe="")}/executions', + json={}, + timeout=15, + ) + self.assertEqual(response.status_code, 200, response.text) + parameters = response.json().get('parameters') + self.assertIsInstance( + parameters, dict, f'no service response came back: {response.text}') + self.assertIs( + parameters.get('success'), True, + f"the member's service did not report success: {parameters}", + ) + self.assertTrue( + parameters.get('message'), + f"the member's service answered with nothing to say: {parameters}", + ) + + def test_a_write_through_an_aggregate_reaches_the_peer_owned_member(self): + """R4 for a write: the topic is published where the member actually is. + + A write resolves its target exactly as a read does, so it lands on the + same gateway. Served here it would publish onto a ROS graph the member is + not on - a publisher nobody is listening to - and answer 200 with the + same echo, which is why the status cannot decide this case and the + serving entity is read instead. The 404 is asserted separately because it + is the other way to get this wrong: consulting the local walk for a + member another gateway runs finds no such topic and refuses a write that + is perfectly valid. + """ + direct = requests.get( + f'{PEER_URL}/apps/pressure_sensor/data/chassis/brakes/pressure', timeout=15) + self.assertEqual(direct.status_code, 200, direct.text) + direct_body = direct.json() + message_type = direct_body.get('x-medkit', {}).get('ros2', {}).get('type') + self.assertTrue(message_type, f'the member does not name its message type: {direct.text}') + + response = requests.put( + f'{PRIMARY_URL}/functions/{MERGED_FUNCTION}/data/' + f'{quote("pressure_sensor:chassis/brakes/pressure", safe="")}', + json={'type': message_type, 'data': direct_body['data']}, + timeout=15, + ) + self.assertNotEqual( + response.status_code, 404, + f'a write to a peer-owned member was refused as an unknown item: {response.text}', + ) + self.assertEqual(response.status_code, 200, response.text) + body = response.json() + self.assertEqual( + body.get('x-medkit', {}).get('ros2', {}).get('topic'), + '/chassis/brakes/pressure', + f'the write echo names a topic the member does not publish: {response.text}', + ) + # Served here the publish would succeed too - a publisher on this + # gateway's graph, which the member is not on, and a 200 that looks + # identical. What tells them apart is which entity answered. + self.assertEqual( + body.get('x-medkit', {}).get('entity_id'), 'pressure_sensor', + f'the write was published by the aggregating gateway, not the member: {body}', + ) + + def test_a_configuration_id_the_list_offers_reads_back_from_its_member(self): + """R5 for configurations, in both directions at once. + + The list of an aggregating entity is assembled from two places - this + gateway's own nodes and the peer fan-out - so it offers ids for members + this gateway cannot reach. Reading one back is what says the id is an + address rather than a label. The value is asserted per member, and the + two members are seeded to different values first, so an id answered by + the wrong copy fails here instead of passing as a plausible 200. + """ + expected = {'primary_calibration': 11.5, 'peer_calibration': 22.5} + self._set_offset(PRIMARY_URL, 'primary_calibration', expected['primary_calibration']) + self._set_offset(PEER_URL, 'peer_calibration', expected['peer_calibration']) + + items = self._items(f'functions/{MERGED_FUNCTION}', 'configurations') + offered = sorted( + item.get('id') for item in items + if str(item.get('id', '')).endswith(f':{CALIBRATION_PARAM}') + ) + self.assertEqual( + offered, + [f'peer_calibration:{CALIBRATION_PARAM}', f'primary_calibration:{CALIBRATION_PARAM}'], + f"the aggregate does not offer both members' copies: " + f"{[item.get('id') for item in items]}", + ) + + for config_id in offered: + with self.subTest(configuration=config_id): + member = config_id.split(':', 1)[0] + response = requests.get( + f'{PRIMARY_URL}/functions/{MERGED_FUNCTION}/configurations/' + f'{quote(config_id, safe="")}', + timeout=15, + ) + self.assertEqual( + response.status_code, 200, + f'the list offers {config_id!r} but reading it answered ' + f'{response.status_code}: {response.text}', + ) + self.assertAlmostEqual( + response.json().get('data'), expected[member], places=6, + msg=f"{config_id} did not answer with its own member's value: " + f'{response.text}', + ) + + def test_a_configuration_of_a_local_member_is_still_served_here(self): + """R5 in the direction that a dispatch fix is most likely to break. + + A member this gateway runs must be answered from this gateway's own + parameter service, with no hop. Both halves are pinned: the entity that + answered is the one the request addressed, and the node behind the value + is the local member's, not its peer namesake's - the two run the same + executable under the same parameter name and differ only in namespace, + so nothing shallower can tell them apart. + """ + local_value = 2.5 + self._set_offset(PRIMARY_URL, 'primary_calibration', local_value) + self._set_offset(PEER_URL, 'peer_calibration', -3.75) + + config_id = f'primary_calibration:{CALIBRATION_PARAM}' + response = requests.get( + f'{PRIMARY_URL}/functions/{MERGED_FUNCTION}/configurations/' + f'{quote(config_id, safe="")}', + timeout=15, + ) + self.assertEqual(response.status_code, 200, response.text) + body = response.json() + self.assertAlmostEqual( + body.get('data'), local_value, places=6, + msg=f"the local member's value did not come back: {body}", + ) + self.assertEqual(body.get('id'), config_id, body) + x_medkit = body.get('x-medkit', {}) + self.assertEqual( + x_medkit.get('entity_id'), MERGED_FUNCTION, + f'a locally owned member was not served here: {body}', + ) + self.assertEqual( + x_medkit.get('source_app'), 'primary_calibration', + f'the answer does not name the member it came from: {body}', + ) + self.assertEqual( + x_medkit.get('ros2', {}).get('node'), '/powertrain/engine/calibration', + f'the value was read from a node the local member is not: {body}', + ) + + def test_a_configuration_of_a_peer_owned_member_is_read_from_that_member(self): + """R5 for configurations, on the half the local walk cannot serve. + + A parameter lives on a node, and the node behind a peer-owned member is + on a ROS graph this gateway is not on. Served here the lookup finds + nothing and the request fails; served on the member's own gateway it + returns that member's value. Both members expose the same parameter + name, seeded to different values, and the answer is checked against the + peer's - a fall-back to the local namesake would otherwise be a 200 + with a number in it. + """ + peer_value = 4.25 + local_value = -1.5 + self._set_offset(PEER_URL, 'peer_calibration', peer_value) + self._set_offset(PRIMARY_URL, 'primary_calibration', local_value) + + config_id = f'peer_calibration:{CALIBRATION_PARAM}' + response = requests.get( + f'{PRIMARY_URL}/functions/{MERGED_FUNCTION}/configurations/' + f'{quote(config_id, safe="")}', + timeout=15, + ) + self.assertEqual(response.status_code, 200, response.text) + body = response.json() + self.assertAlmostEqual( + body.get('data'), peer_value, places=6, + msg=f"the peer member's value did not come back: {body}", + ) + self.assertNotAlmostEqual( + body.get('data'), local_value, places=6, + msg=f'the local namesake answered for a peer-owned member: {body}', + ) + x_medkit = body.get('x-medkit', {}) + # The answer says which entity produced it, and for a peer-owned member + # that is the member itself - the request was served on the member's own + # route, over there. Served here it would name the aggregating entity. + self.assertEqual( + x_medkit.get('entity_id'), 'peer_calibration', + f'the aggregating entity answered for a member it does not run: {body}', + ) + self.assertEqual( + x_medkit.get('ros2', {}).get('node'), '/chassis/brakes/calibration', + f'the value was read from a node the peer member is not: {body}', + ) + + def test_a_configuration_reset_through_an_aggregate_reaches_the_member(self): + """R5 for a reset, the method whose answer carries no payload at all. + + DELETE returns 204 and nothing else, so there is no field in the response + that could say where it landed. Both members are moved off their default + first and the pair is read back from their own gateways afterwards: the + member the id named is back at its default and the local namesake is not. + A reset served here would satisfy neither, and a 204 alone both. + """ + self._set_offset(PEER_URL, 'peer_calibration', 5.5) + self._set_offset(PRIMARY_URL, 'primary_calibration', 6.5) + + config_id = f'peer_calibration:{CALIBRATION_PARAM}' + response = requests.delete( + f'{PRIMARY_URL}/functions/{MERGED_FUNCTION}/configurations/' + f'{quote(config_id, safe="")}', + timeout=15, + ) + self.assertNotEqual( + response.status_code, 404, + f'a reset of a peer-owned member was refused as an unknown member: {response.text}', + ) + self.assertEqual(response.status_code, 204, response.text) + + self.assertAlmostEqual( + self._offset_of(PEER_URL, 'peer_calibration'), 0.0, places=6, + msg='the reset never reached the member it named', + ) + self.assertAlmostEqual( + self._offset_of(PRIMARY_URL, 'primary_calibration'), 6.5, places=6, + msg='the reset landed on the local namesake instead of the member it named', + ) + + def test_a_configuration_write_through_an_aggregate_lands_on_the_member(self): + """R5 for a write: the parameter changes where the member actually is. + + A write echoes what it was handed, so its response cannot show where it + landed. The proof is read back from the PEER's own gateway afterwards, + and the local namesake is read too: a write served here would change + that one, or nothing at all, and either way the echo would look the same. + """ + written = 9.75 + self._set_offset(PEER_URL, 'peer_calibration', 0.0) + self._set_offset(PRIMARY_URL, 'primary_calibration', 0.0) + + config_id = f'peer_calibration:{CALIBRATION_PARAM}' + response = requests.put( + f'{PRIMARY_URL}/functions/{MERGED_FUNCTION}/configurations/' + f'{quote(config_id, safe="")}', + json={'data': written}, + timeout=15, + ) + self.assertNotEqual( + response.status_code, 404, + f'a write to a peer-owned member was refused as an unknown member: {response.text}', + ) + self.assertEqual(response.status_code, 200, response.text) + self.assertEqual( + response.json().get('x-medkit', {}).get('entity_id'), 'peer_calibration', + f'the write was applied by the aggregating gateway, not the member: {response.text}', + ) + + self.assertAlmostEqual( + self._offset_of(PEER_URL, 'peer_calibration'), written, places=6, + msg='the write never reached the member it named', + ) + self.assertAlmostEqual( + self._offset_of(PRIMARY_URL, 'primary_calibration'), 0.0, places=6, + msg='the write landed on the local namesake instead of the member it named', + ) + + def test_an_unknown_member_in_a_compound_id_is_refused(self): + """R5 for the member half: absent is said, not sampled for. + + A qualifier naming nothing must be refused before any gateway is asked, + and the refusal must name the half that was wrong - otherwise a client + cannot tell a mistyped member from a member whose item is missing. The + second half of the case pins the distinction the refusal rests on: an id + that DOES resolve answers 200 and states whether data arrived, so + "present but empty" and "absent" are never the same response. + """ + response = requests.get( + f'{PRIMARY_URL}/functions/{MERGED_FUNCTION}/data/' + f'{quote("no_such_member:chassis/brakes/pressure", safe="")}', + timeout=10, + ) + self.assertEqual(response.status_code, 404, response.text) + body = response.json() + self.assertEqual( + body.get('parameters', {}).get('member_id'), 'no_such_member', + f'the refusal does not name the member half that was wrong: {body}', + ) + + present = requests.get( + f'{PRIMARY_URL}/functions/{MERGED_FUNCTION}/data/' + f'{quote("pressure_sensor:chassis/brakes/pressure", safe="")}', + timeout=15, + ) + self.assertEqual(present.status_code, 200, present.text) + self.assertIn( + present.json().get('x-medkit', {}).get('status'), ('data', 'metadata_only'), + f'a resolvable id does not say whether data arrived: {present.text}', + ) # ---------------------------------------------------------------------- R5 @@ -568,6 +979,11 @@ def test_an_item_no_member_provides_is_refused(self): Today an unknown topic answers 200 metadata-only on every entity, so a client cannot tell a typo from a silent sensor. On an aggregating entity the member set is known, so the answer can be exact. + + The reason is asserted, not just the status: a read that fell through to + the ROS graph and failed to find the topic there also answers 404, so a + bare status check cannot tell "the member does not provide this" from + "nobody happened to be publishing". """ response = requests.get( f'{PRIMARY_URL}/functions/{MERGED_FUNCTION}/data/' @@ -575,6 +991,31 @@ def test_an_item_no_member_provides_is_refused(self): timeout=10, ) self.assertEqual(response.status_code, 404, response.text) + body = response.json() + self.assertEqual(body.get('error_code'), 'resource-not-found', body) + self.assertEqual(body.get('parameters', {}).get('member_id'), 'temp_sensor', body) + self.assertEqual( + body.get('parameters', {}).get('topic_name'), '/no/such/topic', body) + + def test_an_id_that_names_a_member_and_no_item_is_refused(self): + """R5 at the boundary: naming a member is not naming an item. + + The member's own collection route is one trailing slash away from its + item route, so an id that stops at the colon addresses the collection if + it is carried any further - and the caller, having asked for one value, + is handed a list and no sign that anything went wrong. Checked on a + PEER-owned member because the local path never reaches that far. + """ + response = requests.get( + f'{PRIMARY_URL}/functions/{MERGED_FUNCTION}/data/' + f'{quote("pressure_sensor:", safe="")}', + timeout=15, + ) + self.assertEqual(response.status_code, 404, response.text) + self.assertNotIn( + 'items', response.json(), + f'a request for one item was answered with a collection: {response.text}', + ) # ---------------------------------------------------------------------- R7 @@ -598,6 +1039,48 @@ def test_loop_suppression_is_carried_on_every_hop(self): 'suppression header ignored, so a peered pair would not terminate', ) + def test_loop_suppression_does_not_stop_a_request_reaching_its_member(self): + """R7 and R4 together: suppression bounds fan-out, it does not blind a read. + + A request addressed to ONE member is not a fan-out - it names its owner, + and that owner serves it from its own tree without asking anyone else, so + the chain is one hop whatever the header says. Turning the header into a + refusal to dispatch would answer the same request two different ways + depending on a hint about collection listing, and a client that sets it + to keep listings local would silently start reading empty bodies. + + Termination is measured on the clock rather than inferred: a bounce + between peered gateways shows up as a request that never comes back, and + an assertion that only reads the status would report that as an error + from the transport instead of as the loop it is. + """ + started = time.monotonic() + read = requests.get( + f'{PRIMARY_URL}/functions/{MERGED_FUNCTION}/data/' + f'{quote("pressure_sensor:chassis/brakes/pressure", safe="")}', + headers={'X-Medkit-No-Fan-Out': '1'}, + timeout=15, + ) + run = requests.post( + f'{PRIMARY_URL}/functions/{MERGED_FUNCTION}/operations/' + f'{quote("peer_calibration:calibrate", safe="")}/executions', + headers={'X-Medkit-No-Fan-Out': '1'}, + json={}, + timeout=15, + ) + elapsed = time.monotonic() - started + + self.assertEqual(read.status_code, 200, read.text) + self.assertEqual( + read.json().get('x-medkit', {}).get('status'), 'data', + f'suppression turned a member read into an empty success: {read.text}', + ) + self.assertIn(run.status_code, (200, 202), run.text) + self.assertLess( + elapsed, 10.0, + f'two suppressed member requests took {elapsed:.1f}s, which is a bounce, not a hop', + ) + # ------------------------------------------------------------------ R10 # A PEER THAT STOPS ANSWERING. # @@ -940,6 +1423,58 @@ def test_z6_every_id_the_list_offers_is_executable(self): f'{response.status_code}: {response.text}', ) + def test_z6a_a_read_of_a_peer_owned_member_says_not_responding(self): + """R10 for the dispatch path: a dead link is reported, never proxied. + + A forward to a peer that has stopped answering fails at the socket and + comes back 502, which says this gateway broke. The member is retained + precisely so the true answer is available without asking: it is declared, + it is unreachable, and 504 not-responding is the SOVD code for that. The + 502 is asserted against explicitly because it is the regression this + ordering exists to prevent, and a bare `assertEqual(504)` reads the same + whichever wrong status arrives. + """ + response = requests.get( + f'{PRIMARY_URL}/functions/{MERGED_FUNCTION}/data/' + f'{quote("pressure_sensor:chassis/brakes/pressure", safe="")}', + timeout=15, + ) + self.assertNotEqual( + response.status_code, 502, + f'a silent peer was forwarded to instead of answered for: {response.text}', + ) + self.assertEqual(response.status_code, 504, response.text) + body = response.json() + self.assertEqual(body.get('error_code'), 'not-responding', body) + self.assertIn('pressure_sensor', body.get('message', ''), body) + self.assertEqual( + body.get('parameters', {}).get('member_id'), 'pressure_sensor', body) + + def test_z6b_a_configuration_read_of_a_silent_peer_owned_member_says_not_responding(self): + """R10 for the configuration dispatch path, for the same reason as z6a. + + Reachability is settled before anything is forwarded, so a member whose + gateway has stopped answering is reported as unreachable rather than as + a proxy hop that failed at the socket. The 502 is asserted against + explicitly: it is the shape this ordering exists to prevent, and a bare + assertEqual(504) reads the same whichever wrong status arrives. + """ + response = requests.get( + f'{PRIMARY_URL}/functions/{MERGED_FUNCTION}/configurations/' + f'{quote(f"peer_calibration:{CALIBRATION_PARAM}", safe="")}', + timeout=15, + ) + self.assertNotEqual( + response.status_code, 502, + f'a silent peer was forwarded to instead of answered for: {response.text}', + ) + self.assertEqual(response.status_code, 504, response.text) + body = response.json() + self.assertEqual(body.get('error_code'), 'not-responding', body) + self.assertIn('peer_calibration', body.get('message', ''), body) + self.assertEqual( + body.get('parameters', {}).get('member_id'), 'peer_calibration', body) + def test_z7_suppression_omits_the_peer_without_losing_ambiguity(self): """The loop-suppression guard, checked for the reason it was written. From 572a681dc9657e6b90d41093893e8c8da21b7d3b Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Thu, 20 Aug 2026 11:43:05 +0200 Subject: [PATCH 06/22] fix(aggregation): carry the caller's identity to the gateway that judges the lock A lock is held against a client id and every later request is judged against it. The id was not forwarded, so a request that crossed a gateway boundary arrived anonymous and the peer refused to record a lock naming no client. Aggregation therefore had no working locking at all on the entities only a peer owns. The id now travels with a forwarded request. It is not a credential and is not governed by forward_auth: it names the caller rather than granting it anything. The rule this makes true was written down and never verified, so it has a specification now, with a second gateway holding the lock. --- docs/config/aggregation.rst | 10 + .../src/core/aggregation/peer_client.cpp | 9 + .../CMakeLists.txt | 1 + .../test_aggregate_lock_identity.test.py | 262 ++++++++++++++++++ .../test_grouping_entity_aggregation.test.py | 2 + 5 files changed, 284 insertions(+) create mode 100644 src/ros2_medkit_integration_tests/test/features/test_aggregate_lock_identity.test.py diff --git a/docs/config/aggregation.rst b/docs/config/aggregation.rst index 6897270ff..75d02a14d 100644 --- a/docs/config/aggregation.rst +++ b/docs/config/aggregation.rst @@ -312,6 +312,16 @@ Combine static peers for known infrastructure with mDNS for dynamic discovery: configuration. See :ref:`Security Parameters ` for details on securing peer communication. +.. note:: + + ``X-Client-Id`` is always forwarded, and is not governed by + ``forward_auth``. It names the caller rather than granting it anything: + a lock on a peer-owned entity is held on the peer and judged there, so a + forwarded request that arrived without the name would be a different + caller than the one holding the lock. Authority still travels only in + ``Authorization``, which is forwarded when the deployment says the peer + is trusted with it. + Secure Aggregation (TLS + Auth) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/src/ros2_medkit_gateway/src/core/aggregation/peer_client.cpp b/src/ros2_medkit_gateway/src/core/aggregation/peer_client.cpp index 9243bd39e..1e27c8e12 100644 --- a/src/ros2_medkit_gateway/src/core/aggregation/peer_client.cpp +++ b/src/ros2_medkit_gateway/src/core/aggregation/peer_client.cpp @@ -790,6 +790,15 @@ void PeerClient::forward_request(const httplib::Request & req, httplib::Response if (req.has_header("X-Medkit-No-Fan-Out")) { headers.emplace("X-Medkit-No-Fan-Out", "1"); } + // The client's identity, which is what a lock is held against. The peer owns + // the entity, so the peer holds the lock and judges every request against the + // name it recorded; a request that arrives anonymous is a different caller + // than the one that took the lock, whoever sent it. It is not a credential - + // authority travels in Authorization, which is forwarded only when the + // deployment says the peer is trusted with it. + if (req.has_header("X-Client-Id")) { + headers.emplace("X-Client-Id", req.get_header_value("X-Client-Id")); + } httplib::Result result{nullptr, httplib::Error::Unknown}; const std::string path = path_with_query(req); diff --git a/src/ros2_medkit_integration_tests/CMakeLists.txt b/src/ros2_medkit_integration_tests/CMakeLists.txt index 9c5aecc35..aca34ddea 100644 --- a/src/ros2_medkit_integration_tests/CMakeLists.txt +++ b/src/ros2_medkit_integration_tests/CMakeLists.txt @@ -233,6 +233,7 @@ if(BUILD_TESTING) test_cross_ecu_fanout test_daisy_chain_aggregation test_grouping_entity_aggregation + test_aggregate_lock_identity test_leaf_collision_aggregation test_startup_param_clamp_warnings) set(_MULTI_GATEWAY_DOMAINS 4) diff --git a/src/ros2_medkit_integration_tests/test/features/test_aggregate_lock_identity.test.py b/src/ros2_medkit_integration_tests/test/features/test_aggregate_lock_identity.test.py new file mode 100644 index 000000000..3a3a4217f --- /dev/null +++ b/src/ros2_medkit_integration_tests/test/features/test_aggregate_lock_identity.test.py @@ -0,0 +1,262 @@ +#!/usr/bin/env python3 +# +# Copyright 2026 bburda +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Whether a lock survives the hop to the gateway that owns the entity. + +A lock names the client holding it, and every later request is judged against +that name: the owner passes, anyone else is refused. The name travels in +`X-Client-Id`. When the entity lives on a peer, the request crosses a gateway +boundary before it is judged, so the question here is whether the holder is +still recognisable on the other side. + +Both failure directions are wrong and only one of them is loud. If the name is +lost, every request arrives anonymous - including the holder's - and anonymous +matches anonymous, so the lock stops refusing anybody while continuing to look +like it exists. That is why L2 asserts a refusal rather than an acceptance: a +lock that grants everything passes any test written the other way round. + +L1 A lock taken on a peer-owned entity through the aggregating gateway is + recorded on the peer as belonging to the client who took it. Read back + from the peer directly - the aggregating gateway is what is under test, so + its own view cannot be the witness. +L2 While that lock is held, a different client's write to the same entity is + refused. +""" + +import tempfile +import time +import unittest + +from launch import LaunchDescription +from launch.actions import SetEnvironmentVariable, TimerAction +import launch_testing.actions +import requests + +from ros2_medkit_test_utils.constants import ( + ALLOWED_EXIT_CODES, + API_BASE_PATH, + get_test_domain_id, + get_test_port, +) +from ros2_medkit_test_utils.launch_helpers import ( + create_demo_nodes, + create_gateway_node, +) + +PRIMARY_PORT = get_test_port(0) +PEER_PORT = get_test_port(1) +PRIMARY_URL = f'http://localhost:{PRIMARY_PORT}{API_BASE_PATH}' +PEER_URL = f'http://localhost:{PEER_PORT}{API_BASE_PATH}' + +PRIMARY_DOMAIN_ID = get_test_domain_id(0) +PEER_DOMAIN_ID = get_test_domain_id(1) + +HOLDER = 'client-holder' +INTRUDER = 'client-intruder' + +# The peer owns this app, so the primary knows it only through aggregation - +# which is what puts a gateway boundary between the lock and the request being +# judged against it. +PEER_APP = 'peer_sensor' +PEER_DATA_ITEM = 'chassis/brakes/pressure' + +PEER_MANIFEST = f"""\ +manifest_version: "1.0" +metadata: + name: "Peer ECU" + version: "1.0.0" +config: + unmanifested_nodes: ignore +components: + - id: peer-ecu + name: "Peer ECU" +apps: + - id: {PEER_APP} + name: "Peer Sensor" + is_located_on: peer-ecu + ros_binding: + node_name: pressure_sensor + namespace: /chassis/brakes +""" + + +def _write_manifest(text): + handle = tempfile.NamedTemporaryFile( + mode='w', suffix='.yaml', delete=False, prefix='lock_identity_') + handle.write(text) + handle.close() + return handle.name + + +def generate_test_description(): + """Launch an aggregating gateway and the peer that owns the entity.""" + peer_manifest_path = _write_manifest(PEER_MANIFEST) + peer_domain_env = {'ROS_DOMAIN_ID': str(PEER_DOMAIN_ID)} + + locking = { + 'locking.enabled': True, + 'locking.default_max_expiration': 3600, + 'locking.cleanup_interval': 1, + } + + primary_gateway = create_gateway_node( + port=PRIMARY_PORT, + extra_params={ + **locking, + 'aggregation.enabled': True, + 'aggregation.timeout_ms': 5000, + 'aggregation.announce': False, + 'aggregation.discover': False, + 'aggregation.peer_urls': [f'http://localhost:{PEER_PORT}'], + 'aggregation.peer_names': ['peer_gateway'], + }, + ) + + peer_gateway = create_gateway_node( + name='peer_gateway_node', + port=PEER_PORT, + extra_params={ + **locking, + 'discovery.mode': 'hybrid', + 'discovery.manifest_path': peer_manifest_path, + 'discovery.manifest_strict_validation': False, + }, + extra_env=peer_domain_env, + ) + + delayed = TimerAction( + period=2.0, + actions=create_demo_nodes(['pressure_sensor'], extra_env=peer_domain_env), + ) + + launch_description = LaunchDescription([ + SetEnvironmentVariable('ROS_DOMAIN_ID', str(PRIMARY_DOMAIN_ID)), + primary_gateway, + peer_gateway, + delayed, + launch_testing.actions.ReadyToTest(), + ]) + return ( + launch_description, + {'gateway_node': primary_gateway, 'peer_gateway': peer_gateway}, + ) + + +class AggregateLockIdentityTest(unittest.TestCase): + """Drives the aggregating gateway; the peer is only ever used to verify.""" + + _lock_id = None + + def _wait_for_peer_app(self, timeout=60.0): + """Block until the primary has merged the peer's app.""" + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + response = requests.get(f'{PRIMARY_URL}/apps', timeout=5) + if response.status_code == 200: + ids = [item.get('id') for item in response.json().get('items', [])] + if PEER_APP in ids: + return + except requests.RequestException: + pass + time.sleep(0.25) + self.fail(f'{PEER_APP} was never merged into the aggregating gateway') + + def test_a_lock_taken_through_the_aggregate_belongs_to_the_client_who_took_it(self): + """L1: the holder's name reaches the gateway that records the lock.""" + self._wait_for_peer_app() + + acquired = requests.post( + f'{PRIMARY_URL}/apps/{PEER_APP}/locks', + headers={'X-Client-Id': HOLDER}, + json={'lock_expiration': 600}, + timeout=15, + ) + self.assertIn( + acquired.status_code, (200, 201), + f'acquiring a lock on a peer-owned app through the aggregate failed: ' + f'{acquired.status_code} {acquired.text}', + ) + type(self)._lock_id = acquired.json().get('id') + + # Asked of the PEER, not of the gateway under test. `owned` is the only + # place the answer is visible: the wire never carries another client's + # identity, so ownership is reported relative to whoever is asking. + on_peer = requests.get( + f'{PEER_URL}/apps/{PEER_APP}/locks', + headers={'X-Client-Id': HOLDER}, + timeout=15, + ) + self.assertEqual(on_peer.status_code, 200, on_peer.text) + items = on_peer.json().get('items', []) + self.assertTrue(items, 'the peer recorded no lock at all') + self.assertTrue( + any(item.get('owned') for item in items), + f'the peer holds a lock that belongs to nobody it can recognise, so ' + f'the client that took it is not its owner there: {items}', + ) + + def test_b_another_client_is_refused_while_the_lock_is_held(self): + """L2: the property a lock exists for, judged on the peer.""" + self.assertIsNotNone( + type(self)._lock_id, + 'no lock was acquired, so this case would prove nothing', + ) + + refused = requests.put( + f'{PRIMARY_URL}/apps/{PEER_APP}/data/{PEER_DATA_ITEM}', + headers={'X-Client-Id': INTRUDER}, + json={'data': {'pressure': 1.0}}, + timeout=15, + ) + self.assertEqual( + refused.status_code, 409, + f'a client holding no lock was allowed to write to a locked entity: ' + f'{refused.status_code} {refused.text}', + ) + + def test_c_the_holder_is_still_allowed_while_it_holds_the_lock(self): + """The other half of L2: refusing everybody is not the fix either.""" + self.assertIsNotNone( + type(self)._lock_id, + 'no lock was acquired, so this case would prove nothing', + ) + + allowed = requests.put( + f'{PRIMARY_URL}/apps/{PEER_APP}/data/{PEER_DATA_ITEM}', + headers={'X-Client-Id': HOLDER}, + json={'data': {'pressure': 2.0}}, + timeout=15, + ) + self.assertNotEqual( + allowed.status_code, 409, + f'the client holding the lock was refused by its own lock: ' + f'{allowed.status_code} {allowed.text}', + ) + + +@launch_testing.post_shutdown_test() +class TestShutdown(unittest.TestCase): + """Both gateways must come down cleanly.""" + + def test_exit_codes(self, proc_info): + """Check all processes exited cleanly.""" + for info in proc_info: + self.assertIn( + info.returncode, + ALLOWED_EXIT_CODES, + f'{info.process_name} exited with code {info.returncode}', + ) diff --git a/src/ros2_medkit_integration_tests/test/features/test_grouping_entity_aggregation.test.py b/src/ros2_medkit_integration_tests/test/features/test_grouping_entity_aggregation.test.py index 96c9323ce..1e499a6f6 100644 --- a/src/ros2_medkit_integration_tests/test/features/test_grouping_entity_aggregation.test.py +++ b/src/ros2_medkit_integration_tests/test/features/test_grouping_entity_aggregation.test.py @@ -73,6 +73,8 @@ R7 A Component both sides contribute to aggregates. Routing it wholesale to one peer would discard the other half, which is what happens today. R8 A lock on a leaf is honoured by a request dispatched through an aggregate. + Verified in test_aggregate_lock_identity, which needs a second gateway + holding the lock and so cannot share this topology. R9 Peered gateways terminate. R10 A manifest-declared entity outlives the link that reported it: it stays in the tree, keeps the items it last reported, and says it cannot be reached. From 5ebcce6d2f17df71b7c964e884ef301b17b18288 Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Thu, 20 Aug 2026 11:43:06 +0200 Subject: [PATCH 07/22] fix(operations): address an operation by its ROS path where one provider shares the name An operation's wire id is the last segment of its ROS path, so one provider exposing left/calibrate and right/calibrate offers the same id twice. The member half cannot separate those: it separates copies belonging to different members, and these belong to one. The collection listed the duplicate and execution refused it, which leaves the resource unreachable. Those operations now carry the ROS path as the item half, and both forms resolve. A short name never contains a slash, so neither form can be read as the other, and an id that identifies one operation today is untouched. One rule decides this, and the listing, the capability document and the resolver all read it. --- docs/api/rest.rst | 24 +- src/ros2_medkit_gateway/README.md | 19 ++ .../design/aggregation.rst | 12 + .../core/http/member_qualified_id.hpp | 27 +- .../core/http/operation_item_id.hpp | 96 ++++++ .../src/core/openapi/route_registry.cpp | 6 +- .../src/http/handlers/operation_handlers.cpp | 72 +++-- .../src/http/rest_server.cpp | 24 +- .../src/openapi/capability_generator.cpp | 20 +- .../CMakeLists.txt | 5 + .../demo_nodes/dual_calibration_service.cpp | 79 +++++ .../ros2_medkit_test_utils/launch_helpers.py | 4 + .../test_grouping_entity_aggregation.test.py | 296 +++++++++++++++++- 13 files changed, 642 insertions(+), 42 deletions(-) create mode 100644 src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/http/operation_item_id.hpp create mode 100644 src/ros2_medkit_integration_tests/demo_nodes/dual_calibration_service.cpp diff --git a/docs/api/rest.rst b/docs/api/rest.rst index f22c3f259..d6d78166a 100644 --- a/docs/api/rest.rst +++ b/docs/api/rest.rst @@ -645,6 +645,20 @@ consider it unique. In practice: every contributor is named in ``member_ids``. A path is qualified only when two gateways each contribute an item under it. +**When one member carries the short name twice.** An operation's wire id is the +last segment of its ROS path, so ``left/calibrate`` and ``right/calibrate`` on +one node are two operations called ``calibrate``. The member half names that +same member for both copies and separates nothing, so those items take the ROS +path, leading slash stripped, as their item half:: + + robot/left/calibrate # on the App itself + primary_calibration:robot/left/calibrate # on an entity that aggregates it + +The form is decided per provider: a short name its own provider carries once +keeps that short name, whatever another provider does with the same name. The +split at the first colon is unchanged, because a ROS path carries no colon, and +the path form is the one ``/data`` already uses for a topic. + What this means for a request: - A bare id that names one item works, on every route. Every client that sends @@ -654,7 +668,11 @@ What this means for a request: one member provides is refused with ``400 invalid-request``, naming the qualified form and listing the members in ``parameters.member_ids``. Running whichever member was walked first without saying which one ran is the defect - this removes. + this removes. A short name that ONE member carries at two ROS paths is + refused the same way, listing those paths in ``parameters.ros2_paths``. + Either refusal carries ``parameters.operation_ids``: the ids that do address + what collided, as the collection lists them, so the client sends one back + rather than deriving it. - A qualified id is accepted on the single-item routes. A member half that names no member of the entity is ``404``, and so is an item half that member does not provide - which is what tells an absent item apart from an item that @@ -816,7 +834,9 @@ Operations Endpoints Execute ROS 2 services and actions. Operation ids follow :ref:`member-qualified-ids`: a short name that only one member exposes is used -bare, and one that several expose is addressed ``:``. +bare, one that several expose is addressed ``:``, and one +that a single member exposes at two ROS paths is addressed by the path itself, +without its leading slash. List Operations ~~~~~~~~~~~~~~~ diff --git a/src/ros2_medkit_gateway/README.md b/src/ros2_medkit_gateway/README.md index b62aa9574..8f2af7e5d 100644 --- a/src/ros2_medkit_gateway/README.md +++ b/src/ros2_medkit_gateway/README.md @@ -282,10 +282,29 @@ path names one topic however many members publish and subscribe to it, so it stays bare and lists its contributors in `member_ids`; it is qualified only if two gateways each contribute an item under that path. +An operation's short name is the last segment of its ROS path, so ONE member +can carry it twice - `left/calibrate` and `right/calibrate` are two operations +called `calibrate`. The member half names that member for both copies and +separates nothing, so those items take the ROS path, leading slash stripped, as +their item half: + +``` +robot/left/calibrate # on the App itself +primary_calibration:robot/left/calibrate # on an entity that aggregates it +``` + +The form is decided per provider: a short name that its own provider carries +once keeps that short name, whatever any other provider does with it. `/data` +already addresses its items by path, so the split at the first colon is +unchanged - a ROS path carries no colon. + - A bare id that names one item works on every route, which is what the web UI, the Foxglove panel, the MCP tools and the generated OpenAPI document all send. - `POST /{entity}/operations/{id}/executions` with a bare id several members provide is `400 invalid-request`, naming the qualified form and the members. + A short name one member carries twice is `400` too, naming the ROS paths that + collided. Either refusal carries `parameters.operation_ids`: the ids that do + address what collided, as the collection lists them. - A qualified id is accepted on single-item routes; an unknown member half, an item half that member does not provide, or a member half followed by nothing, is `404` - which is what tells an absent item apart from one that exists and diff --git a/src/ros2_medkit_gateway/design/aggregation.rst b/src/ros2_medkit_gateway/design/aggregation.rst index ef0bdfa5e..d1a9bc261 100644 --- a/src/ros2_medkit_gateway/design/aggregation.rst +++ b/src/ros2_medkit_gateway/design/aggregation.rst @@ -445,6 +445,18 @@ the member half is an entity id, and the item half is the id the member's own route uses. Nothing on the owning gateway is aggregating, so the item half is sent bare - a parameter as its plain name, a topic as its plain path. +An operation's item half is its short name, except where the member carrying it +exposes that short name at more than one ROS path. There the member half names +one member for both copies and cannot separate them, so the item half is the +ROS path with its leading slash stripped - and it stays that on the member's own +route too, because the member has the same two operations under the same short +name: + +.. code-block:: text + + POST /api/v1/components/vehicle-ecu/operations/dual_calibration:testrig/dual/left/calibrate/executions + -> POST /api/v1/apps/dual_calibration/operations/testrig/dual/left/calibrate/executions + The member's own gateway is the only one that can answer: the ROS service, the topic and the parameter behind the id exist on its graph and nowhere else. What this gateway holds for a peer-owned member is a declaration, which is why the diff --git a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/http/member_qualified_id.hpp b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/http/member_qualified_id.hpp index 87fb44aec..7340d10c9 100644 --- a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/http/member_qualified_id.hpp +++ b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/http/member_qualified_id.hpp @@ -40,6 +40,10 @@ namespace http { * subscribe to it: that is still one topic, merged into one item, and the bare * path addresses it exactly. It becomes ambiguous only if two gateways each * contribute an item under the same path. + * + * The member half separates copies that belong to DIFFERENT members. Where one + * member carries both copies it has nothing left to separate, and the item half + * carries the ROS path instead - see `path_item_id`. */ struct MemberQualifiedId { std::string member_id; ///< Owning member; empty when the id carries no member half. @@ -77,6 +81,26 @@ inline std::string make_member_qualified_id(const std::string & member_id, const return member_id + ":" + item_id; } +/** + * @brief The item half that addresses an operation by its ROS path. + * + * An operation's short name is the last segment of its ROS path, so one + * provider can carry the same short name twice: `/robot/left/calibrate` and + * `/robot/right/calibrate` are two operations both named `calibrate`. The + * member half cannot tell those apart - it is the same member - while the path + * can, because a ROS graph holds each path once. + * + * The leading slash is dropped so the id reads as the sequence of segments the + * router carries, which is the form `/data` already uses for a topic. A short + * name never contains a slash, so a path item half can never be read as one. + * + * Splitting a member-qualified id still happens at the first colon: a ROS path + * carries no colon, so the member half stays the part before it. + */ +inline std::string path_item_id(const std::string & full_path) { + return (!full_path.empty() && full_path.front() == '/') ? full_path.substr(1) : full_path; +} + /** * @brief Rewrite the id of every item whose id another item in `items` shares. * @@ -108,8 +132,7 @@ void qualify_ambiguous_ids(std::vector & items, MemberIdsOf member_ids_of) } // An id already addressed to this member is left alone. Prefixing it again // yields a form whose first colon splits off the member twice, which names - // nothing - and it happens whenever one member exposes the same short name - // more than once, because both copies then carry the same qualified id. + // nothing. if (item.id.rfind(members->front() + ":", 0) == 0) { continue; } diff --git a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/http/operation_item_id.hpp b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/http/operation_item_id.hpp new file mode 100644 index 000000000..31085d8d7 --- /dev/null +++ b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/http/operation_item_id.hpp @@ -0,0 +1,96 @@ +// Copyright 2026 bburda +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#include +#include + +#include "ros2_medkit_gateway/core/http/member_qualified_id.hpp" +#include "ros2_medkit_gateway/core/models/thread_safe_entity_cache.hpp" + +namespace ros2_medkit_gateway { +namespace http { + +/** + * @brief Which item half an operation's id carries. + * + * An operation's wire id is the last segment of its ROS path, so a short name + * is unique only as far as its provider's own graph makes it so. Two axes + * decide the id, and they are independent: + * + * - the MEMBER half, from `member_qualified_id.hpp`, separates copies that + * belong to different members; + * - the ITEM half, decided here, separates copies that belong to the SAME + * member - there the member half names one member twice and separates + * nothing, so the ROS path stands in for the short name. + * + * Everything that emits an operation id - the collection, the per-entity + * OpenAPI document - and everything that resolves one has to read the same + * rule, or the gateway documents and lists a request it then refuses. + */ + +/// The full paths whose short name their OWN provider carries more than once. +/// +/// Every other operation keeps its short name, which is the id every current +/// client and the generated OpenAPI document send. +/// +/// An entity that exposes its operations directly records no owner for them. +/// The empty owner is one provider, which is exactly what a plain App is. +inline std::unordered_set operation_paths_addressed_by_path(const AggregatedOperations & ops) { + std::map, std::vector> paths_by_provider; + const auto record = [&ops, &paths_by_provider](const std::string & name, const std::string & full_path) { + auto owner = ops.owner_by_path.find(full_path); + const std::string member = owner != ops.owner_by_path.end() ? owner->second : std::string{}; + paths_by_provider[std::make_pair(member, name)].push_back(full_path); + }; + for (const auto & svc : ops.services) { + record(svc.name, svc.full_path); + } + for (const auto & act : ops.actions) { + record(act.name, act.full_path); + } + + std::unordered_set shared; + for (const auto & entry : paths_by_provider) { + if (entry.second.size() < 2) { + continue; + } + shared.insert(entry.second.begin(), entry.second.end()); + } + return shared; +} + +/// The item half `full_path` is addressed by, given the paths its provider +/// carries under a shared short name. +inline std::string operation_item_half(const std::string & name, const std::string & full_path, + const std::unordered_set & addressed_by_path) { + return addressed_by_path.count(full_path) > 0u ? path_item_id(full_path) : name; +} + +/// True when `item_id` addresses this operation. +/// +/// Two forms, both exact: the short name, and the ROS path without its leading +/// slash. A short name never contains a slash, so neither form can be read as +/// the other. +inline bool operation_item_id_names(const std::string & name, const std::string & full_path, + const std::string & item_id) { + return item_id == name || item_id == path_item_id(full_path); +} + +} // namespace http +} // namespace ros2_medkit_gateway diff --git a/src/ros2_medkit_gateway/src/core/openapi/route_registry.cpp b/src/ros2_medkit_gateway/src/core/openapi/route_registry.cpp index 11c0f9abb..65f8cb263 100644 --- a/src/ros2_medkit_gateway/src/core/openapi/route_registry.cpp +++ b/src/ros2_medkit_gateway/src/core/openapi/route_registry.cpp @@ -372,9 +372,13 @@ std::string RouteRegistry::to_regex_path(const std::string & openapi_path, const // Special cases: // - {data_id} at the end of data paths -> (.+) (multi-segment, for slash-containing topic names) // - {config_id} at the end of configuration paths -> (.+) (for slash-containing param names) + // - {operation_id} anywhere -> (.+) (an operation is addressed by ROS path where one + // provider carries its short name twice, and the executions sub-resources sit behind it) // - All other {param} -> ([^/]+) (single segment) // // The "end of path" check ensures only the LAST param on data/config paths gets (.+). + // {operation_id} carries no such check because the sub-resource routes put it mid-path; + // the routes it can then swallow are registered ahead of it (see setup_routes). // Root path "/" -> just optional slash anchor (prefix already has the base path) if (openapi_path == "/") { @@ -395,7 +399,7 @@ std::string RouteRegistry::to_regex_path(const std::string & openapi_path, const bool is_last = (close + 1 >= openapi_path.size()); // Use (.+) for the final segment on data and configuration item paths - if (is_last && (param_name == "data_id" || param_name == "config_id")) { + if (param_name == "operation_id" || (is_last && (param_name == "data_id" || param_name == "config_id"))) { result += "(.+)"; } else { result += "([^/]+)"; diff --git a/src/ros2_medkit_gateway/src/http/handlers/operation_handlers.cpp b/src/ros2_medkit_gateway/src/http/handlers/operation_handlers.cpp index 71d7bed44..ced5b4f60 100644 --- a/src/ros2_medkit_gateway/src/http/handlers/operation_handlers.cpp +++ b/src/ros2_medkit_gateway/src/http/handlers/operation_handlers.cpp @@ -32,6 +32,7 @@ #include "ros2_medkit_gateway/core/http/fan_out_helpers.hpp" #include "ros2_medkit_gateway/core/http/http_utils.hpp" #include "ros2_medkit_gateway/core/http/member_qualified_id.hpp" +#include "ros2_medkit_gateway/core/http/operation_item_id.hpp" #include "ros2_medkit_gateway/core/managers/operation_manager.hpp" #include "ros2_medkit_gateway/core/plugins/plugin_manager.hpp" #include "ros2_medkit_gateway/core/providers/operation_provider.hpp" @@ -149,8 +150,9 @@ struct ResolvedOperation { /// Resolve `parsed` against the entity's operations. /// /// A member half selects among same-named operations using the owner recorded -/// per full ROS path. A bare id keeps the first match, which is the only thing -/// it can mean when it is unique and the only thing this gateway did before. +/// per full ROS path, and an item half that is a ROS path selects one outright. +/// A bare short name keeps the first match, which is the only thing it can mean +/// when it is unique and the only thing this gateway did before. ResolvedOperation resolve_operation(const AggregatedOperations & ops, const http::MemberQualifiedId & parsed) { const auto owned_by_target = [&ops, &parsed](const std::string & full_path) { if (!parsed.has_member) { @@ -162,13 +164,13 @@ ResolvedOperation resolve_operation(const AggregatedOperations & ops, const http ResolvedOperation resolved; for (const auto & svc : ops.services) { - if (svc.name == parsed.item_id && owned_by_target(svc.full_path)) { + if (http::operation_item_id_names(svc.name, svc.full_path, parsed.item_id) && owned_by_target(svc.full_path)) { resolved.service = svc; return resolved; } } for (const auto & act : ops.actions) { - if (act.name == parsed.item_id && owned_by_target(act.full_path)) { + if (http::operation_item_id_names(act.name, act.full_path, parsed.item_id) && owned_by_target(act.full_path)) { resolved.action = act; return resolved; } @@ -200,12 +202,12 @@ std::vector matching_operations(const AggregatedOperations & ops matches.push_back(OperationMatch{full_path, member}); }; for (const auto & svc : ops.services) { - if (svc.name == parsed.item_id) { + if (http::operation_item_id_names(svc.name, svc.full_path, parsed.item_id)) { record(svc.full_path); } } for (const auto & act : ops.actions) { - if (act.name == parsed.item_id) { + if (http::operation_item_id_names(act.name, act.full_path, parsed.item_id)) { record(act.full_path); } } @@ -495,8 +497,8 @@ http::Result> OperationHandlers::list_operat auto data_access_mgr = ctx_.node()->get_data_access_manager(); auto type_introspection = data_access_mgr->get_type_introspection(); - // How many members the DECLARED tree says provide each short name. This is - // the same count `create_execution` refuses on, and it is read here so the + // How many operations the DECLARED tree carries under each short name. This + // is the same count `create_execution` refuses on, and it is read here so the // listing and the execution cannot disagree: an id the tree calls ambiguous // is never offered bare, wherever the copy came from. std::unordered_map declared_providers; @@ -507,6 +509,11 @@ http::Result> OperationHandlers::list_operat ++declared_providers[act.name]; } + // The paths whose short name one provider carries twice. Their item half is + // the path, because the member half they would otherwise be told apart by is + // the same member for both. + const std::unordered_set path_addressed = http::operation_paths_addressed_by_path(ops); + const auto contributed_by_peer = [&cache](const std::string & member_id) { static constexpr std::string_view kPeerPrefix = "peer:"; if (auto app = cache.get_app(member_id)) { @@ -522,13 +529,28 @@ http::Result> OperationHandlers::list_operat return owner != ops.owner_by_path.end() && contributed_by_peer(owner->second); }; + // The ROS path an item names, in the form an id carries it. Empty when the + // item names no path, which is what a peer's malformed item looks like. + const auto path_half_of = [](const dto::OperationItem & item) -> std::string { + if (!item.x_medkit.has_value() || !item.x_medkit->ros2.has_value()) { + return {}; + } + const auto & ros2 = *item.x_medkit->ros2; + return http::path_item_id(ros2.service.value_or(ros2.action.value_or(std::string{}))); + }; + // Qualify from the declared tree rather than by counting copies in this // response. A response can be short a copy - the caller suppressed fan-out, // or a peer did not answer - and counting copies would then hand back a bare // id that the execution refuses. - const auto qualify_from_declared_tree = [&declared_providers](dto::OperationItem & item) { - if (item.id != item.name) { - return; // already qualified, by us or by the peer that sent it + // + // The member half goes in front of whichever item half the id already + // carries, short name or path, so an item this gateway addressed by path is + // still addressed to the member that owns it. + const auto qualify_from_declared_tree = [&declared_providers, &path_half_of](dto::OperationItem & item) { + const std::string path_half = path_half_of(item); + if (item.id != item.name && (path_half.empty() || item.id != path_half)) { + return; // already carries a member half, from us or from the peer that sent it } auto count = declared_providers.find(item.name); if (count == declared_providers.end() || count->second < 2) { @@ -538,12 +560,12 @@ http::Result> OperationHandlers::list_operat item.x_medkit->member_ids->size() != 1) { return; } - item.id = http::make_member_qualified_id(item.x_medkit->member_ids->front(), item.name); + item.id = http::make_member_qualified_id(item.x_medkit->member_ids->front(), item.id); }; const auto build_item = [&](const auto & op, bool asynchronous) { dto::OperationItem item; - item.id = op.name; + item.id = http::operation_item_half(op.name, op.full_path, path_addressed); item.name = op.name; item.proximity_proof_required = false; item.asynchronous_execution = asynchronous; @@ -853,9 +875,8 @@ OperationHandlers::create_execution(const http::TypedRequest & req, dto::Executi // // The qualified form is checked too. Naming the member narrows the set, but // one member that uses the same short name at two ROS paths is still not - // identified by it - and for those two the member half has nothing left to - // add, so the caller is told what collided rather than handed a remedy that - // cannot work. + // identified by it - there the item half has to be the ROS path, which is the + // form the collection offers for exactly those copies. const std::vector matches = matching_operations(ops, parsed); if (matches.size() > 1) { std::vector paths; @@ -868,14 +889,29 @@ OperationHandlers::create_execution(const http::TypedRequest & req, dto::Executi if (!members.empty()) { params["member_ids"] = members; } + // The ids that DO address the operations this one collided with, built by + // the rule the collection lists them under. A refusal that only describes + // the form leaves the caller to re-derive it, and a caller that derives it + // differently is refused again for a reason the answer already knew. + const auto addressed_by_path = http::operation_paths_addressed_by_path(ops); + std::vector addressable; + addressable.reserve(matches.size()); + for (const auto & match : matches) { + std::string item = http::operation_item_half(parsed.item_id, match.full_path, addressed_by_path); + if (ops.is_aggregated && !match.member_id.empty()) { + item = http::make_member_qualified_id(match.member_id, item); + } + addressable.push_back(std::move(item)); + } + params["operation_ids"] = addressable; if (members.size() > 1) { params["details"] = "Use format 'member_id:operation_id' to name the member that runs it"; return tl::make_unexpected( make_error(400, ERR_INVALID_REQUEST, "Ambiguous operation id: more than one member provides it", params)); } params["details"] = - "One provider exposes this short name at more than one ROS path, so naming the member " - "cannot separate them"; + "One provider exposes this short name at more than one ROS path; address the one you mean by that " + "path, without its leading slash"; return tl::make_unexpected( make_error(400, ERR_INVALID_REQUEST, "Ambiguous operation id: it names more than one operation", params)); } diff --git a/src/ros2_medkit_gateway/src/http/rest_server.cpp b/src/ros2_medkit_gateway/src/http/rest_server.cpp index 65efa45af..5559736a0 100644 --- a/src/ros2_medkit_gateway/src/http/rest_server.cpp +++ b/src/ros2_medkit_gateway/src/http/rest_server.cpp @@ -657,16 +657,6 @@ void RESTServer::setup_routes() { .description(std::string("Lists all ROS 2 services and actions available on this ") + et.singular + ".") .operation_id(std::string("list") + capitalize(et.singular) + "Operations"); - reg.get(entity_path + "/operations/{operation_id}", - [this](http::TypedRequest req) -> http::Result { - return operation_handlers_->get_operation(req); - }) - .tag("Operations") - .summary(std::string("Get operation details for ") + et.singular) - .description(std::string("Returns operation details including request/response schema for this ") + - et.singular + ".") - .operation_id(std::string("get") + capitalize(et.singular) + "Operation"); - // Execution endpoints reg.post_alternates( entity_path + "/operations/{operation_id}/executions", @@ -741,6 +731,20 @@ void RESTServer::setup_routes() { nlohmann::json{{"$ref", "#/components/schemas/GenericError"}}) .operation_id(std::string("cancel") + capitalize(et.singular) + "Execution"); + // Operation item, registered after its own sub-resources. {operation_id} + // spans segments, so this pattern also matches an executions URI; the + // router takes the first route that matches, and the specific ones have to + // be reachable. + reg.get(entity_path + "/operations/{operation_id}", + [this](http::TypedRequest req) -> http::Result { + return operation_handlers_->get_operation(req); + }) + .tag("Operations") + .summary(std::string("Get operation details for ") + et.singular) + .description(std::string("Returns operation details including request/response schema for this ") + + et.singular + ".") + .operation_id(std::string("get") + capitalize(et.singular) + "Operation"); + // --- Configurations --- // // PR-403 commit 26: 5 config routes migrate to the typed RouteRegistry diff --git a/src/ros2_medkit_gateway/src/openapi/capability_generator.cpp b/src/ros2_medkit_gateway/src/openapi/capability_generator.cpp index 730fa7bcc..bd9814872 100644 --- a/src/ros2_medkit_gateway/src/openapi/capability_generator.cpp +++ b/src/ros2_medkit_gateway/src/openapi/capability_generator.cpp @@ -24,6 +24,7 @@ #include "ros2_medkit_gateway/core/http/http_utils.hpp" #include "ros2_medkit_gateway/core/http/member_qualified_id.hpp" +#include "ros2_medkit_gateway/core/http/operation_item_id.hpp" #include "ros2_medkit_gateway/core/models/entity_capabilities.hpp" #include "ros2_medkit_gateway/core/models/entity_types.hpp" #include "ros2_medkit_gateway/core/plugins/plugin_manager.hpp" @@ -275,8 +276,11 @@ nlohmann::json CapabilityGenerator::generate_resource_collection(const ResolvedP } paths[collection_path] = path_builder.build_operations_collection(entity_path, ops); - // Short names are not unique across members, and two of them produced the - // same key here, so one item's documentation silently replaced the other's. + // A path key that two operations produce documents one of them and drops + // the other, so the id here has to be the id the collection emits - both + // halves of it. `short_name_counts` decides the member half; a short name + // one provider carries twice takes the ROS path as its item half, because + // the member half is that same provider for both copies. std::unordered_map short_name_counts; for (const auto & svc : ops.services) { ++short_name_counts[svc.name]; @@ -288,16 +292,18 @@ nlohmann::json CapabilityGenerator::generate_resource_collection(const ResolvedP auto owner = ops.owner_by_path.find(full_path); return owner != ops.owner_by_path.end() ? owner->second : std::string{}; }; + const auto addressed_by_path = http::operation_paths_addressed_by_path(ops); + const auto operation_id_of = [&](const std::string & name, const std::string & full_path) { + return qualified_item_id(http::operation_item_half(name, full_path, addressed_by_path), short_name_counts[name], + owner_of(full_path)); + }; for (const auto & svc : ops.services) { - std::string item_path = - collection_path + "/" + qualified_item_id(svc.name, short_name_counts[svc.name], owner_of(svc.full_path)); + std::string item_path = collection_path + "/" + operation_id_of(svc.name, svc.full_path); paths[item_path] = path_builder.build_operation_item(entity_path, svc); } for (const auto & action : ops.actions) { - std::string item_path = - collection_path + "/" + - qualified_item_id(action.name, short_name_counts[action.name], owner_of(action.full_path)); + std::string item_path = collection_path + "/" + operation_id_of(action.name, action.full_path); paths[item_path] = path_builder.build_operation_item(entity_path, action); } } else if (resolved.resource_collection == "configurations") { diff --git a/src/ros2_medkit_integration_tests/CMakeLists.txt b/src/ros2_medkit_integration_tests/CMakeLists.txt index aca34ddea..942327502 100644 --- a/src/ros2_medkit_integration_tests/CMakeLists.txt +++ b/src/ros2_medkit_integration_tests/CMakeLists.txt @@ -75,6 +75,10 @@ add_executable(demo_calibration_service demo_nodes/calibration_service.cpp) target_include_directories(demo_calibration_service PRIVATE ${_demo_include_dir}) medkit_target_dependencies(demo_calibration_service rclcpp rcl_interfaces std_srvs) +add_executable(demo_dual_calibration_service demo_nodes/dual_calibration_service.cpp) +target_include_directories(demo_dual_calibration_service PRIVATE ${_demo_include_dir}) +medkit_target_dependencies(demo_dual_calibration_service rclcpp std_srvs) + add_executable(demo_long_calibration_action demo_nodes/long_calibration_action.cpp) target_include_directories(demo_long_calibration_action PRIVATE ${_demo_include_dir}) medkit_target_dependencies(demo_long_calibration_action rclcpp rclcpp_action example_interfaces) @@ -114,6 +118,7 @@ install(TARGETS demo_brake_actuator demo_light_controller demo_calibration_service + demo_dual_calibration_service demo_long_calibration_action demo_lidar_sensor demo_beacon_publisher diff --git a/src/ros2_medkit_integration_tests/demo_nodes/dual_calibration_service.cpp b/src/ros2_medkit_integration_tests/demo_nodes/dual_calibration_service.cpp new file mode 100644 index 000000000..2ac779aed --- /dev/null +++ b/src/ros2_medkit_integration_tests/demo_nodes/dual_calibration_service.cpp @@ -0,0 +1,79 @@ +// Copyright 2026 bburda +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * @file dual_calibration_service.cpp + * @brief One node exposing two services whose ROS paths differ only above the + * last segment. + * + * `left/calibrate` and `right/calibrate` under the node's namespace are two + * different services with one short name, and the short name is the wire id the + * operations collection uses. One provider carries both, so the member half of + * a qualified id names the same thing for each - which is the case the ROS path + * has to address instead. + * + * Each side answers with a message naming itself, so a caller can tell which + * service ran from the response rather than from the status alone. + */ + +#include +#include + +#include +#include + +#include "ros2_medkit_integration_tests/demo_node_main.hpp" + +class DualCalibrationService : public rclcpp::Node { + public: + DualCalibrationService() : Node("dual_calibration") { + left_srv_ = make_side("left"); + right_srv_ = make_side("right"); + + RCLCPP_INFO(this->get_logger(), "Dual calibration services started"); + } + + // The callbacks capture `this`, so the services have to go before any member + // they touch does. + ~DualCalibrationService() override { + left_srv_.reset(); + right_srv_.reset(); + } + + DualCalibrationService(const DualCalibrationService &) = delete; + DualCalibrationService & operator=(const DualCalibrationService &) = delete; + DualCalibrationService(DualCalibrationService &&) = delete; + DualCalibrationService & operator=(DualCalibrationService &&) = delete; + + private: + rclcpp::Service::SharedPtr make_side(const std::string & side) { + return this->create_service( + side + "/calibrate", [this, side](const std::shared_ptr & request, + const std::shared_ptr & response) { + (void)request; // Trigger has no request fields + response->success = true; + response->message = side + " side calibrated"; + RCLCPP_INFO(this->get_logger(), "Calibration requested: %s", response->message.c_str()); + }); + } + + rclcpp::Service::SharedPtr left_srv_; + rclcpp::Service::SharedPtr right_srv_; +}; + +int main(int argc, char * argv[]) { + return ros2_medkit_integration_tests::run_demo_node(argc, argv, []() -> std::shared_ptr { + return std::make_shared(); + }); +} diff --git a/src/ros2_medkit_integration_tests/ros2_medkit_test_utils/launch_helpers.py b/src/ros2_medkit_integration_tests/ros2_medkit_test_utils/launch_helpers.py index 944ab5ab8..cd4b7ab41 100644 --- a/src/ros2_medkit_integration_tests/ros2_medkit_test_utils/launch_helpers.py +++ b/src/ros2_medkit_integration_tests/ros2_medkit_test_utils/launch_helpers.py @@ -55,6 +55,10 @@ 'controller': ('demo_light_controller', 'controller', '/body/lights'), # Operations (services / actions) 'calibration': ('demo_calibration_service', 'calibration', '/powertrain/engine'), + # Two services under one node whose ROS paths differ only above the last + # segment, so one provider carries the operation short name `calibrate` + # twice. Its own namespace keeps that collision away from `calibration`. + 'dual_calibration': ('demo_dual_calibration_service', 'dual_calibration', '/testrig/dual'), 'long_calibration': ('demo_long_calibration_action', 'long_calibration', '/powertrain/engine'), # Lifecycle demo (stays unconfigured by default; auto_activate:=true activates it) 'managed_lifecycle': ('managed_lifecycle', 'managed_lifecycle', ''), diff --git a/src/ros2_medkit_integration_tests/test/features/test_grouping_entity_aggregation.test.py b/src/ros2_medkit_integration_tests/test/features/test_grouping_entity_aggregation.test.py index 1e499a6f6..1adfd8daa 100644 --- a/src/ros2_medkit_integration_tests/test/features/test_grouping_entity_aggregation.test.py +++ b/src/ros2_medkit_integration_tests/test/features/test_grouping_entity_aggregation.test.py @@ -63,6 +63,12 @@ App hangs off the one host Component, so "aggregating" is the ordinary entity, and qualifying everything there would change the ids of the most used entity in the product and refuse requests every current client sends. + Where ONE leaf provides two operations under one short name - the wire id + is the last segment of the ROS path, so `left/calibrate` and + `right/calibrate` are two operations called `calibrate` - the leaf half + names that same leaf twice and separates nothing. Those take the ROS path, + leading slash stripped, as their item half: "robot/left/calibrate" on the + leaf itself and ":robot/left/calibrate" on an aggregate. R4 A bare id is refused only when it is ambiguous, and the refusal names the form to use. An unambiguous bare id keeps working. R5 A compound id reaches its leaf, local or on a peer, and returns that leaf's @@ -126,7 +132,7 @@ # exist for. With the same namespace on both sides the full paths are identical # and the local walk simply deduplicates the peer's copy away, which builds a # dedup collapse rather than the ambiguity. -PRIMARY_NODES = ['temp_sensor', 'calibration', 'rpm_sensor'] +PRIMARY_NODES = ['temp_sensor', 'calibration', 'dual_calibration', 'rpm_sensor'] PEER_NODES = ['pressure_sensor', 'actuator'] PEER_CALIBRATION_NAMESPACE = '/chassis/brakes' @@ -141,6 +147,17 @@ # the configuration cases below address. CALIBRATION_PARAM = 'calibration_offset' +# One node exposing `left/calibrate` and `right/calibrate` under its own +# namespace: two operations, one short name, one provider. The short name is the +# wire id, and the member half names `dual_calibration` for both copies, so the +# ROS path is the only thing left that tells them apart. It lives outside +# `/powertrain/engine` so this collision stays independent of the one between +# the two calibration Apps. +DUAL_APP = 'dual_calibration' +DUAL_NAMESPACE = '/testrig/dual' +DUAL_LEFT_ID = 'testrig/dual/left/calibrate' +DUAL_RIGHT_ID = 'testrig/dual/right/calibrate' + MERGED_AREA = 'vehicle' MERGED_FUNCTION = 'vehicle_health' PARENT_COMPONENT = 'vehicle-ecu' @@ -176,6 +193,12 @@ ros_binding: node_name: calibration namespace: /powertrain/engine + - id: {DUAL_APP} + name: "Dual Calibration Service" + is_located_on: {PARENT_COMPONENT} + ros_binding: + node_name: dual_calibration + namespace: {DUAL_NAMESPACE} - id: {COLLIDING_LEAF} name: "Shared Sensor (primary)" is_located_on: {PARENT_COMPONENT} @@ -334,7 +357,8 @@ def setUpClass(cls): # completes while the local ROS graph is still binding Apps to nodes, so # a collection read between those two moments is legitimately empty and # would fail every rule below for a reason unrelated to the rule. - cls._wait_for_apps(PRIMARY_URL, {'temp_sensor', 'primary_calibration'}, 'primary') + cls._wait_for_apps( + PRIMARY_URL, {'temp_sensor', 'primary_calibration', DUAL_APP}, 'primary') cls._wait_for_apps(PEER_URL, {'pressure_sensor', 'peer_calibration'}, 'peer') cls._wait_until_merged() @@ -391,6 +415,34 @@ def _items(self, entity_path, collection): self.assertEqual(response.status_code, 200, response.text) return response.json().get('items', []) + @staticmethod + def _run_operation(entity_path, operation_id): + """POST one execution of `operation_id` on the aggregating gateway.""" + return requests.post( + f'{PRIMARY_URL}/{entity_path}/operations/' + f'{quote(operation_id, safe="")}/executions', + json={}, + timeout=15, + ) + + def _assert_side_ran(self, response, side, operation_id): + """Assert the service the id names is the one that answered. + + Both sides return 200, so the status says only that something ran. The + message each service builds names itself, which is the only thing on the + wire that distinguishes a resolved id from one that fell through to + whichever operation was walked first. + """ + self.assertEqual(response.status_code, 200, response.text) + parameters = response.json().get('parameters') + self.assertIsInstance( + parameters, dict, f'no service response came back: {response.text}') + self.assertIs(parameters.get('success'), True, parameters) + self.assertEqual( + parameters.get('message'), f'{side} side calibrated', + f'{operation_id!r} reached the wrong service: {parameters}', + ) + @staticmethod def _set_offset(base_url, app_id, value): """Set `calibration_offset` on one App through ITS OWN gateway. @@ -571,6 +623,246 @@ def test_an_ambiguous_bare_id_is_refused(self): ).lower() self.assertIn('member', message, f'refusal does not name the qualified form: {body}') + # ---------------------------------------------------------------------- R3 + # ONE LEAF, ONE SHORT NAME, TWO ROS PATHS. + # + # The member half separates copies that belong to different leaves. Where + # one leaf holds both copies it names that leaf twice and separates nothing, + # so the item half has to be the ROS path instead. + + def test_a_leaf_naming_two_operations_alike_is_addressable_on_the_leaf(self): + """R3 on the leaf, where there is no member half at all. + + `left/calibrate` and `right/calibrate` are two services whose wire id - + the last segment of the ROS path - is `calibrate` for both. Offering + that id twice hands a client a name that cannot mean one thing, and + execution can then only refuse it, so the collection would be + advertising something unreachable. + """ + items = self._items(f'apps/{DUAL_APP}', 'operations') + ids = [item.get('id') for item in items] + + self.assertEqual(ids.count(DUAL_LEFT_ID), 1, f'ids were {ids}') + self.assertEqual(ids.count(DUAL_RIGHT_ID), 1, f'ids were {ids}') + self.assertEqual( + ids.count('calibrate'), 0, + f'an id naming two operations of one leaf was offered: {ids}', + ) + + for operation_id, side in ((DUAL_LEFT_ID, 'left'), (DUAL_RIGHT_ID, 'right')): + with self.subTest(operation=operation_id): + self._assert_side_ran( + self._run_operation(f'apps/{DUAL_APP}', operation_id), side, operation_id) + + # The detail route resolves the same id to the same operation, so a + # client that reads an item before running it is looking at what it will + # get. + detail = requests.get( + f'{PRIMARY_URL}/apps/{DUAL_APP}/operations/{quote(DUAL_LEFT_ID, safe="")}', + timeout=10, + ) + self.assertEqual(detail.status_code, 200, detail.text) + self.assertEqual( + detail.json().get('item', {}).get('x-medkit', {}).get('ros2', {}).get('service'), + f'/{DUAL_LEFT_ID}', + f'the detail route resolved the id to another service: {detail.text}', + ) + + def test_a_leaf_naming_two_operations_alike_is_refused_by_the_short_name(self): + """R4 for the same collision: the bare short name still names two things. + + It is no longer offered, and a client holding an older copy of it has to + be told what to send instead. The remedy is asserted as ids that run, + not as words in a sentence: a message can be reworded into saying + nothing while every assertion about a word in it still passes. + """ + response = self._run_operation(f'apps/{DUAL_APP}', 'calibrate') + self.assertEqual(response.status_code, 400, response.text) + body = response.json() + parameters = body.get('parameters', {}) + + self.assertEqual( + sorted(parameters.get('ros2_paths', [])), + [f'/{DUAL_LEFT_ID}', f'/{DUAL_RIGHT_ID}'], + f'the refusal does not say what collided: {body}', + ) + + addressable = parameters.get('operation_ids') + self.assertIsInstance( + addressable, list, f'the refusal offers no id to send instead: {body}') + self.assertEqual( + sorted(addressable), sorted([DUAL_LEFT_ID, DUAL_RIGHT_ID]), + f'the refusal does not hand back the ids that work: {body}', + ) + offered = {item.get('id') for item in self._items(f'apps/{DUAL_APP}', 'operations')} + self.assertTrue( + set(addressable) <= offered, + f'the refusal names ids the collection does not offer: ' + f'{sorted(addressable)} against {sorted(offered)}', + ) + + # Taken straight out of the refusal it runs, so the remedy is usable + # rather than merely described. + for operation_id in addressable: + with self.subTest(operation=operation_id): + side = 'left' if operation_id == DUAL_LEFT_ID else 'right' + self._assert_side_ran( + self._run_operation(f'apps/{DUAL_APP}', operation_id), side, operation_id) + + # The sentence a human reads names the one thing a client gets wrong. + self.assertIn( + 'without its leading slash', + str(parameters.get('details', '')), + f'the refusal does not say how the path is written: {body}', + ) + + # The same refusal on an aggregate has to carry the leaf half as well, + # or the id it hands back is one the aggregate does not accept. + through_aggregate = self._run_operation( + f'components/{PARENT_COMPONENT}', f'{DUAL_APP}:calibrate') + self.assertEqual(through_aggregate.status_code, 400, through_aggregate.text) + aggregate_ids = through_aggregate.json().get('parameters', {}).get('operation_ids') + self.assertEqual( + sorted(aggregate_ids or []), + sorted([f'{DUAL_APP}:{DUAL_LEFT_ID}', f'{DUAL_APP}:{DUAL_RIGHT_ID}']), + f'the aggregate refusal hands back ids it would refuse: {through_aggregate.text}', + ) + self._assert_side_ran( + self._run_operation(f'components/{PARENT_COMPONENT}', aggregate_ids[0]), + 'left' if aggregate_ids[0].endswith(DUAL_LEFT_ID) else 'right', + aggregate_ids[0], + ) + + def test_a_leaf_naming_two_operations_alike_is_addressable_on_an_aggregate(self): + """R3 on an aggregate, where both halves of the id are in play. + + The leaf half still says who runs it - the aggregate holds nothing + itself - and the ROS path says which of that leaf's two operations is + meant. Both are asserted, and the run is checked on the response body + because both services answer 200. + """ + items = self._items(f'components/{PARENT_COMPONENT}', 'operations') + ids = [item.get('id') for item in items] + left = f'{DUAL_APP}:{DUAL_LEFT_ID}' + right = f'{DUAL_APP}:{DUAL_RIGHT_ID}' + + self.assertEqual(ids.count(left), 1, f'ids were {ids}') + self.assertEqual(ids.count(right), 1, f'ids were {ids}') + self.assertEqual( + ids.count(f'{DUAL_APP}:calibrate'), 0, + f'a leaf-qualified id still named two operations: {ids}', + ) + + for item in items: + if item.get('id') in (left, right): + self.assertEqual( + item.get('x-medkit', {}).get('member_ids'), [DUAL_APP], + f"{item.get('id')!r} does not name the leaf that runs it: {item}", + ) + + for operation_id, side in ((left, 'left'), (right, 'right')): + with self.subTest(operation=operation_id): + self._assert_side_ran( + self._run_operation(f'components/{PARENT_COMPONENT}', operation_id), + side, operation_id) + + def test_the_capability_description_documents_both_alike_operations(self): + """The generated spec has to describe the ids the collection offers. + + The per-entity capability description keys its paths by operation id, so + two operations that produce one key document one of them and drop the + other without a word - the same "the listing does not match what can be + run" failure, one surface up, and the one a generated client is built + from. + """ + for entity_path, prefix in ( + (f'apps/{DUAL_APP}', ''), + (f'components/{PARENT_COMPONENT}', f'{DUAL_APP}:'), + ): + with self.subTest(entity=entity_path): + response = requests.get( + f'{PRIMARY_URL}/{entity_path}/operations/docs', timeout=10) + self.assertEqual(response.status_code, 200, response.text) + documented = set(response.json().get('paths', {})) + for operation_id in (DUAL_LEFT_ID, DUAL_RIGHT_ID): + self.assertIn( + f'/{entity_path}/operations/{prefix}{operation_id}', documented, + f'{operation_id!r} is not documented: {sorted(documented)}', + ) + + def test_a_short_name_one_leaf_carries_once_keeps_the_id_it_has_today(self): + """The regression guard, and it matters more than the case above. + + Every current client and the generated OpenAPI document send the bare + short name, or the leaf-qualified form where several leaves expose it. + Neither may move because some OTHER operation somewhere collided: an id + is decided by its own provider's use of its own short name. + """ + ids = [item.get('id') for item in self._items('apps/primary_calibration', 'operations')] + self.assertIn( + 'calibrate', ids, f'a short name only one leaf carries was rewritten: {ids}') + + response = self._run_operation('apps/primary_calibration', 'calibrate') + self.assertEqual(response.status_code, 200, response.text) + parameters = response.json().get('parameters') + self.assertIsInstance(parameters, dict, response.text) + self.assertIs(parameters.get('success'), True, parameters) + self.assertIn( + 'Engine calibrated', parameters.get('message', ''), + f'the bare id reached something else: {parameters}', + ) + + merged = [ + item.get('id') + for item in self._items(f'functions/{MERGED_FUNCTION}', 'operations') + ] + self.assertIn( + 'primary_calibration:calibrate', merged, + f'a leaf-qualified id gained a path half it does not need: {merged}', + ) + run = self._run_operation(f'functions/{MERGED_FUNCTION}', 'primary_calibration:calibrate') + self.assertEqual(run.status_code, 200, run.text) + self.assertIs( + run.json().get('parameters', {}).get('success'), True, run.text) + + def test_a_path_shaped_id_that_names_no_operation_is_refused(self): + """R5 for the path form: a path is an address, not a wildcard. + + Asserted beside a sibling path that DOES resolve, because "404 for + everything path-shaped" would satisfy the first half on its own and + would mean the form works nowhere. + """ + missing = 'testrig/dual/middle/calibrate' + + response = self._run_operation(f'apps/{DUAL_APP}', missing) + self.assertEqual(response.status_code, 404, response.text) + body = response.json() + self.assertEqual(body.get('error_code'), 'operation-not-found', body) + self.assertEqual(body.get('parameters', {}).get('operation_id'), missing, body) + + detail = requests.get( + f'{PRIMARY_URL}/apps/{DUAL_APP}/operations/{quote(missing, safe="")}', timeout=10) + self.assertEqual(detail.status_code, 404, detail.text) + + present = requests.get( + f'{PRIMARY_URL}/apps/{DUAL_APP}/operations/{quote(DUAL_LEFT_ID, safe="")}', + timeout=10, + ) + self.assertEqual( + present.status_code, 200, + f'the sibling that exists is refused too, so nothing is addressable: {present.text}', + ) + + # And through the aggregate, where the leaf half is right and only the + # path is wrong. + through_aggregate = self._run_operation( + f'components/{PARENT_COMPONENT}', f'{DUAL_APP}:{missing}') + self.assertEqual(through_aggregate.status_code, 404, through_aggregate.text) + self.assertEqual( + through_aggregate.json().get('error_code'), 'operation-not-found', + through_aggregate.text, + ) + # ---------------------------------------------------------------------- R4 def test_a_compound_id_reaches_a_peer_owned_member(self): From 2cbf1ab11d3d1d274c15b635a60705144af13e13 Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Thu, 20 Aug 2026 11:43:08 +0200 Subject: [PATCH 08/22] fix(configurations): decide the member half by membership, not by a node count A configuration id was split into member and parameter only when the entity counted as aggregating, and that count came from the nodes this gateway can resolve on its own ROS graph. A peer never reports a ROS binding for its apps, so an entity whose members all live on peers looked like it had none, and one with a single local node beside several peer ones looked unaggregated. The split now happens when the prefix names a member of the entity. No id that resolves today changes. Reset on an aggregating entity also stops reporting plain success while leaving peer-owned members untouched. --- docs/api/rest.rst | 56 +- docs/config/aggregation.rst | 11 + src/ros2_medkit_gateway/README.md | 23 +- .../design/aggregation.rst | 28 + .../src/http/handlers/config_handlers.cpp | 177 ++-- .../CMakeLists.txt | 1 + ...est_aggregator_only_configurations.test.py | 757 ++++++++++++++++++ 7 files changed, 1001 insertions(+), 52 deletions(-) create mode 100644 src/ros2_medkit_integration_tests/test/features/test_aggregator_only_configurations.test.py diff --git a/docs/api/rest.rst b/docs/api/rest.rst index d6d78166a..374b31c63 100644 --- a/docs/api/rest.rst +++ b/docs/api/rest.rst @@ -724,6 +724,18 @@ declares the parameter. ``GET /{entity}/configurations`` is unaffected: peer parameters reach that listing through the collection fan-out, and the ids it offers are the ids the single-item routes accept. +A member half is recognised when the text before the first colon names a member +of the addressed entity. How many ROS nodes this gateway resolves for that +entity does not enter into it, because a member another gateway runs reports no +ROS binding here and so resolves none: an aggregating entity whose members are +all peer-owned resolves nothing at all, and one that resolves a single local +node can still have peer-owned members beside it. Both take +``:`` exactly as an entity with several local nodes does. + +A prefix naming no member is part of the parameter name, which is what keeps a +parameter whose own name contains a colon addressable, and an entity's own id is +never read as a member half of itself. + Reachability is decided before anything is forwarded. A member retained while its gateway is silent answers ``504 not-responding`` naming the member (see :ref:`retained-entities`) rather than a ``502`` from a connection that could not @@ -772,8 +784,10 @@ separate questions and are reported separately. ``/configurations`` predates this rule and keeps its own: on an entity whose parameters come from more than one node, **every** parameter id is ``:``, and a bare id is refused on write. Its items carry - ``x-medkit.source`` (a single app id), not ``member_ids``. See - :ref:`configuration-endpoints`. + ``x-medkit.source`` (a single app id), not ``member_ids``. The node count + decides which ids the listing OFFERS; which ids it ACCEPTS is decided by the + member set, so the qualified form works on an entity whose members are all + peer-owned too. See :ref:`configuration-endpoints`. Data Endpoints -------------- @@ -1095,7 +1109,10 @@ Manage ROS 2 node parameters. The ```` half is a member id, so ``GET``, ``PUT`` and ``DELETE`` of a qualified id are served by the gateway that owns that app, on its own ``/apps/{app_id}/configurations/{param_name}`` route. See - :ref:`member-qualified-ids` for the dispatch and its ``504`` case. + :ref:`member-qualified-ids` for the dispatch and its ``504`` case. The + qualified form is accepted on any entity that has the named member, including + one whose members are all peer-owned and one that runs a single node of its + own - the ids the listing offers are unchanged in either case. ``GET /api/v1/components/{id}/configurations`` List all parameters for an entity. @@ -1148,6 +1165,39 @@ Manage ROS 2 node parameters. ``DELETE /api/v1/components/{id}/configurations`` Reset all parameters to default values. + - **204:** every member of the entity was reset + - **207:** some were not, and the body names each one + + This gateway resets a parameter by calling the parameter service on its own + ROS graph, so a member another gateway runs is not reset by this request. Such + a member is listed in the ``207`` body with ``success: false`` and an error + naming the gateway that owns it, so a caller is never told a reset covered + parameters it did not reach. Reset it on that gateway, through the member's + own ``/apps/{app_id}/configurations`` route. + + .. code-block:: json + + { + "entity_id": "vehicle_health", + "results": [ + { + "node": "/powertrain/engine/calibration", + "app_id": "primary_calibration", + "success": true, + "details": {"reset_count": 2, "failed_count": 0} + }, + { + "app_id": "peer_calibration", + "success": false, + "error": "Not reset here: 'peer_calibration' is owned by gateway 'secondary_gateway'. Reset it on that gateway, through its own /apps/peer_calibration/configurations route." + } + ] + } + + ``details`` carries the per-parameter outcome of the nodes this gateway did + reset, including on an entry that failed - a partial reset names the + parameters it could not restore. + Resource Locking ---------------- diff --git a/docs/config/aggregation.rst b/docs/config/aggregation.rst index 75d02a14d..d4f231192 100644 --- a/docs/config/aggregation.rst +++ b/docs/config/aggregation.rst @@ -396,6 +396,17 @@ becomes PUT /api/v1/apps/peer_calibration/configurations/calibration_offset +A member half is recognised when the text before the first colon names a member +of the addressed entity, so the qualified form works on an aggregating entity +whose members are all owned by peers - a parent gateway that runs no ROS node of +its own - and on one that runs a single node beside peer-owned members. Neither +shape changes the ids the entity's listing offers. + +``DELETE /api/v1/{entity_type}/{id}/configurations`` resets the nodes this +gateway runs. A member another gateway runs is not reset by it, and the response +says so: ``207`` instead of ``204``, with that member named and the gateway that +owns it named with it. + Reachability is answered before anything is forwarded, so a member whose gateway is silent gets ``504 not-responding`` rather than a ``502`` from a failed connection. A member this gateway owns is served here, unchanged. diff --git a/src/ros2_medkit_gateway/README.md b/src/ros2_medkit_gateway/README.md index 8f2af7e5d..5def440f7 100644 --- a/src/ros2_medkit_gateway/README.md +++ b/src/ros2_medkit_gateway/README.md @@ -348,7 +348,23 @@ PUT /api/v1/apps/peer_calibration/configurations/calibration_offset and the write lands on the ROS node that declares the parameter. The `GET /{entity}/configurations` listing is unchanged - peer parameters reach it through the collection fan-out, and the ids it offers are the ids the -single-item routes accept. Reachability is settled first, so a +single-item routes accept. + +The member half is recognised when the text before the first colon names a +member of the addressed entity. How many ROS nodes this gateway resolves for the +entity does not enter into it: a member another gateway runs reports no ROS +binding here, so an entity whose members are all peer-owned resolves none and an +entity running one node of its own can still have peer-owned members beside it. +Both take the same id form as an entity with several local nodes, and neither +changes the ids the listing offers. A prefix naming no member is part of the +parameter name. + +`DELETE /{entity}/configurations` resets the nodes this gateway runs, so a +member another gateway runs is not reset by it. That is reported rather than +implied: `207` instead of `204`, with the member named and the gateway that owns +it named with it. + +Reachability is settled first, so a member whose gateway has gone silent answers `504 not-responding` instead of a `502` from a connection that could not be made. `X-Medkit-No-Fan-Out` bounds the collection fan-out and does not change where a member-qualified request is @@ -392,7 +408,10 @@ health are separate questions. `/configurations` predates this rule and keeps its own: on a multi-node entity every parameter id is `:`, a bare id is refused on write, -and items carry `x-medkit.source` rather than `member_ids`. +and items carry `x-medkit.source` rather than `member_ids`. The node count +decides which ids the listing offers; which ids it accepts is decided by the +member set, so the qualified form works on an entity whose members are all +peer-owned too. ### Component Data Read Endpoints diff --git a/src/ros2_medkit_gateway/design/aggregation.rst b/src/ros2_medkit_gateway/design/aggregation.rst index d1a9bc261..1536a7c24 100644 --- a/src/ros2_medkit_gateway/design/aggregation.rst +++ b/src/ros2_medkit_gateway/design/aggregation.rst @@ -445,6 +445,25 @@ the member half is an entity id, and the item half is the id the member's own route uses. Nothing on the owning gateway is aggregating, so the item half is sent bare - a parameter as its plain name, a topic as its plain path. +Which half is which is decided from the entity's MEMBER SET, not from the number +of ROS nodes the local walk resolves for the entity. The two answer different +questions. A member another gateway runs announces no ROS binding, so it +contributes no node here: the node count measures how much of the entity is +local, and reading an id from it refuses the qualified form on exactly the two +deployments that need it most - an aggregator that runs no node of its own, +where the count is zero, and a gateway running one node beside peer-owned +members, where the count is one. The member set includes what the peers +contributed, so it says the same thing on every deployment. + +A prefix that names no member is part of the parameter name. That is what makes +the rule self-protecting: a parameter whose own name contains a colon stays +addressable, and no id that resolves today moves, because a split only happens +where the prefix matches a real member. An entity's own id is not a member half +of itself - it separates nothing - and a member half with an empty item after it +is not one either, since one path segment shorter is the member's configurations +COLLECTION and answering there would hand a list to a caller that asked for one +value. + An operation's item half is its short name, except where the member carrying it exposes that short name at more than one ROS path. There the member half names one member for both copies and cannot separate them, so the item half is the @@ -483,6 +502,15 @@ The order inside ``dispatch_to_member`` is load-bearing: as the two-argument form applies to the incoming one - the target is assembled from client-supplied ids and is exactly as untrusted. +Reset-all, ``DELETE /{entity}/configurations``, is not a member-qualified +request and does not go through this dispatch: it names no member, and its +members can sit on several gateways at once, so there is no single owner to hand +it to. This gateway resets what it runs, by calling the parameter services on +its own ROS graph. A member owned by a peer is therefore not reset, and the +response says so rather than implying otherwise - ``207`` with that member named +and its owning gateway named with it, instead of a ``204`` that would report a +reset of parameters the entity lists and this request never touched. + The wire is committed by the forward, so the handler returns ``HandlerContext::forwarded_sentinel_error()``: the typed router recognises the ``x-medkit-internal-forwarded`` code and renders nothing, the same channel the diff --git a/src/ros2_medkit_gateway/src/http/handlers/config_handlers.cpp b/src/ros2_medkit_gateway/src/http/handlers/config_handlers.cpp index dc2ba0052..4abfff4e2 100644 --- a/src/ros2_medkit_gateway/src/http/handlers/config_handlers.cpp +++ b/src/ros2_medkit_gateway/src/http/handlers/config_handlers.cpp @@ -14,6 +14,7 @@ #include "ros2_medkit_gateway/core/http/handlers/config_handlers.hpp" +#include #include #include #include @@ -87,21 +88,52 @@ struct ParsedParamId { bool has_prefix{false}; ///< Whether the ID had an app prefix }; -/// Parse param_id which may carry an `app_id:param_name` prefix for -/// aggregated configurations. For aggregated entities the prefix -/// disambiguates which app's parameter is targeted; for non-aggregated -/// entities the colon (if any) is treated as part of the parameter name. -ParsedParamId parse_aggregated_param_id(const std::string & param_id, bool is_aggregated) { +/// Split `param_id` into member and parameter halves at its first colon, when +/// the half before that colon names a member of this entity. +/// +/// An entity id is restricted to alphanumerics, underscore and hyphen, so it +/// can never contain a colon and the first one is the only candidate separator. +/// Whether it IS a separator is decided by the entity's member set, because +/// that is the set the member half draws its meaning from. A count of the nodes +/// this gateway resolves for the entity cannot decide it: a member another +/// gateway runs reports no ROS binding here and so contributes no node, which +/// makes the count a measure of how much of the entity is local and not of how +/// its ids are formed. An entity whose members all belong to peers resolves no +/// node at all, and one that resolves a single node can still have peer-owned +/// members alongside it. +/// +/// A prefix that names no member is part of the parameter name. That is what +/// keeps a parameter whose own name contains a colon addressable, and it is why +/// the entity's own id is not read as a member half: it separates nothing, and +/// accepting it would give a parameter a second id nothing offers. +/// +/// A member half followed by an empty parameter name addresses no parameter. +/// Carried further it would name the member's configurations COLLECTION - the +/// route one path segment shorter - and hand a list back to a caller that asked +/// for one value, so it is left unsplit and misses as an ordinary name. +ParsedParamId address_parameter(const ThreadSafeEntityCache & cache, const EntityInfo & entity, + const std::string & param_id) { ParsedParamId result; result.param_name = param_id; - auto colon_pos = param_id.find(':'); - if (colon_pos != std::string::npos && is_aggregated) { - result.app_id = param_id.substr(0, colon_pos); - result.param_name = param_id.substr(colon_pos + 1); - result.has_prefix = true; + const auto colon_pos = param_id.find(':'); + if (colon_pos == std::string::npos || colon_pos == 0 || colon_pos + 1 == param_id.size()) { + return result; + } + + std::string candidate = param_id.substr(0, colon_pos); + if (candidate == entity.id) { + return result; + } + + const auto members = cache.get_members(entity.sovd_type(), entity.id); + if (std::find(members.begin(), members.end(), candidate) == members.end()) { + return result; } + result.app_id = std::move(candidate); + result.param_name = param_id.substr(colon_pos + 1); + result.has_prefix = true; return result; } @@ -482,8 +514,10 @@ http::Result ConfigHandlers::get_configuration(cons if (!entity_result) { return tl::unexpected(flatten_validator_error(entity_result.error())); } + const auto & entity = *entity_result; - // Parameter ID may be prefixed with app_id: for aggregated configs. + // Bounded before anything is parsed: the id may carry a member half, so the + // cap covers an entity id, the separator and a parameter name. if (param_id.empty() || param_id.length() > kMaxAggregatedParamIdLength) { return tl::unexpected(make_error(400, ERR_INVALID_PARAMETER, "Invalid parameter ID", json{{"details", "Parameter ID is empty or too long"}})); @@ -491,6 +525,15 @@ http::Result ConfigHandlers::get_configuration(cons const auto & cache = ctx_.node()->get_thread_safe_cache(); auto agg_configs = cache.get_entity_configurations(entity_id); + auto parsed = address_parameter(cache, entity, param_id); + + // Ownership is settled before the local node set is consulted, because the + // node set describes this gateway's own graph: an entity all of whose members + // belong to peers resolves none, and refusing on that count would refuse the + // one id the entity can actually answer. + if (auto answered = dispatch_configuration(ctx_, req, entity_id, param_id, parsed)) { + return tl::unexpected(*answered); + } if (agg_configs.nodes.empty()) { return tl::unexpected(make_error(404, ERR_RESOURCE_NOT_FOUND, "No nodes available", @@ -498,14 +541,9 @@ http::Result ConfigHandlers::get_configuration(cons } auto * config_mgr = ctx_.node()->get_configuration_manager(); - auto parsed = parse_aggregated_param_id(param_id, agg_configs.is_aggregated); - - if (auto answered = dispatch_configuration(ctx_, req, entity_id, param_id, parsed)) { - return tl::unexpected(*answered); - } - // If targeting a specific app in an aggregated entity, dispatch to that - // app's node directly. + // The id named a member this gateway owns; read the parameter from that + // member's own node rather than probing every node the entity has. if (parsed.has_prefix) { const auto * node_info = find_node_for_app(agg_configs.nodes, parsed.app_id); if (node_info == nullptr) { @@ -608,14 +646,7 @@ http::Result ConfigHandlers::set_configuration(cons const auto & cache = ctx_.node()->get_thread_safe_cache(); auto agg_configs = cache.get_entity_configurations(entity_id); - - if (agg_configs.nodes.empty()) { - return tl::unexpected(make_error(404, ERR_RESOURCE_NOT_FOUND, "No nodes available", - json{{"entity_id", entity_id}, {"id", param_id}})); - } - - auto * config_mgr = ctx_.node()->get_configuration_manager(); - auto parsed = parse_aggregated_param_id(param_id, agg_configs.is_aggregated); + auto parsed = address_parameter(cache, entity, param_id); // Helper: turn a successful set into the typed response. The wire shape // matches the legacy handler exactly - the response id is the original @@ -626,10 +657,20 @@ http::Result ConfigHandlers::set_configuration(cons return make_read_value(entity_id, node_fqn, param_id, source_app, param_data); }; + // Ownership is settled before the local node set is consulted, for the same + // reason as on the read: the node set counts this gateway's own nodes, and a + // member another gateway runs contributes none of them. if (auto answered = dispatch_configuration(ctx_, req, entity_id, param_id, parsed)) { return tl::unexpected(*answered); } + if (agg_configs.nodes.empty()) { + return tl::unexpected(make_error(404, ERR_RESOURCE_NOT_FOUND, "No nodes available", + json{{"entity_id", entity_id}, {"id", param_id}})); + } + + auto * config_mgr = ctx_.node()->get_configuration_manager(); + if (parsed.has_prefix) { const auto * node_info = find_node_for_app(agg_configs.nodes, parsed.app_id); if (node_info == nullptr) { @@ -693,6 +734,14 @@ http::Result ConfigHandlers::delete_configuration(const http::T const auto & cache = ctx_.node()->get_thread_safe_cache(); auto agg_configs = cache.get_entity_configurations(entity_id); + auto parsed = address_parameter(cache, entity, param_id); + + // Ownership is settled before the local node set is consulted, for the same + // reason as on the read: the node set counts this gateway's own nodes, and a + // member another gateway runs contributes none of them. + if (auto answered = dispatch_configuration(ctx_, req, entity_id, param_id, parsed)) { + return tl::unexpected(*answered); + } if (agg_configs.nodes.empty()) { return tl::unexpected(make_error(404, ERR_RESOURCE_NOT_FOUND, "No nodes available", @@ -700,11 +749,6 @@ http::Result ConfigHandlers::delete_configuration(const http::T } auto * config_mgr = ctx_.node()->get_configuration_manager(); - auto parsed = parse_aggregated_param_id(param_id, agg_configs.is_aggregated); - - if (auto answered = dispatch_configuration(ctx_, req, entity_id, param_id, parsed)) { - return tl::unexpected(*answered); - } if (parsed.has_prefix) { const auto * node_info = find_node_for_app(agg_configs.nodes, parsed.app_id); @@ -764,14 +808,8 @@ ConfigHandlers::delete_all_configurations(const http::TypedRequest & req) { const auto & cache = ctx_.node()->get_thread_safe_cache(); auto agg_configs = cache.get_entity_configurations(entity_id); - if (agg_configs.nodes.empty()) { - // No backing nodes means nothing to reset; SOVD treats this as a success - // with no content (204) - matches legacy behaviour. - return ResultVariant{http::NoContent{}}; - } - auto * config_mgr = ctx_.node()->get_configuration_manager(); - bool all_success = true; + bool all_reset = true; dto::ConfigurationDeleteMultiStatus multi_status; multi_status.entity_id = entity_id; @@ -781,20 +819,65 @@ ConfigHandlers::delete_all_configurations(const http::TypedRequest & req) { dto::ConfigurationDeleteResultItem entry; entry.node = node_info.node_fqn; entry.app_id = node_info.app_id; - if (result.success) { - entry.success = true; - if (result.data.is_object() || result.data.is_array()) { - entry.details = result.data; - } - } else { - all_success = false; - entry.success = false; + entry.success = result.success; + if (!result.success) { + all_reset = false; entry.error = result.error_message; } + // Carried on the failing entry as well as the succeeding one: a partial + // reset names the parameters it could not restore, and dropping that leaves + // the caller a verdict with nothing to act on. + if (result.data.is_object() || result.data.is_array()) { + entry.details = result.data; + } multi_status.results.push_back(std::move(entry)); } - if (all_success) { + // Members whose node another gateway runs. This route resets by calling the + // parameter services on this gateway's own ROS graph, and there is no such + // service for them here - so they are not reset, however the local half went. + // They are named rather than omitted because the entity's configurations + // collection LISTS their parameters: a caller who resets the entity and is + // answered 204 has been told the parameters it can see were restored, and for + // these they were not. A member with no parameters at all is a different case + // and is not reported - nothing was left undone for it. + // + // A member routed to a peer never contributes a local node: an id a peer + // announces that this gateway also declares is renamed `__` by the + // merge, so the loop above and this one cannot name the same member. + if (auto * agg = ctx_.aggregation_manager(); agg != nullptr) { + for (const auto & member : cache.get_members(entity.sovd_type(), entity_id)) { + // Apps only: a parameter belongs to a ROS node, an App is what carries + // one, and a peer-owned Component's Apps are members in their own right. + // Naming the Component too would report the same gap twice. + if (!cache.get_app(member)) { + continue; + } + auto peer = agg->find_peer_for_entity(member); + if (!peer) { + continue; + } + + std::string reason = "Not reset here: '"; + reason += member; + reason += "' is owned by gateway '"; + reason += *peer; + reason += "'. Reset it on that gateway, through its own /apps/"; + reason += member; + reason += "/configurations route."; + + dto::ConfigurationDeleteResultItem entry; + entry.app_id = member; + entry.success = false; + entry.error = std::move(reason); + all_reset = false; + multi_status.results.push_back(std::move(entry)); + } + } + + if (all_reset) { + // Every member this entity has was reset - including the case where it has + // none at all, which SOVD answers as a success with no content. return ResultVariant{http::NoContent{}}; } return ResultVariant{std::move(multi_status)}; diff --git a/src/ros2_medkit_integration_tests/CMakeLists.txt b/src/ros2_medkit_integration_tests/CMakeLists.txt index 942327502..42b7f9036 100644 --- a/src/ros2_medkit_integration_tests/CMakeLists.txt +++ b/src/ros2_medkit_integration_tests/CMakeLists.txt @@ -239,6 +239,7 @@ if(BUILD_TESTING) test_daisy_chain_aggregation test_grouping_entity_aggregation test_aggregate_lock_identity + test_aggregator_only_configurations test_leaf_collision_aggregation test_startup_param_clamp_warnings) set(_MULTI_GATEWAY_DOMAINS 4) diff --git a/src/ros2_medkit_integration_tests/test/features/test_aggregator_only_configurations.test.py b/src/ros2_medkit_integration_tests/test/features/test_aggregator_only_configurations.test.py new file mode 100644 index 000000000..c6014a266 --- /dev/null +++ b/src/ros2_medkit_integration_tests/test/features/test_aggregator_only_configurations.test.py @@ -0,0 +1,757 @@ +# Copyright 2026 bburda +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""End-to-end specification for configurations on peer-owned members. + +An entity whose members are not this gateway's to serve still has to answer for +them. + +test_grouping_entity_aggregation covers the shape where the aggregating entity +has SEVERAL locally resolvable nodes. Two deployments it cannot express are the +ones this file exists for, and both are ordinary: + + AGGREGATOR-ONLY a parent gateway that runs no ROS node of its own. Every + member of the entity belongs to a peer. The local walk + resolves NO node for it, so anything that decides what an + id means from the local node count decides it from zero. + + ONE LOCAL, N PEER a gateway that runs one node and aggregates the rest. The + local walk resolves exactly one node, so anything that + treats "more than one local node" as the mark of an + aggregate concludes this entity is not one. + +Both deployments address a parameter the same way every other member-qualified +id is addressed - ``:`` - and the member half means exactly what +it means everywhere else: the entity that owns the parameter. Whether this +gateway happens to run that member's node decides WHERE the request is served, +never whether the id parses. + +THE RULES + +C1 An entity whose members all belong to peers still lists its members' + parameters, and a read of one returns THAT member's value. Asserting the + status alone cannot show this: the failure mode is a 404 raised before the + id is even looked at, and the fix that removes it can still return a value + read from the wrong node. +C2 A write through such an entity lands on the member's own node. Proven by + reading the value back from the peer gateway directly, so the assertion + never passes through the code path that performed the write. +C3 An entity with one local node and peer members reads the member half, for + both halves it has: the local member and the peer-owned one. +C4 An id that works today keeps working and keeps its shape. A bare parameter + name stays a bare parameter name, and the ids the list offers do not move. + This is the constraint the other four are subordinate to: every deployment + in the field addresses parameters by the ids these entities offer now. +C5 A colon in an id is only a separator when the half before it names a member + of this entity. Anything else is part of the parameter name, which is what + keeps a parameter whose name contains a colon addressable. +C6 Reset-all reports what it reset and what it did not. This gateway resets + the nodes it runs; a member another gateway runs is not reached from here, + and a response that says 204 to a caller whose peer-owned members were + never touched reports a reset that did not happen. +""" + +import os +import tempfile +import time +import unittest +from urllib.parse import quote + +from launch import LaunchDescription +from launch.actions import SetEnvironmentVariable, TimerAction +import launch_ros.actions +import launch_testing.actions +import requests +from ros2_medkit_test_utils.constants import ( + ALLOWED_EXIT_CODES, + API_BASE_PATH, + get_test_domain_id, + get_test_port, +) +from ros2_medkit_test_utils.launch_helpers import create_gateway_node + +PRIMARY_PORT = get_test_port(0) +PEER_PORT = get_test_port(1) +PRIMARY_URL = f'http://localhost:{PRIMARY_PORT}{API_BASE_PATH}' +PEER_URL = f'http://localhost:{PEER_PORT}{API_BASE_PATH}' + +PRIMARY_DOMAIN_ID = get_test_domain_id(0) +PEER_DOMAIN_ID = get_test_domain_id(1) + +# Declared by the calibration demo node and writable, so a value put there by +# one request is observable by another. +CALIBRATION_PARAM = 'calibration_offset' + +# The one node the primary runs. It exists so the ONE-LOCAL-N-PEER shape has a +# local half at all, and so the regression guard has a local entity whose ids +# must not move. +LOCAL_APP = 'local_calibration' +LOCAL_NAMESPACE = '/powertrain/engine' + +# Members that live only on the peer. +PEER_CALIBRATION_APP = 'remote_calibration' +PEER_PRESSURE_APP = 'remote_pressure' +PEER_NAMESPACE = '/chassis/brakes' + +# Declared on BOTH gateways, so it merges rather than being routed whole to the +# peer: a Function only one gateway declares gets a routing entry and the whole +# request is handed over, which is a different code path and not the one under +# test. The primary declares no hosts for it - its whole membership arrives +# from the peer, which is what makes it aggregator-only. +AGGREGATOR_ONLY_FUNCTION = 'remote_health' + +# Declared on the primary alone, hosting one local App and one peer-owned one. +MIXED_FUNCTION = 'mixed_health' + +PRIMARY_COMPONENT = 'primary-ecu' +PEER_COMPONENT = 'remote-ecu' + +PRIMARY_MANIFEST = f"""\ +manifest_version: "1.0" +metadata: + name: "Aggregating ECU" + version: "1.0.0" +config: + unmanifested_nodes: ignore +components: + - id: {PRIMARY_COMPONENT} + name: "Primary ECU" +apps: + - id: {LOCAL_APP} + name: "Local Calibration Service" + is_located_on: {PRIMARY_COMPONENT} + ros_binding: + node_name: calibration + namespace: {LOCAL_NAMESPACE} +functions: + # No hosts of its own. The peer declares the same id with its two Apps, the + # two declarations merge, and every member of the merged Function is then + # peer-owned. + - id: {AGGREGATOR_ONLY_FUNCTION} + name: "Remote Health Monitoring" + category: monitoring + - id: {MIXED_FUNCTION} + name: "Mixed Health Monitoring" + category: monitoring + hosted_by: + - {LOCAL_APP} + - {PEER_CALIBRATION_APP} +""" + +PEER_MANIFEST = f"""\ +manifest_version: "1.0" +metadata: + name: "Remote ECU" + version: "1.0.0" +config: + unmanifested_nodes: ignore +components: + - id: {PEER_COMPONENT} + name: "Remote ECU" +apps: + - id: {PEER_CALIBRATION_APP} + name: "Remote Calibration Service" + is_located_on: {PEER_COMPONENT} + ros_binding: + node_name: calibration + namespace: {PEER_NAMESPACE} + - id: {PEER_PRESSURE_APP} + name: "Remote Brake Pressure Sensor" + is_located_on: {PEER_COMPONENT} + ros_binding: + node_name: pressure_sensor + namespace: {PEER_NAMESPACE} +functions: + # The Component is a host alongside its own Apps. A Function is + # cross-component by definition and hosting one is ordinary, and it puts a + # member in the set that carries no parameters of its own - the case a + # per-member report has to leave out, because its Apps already carry the + # parameters and naming it too would report the same gap twice. + - id: {AGGREGATOR_ONLY_FUNCTION} + name: "Remote Health Monitoring" + category: monitoring + hosted_by: + - {PEER_CALIBRATION_APP} + - {PEER_PRESSURE_APP} + - {PEER_COMPONENT} +""" + + +def _write_manifest(content): + """Write manifest YAML to a temporary file and return its path.""" + fd, path = tempfile.mkstemp(suffix='.yaml', prefix='test_aggregator_only_manifest_') + with os.fdopen(fd, 'w') as handle: + handle.write(content) + return path + + +def generate_test_description(): + primary_manifest_path = _write_manifest(PRIMARY_MANIFEST) + peer_manifest_path = _write_manifest(PEER_MANIFEST) + + peer_domain_env = {'ROS_DOMAIN_ID': str(PEER_DOMAIN_ID)} + + primary_gateway = create_gateway_node( + port=PRIMARY_PORT, + extra_params={ + 'discovery.mode': 'hybrid', + 'discovery.manifest_path': primary_manifest_path, + 'discovery.manifest_strict_validation': False, + 'aggregation.enabled': True, + 'aggregation.timeout_ms': 5000, + 'aggregation.announce': False, + 'aggregation.discover': False, + 'aggregation.peer_urls': [f'http://localhost:{PEER_PORT}'], + 'aggregation.peer_names': ['remote_gateway'], + }, + ) + + peer_gateway = create_gateway_node( + name='remote_gateway_node', + port=PEER_PORT, + extra_params={ + 'discovery.mode': 'hybrid', + 'discovery.manifest_path': peer_manifest_path, + 'discovery.manifest_strict_validation': False, + }, + extra_env=peer_domain_env, + ) + + # Built inline rather than through create_demo_nodes because the registry + # binds each key to one fixed namespace, and both gateways need a + # calibration node in a namespace of their own. + local_calibration = launch_ros.actions.Node( + package='ros2_medkit_integration_tests', + executable='demo_calibration_service', + name='calibration', + namespace=LOCAL_NAMESPACE, + output='screen', + ) + peer_calibration = launch_ros.actions.Node( + package='ros2_medkit_integration_tests', + executable='demo_calibration_service', + name='calibration', + namespace=PEER_NAMESPACE, + output='screen', + additional_env=peer_domain_env, + ) + peer_pressure = launch_ros.actions.Node( + package='ros2_medkit_integration_tests', + executable='demo_brake_pressure_sensor', + name='pressure_sensor', + namespace=PEER_NAMESPACE, + output='screen', + additional_env=peer_domain_env, + ) + + delayed = TimerAction( + period=2.0, + actions=[local_calibration, peer_calibration, peer_pressure], + ) + + launch_description = LaunchDescription([ + SetEnvironmentVariable('ROS_DOMAIN_ID', str(PRIMARY_DOMAIN_ID)), + primary_gateway, + peer_gateway, + delayed, + launch_testing.actions.ReadyToTest(), + ]) + + return ( + launch_description, + {'gateway_node': primary_gateway, 'peer_gateway': peer_gateway}, + ) + + +class AggregatorOnlyConfigurationsTest(unittest.TestCase): + """Drives the aggregating gateway; the peer is only ever used to verify.""" + + @classmethod + def setUpClass(cls): + # A manifest App exists before its node does, and an App with no live + # binding contributes no parameters, so presence alone would let a + # collection read run against an entity that is legitimately empty. + cls._wait_for_apps(PRIMARY_URL, {LOCAL_APP}, 'primary') + cls._wait_for_apps( + PEER_URL, {PEER_CALIBRATION_APP, PEER_PRESSURE_APP}, 'peer') + cls._wait_until_merged() + + @classmethod + def _wait_for_apps(cls, base_url, required, label): + """Block until `required` Apps are present AND bound to a live node.""" + deadline = time.monotonic() + 60.0 + while time.monotonic() < deadline: + try: + response = requests.get(f'{base_url}/apps', timeout=5) + if response.status_code == 200: + online = { + item.get('id') + for item in response.json().get('items', []) + if item.get('x-medkit', {}).get('is_online') + } + if required <= online: + return + except requests.RequestException: + pass + time.sleep(1.0) + raise AssertionError(f'{label}: {required} not online within 60s') + + @classmethod + def _wait_until_merged(cls): + """Block until the peer's members are visible on the primary.""" + deadline = time.monotonic() + 60.0 + while time.monotonic() < deadline: + try: + response = requests.get(f'{PRIMARY_URL}/apps', timeout=5) + if response.status_code == 200: + ids = { + item.get('id') for item in response.json().get('items', []) + } + if {PEER_CALIBRATION_APP, PEER_PRESSURE_APP} <= ids: + return + except requests.RequestException: + pass + time.sleep(0.5) + raise AssertionError("the peer's Apps did not merge into the primary in 60s") + + # ------------------------------------------------------------------ helpers + + def _items(self, entity_path, collection): + response = requests.get( + f'{PRIMARY_URL}/{entity_path}/{collection}', timeout=15) + self.assertEqual(response.status_code, 200, response.text) + return response.json().get('items', []) + + @staticmethod + def _config_url(base_url, entity_path, config_id): + return ( + f'{base_url}/{entity_path}/configurations/{quote(config_id, safe="")}' + ) + + def _seed_on_peer(self, app_id, value): + """Put a value on one peer App through THE PEER's own gateway. + + The value the aggregate is then asked for was written by a request that + never went through the aggregate, so a read that returns it cannot have + got it from the write path under test. + """ + response = requests.put( + self._config_url(PEER_URL, f'apps/{app_id}', CALIBRATION_PARAM), + json={'data': value}, + timeout=15, + ) + self.assertEqual( + response.status_code, 200, + f'could not seed {app_id}.{CALIBRATION_PARAM} on the peer: ' + f'{response.status_code} {response.text}', + ) + + def _read_on_peer(self, app_id): + """Read a value from one peer App through THE PEER's own gateway.""" + response = requests.get( + self._config_url(PEER_URL, f'apps/{app_id}', CALIBRATION_PARAM), + timeout=15, + ) + self.assertEqual( + response.status_code, 200, + f'could not read {app_id}.{CALIBRATION_PARAM} on the peer: ' + f'{response.status_code} {response.text}', + ) + return response.json().get('data') + + def _seed_locally(self, value): + """Put a value on the local App through its own route on the primary.""" + response = requests.put( + self._config_url(PRIMARY_URL, f'apps/{LOCAL_APP}', CALIBRATION_PARAM), + json={'data': value}, + timeout=15, + ) + self.assertEqual( + response.status_code, 200, + f'could not seed {LOCAL_APP}.{CALIBRATION_PARAM}: ' + f'{response.status_code} {response.text}', + ) + + # ----------------------------------------------------------------- C1 (list) + + def test_a1_an_aggregator_only_entity_lists_its_peer_members_parameters(self): + """C1, the listing half. + + The entity resolves no local node at all. Everything it can offer comes + from members another gateway runs, so a listing that answers from the + local node set alone has nothing to say and the collection looks empty - + or, worse, refuses outright. + """ + items = self._items(f'functions/{AGGREGATOR_ONLY_FUNCTION}', 'configurations') + ids = [item.get('id') for item in items] + self.assertIn( + f'{PEER_CALIBRATION_APP}:{CALIBRATION_PARAM}', ids, + f'an aggregator-only entity offered no parameter of its members: {ids}', + ) + + # ------------------------------------------------------------- C1 (single read) + + def test_a2_a_read_on_an_aggregator_only_entity_returns_the_members_value(self): + """C1, the read half, asserted on the value and on who served it. + + A 200 says only that something answered. The seeded value says the + answer came from the member's own node, and `x-medkit.entity_id` says + which entity produced it - the member itself, because the member's own + gateway answered on the member's own route. + """ + self._seed_on_peer(PEER_CALIBRATION_APP, 4.25) + + config_id = f'{PEER_CALIBRATION_APP}:{CALIBRATION_PARAM}' + response = requests.get( + self._config_url( + PRIMARY_URL, f'functions/{AGGREGATOR_ONLY_FUNCTION}', config_id), + timeout=15, + ) + self.assertEqual(response.status_code, 200, response.text) + body = response.json() + self.assertAlmostEqual( + body.get('data'), 4.25, places=6, + msg=f'the value did not come from the member that holds it: {body}', + ) + self.assertEqual( + body.get('x-medkit', {}).get('entity_id'), PEER_CALIBRATION_APP, + f'the read was not served by the member that owns the parameter: {body}', + ) + + # ------------------------------------------------------------------ C2 (write) + + def test_a3_a_write_on_an_aggregator_only_entity_lands_on_the_peer(self): + """C2: the write reaches the member's own node. + + Read back from the PEER gateway, not from the aggregate. A write that + was accepted and dropped, or applied to some other node, still returns + 200 here and still reads back through the aggregate if the aggregate + answers from whatever it wrote. + """ + self._seed_on_peer(PEER_CALIBRATION_APP, 0.0) + + config_id = f'{PEER_CALIBRATION_APP}:{CALIBRATION_PARAM}' + response = requests.put( + self._config_url( + PRIMARY_URL, f'functions/{AGGREGATOR_ONLY_FUNCTION}', config_id), + json={'data': 7.5}, + timeout=15, + ) + self.assertEqual(response.status_code, 200, response.text) + + self.assertAlmostEqual( + self._read_on_peer(PEER_CALIBRATION_APP), 7.5, places=6, + msg='the write did not reach the peer that runs the member', + ) + + def test_a3b_a_reset_of_one_parameter_reaches_the_peer_owned_member(self): + """C2 for the third method the single-item route carries. + + Reset takes the same id and the same owner as read and write, so it has + to reach the same node. Checked by reading the value back from the peer: + the parameter is at the default its node declared, which is a state only + a reset on that node produces. + """ + self._seed_on_peer(PEER_CALIBRATION_APP, 9.75) + + config_id = f'{PEER_CALIBRATION_APP}:{CALIBRATION_PARAM}' + response = requests.delete( + self._config_url( + PRIMARY_URL, f'functions/{AGGREGATOR_ONLY_FUNCTION}', config_id), + timeout=15, + ) + self.assertEqual(response.status_code, 204, response.text) + + self.assertAlmostEqual( + self._read_on_peer(PEER_CALIBRATION_APP), 0.0, places=6, + msg='the reset did not reach the peer that runs the member', + ) + + # ------------------------------------------------------------------------- C3 + + def test_a4_one_local_node_plus_peer_members_still_reads_the_member_half(self): + """C3: the shape whose local node count is one. + + One local node is not "not aggregating" - it is an aggregate with one + local member. Both halves have to resolve: the peer-owned member, whose + node this gateway cannot see, and the local one, whose node it runs. + """ + self._seed_on_peer(PEER_CALIBRATION_APP, 3.5) + self._seed_locally(-1.25) + + for member, expected in ( + (PEER_CALIBRATION_APP, 3.5), + (LOCAL_APP, -1.25), + ): + with self.subTest(member=member): + config_id = f'{member}:{CALIBRATION_PARAM}' + response = requests.get( + self._config_url( + PRIMARY_URL, f'functions/{MIXED_FUNCTION}', config_id), + timeout=15, + ) + self.assertEqual(response.status_code, 200, response.text) + body = response.json() + self.assertAlmostEqual( + body.get('data'), expected, places=6, + msg=f"{config_id} did not return {member}'s value: {body}", + ) + + # ------------------------------------------------------------------------- C4 + + def test_a5_a_bare_parameter_name_and_the_ids_the_list_offers_do_not_move(self): + """C4, the constraint every other rule here is subordinate to. + + An App is a single-node entity and its parameter ids are bare names. + Nothing about member addressing may reach that: a client holding + `calibration_offset` today must still be able to send it, and the list + must still offer it in that form. + """ + self._seed_locally(2.5) + + items = self._items(f'apps/{LOCAL_APP}', 'configurations') + ids = [item.get('id') for item in items] + self.assertIn( + CALIBRATION_PARAM, ids, + f'the App no longer offers its parameter under its bare name: {ids}', + ) + self.assertFalse( + [i for i in ids if ':' in str(i)], + f'a single-node entity qualified its parameter ids: {ids}', + ) + + response = requests.get( + self._config_url(PRIMARY_URL, f'apps/{LOCAL_APP}', CALIBRATION_PARAM), + timeout=15, + ) + self.assertEqual(response.status_code, 200, response.text) + self.assertAlmostEqual(response.json().get('data'), 2.5, places=6) + + def test_a6_a_bare_name_on_an_entity_with_members_still_reads_its_local_node(self): + """C4 on the aggregating entity, where the temptation to qualify is. + + The mixed Function resolves one local node, so the ids it offers for + that node are bare today. Splitting on any colon, or qualifying every + id because the entity has members, would move them. + """ + self._seed_locally(6.75) + + items = self._items(f'functions/{MIXED_FUNCTION}', 'configurations') + local_ids = [ + item.get('id') for item in items + if str(item.get('id', '')).endswith(CALIBRATION_PARAM) + ] + self.assertIn( + CALIBRATION_PARAM, local_ids, + f"the local member's parameter id moved off its bare form: {local_ids}", + ) + + response = requests.get( + self._config_url( + PRIMARY_URL, f'functions/{MIXED_FUNCTION}', CALIBRATION_PARAM), + timeout=15, + ) + self.assertEqual(response.status_code, 200, response.text) + self.assertAlmostEqual(response.json().get('data'), 6.75, places=6) + + # ------------------------------------------------------------------------- C5 + + def test_a7_a_colon_that_names_no_member_stays_part_of_the_parameter_name(self): + """C5: a prefix that is not a member is not a member half. + + `not_a_member` names nothing in this entity, so the whole string is a + parameter name - one no node declares, hence a miss on the parameter + and not on the member. The two are distinguishable on the wire and have + to stay so: reporting a bad parameter name as an unreachable member + sends the caller looking for a gateway that is not down. + """ + config_id = f'not_a_member:{CALIBRATION_PARAM}' + response = requests.get( + self._config_url(PRIMARY_URL, f'functions/{MIXED_FUNCTION}', config_id), + timeout=15, + ) + self.assertEqual(response.status_code, 404, response.text) + body = response.json() + # The message is what discriminates. A split id that named nothing is + # refused as a missing member or, for a member whose gateway is silent, + # as `not-responding`; an unsplit one is refused as a parameter the + # nodes do not declare, which is what this id is. + self.assertEqual( + body.get('message'), 'Parameter not found', + f'the prefix was read as a member half: {body}', + ) + self.assertEqual( + body.get('parameters', {}).get('id'), config_id, + f'the id was not carried back whole: {body}', + ) + + def test_a7a_a_member_half_with_no_parameter_after_it_addresses_nothing(self): + """C5: a member half alone names no parameter. + + One path segment shorter is the member's own configurations COLLECTION. + Read as a member half, this id would be re-addressed there and a caller + that asked for one value would be handed a list - with 200 on it, so + nothing downstream would notice. + """ + config_id = f'{PEER_CALIBRATION_APP}:' + response = requests.get( + self._config_url( + PRIMARY_URL, f'functions/{AGGREGATOR_ONLY_FUNCTION}', config_id), + timeout=15, + ) + self.assertNotEqual( + response.status_code, 200, + f'an id naming no parameter was answered: {response.text}', + ) + self.assertNotIn( + 'items', response.json(), + f'a single-value read was answered with a collection: {response.text}', + ) + + def test_a7b_an_entity_is_not_a_member_half_of_itself(self): + """C5: the entity's own id separates nothing. + + `local_calibration:calibration_offset` on the App `local_calibration` + would name the same entity twice. Reading it as a member half gives the + parameter a second id that no listing offers, and every client that + holds an id would then have two ways to spell it and one of them + undocumented. + """ + config_id = f'{LOCAL_APP}:{CALIBRATION_PARAM}' + response = requests.get( + self._config_url(PRIMARY_URL, f'apps/{LOCAL_APP}', config_id), + timeout=15, + ) + self.assertEqual( + response.status_code, 404, + f'an entity answered to its own id as a member half: {response.text}', + ) + + # ------------------------------------------------------------------------- C6 + + def test_a8_reset_all_reports_the_members_it_did_not_reach(self): + """C6: reset-all tells the truth about its own reach. + + This gateway resets the nodes it runs. A member another gateway runs is + not reached from here, and the caller has to be told - a plain success + for an entity whose peer-owned half was never touched is a reset the + caller believes happened. + + Both claims are checked against what actually happened: the local + member's parameter is read back and is at its default, and the peer + member's is read back from the peer and is untouched. + """ + self._seed_on_peer(PEER_CALIBRATION_APP, 5.5) + self._seed_locally(5.5) + + response = requests.delete( + f'{PRIMARY_URL}/functions/{MIXED_FUNCTION}/configurations', timeout=20) + self.assertEqual( + response.status_code, 207, + f'reset-all reported plain success for an entity it could not fully ' + f'reset: {response.status_code} {response.text}', + ) + results = response.json().get('results', []) + by_member = {entry.get('app_id'): entry for entry in results} + + # The member this gateway runs is named, and named as attempted here: + # its entry carries the ROS node the reset was addressed to. + self.assertIn( + LOCAL_APP, by_member, + f'the member this gateway runs is not named: {results}') + self.assertTrue( + by_member[LOCAL_APP].get('node'), + f"the local member's entry names no node: {by_member[LOCAL_APP]}", + ) + # What the reset did on that node, per parameter. A verdict on its own + # leaves the caller nothing to act on, and the entry is the only place + # this ever appears. + self.assertIsInstance( + by_member[LOCAL_APP].get('details'), dict, + f"the local member's entry reports no per-parameter outcome: " + f'{by_member[LOCAL_APP]}', + ) + + # The member another gateway runs is named as not reset, and the entry + # says who does own it - without that the caller has nowhere to go. + self.assertIn( + PEER_CALIBRATION_APP, by_member, + f'the member this gateway did not reach is not named: {results}') + peer_entry = by_member[PEER_CALIBRATION_APP] + self.assertIs(peer_entry.get('success'), False, peer_entry) + self.assertIn( + 'remote_gateway', str(peer_entry.get('error', '')), + f'the entry does not say which gateway owns the member: {peer_entry}', + ) + + # Both claims are checked against reality, so the test fails if the + # response lies in either direction. + response = requests.get( + self._config_url(PRIMARY_URL, f'apps/{LOCAL_APP}', CALIBRATION_PARAM), + timeout=15, + ) + self.assertEqual(response.status_code, 200, response.text) + self.assertAlmostEqual( + response.json().get('data'), 0.0, places=6, + msg='the local member was named as reset here, but its value stands', + ) + self.assertAlmostEqual( + self._read_on_peer(PEER_CALIBRATION_APP), 5.5, places=6, + msg='the response said the peer member was not reset, but it was', + ) + + def test_a9_reset_all_on_an_aggregator_only_entity_is_not_plain_success(self): + """C6 where NOTHING is reachable from here. + + Every member belongs to a peer, so this gateway resets nothing at all. + That is the case where a 204 is most misleading, because it reports a + complete reset of an entity nothing was done to. + """ + self._seed_on_peer(PEER_CALIBRATION_APP, 8.25) + + response = requests.delete( + f'{PRIMARY_URL}/functions/{AGGREGATOR_ONLY_FUNCTION}/configurations', + timeout=20, + ) + self.assertEqual( + response.status_code, 207, + f'an entity with no locally resettable member reported a complete ' + f'reset: {response.status_code} {response.text}', + ) + results = response.json().get('results', []) + not_reached = { + entry.get('app_id') for entry in results + if entry.get('success') is False + } + self.assertEqual( + not_reached, {PEER_CALIBRATION_APP, PEER_PRESSURE_APP}, + f'the members that were not reached are not named: {results}', + ) + + self.assertAlmostEqual( + self._read_on_peer(PEER_CALIBRATION_APP), 8.25, places=6, + msg='the response said nothing was reset, but the peer value moved', + ) + + +@launch_testing.post_shutdown_test() +class TestShutdown(unittest.TestCase): + + def test_exit_codes(self, proc_info): + """Check all processes exited cleanly (SIGTERM allowed).""" + for info in proc_info: + self.assertIn( + info.returncode, ALLOWED_EXIT_CODES, + f'{info.process_name} exited with code {info.returncode}' + ) From ea062c90d7061e84b45d417e5b2363db25af8ec3 Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Thu, 20 Aug 2026 11:43:09 +0200 Subject: [PATCH 09/22] fix(operations): resolve an operation id the same way on every route Listing executions resolved nothing. It joined a Component's namespace to the id to guess a ROS path, and for an App it scanned actions alone and only for a bare name, so a member-qualified or path-shaped id never matched and a service answered "entity not found" for an entity that plainly exists. The route also skipped the entity validation its siblings perform. Reading an operation returned whichever of several matched first, while executing the same id was refused. Both now go through one construction, and the answer names the ids that do address them. A service answers with an empty execution collection: it completes inside its own request and creates no execution resource. --- docs/api/rest.rst | 33 +- src/ros2_medkit_gateway/README.md | 14 +- .../design/aggregation.rst | 13 + .../src/http/handlers/operation_handlers.cpp | 222 ++++++---- .../test/test_operation_handlers.cpp | 185 ++++++++- .../CMakeLists.txt | 2 +- src/ros2_medkit_integration_tests/README.md | 1 + .../demo_nodes/dual_calibration_service.cpp | 114 +++++- .../test_grouping_entity_aggregation.test.py | 386 +++++++++++++++++- 9 files changed, 884 insertions(+), 86 deletions(-) diff --git a/docs/api/rest.rst b/docs/api/rest.rst index 374b31c63..cea5b34ee 100644 --- a/docs/api/rest.rst +++ b/docs/api/rest.rst @@ -678,8 +678,33 @@ What this means for a request: does not provide - which is what tells an absent item apart from an item that exists and currently carries no data. A member half followed by nothing names no item and is ``404`` as well. -- Reads are permissive: ``GET`` of a bare id returns the first match rather - than refusing, which is the behaviour every existing client depends on. +- ``GET /{entity}/operations/{id}`` and + ``GET /{entity}/operations/{id}/executions`` refuse exactly what the execution + refuses, with the same body. Reading an operation under an id that names + several of them would describe one without saying which, and the same id is a + ``400`` the moment the caller runs it. The collection never offers such an id, + so only a stale one arrives, and it leaves with ``parameters.operation_ids``. + An unambiguous bare id reads and lists exactly as before. + +Listing the Executions of an Operation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``GET /{entity}/operations/{id}/executions`` resolves ``{id}`` by the rule +above, so a member-qualified id and a ROS-path id both work, and the answer has +three distinct forms: + +- an **action** returns the goals it holds, newest first; +- a **service** returns ``200`` with an empty ``items`` array. A service call + completes inside its own ``POST`` and leaves no execution resource, so its + collection exists and is permanently empty. Whether an operation can ever have + executions is read from ``asynchronous_execution`` on the operation itself; +- an id that names **no operation** is ``404 operation-not-found``, and an + unknown member half is ``404 resource-not-found`` naming that half - so a typo + is never answered as an operation that simply has not been run. + +Goals live on the gateway that sent them, so an id naming a peer-owned member is +dispatched to that member's own route exactly as the ``POST`` was. A goal +started through an aggregating entity is therefore listed through it too. Where a Member-Qualified Request is Served ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -933,7 +958,9 @@ Execute Operations } ``GET /api/v1/components/{id}/operations/{operation_id}/executions`` - List all executions for an operation. + List all executions for an operation. Actions return their goals; a service + returns an empty ``items`` array, because a service call leaves no execution + resource behind. An id naming no operation is ``404``. ``GET /api/v1/components/{id}/operations/{operation_id}/executions/{execution_id}`` Get execution status and result. diff --git a/src/ros2_medkit_gateway/README.md b/src/ros2_medkit_gateway/README.md index 5def440f7..99f0d3c81 100644 --- a/src/ros2_medkit_gateway/README.md +++ b/src/ros2_medkit_gateway/README.md @@ -52,7 +52,7 @@ All endpoints are prefixed with `/api/v1` for API versioning. - `GET /api/v1/components/{component_id}/operations` - List all services and actions for a component - `GET /api/v1/components/{component_id}/operations/{operation_id}` - Get operation details - `POST /api/v1/components/{component_id}/operations/{operation_id}/executions` - Execute operation (call service or send action goal) -- `GET /api/v1/components/{component_id}/operations/{operation_id}/executions` - List all executions for an operation +- `GET /api/v1/components/{component_id}/operations/{operation_id}/executions` - List all executions for an operation (empty for a service, which leaves no execution resource) - `GET /api/v1/components/{component_id}/operations/{operation_id}/executions/{execution_id}` - Get execution status - `DELETE /api/v1/components/{component_id}/operations/{operation_id}/executions/{execution_id}` - Cancel action execution @@ -309,7 +309,17 @@ unchanged - a ROS path carries no colon. item half that member does not provide, or a member half followed by nothing, is `404` - which is what tells an absent item apart from one that exists and carries no data. -- `GET` of a bare id stays permissive and returns the first match. +- `GET /{entity}/operations/{id}` and `GET /{entity}/operations/{id}/executions` + refuse exactly what the execution refuses, with the same body. An unambiguous + bare id reads and lists unchanged. + +`GET /{entity}/operations/{id}/executions` resolves the id by the same rule, so +a member-qualified id and a ROS-path id both work, and has three distinct +answers: an action returns its goals, a service returns `200` with an empty +`items` array (a service call completes inside its `POST` and leaves no +execution resource), and an id naming no operation is `404`. Goals live on the +gateway that sent them, so an id naming a peer-owned member is dispatched to +that member's own route exactly as the `POST` was. A member-qualified request is served by the gateway that owns that member, on the member's own entity route. An aggregating entity holds nothing itself and diff --git a/src/ros2_medkit_gateway/design/aggregation.rst b/src/ros2_medkit_gateway/design/aggregation.rst index 1536a7c24..6f0e88ef9 100644 --- a/src/ros2_medkit_gateway/design/aggregation.rst +++ b/src/ros2_medkit_gateway/design/aggregation.rst @@ -476,6 +476,19 @@ name: POST /api/v1/components/vehicle-ecu/operations/dual_calibration:testrig/dual/left/calibrate/executions -> POST /api/v1/apps/dual_calibration/operations/testrig/dual/left/calibrate/executions +The executions of an operation are addressed the same way, and dispatched for +the same reason. A goal lives on the gateway that sent it - the one the ``POST`` +was dispatched to - so listing it has to reach that gateway too: + +.. code-block:: text + + GET /api/v1/functions/vehicle_health/operations/peer_long_calibration:long_calibration/executions + -> GET /api/v1/apps/peer_long_calibration/operations/long_calibration/executions + +Answering that from the aggregator's own goal tracking returns an empty +collection for goals that exist, which reads as "this operation has never been +run". + The member's own gateway is the only one that can answer: the ROS service, the topic and the parameter behind the id exist on its graph and nowhere else. What this gateway holds for a peer-owned member is a declaration, which is why the diff --git a/src/ros2_medkit_gateway/src/http/handlers/operation_handlers.cpp b/src/ros2_medkit_gateway/src/http/handlers/operation_handlers.cpp index ced5b4f60..543d1dbab 100644 --- a/src/ros2_medkit_gateway/src/http/handlers/operation_handlers.cpp +++ b/src/ros2_medkit_gateway/src/http/handlers/operation_handlers.cpp @@ -229,6 +229,58 @@ std::vector distinct_members(const std::vector & ma return members; } +/// The refusal `parsed` earns on this entity, or nothing when it names at most +/// one operation. +/// +/// One construction for every route that resolves an operation id. A route that +/// accepted an id another route refuses would serve one of several operations +/// without ever saying which, and two separately built messages for the same +/// collision drift apart, so the caller is told a different remedy depending on +/// which verb it used. +/// +/// `parameters.operation_ids` carries the ids that DO address what collided, +/// built by the rule the collection lists them under, so a caller sends one of +/// them back instead of deriving the form itself. +std::optional refuse_if_ambiguous(const AggregatedOperations & ops, const http::MemberQualifiedId & parsed, + const std::string & entity_id, const std::string & operation_id) { + const std::vector matches = matching_operations(ops, parsed); + if (matches.size() < 2) { + return std::nullopt; + } + + std::vector paths; + paths.reserve(matches.size()); + for (const auto & match : matches) { + paths.push_back(match.full_path); + } + const std::vector members = distinct_members(matches); + json params{{"entity_id", entity_id}, {"operation_id", operation_id}, {"ros2_paths", paths}}; + if (!members.empty()) { + params["member_ids"] = members; + } + + const auto addressed_by_path = http::operation_paths_addressed_by_path(ops); + std::vector addressable; + addressable.reserve(matches.size()); + for (const auto & match : matches) { + std::string item = http::operation_item_half(parsed.item_id, match.full_path, addressed_by_path); + if (ops.is_aggregated && !match.member_id.empty()) { + item = http::make_member_qualified_id(match.member_id, item); + } + addressable.push_back(std::move(item)); + } + params["operation_ids"] = addressable; + + if (members.size() > 1) { + params["details"] = "Use format 'member_id:operation_id' to name the member that runs it"; + return make_error(400, ERR_INVALID_REQUEST, "Ambiguous operation id: more than one member provides it", params); + } + params["details"] = + "One provider exposes this short name at more than one ROS path; address the one you mean by that " + "path, without its leading slash"; + return make_error(400, ERR_INVALID_REQUEST, "Ambiguous operation id: it names more than one operation", params); +} + /// True for a member that is in the tree but whose gateway is silent. /// /// A retained member is kept precisely so that the answer to a request does not @@ -734,6 +786,15 @@ http::Result OperationHandlers::get_operation(const http:: json{{"entity_id", entity_id}, {"operation_id", operation_id}})); } + // A read refuses exactly what an execution refuses. Describing one of several + // operations under an id that names them all tells the caller it holds an + // address it does not hold, and the next thing it does with that id is run + // it - where the same id is a 400. The collection no longer offers such an + // id, so only a stale one arrives here, and it leaves with the ids that work. + if (auto ambiguous = refuse_if_ambiguous(ops, parsed, entity_id, operation_id); ambiguous.has_value()) { + return tl::make_unexpected(std::move(*ambiguous)); + } + auto data_access_mgr = ctx_.node()->get_data_access_manager(); auto type_introspection = data_access_mgr->get_type_introspection(); @@ -877,43 +938,8 @@ OperationHandlers::create_execution(const http::TypedRequest & req, dto::Executi // one member that uses the same short name at two ROS paths is still not // identified by it - there the item half has to be the ROS path, which is the // form the collection offers for exactly those copies. - const std::vector matches = matching_operations(ops, parsed); - if (matches.size() > 1) { - std::vector paths; - paths.reserve(matches.size()); - for (const auto & match : matches) { - paths.push_back(match.full_path); - } - const std::vector members = distinct_members(matches); - json params{{"entity_id", entity_id}, {"operation_id", operation_id}, {"ros2_paths", paths}}; - if (!members.empty()) { - params["member_ids"] = members; - } - // The ids that DO address the operations this one collided with, built by - // the rule the collection lists them under. A refusal that only describes - // the form leaves the caller to re-derive it, and a caller that derives it - // differently is refused again for a reason the answer already knew. - const auto addressed_by_path = http::operation_paths_addressed_by_path(ops); - std::vector addressable; - addressable.reserve(matches.size()); - for (const auto & match : matches) { - std::string item = http::operation_item_half(parsed.item_id, match.full_path, addressed_by_path); - if (ops.is_aggregated && !match.member_id.empty()) { - item = http::make_member_qualified_id(match.member_id, item); - } - addressable.push_back(std::move(item)); - } - params["operation_ids"] = addressable; - if (members.size() > 1) { - params["details"] = "Use format 'member_id:operation_id' to name the member that runs it"; - return tl::make_unexpected( - make_error(400, ERR_INVALID_REQUEST, "Ambiguous operation id: more than one member provides it", params)); - } - params["details"] = - "One provider exposes this short name at more than one ROS path; address the one you mean by that " - "path, without its leading slash"; - return tl::make_unexpected( - make_error(400, ERR_INVALID_REQUEST, "Ambiguous operation id: it names more than one operation", params)); + if (auto ambiguous = refuse_if_ambiguous(ops, parsed, entity_id, operation_id); ambiguous.has_value()) { + return tl::make_unexpected(std::move(*ambiguous)); } // Whoever ends up owning the resolved operation must be reachable, and must @@ -1033,48 +1059,104 @@ http::Result> OperationHandlers::list_executio } const std::string operation_id = *op_id_result; - if (auto vr = ctx_.validate_entity_id(entity_id); !vr) { - return tl::make_unexpected(make_error(400, ERR_INVALID_PARAMETER, "Invalid entity ID", - json{{"details", vr.error()}, {"entity_id", entity_id}})); + auto entity_result = ctx_.validate_entity_for_route(req, entity_id); + if (!entity_result) { + return tl::make_unexpected(flatten_validator_error(entity_result.error())); } + const auto entity_info = *entity_result; - const auto & cache = ctx_.node()->get_thread_safe_cache(); - std::string namespace_path; - bool entity_found = false; - - if (auto component = cache.get_component(entity_id)) { - namespace_path = component->namespace_path; - entity_found = true; - } - if (!entity_found) { - if (auto app = cache.get_app(entity_id)) { - for (const auto & act : app->actions) { - if (act.name == operation_id) { - namespace_path = act.full_path.substr(0, act.full_path.rfind('/')); - entity_found = true; - break; - } + // Typed Collection; the wire shape is `{"items": [{"id": "..."}]}` + // per JsonWriter>::write. + dto::Collection collection; + + // A plugin operation runs to completion inside execute_operation, so nothing + // is ever tracked for it and the collection is empty by construction. The + // provider still decides whether the operation exists at all, so a typo is a + // miss here and not an empty success. + if (entity_info.is_plugin) { + auto * pmgr = ctx_.node()->get_plugin_manager(); + auto * op_prov = pmgr ? pmgr->get_operation_provider_for_entity(entity_id) : nullptr; + if (op_prov == nullptr) { + return tl::make_unexpected( + make_error(404, ERR_OPERATION_NOT_FOUND, "No operation provider for plugin entity '" + entity_id + "'")); + } + try { + auto result = op_prov->get_operation(entity_id, operation_id); + if (!result) { + return tl::make_unexpected(make_provider_error(result.error(), entity_id, operation_id)); } + return collection; + } catch (const std::exception & e) { + RCLCPP_ERROR(HandlerContext::logger(), "Plugin OperationProvider threw for entity '%s': %s", entity_id.c_str(), + e.what()); + return tl::make_unexpected(make_plugin_error(500, "Plugin threw exception", json{{"entity_id", entity_id}})); + } catch (...) { + RCLCPP_ERROR(HandlerContext::logger(), "Plugin OperationProvider threw unknown exception for entity '%s'", + entity_id.c_str()); + return tl::make_unexpected( + make_plugin_error(500, "Plugin threw unknown exception", json{{"entity_id", entity_id}})); } } - if (!entity_found) { + + const auto & cache = ctx_.node()->get_thread_safe_cache(); + auto lookup = resolve_entity_operations(cache, entity_info.sovd_type(), entity_id); + if (!lookup) { + return tl::make_unexpected(lookup.error()); + } + const auto & ops = lookup->ops; + + // The executions of an operation are addressed by the id that addresses the + // operation, resolved by the one rule the collection lists it under. Deriving + // a ROS path by joining the entity's namespace to the id instead names a path + // no member need have: an id can carry a member half or be a ROS path, and a + // member's namespace is its own, not the aggregate's. + auto parsed = http::parse_member_qualified_id(operation_id, ops.is_aggregated); + if (parsed.has_member && !names_a_member(ops, parsed.member_id)) { return tl::make_unexpected( - make_error(404, ERR_ENTITY_NOT_FOUND, "Entity not found", json{{"entity_id", entity_id}})); + make_error(404, ERR_RESOURCE_NOT_FOUND, "Member not found in entity", + json{{"entity_id", entity_id}, {"operation_id", operation_id}, {"member_id", parsed.member_id}})); } - const std::string action_path = namespace_path + "/" + operation_id; - auto * operation_mgr = ctx_.node()->get_operation_manager(); - auto goals = operation_mgr->get_goals_for_action(action_path); + auto resolved = resolve_operation(ops, parsed); + if (!resolved.found()) { + return tl::make_unexpected(make_error(404, ERR_OPERATION_NOT_FOUND, "Operation not found", + json{{"entity_id", entity_id}, {"operation_id", operation_id}})); + } - // Typed Collection - replaces the legacy ad-hoc - // `{"items": [{"id": "..."}]}` JSON literal. The wire shape is identical - // (per JsonWriter>::write) but the per-item schema - // is now enforced by JsonReader on round-trip. - dto::Collection collection; - for (const auto & goal : goals) { - dto::ExecutionId item; - item.id = goal.goal_id; - collection.items.push_back(std::move(item)); + if (auto ambiguous = refuse_if_ambiguous(ops, parsed, entity_id, operation_id); ambiguous.has_value()) { + return tl::make_unexpected(std::move(*ambiguous)); + } + + // The goals of an operation live on the gateway that sent them, which is the + // one that owns the member - the same gateway POST reached to create them. + // Answering from the local tracking map instead reports an empty collection + // for goals that exist. + const std::string & full_path = + resolved.service.has_value() ? resolved.service->full_path : resolved.action->full_path; + if (auto owner = ops.owner_by_path.find(full_path); owner != ops.owner_by_path.end()) { + auto dispatch = ctx_.dispatch_to_member(req, owner->second, "operations/" + parsed.item_id + "/executions", + json{{"entity_id", entity_id}, {"operation_id", operation_id}}); + if (!dispatch) { + return tl::make_unexpected(dispatch.error()); + } + if (*dispatch == MemberDispatch::kForwarded) { + return tl::make_unexpected(HandlerContext::forwarded_sentinel_error()); + } + } + + // Only an action produces an execution resource: a service call returns its + // result inside the POST and leaves nothing to address afterwards. Its + // executions collection therefore exists and is empty, which is a different + // answer from the 404 an id naming no operation gets above - and the two have + // to stay different, because a client reading a collection of executions + // cannot otherwise tell a synchronous operation from a mistyped id. + if (resolved.action.has_value()) { + auto * operation_mgr = ctx_.node()->get_operation_manager(); + for (const auto & goal : operation_mgr->get_goals_for_action(resolved.action->full_path)) { + dto::ExecutionId item; + item.id = goal.goal_id; + collection.items.push_back(std::move(item)); + } } return collection; } diff --git a/src/ros2_medkit_gateway/test/test_operation_handlers.cpp b/src/ros2_medkit_gateway/test/test_operation_handlers.cpp index 5f245ed7e..4a7278436 100644 --- a/src/ros2_medkit_gateway/test/test_operation_handlers.cpp +++ b/src/ros2_medkit_gateway/test/test_operation_handlers.cpp @@ -42,6 +42,8 @@ #include "ros2_medkit_gateway/core/discovery/models/area.hpp" #include "ros2_medkit_gateway/core/http/error_codes.hpp" #include "ros2_medkit_gateway/core/http/handlers/operation_handlers.hpp" +#include "ros2_medkit_gateway/core/plugins/plugin_manager.hpp" +#include "ros2_medkit_gateway/core/providers/operation_provider.hpp" #include "ros2_medkit_gateway/dto/json_writer.hpp" #include "ros2_medkit_gateway/gateway_node.hpp" #include "ros2_medkit_gateway/http/typed_router.hpp" @@ -236,6 +238,45 @@ TEST_F(OperationHandlersValidationTest, ListOperationsInvalidEntityReturns400) { EXPECT_EQ(result.error().code, ros2_medkit_gateway::ERR_INVALID_PARAMETER); } +// A plugin owning one entity and one operation. An operation a plugin serves is +// synchronous - execute_operation returns the result - so the entity has an +// executions collection that is empty, and an id the plugin does not know is +// still a miss. Nothing else in this workspace pairs an OperationProvider with +// a live GatewayNode, so without this the plugin branch could not be driven. +class MockOperationPlugin : public ros2_medkit_gateway::GatewayPlugin, public ros2_medkit_gateway::OperationProvider { + public: + static constexpr const char * kName = "mock_operation_plugin"; + static constexpr const char * kEntityId = "plugin_ecu"; + static constexpr const char * kOperationId = "plugin_op"; + + std::string name() const override { + return kName; + } + void configure(const json & /*config*/) override { + } + void shutdown() override { + } + + tl::expected, + ros2_medkit_gateway::OperationProviderErrorInfo> + list_operations(const std::string & entity_id) override { + ros2_medkit_gateway::dto::Collection coll; + ros2_medkit_gateway::dto::OperationItem item; + item.id = kOperationId; + item.name = kOperationId; + ros2_medkit_gateway::dto::XMedkitOperationItem xm; + xm.entity_id = entity_id; + item.x_medkit = xm; + coll.items.push_back(std::move(item)); + return coll; + } + + tl::expected + execute_operation(const std::string & /*entity_id*/, const std::string & op, const json & /*params*/) override { + return ros2_medkit_gateway::dto::OperationExecutionResult{json{{"executed", op}}}; + } +}; + // ============================================================================= // Fixture-based tests against a live GatewayNode + ROS 2 graph. // ============================================================================= @@ -319,6 +360,7 @@ class OperationHandlersFixtureTest : public ::testing::Test { ASSERT_TRUE(wait_for_discovery_settled(base_generation)) << "action/service discovery did not settle before seeding"; seed_component_cache(); + seed_plugin_entity(); } void TearDown() override { @@ -410,8 +452,23 @@ class OperationHandlersFixtureTest : public ::testing::Test { area.namespace_path = "/powertrain"; area.source = "manifest"; + Component plugin_ecu; + plugin_ecu.id = MockOperationPlugin::kEntityId; + plugin_ecu.name = "Plugin ECU"; + plugin_ecu.namespace_path = "/external"; + plugin_ecu.fqn = "/external"; + plugin_ecu.source = "plugin"; + auto & cache = const_cast(gateway_node_->get_thread_safe_cache()); - cache.update_all({area}, {component, gearbox}, {}, {}); + cache.update_all({area}, {component, gearbox, plugin_ecu}, {}, {}); + } + + /// Give the plugin ECU an owner, so the handlers route it to the provider. + void seed_plugin_entity() { + auto * pmgr = gateway_node_->get_plugin_manager(); + ASSERT_NE(pmgr, nullptr); + pmgr->add_plugin(std::make_unique()); + pmgr->register_entity_ownership(MockOperationPlugin::kName, {MockOperationPlugin::kEntityId}); } /// Drive `create_execution` and assert the typed response carries the async @@ -548,6 +605,132 @@ TEST_F(OperationHandlersFixtureTest, ListExecutionsReturnsTrackedActionGoal) { EXPECT_EQ(collection.items[0].id, execution_id); } +// The executions of an operation are addressed by the id that addresses the +// operation, so a member half has to select among same-named copies here just +// as it does on the execution itself. +TEST_F(OperationHandlersFixtureTest, ListExecutionsResolvesAQualifiedIdToItsMember) { + const auto execution_id = create_action_execution(); + ASSERT_FALSE(execution_id.empty()); + + auto raw_req = make_request_with_match("/api/v1/areas/powertrain/operations/engine:long_calibration/executions", + R"(/api/v1/areas/([^/]+)/operations/([^/]+)/executions)"); + http::TypedRequest typed(raw_req); + + auto result = handlers_->list_executions(typed); + ASSERT_TRUE(result.has_value()) << result.error().code << ": " << result.error().message; + ASSERT_EQ(result->items.size(), 1u); + EXPECT_EQ(result->items[0].id, execution_id); +} + +// A service answers inside its own call, so it never leaves an execution +// behind. The collection is present and empty, which is the answer an id +// naming no operation must NOT get. +TEST_F(OperationHandlersFixtureTest, ListExecutionsOfAServiceIsAnEmptyCollection) { + auto raw_req = make_request_with_match("/api/v1/components/engine/operations/calibrate/executions", + R"(/api/v1/components/([^/]+)/operations/([^/]+)/executions)"); + http::TypedRequest typed(raw_req); + + auto result = handlers_->list_executions(typed); + ASSERT_TRUE(result.has_value()) << result.error().code << ": " << result.error().message; + EXPECT_TRUE(result->items.empty()); +} + +TEST_F(OperationHandlersFixtureTest, ListExecutionsUnknownOperationIsOperationNotFound) { + auto raw_req = make_request_with_match("/api/v1/components/engine/operations/does_not_exist/executions", + R"(/api/v1/components/([^/]+)/operations/([^/]+)/executions)"); + http::TypedRequest typed(raw_req); + + auto result = handlers_->list_executions(typed); + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(result.error().http_status, 404); + EXPECT_EQ(result.error().code, ros2_medkit_gateway::ERR_OPERATION_NOT_FOUND); +} + +TEST_F(OperationHandlersFixtureTest, ListExecutionsUnknownEntityIsEntityNotFound) { + auto raw_req = make_request_with_match("/api/v1/components/no_such_entity/operations/calibrate/executions", + R"(/api/v1/components/([^/]+)/operations/([^/]+)/executions)"); + http::TypedRequest typed(raw_req); + + auto result = handlers_->list_executions(typed); + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(result.error().http_status, 404); + EXPECT_EQ(result.error().code, ros2_medkit_gateway::ERR_ENTITY_NOT_FOUND); +} + +// The bare id names two operations, so it names neither collection of goals. +TEST_F(OperationHandlersFixtureTest, ListExecutionsRefusesAnAmbiguousBareId) { + auto raw_req = make_request_with_match("/api/v1/areas/powertrain/operations/calibrate/executions", + R"(/api/v1/areas/([^/]+)/operations/([^/]+)/executions)"); + http::TypedRequest typed(raw_req); + + auto result = handlers_->list_executions(typed); + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(result.error().http_status, 400); + EXPECT_EQ(result.error().code, ros2_medkit_gateway::ERR_INVALID_REQUEST); + ASSERT_TRUE(result.error().params.contains("operation_ids")); +} + +// A read that resolved this id would describe one of two operations and never +// say which, while running the same id is a 400. One id, one answer. +TEST_F(OperationHandlersFixtureTest, GetOperationRefusesAnAmbiguousBareId) { + auto raw_req = make_request_with_match("/api/v1/areas/powertrain/operations/calibrate", + R"(/api/v1/areas/([^/]+)/operations/([^/]+))"); + http::TypedRequest typed(raw_req); + + auto result = handlers_->get_operation(typed); + + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(result.error().http_status, 400); + EXPECT_EQ(result.error().code, ros2_medkit_gateway::ERR_INVALID_REQUEST); + ASSERT_TRUE(result.error().params.contains("operation_ids")); + std::set offered; + for (const auto & id : result.error().params["operation_ids"]) { + offered.insert(id.get()); + } + EXPECT_EQ(offered, (std::set{"engine:calibrate", "gearbox:calibrate"})); +} + +// The other half of the same rule, and the one every existing client depends +// on: an id its own provider carries once still reads. +TEST_F(OperationHandlersFixtureTest, GetOperationStillResolvesAnUnambiguousBareId) { + auto raw_req = make_request_with_match("/api/v1/areas/powertrain/operations/long_calibration", + R"(/api/v1/areas/([^/]+)/operations/([^/]+))"); + http::TypedRequest typed(raw_req); + + auto result = handlers_->get_operation(typed); + + ASSERT_TRUE(result.has_value()) << result.error().code << ": " << result.error().message; + EXPECT_EQ(result->item.id, "long_calibration"); + ASSERT_TRUE(result->item.x_medkit.has_value()); + ASSERT_TRUE(result->item.x_medkit->ros2.has_value()); + EXPECT_EQ(result->item.x_medkit->ros2->action, "/powertrain/engine/long_calibration"); +} + +TEST_F(OperationHandlersFixtureTest, ListExecutionsOnAPluginEntityIsAnEmptyCollection) { + auto raw_req = make_request_with_match(std::string("/api/v1/components/") + MockOperationPlugin::kEntityId + + "/operations/" + MockOperationPlugin::kOperationId + "/executions", + R"(/api/v1/components/([^/]+)/operations/([^/]+)/executions)"); + http::TypedRequest typed(raw_req); + + auto result = handlers_->list_executions(typed); + ASSERT_TRUE(result.has_value()) << result.error().code << ": " << result.error().message; + EXPECT_TRUE(result->items.empty()); +} + +TEST_F(OperationHandlersFixtureTest, ListExecutionsOnAPluginEntityRefusesAnUnknownOperation) { + auto raw_req = make_request_with_match(std::string("/api/v1/components/") + MockOperationPlugin::kEntityId + + "/operations/does_not_exist/executions", + R"(/api/v1/components/([^/]+)/operations/([^/]+)/executions)"); + http::TypedRequest typed(raw_req); + + auto result = handlers_->list_executions(typed); + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(result.error().http_status, 404); + // The plugin decided this, and the same code the plugin read route answers + // with, so a caller cannot tell the two routes apart by the error alone. + EXPECT_EQ(result.error().code, ros2_medkit_gateway::ERR_PLUGIN_ERROR); +} + TEST_F(OperationHandlersFixtureTest, GetExecutionContainsStatusFields) { const auto execution_id = create_action_execution(); ASSERT_FALSE(execution_id.empty()); diff --git a/src/ros2_medkit_integration_tests/CMakeLists.txt b/src/ros2_medkit_integration_tests/CMakeLists.txt index 42b7f9036..361dbc571 100644 --- a/src/ros2_medkit_integration_tests/CMakeLists.txt +++ b/src/ros2_medkit_integration_tests/CMakeLists.txt @@ -77,7 +77,7 @@ medkit_target_dependencies(demo_calibration_service rclcpp rcl_interfaces std_sr add_executable(demo_dual_calibration_service demo_nodes/dual_calibration_service.cpp) target_include_directories(demo_dual_calibration_service PRIVATE ${_demo_include_dir}) -medkit_target_dependencies(demo_dual_calibration_service rclcpp std_srvs) +medkit_target_dependencies(demo_dual_calibration_service rclcpp rclcpp_action example_interfaces std_srvs) add_executable(demo_long_calibration_action demo_nodes/long_calibration_action.cpp) target_include_directories(demo_long_calibration_action PRIVATE ${_demo_include_dir}) diff --git a/src/ros2_medkit_integration_tests/README.md b/src/ros2_medkit_integration_tests/README.md index 6a8b7cbba..65cd04722 100644 --- a/src/ros2_medkit_integration_tests/README.md +++ b/src/ros2_medkit_integration_tests/README.md @@ -76,6 +76,7 @@ ros2 launch ros2_medkit_integration_tests demo_nodes.launch.py | `controller` | `/body/lights` | Subscriber + Publisher | Light controller (command/status) | | `calibration` | `/powertrain/engine` | Service | Trigger-based calibration | | `long_calibration` | `/powertrain/engine` | Action | Fibonacci-based long-running action | +| `dual_calibration` | `/testrig/dual` | Services + Actions | `left/calibrate` and `right/calibrate`, `left/sweep` and `right/sweep` - one provider carrying each operation short name twice, so the ROS path is the only id that separates the copies | ## Writing New Tests diff --git a/src/ros2_medkit_integration_tests/demo_nodes/dual_calibration_service.cpp b/src/ros2_medkit_integration_tests/demo_nodes/dual_calibration_service.cpp index 2ac779aed..a8ff88696 100644 --- a/src/ros2_medkit_integration_tests/demo_nodes/dual_calibration_service.cpp +++ b/src/ros2_medkit_integration_tests/demo_nodes/dual_calibration_service.cpp @@ -14,8 +14,8 @@ /** * @file dual_calibration_service.cpp - * @brief One node exposing two services whose ROS paths differ only above the - * last segment. + * @brief One node exposing two services AND two actions whose ROS paths differ + * only above the last segment. * * `left/calibrate` and `right/calibrate` under the node's namespace are two * different services with one short name, and the short name is the wire id the @@ -23,30 +23,63 @@ * a qualified id names the same thing for each - which is the case the ROS path * has to address instead. * + * `left/sweep` and `right/sweep` are the same collision on the action side, and + * it has to exist separately: only an action produces executions, so a ROS path + * is the only id under which a goal of a twice-named operation can be listed at + * all. A service answers inside its call and leaves nothing to address. + * * Each side answers with a message naming itself, so a caller can tell which - * service ran from the response rather than from the status alone. + * service ran from the response rather than from the status alone. Each sweep + * runs one step per goal `order` at 2 Hz, so a goal started with a large order + * is still running when the execution that started it is looked up. */ +#include + +#include +#include +#include #include #include +#include +#include #include +#include #include #include "ros2_medkit_integration_tests/demo_node_main.hpp" class DualCalibrationService : public rclcpp::Node { public: + using Fibonacci = example_interfaces::action::Fibonacci; + using GoalHandleFibonacci = rclcpp_action::ServerGoalHandle; + DualCalibrationService() : Node("dual_calibration") { left_srv_ = make_side("left"); right_srv_ = make_side("right"); + left_sweep_ = make_sweep("left"); + right_sweep_ = make_sweep("right"); - RCLCPP_INFO(this->get_logger(), "Dual calibration services started"); + RCLCPP_INFO(this->get_logger(), "Dual calibration services and sweeps started"); } - // The callbacks capture `this`, so the services have to go before any member - // they touch does. + void prepare_shutdown() { + shutdown_.store(true); + if (left_thread_.joinable()) { + left_thread_.join(); + } + if (right_thread_.joinable()) { + right_thread_.join(); + } + left_sweep_.reset(); + right_sweep_.reset(); + } + + // The callbacks capture `this`, so the services and the goal-executing + // threads have to go before any member they touch does. ~DualCalibrationService() override { + prepare_shutdown(); left_srv_.reset(); right_srv_.reset(); } @@ -68,11 +101,80 @@ class DualCalibrationService : public rclcpp::Node { }); } + rclcpp_action::Server::SharedPtr make_sweep(const std::string & side) { + return rclcpp_action::create_server( + this, side + "/sweep", + [](const rclcpp_action::GoalUUID &, std::shared_ptr) { + return rclcpp_action::GoalResponse::ACCEPT_AND_EXECUTE; + }, + [](const std::shared_ptr &) { + return rclcpp_action::CancelResponse::ACCEPT; + }, + [this, side](const std::shared_ptr & goal_handle) { + std::thread & slot = side == "left" ? left_thread_ : right_thread_; + if (slot.joinable()) { + slot.join(); + } + slot = std::thread(&DualCalibrationService::sweep, this, side, goal_handle); + }); + } + + /// One sweep step per goal `order` at 2 Hz, so a goal ordered generously is + /// still running while the caller looks it up. + /// + /// A goal in flight when the node goes down is left alone: rclcpp_action + /// clears its own tracking during shutdown, and reporting an outcome into + /// that window throws. + void sweep(const std::string & side, const std::shared_ptr & goal_handle) { + try { + auto feedback = std::make_shared(); + auto result = std::make_shared(); + feedback->sequence.push_back(0); + rclcpp::Rate rate(2); + + for (int step = 1; step < goal_handle->get_goal()->order && rclcpp::ok() && !shutdown_.load(); ++step) { + if (goal_handle->is_canceling()) { + result->sequence = feedback->sequence; + goal_handle->canceled(result); + return; + } + feedback->sequence.push_back(step); + goal_handle->publish_feedback(feedback); + rate.sleep(); + } + + if (!rclcpp::ok() || shutdown_.load()) { + return; + } + result->sequence = feedback->sequence; + if (goal_handle->is_canceling()) { + goal_handle->canceled(result); + return; + } + goal_handle->succeed(result); + RCLCPP_INFO(this->get_logger(), "%s side swept", side.c_str()); + } catch (const std::exception & e) { + RCLCPP_WARN(this->get_logger(), "Sweep interrupted: %s", e.what()); + } + } + rclcpp::Service::SharedPtr left_srv_; rclcpp::Service::SharedPtr right_srv_; + rclcpp_action::Server::SharedPtr left_sweep_; + rclcpp_action::Server::SharedPtr right_sweep_; + std::thread left_thread_; + std::thread right_thread_; + std::atomic shutdown_{false}; }; int main(int argc, char * argv[]) { + // rclcpp_action may throw "Asked to publish result for goal that does not + // exist" during SIGINT shutdown if a goal is in flight: the executor can run + // a goal state callback after the action server has cleared its tracking. + // Exiting cleanly there keeps a torn-down demo node from reading as a crash. + std::set_terminate([]() { + _exit(0); + }); return ros2_medkit_integration_tests::run_demo_node(argc, argv, []() -> std::shared_ptr { return std::make_shared(); }); diff --git a/src/ros2_medkit_integration_tests/test/features/test_grouping_entity_aggregation.test.py b/src/ros2_medkit_integration_tests/test/features/test_grouping_entity_aggregation.test.py index 1adfd8daa..9ab7a0c70 100644 --- a/src/ros2_medkit_integration_tests/test/features/test_grouping_entity_aggregation.test.py +++ b/src/ros2_medkit_integration_tests/test/features/test_grouping_entity_aggregation.test.py @@ -132,10 +132,23 @@ # exist for. With the same namespace on both sides the full paths are identical # and the local walk simply deduplicates the peer's copy away, which builds a # dedup collapse rather than the ambiguity. -PRIMARY_NODES = ['temp_sensor', 'calibration', 'dual_calibration', 'rpm_sensor'] +PRIMARY_NODES = [ + 'temp_sensor', 'calibration', 'dual_calibration', 'rpm_sensor', 'long_calibration', +] PEER_NODES = ['pressure_sensor', 'actuator'] PEER_CALIBRATION_NAMESPACE = '/chassis/brakes' +# An action on each side, because only an action leaves an execution behind: a +# service answers inside its own call, so a topology of services alone cannot +# show whether listing executions reaches the gateway that holds the goals. The +# peer's copy runs in the peer's namespace for the same reason its calibration +# service does - identical full paths would deduplicate into one item and the +# member half would have nothing to separate. +PRIMARY_LONG_APP = 'primary_long_calibration' +PEER_LONG_APP = 'peer_long_calibration' +LONG_OPERATION = 'long_calibration' +PEER_LONG_NAMESPACE = '/chassis/brakes' + # Declared with the SAME id on both gateways. Apps are renamed on collision, # Components are not, so this is the case that shows whether leaf identity # survives the merge - R1, the precondition for every addressing rule. @@ -158,6 +171,12 @@ DUAL_LEFT_ID = 'testrig/dual/left/calibrate' DUAL_RIGHT_ID = 'testrig/dual/right/calibrate' +# The same collision on the action side. A service has no executions at all, so +# the path form can only be shown to select the operation it names - rather than +# merely to resolve - where the two copies hold goals that differ. +DUAL_LEFT_SWEEP_ID = 'testrig/dual/left/sweep' +DUAL_RIGHT_SWEEP_ID = 'testrig/dual/right/sweep' + MERGED_AREA = 'vehicle' MERGED_FUNCTION = 'vehicle_health' PARENT_COMPONENT = 'vehicle-ecu' @@ -199,6 +218,12 @@ ros_binding: node_name: dual_calibration namespace: {DUAL_NAMESPACE} + - id: {PRIMARY_LONG_APP} + name: "Primary Long Calibration" + is_located_on: {PARENT_COMPONENT} + ros_binding: + node_name: long_calibration + namespace: /powertrain/engine - id: {COLLIDING_LEAF} name: "Shared Sensor (primary)" is_located_on: {PARENT_COMPONENT} @@ -212,6 +237,7 @@ hosted_by: - temp_sensor - primary_calibration + - {PRIMARY_LONG_APP} - {COLLIDING_LEAF} """ @@ -255,6 +281,12 @@ ros_binding: node_name: calibration namespace: {PEER_CALIBRATION_NAMESPACE} + - id: {PEER_LONG_APP} + name: "Peer Long Calibration" + is_located_on: {PEER_SUBCOMPONENT} + ros_binding: + node_name: long_calibration + namespace: {PEER_LONG_NAMESPACE} - id: {COLLIDING_LEAF} name: "Shared Sensor (peer)" is_located_on: {PEER_SUBCOMPONENT} @@ -268,6 +300,7 @@ hosted_by: - pressure_sensor - peer_calibration + - {PEER_LONG_APP} - {COLLIDING_LEAF} """ @@ -327,6 +360,14 @@ def generate_test_description(): output='screen', additional_env=peer_domain_env, )] + + [launch_ros.actions.Node( + package='ros2_medkit_integration_tests', + executable='demo_long_calibration_action', + name='long_calibration', + namespace=PEER_LONG_NAMESPACE, + output='screen', + additional_env=peer_domain_env, + )] + [ create_fault_manager_node(rosbag_enabled=False), create_fault_manager_node(rosbag_enabled=False, extra_env=peer_domain_env), @@ -358,8 +399,11 @@ def setUpClass(cls): # a collection read between those two moments is legitimately empty and # would fail every rule below for a reason unrelated to the rule. cls._wait_for_apps( - PRIMARY_URL, {'temp_sensor', 'primary_calibration', DUAL_APP}, 'primary') - cls._wait_for_apps(PEER_URL, {'pressure_sensor', 'peer_calibration'}, 'peer') + PRIMARY_URL, + {'temp_sensor', 'primary_calibration', DUAL_APP, PRIMARY_LONG_APP}, + 'primary') + cls._wait_for_apps( + PEER_URL, {'pressure_sensor', 'peer_calibration', PEER_LONG_APP}, 'peer') cls._wait_until_merged() @classmethod @@ -425,6 +469,59 @@ def _run_operation(entity_path, operation_id): timeout=15, ) + @staticmethod + def _executions_url(entity_path, operation_id, base_url=PRIMARY_URL): + return (f'{base_url}/{entity_path}/operations/' + f'{quote(operation_id, safe="")}/executions') + + def _wait_for_operation(self, entity_path, operation_id, timeout=60.0): + """Block until `operation_id` is listed AND carries its ROS type. + + An operation appears in the collection before the gateway has resolved + the interface type behind it, and a goal sent in that window is refused + for a reason that has nothing to do with the id under test. + """ + deadline = time.monotonic() + timeout + seen = [] + while time.monotonic() < deadline: + response = requests.get( + f'{PRIMARY_URL}/{entity_path}/operations', timeout=10) + if response.status_code == 200: + seen = [] + for item in response.json().get('items', []): + seen.append(item.get('id')) + if item.get('id') != operation_id: + continue + ros2 = item.get('x-medkit', {}).get('ros2', {}) + if ros2.get('type'): + return + time.sleep(0.5) + raise AssertionError( + f'{entity_path}: {operation_id!r} not usable within {timeout}s; offered {seen}') + + def _start_goal(self, entity_path, operation_id, order=30): + """Send one action goal and return the execution id it was given.""" + response = requests.post( + self._executions_url(entity_path, operation_id), + json={'parameters': {'order': order}}, + timeout=20, + ) + self.assertEqual( + response.status_code, 202, + f'{operation_id!r} on {entity_path} did not start: {response.text}') + execution_id = response.json().get('id') + self.assertTrue(execution_id, response.text) + return execution_id + + def _execution_ids(self, entity_path, operation_id, base_url=PRIMARY_URL): + """Read the ids in the executions collection of one operation.""" + response = requests.get( + self._executions_url(entity_path, operation_id, base_url), timeout=20) + self.assertEqual(response.status_code, 200, response.text) + body = response.json() + self.assertIn('items', body, response.text) + return [item.get('id') for item in body['items']] + def _assert_side_ran(self, response, side, operation_id): """Assert the service the id names is the one that answered. @@ -863,6 +960,289 @@ def test_a_path_shaped_id_that_names_no_operation_is_refused(self): through_aggregate.text, ) + # ---------------------------------------------------------------------- R5 + # LISTING THE EXECUTIONS OF AN OPERATION. + # + # An execution exists only for an action: a service answers inside its own + # call and leaves nothing behind. So the collection has three distinct + # answers, and a client has to be able to tell them apart - the goals of an + # action, the empty collection of a service, and the refusal of an id that + # names no operation at all. + + def test_a_started_goal_is_listed_under_the_id_that_started_it(self): + """The goals an aggregate reports are the ones that exist. + + Asserted by the id that came back from starting it, not by the status: + a listing that resolves nothing answers 200 with an empty array, and + that is indistinguishable from a working listing of an action nobody + has run. The second half pins that the listing is per operation - a + collection that reported every goal the gateway tracks would satisfy + the first half on its own. + """ + entity_path = f'functions/{MERGED_FUNCTION}' + local_id = f'{PRIMARY_LONG_APP}:{LONG_OPERATION}' + peer_id = f'{PEER_LONG_APP}:{LONG_OPERATION}' + self._wait_for_operation(entity_path, local_id) + self._wait_for_operation(entity_path, peer_id) + + execution_id = self._start_goal(entity_path, local_id) + + listed = self._execution_ids(entity_path, local_id) + self.assertIn( + execution_id, listed, + f'the goal that was just started is not among {listed}') + + other = self._execution_ids(entity_path, peer_id) + self.assertNotIn( + execution_id, other, + f'a goal of {local_id!r} is reported under {peer_id!r}: {other}', + ) + + def test_a_goal_started_on_a_peer_owned_member_is_listed_through_the_aggregate(self): + """R5 for the goals themselves: they live where they were sent. + + The POST is dispatched to the member's own gateway, so the goal is on + the peer and this gateway tracks nothing for it. A listing answered from + the local tracking map returns an empty array with status 200, which is + exactly the false success this asserts against - so the peer is asked + directly as well, proving the goal the aggregate reported is the one + that actually exists over there. + """ + entity_path = f'functions/{MERGED_FUNCTION}' + peer_id = f'{PEER_LONG_APP}:{LONG_OPERATION}' + self._wait_for_operation(entity_path, peer_id) + + execution_id = self._start_goal(entity_path, peer_id) + + on_the_peer = self._execution_ids( + f'apps/{PEER_LONG_APP}', LONG_OPERATION, base_url=PEER_URL) + self.assertIn( + execution_id, on_the_peer, + f'the goal was not started on the peer at all: {on_the_peer}') + + through_aggregate = self._execution_ids(entity_path, peer_id) + self.assertIn( + execution_id, through_aggregate, + f'the aggregate answered from its own tracking map: {through_aggregate}', + ) + + # The member's own route on THIS gateway names an entity the peer owns + # wholesale, so the whole request belongs on the peer. + forwarded = self._execution_ids(f'apps/{PEER_LONG_APP}', LONG_OPERATION) + self.assertIn( + execution_id, forwarded, + f'a peer-owned entity was answered locally: {forwarded}') + + local_id = f'{PRIMARY_LONG_APP}:{LONG_OPERATION}' + here = self._execution_ids(entity_path, local_id) + self.assertNotIn( + execution_id, here, + f"a peer's goal is reported under the local member: {here}") + + def test_a_service_operation_has_an_empty_execution_collection_not_a_miss(self): + """R6 for executions: present-but-empty is not the same as absent. + + A service runs to completion inside the POST, so its executions + collection exists and is empty - forever. An id that names no operation + does not exist at all. Answering both the same way tells a client its + typo worked, and answering the service with `entity-not-found` names the + wrong thing entirely: the entity is right there. + """ + entity_path = f'functions/{MERGED_FUNCTION}' + service_id = 'primary_calibration:calibrate' + + response = requests.get( + self._executions_url(entity_path, service_id), timeout=20) + self.assertEqual(response.status_code, 200, response.text) + self.assertEqual( + response.json().get('items'), [], + f'a synchronous operation reported executions: {response.text}') + + for missing, label in ( + ('primary_calibration:no_such_operation', 'a member that exists'), + ('no_such_operation', 'no member half at all'), + ): + with self.subTest(operation=missing): + refused = requests.get( + self._executions_url(entity_path, missing), timeout=20) + self.assertEqual( + refused.status_code, 404, + f'{label}: {missing!r} answered {refused.status_code}: {refused.text}') + body = refused.json() + self.assertEqual( + body.get('error_code'), 'operation-not-found', + f'the refusal blames the entity rather than the id: {body}') + self.assertEqual( + body.get('parameters', {}).get('operation_id'), missing, body) + + # A member half naming no member of this entity is wrong in a different + # place, and the refusal has to say which half - otherwise a mistyped + # member reads the same as a member whose operation is missing. + unknown_member = requests.get( + self._executions_url(entity_path, f'no_such_member:{LONG_OPERATION}'), timeout=20) + self.assertEqual(unknown_member.status_code, 404, unknown_member.text) + body = unknown_member.json() + self.assertEqual(body.get('error_code'), 'resource-not-found', body) + self.assertEqual( + body.get('parameters', {}).get('member_id'), 'no_such_member', + f'the refusal does not name the member half that was wrong: {body}') + + def test_a_path_shaped_id_lists_the_goals_of_the_operation_it_names(self): + """R3 and R5 together, for the id form that has no member half to use. + + `left/sweep` and `right/sweep` are one provider's two actions under one + short name. Both sides are driven and each list is checked for its OWN + goal AND against the other's, because a resolver that always picked the + first match would list the left goal under both ids and every + single-sided assertion would still pass. + """ + entity_path = f'apps/{DUAL_APP}' + self._wait_for_operation(entity_path, DUAL_LEFT_SWEEP_ID) + self._wait_for_operation(entity_path, DUAL_RIGHT_SWEEP_ID) + + left_goal = self._start_goal(entity_path, DUAL_LEFT_SWEEP_ID) + right_goal = self._start_goal(entity_path, DUAL_RIGHT_SWEEP_ID) + self.assertNotEqual(left_goal, right_goal) + + left_listed = self._execution_ids(entity_path, DUAL_LEFT_SWEEP_ID) + right_listed = self._execution_ids(entity_path, DUAL_RIGHT_SWEEP_ID) + self.assertIn(left_goal, left_listed, left_listed) + self.assertNotIn( + right_goal, left_listed, + f'the left list carries the right goal: {left_listed}') + self.assertIn(right_goal, right_listed, right_listed) + self.assertNotIn( + left_goal, right_listed, + f'the right list carries the left goal: {right_listed}') + + # The bare short name names both, so listing it names neither, and the + # refusal hands back the ids that do work. + bare = requests.get(self._executions_url(entity_path, 'sweep'), timeout=20) + self.assertEqual(bare.status_code, 400, bare.text) + self.assertEqual( + sorted(bare.json().get('parameters', {}).get('operation_ids') or []), + sorted([DUAL_LEFT_SWEEP_ID, DUAL_RIGHT_SWEEP_ID]), + f'the refusal does not hand back the ids that work: {bare.text}', + ) + + # And through an aggregate, where the member half and the path half are + # both in play. + aggregate_left = f'{DUAL_APP}:{DUAL_LEFT_SWEEP_ID}' + through_aggregate = self._execution_ids( + f'components/{PARENT_COMPONENT}', aggregate_left) + self.assertIn( + left_goal, through_aggregate, + f'{aggregate_left!r} lost the goal it names: {through_aggregate}') + + def test_reading_an_id_that_names_two_operations_is_refused(self): + """R4 on the read, which is the half that was still permissive. + + Returning the first match hands the caller one of several operations and + never says which - and the very next thing it does with that id, running + it, is a 400. The remedy is asserted as an id that reads, not as words + in a sentence. + """ + cases = ( + # one provider, one short name, two ROS paths + (f'apps/{DUAL_APP}', 'calibrate', [DUAL_LEFT_ID, DUAL_RIGHT_ID]), + (f'components/{PARENT_COMPONENT}', f'{DUAL_APP}:calibrate', + [f'{DUAL_APP}:{DUAL_LEFT_ID}', f'{DUAL_APP}:{DUAL_RIGHT_ID}']), + # two members, one short name + (f'functions/{MERGED_FUNCTION}', 'calibrate', + ['primary_calibration:calibrate', 'peer_calibration:calibrate']), + ) + for entity_path, operation_id, expected in cases: + with self.subTest(entity=entity_path, operation=operation_id): + refused = requests.get( + f'{PRIMARY_URL}/{entity_path}/operations/' + f'{quote(operation_id, safe="")}', + timeout=20, + ) + self.assertEqual( + refused.status_code, 400, + f'a read resolved an id that names two operations: {refused.text}') + body = refused.json() + self.assertEqual(body.get('error_code'), 'invalid-request', body) + offered = body.get('parameters', {}).get('operation_ids') + self.assertEqual( + sorted(offered or []), sorted(expected), + f'the refusal does not name the ids that work: {body}') + + # Taken straight out of the refusal it reads, so the remedy is + # usable rather than merely described. + detail = requests.get( + f'{PRIMARY_URL}/{entity_path}/operations/' + f'{quote(offered[0], safe="")}', + timeout=20, + ) + self.assertEqual( + detail.status_code, 200, + f'{offered[0]!r} came out of the refusal and is refused too: ' + f'{detail.text}') + self.assertEqual(detail.json().get('item', {}).get('id'), offered[0], detail.text) + + def test_an_unambiguous_bare_id_still_reads_and_lists(self): + """The regression guard, and it matters more than the refusal above. + + Every current client, the web UI, the Foxglove panel, the MCP tools and + the generated OpenAPI document send the bare short name. An id its own + provider carries once must read and list exactly as it did, whatever + some other provider does with the same name. + """ + detail = requests.get( + f'{PRIMARY_URL}/apps/primary_calibration/operations/calibrate', timeout=20) + self.assertEqual(detail.status_code, 200, detail.text) + item = detail.json().get('item', {}) + self.assertEqual(item.get('id'), 'calibrate', detail.text) + self.assertEqual( + item.get('x-medkit', {}).get('ros2', {}).get('service'), + '/powertrain/engine/calibrate', detail.text, + ) + + self.assertEqual( + self._execution_ids('apps/primary_calibration', 'calibrate'), [], + 'a service reported executions', + ) + + entity_path = f'apps/{PRIMARY_LONG_APP}' + self._wait_for_operation(entity_path, LONG_OPERATION) + execution_id = self._start_goal(entity_path, LONG_OPERATION) + listed = self._execution_ids(entity_path, LONG_OPERATION) + self.assertIn( + execution_id, listed, + f'a bare id started a goal it then could not list: {listed}') + + def test_the_executions_route_validates_the_entity_like_its_siblings(self): + """The executions collection is a route on an entity, not a free path. + + Every other operations route settles the entity first: the id has to + name an entity of the type the route is registered for, and an entity a + peer owns wholesale belongs on that peer. Reading the entity straight + out of the cache instead answers a Component asked for on the apps route + as though the route said nothing, and never forwards. + """ + # An id that IS unambiguous on that Component, so the only thing left to + # refuse it for is the collection it was asked on. A colliding id would + # answer 400 either way and prove nothing. + wrong_type = requests.get( + self._executions_url( + f'apps/{PARENT_COMPONENT}', 'primary_calibration:calibrate'), + timeout=20, + ) + self.assertEqual( + wrong_type.status_code, 400, + f'a Component was served on the apps route: {wrong_type.text}') + body = wrong_type.json() + self.assertEqual(body.get('error_code'), 'invalid-parameter', body) + self.assertEqual( + body.get('parameters', {}).get('actual_type'), 'Component', + f'the refusal is not about the route the entity was asked on: {body}') + + absent = requests.get( + self._executions_url('functions/no_such_entity', 'calibrate'), timeout=20) + self.assertEqual(absent.status_code, 404, absent.text) + self.assertEqual(absent.json().get('error_code'), 'entity-not-found', absent.text) + # ---------------------------------------------------------------------- R4 def test_a_compound_id_reaches_a_peer_owned_member(self): From df92fc4415332c68b9ec01808160761b277afa79 Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Thu, 20 Aug 2026 11:44:30 +0200 Subject: [PATCH 10/22] fix(aggregation): carry an entity's unreachability across every hop A peer's statement that an entity cannot be reached was never read back, so a gateway two hops from a dead leaf presented it as reachable. An App leaked the news through is_online; a Component, which has no such field, said nothing. A nested collection answering 504 not-responding was also read as a failed request rather than as a statement about one member, so a single unreachable member behind a peer aborted that peer's whole refresh and the aggregator replayed its last pre-failure picture indefinitely. A 504 is now carried where it names an entity and still fails the fetch where it does not. fetch_all_peer_entities is deleted. It dropped a failed peer fetch silently and had no caller outside its own tests. --- docs/api/rest.rst | 23 +++ docs/config/aggregation.rst | 26 ++- src/ros2_medkit_gateway/README.md | 21 +- .../design/aggregation.rst | 47 ++++- .../aggregation/aggregation_manager.hpp | 6 - .../src/aggregation/aggregation_manager.cpp | 35 +--- .../src/core/aggregation/peer_client.cpp | 47 ++++- .../test/test_aggregation_manager.cpp | 55 ------ .../test/test_peer_client.cpp | 187 ++++++++++++++++++ .../test_daisy_chain_aggregation.test.py | 82 ++++++++ 10 files changed, 420 insertions(+), 109 deletions(-) diff --git a/docs/api/rest.rst b/docs/api/rest.rst index cea5b34ee..dd52d0dcc 100644 --- a/docs/api/rest.rst +++ b/docs/api/rest.rst @@ -804,6 +804,14 @@ retained entity: ``status: "offline"``. Availability of an entity and health of a peer are separate questions and are reported separately. +``x-medkit.available`` is emitted **only when false**, so an absent field means +the entity is reachable. An aggregating gateway reads the field back off its +peers with that same default, which is what carries the fact past one hop: in a +chain ``A <- B <- C``, ``B`` marks ``C``'s declared entities unavailable when +``C`` goes quiet, and ``A`` reports them the same way. An App also carries +``x-medkit.is_online``; a Component has no second signal, so for a Component +this field is the only one. + .. note:: ``/configurations`` predates this rule and keeps its own: on an entity whose @@ -1045,6 +1053,21 @@ Execute Operations the status stream does not show the goal cancelling: the outcome is unknown - poll the execution status resource (``not-responding``) +.. note:: + + **Executions on an aggregate.** ``GET``, ``PUT`` and ``DELETE`` on a single + execution resolve the operation id in the route to the member that owns it + and are dispatched to that member's gateway, exactly as ``POST`` and the + executions listing are - a goal lives on the gateway that sent it. So every + id the listing hands out is addressable through the same path it was listed + under, and the ``Location`` a dispatched ``PUT`` or ``POST`` returns names + the member's own route, which this gateway resolves to the same member. An + operation id that does not resolve to exactly one owned operation is answered + locally, keyed on the execution id alone, so a locally-owned execution is + unaffected and an id naming no goal still gets ``404``. A member whose + gateway is silent answers ``504 not-responding`` before anything is + forwarded. + .. note:: **Cancel budget.** Both routes above are bounded by diff --git a/docs/config/aggregation.rst b/docs/config/aggregation.rst index d4f231192..246f4e491 100644 --- a/docs/config/aggregation.rst +++ b/docs/config/aggregation.rst @@ -471,12 +471,26 @@ Two statuses are read rather than treated as failures: that does not expose the route. Those members are omitted, the rest of the peer merges normally, and the absent routes are logged once per refresh at ``WARN``. -- ``504`` with error code ``not-responding`` on a Component's detail means the - peer holds that id and the gateway contributing it has gone quiet - the - answer an aggregating peer gives for a declaration it is retaining. In a - chain topology this is how the far end reports a dead leaf, so the Component - is kept as the peer's list named it and marked - ``x-medkit.available: false``. +- ``504`` with error code ``not-responding`` on any route hanging off an entity + - its detail, or one of its nested collections - means the peer holds that id + and the gateway contributing it has gone quiet, which is the answer an + aggregating peer gives for a declaration it is retaining. In a chain topology + this is how the far end reports a dead leaf, so the entity is kept as the + peer's list named it and marked ``x-medkit.available: false``. A nested + collection answering that way costs only the members that route carries; + treated as a failure it would discard the whole peer on every refresh, so one + unreachable member would freeze this gateway's view of everything that peer + holds. A ``504`` without ``not-responding`` is not a statement about an entity + and still discards the refresh. + +Availability is also read back off the wire. ``x-medkit.available`` is emitted +only when false, so an absent field means the entity is reachable, and that is +the default this gateway parses it with. It matters most beyond one hop: an App +also carries ``x-medkit.is_online``, but a Component has no second signal, so +without the read-back the head of a three-gateway chain reports a leaf behind a +dead gateway as reachable. Retention never contradicts what a peer said - it +only ever sets ``available`` to false, and it does so when the peer itself +stopped answering, which already covers everything behind it. .. _aggregation-breaking-changes: diff --git a/src/ros2_medkit_gateway/README.md b/src/ros2_medkit_gateway/README.md index 99f0d3c81..879881bdf 100644 --- a/src/ros2_medkit_gateway/README.md +++ b/src/ros2_medkit_gateway/README.md @@ -394,11 +394,22 @@ absent. The peer's last complete declaration stands for another cycle. Two statu carry a meaning of their own and are read instead: a `404` on a nested collection route means the peer runs a gateway that predates that route, so those members are omitted, the rest of the peer merges normally and the absent -routes are logged once per refresh; a `504 not-responding` on a Component's -detail is the peer saying it holds that id and whoever contributes it has gone -quiet, which is what an aggregating peer answers for a declaration it is -retaining, so the Component is kept as its list named it and marked -`x-medkit.available: false`. +routes are logged once per refresh; a `504 not-responding` on any route hanging +off an entity - its detail, or one of its nested collections - is the peer +saying it holds that id and whoever contributes it has gone quiet, which is what +an aggregating peer answers for a declaration it is retaining, so the entity is +kept as its list named it and marked `x-medkit.available: false`. A nested +collection answering that way costs only the members that route carries; read as +a failed request it would discard the whole peer, so one unreachable member +anywhere behind it would freeze this gateway's view of that peer. A `504` +without `not-responding` says nothing about an entity and still drops the +refresh. + +`x-medkit.available` is read back off a peer's response with a default of +`true`, since it is emitted only when false. Beyond one hop that is the only +thing carrying the fact: an App also has `is_online`, a Component has nothing +else, so without it the head of a three-gateway chain reports a leaf behind a +dead gateway as reachable. When a peer stops answering, the entities it declared in its manifest are retained and marked unavailable (`x-medkit.available: false`, diff --git a/src/ros2_medkit_gateway/design/aggregation.rst b/src/ros2_medkit_gateway/design/aggregation.rst index 6f0e88ef9..448b2bff0 100644 --- a/src/ros2_medkit_gateway/design/aggregation.rst +++ b/src/ros2_medkit_gateway/design/aggregation.rst @@ -31,7 +31,7 @@ not need to know which gateway owns which entity. package "Primary Gateway" { class AggregationManager { - + fetch_all_peer_entities() + + fetch_and_merge_peer_entities() + fan_out_get() + forward_request() + check_all_health() @@ -489,6 +489,25 @@ Answering that from the aggregator's own goal tracking returns an empty collection for goals that exist, which reads as "this operation has never been run". +Every verb that takes one of those execution ids back resolves it the same way, +for the same reason - ``GET`` for its status, ``PUT`` to apply a capability and +``DELETE`` to cancel: + +.. code-block:: text + + GET /api/v1/functions/vehicle_health/operations/peer_long_calibration:long_calibration/executions/{exec} + -> GET /api/v1/apps/peer_long_calibration/operations/long_calibration/executions/{exec} + +An id that does not resolve to exactly one owned operation is left to the local +path, whose key is the execution id alone: resolving is how the owning gateway +is found, not a second place for these routes to refuse a request. So a +locally-owned execution is answered here exactly as before, and an id naming no +goal gets the same ``404`` it always did. + +The peer's ``Location`` header is carried back through the forward, because it +names the member's own route - a path the aggregator resolves to the same +member - and it is the only address of a resource that lives on the other side. + The member's own gateway is the only one that can answer: the ROS service, the topic and the parameter behind the id exist on its graph and nowhere else. What this gateway holds for a peer-owned member is a declaration, which is why the @@ -649,10 +668,28 @@ their own: a ``404`` on a nested collection route (``/subareas``, ``/subcomponents``, an app's ``/operations``) identifies a peer running a gateway that predates the route and is reported in ``PeerEntities::absent_routes`` for the caller to log; a ``504`` with error code -``not-responding`` on a Component's detail is the peer reporting that the -gateway contributing that Component has gone quiet - what a middle gateway in a -chain answers for a declaration it is retaining - so the Component is kept as -the list named it and marked unavailable. +``not-responding`` on any route hanging off an entity - its detail, or one of +its nested collections - is the peer reporting that the gateway contributing +that entity has gone quiet, which is what a middle gateway in a chain answers +for a declaration it is retaining. The entity is kept as the list named it and +marked unavailable, and a nested collection answering that way costs the members +that route carries and nothing else. Read as a failed request instead, one +unreachable member anywhere behind a peer would discard that peer's whole +picture on every refresh, and the aggregator would go on serving its last +pre-failure view indefinitely. A ``504`` that does not carry +``not-responding`` says nothing about an entity and still fails the fetch. + +Availability travels the same way in the other direction. ``x-medkit.available`` +is emitted only when false, so absence means reachable, and both +``parse_component`` and ``parse_app`` read it back with a default of ``true``. +Without that, a chain of three gateways loses the fact at the second hop: an App +still carries ``is_online``, but a Component has no other signal, and the head of +the chain would report an unreachable leaf as reachable. Local retention only +ever sets the flag false and never back to true, so a peer's own statement that +something it holds is unreachable survives being replayed, and where the marking +does apply over a peer's ``available`` it is because the peer itself stopped +answering - the stronger statement, since everything it holds sits behind the +link that is down. ``AggregationManager`` never records a failed fetch as the peer's declaration, so the last complete one survives; it re-checks that peer's health to decide whether to replay it marked unavailable (health check failed) or exactly as it diff --git a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/aggregation/aggregation_manager.hpp b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/aggregation/aggregation_manager.hpp index 65417adc6..6339c5ef0 100644 --- a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/aggregation/aggregation_manager.hpp +++ b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/aggregation/aggregation_manager.hpp @@ -162,12 +162,6 @@ class AggregationManager { */ size_t healthy_peer_count() const; - /** - * @brief Fetch entities from all healthy peers and merge them - * @return Merged PeerEntities from all reachable peers - */ - PeerEntities fetch_all_peer_entities(); - /** * @brief Fetch entities from all healthy peers, merge with local entities, and build routing table * diff --git a/src/ros2_medkit_gateway/src/aggregation/aggregation_manager.cpp b/src/ros2_medkit_gateway/src/aggregation/aggregation_manager.cpp index 859cf8b3e..2d77badc3 100644 --- a/src/ros2_medkit_gateway/src/aggregation/aggregation_manager.cpp +++ b/src/ros2_medkit_gateway/src/aggregation/aggregation_manager.cpp @@ -326,35 +326,6 @@ size_t AggregationManager::healthy_peer_count() const { return count; } -PeerEntities AggregationManager::fetch_all_peer_entities() { - // Snapshot healthy peers under lock, release before network I/O. - std::vector> snapshot; - { - std::shared_lock lock(mutex_); - for (const auto & peer : peers_) { - if (peer->is_healthy()) { - snapshot.push_back(peer); - } - } - } - - PeerEntities merged; - for (auto & peer : snapshot) { - auto result = peer->fetch_entities(); - if (!result.has_value()) { - continue; - } - - const auto & entities = result.value(); - merged.areas.insert(merged.areas.end(), entities.areas.begin(), entities.areas.end()); - merged.components.insert(merged.components.end(), entities.components.begin(), entities.components.end()); - merged.apps.insert(merged.apps.end(), entities.apps.begin(), entities.apps.end()); - merged.functions.insert(merged.functions.end(), entities.functions.begin(), entities.functions.end()); - } - - return merged; -} - namespace { /// True for an entity a peer said it had DECLARED, rather than discovered from @@ -398,6 +369,12 @@ std::vector declared_only(const std::vector & src) { } /// Mark every addressable entity of a replayed declaration unreachable. +/// +/// The marking is one-way: it only ever sets the flag false, never back to +/// true. So an entity a peer itself reported as unreachable keeps that from its +/// own account, and where the marking does apply over a peer's `available` it +/// is because the peer stopped answering - which is the stronger statement, +/// since everything it holds sits behind the link that is down. template void mark_all_unreachable(std::vector & entities) { for (auto & entity : entities) { diff --git a/src/ros2_medkit_gateway/src/core/aggregation/peer_client.cpp b/src/ros2_medkit_gateway/src/core/aggregation/peer_client.cpp index 1e27c8e12..976532854 100644 --- a/src/ros2_medkit_gateway/src/core/aggregation/peer_client.cpp +++ b/src/ros2_medkit_gateway/src/core/aggregation/peer_client.cpp @@ -187,6 +187,11 @@ Component parse_component(const nlohmann::json & j) { if (xm.contains("identity") && xm["identity"].is_object()) { comp.identity = AssetIdentity::from_json(xm["identity"]); } + // A peer emits `available` only to say false, so absence is the peer + // stating the entity is reachable and the default has to be true. A + // Component carries no second signal - an App has `is_online` - so this is + // the only way an unreachable one stays unreachable past another hop. + comp.available = xm.value("available", true); } if (j.contains("translationId")) { comp.translation_id = j["translationId"].get(); @@ -321,6 +326,11 @@ App parse_app(const nlohmann::json & j) { } app.source = xm.value("source", ""); app.is_online = xm.value("is_online", false); + // Emitted only to say false, so absence means reachable. Read back for the + // same reason `is_online` is: a leaf that went quiet several hops away is + // described by the gateway that still holds its declaration, and that + // description is the only account of it this gateway can get. + app.available = xm.value("available", true); if (app.description.empty()) { app.description = xm.value("description", ""); } @@ -369,6 +379,13 @@ enum class RouteKind { /// ``/apps/{id}/operations``. A gateway old enough not to have the route answers /// 404, and aggregation has to keep working across that version boundary, so a /// 404 here means "not offered" rather than "could not be read". + /// + /// It also carries ``504 not-responding``, and for the same reason the detail + /// of an addressable entity does: the route hangs off an entity, and a peer + /// holding a declaration for a gateway that went quiet answers 504 on every + /// route of that entity, this one included. Read as a hole in the picture it + /// would abort the whole fetch, and one unreachable member anywhere behind a + /// peer would freeze this gateway's view of everything that peer holds. kNestedCollection, /// The detail of an entity that carries availability of its own (a Component). /// ``504 not-responding`` is the peer describing that entity as unreachable - @@ -428,7 +445,8 @@ SubResponse read_sub_response(const httplib::Result & result, const std::string out.kind = SubResponse::Kind::kRouteAbsent; return out; } - if (result->status == 504 && kind == RouteKind::kAddressableDetail && says_not_responding(result->body)) { + if (result->status == 504 && says_not_responding(result->body) && + (kind == RouteKind::kAddressableDetail || kind == RouteKind::kNestedCollection)) { out.kind = SubResponse::Kind::kEntityUnreachable; return out; } @@ -567,6 +585,12 @@ tl::expected PeerClient::fetch_entities() { note_absent_route("/areas/{id}/subareas"); continue; } + if (sub.kind == SubResponse::Kind::kEntityUnreachable) { + // The peer holds this Area's id but the gateway contributing it is + // silent, so its members are out of reach. That is one Area's worth of + // detail missing, not a failed read of the peer. + continue; + } auto subareas = parse_collection(sub.body, parse_area); for (auto & subarea : subareas) { if (!is_valid_entity_id(subarea.id)) { @@ -636,6 +660,11 @@ tl::expected PeerClient::fetch_entities() { note_absent_route("/components/{id}/subcomponents"); continue; } + if (sub.kind == SubResponse::Kind::kEntityUnreachable) { + // Same as for subareas: the parent is retained and unreachable, so what + // it contains cannot be read. The parent itself already carries that. + continue; + } auto subcomps = parse_collection(sub.body, parse_component); for (auto & subcomp : subcomps) { if (!is_valid_entity_id(subcomp.id)) { @@ -721,6 +750,12 @@ tl::expected PeerClient::fetch_entities() { note_absent_route("/apps/{id}/operations"); continue; } + if (ops.kind == SubResponse::Kind::kEntityUnreachable) { + // The App is retained and its gateway is silent, so the peer answers + // for it rather than proxying. It keeps the operations the peer already + // reported; there is nothing further to read. + continue; + } parse_operations_into(ops.body, app); } } @@ -836,8 +871,14 @@ void PeerClient::forward_request(const httplib::Request & req, httplib::Response // The x-medkit header allowlist must match headers the gateway actually produces. // Currently the only x-medkit HTTP header is X-Medkit-Local-Only (fault_handlers.cpp). // Update this list when adding new x-medkit HTTP response headers. - static const std::set allowed_headers = {"content-type", "etag", "cache-control", "last-modified", - "x-medkit-local-only"}; + // + // `location` is what a 201 or 202 hands the client to address the resource it + // just created. The peer builds it from the request it received, so it names + // the member's own route - a path this gateway resolves to the same member, + // and the only address of a resource that lives on the other side. Dropping it + // leaves the client a status with nothing to follow. + static const std::set allowed_headers = {"content-type", "etag", "cache-control", + "last-modified", "location", "x-medkit-local-only"}; res.status = result->status; res.body = result->body; diff --git a/src/ros2_medkit_gateway/test/test_aggregation_manager.cpp b/src/ros2_medkit_gateway/test/test_aggregation_manager.cpp index 388952f67..a2975c87d 100644 --- a/src/ros2_medkit_gateway/test/test_aggregation_manager.cpp +++ b/src/ros2_medkit_gateway/test/test_aggregation_manager.cpp @@ -498,23 +498,6 @@ TEST(AggregationManager, get_peer_url_returns_empty_for_unknown_peer) { EXPECT_TRUE(url.empty()); } -// ============================================================================= -// fetch_all_peer_entities tests (with unreachable peers) -// ============================================================================= - -TEST(AggregationManager, fetch_all_peer_entities_returns_empty_when_none_healthy) { - auto config = make_config(2); - AggregationManager manager(config); - - // No peers healthy -> empty result - auto entities = manager.fetch_all_peer_entities(); - - EXPECT_TRUE(entities.areas.empty()); - EXPECT_TRUE(entities.components.empty()); - EXPECT_TRUE(entities.apps.empty()); - EXPECT_TRUE(entities.functions.empty()); -} - // ============================================================================= // Prefix stripping tests (forward_request) // ============================================================================= @@ -1946,44 +1929,6 @@ TEST(AggregationManager, concurrent_fan_out_with_peer_mutations) { SUCCEED(); } -// ============================================================================= -// fetch_all_peer_entities happy-path with mock server -// ============================================================================= - -TEST(AggregationManager, fetch_all_peer_entities_returns_entities_from_healthy_peer) { - MockPeerServer mock; - install_entity_endpoints(mock.server(), 2, 1, 3, 0); - int port = mock.start(); - - AggregationConfig config; - config.enabled = true; - config.timeout_ms = 5000; - - AggregationConfig::PeerConfig peer; - peer.url = "http://127.0.0.1:" + std::to_string(port); - peer.name = "entity_peer"; - config.peers.push_back(peer); - - AggregationManager manager(config); - manager.check_all_health(); - ASSERT_EQ(manager.healthy_peer_count(), 1u); - - auto entities = manager.fetch_all_peer_entities(); - - EXPECT_EQ(entities.areas.size(), 2u); - EXPECT_EQ(entities.components.size(), 1u); - EXPECT_EQ(entities.apps.size(), 3u); - EXPECT_EQ(entities.functions.size(), 0u); - - // Verify source tagging - for (const auto & area : entities.areas) { - EXPECT_EQ(area.source, "peer:entity_peer"); - } - for (const auto & app : entities.apps) { - EXPECT_EQ(app.source, "peer:entity_peer"); - } -} - // ============================================================================= // forward_request happy-path with mock server // ============================================================================= diff --git a/src/ros2_medkit_gateway/test/test_peer_client.cpp b/src/ros2_medkit_gateway/test/test_peer_client.cpp index a5dd99be1..9d1c8a790 100644 --- a/src/ros2_medkit_gateway/test/test_peer_client.cpp +++ b/src/ros2_medkit_gateway/test/test_peer_client.cpp @@ -20,6 +20,7 @@ #include "ros2_medkit_gateway/core/aggregation/entity_merger.hpp" #include "ros2_medkit_gateway/core/aggregation/peer_client.hpp" +#include "ros2_medkit_gateway/core/http/error_codes.hpp" #include "ros2_medkit_gateway/http/handlers/handler_context.hpp" using namespace ros2_medkit_gateway; @@ -1004,3 +1005,189 @@ TEST(PeerClientHappyPath, asset_identity_survives_fetch_and_merge) { EXPECT_EQ(merged_id.firmware_version, "2.9.4"); EXPECT_EQ(merged_id.extra.at("slot"), "3"); } + +// ============================================================================= +// Availability read-back +// +// `x-medkit.available` is emitted only when false, so absence is the peer +// saying the entity is reachable. Both directions are pinned here because the +// two failures are opposite and equally silent: dropping the read reports every +// unreachable entity of every peer as reachable, and defaulting to false +// reports every healthy one as unreachable. +// ============================================================================= + +namespace { + +/// Serve the four collection roots a fetch always reads, so a test only has to +/// describe the routes it is actually about. httplib answers with the first +/// handler that matches, so this is installed AFTER the routes a test defines +/// itself and only fills in the ones it left out. +void install_empty_roots(httplib::Server & svr) { + svr.Get("/api/v1/areas", [](const httplib::Request &, httplib::Response & res) { + res.set_content(R"({"items":[]})", "application/json"); + }); + svr.Get("/api/v1/components", [](const httplib::Request &, httplib::Response & res) { + res.set_content(R"({"items":[]})", "application/json"); + }); + svr.Get("/api/v1/apps", [](const httplib::Request &, httplib::Response & res) { + res.set_content(R"({"items":[]})", "application/json"); + }); + svr.Get("/api/v1/functions", [](const httplib::Request &, httplib::Response & res) { + res.set_content(R"({"items":[]})", "application/json"); + }); +} + +/// A SOVD error body carrying the ``not-responding`` code, which is what a +/// gateway answers on every route of an entity whose contributor went quiet. +std::string not_responding_body(const std::string & member_id) { + return nlohmann::json( + {{"error_code", ERR_NOT_RESPONDING}, {"message", "Member '" + member_id + "' is not available"}}) + .dump(); +} + +/// Listen on a random port for the life of the scope. A failed ASSERT returns +/// from the test body, so a hand-written stop/join at the end of it never runs +/// and the listening thread outlives its server. +class ScopedServer { + public: + explicit ScopedServer(httplib::Server & svr) : svr_(svr) { + port_ = svr_.bind_to_any_port("127.0.0.1"); + thread_ = std::thread([this]() { + svr_.listen_after_bind(); + }); + } + + ~ScopedServer() { + svr_.stop(); + if (thread_.joinable()) { + thread_.join(); + } + } + + ScopedServer(const ScopedServer &) = delete; + ScopedServer & operator=(const ScopedServer &) = delete; + ScopedServer(ScopedServer &&) = delete; + ScopedServer & operator=(ScopedServer &&) = delete; + + std::string url() const { + return "http://127.0.0.1:" + std::to_string(port_); + } + + private: + httplib::Server & svr_; + std::thread thread_; + int port_{0}; +}; + +} // namespace + +TEST(PeerClientAvailability, a_peers_statement_that_an_entity_is_unreachable_is_carried) { + httplib::Server svr; + + svr.Get("/api/v1/components", [](const httplib::Request &, httplib::Response & res) { + res.set_content(R"({"items":[{"id":"ecu-live","name":"ecu-live"},{"id":"ecu-quiet","name":"ecu-quiet"}]})", + "application/json"); + }); + // A 200 detail that names the entity unreachable. This is the only account of + // the entity the aggregator gets, so the flag has to come off the body. + svr.Get("/api/v1/components/ecu-quiet", [](const httplib::Request &, httplib::Response & res) { + res.set_content(R"({"id":"ecu-quiet","name":"ecu-quiet","x-medkit":{"source":"manifest","available":false}})", + "application/json"); + }); + svr.Get("/api/v1/components/ecu-live", [](const httplib::Request &, httplib::Response & res) { + res.set_content(R"({"id":"ecu-live","name":"ecu-live","x-medkit":{"source":"manifest"}})", "application/json"); + }); + svr.Get(R"(/api/v1/components/([^/]+)/subcomponents)", [](const httplib::Request &, httplib::Response & res) { + res.set_content(R"({"items":[]})", "application/json"); + }); + svr.Get("/api/v1/apps", [](const httplib::Request &, httplib::Response & res) { + res.set_content( + R"({"items":[ + {"id":"app-live","name":"app-live","x-medkit":{"source":"manifest","is_online":true}}, + {"id":"app-quiet","name":"app-quiet","x-medkit":{"source":"manifest","available":false}} + ]})", + "application/json"); + }); + svr.Get(R"(/api/v1/apps/([^/]+)/operations)", [](const httplib::Request &, httplib::Response & res) { + res.set_content(R"({"items":[]})", "application/json"); + }); + install_empty_roots(svr); + + ScopedServer running(svr); + PeerClient client(running.url(), "peer_b", 5000); + auto result = client.fetch_entities(); + ASSERT_TRUE(result.has_value()) << result.error(); + + ASSERT_EQ(result->components.size(), 2u); + EXPECT_EQ(result->components[0].id, "ecu-live"); + EXPECT_TRUE(result->components[0].available) << "an entity the peer said nothing about was marked unreachable"; + EXPECT_EQ(result->components[1].id, "ecu-quiet"); + EXPECT_FALSE(result->components[1].available) << "the peer's statement that ecu-quiet is unreachable was dropped"; + + ASSERT_EQ(result->apps.size(), 2u); + EXPECT_EQ(result->apps[0].id, "app-live"); + EXPECT_TRUE(result->apps[0].available) << "an app the peer said nothing about was marked unreachable"; + EXPECT_EQ(result->apps[1].id, "app-quiet"); + EXPECT_FALSE(result->apps[1].available) << "the peer's statement that app-quiet is unreachable was dropped"; +} + +TEST(PeerClientAvailability, a_nested_collection_that_says_not_responding_does_not_abort_the_fetch) { + httplib::Server svr; + + svr.Get("/api/v1/components", [](const httplib::Request &, httplib::Response & res) { + res.set_content(R"({"items":[{"id":"ecu-quiet","name":"ecu-quiet"}]})", "application/json"); + }); + svr.Get("/api/v1/components/ecu-quiet", [](const httplib::Request &, httplib::Response & res) { + res.set_content(R"({"id":"ecu-quiet","name":"ecu-quiet","x-medkit":{"source":"manifest","available":false}})", + "application/json"); + }); + // Every route of a retained member answers 504 not-responding, the nested + // collections included. Read as a failed request it would discard the peer's + // whole picture, including the entities that are perfectly reachable. + svr.Get("/api/v1/components/ecu-quiet/subcomponents", [](const httplib::Request &, httplib::Response & res) { + res.status = 504; + res.set_content(not_responding_body("ecu-quiet"), "application/json"); + }); + svr.Get("/api/v1/apps", [](const httplib::Request &, httplib::Response & res) { + res.set_content(R"({"items":[{"id":"app-quiet","name":"app-quiet","x-medkit":{"available":false}}]})", + "application/json"); + }); + svr.Get("/api/v1/apps/app-quiet/operations", [](const httplib::Request &, httplib::Response & res) { + res.status = 504; + res.set_content(not_responding_body("app-quiet"), "application/json"); + }); + install_empty_roots(svr); + + ScopedServer running(svr); + PeerClient client(running.url(), "peer_b", 5000); + auto result = client.fetch_entities(); + ASSERT_TRUE(result.has_value()) << "one unreachable member discarded the peer's whole picture: " << result.error(); + + ASSERT_EQ(result->components.size(), 1u); + EXPECT_FALSE(result->components[0].available); + ASSERT_EQ(result->apps.size(), 1u); + EXPECT_FALSE(result->apps[0].available); +} + +TEST(PeerClientAvailability, a_504_that_is_not_a_statement_about_an_entity_still_fails_the_fetch) { + httplib::Server svr; + + svr.Get("/api/v1/components", [](const httplib::Request &, httplib::Response & res) { + res.set_content(R"({"items":[{"id":"ecu-a","name":"ecu-a"}]})", "application/json"); + }); + svr.Get("/api/v1/components/ecu-a", [](const httplib::Request &, httplib::Response & res) { + res.set_content(R"({"id":"ecu-a","name":"ecu-a"})", "application/json"); + }); + // 504 without the not-responding code is an upstream proxy timing out, which + // says nothing about the entity and leaves a hole in the picture. + svr.Get("/api/v1/components/ecu-a/subcomponents", [](const httplib::Request &, httplib::Response & res) { + res.status = 504; + res.set_content(R"({"error_code":"vendor-error","message":"upstream timed out"})", "application/json"); + }); + install_empty_roots(svr); + + ScopedServer running(svr); + PeerClient client(running.url(), "peer_b", 5000); + auto result = client.fetch_entities(); + EXPECT_FALSE(result.has_value()) << "a gateway timeout was read as a statement that an entity is unreachable"; +} diff --git a/src/ros2_medkit_integration_tests/test/features/test_daisy_chain_aggregation.test.py b/src/ros2_medkit_integration_tests/test/features/test_daisy_chain_aggregation.test.py index e415d339c..4a9203006 100644 --- a/src/ros2_medkit_integration_tests/test/features/test_daisy_chain_aggregation.test.py +++ b/src/ros2_medkit_integration_tests/test/features/test_daisy_chain_aggregation.test.py @@ -27,6 +27,8 @@ plus "peer:peer_b"; each hop surfaces only its direct upstream. - ``/health.warnings`` is an empty array when there are no deployment anomalies. +- Killing ``peer_C`` leaves its declared entities in the primary's tree + marked unavailable, two hops from where they were declared. DDS isolation uses three distinct domain IDs so that the gateways cannot discover each other's nodes via ROS 2 graph introspection - the only @@ -34,6 +36,7 @@ """ import os +import signal import tempfile import textwrap import time @@ -326,6 +329,85 @@ def test_app_detail_contributors_present(self): contributors = r.json().get('x-medkit', {}).get('contributors', []) self.assertIn('local', contributors) + # --- Unreachability across two hops ---------------------------------- + # + # Named to sort last in the class: it kills the tail of the chain, and + # every case above needs all three gateways answering. + + @staticmethod + def _primary_app(app_id): + """One App as the head of the chain currently lists it, or None.""" + response = requests.get(f'{PRIMARY_URL}/apps', timeout=10) + if response.status_code != 200: + return None + for item in response.json().get('items', []): + if item.get('id') == app_id: + return item + return None + + @staticmethod + def _primary_subcomponent(parent_id, subcomponent_id): + """One subcomponent as the head of the chain currently lists it.""" + response = requests.get( + f'{PRIMARY_URL}/components/{parent_id}/subcomponents', timeout=10) + if response.status_code != 200: + return None + for item in response.json().get('items', []): + if item.get('id') == subcomponent_id: + return item + return None + + def test_z_a_dead_tail_is_unreachable_at_the_head_of_the_chain(self, peer_c): + """Unreachability is a fact about an entity, not about one link. + + peer_B holds peer_C's declaration and marks it unavailable when peer_C + stops answering. The primary is a hop further out and never talks to + peer_C at all, so the only thing that can tell it is peer_B's answer - + which it has to read back. Asserted on the flag rather than on a status + code, because an entity reported as reachable is a plausible-looking + 200 built from a stale picture, and both entity kinds are checked: an + App also carries ``is_online``, a Component carries nothing else. + """ + comp_before = self._primary_subcomponent('robot-x', 'ecu-c') + self.assertIsNotNone( + comp_before, 'ecu-c was never merged two hops out; nothing to test') + self.assertNotEqual( + comp_before.get('x-medkit', {}).get('available'), False, + f'ecu-c was already unavailable before peer_C was killed: {comp_before}') + app_before = self._primary_app('ecu-c-app') + self.assertIsNotNone( + app_before, 'ecu-c-app was never merged two hops out; nothing to test') + self.assertNotEqual( + app_before.get('x-medkit', {}).get('available'), False, + f'ecu-c-app was already unavailable before peer_C was killed: {app_before}') + + os.kill(peer_c.process_details['pid'], signal.SIGTERM) + + # Two refreshes have to land in sequence - peer_B notices, then the + # primary reads what peer_B now says - so the window is wider than the + # single-hop case. It is still a small multiple of refresh_interval_ms + # (1000 ms for a test gateway), not a wait for something unbounded. + deadline = time.monotonic() + 45.0 + comp = app = None + while time.monotonic() < deadline: + comp = self._primary_subcomponent('robot-x', 'ecu-c') + app = self._primary_app('ecu-c-app') + if (comp is not None and comp.get('x-medkit', {}).get('available') is False + and app is not None and app.get('x-medkit', {}).get('available') is False): + break + time.sleep(0.5) + + self.assertIsNotNone( + comp, 'ecu-c vanished from the head of the chain instead of being retained') + self.assertIs( + comp.get('x-medkit', {}).get('available'), False, + f'a Component behind a dead gateway is reported reachable: {comp}') + self.assertIsNotNone( + app, 'ecu-c-app vanished from the head of the chain instead of being retained') + self.assertIs( + app.get('x-medkit', {}).get('available'), False, + f'an App behind a dead gateway is reported reachable: {app}') + @launch_testing.post_shutdown_test() class TestDaisyShutdown(unittest.TestCase): From 0fc6388318f9b66182815c54e81800b4c587606b Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Thu, 20 Aug 2026 11:44:32 +0200 Subject: [PATCH 11/22] fix(operations): reach a peer-owned execution through the aggregate Reading, updating and cancelling an execution resolved only goals tracked here, so a member's execution on another gateway answered not-found through the aggregate while the same request against the peer's own route answered normally. The operation id is in the route, so the owning member can be resolved exactly as listing them now does. The forwarded Location that should have pointed clients at the owning gateway was being removed by the response header allowlist, so this was a broken promise rather than an inconsistency. --- .../src/http/handlers/operation_handlers.cpp | 71 ++++++ .../test_grouping_entity_aggregation.test.py | 221 ++++++++++++++++++ 2 files changed, 292 insertions(+) diff --git a/src/ros2_medkit_gateway/src/http/handlers/operation_handlers.cpp b/src/ros2_medkit_gateway/src/http/handlers/operation_handlers.cpp index 543d1dbab..f4373856d 100644 --- a/src/ros2_medkit_gateway/src/http/handlers/operation_handlers.cpp +++ b/src/ros2_medkit_gateway/src/http/handlers/operation_handlers.cpp @@ -281,6 +281,47 @@ std::optional refuse_if_ambiguous(const AggregatedOperations & ops, c return make_error(400, ERR_INVALID_REQUEST, "Ambiguous operation id: it names more than one operation", params); } +/// Settle which gateway holds the execution this request addresses. +/// +/// An execution lives on the gateway that started it, which is the gateway that +/// owns the member behind the operation id - the same one POST was dispatched +/// to. The operation id is in the route, so the owner is resolvable by the rule +/// the executions collection already resolves it by, and the three follow-up +/// verbs reach the goals the collection hands out. +/// +/// An id this gateway cannot resolve to exactly one owned operation is left to +/// the local path: the execution id is the key there, and the answer it gives +/// for an id naming no goal is the answer this route already owes. Resolving is +/// how the owner is found, never a second place for the route to refuse. +http::Result dispatch_execution_to_owner(const HandlerContext & ctx, const http::TypedRequest & req, + const std::string & entity_id, + const std::string & operation_id, + const std::string & execution_id) { + const auto entity_info = ctx.get_entity_info(entity_id); + const auto & cache = ctx.node()->get_thread_safe_cache(); + auto lookup = resolve_entity_operations(cache, entity_info.sovd_type(), entity_id); + if (!lookup) { + return MemberDispatch::kServeLocally; + } + const auto & ops = lookup->ops; + + auto parsed = http::parse_member_qualified_id(operation_id, ops.is_aggregated); + auto resolved = resolve_operation(ops, parsed); + if (!resolved.found() || refuse_if_ambiguous(ops, parsed, entity_id, operation_id).has_value()) { + return MemberDispatch::kServeLocally; + } + + const std::string & full_path = + resolved.service.has_value() ? resolved.service->full_path : resolved.action->full_path; + auto owner = ops.owner_by_path.find(full_path); + if (owner == ops.owner_by_path.end()) { + return MemberDispatch::kServeLocally; + } + return ctx.dispatch_to_member( + req, owner->second, "operations/" + parsed.item_id + "/executions/" + execution_id, + json{{"entity_id", entity_id}, {"operation_id", operation_id}, {"execution_id", execution_id}}); +} + /// True for a member that is in the tree but whose gateway is silent. /// /// A retained member is kept precisely so that the answer to a request does not @@ -1189,6 +1230,16 @@ http::Result OperationHandlers::get_execution(const htt json{{"details", vr.error()}, {"entity_id", entity_id}})); } + { + auto dispatch = dispatch_execution_to_owner(ctx_, req, entity_id, operation_id, execution_id); + if (!dispatch) { + return tl::make_unexpected(dispatch.error()); + } + if (*dispatch == MemberDispatch::kForwarded) { + return tl::make_unexpected(HandlerContext::forwarded_sentinel_error()); + } + } + auto * operation_mgr = ctx_.node()->get_operation_manager(); auto goal_info = operation_mgr->get_tracked_goal(execution_id); if (!goal_info.has_value()) { @@ -1253,6 +1304,16 @@ http::Result OperationHandlers::cancel_execution(const http::Ty return tl::make_unexpected(lock_err.error()); } + { + auto dispatch = dispatch_execution_to_owner(ctx_, req, entity_id, operation_id, execution_id); + if (!dispatch) { + return tl::make_unexpected(dispatch.error()); + } + if (*dispatch == MemberDispatch::kForwarded) { + return tl::make_unexpected(HandlerContext::forwarded_sentinel_error()); + } + } + auto * operation_mgr = ctx_.node()->get_operation_manager(); auto goal_info = operation_mgr->get_tracked_goal(execution_id); if (!goal_info.has_value()) { @@ -1313,6 +1374,16 @@ OperationHandlers::update_execution(const http::TypedRequest & req, const dto::E return tl::make_unexpected(lock_err.error()); } + { + auto dispatch = dispatch_execution_to_owner(ctx_, req, entity_id, operation_id, execution_id); + if (!dispatch) { + return tl::make_unexpected(dispatch.error()); + } + if (*dispatch == MemberDispatch::kForwarded) { + return tl::make_unexpected(HandlerContext::forwarded_sentinel_error()); + } + } + const std::string capability = body.capability; auto * operation_mgr = ctx_.node()->get_operation_manager(); diff --git a/src/ros2_medkit_integration_tests/test/features/test_grouping_entity_aggregation.test.py b/src/ros2_medkit_integration_tests/test/features/test_grouping_entity_aggregation.test.py index 9ab7a0c70..371e2dca6 100644 --- a/src/ros2_medkit_integration_tests/test/features/test_grouping_entity_aggregation.test.py +++ b/src/ros2_medkit_integration_tests/test/features/test_grouping_entity_aggregation.test.py @@ -147,6 +147,10 @@ PRIMARY_LONG_APP = 'primary_long_calibration' PEER_LONG_APP = 'peer_long_calibration' LONG_OPERATION = 'long_calibration' +# Calibration steps for a goal a case means to stop rather than watch finish. +# The demo action server runs one goal at a time at 2 Hz, so a long goal left +# behind by one case is a wait every later case pays. +SHORT_GOAL_ORDER = 6 PEER_LONG_NAMESPACE = '/chassis/brakes' # Declared with the SAME id on both gateways. Apps are renamed on collision, @@ -1039,6 +1043,223 @@ def test_a_goal_started_on_a_peer_owned_member_is_listed_through_the_aggregate(s execution_id, here, f"a peer's goal is reported under the local member: {here}") + # ---------------------------------------------------------------- R5b + # ADDRESSING ONE EXECUTION. + # + # The collection hands out ids. Every verb that takes one back has to + # resolve it the same way the collection did, or the aggregate offers + # addresses its own siblings answer 404 for. + + def _one_execution_url(self, entity_path, operation_id, execution_id, base_url=PRIMARY_URL): + return (f'{self._executions_url(entity_path, operation_id, base_url)}' + f'/{quote(execution_id, safe="")}') + + def _wait_for_action_server_free(self, entity_path, operation_id, base_url, timeout=30.0): + """Block until nothing is running on the operation's action server. + + The demo action server runs one goal at a time: its accept callback + joins the previous execution thread before starting the next, so a goal + an earlier case left running blocks every request to that server for as + long as it lasts - the cancels this section is about included. Read + through the member's own route so the wait never depends on the + addressing path under test. + """ + deadline = time.monotonic() + timeout + running = None + while time.monotonic() < deadline: + running = [] + for execution_id in self._execution_ids(entity_path, operation_id, base_url): + response = requests.get( + self._one_execution_url(entity_path, operation_id, execution_id, base_url), + timeout=10) + if response.status_code == 200 and response.json().get('status') == 'running': + running.append(execution_id) + if not running: + return + time.sleep(0.5) + raise AssertionError( + f'{operation_id!r} on {entity_path} still has {running} running after {timeout}s') + + def _peer_action_server_free(self): + self._wait_for_action_server_free( + f'apps/{PEER_LONG_APP}', LONG_OPERATION, PEER_URL) + + def _local_action_server_free(self): + self._wait_for_action_server_free( + f'apps/{PRIMARY_LONG_APP}', LONG_OPERATION, PRIMARY_URL) + + def _wait_until_executing(self, execution_url, timeout=15.0): + """Block until the action server has picked the goal up. + + A goal is addressable the moment it is accepted, but in that window the + server has not started it, and a cancel sent there stays outstanding + until it does - long enough for the aggregate's forward to time out and + report a broken link instead of the answer. ``ros2_status`` is the field + that separates accepted from executing; the SOVD ``status`` calls both + "running". + """ + deadline = time.monotonic() + timeout + last = None + while time.monotonic() < deadline: + response = requests.get(execution_url, timeout=10) + if response.status_code == 200: + last = response.json() + if last.get('x-medkit', {}).get('ros2_status') == 'executing': + return last + time.sleep(0.1) + raise AssertionError( + f'{execution_url} never reported itself executing within {timeout}s; last read {last}') + + def test_a_peer_owned_execution_is_readable_through_the_aggregate(self): + """R5b for GET: the id the collection offered resolves through it. + + The goal lives on the peer, so this gateway tracks nothing for it. + Asserted on the body rather than on 200 alone, and against what the + peer itself answers for the same id: the failure mode being excluded + is a plausible-looking status built somewhere other than where the + goal is. + """ + entity_path = f'functions/{MERGED_FUNCTION}' + peer_id = f'{PEER_LONG_APP}:{LONG_OPERATION}' + self._wait_for_operation(entity_path, peer_id) + self._peer_action_server_free() + execution_id = self._start_goal(entity_path, peer_id, order=SHORT_GOAL_ORDER) + + through_aggregate = requests.get( + self._one_execution_url(entity_path, peer_id, execution_id), timeout=20) + self.assertEqual( + through_aggregate.status_code, 200, + f'an id the executions collection offers is unreadable through it: ' + f'{through_aggregate.text}') + body = through_aggregate.json() + x_medkit = body.get('x-medkit', {}) + self.assertEqual(x_medkit.get('goal_id'), execution_id, body) + + on_the_peer = requests.get( + self._one_execution_url( + f'apps/{PEER_LONG_APP}', LONG_OPERATION, execution_id, base_url=PEER_URL), + timeout=20) + self.assertEqual(on_the_peer.status_code, 200, on_the_peer.text) + peer_x_medkit = on_the_peer.json().get('x-medkit', {}) + # The ROS action path is the member's own and is not derivable from the + # aggregate's namespace, so matching it is what says the answer came + # from the gateway that holds the goal. + self.assertEqual( + x_medkit.get('ros2', {}).get('action'), + peer_x_medkit.get('ros2', {}).get('action'), + f'the aggregate answered for a different action than the peer holds: {body}', + ) + self.assertEqual(x_medkit.get('goal_id'), peer_x_medkit.get('goal_id'), body) + + def test_a_peer_owned_execution_is_stoppable_through_the_aggregate(self): + """R5b for PUT: the capability lands on the gateway holding the goal. + + The 202 carries a Location, and that Location has to name the member's + own route - a Location rebuilt from the aggregate's path would send the + client back to a gateway that cannot resolve the id. + """ + entity_path = f'functions/{MERGED_FUNCTION}' + peer_id = f'{PEER_LONG_APP}:{LONG_OPERATION}' + self._wait_for_operation(entity_path, peer_id) + self._peer_action_server_free() + execution_id = self._start_goal(entity_path, peer_id, order=SHORT_GOAL_ORDER) + + self._wait_until_executing( + self._one_execution_url(entity_path, peer_id, execution_id)) + + stopped = requests.put( + self._one_execution_url(entity_path, peer_id, execution_id), + json={'capability': 'stop'}, + timeout=20, + ) + self.assertEqual( + stopped.status_code, 202, + f'stopping a peer-owned execution through the aggregate failed: {stopped.text}') + self.assertEqual(stopped.json().get('id'), execution_id, stopped.text) + self.assertIn( + PEER_LONG_APP, stopped.headers.get('Location', ''), + f'the Location does not name the member that holds the goal: {stopped.headers}') + + # And the goal really stopped, on the peer, where it was running. + self._assert_goal_stops( + self._one_execution_url( + f'apps/{PEER_LONG_APP}', LONG_OPERATION, execution_id, base_url=PEER_URL)) + + def test_a_peer_owned_execution_is_cancellable_through_the_aggregate(self): + """R5b for DELETE, and the case where a false success costs the most. + + A 204 produced locally reads exactly like a cancel that worked while + the goal keeps running on the peer, so the peer is asked afterwards. + """ + entity_path = f'functions/{MERGED_FUNCTION}' + peer_id = f'{PEER_LONG_APP}:{LONG_OPERATION}' + self._wait_for_operation(entity_path, peer_id) + self._peer_action_server_free() + execution_id = self._start_goal(entity_path, peer_id, order=SHORT_GOAL_ORDER) + + self._wait_until_executing( + self._one_execution_url(entity_path, peer_id, execution_id)) + + cancelled = requests.delete( + self._one_execution_url(entity_path, peer_id, execution_id), timeout=20) + self.assertEqual( + cancelled.status_code, 204, + f'cancelling a peer-owned execution through the aggregate failed: {cancelled.text}') + + self._assert_goal_stops( + self._one_execution_url( + f'apps/{PEER_LONG_APP}', LONG_OPERATION, execution_id, base_url=PEER_URL)) + + def test_a_locally_owned_execution_is_still_answered_here(self): + """R5b's other half: dispatching did not move the local case anywhere. + + The local member's goal is tracked on this gateway, so all three verbs + have to keep answering from here - and the peer, asked for the same id, + has to say it never heard of it. + """ + entity_path = f'functions/{MERGED_FUNCTION}' + local_id = f'{PRIMARY_LONG_APP}:{LONG_OPERATION}' + self._wait_for_operation(entity_path, local_id) + self._local_action_server_free() + execution_id = self._start_goal(entity_path, local_id, order=SHORT_GOAL_ORDER) + + here = requests.get( + self._one_execution_url(entity_path, local_id, execution_id), timeout=20) + self.assertEqual(here.status_code, 200, here.text) + self.assertEqual( + here.json().get('x-medkit', {}).get('goal_id'), execution_id, here.text) + + not_there = requests.get( + self._one_execution_url( + f'apps/{PEER_LONG_APP}', LONG_OPERATION, execution_id, base_url=PEER_URL), + timeout=20) + self.assertEqual( + not_there.status_code, 404, + f'a local goal was found on the peer, so the ids are not distinguishing: ' + f'{not_there.text}') + + self._wait_until_executing( + self._one_execution_url(entity_path, local_id, execution_id)) + cancelled = requests.delete( + self._one_execution_url(entity_path, local_id, execution_id), timeout=20) + self.assertEqual(cancelled.status_code, 204, cancelled.text) + + def _assert_goal_stops(self, execution_url, timeout=20.0): + """Block until the execution at `execution_url` is no longer running.""" + deadline = time.monotonic() + timeout + last = None + while time.monotonic() < deadline: + response = requests.get(execution_url, timeout=10) + if response.status_code == 404: + return + if response.status_code == 200: + last = response.json().get('status') + if last != 'running': + return + time.sleep(0.25) + raise AssertionError( + f'{execution_url} still reports {last!r} {timeout}s after it was stopped') + def test_a_service_operation_has_an_empty_execution_collection_not_a_miss(self): """R6 for executions: present-but-empty is not the same as absent. From b2e8713b94df04677041c11dda66eaa751f066d3 Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Thu, 20 Aug 2026 11:44:33 +0200 Subject: [PATCH 12/22] test(triggers): start the second gateway when the trigger exists A restart restores what the persistent store holds at the moment the gateway is constructed. The test started its second gateway on a fixed timer, so a slow setup left the store empty when that gateway came up and the later cases failed on a trigger that had never been restored. The gateway now starts because the trigger is in the store, not because a clock said so. --- .../features/test_triggers_persistent.test.py | 76 ++++++++++++++----- 1 file changed, 58 insertions(+), 18 deletions(-) diff --git a/src/ros2_medkit_integration_tests/test/features/test_triggers_persistent.test.py b/src/ros2_medkit_integration_tests/test/features/test_triggers_persistent.test.py index f63ea856b..d5fc08ae4 100644 --- a/src/ros2_medkit_integration_tests/test/features/test_triggers_persistent.test.py +++ b/src/ros2_medkit_integration_tests/test/features/test_triggers_persistent.test.py @@ -36,7 +36,8 @@ import unittest from launch import LaunchDescription -from launch.actions import TimerAction +from launch.actions import ExecuteProcess, RegisterEventHandler, TimerAction +from launch.event_handlers import OnProcessExit import launch_testing import launch_testing.actions import requests @@ -67,6 +68,19 @@ f'test_triggers_persist_{os.getpid()}.db', ) +# The secondary gateway stands in for a restart, and a restart only restores +# what the store already held: ``load_persistent_triggers()`` runs once, while +# the gateway is being constructed. So the secondary must not be started until +# the trigger exists in the shared DB, and the only thing that knows when that +# is true is the test that created it. This path is the handshake - test_01 +# creates it, and the secondary's start is chained to a process that waits for +# it - so the order is a consequence of the work rather than of how long +# ``setUpClass`` happened to take. +GATE_PATH = os.path.join( + tempfile.gettempdir(), + f'test_triggers_persist_gate_{os.getpid()}', +) + APP_ID = 'temp_sensor' RESOURCE_URI = f'/api/v1/apps/{APP_ID}/faults' @@ -98,17 +112,20 @@ def generate_test_description(): actions=demo + [launch_testing.actions.ReadyToTest()], ) - # Secondary gateway delayed 25s - tests 01-02 create the trigger on - # primary first, then secondary starts and loads it from shared DB. - # This simulates a gateway restart: primary creates trigger -> "restart" - # (secondary starts with same DB) -> secondary loads persistent triggers. - delayed_secondary = TimerAction( - period=25.0, - actions=[secondary], + # The secondary starts when the gate file appears, never on a clock: it + # simulates a gateway restart, and a restart that happens before the + # trigger is created restores nothing. + gate = ExecuteProcess( + cmd=['sh', '-c', f'while [ ! -e "{GATE_PATH}" ]; do sleep 0.2; done'], + name='secondary_start_gate', + output='screen', + ) + gated_secondary = RegisterEventHandler( + OnProcessExit(target_action=gate, on_exit=[secondary]), ) return ( - LaunchDescription([primary, delayed_secondary, delayed_demo]), + LaunchDescription([primary, gate, gated_secondary, delayed_demo]), {'primary': primary, 'secondary': secondary}, ) @@ -117,6 +134,15 @@ def generate_test_description(): # Helpers # --------------------------------------------------------------------------- +def _remove_gate(): + """Drop the gate file so a rerun does not inherit an open gate.""" + if os.path.exists(GATE_PATH): + try: + os.unlink(GATE_PATH) + except OSError: + pass + + def _wait_for_health(base_url, *, timeout=30.0): """Poll /health until 200 or timeout.""" deadline = time.monotonic() + timeout @@ -173,7 +199,11 @@ def setUpClass(cls): _wait_for_app(BASE_URL_PRIMARY, APP_ID, timeout=60.0) # Allow a few discovery refresh cycles so the entity is stable time.sleep(3.0) - # Secondary gateway starts later (25s delay) - waited in test_03 + # The secondary is not running yet: it starts once test_01 opens the + # gate, and test_03 waits for it there. The gate file is dropped at the + # end of the class so a rerun in the same temp directory cannot inherit + # an open one. + cls.addClassCleanup(_remove_gate) # ------------------------------------------------------------------ # Test 01: create a persistent trigger on the PRIMARY gateway @@ -210,6 +240,12 @@ def test_01_create_persistent_trigger(self): # Stash the ID for subsequent tests. TestTriggersPersistent._trigger_id = trig['id'] + # The persistent trigger is now a row in the shared store, which is the + # precondition the secondary's restore path needs. Opening the gate + # starts it. + with open(GATE_PATH, 'w', encoding='utf-8') as gate: + gate.write(trig['id']) + # ------------------------------------------------------------------ # Test 02: trigger is listed on the PRIMARY gateway # ------------------------------------------------------------------ @@ -248,8 +284,8 @@ def test_03_trigger_restored_on_secondary(self): 'test_01 must set _trigger_id before test_03 runs', ) - # Wait for the secondary gateway to start (delayed 25s from launch) - # and discover the entity. + # The secondary starts only after test_01 opened the gate, so this is + # the first point at which it can be up at all. _wait_for_health(BASE_URL_SECONDARY, timeout=60.0) _wait_for_app(BASE_URL_SECONDARY, APP_ID, timeout=30.0) time.sleep(2.0) # Allow discovery to stabilize @@ -391,9 +427,13 @@ def test_exit_codes(self, proc_info): f'{info.process_name} exited with code {info.returncode}', ) - # Clean up the shared SQLite DB if it was created. - if os.path.exists(DB_PATH): - try: - os.unlink(DB_PATH) - except OSError: - pass + # Clean up the shared SQLite DB if it was created. SQLite in WAL mode + # writes two sidecars next to it, and a run that leaves them behind + # leaves state a later run can find. + for path in (DB_PATH, f'{DB_PATH}-wal', f'{DB_PATH}-shm'): + if os.path.exists(path): + try: + os.unlink(path) + except OSError: + pass + _remove_gate() From e08e67faad332b94e2b22daaa9e07238f0b9ee6f Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Thu, 20 Aug 2026 11:44:34 +0200 Subject: [PATCH 13/22] fix(subscriptions): refuse a resource path the collection cannot stream A subscription URI may name a single resource after the collection, and only data required one. For faults, configurations and logs the path was parsed, recorded on the subscription and then discarded by the sampler, which streams those collections whole. A client asking for one parameter was told the subscription existed and then handed everything, every tick. A sampler now declares whether it honours a resource path, and a path given to one that does not is refused with the collection named. Not declaring means not honouring, so refusing is the direction that cannot silently answer a different question. --- docs/api/rest.rst | 23 ++- docs/tutorials/graph-provider.rst | 4 + docs/tutorials/plugin-system.rst | 50 ++++-- .../core/plugins/plugin_context.hpp | 9 +- .../core/plugins/plugin_types.hpp | 9 +- .../core/resource_sampler.hpp | 19 ++- .../handlers/cyclic_subscription_handlers.hpp | 10 ++ .../src/core/resource_sampler.cpp | 11 +- src/ros2_medkit_gateway/src/gateway_node.cpp | 10 +- .../handlers/cyclic_subscription_handlers.cpp | 17 ++ .../src/plugins/plugin_context.cpp | 6 +- .../test_cyclic_subscription_handlers.cpp | 150 ++++++++++++++++++ .../test/test_plugin_manager.cpp | 32 +++- .../test/test_resource_sampler_registry.cpp | 83 ++++++++++ ...est_multi_collection_subscriptions.test.py | 112 ++++++++++++- .../design/index.rst | 5 + .../test/test_graph_provider_plugin.cpp | 19 ++- .../test/test_opcua_identity.cpp | 4 +- .../test/test_opcua_plugin.cpp | 3 +- .../test/test_sovd_service_interface.cpp | 3 +- 20 files changed, 536 insertions(+), 43 deletions(-) diff --git a/docs/api/rest.rst b/docs/api/rest.rst index dd52d0dcc..e2eab8374 100644 --- a/docs/api/rest.rst +++ b/docs/api/rest.rst @@ -1921,11 +1921,17 @@ Subscriptions are temporary - they do not survive server restart. **Supported collections:** -- ``data`` - Topic data (requires a resource path, e.g. ``/data/temperature``) -- ``faults`` - Fault list (resource path optional, e.g. ``/faults`` or ``/faults/fault_001``) -- ``configurations`` - Parameter values (resource path optional) -- ``logs`` - Application log entries from ``/rosout`` -- ``x-*`` - Vendor extensions (e.g. ``x-medkit-graph``) +- ``data`` - Topic data. Requires a resource path naming the topic, e.g. ``/data/temperature`` +- ``faults`` - Fault list. Streamed as a whole; no resource path +- ``configurations`` - Parameter values. Streamed as a whole; no resource path +- ``logs`` - Application log entries from ``/rosout``. Streamed as a whole; no resource path +- ``x-*`` - Vendor extensions (e.g. ``x-medkit-graph``). Streamed as a whole unless the + plugin registering the sampler declares that it narrows its payload to a named resource + +A collection that is streamed as a whole delivers every item of that collection on +every tick. A resource URI naming a single item of such a collection is refused with +400 ``x-medkit-invalid-resource-uri`` rather than accepted and answered with the whole +collection. **Interval values:** @@ -1953,7 +1959,8 @@ Subscriptions are temporary - they do not survive server restart. - ``resource`` (string, required): Full SOVD resource URI to observe (e.g. ``/api/v1/apps/{id}/data/{topic}``, ``/api/v1/apps/{id}/faults``, - ``/api/v1/functions/{id}/x-medkit-graph``) + ``/api/v1/functions/{id}/x-medkit-graph``). The URI ends at the collection + unless that collection's sampler narrows its payload to a named resource - ``protocol`` (string, optional): Transport protocol. Only ``"sse"`` supported. Default: ``"sse"`` - ``interval`` (string, required): One of ``fast``, ``normal``, ``slow`` - ``duration`` (integer, required): Subscription lifetime in seconds. @@ -1962,7 +1969,9 @@ Subscriptions are temporary - they do not survive server restart. **Error responses:** - **400** ``invalid-parameter`` - Invalid interval, duration <= 0, or duration exceeds max - - **400** ``x-medkit-invalid-resource-uri`` - Malformed resource URI or path traversal + - **400** ``x-medkit-invalid-resource-uri`` - Malformed resource URI, path traversal, + ``data`` without a topic path, or a resource path on a collection that is streamed + as a whole - **400** ``x-medkit-entity-mismatch`` - Resource URI references different entity than route - **400** ``x-medkit-collection-not-supported`` - Entity doesn't support the collection - **400** ``x-medkit-collection-not-available`` - No data provider registered for collection diff --git a/docs/tutorials/graph-provider.rst b/docs/tutorials/graph-provider.rst index 8e34255b9..eb305285f 100644 --- a/docs/tutorials/graph-provider.rst +++ b/docs/tutorials/graph-provider.rst @@ -317,6 +317,10 @@ resource, so a client can receive periodic graph snapshots over Server-Sent Events instead of polling. See :doc:`/api/rest` for the general cyclic-subscription API; the graph-specific parts are below. +Each tick carries the whole graph document for the function. The resource URI +must therefore end at ``x-medkit-graph`` - appending a path below it is refused +with 400. + Create the subscription: .. code-block:: bash diff --git a/docs/tutorials/plugin-system.rst b/docs/tutorials/plugin-system.rst index 68af2b67b..9851103c8 100644 --- a/docs/tutorials/plugin-system.rst +++ b/docs/tutorials/plugin-system.rst @@ -295,7 +295,8 @@ providing access to gateway data and utilities: - ``acquire_lock()`` / ``release_lock()`` - acquire and release entity locks with optional scope and TTL - ``get_entity_snapshot()`` - returns an ``IntrospectionInput`` populated from the current entity cache - ``list_all_faults()`` - returns JSON object with a ``"faults"`` array containing all active faults across all entities -- ``register_sampler(collection, fn)`` - registers a cyclic subscription sampler for a custom collection name +- ``register_sampler(collection, fn, honours_resource_path = false)`` - registers a cyclic + subscription sampler for a custom collection name .. code-block:: cpp @@ -333,11 +334,18 @@ and reflects the state of the gateway's thread-safe entity cache. ``list_all_faults()`` is useful for plugins that need cross-entity fault visibility (e.g. mapping fault codes to topics). Returns ``{}`` if the fault manager is unavailable. -``register_sampler(collection, fn)`` wires a sampler into the ``ResourceSamplerRegistry`` -so that cyclic subscriptions created for ``collection`` (e.g. ``"x-medkit-metrics"``) -call ``fn(entity_id, resource_path)`` on each tick. The function must return -``tl::expected``. See `Cyclic Subscription Extensions`_ -for the lower-level registry API. +``register_sampler(collection, fn, honours_resource_path)`` wires a sampler into the +``ResourceSamplerRegistry`` so that cyclic subscriptions created for ``collection`` +(e.g. ``"x-medkit-metrics"``) call ``fn(entity_id, resource_path)`` on each tick. The +function must return ``tl::expected``. See +`Cyclic Subscription Extensions`_ for the lower-level registry API. + +``honours_resource_path`` defaults to ``false`` and states whether ``fn`` narrows its +payload to the one resource named by ``resource_path``. Leave it ``false`` for a sampler +that ignores that argument: the gateway then refuses a subscription whose resource URI +names a single item of the collection with 400 ``x-medkit-invalid-resource-uri``, instead +of accepting it and streaming the whole collection on every tick. Set it to ``true`` only +when ``fn`` actually reads ``resource_path``. .. note:: @@ -404,7 +412,7 @@ all entities. Returns an empty object if the fault manager is unavailable: // Process each fault } -**register_sampler(collection, fn)** +**register_sampler(collection, fn, honours_resource_path = false)** Registers a cyclic subscription sampler for a custom collection name. Once registered, clients can create cyclic subscriptions on that collection for any @@ -420,6 +428,21 @@ entity: return *data; }); +The sampler above ignores ``resource_path`` and so answers with the whole +collection; leaving ``honours_resource_path`` at its ``false`` default is what +makes the gateway refuse ``/x-medkit-metrics/{item}`` instead of streaming +everything in response to it. A sampler that does read ``resource_path`` passes +``true``: + +.. code-block:: cpp + + ctx_->register_sampler("x-medkit-metrics", + [this](const std::string& entity_id, const std::string& resource_path) + -> tl::expected { + return collect_one_metric(entity_id, resource_path); + }, + /*honours_resource_path=*/true); + This is a convenience wrapper around the lower-level ``ResourceSamplerRegistry`` API described in `Cyclic Subscription Extensions`_. @@ -468,8 +491,9 @@ Plugins can extend cyclic subscriptions by registering custom resource samplers and transport providers during ``set_context()``. **Resource Samplers** provide the data for a collection when sampled by a subscription. -Built-in samplers (``data``, ``faults``, ``configurations``, ``updates``) are registered -by the gateway during startup. Custom samplers are registered via ``ResourceSamplerRegistry`` +Built-in samplers (``data``, ``faults``, ``configurations``, ``logs``, ``updates``) are +registered by the gateway during startup. Of these only ``data`` and ``updates`` narrow +their payload to a named resource; the rest stream their whole collection. Custom samplers are registered via ``ResourceSamplerRegistry`` on the ``GatewayNode``: .. code-block:: cpp @@ -479,10 +503,14 @@ on the ``GatewayNode``: [this](const std::string& entity_id, const std::string& resource_path) -> tl::expected { return get_metrics(entity_id, resource_path); - }); + }, + /*is_builtin=*/false, /*honours_resource_path=*/true); Once registered, clients can create cyclic subscriptions on the ``x-medkit-metrics`` -collection for any entity. +collection for any entity. The trailing ``honours_resource_path`` argument declares +that the sampler narrows its payload to the resource named in the subscription URI; +without it the gateway refuses such a URI with 400, because a sampler that ignores +``resource_path`` would answer a request for one item with the whole collection. **Transport Providers** deliver subscription data via alternative protocols (beyond the built-in SSE transport). Register via ``TransportRegistry`` on the ``GatewayNode``: diff --git a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/plugins/plugin_context.hpp b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/plugins/plugin_context.hpp index 0212eaceb..5db68a96b 100644 --- a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/plugins/plugin_context.hpp +++ b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/plugins/plugin_context.hpp @@ -186,10 +186,17 @@ class PluginContext { // ---- Resource sampler registration ---- /// Register a cyclic subscription sampler for a custom collection. + /// + /// `honours_resource_path` states that `fn` narrows its payload to the one + /// resource named by its second argument. Leave it false for a sampler that + /// ignores that argument: the gateway then refuses a subscription whose URI + /// names a single item of this collection, instead of accepting it and + /// streaming the whole collection on every tick. virtual void register_sampler( const std::string & /*collection*/, const std::function(const std::string &, const std::string &)> & - /*fn*/) { + /*fn*/, + bool /*honours_resource_path*/ = false) { } // ---- Trigger infrastructure access ---- diff --git a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/plugins/plugin_types.hpp b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/plugins/plugin_types.hpp index d4598fbb3..10dcfc71b 100644 --- a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/plugins/plugin_types.hpp +++ b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/plugins/plugin_types.hpp @@ -40,7 +40,14 @@ namespace ros2_medkit_gateway { /// so a pre-compiled v6 `.so` is rejected. Out-of-tree plugins must be /// recompiled against v7 headers; in-tree plugins that `return /// PLUGIN_API_VERSION` pick up the bump automatically. -constexpr int PLUGIN_API_VERSION = 7; +/// - v8: PluginContext::register_sampler() takes a trailing +/// `honours_resource_path` flag, defaulted to false, so a plugin can +/// declare that its sampler narrows its payload to a named resource. +/// Plugin SOURCE written against v7 compiles unchanged. The parameter +/// sits in an existing vtable slot, so a pre-compiled v7 `.so` would +/// call it with one argument missing; the loader's strict equality +/// against this value is what keeps such a `.so` out. +constexpr int PLUGIN_API_VERSION = 8; /// Log severity levels for plugin logging callback enum class PluginLogLevel { kInfo, kWarn, kError }; diff --git a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/resource_sampler.hpp b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/resource_sampler.hpp index 58fa9643e..a6465fe81 100644 --- a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/resource_sampler.hpp +++ b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/resource_sampler.hpp @@ -72,10 +72,24 @@ using ResourceSamplerFn = std::function get_sampler(const std::string & collection) const; bool has_sampler(const std::string & collection) const; + /// Whether the sampler registered for `collection` narrows its payload to + /// the resource named by a resource path. False for a collection with no + /// sampler, and for any sampler registered without declaring it. + bool honours_resource_path(const std::string & collection) const; + /// Remove a previously registered sampler. A no-op if the collection was /// never registered, or if it was registered with `is_builtin = true` - /// built-ins are owned by the gateway node, not by any plugin, and removing @@ -124,6 +138,9 @@ class ResourceSamplerRegistry { /// exactly what `get_sampler()` copies out. ResourceSamplerFn fn; bool is_builtin; + /// See `register_sampler`. The subscription layer reads this to decide + /// whether a resource URI may name a single item of this collection. + bool honours_resource_path; /// Shared with every copy of `fn` handed out via `get_sampler()`. See /// `ControlBlock` and the class comment. std::shared_ptr control; diff --git a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/http/handlers/cyclic_subscription_handlers.hpp b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/http/handlers/cyclic_subscription_handlers.hpp index ca6db0823..2e8c64436 100644 --- a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/http/handlers/cyclic_subscription_handlers.hpp +++ b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/http/handlers/cyclic_subscription_handlers.hpp @@ -96,6 +96,16 @@ class CyclicSubscriptionHandlers { /// Parse resource URI to extract entity type, entity id, collection, and resource path. static tl::expected parse_resource_uri(const std::string & resource); + /// Refuse a resource path the collection's sampler cannot act on. + /// + /// A sampler registered without `honours_resource_path` answers with its + /// whole collection on every tick, so a URI naming a single item of that + /// collection cannot be served as written. The refusal keeps a narrow + /// request from being answered with a wide stream. + static tl::expected validate_resource_path_support(const ResourceSamplerRegistry & registry, + const ParsedResourceUri & parsed, + const std::string & resource); + private: /// Build event_source URI from subscription info static std::string build_event_source(const CyclicSubscriptionInfo & info); diff --git a/src/ros2_medkit_gateway/src/core/resource_sampler.cpp b/src/ros2_medkit_gateway/src/core/resource_sampler.cpp index fff2c6a78..1e2227198 100644 --- a/src/ros2_medkit_gateway/src/core/resource_sampler.cpp +++ b/src/ros2_medkit_gateway/src/core/resource_sampler.cpp @@ -19,7 +19,8 @@ namespace ros2_medkit_gateway { -void ResourceSamplerRegistry::register_sampler(const std::string & collection, ResourceSamplerFn fn, bool is_builtin) { +void ResourceSamplerRegistry::register_sampler(const std::string & collection, ResourceSamplerFn fn, bool is_builtin, + bool honours_resource_path) { std::unique_lock lock(mutex_); if (!is_builtin) { @@ -48,7 +49,7 @@ void ResourceSamplerRegistry::register_sampler(const std::string & collection, R return fn(entity_id, resource_path); }; - samplers_[collection] = Entry{std::move(wrapped), is_builtin, std::move(control)}; + samplers_[collection] = Entry{std::move(wrapped), is_builtin, honours_resource_path, std::move(control)}; } std::optional ResourceSamplerRegistry::get_sampler(const std::string & collection) const { @@ -65,6 +66,12 @@ bool ResourceSamplerRegistry::has_sampler(const std::string & collection) const return samplers_.count(collection) > 0; } +bool ResourceSamplerRegistry::honours_resource_path(const std::string & collection) const { + std::shared_lock lock(mutex_); + auto it = samplers_.find(collection); + return it != samplers_.end() && it->second.honours_resource_path; +} + void ResourceSamplerRegistry::remove_sampler(const std::string & collection) { std::shared_ptr control; { diff --git a/src/ros2_medkit_gateway/src/gateway_node.cpp b/src/ros2_medkit_gateway/src/gateway_node.cpp index 71b942873..a8debcc6e 100644 --- a/src/ros2_medkit_gateway/src/gateway_node.cpp +++ b/src/ros2_medkit_gateway/src/gateway_node.cpp @@ -1310,7 +1310,7 @@ GatewayNode::GatewayNode(const rclcpp::NodeOptions & options) : Node("ros2_medki } return tl::make_unexpected("Topic data not available: " + resource_path); }, - true); + /*is_builtin=*/true, /*honours_resource_path=*/true); sampler_registry_->register_sampler( "faults", @@ -1340,7 +1340,7 @@ GatewayNode::GatewayNode(const rclcpp::NodeOptions & options) : Node("ros2_medki result.data["faults"] = std::move(filtered); return result.data; }, - true); + /*is_builtin=*/true); sampler_registry_->register_sampler( "configurations", @@ -1365,7 +1365,7 @@ GatewayNode::GatewayNode(const rclcpp::NodeOptions & options) : Node("ros2_medki payload["items"] = std::move(items); return payload; }, - true); + /*is_builtin=*/true); sampler_registry_->register_sampler( "logs", @@ -1405,7 +1405,7 @@ GatewayNode::GatewayNode(const rclcpp::NodeOptions & options) : Node("ros2_medki payload["items"] = std::move(*result); return payload; }, - true); + /*is_builtin=*/true); // Register update status sampler (server-level, uses UpdateManager) if (update_mgr_) { @@ -1423,7 +1423,7 @@ GatewayNode::GatewayNode(const rclcpp::NodeOptions & options) : Node("ros2_medki } return update_status_to_json(*result); }, - true); + /*is_builtin=*/true, /*honours_resource_path=*/true); } RCLCPP_INFO(get_logger(), "Registered built-in resource samplers: data, faults, configurations, logs%s", diff --git a/src/ros2_medkit_gateway/src/http/handlers/cyclic_subscription_handlers.cpp b/src/ros2_medkit_gateway/src/http/handlers/cyclic_subscription_handlers.cpp index fa67b6584..990c1fd8d 100644 --- a/src/ros2_medkit_gateway/src/http/handlers/cyclic_subscription_handlers.cpp +++ b/src/ros2_medkit_gateway/src/http/handlers/cyclic_subscription_handlers.cpp @@ -184,6 +184,10 @@ CyclicSubscriptionHandlers::post_subscription(const http::TypedRequest & req, json{{"collection", parsed->collection}})); } + if (auto path_ok = validate_resource_path_support(sampler_registry_, *parsed, resource); !path_ok) { + return tl::unexpected(path_ok.error()); + } + // Create subscription auto result = sub_mgr_.create(entity_id, entity_type, resource, parsed->collection, parsed->resource_path, protocol, interval, duration); @@ -485,5 +489,18 @@ CyclicSubscriptionHandlers::parse_resource_uri(const std::string & resource) { "or /api/v1/updates/{id}/status"); } +tl::expected CyclicSubscriptionHandlers::validate_resource_path_support( + const ResourceSamplerRegistry & registry, const ParsedResourceUri & parsed, const std::string & resource) { + if (parsed.resource_path.empty() || registry.honours_resource_path(parsed.collection)) { + return {}; + } + return tl::unexpected(make_error( + 400, ERR_X_MEDKIT_INVALID_RESOURCE_URI, + "Collection '" + parsed.collection + + "' is streamed as a whole and does not support a resource path. Subscribe to the collection URI without a " + "trailing resource path.", + json{{"parameter", "resource"}, {"value", resource}, {"collection", parsed.collection}})); +} + } // namespace handlers } // namespace ros2_medkit_gateway diff --git a/src/ros2_medkit_gateway/src/plugins/plugin_context.cpp b/src/ros2_medkit_gateway/src/plugins/plugin_context.cpp index 4cec62393..00a09aded 100644 --- a/src/ros2_medkit_gateway/src/plugins/plugin_context.cpp +++ b/src/ros2_medkit_gateway/src/plugins/plugin_context.cpp @@ -299,10 +299,10 @@ class GatewayPluginContext : public RosPluginContext { void register_sampler( const std::string & collection, - const std::function(const std::string &, const std::string &)> & fn) - override { + const std::function(const std::string &, const std::string &)> & fn, + bool honours_resource_path) override { if (sampler_registry_) { - sampler_registry_->register_sampler(collection, fn); + sampler_registry_->register_sampler(collection, fn, /*is_builtin=*/false, honours_resource_path); } } diff --git a/src/ros2_medkit_gateway/test/test_cyclic_subscription_handlers.cpp b/src/ros2_medkit_gateway/test/test_cyclic_subscription_handlers.cpp index 7f2a93efb..37a07d47e 100644 --- a/src/ros2_medkit_gateway/test/test_cyclic_subscription_handlers.cpp +++ b/src/ros2_medkit_gateway/test/test_cyclic_subscription_handlers.cpp @@ -174,3 +174,153 @@ TEST(ParseResourceUriTest, UpdatesListNotSubscribable) { // test_primitives.cpp (write_generic_error / write_oauth2_error suites) // and the per-route handler tests assert error bodies end-to-end via the // typed router. + +// --- validate_resource_path_support tests --- + +namespace { + +/// Fill `registry` the way the gateway fills its own: `data` and `updates` +/// narrow their payload to a named resource, the rest stream their whole +/// collection. ResourceSamplerRegistry holds a shared_mutex and so is neither +/// copyable nor movable - the caller owns the instance and passes it in. +void register_builtin_like_samplers(ResourceSamplerRegistry & registry) { + auto stub = [](const std::string &, const std::string &) -> tl::expected { + return nlohmann::json::object(); + }; + registry.register_sampler("data", stub, /*is_builtin=*/true, /*honours_resource_path=*/true); + registry.register_sampler("updates", stub, /*is_builtin=*/true, /*honours_resource_path=*/true); + registry.register_sampler("faults", stub, /*is_builtin=*/true); + registry.register_sampler("configurations", stub, /*is_builtin=*/true); + registry.register_sampler("logs", stub, /*is_builtin=*/true); +} + +ParsedResourceUri parsed_uri(const std::string & resource) { + auto parsed = CyclicSubscriptionHandlers::parse_resource_uri(resource); + EXPECT_TRUE(parsed.has_value()) << "fixture URI must parse: " << resource; + return parsed.value_or(ParsedResourceUri{}); +} + +} // namespace + +TEST(ValidateResourcePathSupportTest, ConfigurationsWithResourcePathRefused) { + ResourceSamplerRegistry registry; + register_builtin_like_samplers(registry); + const std::string resource = "/api/v1/components/ecu1/configurations/param1"; + + auto result = CyclicSubscriptionHandlers::validate_resource_path_support(registry, parsed_uri(resource), resource); + + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(result.error().http_status, 400); + EXPECT_EQ(result.error().code, ERR_X_MEDKIT_INVALID_RESOURCE_URI); + EXPECT_NE(result.error().message.find("configurations"), std::string::npos); + EXPECT_NE(result.error().message.find("streamed as a whole"), std::string::npos); + EXPECT_EQ(result.error().params["collection"], "configurations"); + EXPECT_EQ(result.error().params["value"], resource); +} + +TEST(ValidateResourcePathSupportTest, FaultsWithResourcePathRefused) { + ResourceSamplerRegistry registry; + register_builtin_like_samplers(registry); + const std::string resource = "/api/v1/apps/node1/faults/fault_001"; + + auto result = CyclicSubscriptionHandlers::validate_resource_path_support(registry, parsed_uri(resource), resource); + + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(result.error().http_status, 400); + EXPECT_NE(result.error().message.find("faults"), std::string::npos); + EXPECT_EQ(result.error().params["collection"], "faults"); +} + +TEST(ValidateResourcePathSupportTest, LogsWithResourcePathRefused) { + ResourceSamplerRegistry registry; + register_builtin_like_samplers(registry); + const std::string resource = "/api/v1/apps/node1/logs/entry_7"; + + auto result = CyclicSubscriptionHandlers::validate_resource_path_support(registry, parsed_uri(resource), resource); + + ASSERT_FALSE(result.has_value()); + EXPECT_NE(result.error().message.find("logs"), std::string::npos); +} + +TEST(ValidateResourcePathSupportTest, ConfigurationsWithoutResourcePathAccepted) { + ResourceSamplerRegistry registry; + register_builtin_like_samplers(registry); + const std::string resource = "/api/v1/components/ecu1/configurations"; + + auto result = CyclicSubscriptionHandlers::validate_resource_path_support(registry, parsed_uri(resource), resource); + + EXPECT_TRUE(result.has_value()); +} + +TEST(ValidateResourcePathSupportTest, FaultsWithoutResourcePathAccepted) { + ResourceSamplerRegistry registry; + register_builtin_like_samplers(registry); + const std::string resource = "/api/v1/apps/node1/faults"; + + auto result = CyclicSubscriptionHandlers::validate_resource_path_support(registry, parsed_uri(resource), resource); + + EXPECT_TRUE(result.has_value()); +} + +TEST(ValidateResourcePathSupportTest, DataWithTopicAccepted) { + ResourceSamplerRegistry registry; + register_builtin_like_samplers(registry); + const std::string resource = "/api/v1/apps/node1/data/temperature"; + + auto result = CyclicSubscriptionHandlers::validate_resource_path_support(registry, parsed_uri(resource), resource); + + EXPECT_TRUE(result.has_value()); +} + +TEST(ValidateResourcePathSupportTest, UpdateStatusPackageIdAccepted) { + ResourceSamplerRegistry registry; + register_builtin_like_samplers(registry); + const std::string resource = "/api/v1/updates/my-package/status"; + + auto result = CyclicSubscriptionHandlers::validate_resource_path_support(registry, parsed_uri(resource), resource); + + EXPECT_TRUE(result.has_value()); +} + +TEST(ValidateResourcePathSupportTest, UndeclaredPluginSamplerRefusesResourcePath) { + ResourceSamplerRegistry registry; + registry.register_sampler("x-medkit-metrics", + [](const std::string &, const std::string &) -> tl::expected { + return nlohmann::json::object(); + }); + const std::string resource = "/api/v1/apps/node1/x-medkit-metrics/cpu_usage"; + + auto result = CyclicSubscriptionHandlers::validate_resource_path_support(registry, parsed_uri(resource), resource); + + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(result.error().http_status, 400); + EXPECT_EQ(result.error().params["collection"], "x-medkit-metrics"); +} + +TEST(ValidateResourcePathSupportTest, DeclaredPluginSamplerAcceptsResourcePath) { + ResourceSamplerRegistry registry; + registry.register_sampler( + "x-medkit-metrics", + [](const std::string &, const std::string &) -> tl::expected { + return nlohmann::json::object(); + }, + /*is_builtin=*/false, /*honours_resource_path=*/true); + const std::string resource = "/api/v1/apps/node1/x-medkit-metrics/cpu_usage"; + + auto result = CyclicSubscriptionHandlers::validate_resource_path_support(registry, parsed_uri(resource), resource); + + EXPECT_TRUE(result.has_value()); +} + +TEST(ValidateResourcePathSupportTest, UndeclaredPluginSamplerAcceptsCollectionUri) { + ResourceSamplerRegistry registry; + registry.register_sampler("x-medkit-graph", + [](const std::string &, const std::string &) -> tl::expected { + return nlohmann::json::object(); + }); + const std::string resource = "/api/v1/functions/func1/x-medkit-graph"; + + auto result = CyclicSubscriptionHandlers::validate_resource_path_support(registry, parsed_uri(resource), resource); + + EXPECT_TRUE(result.has_value()); +} diff --git a/src/ros2_medkit_gateway/test/test_plugin_manager.cpp b/src/ros2_medkit_gateway/test/test_plugin_manager.cpp index 9fb5d457c..fc2621f40 100644 --- a/src/ros2_medkit_gateway/test/test_plugin_manager.cpp +++ b/src/ros2_medkit_gateway/test/test_plugin_manager.cpp @@ -233,7 +233,8 @@ class MockThrowOnShutdown : public GatewayPlugin { /// so a dangling registration after teardown would be a use-after-free. class MockSamplerPlugin : public GatewayPlugin { public: - explicit MockSamplerPlugin(std::string collection) : collection_(std::move(collection)) { + explicit MockSamplerPlugin(std::string collection, bool honours_resource_path = false) + : collection_(std::move(collection)), honours_resource_path_(honours_resource_path) { } std::string name() const override { return "sampler_plugin"; @@ -241,14 +242,17 @@ class MockSamplerPlugin : public GatewayPlugin { void configure(const json & /*cfg*/) override { } void set_context(PluginContext & context) override { - context.register_sampler(collection_, - [this](const std::string &, const std::string &) -> tl::expected { - return json{{"plugin", name()}}; - }); + context.register_sampler( + collection_, + [this](const std::string &, const std::string &) -> tl::expected { + return json{{"plugin", name()}}; + }, + honours_resource_path_); } private: std::string collection_; + bool honours_resource_path_; }; /// Same sampler-registering behavior as MockSamplerPlugin, but also throws @@ -449,6 +453,24 @@ TEST(PluginManagerTest, ShutdownAllRemovesPluginSamplers) { EXPECT_FALSE(sampler_registry.has_sampler("x-mock-shutdown-sampler")); } +TEST(PluginManagerTest, PluginResourcePathDeclarationReachesRegistry) { + ResourceSamplerRegistry sampler_registry; + TransportRegistry transport_registry; + PluginManager mgr; + mgr.set_registries(sampler_registry, transport_registry); + mgr.add_plugin(std::make_unique("x-mock-whole-collection")); + mgr.add_plugin(std::make_unique("x-mock-per-item", /*honours_resource_path=*/true)); + mgr.configure_plugins(); + + auto ctx = make_gateway_plugin_context(nullptr, nullptr, &sampler_registry); + mgr.set_context(*ctx); + + ASSERT_TRUE(sampler_registry.has_sampler("x-mock-whole-collection")); + ASSERT_TRUE(sampler_registry.has_sampler("x-mock-per-item")); + EXPECT_FALSE(sampler_registry.honours_resource_path("x-mock-whole-collection")); + EXPECT_TRUE(sampler_registry.honours_resource_path("x-mock-per-item")); +} + TEST(PluginManagerTest, BuiltinSamplerSurvivesPluginDisable) { ResourceSamplerRegistry sampler_registry; TransportRegistry transport_registry; diff --git a/src/ros2_medkit_gateway/test/test_resource_sampler_registry.cpp b/src/ros2_medkit_gateway/test/test_resource_sampler_registry.cpp index 52d53744a..288bbdfe2 100644 --- a/src/ros2_medkit_gateway/test/test_resource_sampler_registry.cpp +++ b/src/ros2_medkit_gateway/test/test_resource_sampler_registry.cpp @@ -338,3 +338,86 @@ TEST(ResourceSamplerRegistryTest, RemoveSamplerBlocksUntilInFlightCallCompletes) invoker.join(); EXPECT_TRUE(invocation_succeeded); } + +// --- resource-path support declaration --- + +TEST(ResourceSamplerRegistryTest, UndeclaredSamplerDoesNotHonourResourcePath) { + ResourceSamplerRegistry registry; + registry.register_sampler("x-medkit-metrics", + [](const std::string &, const std::string &) -> tl::expected { + return nlohmann::json{{"cpu", 42}}; + }); + + EXPECT_FALSE(registry.honours_resource_path("x-medkit-metrics")); +} + +TEST(ResourceSamplerRegistryTest, DeclaredSamplerHonoursResourcePath) { + ResourceSamplerRegistry registry; + registry.register_sampler( + "x-medkit-metrics", + [](const std::string &, const std::string &) -> tl::expected { + return nlohmann::json{{"cpu", 42}}; + }, + /*is_builtin=*/false, /*honours_resource_path=*/true); + + EXPECT_TRUE(registry.honours_resource_path("x-medkit-metrics")); +} + +TEST(ResourceSamplerRegistryTest, BuiltinSamplerCanHonourResourcePath) { + ResourceSamplerRegistry registry; + registry.register_sampler( + "data", + [](const std::string &, const std::string &) -> tl::expected { + return nlohmann::json{{"value", 1}}; + }, + /*is_builtin=*/true, /*honours_resource_path=*/true); + registry.register_sampler( + "configurations", + [](const std::string &, const std::string &) -> tl::expected { + return nlohmann::json{{"items", nlohmann::json::array()}}; + }, + /*is_builtin=*/true); + + EXPECT_TRUE(registry.honours_resource_path("data")); + EXPECT_FALSE(registry.honours_resource_path("configurations")); +} + +TEST(ResourceSamplerRegistryTest, UnregisteredCollectionDoesNotHonourResourcePath) { + ResourceSamplerRegistry registry; + EXPECT_FALSE(registry.honours_resource_path("nonexistent")); +} + +TEST(ResourceSamplerRegistryTest, ReregisteringBuiltinReplacesResourcePathDeclaration) { + ResourceSamplerRegistry registry; + registry.register_sampler( + "data", + [](const std::string &, const std::string &) -> tl::expected { + return nlohmann::json{{"version", 1}}; + }, + /*is_builtin=*/true, /*honours_resource_path=*/true); + ASSERT_TRUE(registry.honours_resource_path("data")); + + registry.register_sampler( + "data", + [](const std::string &, const std::string &) -> tl::expected { + return nlohmann::json{{"version", 2}}; + }, + /*is_builtin=*/true); + + EXPECT_FALSE(registry.honours_resource_path("data")); +} + +TEST(ResourceSamplerRegistryTest, RemovedSamplerNoLongerHonoursResourcePath) { + ResourceSamplerRegistry registry; + registry.register_sampler( + "x-medkit-metrics", + [](const std::string &, const std::string &) -> tl::expected { + return nlohmann::json{{"cpu", 42}}; + }, + /*is_builtin=*/false, /*honours_resource_path=*/true); + ASSERT_TRUE(registry.honours_resource_path("x-medkit-metrics")); + + registry.remove_sampler("x-medkit-metrics"); + + EXPECT_FALSE(registry.honours_resource_path("x-medkit-metrics")); +} diff --git a/src/ros2_medkit_integration_tests/test/features/test_multi_collection_subscriptions.test.py b/src/ros2_medkit_integration_tests/test/features/test_multi_collection_subscriptions.test.py index b0ad0de05..f5e09f946 100644 --- a/src/ros2_medkit_integration_tests/test/features/test_multi_collection_subscriptions.test.py +++ b/src/ros2_medkit_integration_tests/test/features/test_multi_collection_subscriptions.test.py @@ -18,7 +18,8 @@ Validates that cyclic subscriptions can be created for data, faults, and configurations collections, and that error cases (unsupported collection, invalid URI, entity mismatch, unsupported protocol, -path traversal) return 400. +path traversal, resource path on a collection that is streamed whole) +return 400. """ @@ -273,6 +274,66 @@ def collect_events(): self.assertIn('severity', entry) self.assertIn('message', entry) + def _collect_sse_events(self, event_source, wanted=2, timeout=15): + """Open the SSE stream and return up to `wanted` parsed events.""" + events_url = ( + f'{self.BASE_URL}{event_source.removeprefix(API_BASE_PATH)}' + ) + received = [] + stop_event = threading.Event() + + def collect(): + try: + with requests.get( + events_url, stream=True, timeout=timeout, + ) as resp: + for line in resp.iter_lines(decode_unicode=True): + if stop_event.is_set(): + break + if line and line.startswith('data: '): + received.append(json.loads(line[6:])) + if len(received) >= wanted: + stop_event.set() + break + except requests.exceptions.RequestException: + pass + + thread = threading.Thread(target=collect, daemon=True) + thread.start() + stop_event.wait(timeout=timeout) + thread.join(timeout=5) + return received + + def test_configurations_subscription_streams_whole_collection(self): + """Configurations without a resource path is accepted and streams. + + The gateway refuses a configurations URI that names a single + parameter because the sampler answers with the whole collection. + This pins the other side of that contract: the collection URI is + still accepted and still delivers parameter items on the stream. + + @verifies REQ_INTEROP_089 + @verifies REQ_INTEROP_090 + """ + resource = f'/api/v1/apps/{self.app_id}/configurations' + r = self._create_subscription(resource, interval='fast', duration=30) + self.assertEqual(r.status_code, 201, f'Create failed: {r.text}') + + data = r.json() + self.addCleanup(self._delete_subscription, data['id']) + + events = self._collect_sse_events(data['event_source']) + self.assertGreaterEqual( + len(events), 1, + f'Expected at least 1 SSE configurations event, got {len(events)}', + ) + for event in events: + self.assertIn('payload', event) + self.assertIn( + 'items', event['payload'], + 'Configurations payload must have items array', + ) + # =================================================================== # CRUD operations: list, get, update, delete # =================================================================== @@ -410,6 +471,55 @@ def test_path_traversal_returns_400(self): r = self._create_subscription(resource) self.assertEqual(r.status_code, 400) + def _assert_resource_path_refused(self, collection, resource_path): + """Assert a resource path on a whole-collection stream is refused.""" + resource = ( + f'/api/v1/apps/{self.app_id}/{collection}/{resource_path}' + ) + r = self._create_subscription(resource) + self.assertEqual( + r.status_code, 400, + f'Expected 400 for {resource}, got {r.status_code}: {r.text}', + ) + body = r.json() + self.assertEqual(body['error_code'], 'vendor-error') + self.assertEqual( + body['vendor_code'], 'x-medkit-invalid-resource-uri', + ) + self.assertIn( + collection, body['message'], + 'Refusal must name the collection it applies to', + ) + self.assertIn( + 'streamed as a whole', body['message'], + 'Refusal must say the collection is streamed whole', + ) + self.assertEqual(body['parameters']['collection'], collection) + + def test_configurations_with_resource_path_returns_400(self): + """A single-parameter configurations URI is refused, not widened.""" + self._assert_resource_path_refused('configurations', 'use_sim_time') + + def test_faults_with_resource_path_returns_400(self): + """A single-fault URI is refused, not widened to the fault list.""" + self._assert_resource_path_refused('faults', 'fault_001') + + def test_logs_with_resource_path_returns_400(self): + """A single-log-entry URI is refused, not widened to the log list.""" + self._assert_resource_path_refused('logs', 'entry_7') + + def test_data_without_resource_path_returns_400(self): + """Data without a topic is still refused - the sampler needs one.""" + resource = f'/api/v1/apps/{self.app_id}/data' + r = self._create_subscription(resource) + self.assertEqual(r.status_code, 400, f'Got {r.status_code}: {r.text}') + body = r.json() + self.assertEqual(body['error_code'], 'vendor-error') + self.assertEqual( + body['vendor_code'], 'x-medkit-invalid-resource-uri', + ) + self.assertIn('requires a resource path', body['message']) + @launch_testing.post_shutdown_test() class TestShutdown(unittest.TestCase): diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_provider/design/index.rst b/src/ros2_medkit_plugins/ros2_medkit_graph_provider/design/index.rst index c1e56f2be..ffd26324e 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_graph_provider/design/index.rst +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_provider/design/index.rst @@ -300,3 +300,8 @@ The plugin registers a sampler via ``PluginContext::register_sampler()`` for the ``x-medkit-graph`` resource. This allows clients to create cyclic subscriptions that receive periodic graph snapshots over SSE, enabling live dashboard updates without polling the HTTP endpoint. + +The sampler builds one document for the whole function and ignores the resource +path, so it registers without ``honours_resource_path``. A subscription URI that +appends a path below ``x-medkit-graph`` is refused with 400 rather than answered +with the full graph document. diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_provider/test/test_graph_provider_plugin.cpp b/src/ros2_medkit_plugins/ros2_medkit_graph_provider/test/test_graph_provider_plugin.cpp index 59bd5f194..7d8e40819 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_graph_provider/test/test_graph_provider_plugin.cpp +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_provider/test/test_graph_provider_plugin.cpp @@ -345,9 +345,10 @@ class FakePluginContext : public RosPluginContext { void register_sampler( const std::string & collection, - const std::function(const std::string &, const std::string &)> & fn) - override { + const std::function(const std::string &, const std::string &)> & fn, + bool honours_resource_path) override { registered_samplers_[collection] = fn; + sampler_honours_resource_path_[collection] = honours_resource_path; } ResourceChangeNotifier * get_resource_change_notifier() override { @@ -370,6 +371,7 @@ class FakePluginContext : public RosPluginContext { std::unordered_map(const std::string &, const std::string &)>> registered_samplers_; + std::unordered_map sampler_honours_resource_path_; }; class LocalHttpServer { @@ -1650,6 +1652,19 @@ TEST(GraphProviderPluginRouteTest, RegistersSamplerForCyclicSubscriptions) { ASSERT_TRUE(result->contains("x-medkit-graph")); } +// The sampler builds one document for the whole function and ignores its +// resource_path argument, so it must register as not honouring a resource +// path - the gateway refuses a per-item subscription URI on that basis. +TEST(GraphProviderPluginRouteTest, SamplerDeclaresItDoesNotHonourResourcePath) { + GraphProviderPlugin plugin; + FakePluginContext ctx({{"f1", PluginEntityInfo{SovdEntityType::FUNCTION, "f1", "", ""}}}); + plugin.configure({}); + plugin.set_context(ctx); + + ASSERT_EQ(ctx.sampler_honours_resource_path_.count("x-medkit-graph"), 1u); + EXPECT_FALSE(ctx.sampler_honours_resource_path_["x-medkit-graph"]); +} + TEST(GraphProviderPluginRouteTest, AppliesConfigFromConfigure) { nlohmann::json config = { {"expected_frequency_hz_default", 10.0}, {"degraded_frequency_ratio", 0.8}, {"drop_rate_percent_threshold", 2.0}}; diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/test/test_opcua_identity.cpp b/src/ros2_medkit_plugins/ros2_medkit_opcua/test/test_opcua_identity.cpp index 48294503f..acacbe21a 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/test/test_opcua_identity.cpp +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/test/test_opcua_identity.cpp @@ -135,8 +135,8 @@ class FakePluginContext : public RosPluginContext { } void register_sampler( const std::string &, - const std::function(const std::string &, const std::string &)> &) - override { + const std::function(const std::string &, const std::string &)> &, + bool) override { } ResourceChangeNotifier * get_resource_change_notifier() override { return nullptr; diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/test/test_opcua_plugin.cpp b/src/ros2_medkit_plugins/ros2_medkit_opcua/test/test_opcua_plugin.cpp index 4bfdf4bc6..8c823769d 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/test/test_opcua_plugin.cpp +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/test/test_opcua_plugin.cpp @@ -157,7 +157,8 @@ class FakePluginContext : public RosPluginContext { void register_sampler( const std::string & /*topic*/, const std::function(const std::string &, const std::string &)> & - /*sampler*/) override { + /*sampler*/, + bool /*honours_resource_path*/) override { } ResourceChangeNotifier * get_resource_change_notifier() override { return nullptr; diff --git a/src/ros2_medkit_plugins/ros2_medkit_sovd_service_interface/test/test_sovd_service_interface.cpp b/src/ros2_medkit_plugins/ros2_medkit_sovd_service_interface/test/test_sovd_service_interface.cpp index 8d69b44a3..e45924501 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_sovd_service_interface/test/test_sovd_service_interface.cpp +++ b/src/ros2_medkit_plugins/ros2_medkit_sovd_service_interface/test/test_sovd_service_interface.cpp @@ -152,7 +152,8 @@ class FakePluginContext : public RosPluginContext { void register_sampler( const std::string & /*collection*/, const std::function(const std::string &, const std::string &)> & - /*fn*/) override { + /*fn*/, + bool /*honours_resource_path*/) override { } ResourceChangeNotifier * get_resource_change_notifier() override { From 0e559b6f0780658d320e97547c77e8ba36c56cad Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Thu, 20 Aug 2026 11:44:35 +0200 Subject: [PATCH 14/22] fix(triggers): keep a restored trigger until its entity has been seen at least once The orphan sweep treated "entity not in the discovery cache" as "entity gone", and removing a persistent trigger deletes its row from the shared store. Restore runs once, while the gateway is constructing, so a trigger restored on startup was raced by the first sweep tick against a cache that had not yet heard from nodes which were already running. Measured on a restarted gateway: restored, then deleted 109 ms later, with nothing to bring it back and nothing said about it. A trigger now records whether its entity has ever been observed. Only one that was seen and is now missing is an orphan. Restore and the sweep both name what they discarded and why. --- docs/api/rest.rst | 10 + .../core/managers/trigger_manager.hpp | 16 +- .../src/core/managers/trigger_manager.cpp | 194 +++++++---- src/ros2_medkit_gateway/src/gateway_node.cpp | 16 +- .../test/test_trigger_manager.cpp | 74 ++++ .../CMakeLists.txt | 13 + ..._triggers_restore_before_discovery.test.py | 319 ++++++++++++++++++ 7 files changed, 574 insertions(+), 68 deletions(-) create mode 100644 src/ros2_medkit_integration_tests/test/features/test_triggers_restore_before_discovery.test.py diff --git a/docs/api/rest.rst b/docs/api/rest.rst index e2eab8374..0bc4aea05 100644 --- a/docs/api/rest.rst +++ b/docs/api/rest.rst @@ -2510,6 +2510,16 @@ configuration: Non-persistent triggers are always cleared on restart. +Restore happens once, while the gateway starts. The number of triggers it put +back is logged, so a restart that restored fewer than expected is visible in +the gateway log rather than only in a later 404. + +The gateway also removes triggers whose entity has left discovery. A restored +trigger is exempt from that until its entity has been discovered at least once: +immediately after a restart nothing has been discovered yet, and an entity that +has merely not been reported yet has not disappeared. A restored trigger whose +entity never appears stays listed and can be deleted through the API. + Fault Triggers (threshold rules) -------------------------------- diff --git a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/managers/trigger_manager.hpp b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/managers/trigger_manager.hpp index 3009c4b28..8107fe2e7 100644 --- a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/managers/trigger_manager.hpp +++ b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/managers/trigger_manager.hpp @@ -142,7 +142,11 @@ class TriggerManager { void shutdown(); /// Load persistent triggers from the store (on gateway restart). - void load_persistent_triggers(); + /// + /// Runs once, while the gateway is being constructed, and returns how many + /// triggers it put back. Nothing retries it, so a trigger that is not + /// restored here is absent for the life of the process. + size_t load_persistent_triggers(); // --- Hierarchy matching --------------------------------------------------- @@ -199,6 +203,7 @@ class TriggerManager { /// - `info.entity_id` and `info.entity_type` are immutable after creation /// and safe to read without `mtx` (e.g. in matches_entity()). /// - `active` is atomic and can be read/written without `mtx`. + /// - `entity_seen` is atomic and can be read/written without `mtx`. struct TriggerState { TriggerInfo info; nlohmann::json previous_value; @@ -206,6 +211,15 @@ class TriggerManager { std::mutex mtx; std::condition_variable cv; std::atomic active{true}; + /// Whether this gateway has ever found the trigger's entity in discovery. + /// + /// The orphan sweep removes a trigger whose entity is missing, and for a + /// trigger created through the API the entity was validated to exist, so + /// missing can only mean gone: true is the right starting point. A trigger + /// restored from the store is the exception - the process that created it + /// is not this one, so until discovery reports the entity, missing means + /// "not discovered yet" and there is nothing to conclude. + std::atomic entity_seen{true}; std::deque pending_events; static constexpr size_t kMaxPendingEvents = 100; std::atomic event_counter{0}; diff --git a/src/ros2_medkit_gateway/src/core/managers/trigger_manager.cpp b/src/ros2_medkit_gateway/src/core/managers/trigger_manager.cpp index db8aa61f5..3422adb5b 100644 --- a/src/ros2_medkit_gateway/src/core/managers/trigger_manager.cpp +++ b/src/ros2_medkit_gateway/src/core/managers/trigger_manager.cpp @@ -223,25 +223,50 @@ void TriggerManager::set_entity_exists_fn(EntityExistsFn fn) { } void TriggerManager::sweep_orphaned_triggers() { - // Phase 1: collect orphaned trigger IDs under lock - std::vector orphaned; + // Phase 1: collect orphaned triggers under lock + std::vector> orphaned; // trigger id, entity id + WarnLogFn warn_fn; { std::lock_guard lock(triggers_mutex_); std::lock_guard elock(entity_exists_mutex_); if (!entity_exists_fn_) { return; } + warn_fn = warn_log_fn_; for (const auto & [id, state] : triggers_) { // entity_id and entity_type are immutable after creation - safe to read without state->mtx - if (state->active.load() && !entity_exists_fn_(state->info.entity_id, state->info.entity_type)) { - orphaned.push_back(id); + if (!state->active.load()) { + continue; + } + if (entity_exists_fn_(state->info.entity_id, state->info.entity_type)) { + state->entity_seen.store(true); + continue; + } + // An entity this gateway has never seen has not disappeared from its + // discovery, so a trigger on it is not orphaned - the sweep runs from + // the first tick after startup, long before DDS has reported nodes that + // were already running when the gateway came up. Sweeping it there would + // delete a restored persistent trigger from the shared store, and + // restore runs once per process, so nothing would ever bring it back. + if (!state->entity_seen.load()) { + continue; } + orphaned.emplace_back(id, state->info.entity_id); } } // Phase 2: remove without holding triggers_mutex_ (remove() re-acquires it) - for (const auto & id : orphaned) { - remove(id); + for (const auto & entry : orphaned) { + remove(entry.first); + } + + // A sweep deletes a persistent trigger from the store as well, so the + // operator loses it permanently. Name what went and why. + if (warn_fn) { + for (const auto & entry : orphaned) { + warn_fn("orphan sweep: removed trigger '" + entry.first + "' because entity '" + entry.second + + "' is not in the discovery cache."); + } } } @@ -657,79 +682,124 @@ void TriggerManager::set_on_removed(OnRemovedCallback callback) { // Persistent trigger loading // --------------------------------------------------------------------------- -void TriggerManager::load_persistent_triggers() { +size_t TriggerManager::load_persistent_triggers() { if (config_.on_restart_behavior != "restore") { - return; + return 0; } - auto load_result = store_.load_all(); - if (!load_result.has_value()) { - return; + // Collected under the lock, emitted after release: warn_log_fn_ is an + // arbitrary external callback and must not run under triggers_mutex_. + std::vector warnings; + WarnLogFn warn_fn; + { + std::lock_guard lock(triggers_mutex_); + warn_fn = warn_log_fn_; } - std::lock_guard lock(triggers_mutex_); - for (auto & info : load_result.value()) { - if (info.status != TriggerStatus::ACTIVE) { - continue; + auto load_result = store_.load_all(); + if (!load_result.has_value()) { + // Restore runs once, during construction. A store that cannot be read + // therefore does not delay the operator's persistent triggers, it ends + // them: the REST API answers 404 for the life of the process, exactly as + // if they had never been created. + if (warn_fn) { + warn_fn("persistent trigger restore: reading the trigger store failed (" + load_result.error() + + "). No persistent trigger was restored, and restore is not retried, so they stay absent until this " + "gateway is restarted."); } + return 0; + } - // Check if expired - if (info.expires_at.has_value() && std::chrono::system_clock::now() >= info.expires_at.value()) { - nlohmann::json fields; - fields["status"] = "TERMINATED"; - (void)store_.update(info.id, fields); - continue; - } + size_t restored = 0; + { + std::lock_guard lock(triggers_mutex_); + for (auto & info : load_result.value()) { + // A TERMINATED row is a trigger that already ran its course; the store + // keeps it as a record, and not restoring it is what TERMINATED means. + if (info.status != TriggerStatus::ACTIVE) { + continue; + } - auto state = std::make_shared(); - state->info = std::move(info); + if (info.expires_at.has_value() && std::chrono::system_clock::now() >= info.expires_at.value()) { + // The operator created this one and it is not coming back. Say so once + // - the write-back below makes it TERMINATED, so the next restart + // takes the branch above instead. + warnings.push_back("persistent trigger restore: trigger '" + info.id + "' on entity '" + info.entity_id + + "' expired while the gateway was down and was not restored."); + nlohmann::json fields; + fields["status"] = "TERMINATED"; + auto update_result = store_.update(info.id, fields); + if (!update_result.has_value()) { + warnings.push_back("persistent trigger restore: marking expired trigger '" + info.id + + "' TERMINATED in the store failed (" + update_result.error() + + "). It stays ACTIVE on disk and is re-examined on every restart."); + } + continue; + } - // Restore previous value state if available - auto state_result = store_.load_state(state->info.id); - if (state_result.has_value() && state_result.value().has_value()) { - state->previous_value = state_result.value().value(); - state->has_previous_value = true; - } + auto state = std::make_shared(); + state->info = std::move(info); + // This process has never discovered the entity, so the orphan sweep must + // not read "missing from the cache" as "gone" until discovery has had + // its say. + state->entity_seen.store(false); + + // Restore previous value state if available + auto state_result = store_.load_state(state->info.id); + if (state_result.has_value() && state_result.value().has_value()) { + state->previous_value = state_result.value().value(); + state->has_previous_value = true; + } - // Update next_id_ to avoid collisions - // IDs are "trig_N" - extract N and ensure next_id_ is beyond it - auto underscore_pos = state->info.id.find('_'); - if (underscore_pos != std::string::npos) { - try { - auto loaded_id = std::stoull(state->info.id.substr(underscore_pos + 1)); - uint64_t current = next_id_.load(); - while (current <= loaded_id && !next_id_.compare_exchange_weak(current, loaded_id + 1)) { + // Update next_id_ to avoid collisions + // IDs are "trig_N" - extract N and ensure next_id_ is beyond it + auto underscore_pos = state->info.id.find('_'); + if (underscore_pos != std::string::npos) { + try { + auto loaded_id = std::stoull(state->info.id.substr(underscore_pos + 1)); + uint64_t current = next_id_.load(); + while (current <= loaded_id && !next_id_.compare_exchange_weak(current, loaded_id + 1)) { + } + } catch (...) { + // Non-numeric suffix - skip ID adjustment } - } catch (...) { - // Non-numeric suffix - skip ID adjustment } - } - - add_to_dispatch_index(state->info.id, state->info.collection, state->info.entity_id); - // Re-subscribe to topic for restored data triggers via the transport. - if (topic_transport_ && state->info.collection == "data" && !state->info.resolved_topic_name.empty()) { - const std::string trigger_id = state->info.id; - const std::string entity_id = state->info.entity_id; - const std::string resource_path = state->info.resource_path; - auto handle = - topic_transport_->subscribe(state->info.resolved_topic_name, /*msg_type=*/"", - [this, entity_id, resource_path](const nlohmann::json & sample) { - notifier_.notify("data", entity_id, resource_path, sample, ChangeType::UPDATED); - }); - if (handle) { - topic_handles_[trigger_id] = std::move(handle); - } else { - // Subscribe failed during persistent-trigger restore (e.g. the - // topic disappeared between shutdown and restart, or rclcpp threw - // inside TriggerTopicSubscriber). Queue the trigger for retry on - // the next refresh tick instead of leaving it active-but-silent. - unresolved_data_triggers_.push_back({trigger_id, entity_id, resource_path, std::chrono::steady_clock::now()}); + add_to_dispatch_index(state->info.id, state->info.collection, state->info.entity_id); + + // Re-subscribe to topic for restored data triggers via the transport. + if (topic_transport_ && state->info.collection == "data" && !state->info.resolved_topic_name.empty()) { + const std::string trigger_id = state->info.id; + const std::string entity_id = state->info.entity_id; + const std::string resource_path = state->info.resource_path; + auto handle = topic_transport_->subscribe(state->info.resolved_topic_name, /*msg_type=*/"", + [this, entity_id, resource_path](const nlohmann::json & sample) { + notifier_.notify("data", entity_id, resource_path, sample, + ChangeType::UPDATED); + }); + if (handle) { + topic_handles_[trigger_id] = std::move(handle); + } else { + // Subscribe failed during persistent-trigger restore (e.g. the + // topic disappeared between shutdown and restart, or rclcpp threw + // inside TriggerTopicSubscriber). Queue the trigger for retry on + // the next refresh tick instead of leaving it active-but-silent. + unresolved_data_triggers_.push_back({trigger_id, entity_id, resource_path, std::chrono::steady_clock::now()}); + } } + + triggers_[state->info.id] = std::move(state); + ++restored; } + } - triggers_[state->info.id] = std::move(state); + if (warn_fn) { + for (const auto & w : warnings) { + warn_fn(w); + } } + + return restored; } // --------------------------------------------------------------------------- diff --git a/src/ros2_medkit_gateway/src/gateway_node.cpp b/src/ros2_medkit_gateway/src/gateway_node.cpp index a8debcc6e..09ac0fe53 100644 --- a/src/ros2_medkit_gateway/src/gateway_node.cpp +++ b/src/ros2_medkit_gateway/src/gateway_node.cpp @@ -1058,8 +1058,17 @@ GatewayNode::GatewayNode(const rclcpp::NodeOptions & options) : Node("ros2_medki return cache.find_entity(entity_id).has_value(); }); - // Load persistent triggers - trigger_mgr_->load_persistent_triggers(); + trigger_mgr_->set_warn_log_fn([this](const std::string & message) { + RCLCPP_WARN(get_logger(), "%s", message.c_str()); + }); + // Load persistent triggers. This is the only pass: a trigger the store + // holds but this call does not put back stays absent until the next + // restart, so the count is worth having in the log of every start that + // asked for a restore. + const size_t restored_triggers = trigger_mgr_->load_persistent_triggers(); + if (trigger_config.on_restart_behavior == "restore") { + RCLCPP_INFO(get_logger(), "Restored %zu persistent trigger(s) from the trigger store", restored_triggers); + } // Wire notifier to managers so they emit events for trigger evaluation if (update_mgr_) { @@ -1092,9 +1101,6 @@ GatewayNode::GatewayNode(const rclcpp::NodeOptions & options) : Node("ros2_medki } return ""; }); - trigger_mgr_->set_warn_log_fn([this](const std::string & message) { - RCLCPP_WARN(get_logger(), "%s", message.c_str()); - }); // The deferred-resolution budget must outlive at least one full discovery // refresh: plugin-declared topics only reach the entity cache on a refresh // pass, and refresh_interval_ms is configurable up to 60 s - equal to the diff --git a/src/ros2_medkit_gateway/test/test_trigger_manager.cpp b/src/ros2_medkit_gateway/test/test_trigger_manager.cpp index 104bd661a..4097dfb81 100644 --- a/src/ros2_medkit_gateway/test/test_trigger_manager.cpp +++ b/src/ros2_medkit_gateway/test/test_trigger_manager.cpp @@ -1076,6 +1076,80 @@ TEST_F(TriggerManagerTest, Sweep_FreesCapacitySlots) { EXPECT_TRUE(after_sweep.has_value()) << "Should have capacity after sweep: " << after_sweep.error().message; } +/// A gateway that has never seen an entity cannot conclude the entity is gone. +/// Restore runs once, during construction, and the sweep runs from the first +/// tick after that - long before DDS reports nodes that were already up. A +/// sweep there would delete the trigger from the shared store for good. +TEST(LoadPersistentTriggers, SweepKeepsRestoredTriggerBeforeItsEntityIsDiscovered) { + ResourceChangeNotifier notifier; + ConditionRegistry registry; + registry.register_condition("OnChange", std::make_shared()); + + SqliteTriggerStore store(":memory:"); + ASSERT_TRUE(store.save(make_persistent_trigger("trig_7")).has_value()); + + TriggerConfig config; + config.max_triggers = 100; + config.on_restart_behavior = "restore"; + TriggerManager manager(notifier, registry, store, config); + manager.load_persistent_triggers(); + ASSERT_TRUE(manager.get("trig_7").has_value()) << "Trigger should have been restored"; + + // Discovery has not reported the entity yet - the state every restart passes + // through, for as long as DDS takes. + manager.set_entity_exists_fn([](const std::string & /*id*/, const std::string & /*entity_type*/) { + return false; + }); + manager.sweep_orphaned_triggers(); + manager.sweep_orphaned_triggers(); + + EXPECT_TRUE(manager.get("trig_7").has_value()) + << "Restored trigger was swept before its entity had a chance to be discovered"; + auto loaded = store.load_all(); + ASSERT_TRUE(loaded.has_value()); + EXPECT_EQ(loaded->size(), 1u) << "The store row must survive too - restore never runs again"; + + manager.shutdown(); + notifier.shutdown(); +} + +/// The relaxation above is bounded by having seen the entity once: after that, +/// a missing entity is a real orphan and the trigger goes, store row included. +TEST(LoadPersistentTriggers, SweepRemovesRestoredTriggerAfterItsEntityIsDiscoveredAndLost) { + ResourceChangeNotifier notifier; + ConditionRegistry registry; + registry.register_condition("OnChange", std::make_shared()); + + SqliteTriggerStore store(":memory:"); + ASSERT_TRUE(store.save(make_persistent_trigger("trig_8")).has_value()); + + TriggerConfig config; + config.max_triggers = 100; + config.on_restart_behavior = "restore"; + TriggerManager manager(notifier, registry, store, config); + manager.load_persistent_triggers(); + ASSERT_TRUE(manager.get("trig_8").has_value()); + + bool entity_present = true; + manager.set_entity_exists_fn([&entity_present](const std::string & /*id*/, const std::string & /*entity_type*/) { + return entity_present; + }); + + manager.sweep_orphaned_triggers(); + ASSERT_TRUE(manager.get("trig_8").has_value()) << "A discovered entity is not an orphan"; + + entity_present = false; + manager.sweep_orphaned_triggers(); + + EXPECT_FALSE(manager.get("trig_8").has_value()) << "An entity that was discovered and then lost is an orphan"; + auto loaded = store.load_all(); + ASSERT_TRUE(loaded.has_value()); + EXPECT_TRUE(loaded->empty()) << "Sweeping an orphan drops its store row as well"; + + manager.shutdown(); + notifier.shutdown(); +} + // =========================================================================== // Deferred topic resolution tests // =========================================================================== diff --git a/src/ros2_medkit_integration_tests/CMakeLists.txt b/src/ros2_medkit_integration_tests/CMakeLists.txt index 361dbc571..4656ebeb3 100644 --- a/src/ros2_medkit_integration_tests/CMakeLists.txt +++ b/src/ros2_medkit_integration_tests/CMakeLists.txt @@ -244,6 +244,15 @@ if(BUILD_TESTING) test_startup_param_clamp_warnings) set(_MULTI_GATEWAY_DOMAINS 4) + # Two-gateway tests that need the second gateway isolated from the first and + # ask for offset 1 only. Listed apart from the four-domain set above because + # a test holds every domain it is given for as long as it runs: asking for + # four where two are used takes two away from whatever the runner could + # otherwise have started alongside it. + set(_TWO_GATEWAY_TESTS + test_triggers_restore_before_discovery) + set(_TWO_GATEWAY_DOMAINS 2) + # Tests whose polling budgets do not fit the glob's default timeout. Keyed by # test name; anything not listed takes the default for its glob. # @@ -290,6 +299,8 @@ if(BUILD_TESTING) set(_test_domains 1) if(test_name IN_LIST _MULTI_GATEWAY_TESTS) set(_test_domains ${_MULTI_GATEWAY_DOMAINS}) + elseif(test_name IN_LIST _TWO_GATEWAY_TESTS) + set(_test_domains ${_TWO_GATEWAY_DOMAINS}) endif() set(_test_timeout 120) list(FIND _MEDKIT_TEST_TIMEOUT_OVERRIDES ${test_name} _test_timeout_idx) @@ -319,6 +330,8 @@ if(BUILD_TESTING) set(_test_domains 1) if(test_name IN_LIST _MULTI_GATEWAY_TESTS) set(_test_domains ${_MULTI_GATEWAY_DOMAINS}) + elseif(test_name IN_LIST _TWO_GATEWAY_TESTS) + set(_test_domains ${_TWO_GATEWAY_DOMAINS}) endif() set(_test_timeout 300) list(FIND _MEDKIT_TEST_TIMEOUT_OVERRIDES ${test_name} _test_timeout_idx) diff --git a/src/ros2_medkit_integration_tests/test/features/test_triggers_restore_before_discovery.test.py b/src/ros2_medkit_integration_tests/test/features/test_triggers_restore_before_discovery.test.py new file mode 100644 index 000000000..4135439cb --- /dev/null +++ b/src/ros2_medkit_integration_tests/test/features/test_triggers_restore_before_discovery.test.py @@ -0,0 +1,319 @@ +#!/usr/bin/env python3 +# Copyright 2026 bburda +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Feature test: a restored persistent trigger outlives its entity's discovery. + +A gateway restores persistent triggers once, while it is being constructed, +and then sweeps triggers whose entity is missing from the discovery cache. +Straight after a restart the cache is empty for as long as DDS takes to report +the nodes that are already running, so for a restored trigger "missing from the +cache" means "not discovered yet", not "gone". A sweep that cannot tell those +apart deletes the trigger from the shared store, and because restore never runs +again the trigger is unrecoverable for the life of the process. + +Nothing here is raced or timed. The restarted gateway lives on a DDS domain of +its own, the only node that can put its entity on that domain is started by the +test rather than by a clock, and the sweep cadence is pinned to the fastest the +parameter allows - so the window in which the entity is provably absent is as +long as the test says, whatever the machine is doing. +""" + +import os +import tempfile +import time +import unittest + +from launch import LaunchDescription +from launch.actions import ExecuteProcess, RegisterEventHandler, TimerAction +from launch.event_handlers import OnProcessExit +import launch_testing +import launch_testing.actions +import requests + +from ros2_medkit_test_utils.constants import ( + ALLOWED_EXIT_CODES, + API_BASE_PATH, + get_test_domain_id, + get_test_port, +) +from ros2_medkit_test_utils.gateway_test_case import GatewayTestCase +from ros2_medkit_test_utils.launch_helpers import create_demo_nodes, create_gateway_node + +PORT_PRIMARY = get_test_port(0) +PORT_RESTARTED = get_test_port(1) + +BASE_URL_PRIMARY = f'http://localhost:{PORT_PRIMARY}{API_BASE_PATH}' +BASE_URL_RESTARTED = f'http://localhost:{PORT_RESTARTED}{API_BASE_PATH}' + +# The restarted gateway gets a domain to itself so the primary's demo node is +# invisible to it: the only ``temp_sensor`` it can ever discover is the one the +# test starts on that domain. +PRIMARY_DOMAIN = get_test_domain_id(0) +RESTARTED_DOMAIN = get_test_domain_id(1) + +# Sweep cadence of the restarted gateway. 100 ms is the smallest +# ``refresh_interval_ms`` the gateway accepts, so this is the configuration in +# which the sweep is furthest from being a proxy for DDS convergence. +SWEEP_INTERVAL_MS = 100 + +# How long the restarted gateway is left running with its entity provably +# absent. Two orders of magnitude above the sweep cadence, so the restored +# trigger is offered to the sweep many times over. +SWEEP_WINDOW_SECONDS = 3.0 + +DB_PATH = os.path.join( + tempfile.gettempdir(), + f'test_triggers_restore_before_discovery_{os.getpid()}.db', +) + +# Two handshakes, both written by the test. The restarted gateway must not +# start before the trigger is in the shared store, and its entity's node must +# not start before the test has watched the gateway run without it. +GATE_RESTART = os.path.join( + tempfile.gettempdir(), + f'test_triggers_restore_before_discovery_restart_{os.getpid()}', +) +GATE_ENTITY = os.path.join( + tempfile.gettempdir(), + f'test_triggers_restore_before_discovery_entity_{os.getpid()}', +) + +APP_ID = 'temp_sensor' +RESOURCE_URI = f'/api/v1/apps/{APP_ID}/faults' + + +def _gate_process(name, path): + """Return a process that exits once ``path`` exists.""" + return ExecuteProcess( + cmd=['sh', '-c', f'while [ ! -e "{path}" ]; do sleep 0.2; done'], + name=name, + output='screen', + ) + + +def generate_test_description(): + """Launch a primary gateway and a restarted one whose entity arrives late.""" + primary = create_gateway_node( + name=f'ros2_medkit_gateway_{PORT_PRIMARY}', + port=PORT_PRIMARY, + extra_params={ + 'triggers.enabled': True, + 'triggers.storage.path': DB_PATH, + 'triggers.on_restart_behavior': 'reset', + }, + extra_env={'ROS_DOMAIN_ID': str(PRIMARY_DOMAIN)}, + ) + restarted = create_gateway_node( + name=f'ros2_medkit_gateway_{PORT_RESTARTED}', + port=PORT_RESTARTED, + extra_params={ + 'triggers.enabled': True, + 'triggers.storage.path': DB_PATH, + 'triggers.on_restart_behavior': 'restore', + 'refresh_interval_ms': SWEEP_INTERVAL_MS, + }, + extra_env={'ROS_DOMAIN_ID': str(RESTARTED_DOMAIN)}, + ) + + primary_demo = create_demo_nodes( + [APP_ID], lidar_faulty=False, + extra_env={'ROS_DOMAIN_ID': str(PRIMARY_DOMAIN)}, + ) + restarted_demo = create_demo_nodes( + [APP_ID], lidar_faulty=False, + extra_env={'ROS_DOMAIN_ID': str(RESTARTED_DOMAIN)}, + ) + + delayed_primary_demo = TimerAction( + period=2.0, + actions=primary_demo + [launch_testing.actions.ReadyToTest()], + ) + + restart_gate = _gate_process('restart_gate', GATE_RESTART) + entity_gate = _gate_process('entity_gate', GATE_ENTITY) + + return ( + LaunchDescription([ + primary, + restart_gate, + RegisterEventHandler( + OnProcessExit(target_action=restart_gate, on_exit=[restarted]), + ), + entity_gate, + RegisterEventHandler( + OnProcessExit(target_action=entity_gate, on_exit=restarted_demo), + ), + delayed_primary_demo, + ]), + {'primary': primary, 'restarted': restarted}, + ) + + +def _remove_gates(): + """Drop the gate files so a rerun does not inherit an open gate.""" + for path in (GATE_RESTART, GATE_ENTITY): + if os.path.exists(path): + try: + os.unlink(path) + except OSError: + pass + + +def _open_gate(path): + """Write a gate file, releasing the process that waits on it.""" + with open(path, 'w', encoding='utf-8') as gate: + gate.write('open') + + +def _wait_for_health(base_url, *, timeout=60.0): + """Poll /health until 200 or timeout.""" + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + r = requests.get(f'{base_url}/health', timeout=2) + if r.status_code == 200: + return + except requests.exceptions.RequestException: + pass + time.sleep(0.5) + raise AssertionError(f'Gateway at {base_url} not healthy after {timeout}s') + + +def _wait_for_app(base_url, app_id, *, timeout=60.0): + """Poll /apps until the given app_id is discovered.""" + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + r = requests.get(f'{base_url}/apps/{app_id}', timeout=2) + if r.status_code == 200: + return + except requests.exceptions.RequestException: + pass + time.sleep(0.5) + raise AssertionError( + f'App {app_id!r} not discovered at {base_url} after {timeout}s' + ) + + +class TestTriggersRestoreBeforeDiscovery(GatewayTestCase): + """A restored trigger survives until its entity is discovered.""" + + BASE_URL = BASE_URL_PRIMARY + + MIN_EXPECTED_APPS = 0 + REQUIRED_APPS: set = set() + REQUIRED_AREAS: set = set() + + _trigger_id: str = '' + + @classmethod + def setUpClass(cls): + """Wait for the primary gateway and its demo node.""" + _wait_for_health(BASE_URL_PRIMARY, timeout=60.0) + _wait_for_app(BASE_URL_PRIMARY, APP_ID, timeout=60.0) + cls.addClassCleanup(_remove_gates) + + # @verifies REQ_INTEROP_029 + def test_01_create_persistent_trigger(self): + """POST a persistent trigger on the primary gateway, then restart.""" + body = { + 'resource': RESOURCE_URI, + 'trigger_condition': {'condition_type': 'OnChange'}, + 'multishot': True, + 'persistent': True, + 'lifetime': 3600, + } + r = requests.post( + f'{BASE_URL_PRIMARY}/apps/{APP_ID}/triggers', + json=body, + timeout=5, + ) + self.assertEqual(r.status_code, 201, f'Create failed: {r.text}') + trig = r.json() + self.assertTrue(trig.get('persistent'), 'trigger must be persistent') + TestTriggersRestoreBeforeDiscovery._trigger_id = trig['id'] + + # The row is in the shared store, which is the precondition the restore + # path needs. Opening this gate starts the restarted gateway. + _open_gate(GATE_RESTART) + + # @verifies REQ_INTEROP_096 + def test_02_restored_trigger_survives_until_entity_is_discovered(self): + """The restored trigger is still there once its entity finally appears. + + The restarted gateway sweeps for orphaned triggers throughout a window + in which its entity does not exist on its domain at all. The trigger + must outlive that window: nothing has disappeared from this gateway's + discovery, so nothing is orphaned. + """ + self.assertTrue( + self._trigger_id, + 'test_01 must set _trigger_id before test_02 runs', + ) + + _wait_for_health(BASE_URL_RESTARTED, timeout=60.0) + + # The entity is absent by construction: the only node that could put it + # on this domain is started by the gate below, and nothing has opened + # that gate yet. + r = requests.get(f'{BASE_URL_RESTARTED}/apps/{APP_ID}', timeout=5) + self.assertEqual( + r.status_code, 404, + f'{APP_ID!r} must not exist on the restarted gateway before its ' + f'gate is opened - without that window this test proves nothing, ' + f'got {r.status_code}', + ) + + time.sleep(SWEEP_WINDOW_SECONDS) + + _open_gate(GATE_ENTITY) + _wait_for_app(BASE_URL_RESTARTED, APP_ID, timeout=60.0) + + r = requests.get( + f'{BASE_URL_RESTARTED}/apps/{APP_ID}/triggers/{self._trigger_id}', + timeout=5, + ) + self.assertEqual( + r.status_code, 200, + f'Restored trigger {self._trigger_id!r} was dropped while its ' + f'entity was still being discovered: GET returned {r.status_code}: ' + f'{r.text}', + ) + trig = r.json() + self.assertEqual(trig['status'], 'active') + self.assertTrue(trig.get('persistent')) + self.assertEqual(trig.get('observed_resource'), RESOURCE_URI) + + +@launch_testing.post_shutdown_test() +class TestShutdown(unittest.TestCase): + + def test_exit_codes(self, proc_info): + """Check all processes exited cleanly (SIGTERM allowed).""" + for info in proc_info: + self.assertIn( + info.returncode, ALLOWED_EXIT_CODES, + f'{info.process_name} exited with code {info.returncode}', + ) + + # SQLite in WAL mode writes two sidecars next to the DB, and a run that + # leaves them behind leaves state a later run can find. + for path in (DB_PATH, f'{DB_PATH}-wal', f'{DB_PATH}-shm'): + if os.path.exists(path): + try: + os.unlink(path) + except OSError: + pass + _remove_gates() From 978447397e0b102910cfcb02955c81f0de226153 Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Thu, 20 Aug 2026 11:44:36 +0200 Subject: [PATCH 15/22] fix(subscriptions): keep the sampler declaration inside the gateway Whether a sampler narrows its payload to one named resource is something the gateway's own registry needs to know, and nothing else. Putting it on the plugin context made it part of an interface plugins call through, and because the ROS-facing context derives from that interface, any change to the base shifts the derived slots too - so every plugin binary would have needed rebuilding for a capability no plugin in the tree asks for. A sampler a plugin registers is recorded as streaming its collection whole, so a subscription naming a single resource on it is refused exactly as before. --- docs/api/rest.rst | 3 +- docs/tutorials/plugin-system.rst | 51 +++++-------------- .../core/plugins/plugin_context.hpp | 9 +--- .../core/plugins/plugin_types.hpp | 9 +--- .../src/plugins/plugin_context.cpp | 10 ++-- .../test/test_plugin_manager.cpp | 49 ++++++++++++------ .../design/index.rst | 5 -- .../test/test_graph_provider_plugin.cpp | 19 +------ .../test/test_opcua_identity.cpp | 4 +- .../test/test_opcua_plugin.cpp | 3 +- .../test/test_sovd_service_interface.cpp | 3 +- 11 files changed, 63 insertions(+), 102 deletions(-) diff --git a/docs/api/rest.rst b/docs/api/rest.rst index 0bc4aea05..c9d5eddf9 100644 --- a/docs/api/rest.rst +++ b/docs/api/rest.rst @@ -1925,8 +1925,7 @@ Subscriptions are temporary - they do not survive server restart. - ``faults`` - Fault list. Streamed as a whole; no resource path - ``configurations`` - Parameter values. Streamed as a whole; no resource path - ``logs`` - Application log entries from ``/rosout``. Streamed as a whole; no resource path -- ``x-*`` - Vendor extensions (e.g. ``x-medkit-graph``). Streamed as a whole unless the - plugin registering the sampler declares that it narrows its payload to a named resource +- ``x-*`` - Vendor extensions (e.g. ``x-medkit-graph``). Streamed as a whole; no resource path A collection that is streamed as a whole delivers every item of that collection on every tick. A resource URI naming a single item of such a collection is refused with diff --git a/docs/tutorials/plugin-system.rst b/docs/tutorials/plugin-system.rst index 9851103c8..54187770e 100644 --- a/docs/tutorials/plugin-system.rst +++ b/docs/tutorials/plugin-system.rst @@ -295,8 +295,7 @@ providing access to gateway data and utilities: - ``acquire_lock()`` / ``release_lock()`` - acquire and release entity locks with optional scope and TTL - ``get_entity_snapshot()`` - returns an ``IntrospectionInput`` populated from the current entity cache - ``list_all_faults()`` - returns JSON object with a ``"faults"`` array containing all active faults across all entities -- ``register_sampler(collection, fn, honours_resource_path = false)`` - registers a cyclic - subscription sampler for a custom collection name +- ``register_sampler(collection, fn)`` - registers a cyclic subscription sampler for a custom collection name .. code-block:: cpp @@ -334,18 +333,16 @@ and reflects the state of the gateway's thread-safe entity cache. ``list_all_faults()`` is useful for plugins that need cross-entity fault visibility (e.g. mapping fault codes to topics). Returns ``{}`` if the fault manager is unavailable. -``register_sampler(collection, fn, honours_resource_path)`` wires a sampler into the -``ResourceSamplerRegistry`` so that cyclic subscriptions created for ``collection`` -(e.g. ``"x-medkit-metrics"``) call ``fn(entity_id, resource_path)`` on each tick. The -function must return ``tl::expected``. See -`Cyclic Subscription Extensions`_ for the lower-level registry API. +``register_sampler(collection, fn)`` wires a sampler into the ``ResourceSamplerRegistry`` +so that cyclic subscriptions created for ``collection`` (e.g. ``"x-medkit-metrics"``) +call ``fn(entity_id, resource_path)`` on each tick. The function must return +``tl::expected``. See `Cyclic Subscription Extensions`_ +for the lower-level registry API. -``honours_resource_path`` defaults to ``false`` and states whether ``fn`` narrows its -payload to the one resource named by ``resource_path``. Leave it ``false`` for a sampler -that ignores that argument: the gateway then refuses a subscription whose resource URI -names a single item of the collection with 400 ``x-medkit-invalid-resource-uri``, instead -of accepting it and streaming the whole collection on every tick. Set it to ``true`` only -when ``fn`` actually reads ``resource_path``. +A sampler registered through the plugin context streams its whole collection on every +tick. A subscription whose resource URI names a single item of that collection is +refused with 400 ``x-medkit-invalid-resource-uri`` rather than accepted and answered +with everything, so the URI must end at the collection. .. note:: @@ -412,7 +409,7 @@ all entities. Returns an empty object if the fault manager is unavailable: // Process each fault } -**register_sampler(collection, fn, honours_resource_path = false)** +**register_sampler(collection, fn)** Registers a cyclic subscription sampler for a custom collection name. Once registered, clients can create cyclic subscriptions on that collection for any @@ -428,21 +425,6 @@ entity: return *data; }); -The sampler above ignores ``resource_path`` and so answers with the whole -collection; leaving ``honours_resource_path`` at its ``false`` default is what -makes the gateway refuse ``/x-medkit-metrics/{item}`` instead of streaming -everything in response to it. A sampler that does read ``resource_path`` passes -``true``: - -.. code-block:: cpp - - ctx_->register_sampler("x-medkit-metrics", - [this](const std::string& entity_id, const std::string& resource_path) - -> tl::expected { - return collect_one_metric(entity_id, resource_path); - }, - /*honours_resource_path=*/true); - This is a convenience wrapper around the lower-level ``ResourceSamplerRegistry`` API described in `Cyclic Subscription Extensions`_. @@ -492,8 +474,7 @@ and transport providers during ``set_context()``. **Resource Samplers** provide the data for a collection when sampled by a subscription. Built-in samplers (``data``, ``faults``, ``configurations``, ``logs``, ``updates``) are -registered by the gateway during startup. Of these only ``data`` and ``updates`` narrow -their payload to a named resource; the rest stream their whole collection. Custom samplers are registered via ``ResourceSamplerRegistry`` +registered by the gateway during startup. Custom samplers are registered via ``ResourceSamplerRegistry`` on the ``GatewayNode``: .. code-block:: cpp @@ -503,14 +484,10 @@ on the ``GatewayNode``: [this](const std::string& entity_id, const std::string& resource_path) -> tl::expected { return get_metrics(entity_id, resource_path); - }, - /*is_builtin=*/false, /*honours_resource_path=*/true); + }); Once registered, clients can create cyclic subscriptions on the ``x-medkit-metrics`` -collection for any entity. The trailing ``honours_resource_path`` argument declares -that the sampler narrows its payload to the resource named in the subscription URI; -without it the gateway refuses such a URI with 400, because a sampler that ignores -``resource_path`` would answer a request for one item with the whole collection. +collection for any entity. **Transport Providers** deliver subscription data via alternative protocols (beyond the built-in SSE transport). Register via ``TransportRegistry`` on the ``GatewayNode``: diff --git a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/plugins/plugin_context.hpp b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/plugins/plugin_context.hpp index 5db68a96b..0212eaceb 100644 --- a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/plugins/plugin_context.hpp +++ b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/plugins/plugin_context.hpp @@ -186,17 +186,10 @@ class PluginContext { // ---- Resource sampler registration ---- /// Register a cyclic subscription sampler for a custom collection. - /// - /// `honours_resource_path` states that `fn` narrows its payload to the one - /// resource named by its second argument. Leave it false for a sampler that - /// ignores that argument: the gateway then refuses a subscription whose URI - /// names a single item of this collection, instead of accepting it and - /// streaming the whole collection on every tick. virtual void register_sampler( const std::string & /*collection*/, const std::function(const std::string &, const std::string &)> & - /*fn*/, - bool /*honours_resource_path*/ = false) { + /*fn*/) { } // ---- Trigger infrastructure access ---- diff --git a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/plugins/plugin_types.hpp b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/plugins/plugin_types.hpp index 10dcfc71b..d4598fbb3 100644 --- a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/plugins/plugin_types.hpp +++ b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/plugins/plugin_types.hpp @@ -40,14 +40,7 @@ namespace ros2_medkit_gateway { /// so a pre-compiled v6 `.so` is rejected. Out-of-tree plugins must be /// recompiled against v7 headers; in-tree plugins that `return /// PLUGIN_API_VERSION` pick up the bump automatically. -/// - v8: PluginContext::register_sampler() takes a trailing -/// `honours_resource_path` flag, defaulted to false, so a plugin can -/// declare that its sampler narrows its payload to a named resource. -/// Plugin SOURCE written against v7 compiles unchanged. The parameter -/// sits in an existing vtable slot, so a pre-compiled v7 `.so` would -/// call it with one argument missing; the loader's strict equality -/// against this value is what keeps such a `.so` out. -constexpr int PLUGIN_API_VERSION = 8; +constexpr int PLUGIN_API_VERSION = 7; /// Log severity levels for plugin logging callback enum class PluginLogLevel { kInfo, kWarn, kError }; diff --git a/src/ros2_medkit_gateway/src/plugins/plugin_context.cpp b/src/ros2_medkit_gateway/src/plugins/plugin_context.cpp index 00a09aded..47bc64937 100644 --- a/src/ros2_medkit_gateway/src/plugins/plugin_context.cpp +++ b/src/ros2_medkit_gateway/src/plugins/plugin_context.cpp @@ -297,12 +297,16 @@ class GatewayPluginContext : public RosPluginContext { return response; } + /// A plugin sampler is sampled with no way of saying that it narrows its + /// payload to the resource named by its second argument, so it is registered + /// as streaming its collection whole. A subscription URI naming a single item + /// of that collection is then refused rather than answered with everything. void register_sampler( const std::string & collection, - const std::function(const std::string &, const std::string &)> & fn, - bool honours_resource_path) override { + const std::function(const std::string &, const std::string &)> & fn) + override { if (sampler_registry_) { - sampler_registry_->register_sampler(collection, fn, /*is_builtin=*/false, honours_resource_path); + sampler_registry_->register_sampler(collection, fn, /*is_builtin=*/false, /*honours_resource_path=*/false); } } diff --git a/src/ros2_medkit_gateway/test/test_plugin_manager.cpp b/src/ros2_medkit_gateway/test/test_plugin_manager.cpp index fc2621f40..09b0f61e5 100644 --- a/src/ros2_medkit_gateway/test/test_plugin_manager.cpp +++ b/src/ros2_medkit_gateway/test/test_plugin_manager.cpp @@ -19,8 +19,10 @@ #include #include +#include "ros2_medkit_gateway/core/http/error_codes.hpp" #include "ros2_medkit_gateway/core/plugins/plugin_manager.hpp" #include "ros2_medkit_gateway/core/providers/introspection_provider.hpp" +#include "ros2_medkit_gateway/http/handlers/cyclic_subscription_handlers.hpp" #include "ros2_medkit_gateway/plugins/ros_plugin_context.hpp" using namespace ros2_medkit_gateway; @@ -233,8 +235,7 @@ class MockThrowOnShutdown : public GatewayPlugin { /// so a dangling registration after teardown would be a use-after-free. class MockSamplerPlugin : public GatewayPlugin { public: - explicit MockSamplerPlugin(std::string collection, bool honours_resource_path = false) - : collection_(std::move(collection)), honours_resource_path_(honours_resource_path) { + explicit MockSamplerPlugin(std::string collection) : collection_(std::move(collection)) { } std::string name() const override { return "sampler_plugin"; @@ -242,17 +243,14 @@ class MockSamplerPlugin : public GatewayPlugin { void configure(const json & /*cfg*/) override { } void set_context(PluginContext & context) override { - context.register_sampler( - collection_, - [this](const std::string &, const std::string &) -> tl::expected { - return json{{"plugin", name()}}; - }, - honours_resource_path_); + context.register_sampler(collection_, + [this](const std::string &, const std::string &) -> tl::expected { + return json{{"plugin", name()}}; + }); } private: std::string collection_; - bool honours_resource_path_; }; /// Same sampler-registering behavior as MockSamplerPlugin, but also throws @@ -453,22 +451,41 @@ TEST(PluginManagerTest, ShutdownAllRemovesPluginSamplers) { EXPECT_FALSE(sampler_registry.has_sampler("x-mock-shutdown-sampler")); } -TEST(PluginManagerTest, PluginResourcePathDeclarationReachesRegistry) { +/// A plugin registers a sampler through PluginContext, which carries no way of +/// saying that the sampler narrows its payload to a named resource. Such a +/// sampler therefore streams its whole collection, and a subscription URI +/// naming a single item of that collection is refused rather than accepted and +/// answered with everything on every tick. +TEST(PluginManagerTest, PluginSamplerRefusesAResourcePath) { ResourceSamplerRegistry sampler_registry; TransportRegistry transport_registry; PluginManager mgr; mgr.set_registries(sampler_registry, transport_registry); - mgr.add_plugin(std::make_unique("x-mock-whole-collection")); - mgr.add_plugin(std::make_unique("x-mock-per-item", /*honours_resource_path=*/true)); + mgr.add_plugin(std::make_unique("x-mock-metrics")); mgr.configure_plugins(); auto ctx = make_gateway_plugin_context(nullptr, nullptr, &sampler_registry); mgr.set_context(*ctx); - ASSERT_TRUE(sampler_registry.has_sampler("x-mock-whole-collection")); - ASSERT_TRUE(sampler_registry.has_sampler("x-mock-per-item")); - EXPECT_FALSE(sampler_registry.honours_resource_path("x-mock-whole-collection")); - EXPECT_TRUE(sampler_registry.honours_resource_path("x-mock-per-item")); + ASSERT_TRUE(sampler_registry.has_sampler("x-mock-metrics")); + + const std::string item_uri = "/api/v1/apps/node1/x-mock-metrics/cpu_usage"; + auto item_parsed = handlers::CyclicSubscriptionHandlers::parse_resource_uri(item_uri); + ASSERT_TRUE(item_parsed.has_value()); + auto item_result = + handlers::CyclicSubscriptionHandlers::validate_resource_path_support(sampler_registry, *item_parsed, item_uri); + ASSERT_FALSE(item_result.has_value()); + EXPECT_EQ(item_result.error().http_status, 400); + EXPECT_EQ(item_result.error().code, ERR_X_MEDKIT_INVALID_RESOURCE_URI); + EXPECT_EQ(item_result.error().params["collection"], "x-mock-metrics"); + + // The collection URI is the one form such a sampler can answer. + const std::string collection_uri = "/api/v1/apps/node1/x-mock-metrics"; + auto collection_parsed = handlers::CyclicSubscriptionHandlers::parse_resource_uri(collection_uri); + ASSERT_TRUE(collection_parsed.has_value()); + EXPECT_TRUE(handlers::CyclicSubscriptionHandlers::validate_resource_path_support(sampler_registry, *collection_parsed, + collection_uri) + .has_value()); } TEST(PluginManagerTest, BuiltinSamplerSurvivesPluginDisable) { diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_provider/design/index.rst b/src/ros2_medkit_plugins/ros2_medkit_graph_provider/design/index.rst index ffd26324e..c1e56f2be 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_graph_provider/design/index.rst +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_provider/design/index.rst @@ -300,8 +300,3 @@ The plugin registers a sampler via ``PluginContext::register_sampler()`` for the ``x-medkit-graph`` resource. This allows clients to create cyclic subscriptions that receive periodic graph snapshots over SSE, enabling live dashboard updates without polling the HTTP endpoint. - -The sampler builds one document for the whole function and ignores the resource -path, so it registers without ``honours_resource_path``. A subscription URI that -appends a path below ``x-medkit-graph`` is refused with 400 rather than answered -with the full graph document. diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_provider/test/test_graph_provider_plugin.cpp b/src/ros2_medkit_plugins/ros2_medkit_graph_provider/test/test_graph_provider_plugin.cpp index 7d8e40819..59bd5f194 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_graph_provider/test/test_graph_provider_plugin.cpp +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_provider/test/test_graph_provider_plugin.cpp @@ -345,10 +345,9 @@ class FakePluginContext : public RosPluginContext { void register_sampler( const std::string & collection, - const std::function(const std::string &, const std::string &)> & fn, - bool honours_resource_path) override { + const std::function(const std::string &, const std::string &)> & fn) + override { registered_samplers_[collection] = fn; - sampler_honours_resource_path_[collection] = honours_resource_path; } ResourceChangeNotifier * get_resource_change_notifier() override { @@ -371,7 +370,6 @@ class FakePluginContext : public RosPluginContext { std::unordered_map(const std::string &, const std::string &)>> registered_samplers_; - std::unordered_map sampler_honours_resource_path_; }; class LocalHttpServer { @@ -1652,19 +1650,6 @@ TEST(GraphProviderPluginRouteTest, RegistersSamplerForCyclicSubscriptions) { ASSERT_TRUE(result->contains("x-medkit-graph")); } -// The sampler builds one document for the whole function and ignores its -// resource_path argument, so it must register as not honouring a resource -// path - the gateway refuses a per-item subscription URI on that basis. -TEST(GraphProviderPluginRouteTest, SamplerDeclaresItDoesNotHonourResourcePath) { - GraphProviderPlugin plugin; - FakePluginContext ctx({{"f1", PluginEntityInfo{SovdEntityType::FUNCTION, "f1", "", ""}}}); - plugin.configure({}); - plugin.set_context(ctx); - - ASSERT_EQ(ctx.sampler_honours_resource_path_.count("x-medkit-graph"), 1u); - EXPECT_FALSE(ctx.sampler_honours_resource_path_["x-medkit-graph"]); -} - TEST(GraphProviderPluginRouteTest, AppliesConfigFromConfigure) { nlohmann::json config = { {"expected_frequency_hz_default", 10.0}, {"degraded_frequency_ratio", 0.8}, {"drop_rate_percent_threshold", 2.0}}; diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/test/test_opcua_identity.cpp b/src/ros2_medkit_plugins/ros2_medkit_opcua/test/test_opcua_identity.cpp index acacbe21a..48294503f 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/test/test_opcua_identity.cpp +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/test/test_opcua_identity.cpp @@ -135,8 +135,8 @@ class FakePluginContext : public RosPluginContext { } void register_sampler( const std::string &, - const std::function(const std::string &, const std::string &)> &, - bool) override { + const std::function(const std::string &, const std::string &)> &) + override { } ResourceChangeNotifier * get_resource_change_notifier() override { return nullptr; diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/test/test_opcua_plugin.cpp b/src/ros2_medkit_plugins/ros2_medkit_opcua/test/test_opcua_plugin.cpp index 8c823769d..4bfdf4bc6 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/test/test_opcua_plugin.cpp +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/test/test_opcua_plugin.cpp @@ -157,8 +157,7 @@ class FakePluginContext : public RosPluginContext { void register_sampler( const std::string & /*topic*/, const std::function(const std::string &, const std::string &)> & - /*sampler*/, - bool /*honours_resource_path*/) override { + /*sampler*/) override { } ResourceChangeNotifier * get_resource_change_notifier() override { return nullptr; diff --git a/src/ros2_medkit_plugins/ros2_medkit_sovd_service_interface/test/test_sovd_service_interface.cpp b/src/ros2_medkit_plugins/ros2_medkit_sovd_service_interface/test/test_sovd_service_interface.cpp index e45924501..8d69b44a3 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_sovd_service_interface/test/test_sovd_service_interface.cpp +++ b/src/ros2_medkit_plugins/ros2_medkit_sovd_service_interface/test/test_sovd_service_interface.cpp @@ -152,8 +152,7 @@ class FakePluginContext : public RosPluginContext { void register_sampler( const std::string & /*collection*/, const std::function(const std::string &, const std::string &)> & - /*fn*/, - bool /*honours_resource_path*/) override { + /*fn*/) override { } ResourceChangeNotifier * get_resource_change_notifier() override { From 46c214c58521e8bbf3bc0c55ce0262bceb24823d Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Thu, 20 Aug 2026 13:37:00 +0200 Subject: [PATCH 16/22] test: wait for a mock peer to listen before handing back its port A test server was started on a thread and its port returned immediately. stop() only interrupts a server that is already listening, so a teardown reaching it first left the listen running with nothing to end it, and the join never returned. The binary then hung until ctest killed it, naming whichever case happened to be running rather than the one that lost the race. Five files start a server this way; the rest of the suite already waits for readiness. One of them slept instead, which is the same race with a number attached to it. --- .../test/test_aggregation_manager.cpp | 4 ++ .../test/test_fan_out_helpers.cpp | 2 + .../test/test_peer_client.cpp | 42 +++++++++++++++++++ .../test/test_plugin_context_aggregation.cpp | 2 + .../test/test_graph_provider_plugin.cpp | 4 +- 5 files changed, 53 insertions(+), 1 deletion(-) diff --git a/src/ros2_medkit_gateway/test/test_aggregation_manager.cpp b/src/ros2_medkit_gateway/test/test_aggregation_manager.cpp index a2975c87d..89e6ab48f 100644 --- a/src/ros2_medkit_gateway/test/test_aggregation_manager.cpp +++ b/src/ros2_medkit_gateway/test/test_aggregation_manager.cpp @@ -735,6 +735,10 @@ class MockPeerServer { thread_ = std::thread([this]() { server_->listen_after_bind(); }); + // stop() only interrupts a server that is already listening. Returning + // before that leaves a teardown able to run first, and the listen then + // never ends. + server_->wait_until_ready(); return port_; } diff --git a/src/ros2_medkit_gateway/test/test_fan_out_helpers.cpp b/src/ros2_medkit_gateway/test/test_fan_out_helpers.cpp index e4d5e2819..bb835e8f4 100644 --- a/src/ros2_medkit_gateway/test/test_fan_out_helpers.cpp +++ b/src/ros2_medkit_gateway/test/test_fan_out_helpers.cpp @@ -209,6 +209,8 @@ class MockServer { thread_ = std::thread([this]() { server_->listen_after_bind(); }); + // stop() only interrupts a server that is already listening. + server_->wait_until_ready(); return port; } diff --git a/src/ros2_medkit_gateway/test/test_peer_client.cpp b/src/ros2_medkit_gateway/test/test_peer_client.cpp index 9d1c8a790..b65598289 100644 --- a/src/ros2_medkit_gateway/test/test_peer_client.cpp +++ b/src/ros2_medkit_gateway/test/test_peer_client.cpp @@ -148,6 +148,8 @@ TEST(PeerClientHappyPath, health_check_marks_healthy) { std::thread t([&]() { svr.listen_after_bind(); }); + // stop() only interrupts a server that is already listening. + svr.wait_until_ready(); PeerClient client("http://127.0.0.1:" + std::to_string(port), "test_peer", 5000); EXPECT_FALSE(client.is_healthy()); @@ -170,6 +172,8 @@ TEST(PeerClientHappyPath, health_check_unhealthy_on_500) { std::thread t([&]() { svr.listen_after_bind(); }); + // stop() only interrupts a server that is already listening. + svr.wait_until_ready(); PeerClient client("http://127.0.0.1:" + std::to_string(port), "test_peer", 5000); client.check_health(); @@ -203,6 +207,8 @@ TEST(PeerClientHappyPath, fetch_entities_parses_collections) { std::thread t([&]() { svr.listen_after_bind(); }); + // stop() only interrupts a server that is already listening. + svr.wait_until_ready(); PeerClient client("http://127.0.0.1:" + std::to_string(port), "test_peer", 5000); auto result = client.fetch_entities(); @@ -312,6 +318,8 @@ TEST(PeerClientHappyPath, fetch_entities_parses_relationship_fields) { std::thread t([&]() { svr.listen_after_bind(); }); + // stop() only interrupts a server that is already listening. + svr.wait_until_ready(); PeerClient client("http://127.0.0.1:" + std::to_string(port), "peer_ecu", 5000); auto result = client.fetch_entities(); @@ -393,6 +401,8 @@ TEST(PeerClientHappyPath, fetch_entities_parses_is_located_on_without_vendor_ext std::thread t([&]() { svr.listen_after_bind(); }); + // stop() only interrupts a server that is already listening. + svr.wait_until_ready(); PeerClient client("http://127.0.0.1:" + std::to_string(port), "peer_sovd", 5000); auto result = client.fetch_entities(); @@ -451,6 +461,8 @@ TEST(PeerClientHappyPath, fetch_entities_rejects_malicious_component_id_in_locat std::thread t([&]() { svr.listen_after_bind(); }); + // stop() only interrupts a server that is already listening. + svr.wait_until_ready(); PeerClient client("http://127.0.0.1:" + std::to_string(port), "peer_hostile", 5000); auto result = client.fetch_entities(); @@ -501,6 +513,8 @@ TEST(PeerClientHappyPath, fetch_entities_parses_vendor_only_component_id_fallbac std::thread t([&]() { svr.listen_after_bind(); }); + // stop() only interrupts a server that is already listening. + svr.wait_until_ready(); PeerClient client("http://127.0.0.1:" + std::to_string(port), "peer_vendor", 5000); auto result = client.fetch_entities(); @@ -531,6 +545,8 @@ TEST(PeerClientHappyPath, forward_request_proxies_response_with_auth) { std::thread t([&]() { svr.listen_after_bind(); }); + // stop() only interrupts a server that is already listening. + svr.wait_until_ready(); // forward_auth=true: Authorization header should be forwarded PeerClient client("http://127.0.0.1:" + std::to_string(port), "test_peer", 5000, true); @@ -566,6 +582,8 @@ TEST(PeerClientHappyPath, forward_request_does_not_forward_auth_by_default) { std::thread t([&]() { svr.listen_after_bind(); }); + // stop() only interrupts a server that is already listening. + svr.wait_until_ready(); // forward_auth=false (default): Authorization header should NOT be forwarded PeerClient client("http://127.0.0.1:" + std::to_string(port), "test_peer", 5000); @@ -603,6 +621,8 @@ TEST(PeerClientHappyPath, forward_filters_response_headers) { std::thread t([&]() { svr.listen_after_bind(); }); + // stop() only interrupts a server that is already listening. + svr.wait_until_ready(); PeerClient client("http://127.0.0.1:" + std::to_string(port), "test_peer", 5000); @@ -640,6 +660,8 @@ TEST(PeerClientHappyPath, forward_and_get_json_returns_parsed_json) { std::thread t([&]() { svr.listen_after_bind(); }); + // stop() only interrupts a server that is already listening. + svr.wait_until_ready(); PeerClient client("http://127.0.0.1:" + std::to_string(port), "test_peer", 5000); auto result = client.forward_and_get_json("GET", "/api/v1/components/ecu/data"); @@ -667,6 +689,8 @@ TEST(PeerClientHappyPath, forward_and_get_json_with_auth_header_when_enabled) { std::thread t([&]() { svr.listen_after_bind(); }); + // stop() only interrupts a server that is already listening. + svr.wait_until_ready(); // forward_auth=true: auth header should be sent PeerClient client("http://127.0.0.1:" + std::to_string(port), "test_peer", 5000, true); @@ -692,6 +716,8 @@ TEST(PeerClientHappyPath, forward_and_get_json_does_not_forward_auth_by_default) std::thread t([&]() { svr.listen_after_bind(); }); + // stop() only interrupts a server that is already listening. + svr.wait_until_ready(); // forward_auth=false (default): auth header should NOT be sent PeerClient client("http://127.0.0.1:" + std::to_string(port), "test_peer", 5000); @@ -715,6 +741,8 @@ TEST(PeerClientHappyPath, forward_and_get_json_error_on_non_2xx) { std::thread t([&]() { svr.listen_after_bind(); }); + // stop() only interrupts a server that is already listening. + svr.wait_until_ready(); PeerClient client("http://127.0.0.1:" + std::to_string(port), "test_peer", 5000); auto result = client.forward_and_get_json("GET", "/api/v1/missing"); @@ -742,6 +770,8 @@ TEST(PeerClientHappyPath, forward_request_rejects_oversized_response) { std::thread t([&]() { svr.listen_after_bind(); }); + // stop() only interrupts a server that is already listening. + svr.wait_until_ready(); PeerClient client("http://127.0.0.1:" + std::to_string(port), "big_peer", 10000); @@ -779,6 +809,8 @@ TEST(PeerClientHappyPath, forward_and_get_json_rejects_oversized_response) { std::thread t([&]() { svr.listen_after_bind(); }); + // stop() only interrupts a server that is already listening. + svr.wait_until_ready(); PeerClient client("http://127.0.0.1:" + std::to_string(port), "big_peer", 10000); auto result = client.forward_and_get_json("GET", "/api/v1/components/big/data"); @@ -805,6 +837,8 @@ TEST(PeerClientHappyPath, forward_post_request) { std::thread t([&]() { svr.listen_after_bind(); }); + // stop() only interrupts a server that is already listening. + svr.wait_until_ready(); PeerClient client("http://127.0.0.1:" + std::to_string(port), "test_peer", 5000); @@ -879,6 +913,8 @@ TEST(PeerClientHappyPath, fetch_entities_skips_entities_with_invalid_ids) { std::thread t([&]() { svr.listen_after_bind(); }); + // stop() only interrupts a server that is already listening. + svr.wait_until_ready(); PeerClient client("http://127.0.0.1:" + std::to_string(port), "malicious_peer", 5000); auto result = client.fetch_entities(); @@ -917,6 +953,8 @@ TEST(PeerClientHappyPath, fetch_entities_rejects_collection_exceeding_limit) { std::thread t([&]() { svr.listen_after_bind(); }); + // stop() only interrupts a server that is already listening. + svr.wait_until_ready(); PeerClient client("http://127.0.0.1:" + std::to_string(port), "oversized_peer", 5000); auto result = client.fetch_entities(); @@ -972,6 +1010,8 @@ TEST(PeerClientHappyPath, asset_identity_survives_fetch_and_merge) { std::thread t([&]() { svr.listen_after_bind(); }); + // stop() only interrupts a server that is already listening. + svr.wait_until_ready(); PeerClient client("http://127.0.0.1:" + std::to_string(port), "peer_plc", 5000); auto result = client.fetch_entities(); @@ -1055,6 +1095,8 @@ class ScopedServer { thread_ = std::thread([this]() { svr_.listen_after_bind(); }); + // stop() only interrupts a server that is already listening. + svr_.wait_until_ready(); } ~ScopedServer() { diff --git a/src/ros2_medkit_gateway/test/test_plugin_context_aggregation.cpp b/src/ros2_medkit_gateway/test/test_plugin_context_aggregation.cpp index afaf0640c..9119cd2f1 100644 --- a/src/ros2_medkit_gateway/test/test_plugin_context_aggregation.cpp +++ b/src/ros2_medkit_gateway/test/test_plugin_context_aggregation.cpp @@ -123,6 +123,8 @@ class MockPeerServer { thread_ = std::thread([this]() { server_->listen_after_bind(); }); + // stop() only interrupts a server that is already listening. + server_->wait_until_ready(); return port_; } diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_provider/test/test_graph_provider_plugin.cpp b/src/ros2_medkit_plugins/ros2_medkit_graph_provider/test/test_graph_provider_plugin.cpp index 59bd5f194..2cf8b5f2e 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_graph_provider/test/test_graph_provider_plugin.cpp +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_provider/test/test_graph_provider_plugin.cpp @@ -394,7 +394,9 @@ class LocalHttpServer { thread_ = std::thread([&server]() { server.listen_after_bind(); }); - std::this_thread::sleep_for(50ms); + // stop() only interrupts a server that is already listening, so a teardown + // that runs first would leave the listen with nothing to end it. + server.wait_until_ready(); } void stop() { From 3aa545d0539de93027f2aa4613595496bb0ff28c Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Thu, 20 Aug 2026 13:37:01 +0200 Subject: [PATCH 17/22] test(aggregation): give the grouping suite a budget its own waits fit inside The suite polls for up to a minute several times over: once per gateway while it comes up, and again after a peer is killed. Those add to more than the feature glob's default kill, so a slow runner takes the whole file down at once and reports no test name at all. --- src/ros2_medkit_integration_tests/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ros2_medkit_integration_tests/CMakeLists.txt b/src/ros2_medkit_integration_tests/CMakeLists.txt index 4656ebeb3..e297d559f 100644 --- a/src/ros2_medkit_integration_tests/CMakeLists.txt +++ b/src/ros2_medkit_integration_tests/CMakeLists.txt @@ -276,6 +276,7 @@ if(BUILD_TESTING) # polling budgets are similarly generous - both widened to match. set(_MEDKIT_TEST_TIMEOUT_OVERRIDES test_rosbag_boundary_download 180 + test_grouping_entity_aggregation 300 test_graph_provider_greenwave 300 test_graph_provider_stale 300 test_graph_provider_sse 300) From b6a49d372fc9c8fe31e59825d0051cffee2e82b8 Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Thu, 20 Aug 2026 16:07:05 +0200 Subject: [PATCH 18/22] docs(openapi): name every form an operation id can take The generated document described the short name and the member-qualified form. An operation whose short name is shared inside one provider is addressed by its ROS path, and a reader of the document had no way to know that. --- src/ros2_medkit_gateway/src/core/openapi/route_registry.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ros2_medkit_gateway/src/core/openapi/route_registry.cpp b/src/ros2_medkit_gateway/src/core/openapi/route_registry.cpp index 65f8cb263..b56716dc6 100644 --- a/src/ros2_medkit_gateway/src/core/openapi/route_registry.cpp +++ b/src/ros2_medkit_gateway/src/core/openapi/route_registry.cpp @@ -502,7 +502,8 @@ nlohmann::json RouteRegistry::to_openapi_paths() const { "the entity contributes an item under that name"}, {"operation_id", "The operation identifier (service or action short name), or 'member_id:operation' when more than " - "one member of the entity exposes that name"}, + "one member of the entity exposes that name, or the ROS 2 path without its leading slash when one " + "provider exposes that short name at more than one path"}, {"execution_id", "The execution identifier"}, {"config_id", "The configuration parameter identifier (ROS 2 parameter name)"}, {"fault_code", "The fault code identifier"}, From 281ba791850de1dfbbeb3a3def429568a673d460 Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Thu, 20 Aug 2026 16:07:32 +0200 Subject: [PATCH 19/22] fix(aggregation): read a peer item's member and its availability from this gateway's tree Both halves of a fan-out item were taken on the peer's word. A peer names its members as it knows them. When both gateways declare an app under one id the merge renames the peer's copy, so the peer's naming is not this gateway's, and a listed item carried the local member's name over the peer's operation. The collection then offered one id twice, and a client following it ran the local copy for both. Each item's member is now resolved to the id the merge gave its owner, in the collection and in the id itself. Availability was inferred from whether the peer's copy came back in the fan-out. A fan-out does not run at all for an entity no peer contributes, so a member on a healthy peer was reported unreachable on any entity declared only here, while a request addressed to it ran perfectly well. It is now read from the member, which is what the field is defined to describe and what the request itself acts on, so a listing and a request cannot disagree. The colliding leaf could not be driven through operations before: both sides bound it to nodes exposing none. It now serves one, and names its own node in the answer, which is the only thing on the wire that separates two copies that both succeed. --- docs/api/rest.rst | 16 ++ src/ros2_medkit_gateway/README.md | 15 ++ .../design/aggregation.rst | 25 +++ .../aggregation/aggregation_manager.hpp | 24 ++- .../core/http/fan_out_helpers.hpp | 29 ++- .../src/aggregation/aggregation_manager.cpp | 14 ++ .../src/http/handlers/data_handlers.cpp | 19 +- .../src/http/handlers/operation_handlers.cpp | 46 +++- .../test/test_aggregation_manager.cpp | 26 +++ .../CMakeLists.txt | 5 + .../demo_nodes/shared_leaf.cpp | 103 +++++++++ ...est_aggregator_only_configurations.test.py | 44 ++++ .../test_grouping_entity_aggregation.test.py | 201 +++++++++++++++++- 13 files changed, 543 insertions(+), 24 deletions(-) create mode 100644 src/ros2_medkit_integration_tests/demo_nodes/shared_leaf.cpp diff --git a/docs/api/rest.rst b/docs/api/rest.rst index c9d5eddf9..69a60c94b 100644 --- a/docs/api/rest.rst +++ b/docs/api/rest.rst @@ -659,6 +659,22 @@ keeps that short name, whatever another provider does with the same name. The split at the first colon is unchanged, because a ROS path carries no colon, and the path form is the one ``/data`` already uses for a topic. +**A member half from a peer is read through the collision rename.** A peer names +its own leaves as its own tree names them, and an App whose id collided with a +local one is merged here under ``__`` - so the name the peer sends +names the LOCAL leaf. Items arriving through the peer fan-out are re-attributed +to the id the merge gave their owner, both in ``x-medkit.member_ids`` and in the +member half of the item id, so ``secondary_gateway__shared_sensor:calibrate`` +addresses the peer's copy and ``shared_sensor:calibrate`` the local one. + +**Availability on a listed item describes its member.** ``x-medkit.available`` +is emitted only as ``false``, and only when the gateway that owns the item is +not answering; absence means the item can be served. An entity declared on this +gateway alone can host a member another gateway runs - no peer contributes the +entity, so its collection fan-out never runs - and that says nothing about the +member. The item is listed as usual, and the request for it is dispatched to the +member's own route. + What this means for a request: - A bare id that names one item works, on every route. Every client that sends diff --git a/src/ros2_medkit_gateway/README.md b/src/ros2_medkit_gateway/README.md index 879881bdf..43fe167cb 100644 --- a/src/ros2_medkit_gateway/README.md +++ b/src/ros2_medkit_gateway/README.md @@ -298,6 +298,21 @@ once keeps that short name, whatever any other provider does with it. `/data` already addresses its items by path, so the split at the first colon is unchanged - a ROS path carries no colon. +A member half a peer supplied is read through the collision rename before it is +used. A peer names its own leaves as its own tree names them, and an App whose +id collided with a local one is merged here under `__` - so the name +the peer sends names the LOCAL leaf. Items that arrive through the peer fan-out +are re-attributed to the id the merge gave their owner, in `member_ids` and in +the member half of the id itself, so every id the collection offers addresses +the copy it names. + +`x-medkit.available` on a listed item is a statement about its MEMBER: it +appears, as `false`, only when the gateway that owns the item is not answering. +An entity declared on this gateway alone can still host a member another gateway +runs; no peer contributes the entity, so its collection fan-out never runs, and +that says nothing about the member. Such an item is listed as usual and the +request for it is dispatched to the member's own route. + - A bare id that names one item works on every route, which is what the web UI, the Foxglove panel, the MCP tools and the generated OpenAPI document all send. - `POST /{entity}/operations/{id}/executions` with a bare id several members diff --git a/src/ros2_medkit_gateway/design/aggregation.rst b/src/ros2_medkit_gateway/design/aggregation.rst index 448b2bff0..5f493c4d3 100644 --- a/src/ros2_medkit_gateway/design/aggregation.rst +++ b/src/ros2_medkit_gateway/design/aggregation.rst @@ -436,6 +436,19 @@ member: PUT /api/v1/functions/vehicle_health/configurations/peer_calibration:calibration_offset -> PUT /api/v1/apps/peer_calibration/configurations/calibration_offset (on the peer) +A member half a peer supplied is read through the collision rename before it +means anything here. A peer describes its own tree in its own names, and an App +whose id collided with a local one was merged under ``__``, so the +name the peer sends names the LOCAL leaf on this gateway. ``fan_out_get`` +therefore records the peer each item came from in +``FanOutResult::item_peers``, and ``AggregationManager::local_member_id`` +resolves that name against the routing table: an item arriving through the +fan-out is re-attributed to the id the merge gave its owner, in +``x-medkit.member_ids`` and in the member half of the item id, before any of it +is offered to a client. Read verbatim instead, the peer's item is attributed to +a member that does not own it, the collection offers one id for two operations, +and the peer's copy is not addressable through the aggregate at all. + Each collection keeps its own id scheme and each hands the same two halves to the dispatch. ``/data`` and ``/operations`` qualify only an ambiguous id and carry ``x-medkit.member_ids``; ``/configurations`` qualifies every id on a @@ -695,6 +708,18 @@ so the last complete one survives; it re-checks that peer's health to decide whether to replay it marked unavailable (health check failed) or exactly as it was last read (health check still passes). +The same field on a listed ITEM answers for that item's member and for nothing +else. ``/operations`` holds back the copies its declared tree carries for +peer-owned members and offers them only when the fan-out did not bring the +owner's own copy, and whether such a copy is marked unavailable is decided from +the member's reachability - the same reading ``dispatch_to_member`` acts on, so +the listing and the request cannot disagree. A fan-out that produced nothing is +not evidence on its own: it also never runs when no peer contributes the entity, +which is the ordinary shape of a grouping declared on this gateway alone that +hosts a member another gateway runs. Deciding from the fan-out there marks every +peer-owned item of that entity unreachable while its gateway is answering +normally. + The aggregator also publishes its own ``/health`` response with two additional fields when aggregation is enabled (x-medkit extensions on our own endpoint, outside the SOVD core contract): diff --git a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/aggregation/aggregation_manager.hpp b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/aggregation/aggregation_manager.hpp index 6339c5ef0..59960c797 100644 --- a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/aggregation/aggregation_manager.hpp +++ b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/aggregation/aggregation_manager.hpp @@ -97,7 +97,15 @@ class AggregationManager { * @brief Result of a fan-out GET across all healthy peers */ struct FanOutResult { - nlohmann::json merged_items; ///< Merged "items" array from all peers + nlohmann::json merged_items; ///< Merged "items" array from all peers + /// Name of the peer that sent `merged_items[i]`, same order and length. + /// + /// A peer describes its own tree in its own names, and an id this gateway + /// renamed on collision means something else here, so an item can only be + /// re-addressed once the peer it came from is known. Merging the responses + /// into one array without this loses that, and the attribution cannot be + /// recovered afterwards from the item alone. + std::vector item_peers; bool is_partial{false}; ///< True if some peers failed std::vector failed_peers; ///< Names of peers that failed }; @@ -211,6 +219,20 @@ class AggregationManager { */ std::optional find_peer_for_entity(const std::string & entity_id) const; + /** + * @brief The id this gateway uses for a member `peer_name` calls `member_id`. + * + * An App whose id collides with a local one is merged under `__`, + * and the routing table is where that renaming is recorded. A peer knows + * nothing of it and keeps naming the member as it always did, so anything a + * peer says about its own members has to be read through this before it can + * be used to address something here - the unprefixed id names the LOCAL leaf. + * + * Returns `member_id` unchanged when nothing was renamed, which is the + * ordinary case. Thread-safe. + */ + std::string local_member_id(const std::string & peer_name, const std::string & member_id) const; + /** * @brief Replace the map of per-entity peer contributors. * diff --git a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/http/fan_out_helpers.hpp b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/http/fan_out_helpers.hpp index 68882a095..6fa1e9573 100644 --- a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/http/fan_out_helpers.hpp +++ b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/http/fan_out_helpers.hpp @@ -160,6 +160,14 @@ template struct FanOutResult { /// Typed peer items that successfully parsed as `T`. std::vector items; + /// Name of the peer that sent `items[i]`, same order and length as `items`. + /// + /// A peer names its own members, and a member this gateway renamed on + /// collision (`__`) is called something else here, so an item can + /// only be re-addressed to the leaf that owns it once the sending peer is + /// known. Kept parallel rather than folded into `T` because `T` is the wire + /// DTO and provenance is not part of the wire shape. + std::vector item_peers; /// True if at least one targeted peer failed. bool partial{false}; /// Names of peers that failed (matches AggregationManager::FanOutResult.failed_peers). @@ -185,10 +193,7 @@ struct FanOutResult { /// raw JSON. Items that fail validation are dropped from `items` and /// recorded in `dropped_items` with the JsonReader error message plus a /// best-effort `source_id`. A WARN is logged for each drop. -/// - the `peer` field on each DroppedItem is left empty in this commit -/// because AggregationManager::fan_out_get coalesces all peer responses -/// into one `merged_items` array without per-item provenance. Future work -/// can thread per-peer attribution through if needed. +/// - every item that parsed carries the peer it came from, in `item_peers`. template inline FanOutResult fan_out_collection(AggregationManager * agg, const httplib::Request & req) { FanOutResult result; @@ -216,21 +221,25 @@ inline FanOutResult fan_out_collection(AggregationManager * agg, const httpli auto fan_result = agg->fan_out_get(fan_path, req.get_header_value("Authorization"), target_peers); if (fan_result.merged_items.is_array()) { - for (const auto & item : fan_result.merged_items) { + for (size_t index = 0; index < fan_result.merged_items.size(); ++index) { + const auto & item = fan_result.merged_items[index]; + // `item_peers` is built alongside `merged_items` and is the same length; + // the bound is checked so a shorter list degrades to an unattributed item + // rather than reading past the end. + const std::string peer_name = index < fan_result.item_peers.size() ? fan_result.item_peers[index] : std::string{}; if (!item.is_object()) { continue; } auto parsed = dto::JsonReader::read(item); if (parsed.has_value()) { result.items.push_back(std::move(parsed.value())); + result.item_peers.push_back(peer_name); continue; } dto::DroppedItem dropped; - // Best-effort peer URL: per-item provenance is not available from - // AggregationManager::fan_out_get today (it coalesces peer responses - // into a single merged array). Left empty intentionally; if a future - // commit threads per-peer attribution through, this is the place to - // populate it. + // The wire key is a peer URL and what the fan-out carries is a peer name, + // so the two are not interchangeable. Left empty rather than filled with + // the wrong kind of identifier. dropped.peer = ""; // Best-effort source_id: scan a small set of common id keys. static constexpr std::array kIdKeys = {"id", "name", "fault_id", "data_id", "operation_id"}; diff --git a/src/ros2_medkit_gateway/src/aggregation/aggregation_manager.cpp b/src/ros2_medkit_gateway/src/aggregation/aggregation_manager.cpp index 2d77badc3..f13ca49d0 100644 --- a/src/ros2_medkit_gateway/src/aggregation/aggregation_manager.cpp +++ b/src/ros2_medkit_gateway/src/aggregation/aggregation_manager.cpp @@ -693,6 +693,19 @@ std::optional AggregationManager::find_peer_for_entity(const std::s return std::nullopt; } +std::string AggregationManager::local_member_id(const std::string & peer_name, const std::string & member_id) const { + if (peer_name.empty() || member_id.empty()) { + return member_id; + } + const std::string prefixed = peer_name + EntityMerger::SEPARATOR + member_id; + std::shared_lock lock(mutex_); + auto it = routing_table_.find(prefixed); + // The routing entry has to belong to THIS peer. Another peer can own an + // entity whose id happens to read like this peer's prefix, and answering with + // it would re-address the item to a gateway that never sent it. + return (it != routing_table_.end() && it->second == peer_name) ? prefixed : member_id; +} + void AggregationManager::update_peer_contributors( std::unordered_map> contributors) { // Drop entries with empty contributor lists - they would be indistinguishable @@ -878,6 +891,7 @@ AggregationManager::FanOutResult AggregationManager::fan_out_get(const std::stri } for (auto & item : pr.items) { fan_out_result.merged_items.push_back(std::move(item)); + fan_out_result.item_peers.push_back(pr.peer_name); } } diff --git a/src/ros2_medkit_gateway/src/http/handlers/data_handlers.cpp b/src/ros2_medkit_gateway/src/http/handlers/data_handlers.cpp index 6deabfb51..c7ad93d50 100644 --- a/src/ros2_medkit_gateway/src/http/handlers/data_handlers.cpp +++ b/src/ros2_medkit_gateway/src/http/handlers/data_handlers.cpp @@ -432,8 +432,23 @@ http::Result DataHandlers::list_data(const http::TypedReque #pragma GCC diagnostic ignored "-Wdeprecated-declarations" const auto & raw_req = req.raw_for_framework(); #pragma GCC diagnostic pop - auto fan_out = fan_out_collection(ctx_.aggregation_manager(), raw_req); - for (auto & item : fan_out.items) { + auto * agg = ctx_.aggregation_manager(); + auto fan_out = fan_out_collection(agg, raw_req); + + // A peer names its members as its own tree names them, and an App whose id + // collided with a local one was merged under `__` - so the name + // the peer sends names the LOCAL leaf here. A merged App carries no topics, + // so this attribution is the only account of who owns a peer's item, and a + // client that builds `:` out of it addresses a member that + // does not publish that topic at all. + for (size_t index = 0; index < fan_out.items.size(); ++index) { + auto & item = fan_out.items[index]; + const std::string peer_name = index < fan_out.item_peers.size() ? fan_out.item_peers[index] : std::string{}; + if (agg != nullptr && !peer_name.empty() && item.x_medkit.has_value() && item.x_medkit->member_ids.has_value()) { + for (auto & member_id : *item.x_medkit->member_ids) { + member_id = agg->local_member_id(peer_name, member_id); + } + } response.items.push_back(std::move(item)); } diff --git a/src/ros2_medkit_gateway/src/http/handlers/operation_handlers.cpp b/src/ros2_medkit_gateway/src/http/handlers/operation_handlers.cpp index f4373856d..7aa4a867c 100644 --- a/src/ros2_medkit_gateway/src/http/handlers/operation_handlers.cpp +++ b/src/ros2_medkit_gateway/src/http/handlers/operation_handlers.cpp @@ -706,9 +706,36 @@ http::Result> OperationHandlers::list_operat // peered pair into a bounce. A fan-out that ran and came back without them // means the peer is not answering, and the tree still knows what it declared. const bool fan_out_suppressed = raw_req.has_header("X-Medkit-No-Fan-Out"); - auto fan_out = fan_out_collection(ctx_.aggregation_manager(), raw_req); + auto * agg = ctx_.aggregation_manager(); + auto fan_out = fan_out_collection(agg, raw_req); + + // A peer names its members as its own tree names them, and an App whose id + // collided with a local one was merged under `__` - so the name the + // peer sends names the LOCAL leaf here. Re-emitted verbatim it attributes the + // peer's operation to a member that does not own it, and every id built from + // that attribution - by this gateway or by a client reading the list - is + // resolved against the wrong member. The member half of the id the peer + // already qualified carries the same name and is rewritten with it. + const auto retarget_to_local_member = [agg](dto::OperationItem & item, const std::string & peer_name) { + if (agg == nullptr || peer_name.empty() || !item.x_medkit.has_value() || !item.x_medkit->member_ids.has_value() || + item.x_medkit->member_ids->size() != 1) { + return; + } + const std::string reported = item.x_medkit->member_ids->front(); + const std::string local = agg->local_member_id(peer_name, reported); + if (local == reported) { + return; + } + item.x_medkit->member_ids = std::vector{local}; + auto parsed = http::parse_member_qualified_id(item.id, true); + if (parsed.has_member && parsed.member_id == reported) { + item.id = http::make_member_qualified_id(local, parsed.item_id); + } + }; + std::unordered_set paths_from_peers; - for (auto & item : fan_out.items) { + for (size_t index = 0; index < fan_out.items.size(); ++index) { + auto & item = fan_out.items[index]; if (item.x_medkit.has_value() && item.x_medkit->ros2.has_value()) { const auto & ros2 = *item.x_medkit->ros2; auto path = ros2.service.value_or(ros2.action.value_or(std::string{})); @@ -716,6 +743,7 @@ http::Result> OperationHandlers::list_operat paths_from_peers.insert(std::move(path)); } } + retarget_to_local_member(item, index < fan_out.item_peers.size() ? fan_out.item_peers[index] : std::string{}); qualify_from_declared_tree(item); collection.items.push_back(std::move(item)); } @@ -727,7 +755,19 @@ http::Result> OperationHandlers::list_operat if (!path.empty() && paths_from_peers.count(path) > 0u) { continue; // the owner answered for itself, which is the better copy } - item.x_medkit->available = false; + // `available` is a statement about the MEMBER, not about the fan-out: + // false means the gateway that owns the item is not answering, so a + // request for it cannot be served. A fan-out reaching this gateway with + // nothing for this path says nothing on its own - it also never ran when + // no peer contributes this entity, which is the ordinary shape of a + // grouping declared here that hosts a member another gateway runs. The + // member's own reachability is what a request for the item will meet, + // and it is the same reading `dispatch_to_member` acts on, so the listing + // and the execution cannot disagree. + auto owner = ops.owner_by_path.find(path); + if (owner != ops.owner_by_path.end() && member_is_unreachable(cache, owner->second)) { + item.x_medkit->available = false; + } collection.items.push_back(std::move(item)); } } diff --git a/src/ros2_medkit_gateway/test/test_aggregation_manager.cpp b/src/ros2_medkit_gateway/test/test_aggregation_manager.cpp index 89e6ab48f..9967b70fd 100644 --- a/src/ros2_medkit_gateway/test/test_aggregation_manager.cpp +++ b/src/ros2_medkit_gateway/test/test_aggregation_manager.cpp @@ -435,6 +435,32 @@ TEST(AggregationManager, get_peer_contributors_unions_routing_and_contributor_ma EXPECT_EQ(both_peers[1], "peer_y"); } +TEST(AggregationManager, local_member_id_reads_a_peers_own_name_through_the_collision_rename) { + auto config = make_config(0); + AggregationManager manager(config); + + // A member nothing collided with keeps the name its peer uses. + manager.update_routing_table({{"pressure_sensor", "peer_a"}}); + EXPECT_EQ(manager.local_member_id("peer_a", "pressure_sensor"), "pressure_sensor"); + + // A member whose id collided is merged under the prefixed id, and that is + // the only name that addresses it here. + manager.update_routing_table({{"peer_a__shared_sensor", "peer_a"}}); + EXPECT_EQ(manager.local_member_id("peer_a", "shared_sensor"), "peer_a__shared_sensor"); + + // The prefixed id has to be routed to the peer that sent the item. Another + // peer can own an entity whose id reads like this peer's prefix, and + // answering with it would re-address the item to a gateway that never sent + // it - here peer_b owns an entity literally called `peer_a__shared_sensor`, + // so peer_a's own uncollided `shared_sensor` must not be rewritten to it. + manager.update_routing_table({{"peer_a__shared_sensor", "peer_b"}}); + EXPECT_EQ(manager.local_member_id("peer_a", "shared_sensor"), "shared_sensor"); + + // An empty half names nothing to look up. + EXPECT_EQ(manager.local_member_id("", "shared_sensor"), "shared_sensor"); + EXPECT_EQ(manager.local_member_id("peer_a", ""), ""); +} + TEST(AggregationManager, update_peer_contributors_drops_empty_entries) { auto config = make_config(0); AggregationManager manager(config); diff --git a/src/ros2_medkit_integration_tests/CMakeLists.txt b/src/ros2_medkit_integration_tests/CMakeLists.txt index e297d559f..d6557f8a0 100644 --- a/src/ros2_medkit_integration_tests/CMakeLists.txt +++ b/src/ros2_medkit_integration_tests/CMakeLists.txt @@ -83,6 +83,10 @@ add_executable(demo_long_calibration_action demo_nodes/long_calibration_action.c target_include_directories(demo_long_calibration_action PRIVATE ${_demo_include_dir}) medkit_target_dependencies(demo_long_calibration_action rclcpp rclcpp_action example_interfaces) +add_executable(demo_shared_leaf demo_nodes/shared_leaf.cpp) +target_include_directories(demo_shared_leaf PRIVATE ${_demo_include_dir}) +medkit_target_dependencies(demo_shared_leaf rclcpp std_msgs std_srvs) + add_executable(demo_lidar_sensor demo_nodes/lidar_sensor.cpp) target_include_directories(demo_lidar_sensor PRIVATE ${_demo_include_dir}) medkit_target_dependencies(demo_lidar_sensor rclcpp rcl_interfaces sensor_msgs std_srvs ros2_medkit_msgs) @@ -120,6 +124,7 @@ install(TARGETS demo_calibration_service demo_dual_calibration_service demo_long_calibration_action + demo_shared_leaf demo_lidar_sensor demo_beacon_publisher demo_param_beacon_node diff --git a/src/ros2_medkit_integration_tests/demo_nodes/shared_leaf.cpp b/src/ros2_medkit_integration_tests/demo_nodes/shared_leaf.cpp new file mode 100644 index 000000000..f5b2e947b --- /dev/null +++ b/src/ros2_medkit_integration_tests/demo_nodes/shared_leaf.cpp @@ -0,0 +1,103 @@ +// Copyright 2026 bburda +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * @file shared_leaf.cpp + * @brief The node behind a leaf two gateways declare under one id. + * + * An App id declared on both sides of an aggregating pair is renamed on merge + * (`__`), and that rename only means something where an id built from + * the leaf id addresses something. So this node carries one resource of each + * kind that is addressed that way: `calibrate`, whose short name is the wire id + * of an operation, and `reading`, whose path is the wire id of a data item. + * + * The service answers with the node's fully qualified name. Two copies of this + * node both answer 200, so the name in the response is the only thing on the + * wire that says WHICH copy ran. + */ + +#include +#include +#include + +#include +#include +#include + +#include "ros2_medkit_integration_tests/demo_node_main.hpp" + +class SharedLeaf : public rclcpp::Node { + public: + SharedLeaf() : Node("shared_leaf") { + reading_pub_ = this->create_publisher("reading", 10); + + calibrate_srv_ = this->create_service( + "calibrate", [this](const std::shared_ptr & request, + const std::shared_ptr & response) { + (void)request; // Trigger has no request fields + response->success = true; + response->message = std::string(this->get_fully_qualified_name()) + " calibrated"; + RCLCPP_INFO(this->get_logger(), "Calibration requested: %s", response->message.c_str()); + }); + + timer_ = this->create_wall_timer(std::chrono::milliseconds(500), [this]() { + publish_reading(); + }); + + RCLCPP_INFO(this->get_logger(), "Shared leaf started"); + } + + // The service callback and the timer callback both capture `this`, so they + // have to be torn down before any member they touch is. + ~SharedLeaf() override { + timer_->cancel(); + std::lock_guard lock(callback_mutex_); + timer_.reset(); + calibrate_srv_.reset(); + reading_pub_.reset(); + } + + SharedLeaf(const SharedLeaf &) = delete; + SharedLeaf & operator=(const SharedLeaf &) = delete; + SharedLeaf(SharedLeaf &&) = delete; + SharedLeaf & operator=(SharedLeaf &&) = delete; + + private: + void publish_reading() { + std::lock_guard lock(callback_mutex_); + if (!reading_pub_) { + return; + } + reading_ += 1.0; + if (reading_ > 100.0) { + reading_ = 1.0; + } + + auto msg = std_msgs::msg::Float32(); + msg.data = static_cast(reading_); + reading_pub_->publish(msg); + } + + std::mutex callback_mutex_; + rclcpp::Publisher::SharedPtr reading_pub_; + rclcpp::Service::SharedPtr calibrate_srv_; + rclcpp::TimerBase::SharedPtr timer_; + double reading_ = 1.0; +}; + +int main(int argc, char * argv[]) { + return ros2_medkit_integration_tests::run_demo_node(argc, argv, []() -> std::shared_ptr { + return std::make_shared(); + }); +} diff --git a/src/ros2_medkit_integration_tests/test/features/test_aggregator_only_configurations.test.py b/src/ros2_medkit_integration_tests/test/features/test_aggregator_only_configurations.test.py index c6014a266..18dc0eb20 100644 --- a/src/ros2_medkit_integration_tests/test/features/test_aggregator_only_configurations.test.py +++ b/src/ros2_medkit_integration_tests/test/features/test_aggregator_only_configurations.test.py @@ -744,6 +744,50 @@ def test_a9_reset_all_on_an_aggregator_only_entity_is_not_plain_success(self): msg='the response said nothing was reset, but the peer value moved', ) + def test_b1_a_peer_owned_operation_of_a_local_entity_is_not_reported_unreachable(self): + """`available: false` is a statement about the member, not about the fan-out. + + Only this gateway declares the mixed Function, so no peer contributes + the entity and the peer collection fan-out for it never runs - there is + nobody to ask. That says nothing about the member: its gateway is up, + and the request for the operation is dispatched to the member's own + route and served. Marking the item unavailable there contradicts both + the field's meaning and what the very next request does, so the two are + checked together and the execution is asserted on its body - a status + alone cannot tell a service that ran from one that was never called. + """ + items = self._items(f'functions/{MIXED_FUNCTION}', 'operations') + by_id = {item.get('id'): item for item in items} + operation_id = f'{PEER_CALIBRATION_APP}:calibrate' + self.assertIn( + operation_id, by_id, + f'the mixed Function does not offer its peer-owned operation: {sorted(by_id)}', + ) + self.assertNotEqual( + by_id[operation_id].get('x-medkit', {}).get('available'), False, + f'a peer-owned operation was reported unreachable while its gateway ' + f'answers: {by_id[operation_id]}', + ) + + response = requests.post( + f'{PRIMARY_URL}/functions/{MIXED_FUNCTION}/operations/' + f'{quote(operation_id, safe="")}/executions', + json={}, + timeout=15, + ) + self.assertEqual(response.status_code, 200, response.text) + parameters = response.json().get('parameters') + self.assertIsInstance( + parameters, dict, f'no service response came back: {response.text}') + self.assertIs( + parameters.get('success'), True, + f"the member's service did not report success: {parameters}", + ) + self.assertTrue( + parameters.get('message'), + f"the member's service answered with nothing to say: {parameters}", + ) + @launch_testing.post_shutdown_test() class TestShutdown(unittest.TestCase): diff --git a/src/ros2_medkit_integration_tests/test/features/test_grouping_entity_aggregation.test.py b/src/ros2_medkit_integration_tests/test/features/test_grouping_entity_aggregation.test.py index 371e2dca6..e5082ba80 100644 --- a/src/ros2_medkit_integration_tests/test/features/test_grouping_entity_aggregation.test.py +++ b/src/ros2_medkit_integration_tests/test/features/test_grouping_entity_aggregation.test.py @@ -158,6 +158,31 @@ # survives the merge - R1, the precondition for every addressing rule. COLLIDING_LEAF = 'shared_sensor' +# The id the merge gives the peer's copy of the colliding leaf. Written out +# rather than derived, because it is the string a client is handed and the +# whole point of the rename is that it is stable and addressable. +RENAMED_LEAF = f'secondary_gateway__{COLLIDING_LEAF}' + +# Each gateway backs its half of the colliding leaf with a node of its own, +# under a namespace of its own so the two copies keep distinct ROS paths - the +# same reason the two calibration nodes are placed apart. The node carries an +# operation and a topic, which are the two collections whose wire ids are built +# out of the leaf id, so the rename is exercised where it can go wrong. +SHARED_LEAF_NODE = 'shared_leaf' +PRIMARY_SHARED_NAMESPACE = '/powertrain/shared' +PEER_SHARED_NAMESPACE = '/chassis/shared' +PRIMARY_SHARED_SERVICE = f'{PRIMARY_SHARED_NAMESPACE}/calibrate' +PEER_SHARED_SERVICE = f'{PEER_SHARED_NAMESPACE}/calibrate' +PRIMARY_SHARED_TOPIC = f'{PRIMARY_SHARED_NAMESPACE}/reading' +PEER_SHARED_TOPIC = f'{PEER_SHARED_NAMESPACE}/reading' +SHARED_OPERATION = 'calibrate' + +# Members whose ids collide with nothing, one on each side. They put topics +# into the merged Function that no addressing rule has to disambiguate, which +# is what makes "an unambiguous id is left alone" checkable at all. +PRIMARY_RPM_APP = 'rpm_sensor' +PEER_ACTUATOR_APP = 'brake_actuator' + # Declared by the calibration demo node, so BOTH `primary_calibration` and # `peer_calibration` expose it under one name. A member-qualified id is the only # thing that separates the two copies, which is what makes this the parameter @@ -228,12 +253,18 @@ ros_binding: node_name: long_calibration namespace: /powertrain/engine - - id: {COLLIDING_LEAF} - name: "Shared Sensor (primary)" + - id: {PRIMARY_RPM_APP} + name: "Engine RPM Sensor" is_located_on: {PARENT_COMPONENT} ros_binding: node_name: rpm_sensor namespace: /powertrain/engine + - id: {COLLIDING_LEAF} + name: "Shared Leaf (primary)" + is_located_on: {PARENT_COMPONENT} + ros_binding: + node_name: {SHARED_LEAF_NODE} + namespace: {PRIMARY_SHARED_NAMESPACE} functions: - id: {MERGED_FUNCTION} name: "Vehicle Health Monitoring" @@ -242,6 +273,7 @@ - temp_sensor - primary_calibration - {PRIMARY_LONG_APP} + - {PRIMARY_RPM_APP} - {COLLIDING_LEAF} """ @@ -291,12 +323,18 @@ ros_binding: node_name: long_calibration namespace: {PEER_LONG_NAMESPACE} - - id: {COLLIDING_LEAF} - name: "Shared Sensor (peer)" + - id: {PEER_ACTUATOR_APP} + name: "Brake Actuator" is_located_on: {PEER_SUBCOMPONENT} ros_binding: node_name: actuator namespace: /chassis/brakes + - id: {COLLIDING_LEAF} + name: "Shared Leaf (peer)" + is_located_on: {PEER_SUBCOMPONENT} + ros_binding: + node_name: {SHARED_LEAF_NODE} + namespace: {PEER_SHARED_NAMESPACE} functions: - id: {MERGED_FUNCTION} name: "Vehicle Health Monitoring" @@ -305,6 +343,7 @@ - pressure_sensor - peer_calibration - {PEER_LONG_APP} + - {PEER_ACTUATOR_APP} - {COLLIDING_LEAF} """ @@ -372,6 +411,21 @@ def generate_test_description(): output='screen', additional_env=peer_domain_env, )] + + [launch_ros.actions.Node( + package='ros2_medkit_integration_tests', + executable='demo_shared_leaf', + name=SHARED_LEAF_NODE, + namespace=PRIMARY_SHARED_NAMESPACE, + output='screen', + )] + + [launch_ros.actions.Node( + package='ros2_medkit_integration_tests', + executable='demo_shared_leaf', + name=SHARED_LEAF_NODE, + namespace=PEER_SHARED_NAMESPACE, + output='screen', + additional_env=peer_domain_env, + )] + [ create_fault_manager_node(rosbag_enabled=False), create_fault_manager_node(rosbag_enabled=False, extra_env=peer_domain_env), @@ -404,10 +458,13 @@ def setUpClass(cls): # would fail every rule below for a reason unrelated to the rule. cls._wait_for_apps( PRIMARY_URL, - {'temp_sensor', 'primary_calibration', DUAL_APP, PRIMARY_LONG_APP}, + {'temp_sensor', 'primary_calibration', DUAL_APP, PRIMARY_LONG_APP, + COLLIDING_LEAF}, 'primary') cls._wait_for_apps( - PEER_URL, {'pressure_sensor', 'peer_calibration', PEER_LONG_APP}, 'peer') + PEER_URL, + {'pressure_sensor', 'peer_calibration', PEER_LONG_APP, COLLIDING_LEAF}, + 'peer') cls._wait_until_merged() @classmethod @@ -602,6 +659,131 @@ def test_a_leaf_contributed_by_both_gateways_stays_two_addressable_leaves(self): detail = requests.get(f'{PRIMARY_URL}/apps/{leaf_id}', timeout=10) self.assertEqual(detail.status_code, 200, f'{leaf_id} is listed but not addressable') + def test_a_renamed_leaf_is_named_by_the_items_it_owns(self): + """R1 carried into the ids that address items, which is what it is for. + + Both copies of the colliding leaf expose `calibrate`, so the merged + Function offers that short name twice and the member half is all that + separates the two. The peer calls its half `shared_sensor` because that + is what its own tree calls it, and here that name is the LOCAL leaf - + so an attribution passed through unchanged offers one id twice, and the + copy it resolves to is whichever the local walk holds. The peer's + operation is then not addressable through the aggregate at all. + """ + items = self._items(f'functions/{MERGED_FUNCTION}', 'operations') + by_path = {} + for item in items: + service = item.get('x-medkit', {}).get('ros2', {}).get('service') + if service in (PRIMARY_SHARED_SERVICE, PEER_SHARED_SERVICE): + by_path.setdefault(service, []).append(item) + + offered_ids = [item.get('id') for item in items] + for path in (PRIMARY_SHARED_SERVICE, PEER_SHARED_SERVICE): + self.assertEqual( + len(by_path.get(path, [])), 1, + f'{path} is not offered exactly once: {offered_ids}', + ) + + local_item = by_path[PRIMARY_SHARED_SERVICE][0] + peer_item = by_path[PEER_SHARED_SERVICE][0] + self.assertEqual( + local_item.get('x-medkit', {}).get('member_ids'), [COLLIDING_LEAF], + f'the local half names something other than the local leaf: {local_item}', + ) + self.assertEqual( + peer_item.get('x-medkit', {}).get('member_ids'), [RENAMED_LEAF], + f'the peer half is attributed to the local leaf: {peer_item}', + ) + self.assertEqual( + local_item.get('id'), f'{COLLIDING_LEAF}:{SHARED_OPERATION}', local_item) + self.assertEqual( + peer_item.get('id'), f'{RENAMED_LEAF}:{SHARED_OPERATION}', + f'the peer half is offered under the local leaf id: {peer_item}', + ) + + def test_a_renamed_leafs_operation_runs_the_copy_the_id_names(self): + """The list and the execution have to agree about WHICH copy an id names. + + The ids are taken from the collection rather than written out, because + the agreement is what is under test: an id nobody is offered proves + nothing. Both copies answer 200, so the status says only that something + ran - each node names itself in the response, and that is the only thing + on the wire separating the copy the id named from the copy the local + walk reaches first. + """ + offered = {} + for item in self._items(f'functions/{MERGED_FUNCTION}', 'operations'): + service = item.get('x-medkit', {}).get('ros2', {}).get('service') + if service in (PRIMARY_SHARED_SERVICE, PEER_SHARED_SERVICE): + offered[service] = item.get('id') + self.assertEqual( + sorted(offered), sorted([PRIMARY_SHARED_SERVICE, PEER_SHARED_SERVICE]), + f'the collection does not offer both copies: {offered}', + ) + + for service_path, operation_id in sorted(offered.items()): + with self.subTest(operation=operation_id): + response = self._run_operation( + f'functions/{MERGED_FUNCTION}', operation_id) + self.assertEqual(response.status_code, 200, response.text) + parameters = response.json().get('parameters') + self.assertIsInstance( + parameters, dict, f'no service response came back: {response.text}') + self.assertIs(parameters.get('success'), True, parameters) + node_fqn = f"{service_path.rsplit('/', 1)[0]}/{SHARED_LEAF_NODE}" + self.assertEqual( + parameters.get('message'), f'{node_fqn} calibrated', + f'{operation_id!r} ran the other copy: {parameters}', + ) + + def test_a_renamed_leafs_data_reads_back_through_the_id_the_list_offers(self): + """R4 for data, driven from what the collection offers rather than by hand. + + A merged App carries no topics of its own, so the attribution the peer + sends with an item is the ONLY account of who owns it - and the peer + names the leaf as its own tree does. A client that builds + `:` out of that reaches a member which does not publish + the topic, and the read comes back empty rather than refused. + """ + items = self._items(f'functions/{MERGED_FUNCTION}', 'data') + peer_items = [ + item for item in items + if item.get('x-medkit', {}).get('ros2', {}).get('topic') == PEER_SHARED_TOPIC + ] + self.assertEqual( + len(peer_items), 1, + f'the peer half of the colliding leaf offers no reading: ' + f'{[item.get("id") for item in items]}', + ) + members = peer_items[0].get('x-medkit', {}).get('member_ids') + self.assertEqual( + members, [RENAMED_LEAF], + f'the peer half is attributed to the local leaf: {peer_items[0]}', + ) + + item_id = f'{members[0]}:{PEER_SHARED_TOPIC.lstrip("/")}' + response = requests.get( + f'{PRIMARY_URL}/functions/{MERGED_FUNCTION}/data/{quote(item_id, safe="")}', + timeout=15, + ) + self.assertEqual(response.status_code, 200, response.text) + body = response.json() + self.assertEqual( + body.get('x-medkit', {}).get('status'), 'data', + f'the id the list offers read nothing: {body}', + ) + self.assertEqual( + body.get('x-medkit', {}).get('ros2', {}).get('topic'), PEER_SHARED_TOPIC, + f'the answer names a topic the member does not publish: {body}', + ) + self.assertTrue(body.get('data'), 'the peer member returned an empty payload') + # The member's own gateway answered, which is the only place that topic + # exists. Served here the answer would name the aggregating entity. + self.assertEqual( + body.get('x-medkit', {}).get('entity_id'), COLLIDING_LEAF, + f'the aggregating entity answered for a member it does not run: {body}', + ) + # ---------------------------------------------------------------------- R7 def test_a_component_both_sides_contribute_to_aggregates(self): @@ -1368,9 +1550,12 @@ def test_reading_an_id_that_names_two_operations_is_refused(self): (f'apps/{DUAL_APP}', 'calibrate', [DUAL_LEFT_ID, DUAL_RIGHT_ID]), (f'components/{PARENT_COMPONENT}', f'{DUAL_APP}:calibrate', [f'{DUAL_APP}:{DUAL_LEFT_ID}', f'{DUAL_APP}:{DUAL_RIGHT_ID}']), - # two members, one short name + # several members, one short name - including the two halves of the + # colliding leaf, whose member half is the id the merge gave them (f'functions/{MERGED_FUNCTION}', 'calibrate', - ['primary_calibration:calibrate', 'peer_calibration:calibrate']), + ['primary_calibration:calibrate', 'peer_calibration:calibrate', + f'{COLLIDING_LEAF}:{SHARED_OPERATION}', + f'{RENAMED_LEAF}:{SHARED_OPERATION}']), ) for entity_path, operation_id, expected in cases: with self.subTest(entity=entity_path, operation=operation_id): From bf7bf4c00b2e52b0aac48ff173ff6e933aaf710a Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Thu, 20 Aug 2026 17:27:59 +0200 Subject: [PATCH 20/22] fix(triggers): let a restored trigger keep resolving as long as its record lives Keeping a restored trigger out of the orphan sweep bought it nothing on its own. The subscription attempt still ran out after a minute measured from the moment it was queued, and at restore time that clock is certain to expire: restore runs inside the constructor, the subscription executor is wired later, so the attempt is always deferred and always stamped before the graph could answer. An entity appearing after that left a trigger reporting itself active with no subscription behind it - the state the sweep exemption exists to reach, made useless. Resolution now runs on the same rule the record does. Its budget starts when discovery first reports the entity, so while the entity has never been seen it retries for as long as the trigger is kept, and the wait is named in the log rather than passing in silence. A stored row with a topic and nothing to re-resolve from still subscribes directly, and now says when that fails. --- docs/api/rest.rst | 11 + .../core/managers/trigger_manager.hpp | 25 +- .../src/core/managers/trigger_manager.cpp | 112 ++++++--- .../test/test_trigger_manager_routing.cpp | 225 ++++++++++++++---- .../CMakeLists.txt | 10 +- ..._triggers_restore_before_discovery.test.py | 209 +++++++++++++++- 6 files changed, 496 insertions(+), 96 deletions(-) diff --git a/docs/api/rest.rst b/docs/api/rest.rst index 69a60c94b..175e7dd84 100644 --- a/docs/api/rest.rst +++ b/docs/api/rest.rst @@ -2535,6 +2535,17 @@ immediately after a restart nothing has been discovered yet, and an entity that has merely not been reported yet has not disappeared. A restored trigger whose entity never appears stays listed and can be deleted through the API. +A restored ``data`` trigger re-resolves its topic from the entity cache rather +than from the topic name it was stored with, and that attempt is governed by +the same rule as the record: while the entity has never been discovered the +attempt keeps running and never gives up, so a trigger whose entity takes +minutes to appear still subscribes and still fires. The gateway logs a warning +naming the trigger once the entity has been missing for longer than the +resolution budget, so an entity that never appears is visible rather than +silent. Once the entity has been discovered, the budget applies as usual and a +resource path that still cannot be resolved to a topic is given up on with a +warning naming the trigger. + Fault Triggers (threshold rules) -------------------------------- diff --git a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/managers/trigger_manager.hpp b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/managers/trigger_manager.hpp index 8107fe2e7..82218ab61 100644 --- a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/managers/trigger_manager.hpp +++ b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/managers/trigger_manager.hpp @@ -173,8 +173,10 @@ class TriggerManager { using WarnLogFn = std::function; void set_warn_log_fn(WarnLogFn fn); - /// How long deferred resolution keeps retrying before giving up. - /// Default 60 s; tests shrink it to exercise the expiry path. + /// How long deferred resolution keeps retrying after the trigger's entity has + /// been discovered before giving up, and how long it waits for an entity that + /// has never been discovered before saying so once. Default 60 s; tests + /// shrink it to exercise both paths. void set_unresolved_timeout(std::chrono::seconds timeout); /// Retry resolving data triggers whose topic names were unknown at creation. @@ -306,13 +308,28 @@ class TriggerManager { // worker thread (single-threaded dispatch), so no synchronization needed. bool evaluating_trigger_{false}; - // Deferred resolution for data triggers created before topic was discoverable. - // Periodically retried via retry_unresolved_triggers(). + // Deferred resolution for data triggers whose topic was not discoverable when + // the trigger entered this process - created through the API before the topic + // existed, or restored from the store. Periodically retried via + // retry_unresolved_triggers(). struct UnresolvedTrigger { std::string trigger_id; std::string entity_id; std::string resource_path; std::chrono::steady_clock::time_point created_at; + /// When this gateway first found the trigger's entity in discovery, empty + /// until then. The give-up budget runs from this instant, not from + /// created_at: an entity that has never been discovered has not failed to + /// resolve, it has simply not been reported yet, which is the same + /// distinction TriggerState::entity_seen draws for the orphan sweep. Tying + /// both to that one signal is what makes the subscription attempt live + /// exactly as long as the trigger record does. + std::optional entity_seen_at; + /// Whether the "entity still undiscovered" notice has been emitted for this + /// trigger. Retrying for an entity that has never been seen is correct and + /// unbounded, so it is reported once - an unbounded wait nobody is told + /// about is the same dead alarm as a silent give-up. + bool waiting_reported{false}; }; std::vector unresolved_data_triggers_; // guarded by triggers_mutex_ ResolveTopicFn resolve_topic_fn_; // guarded by triggers_mutex_ diff --git a/src/ros2_medkit_gateway/src/core/managers/trigger_manager.cpp b/src/ros2_medkit_gateway/src/core/managers/trigger_manager.cpp index 3422adb5b..d43a7b26b 100644 --- a/src/ros2_medkit_gateway/src/core/managers/trigger_manager.cpp +++ b/src/ros2_medkit_gateway/src/core/managers/trigger_manager.cpp @@ -151,32 +151,62 @@ void TriggerManager::retry_unresolved_triggers() { auto now = std::chrono::steady_clock::now(); std::vector resolved_indices; std::vector expired_indices; + std::vector stale_indices; for (size_t i = 0; i < unresolved_data_triggers_.size(); ++i) { auto & entry = unresolved_data_triggers_[i]; - if (now - entry.created_at > unresolved_timeout_) { - // Give up loudly: the trigger stays registered and ACTIVE, but with no - // subscription it will never fire. Silence here is a dead alarm the - // operator believes in. - expired_indices.push_back(i); - warnings.push_back("trigger '" + entry.trigger_id + "': gave up resolving resource_path '" + - entry.resource_path + "' to a topic for entity '" + entry.entity_id + "' after " + - std::to_string(unresolved_timeout_.count()) + - " s. The trigger stays active but cannot fire; delete it and recreate it once the topic " - "exists."); + auto trigger_it = triggers_.find(entry.trigger_id); + if (trigger_it == triggers_.end()) { + // The trigger was deleted or expired while its topic was still + // unresolved. Subscribing for it now would install a handle under a + // trigger id nothing owns, and nothing would ever release it. + stale_indices.push_back(i); continue; } + // The give-up budget starts when discovery first reports the entity. An + // entity this gateway has never seen is not a resolution failure, and the + // orphan sweep leaves the record alone for exactly the same reason - the + // attempt and the record therefore end together, never one before the + // other. + if (!entry.entity_seen_at.has_value() && trigger_it->second->entity_seen.load()) { + entry.entity_seen_at = now; + } + + if (entry.entity_seen_at.has_value()) { + if (now - *entry.entity_seen_at > unresolved_timeout_) { + // Give up loudly: the trigger stays registered and ACTIVE, but with no + // subscription it will never fire. Silence here is a dead alarm the + // operator believes in. + expired_indices.push_back(i); + warnings.push_back("trigger '" + entry.trigger_id + "': gave up resolving resource_path '" + + entry.resource_path + "' to a topic for entity '" + entry.entity_id + "' after " + + std::to_string(unresolved_timeout_.count()) + + " s of the entity being discovered. The trigger stays active but cannot fire; delete it " + "and recreate it once the topic exists."); + continue; + } + } else if (!entry.waiting_reported && now - entry.created_at > unresolved_timeout_) { + // Waiting on an undiscovered entity is unbounded on purpose, so it is + // announced once. Without this the trigger is indistinguishable from + // one that resolved. + entry.waiting_reported = true; + warnings.push_back("trigger '" + entry.trigger_id + "': entity '" + entry.entity_id + + "' has not appeared in discovery after " + std::to_string(unresolved_timeout_.count()) + + " s, so resource_path '" + entry.resource_path + + "' cannot be resolved to a topic. The trigger stays active and keeps retrying, and it " + "cannot fire until that entity is discovered."); + } + std::string topic_name = resolve_topic_fn_(entry.entity_id, entry.resource_path); if (!topic_name.empty()) { // Update the trigger's resolved_topic_name + register a transport // handle whose lifetime is tied to the trigger entry (cleared on // remove()/cleanup_expired_trigger()). - auto it = triggers_.find(entry.trigger_id); - if (it != triggers_.end()) { - std::lock_guard state_lock(it->second->mtx); - it->second->info.resolved_topic_name = topic_name; + { + std::lock_guard state_lock(trigger_it->second->mtx); + trigger_it->second->info.resolved_topic_name = topic_name; } const std::string trigger_id = entry.trigger_id; const std::string entity_id = entry.entity_id; @@ -199,10 +229,11 @@ void TriggerManager::retry_unresolved_triggers() { } } - // Remove resolved and expired entries (reverse order to maintain indices) + // Remove resolved, expired and stale entries (reverse order to maintain indices) std::vector to_remove; to_remove.insert(to_remove.end(), resolved_indices.begin(), resolved_indices.end()); to_remove.insert(to_remove.end(), expired_indices.begin(), expired_indices.end()); + to_remove.insert(to_remove.end(), stale_indices.begin(), stale_indices.end()); std::sort(to_remove.rbegin(), to_remove.rend()); for (size_t idx : to_remove) { unresolved_data_triggers_.erase(unresolved_data_triggers_.begin() + static_cast(idx)); @@ -420,8 +451,11 @@ tl::expected TriggerManager::create(const Trigg } } else if (!req.resource_path.empty()) { std::lock_guard lock(triggers_mutex_); + // The entity was validated to exist before this request was accepted, so + // it counts as seen and the give-up budget runs from now. + const auto queued_at = std::chrono::steady_clock::now(); unresolved_data_triggers_.push_back( - {info_copy.id, req.entity_id, req.resource_path, std::chrono::steady_clock::now()}); + {info_copy.id, req.entity_id, req.resource_path, queued_at, queued_at, /*waiting_reported=*/false}); } } @@ -767,24 +801,40 @@ size_t TriggerManager::load_persistent_triggers() { add_to_dispatch_index(state->info.id, state->info.collection, state->info.entity_id); - // Re-subscribe to topic for restored data triggers via the transport. - if (topic_transport_ && state->info.collection == "data" && !state->info.resolved_topic_name.empty()) { + // Restored data triggers go through deferred resolution rather than + // straight onto the persisted topic name. The entity belongs to a process + // this gateway has not met yet, so the subscription attempt has to stay + // open for as long as the record does, and deferred resolution is the + // only path whose budget is tied to entity_seen. Subscribing here could + // not honour that in any case: restore runs inside the constructor, the + // transport's executor is wired after it returns, so the attempt would be + // parked on a clock that starts now and runs out long before a late + // entity arrives. + if (topic_transport_ && state->info.collection == "data") { const std::string trigger_id = state->info.id; const std::string entity_id = state->info.entity_id; const std::string resource_path = state->info.resource_path; - auto handle = topic_transport_->subscribe(state->info.resolved_topic_name, /*msg_type=*/"", - [this, entity_id, resource_path](const nlohmann::json & sample) { - notifier_.notify("data", entity_id, resource_path, sample, - ChangeType::UPDATED); - }); - if (handle) { - topic_handles_[trigger_id] = std::move(handle); - } else { - // Subscribe failed during persistent-trigger restore (e.g. the - // topic disappeared between shutdown and restart, or rclcpp threw - // inside TriggerTopicSubscriber). Queue the trigger for retry on - // the next refresh tick instead of leaving it active-but-silent. - unresolved_data_triggers_.push_back({trigger_id, entity_id, resource_path, std::chrono::steady_clock::now()}); + if (!resource_path.empty()) { + unresolved_data_triggers_.push_back({trigger_id, entity_id, resource_path, std::chrono::steady_clock::now(), + std::nullopt, + /*waiting_reported=*/false}); + } else if (!state->info.resolved_topic_name.empty()) { + // A stored row carrying a topic but no resource_path has nothing for + // the entity cache to re-resolve, so the persisted topic is all there + // is to go on. + auto handle = topic_transport_->subscribe(state->info.resolved_topic_name, /*msg_type=*/"", + [this, entity_id, resource_path](const nlohmann::json & sample) { + notifier_.notify("data", entity_id, resource_path, sample, + ChangeType::UPDATED); + }); + if (handle) { + topic_handles_[trigger_id] = std::move(handle); + } else { + warnings.push_back("persistent trigger restore: trigger '" + trigger_id + "' could not subscribe to '" + + state->info.resolved_topic_name + + "' and carries no resource_path to resolve again from. It stays active but cannot " + "fire; delete it and recreate it."); + } } } diff --git a/src/ros2_medkit_gateway/test/test_trigger_manager_routing.cpp b/src/ros2_medkit_gateway/test/test_trigger_manager_routing.cpp index 5035881f3..cc89a6095 100644 --- a/src/ros2_medkit_gateway/test/test_trigger_manager_routing.cpp +++ b/src/ros2_medkit_gateway/test/test_trigger_manager_routing.cpp @@ -223,6 +223,37 @@ class TriggerManagerRoutingTest : public ::testing::Test { return req; } + /// Write an ACTIVE persistent data trigger straight into the store, so a + /// manager built with on_restart_behavior=restore has something to put back. + void persist_data_trigger(const std::string & id, const std::string & entity_id, const std::string & resource_path, + const std::string & resolved_topic) { + TriggerInfo persisted; + persisted.id = id; + persisted.entity_id = entity_id; + persisted.entity_type = "apps"; + persisted.resource_uri = "/api/v1/apps/" + entity_id + "/data" + resource_path; + persisted.collection = "data"; + persisted.resource_path = resource_path; + persisted.resolved_topic_name = resolved_topic; + persisted.path = ""; + persisted.condition_type = "OnChange"; + persisted.condition_params = nlohmann::json::object(); + persisted.protocol = "sse"; + persisted.multishot = true; + persisted.persistent = true; + persisted.status = TriggerStatus::ACTIVE; + persisted.created_at = std::chrono::system_clock::now(); + ASSERT_TRUE(store_.save(persisted).has_value()); + } + + /// Build a second manager over the same store, configured to restore. + std::unique_ptr make_restoring_manager() { + TriggerConfig restore_config; + restore_config.max_triggers = 50; + restore_config.on_restart_behavior = "restore"; + return std::make_unique(notifier_, registry_, store_, restore_config, transport_); + } + ResourceChangeNotifier notifier_; ConditionRegistry registry_; SqliteTriggerStore store_{":memory:"}; @@ -332,70 +363,168 @@ TEST_F(TriggerManagerRoutingTest, CreateReturnsErrorWhenSubscribeReturnsNullHand EXPECT_EQ(transport_->alive_count(), 0u); } -TEST_F(TriggerManagerRoutingTest, RestoreQueuesPersistentTriggerOnSubscribeFailure) { - // Pre-populate the SQLite store with a persistent data trigger so that the - // fresh manager's load_persistent_triggers() call has something to restore. - TriggerInfo persisted; - persisted.id = "trig_restore_fail_1"; - persisted.entity_id = "sensor_a"; - persisted.entity_type = "apps"; - persisted.resource_uri = "/api/v1/apps/sensor_a/data/temperature"; - persisted.collection = "data"; - persisted.resource_path = "/temperature"; - persisted.resolved_topic_name = "/sensor_a/temperature"; - persisted.path = ""; - persisted.condition_type = "OnChange"; - persisted.condition_params = nlohmann::json::object(); - persisted.protocol = "sse"; - persisted.multishot = true; - persisted.persistent = true; - persisted.status = TriggerStatus::ACTIVE; - persisted.created_at = std::chrono::system_clock::now(); - ASSERT_TRUE(store_.save(persisted).has_value()); - - // Force the next subscribe() call (made by load_persistent_triggers() for - // the restored data trigger) to return a null handle, simulating the - // "topic disappeared between shutdown and restart" race that the production - // code path guards against by queuing the trigger onto - // unresolved_data_triggers_ for retry. - transport_->fail_next_subscribe(); +TEST_F(TriggerManagerRoutingTest, RestoreDefersSubscriptionInsteadOfSubscribingDirectly) { + persist_data_trigger("trig_restore_defer_1", "sensor_a", "/temperature", "/sensor_a/temperature"); - // Build a fresh manager configured for restore. The default fixture manager - // uses on_restart_behavior=reset, which makes load_persistent_triggers() a - // no-op - we need restore semantics to exercise the failure path. - TriggerConfig restore_config; - restore_config.max_triggers = 50; - restore_config.on_restart_behavior = "restore"; - auto restored_mgr = std::make_unique(notifier_, registry_, store_, restore_config, transport_); + auto restored_mgr = make_restoring_manager(); restored_mgr->load_persistent_triggers(); - // The trigger is visible in the manager despite the subscribe failure - - // load_persistent_triggers() must not silently drop persistent triggers. + // The trigger is visible, and nothing has been subscribed yet: restore hands + // the trigger to deferred resolution, whose budget is tied to the entity + // being discovered, rather than parking an attempt on a clock of its own. auto triggers = restored_mgr->list("sensor_a"); ASSERT_EQ(triggers.size(), 1u); - EXPECT_EQ(triggers[0].id, "trig_restore_fail_1"); + EXPECT_EQ(triggers[0].id, "trig_restore_defer_1"); + EXPECT_EQ(transport_->total_subscribes(), 0u) << "restore must not subscribe straight from the persisted topic"; - // The first failed subscribe is recorded. No alive handle yet. - const auto subscribes_after_fail = transport_->total_subscribes(); - EXPECT_GE(subscribes_after_fail, 1u); - EXPECT_EQ(transport_->alive_count(), 0u); - - // Now exercise the retry path. retry_unresolved_triggers() requires both a - // resolve_topic_fn_ and a non-empty resolved topic; provide a resolver that - // returns the persisted topic name so the manager can re-subscribe. + // Deferred resolution then does the subscribing, from the entity cache. restored_mgr->set_resolve_topic_fn([](const std::string & entity_id, const std::string & resource_path) { return "/" + entity_id + resource_path; }); restored_mgr->retry_unresolved_triggers(); - // The retry must have called subscribe() again. This time it succeeds and - // an alive handle backs the restored trigger. - EXPECT_GT(transport_->total_subscribes(), subscribes_after_fail); + EXPECT_EQ(transport_->total_subscribes(), 1u); + EXPECT_EQ(transport_->alive_count(), 1u); + EXPECT_EQ(transport_->last_subscribed_topic(), "/sensor_a/temperature"); + + restored_mgr->shutdown(); +} + +TEST_F(TriggerManagerRoutingTest, RestoreSubscribesDirectlyWhenThereIsNothingToResolveFrom) { + // A stored row with a topic but no resource_path gives the entity cache + // nothing to match on, so the persisted topic is the only thing left. It is + // still better than restoring the record with no subscription at all. + persist_data_trigger("trig_restore_no_path_1", "sensor_a", /*resource_path=*/"", "/sensor_a/temperature"); + + auto restored_mgr = make_restoring_manager(); + restored_mgr->load_persistent_triggers(); + + EXPECT_EQ(restored_mgr->list("sensor_a").size(), 1u); + EXPECT_EQ(transport_->total_subscribes(), 1u); + EXPECT_EQ(transport_->alive_count(), 1u); + EXPECT_EQ(transport_->last_subscribed_topic(), "/sensor_a/temperature"); + + restored_mgr->shutdown(); +} + +TEST_F(TriggerManagerRoutingTest, RestoredTriggerKeepsResolvingPastTheBudgetWhileItsEntityIsUndiscovered) { + persist_data_trigger("trig_restore_wait_1", "sensor_a", "/temperature", "/sensor_a/temperature"); + + auto restored_mgr = make_restoring_manager(); + std::vector warnings; + restored_mgr->set_warn_log_fn([&warnings](const std::string & m) { + warnings.push_back(m); + }); + restored_mgr->load_persistent_triggers(); + + // Nothing resolves while the entity is undiscovered, and the budget is spent + // several times over. + bool entity_known = false; + restored_mgr->set_resolve_topic_fn([&entity_known](const std::string & entity_id, const std::string & resource_path) { + return entity_known ? "/" + entity_id + resource_path : std::string{}; + }); + restored_mgr->set_unresolved_timeout(std::chrono::seconds(0)); + restored_mgr->retry_unresolved_triggers(); + restored_mgr->retry_unresolved_triggers(); + restored_mgr->retry_unresolved_triggers(); + + // An unbounded wait is announced exactly once, naming the trigger and the + // entity it is waiting for. + ASSERT_EQ(warnings.size(), 1u); + EXPECT_NE(warnings[0].find("trig_restore_wait_1"), std::string::npos) << warnings[0]; + EXPECT_NE(warnings[0].find("sensor_a"), std::string::npos) << warnings[0]; + EXPECT_NE(warnings[0].find("has not appeared in discovery"), std::string::npos) << warnings[0]; + + // The entity finally shows up, long past the budget. The attempt is still + // there to take it - that is the whole point of holding it open. + entity_known = true; + restored_mgr->retry_unresolved_triggers(); + + EXPECT_EQ(transport_->total_subscribes(), 1u) << "a trigger whose entity arrives late must still subscribe"; EXPECT_EQ(transport_->alive_count(), 1u); restored_mgr->shutdown(); } +TEST_F(TriggerManagerRoutingTest, RestoredTriggerGivesUpLoudlyOnceItsEntityHasBeenSeen) { + persist_data_trigger("trig_restore_giveup_1", "sensor_a", "/temperature", "/sensor_a/temperature"); + + auto restored_mgr = make_restoring_manager(); + std::vector warnings; + restored_mgr->set_warn_log_fn([&warnings](const std::string & m) { + warnings.push_back(m); + }); + restored_mgr->load_persistent_triggers(); + restored_mgr->set_resolve_topic_fn([](const std::string &, const std::string &) { + return std::string{}; + }); + restored_mgr->set_unresolved_timeout(std::chrono::seconds(0)); + + // Discovery reports the entity. From here the trigger is sweepable and its + // resolution budget runs - the two start together. + restored_mgr->set_entity_exists_fn([](const std::string &, const std::string &) { + return true; + }); + restored_mgr->sweep_orphaned_triggers(); + + // The tick that observes the entity starts the budget, it does not spend it. + restored_mgr->retry_unresolved_triggers(); + EXPECT_TRUE(warnings.empty()) << "the budget must start when the entity is seen, not end there"; + + restored_mgr->retry_unresolved_triggers(); + ASSERT_EQ(warnings.size(), 1u); + EXPECT_NE(warnings[0].find("trig_restore_giveup_1"), std::string::npos) << warnings[0]; + EXPECT_NE(warnings[0].find("gave up resolving"), std::string::npos) << warnings[0]; + + // Given up means stopped: later ticks are silent and nothing subscribes. + restored_mgr->retry_unresolved_triggers(); + EXPECT_EQ(warnings.size(), 1u); + EXPECT_EQ(transport_->total_subscribes(), 0u); + + restored_mgr->shutdown(); +} + +TEST_F(TriggerManagerRoutingTest, ApiCreatedTriggerBudgetRunsFromCreationNotFromTheFirstTick) { + // The entity behind an accepted request was validated to exist, so nothing + // about this trigger is waiting on discovery: its budget is already running + // by the time the first retry tick arrives, and that tick can spend it. + // Starting the budget at the first tick instead would silently hand every + // trigger a whole extra tick. + std::vector warnings; + manager_->set_warn_log_fn([&warnings](const std::string & m) { + warnings.push_back(m); + }); + manager_->set_resolve_topic_fn([](const std::string &, const std::string &) { + return std::string{}; + }); + manager_->set_unresolved_timeout(std::chrono::seconds(0)); + + auto created = manager_->create(make_data_request("plc_app", /*resolved_topic=*/"", "/counter")); + ASSERT_TRUE(created.has_value()) << created.error().message; + + manager_->retry_unresolved_triggers(); + + ASSERT_EQ(warnings.size(), 1u) << "the first tick after creation must find the budget already spent"; + EXPECT_NE(warnings[0].find("gave up resolving"), std::string::npos) << warnings[0]; + EXPECT_NE(warnings[0].find(created->id), std::string::npos) << warnings[0]; +} + +TEST_F(TriggerManagerRoutingTest, DeferredResolutionDropsEntriesWhoseTriggerIsGone) { + // A deferred entry outliving its trigger would subscribe under a trigger id + // nothing owns, so nothing would ever release the handle. + auto created = manager_->create(make_data_request("plc_app", /*resolved_topic=*/"", "/counter")); + ASSERT_TRUE(created.has_value()) << created.error().message; + ASSERT_TRUE(manager_->remove(created->id)); + + manager_->set_resolve_topic_fn([](const std::string & entity_id, const std::string & resource_path) { + return "/" + entity_id + resource_path; + }); + manager_->retry_unresolved_triggers(); + + EXPECT_EQ(transport_->total_subscribes(), 0u) << "a removed trigger must not acquire a subscription"; + EXPECT_EQ(transport_->alive_count(), 0u); +} + TEST_F(TriggerManagerRoutingTest, UnresolvedTriggerExpiryWarnsAndStopsRetrying) { // #584: an unresolved data trigger used to be dropped from the retry list // after the timeout with no trace, leaving a permanently ACTIVE trigger diff --git a/src/ros2_medkit_integration_tests/CMakeLists.txt b/src/ros2_medkit_integration_tests/CMakeLists.txt index d6557f8a0..a4188d658 100644 --- a/src/ros2_medkit_integration_tests/CMakeLists.txt +++ b/src/ros2_medkit_integration_tests/CMakeLists.txt @@ -279,12 +279,20 @@ if(BUILD_TESTING) # greenwave_monitor node through the same 10s launch-ordering delay plus # warmup as test_graph_provider_greenwave above, and their staleness/SSE # polling budgets are similarly generous - both widened to match. + # + # test_triggers_restore_before_discovery keeps its entity provably absent for + # longer than the 60s budget a trigger subscription attempt gets, because a + # shorter window is one a gateway that abandons the attempt would also pass. + # That window plus two gateway startups and the SSE wait after it exceed the + # 120s the feature glob assigns, so it gets its own budget rather than have + # the file killed with no test name in the output. set(_MEDKIT_TEST_TIMEOUT_OVERRIDES test_rosbag_boundary_download 180 test_grouping_entity_aggregation 300 test_graph_provider_greenwave 300 test_graph_provider_stale 300 - test_graph_provider_sse 300) + test_graph_provider_sse 300 + test_triggers_restore_before_discovery 300) # Names actually matched against a discovered test_name in the two loops # below. Checked against _MEDKIT_TEST_TIMEOUT_OVERRIDES itself after both # loops finish - see the FATAL_ERROR check at the end of this block - so a diff --git a/src/ros2_medkit_integration_tests/test/features/test_triggers_restore_before_discovery.test.py b/src/ros2_medkit_integration_tests/test/features/test_triggers_restore_before_discovery.test.py index 4135439cb..73d43e49f 100644 --- a/src/ros2_medkit_integration_tests/test/features/test_triggers_restore_before_discovery.test.py +++ b/src/ros2_medkit_integration_tests/test/features/test_triggers_restore_before_discovery.test.py @@ -23,6 +23,15 @@ apart deletes the trigger from the shared store, and because restore never runs again the trigger is unrecoverable for the life of the process. +Surviving the sweep is only half of it. A restored data trigger also has to end +up subscribed to its topic, and it is the subscription - not the record - that +decides whether the trigger can ever fire. A trigger that keeps its record but +loses its subscription reports itself active and stays silent, which is worse +than one that was deleted, because nothing says so. So the same window that +proves the record survives is used to prove the subscription attempt does: it +is longer than any budget the trigger subsystem gives a single attempt, and at +the end of it the trigger is required to deliver a real event. + Nothing here is raced or timed. The restarted gateway lives on a DDS domain of its own, the only node that can put its entity on that domain is started by the test rather than by a clock, and the sweep cadence is pinned to the fastest the @@ -30,8 +39,10 @@ long as the test says, whatever the machine is doing. """ +import json import os import tempfile +import threading import time import unittest @@ -69,9 +80,12 @@ SWEEP_INTERVAL_MS = 100 # How long the restarted gateway is left running with its entity provably -# absent. Two orders of magnitude above the sweep cadence, so the restored -# trigger is offered to the sweep many times over. -SWEEP_WINDOW_SECONDS = 3.0 +# absent. Two things set this length. It is hundreds of sweep cadences, so the +# restored trigger is offered to the sweep many times over. And it outlasts the +# 60 s budget the trigger subsystem gives a single subscription attempt, which +# is what makes the firing check below discriminating: a gateway that abandons +# the attempt on that budget survives any shorter window unnoticed. +ABSENCE_WINDOW_SECONDS = 75.0 DB_PATH = os.path.join( tempfile.gettempdir(), @@ -93,6 +107,12 @@ APP_ID = 'temp_sensor' RESOURCE_URI = f'/api/v1/apps/{APP_ID}/faults' +# The topic the temp_sensor demo node publishes, and the data resource that +# maps onto it. A data trigger is the one that needs a live topic subscription +# behind it, so it is the one that can be silently dead. +DATA_TOPIC = '/powertrain/engine/temperature' +DATA_RESOURCE_URI = f'/api/v1/apps/{APP_ID}/data{DATA_TOPIC}' + def _gate_process(name, path): """Return a process that exits once ``path`` exists.""" @@ -207,6 +227,63 @@ def _wait_for_app(base_url, app_id, *, timeout=60.0): ) +def _wait_for_data_items(base_url, app_id, *, timeout=60.0): + """Poll /apps/{id}/data until the app's topics are in the entity cache. + + The data trigger created below is meant to be a fully resolved one, the + same shape an operator's is: that needs the app's topics discovered, which + happens a refresh cycle after the app itself. + """ + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + r = requests.get(f'{base_url}/apps/{app_id}/data', timeout=5) + if r.status_code == 200 and r.json().get('items'): + return + except requests.exceptions.RequestException: + pass + time.sleep(0.5) + raise AssertionError( + f'App {app_id!r} exposed no data items at {base_url} after {timeout}s' + ) + + +def _collect_trigger_events(events_url, wanted, *, timeout): + """Read up to ``wanted`` SSE events from a trigger stream. + + The read runs on its own daemon thread and the caller's wait is bounded by + a wall clock. A trigger with no subscription behind it holds its stream + open and puts nothing on it, and a blocking read there hands ctest a killed + file with no test name in it instead of a named assertion failure. + """ + received = [] + done = threading.Event() + + def collect(): + try: + with requests.get(events_url, stream=True, timeout=timeout) as resp: + if resp.status_code != 200: + return + for line in resp.iter_lines(decode_unicode=True): + if done.is_set(): + return + if line and line.startswith('data: '): + received.append(json.loads(line[6:])) + if len(received) >= wanted: + return + except requests.exceptions.RequestException: + pass + finally: + done.set() + + thread = threading.Thread(target=collect, daemon=True) + thread.start() + done.wait(timeout=timeout) + done.set() + thread.join(timeout=5) + return list(received) + + class TestTriggersRestoreBeforeDiscovery(GatewayTestCase): """A restored trigger survives until its entity is discovered.""" @@ -217,19 +294,20 @@ class TestTriggersRestoreBeforeDiscovery(GatewayTestCase): REQUIRED_AREAS: set = set() _trigger_id: str = '' + _data_trigger_id: str = '' @classmethod def setUpClass(cls): """Wait for the primary gateway and its demo node.""" _wait_for_health(BASE_URL_PRIMARY, timeout=60.0) _wait_for_app(BASE_URL_PRIMARY, APP_ID, timeout=60.0) + _wait_for_data_items(BASE_URL_PRIMARY, APP_ID, timeout=60.0) cls.addClassCleanup(_remove_gates) - # @verifies REQ_INTEROP_029 - def test_01_create_persistent_trigger(self): - """POST a persistent trigger on the primary gateway, then restart.""" + def _create_persistent_trigger(self, resource_uri): + """POST a persistent multishot OnChange trigger and return its body.""" body = { - 'resource': RESOURCE_URI, + 'resource': resource_uri, 'trigger_condition': {'condition_type': 'OnChange'}, 'multishot': True, 'persistent': True, @@ -240,13 +318,29 @@ def test_01_create_persistent_trigger(self): json=body, timeout=5, ) - self.assertEqual(r.status_code, 201, f'Create failed: {r.text}') + self.assertEqual( + r.status_code, 201, f'Create failed for {resource_uri}: {r.text}', + ) trig = r.json() self.assertTrue(trig.get('persistent'), 'trigger must be persistent') - TestTriggersRestoreBeforeDiscovery._trigger_id = trig['id'] + return trig + + # @verifies REQ_INTEROP_029 + def test_01_create_persistent_triggers(self): + """POST both persistent triggers on the primary gateway, then restart. - # The row is in the shared store, which is the precondition the restore - # path needs. Opening this gate starts the restarted gateway. + The faults trigger carries no subscription, so it only exercises the + record. The data trigger needs a live topic subscription behind it, + which is what the later window puts under strain. + """ + cls = TestTriggersRestoreBeforeDiscovery + cls._trigger_id = self._create_persistent_trigger(RESOURCE_URI)['id'] + cls._data_trigger_id = self._create_persistent_trigger( + DATA_RESOURCE_URI, + )['id'] + + # Both rows are in the shared store, which is the precondition the + # restore path needs. Opening this gate starts the restarted gateway. _open_gate(GATE_RESTART) # @verifies REQ_INTEROP_096 @@ -276,7 +370,17 @@ def test_02_restored_trigger_survives_until_entity_is_discovered(self): f'got {r.status_code}', ) - time.sleep(SWEEP_WINDOW_SECONDS) + time.sleep(ABSENCE_WINDOW_SECONDS) + + # Still absent after the window. If this fails the window was not an + # absence window at all and neither this case nor test_03 means + # anything, so it is checked before the gate is opened. + r = requests.get(f'{BASE_URL_RESTARTED}/apps/{APP_ID}', timeout=5) + self.assertEqual( + r.status_code, 404, + f'{APP_ID!r} appeared on the restarted gateway during the window ' + f'that is supposed to prove its absence, got {r.status_code}', + ) _open_gate(GATE_ENTITY) _wait_for_app(BASE_URL_RESTARTED, APP_ID, timeout=60.0) @@ -296,6 +400,87 @@ def test_02_restored_trigger_survives_until_entity_is_discovered(self): self.assertTrue(trig.get('persistent')) self.assertEqual(trig.get('observed_resource'), RESOURCE_URI) + # @verifies REQ_INTEROP_097 + def test_03_restored_data_trigger_fires_after_late_discovery(self): + """The restored data trigger delivers events once its entity arrives. + + Its entity turned up later than the budget a single subscription + attempt gets, so this is the case in which a trigger that keeps only + its record goes quiet. Being listed is not evidence here - the trigger + has to put a real sample on its event stream. + """ + self.assertTrue( + self._data_trigger_id, + 'test_01 must set _data_trigger_id before test_03 runs', + ) + # test_02 opened the entity gate; the node needs to be discovered + # before its topic can be subscribed. + _wait_for_app(BASE_URL_RESTARTED, APP_ID, timeout=60.0) + + url = ( + f'{BASE_URL_RESTARTED}/apps/{APP_ID}/triggers/' + f'{self._data_trigger_id}' + ) + r = requests.get(url, timeout=5) + self.assertEqual( + r.status_code, 200, + f'Restored data trigger {self._data_trigger_id!r} is not there: ' + f'{r.status_code}: {r.text}', + ) + trig = r.json() + self.assertEqual(trig['status'], 'active') + self.assertEqual(trig.get('observed_resource'), DATA_RESOURCE_URI) + + events_url = ( + f'{BASE_URL_RESTARTED}' + f'{trig["event_source"].removeprefix(API_BASE_PATH)}' + ) + # The demo node publishes at 2 Hz and deferred resolution runs on a 5 s + # tick, so this covers several ticks over: a failure here means "never + # fired", not "not yet". + events = _collect_trigger_events(events_url, 1, timeout=45) + + self.assertGreaterEqual( + len(events), 1, + f'Restored data trigger {self._data_trigger_id!r} reports itself ' + f'active but never delivered an event after its entity appeared - ' + f'its subscription did not outlive the wait for that entity', + ) + for event in events: + self.assertIn('timestamp', event) + self.assertIn('payload', event) + + def test_04_waiting_for_an_undiscovered_entity_is_reported(self, proc_output, restarted): + """The gateway says which trigger is still waiting for its entity. + + Holding the attempt open indefinitely is right, and invisible only if + nobody is told. The notice is what keeps an entity that never appears + from looking like a trigger that resolved. + """ + self.assertTrue( + self._data_trigger_id, + 'test_01 must set _data_trigger_id before test_04 runs', + ) + # Concatenated with no separator: proc_output yields raw stream chunks, + # and joining with a newline splices one into the middle of a log line. + text = ''.join( + output.text.decode(errors='replace') + for output in proc_output[restarted] + ) + notices = [ + line for line in text.splitlines() + if 'has not appeared in discovery' in line + ] + self.assertTrue( + notices, + 'The restarted gateway spent the whole absence window unable to ' + 'resolve a restored data trigger and never said so', + ) + self.assertTrue( + any(self._data_trigger_id in line for line in notices), + f'The notice does not name the trigger that is waiting: {notices}', + ) + @launch_testing.post_shutdown_test() class TestShutdown(unittest.TestCase): From 15a36169296905a4d319ae5df324839a8c5a7abb Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Thu, 20 Aug 2026 19:54:27 +0200 Subject: [PATCH 21/22] fix(data): let a bare topic id find the gateway that owns it An operation keeps its bare id and is still routable, because the entity's operations carry the member that owns each ROS path. Data had no such map: a peer never described its topics, so a merged peer App held none. A topic only a peer publishes was therefore listed bare - correctly, one member provides it - and reading that id found nothing on this graph and answered as though the topic were gone, while its member and gateway were both healthy. A peer's apps now describe their topics, which puts them in the same field a local app uses and so in the same owner map both the listing and the read already consult. A bare id resolves its owner there and is served on that member's gateway; a locally-owned topic is untouched, and qualification still follows ambiguity rather than aggregation, so the id itself does not change. Ownership is by declaration rather than by counting: one topic can be published by one member and subscribed by another, so a sole-owner rule would have left the case this fixes broken. The listing holds a peer copy back and emits it only when the fan-out did not carry that path, keyed on the path itself, because otherwise one topic appears twice and neither copy keeps the bare id. --- docs/api/rest.rst | 22 ++ src/ros2_medkit_gateway/README.md | 22 ++ .../design/aggregation.rst | 55 ++- .../include/ros2_medkit_gateway/dto/data.hpp | 23 +- .../http/handlers/handler_support.hpp | 35 ++ .../src/core/aggregation/peer_client.cpp | 90 ++++- .../src/http/handlers/data_handlers.cpp | 181 ++++++++- .../src/http/handlers/operation_handlers.cpp | 15 +- .../test/test_peer_client.cpp | 144 +++++++ .../test_grouping_entity_aggregation.test.py | 359 ++++++++++++++++++ 10 files changed, 883 insertions(+), 63 deletions(-) diff --git a/docs/api/rest.rst b/docs/api/rest.rst index 175e7dd84..cfb4c879e 100644 --- a/docs/api/rest.rst +++ b/docs/api/rest.rst @@ -746,6 +746,28 @@ before. This applies to ``GET`` and ``PUT`` of a single ``/data`` item, to ``POST`` of an ``/operations`` execution, and to ``GET``, ``PUT`` and ``DELETE`` of a single ``/configurations`` item. +**An id needs no member half to reach its owner.** A ``/data`` item one member +provides keeps its bare id - qualification follows ambiguity, not aggregation - +so the bare topic path is the id the collection hands back, and it is dispatched +by the member the tree records as providing that topic: + +.. code-block:: text + + GET /api/v1/functions/vehicle_health/data/chassis%2Fbrakes%2Fpressure + +is answered, when every member providing ``/chassis/brakes/pressure`` belongs to +one peer, by + +.. code-block:: text + + GET /api/v1/apps/pressure_sensor/data/chassis/brakes/pressure + +on that peer. A topic a member this gateway runs provides is sampled here, +unchanged; a topic whose providers are spread across gateways names no single +place, and the local graph answers it as before. ``/operations`` resolves a bare +id the same way - the operation is resolved first, and the member owning its ROS +path is where the execution is sent. + ``/configurations`` keeps its own id scheme, ``:``, and the member half is the app id. Because nothing on the owning gateway is aggregating, the parameter is addressed there by its bare name: diff --git a/src/ros2_medkit_gateway/README.md b/src/ros2_medkit_gateway/README.md index 43fe167cb..cc3201d10 100644 --- a/src/ros2_medkit_gateway/README.md +++ b/src/ros2_medkit_gateway/README.md @@ -356,6 +356,28 @@ member is served here as before. This covers `GET` and `PUT` of one `/data` item `POST` of an `/operations` execution, and `GET`, `PUT` and `DELETE` of one `/configurations` item. +An id needs no member half to reach its owner. A `/data` item one member provides +keeps its bare id - qualification follows ambiguity, not aggregation - so the bare +topic path is the id the collection hands back, and it is dispatched by the member +the tree records as providing that topic: + +``` +GET /api/v1/functions/vehicle_health/data/chassis%2Fbrakes%2Fpressure +``` + +becomes, when every member providing `/chassis/brakes/pressure` belongs to one +peer, + +``` +GET /api/v1/apps/pressure_sensor/data/chassis/brakes/pressure +``` + +A topic a member this gateway runs provides is sampled here, unchanged; a topic +whose providers are spread across gateways names no single place, and the local +graph answers it as before. `/operations` resolves a bare id the same way: the +operation is resolved first, and the member owning its ROS path is where the +execution is sent. + `/configurations` keeps its own id scheme, `:`, whose member half is the app id. Nothing on the owning gateway is aggregating, so the parameter is addressed there by its bare name: diff --git a/src/ros2_medkit_gateway/design/aggregation.rst b/src/ros2_medkit_gateway/design/aggregation.rst index 5f493c4d3..0a89bc02a 100644 --- a/src/ros2_medkit_gateway/design/aggregation.rst +++ b/src/ros2_medkit_gateway/design/aggregation.rst @@ -436,6 +436,28 @@ member: PUT /api/v1/functions/vehicle_health/configurations/peer_calibration:calibration_offset -> PUT /api/v1/apps/peer_calibration/configurations/calibration_offset (on the peer) +An id with no member half is the same question asked differently, and it has to +have the same answer. Qualification follows ambiguity, so an item a single member +provides keeps its bare id, and that bare id is what the collection hands a +client - it has to reach the owner too, or the most ordinary item in an +aggregating entity is the one that cannot be read. Each collection recovers the +owner from what the tree already records: +``AggregatedOperations::owner_by_path`` maps a ROS path to the member that owns +the operation, and ``AggregatedData::owners_by_topic`` maps a topic path to the +members that provide it. The dispatch is then identical: + +.. code-block:: text + + GET /api/v1/functions/vehicle_health/data/chassis%2Fbrakes%2Fpressure + -> GET /api/v1/apps/pressure_sensor/data/chassis/brakes/pressure (on the peer) + +A topic differs from an operation in having a LIST of owners, because one topic +that a member publishes and another subscribes to is one item. That list settles +where, not how many: an owner this gateway runs means the topic is on this graph +and is sampled here; owners all on one peer mean the topic is on that peer, and +any one of them addresses it there; owners spread across gateways mean the bare +id names no single place, and the local graph answers it as it always did. + A member half a peer supplied is read through the collision rename before it means anything here. A peer describes its own tree in its own names, and an App whose id collided with a local one was merged under ``__``, so the @@ -525,9 +547,11 @@ The member's own gateway is the only one that can answer: the ROS service, the topic and the parameter behind the id exist on its graph and nowhere else. What this gateway holds for a peer-owned member is a declaration, which is why the local walk's record of "does this member provide this item" is consulted only -once the member is known to be served here - on a peer-owned member it holds no -topics at all, and no node FQN to ask for a parameter, so every one of them -would be a miss. +once the member is known to be served here. What that declaration carries is +whatever the peer last reported - the topics and operations it has, and no node +FQN to ask for a parameter - so a member whose report has not arrived yet, or +whose parameters were never in it, would have every one of its items read as a +miss. The order inside ``dispatch_to_member`` is load-bearing: @@ -678,7 +702,8 @@ no other meaning for, an oversized body or unparsable JSON on any of them fails the fetch, because a picture missing a branch is indistinguishable on the wire from a peer that does not have that branch. Two statuses carry a meaning of their own: a ``404`` on a nested collection route (``/subareas``, -``/subcomponents``, an app's ``/operations``) identifies a peer running a +``/subcomponents``, an app's ``/operations`` or ``/data``) identifies a peer +running a gateway that predates the route and is reported in ``PeerEntities::absent_routes`` for the caller to log; a ``504`` with error code ``not-responding`` on any route hanging off an entity - its detail, or one of @@ -692,6 +717,21 @@ picture on every refresh, and the aggregator would go on serving its last pre-failure view indefinitely. A ``504`` that does not carry ``not-responding`` says nothing about an entity and still fails the fetch. +Two of those requests are made per App: its ``/operations`` and its ``/data``. +Neither collection is ever declared in a manifest - both are discovered from the +ROS graph - so what the peer reports on those routes is the only record of them +this gateway can have, and both records are what addressing is built on. Without +the operations, ambiguity between two members sharing an operation short name +could only be settled by asking at request time, and an answer that depends on +who is reachable is not one a client can rely on. Without the topics, nothing +here maps a peer's topic to the member that provides it, so the bare path a +single-provider topic is listed under resolves to no owner, is served from the +local graph, and is refused as an unknown topic while the member and its gateway +are both healthy. Both requests carry ``X-Medkit-No-Fan-Out``, which keeps the +peer from re-asking ITS peers: each gateway reports what it holds, the hop that +owns an entity is the hop that answers for it, and that is also what makes the +walk terminate. + Availability travels the same way in the other direction. ``x-medkit.available`` is emitted only when false, so absence means reachable, and both ``parse_component`` and ``parse_app`` read it back with a default of ``true``. @@ -709,9 +749,10 @@ whether to replay it marked unavailable (health check failed) or exactly as it was last read (health check still passes). The same field on a listed ITEM answers for that item's member and for nothing -else. ``/operations`` holds back the copies its declared tree carries for -peer-owned members and offers them only when the fan-out did not bring the -owner's own copy, and whether such a copy is marked unavailable is decided from +else. ``/operations`` and ``/data`` hold back the copies their declared tree +carries for peer-owned members and offer them only when the fan-out did not bring +the owner's own copy - keyed on the full ROS path, which is what makes two copies +one item - and whether such a copy is marked unavailable is decided from the member's reachability - the same reading ``dispatch_to_member`` acts on, so the listing and the request cannot disagree. A fan-out that produced nothing is not evidence on its own: it also never runs when no peer contributes the entity, diff --git a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/dto/data.hpp b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/dto/data.hpp index 511f0e8e0..a38ec6c6a 100644 --- a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/dto/data.hpp +++ b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/dto/data.hpp @@ -59,6 +59,11 @@ namespace dto { // timestamp - sample timestamp in nanoseconds since epoch (int64) // publisher_count - number of publishers on the topic at sample time (int64) // subscriber_count - number of subscribers on the topic at sample time (int64) +// +// Additional keys on an aggregating entity's list items: +// member_ids - the members of the grouping that provide the topic +// available - present only as false, marking an item whose provider is not +// answering; absent means it can be served // ============================================================================= struct XMedkitDataItem { std::optional ros2; @@ -78,15 +83,21 @@ struct XMedkitDataItem { /// short name. More than one entry is what makes the bare item id /// ambiguous for addressing. std::optional> member_ids; + /// Absent while the item can be served. False marks an item listed from a + /// retained declaration because the member that owns it is not answering - + /// the item is still part of the tree, and still addressable. + std::optional available; }; template <> -inline constexpr auto dto_fields = std::make_tuple( - field("ros2", &XMedkitDataItem::ros2), field("type_info", &XMedkitDataItem::type_info), - field("entity_id", &XMedkitDataItem::entity_id), field("status", &XMedkitDataItem::status), - field("publisher_created", &XMedkitDataItem::publisher_created), field("timestamp", &XMedkitDataItem::timestamp), - field("publisher_count", &XMedkitDataItem::publisher_count), - field("subscriber_count", &XMedkitDataItem::subscriber_count), field("member_ids", &XMedkitDataItem::member_ids)); +inline constexpr auto dto_fields = + std::make_tuple(field("ros2", &XMedkitDataItem::ros2), field("type_info", &XMedkitDataItem::type_info), + field("entity_id", &XMedkitDataItem::entity_id), field("status", &XMedkitDataItem::status), + field("publisher_created", &XMedkitDataItem::publisher_created), + field("timestamp", &XMedkitDataItem::timestamp), + field("publisher_count", &XMedkitDataItem::publisher_count), + field("subscriber_count", &XMedkitDataItem::subscriber_count), + field("member_ids", &XMedkitDataItem::member_ids), field("available", &XMedkitDataItem::available)); template <> inline constexpr std::string_view dto_name = "XMedkitDataItem"; diff --git a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/http/handlers/handler_support.hpp b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/http/handlers/handler_support.hpp index 6b7b4452a..36f0430ff 100644 --- a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/http/handlers/handler_support.hpp +++ b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/http/handlers/handler_support.hpp @@ -15,6 +15,7 @@ #pragma once #include +#include #include #include #include @@ -22,12 +23,46 @@ #include #include "ros2_medkit_gateway/core/models/error_info.hpp" +#include "ros2_medkit_gateway/core/models/thread_safe_entity_cache.hpp" #include "ros2_medkit_gateway/http/handlers/handler_context.hpp" #include "ros2_medkit_gateway/http/typed_router.hpp" namespace ros2_medkit_gateway { namespace handlers { +/// The `peer:` source of a member another gateway contributed, or empty +/// for a member this gateway runs or has never heard of. +/// +/// One definition for every collection, because the reading decides three +/// things that have to agree: whether this gateway's own walk may report an +/// item, whether the fan-out copy is the one that counts, and which gateway a +/// request for it goes to. Two collections reading it apart would list an item +/// from one place and address it to another. +/// +/// The declared source rather than the routing table: a member whose gateway has +/// gone quiet is retained under the same source, and a request for its items has +/// to keep resolving to it in order to be answered "not responding" rather than +/// sampled here and reported absent. +inline std::string peer_source_of_member(const ThreadSafeEntityCache & cache, const std::string & member_id) { + static constexpr std::string_view kPeerPrefix = "peer:"; + const auto peer_source = [](const std::string & source) { + return source.rfind(kPeerPrefix, 0) == 0 ? source : std::string{}; + }; + if (auto app = cache.get_app(member_id)) { + return peer_source(app->source); + } + if (auto component = cache.get_component(member_id)) { + return peer_source(component->source); + } + return {}; +} + +/// True for a member whose declaration reached this gateway from a peer, and so +/// whose items another gateway serves. +inline bool member_is_peer_contributed(const ThreadSafeEntityCache & cache, const std::string & member_id) { + return !peer_source_of_member(cache, member_id).empty(); +} + /// Build a SOVD-shaped ErrorInfo. Empty `params` are dropped so the wire body /// matches the legacy `send_error` default and integration tests stay byte- /// identical. Shared by every typed handler (was duplicated per handler TU). diff --git a/src/ros2_medkit_gateway/src/core/aggregation/peer_client.cpp b/src/ros2_medkit_gateway/src/core/aggregation/peer_client.cpp index 976532854..2031792c9 100644 --- a/src/ros2_medkit_gateway/src/core/aggregation/peer_client.cpp +++ b/src/ros2_medkit_gateway/src/core/aggregation/peer_client.cpp @@ -281,6 +281,47 @@ void parse_operations_into(const nlohmann::json & j, App & app) { } } +/** + * @brief Read a peer's data collection into the App's topics. + * + * A data item names its ROS topic by full path in ``x-medkit.ros2.topic``. An + * item without one describes something that is not a topic - a plugin's data + * point, for instance - and carries nothing this gateway can address as one, so + * it is skipped rather than recorded under its wire id. + * + * ``direction`` decides which side of the topic the App is on, and only the + * three values a gateway emits are accepted. An item that says anything else + * leaves the App unattributed for that topic instead of being recorded as a + * publisher it may not be: the ownership built out of these lists is what a + * bare item id is dispatched by, and a guess there sends a read to a gateway + * that does not have the topic. + */ +void parse_data_items_into(const nlohmann::json & j, App & app) { + if (!j.contains("items") || !j["items"].is_array()) { + return; + } + for (const auto & item : j["items"]) { + if (!item.is_object() || !item.contains("x-medkit") || !item["x-medkit"].is_object()) { + continue; + } + const auto & xm = item["x-medkit"]; + if (!xm.contains("ros2") || !xm["ros2"].is_object()) { + continue; + } + const std::string topic = xm["ros2"].value("topic", ""); + const std::string direction = xm["ros2"].value("direction", ""); + if (topic.empty()) { + continue; + } + if (direction == "publish" || direction == "both") { + app.topics.publishes.push_back(topic); + } + if (direction == "subscribe" || direction == "both") { + app.topics.subscribes.push_back(topic); + } + } +} + /** * @brief Parse an App from JSON. * @@ -728,35 +769,52 @@ tl::expected PeerClient::fetch_entities() { }), entities.apps.end()); - // Fetch each app's operations. An operation is never declared in a - // manifest - it is discovered from the ROS graph - so the only record of - // what a peer's app exposes is what the peer reports. Without it the - // aggregator cannot tell that two members share an operation short name - // except by asking at request time, and an answer that depends on who is - // reachable is not an answer a client can rely on. + // Fetch each app's operations and data. Neither is ever declared in a + // manifest - both are discovered from the ROS graph - so the only record of + // what a peer's app exposes is what the peer reports. Without the + // operations the aggregator cannot tell that two members share an operation + // short name except by asking at request time, and an answer that depends + // on who is reachable is not an answer a client can rely on. Without the + // topics it holds no record of which member owns one, so an item id that + // names no member - the form a topic with a single provider is listed under + // - cannot be routed to the gateway that has the topic. // // `X-Medkit-No-Fan-Out` keeps the peer from re-asking ITS peers: each // gateway reports what it holds, and the hop that owns the entity is the // hop that answers for it. It is also what makes this terminate. + // + // A route that is absent or answers for an unreachable entity is skipped on + // its own, never for the app: the two collections are read independently + // and one missing must not cost the other. for (auto & app : entities.apps) { - httplib::Headers no_fan_out{{"X-Medkit-No-Fan-Out", "1"}}; - const std::string route = "/apps/" + app.id + "/operations"; - auto ops = read_sub_response(cli.Get(std::string(API_PREFIX) + route, no_fan_out), name_, route, + const httplib::Headers no_fan_out{{"X-Medkit-No-Fan-Out", "1"}}; + + const std::string ops_route = "/apps/" + app.id + "/operations"; + auto ops = read_sub_response(cli.Get(std::string(API_PREFIX) + ops_route, no_fan_out), name_, ops_route, RouteKind::kNestedCollection); if (ops.kind == SubResponse::Kind::kIncomplete) { return tl::unexpected(ops.error); } if (ops.kind == SubResponse::Kind::kRouteAbsent) { note_absent_route("/apps/{id}/operations"); - continue; + } else if (ops.kind == SubResponse::Kind::kBody) { + parse_operations_into(ops.body, app); } - if (ops.kind == SubResponse::Kind::kEntityUnreachable) { - // The App is retained and its gateway is silent, so the peer answers - // for it rather than proxying. It keeps the operations the peer already - // reported; there is nothing further to read. - continue; + // kEntityUnreachable: the App is retained and its gateway is silent, so + // the peer answers for it rather than proxying. It keeps what the peer + // already reported; there is nothing further to read. + + const std::string data_route = "/apps/" + app.id + "/data"; + auto data = read_sub_response(cli.Get(std::string(API_PREFIX) + data_route, no_fan_out), name_, data_route, + RouteKind::kNestedCollection); + if (data.kind == SubResponse::Kind::kIncomplete) { + return tl::unexpected(data.error); + } + if (data.kind == SubResponse::Kind::kRouteAbsent) { + note_absent_route("/apps/{id}/data"); + } else if (data.kind == SubResponse::Kind::kBody) { + parse_data_items_into(data.body, app); } - parse_operations_into(ops.body, app); } } diff --git a/src/ros2_medkit_gateway/src/http/handlers/data_handlers.cpp b/src/ros2_medkit_gateway/src/http/handlers/data_handlers.cpp index c7ad93d50..89c8fd1b2 100644 --- a/src/ros2_medkit_gateway/src/http/handlers/data_handlers.cpp +++ b/src/ros2_medkit_gateway/src/http/handlers/data_handlers.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -98,15 +99,60 @@ std::string to_full_topic_path(const std::string & topic_name) { } /// What one addressed data item resolves to: the ROS topic to act on, the id to -/// echo back to the caller, the member the id named, and whether this gateway's -/// own walk records that member as a provider of the topic. +/// echo back to the caller, the member the id named, whether this gateway's own +/// walk records that member as a provider of the topic, and the member the tree +/// dispatches the request through when the id named none. struct AddressedDataItem { std::string full_topic_path; std::string item_id; std::string member_id; ///< Empty when the id carried no member half. bool provided_by_named_member{false}; ///< Meaningful only when member_id is set. + std::string owner_member_id; ///< The member a bare id is dispatched through; empty means serve here. }; +/// The member a BARE id is dispatched through, or empty when this gateway is the +/// one that serves it. +/// +/// A topic with a single provider keeps its bare id in the collection - +/// qualification follows ambiguity, not aggregation - so the bare form is the id +/// a client is handed back, and it has to reach the gateway that has the topic. +/// That is what the owners record makes possible, exactly as +/// `AggregatedOperations::owner_by_path` makes a bare operation id routable. +/// +/// What is settled here is WHERE, and the owner COUNT does not settle it: a +/// topic is one topic on one ROS graph, and its owners are the members that +/// touch it, not several copies of it. Two members of one gateway publishing and +/// subscribing to a topic name one place between them, and either of them +/// addresses it there. So the declarations decide: +/// +/// * an owner this gateway runs - the topic is on this graph, serve it here; +/// * every owner on one peer - the topic is on that peer, and any of its +/// members reaches it; +/// * owners spread across gateways - the bare id names no single place, and +/// the local walk answers, which is what it has always done. +/// +/// A leaf has no members at all: it is its own sole contributor, and there is +/// nothing to hand the request to. +std::string serving_member_for_topic(const ThreadSafeEntityCache & cache, const AggregatedData & aggregated, + const std::string & full_topic_path) { + if (!aggregated.is_aggregated) { + return {}; + } + auto owners = aggregated.owners_by_topic.find(full_topic_path); + if (owners == aggregated.owners_by_topic.end() || owners->second.empty()) { + return {}; + } + std::string peer; + for (const auto & owner : owners->second) { + const std::string owner_peer = peer_source_of_member(cache, owner); + if (owner_peer.empty() || (!peer.empty() && owner_peer != peer)) { + return {}; + } + peer = owner_peer; + } + return owners->second.front(); +} + /// Resolve the id in the route against the entity, for reads and writes alike. /// /// A qualified id is answered exactly, because the member set is known here: an @@ -114,28 +160,39 @@ struct AddressedDataItem { /// gateway samples the local graph, finds nothing, and returns 200 with an empty /// body and status `metadata_only` - a typo reported as success. /// -/// Whether the named member provides the item is REPORTED, not decided: the -/// answer comes from this gateway's own walk, which holds no topics for a member -/// another gateway runs, so acting on it here would turn every peer-owned item -/// into a miss. The caller resolves ownership first and only then reads the flag. +/// Whether the named member provides the item is REPORTED, not decided: this +/// gateway's walk holds a member another gateway runs only as that gateway last +/// reported it, so a member whose report has not arrived yet would have every +/// one of its items read as a miss. The caller settles ownership first - a +/// member another gateway runs is answered by that gateway - and only reads the +/// flag for a member it serves itself. /// /// A ROS topic name cannot contain a colon, so one in the id can only be the -/// member separator. Building the member set is not free, so the cache is only -/// consulted for an id that carries one; a bare id addresses a topic path, -/// which names one topic on its own and keeps its existing behaviour. -tl::expected -address_data_item(const ThreadSafeEntityCache & cache, const std::string & entity_id, const std::string & topic_name) { +/// member separator. A bare id addresses a topic path, which names one topic on +/// its own; it still has an owner, and `resolve_owner` says whether finding it +/// can change where the request is served. Walking the entity is not free, so a +/// gateway with no peers - where every member is served here anyway - skips it. +tl::expected address_data_item(const ThreadSafeEntityCache & cache, + const std::string & entity_id, + const std::string & topic_name, bool resolve_owner) { AddressedDataItem addressed; addressed.full_topic_path = to_full_topic_path(topic_name); addressed.item_id = addressed.full_topic_path; if (topic_name.find(':') == std::string::npos) { + if (resolve_owner) { + addressed.owner_member_id = + serving_member_for_topic(cache, cache.get_entity_data(entity_id), addressed.full_topic_path); + } return addressed; } auto aggregated = cache.get_entity_data(entity_id); auto parsed = http::parse_member_qualified_id(topic_name, aggregated.is_aggregated); if (!parsed.has_member) { + if (resolve_owner) { + addressed.owner_member_id = serving_member_for_topic(cache, aggregated, addressed.full_topic_path); + } return addressed; } @@ -182,12 +239,19 @@ std::string member_data_resource_path(const std::string & full_topic_path) { /// Returns the answer the handler must return - including the sentinel that says /// the owning peer has already committed the wire - or nullopt when this gateway /// serves the item itself. The "member does not provide it" refusal is decided -/// here rather than while addressing, because it rests on the local walk, which -/// says nothing about a member another gateway runs. +/// here rather than while addressing, because it rests on the local walk, and +/// the walk only describes a member another gateway runs as well as that +/// gateway's last report did. +/// +/// An id that named a member is served by that member. An id that named none is +/// served by the member the tree says owns the topic, so the bare form a +/// single-provider item is listed under reaches the same gateway the qualified +/// form does. Where neither applies the entity serves it itself. std::optional dispatch_data_item(const HandlerContext & ctx, const http::TypedRequest & req, const std::string & entity_id, const std::string & topic_name, const AddressedDataItem & addressed) { - auto dispatch = ctx.dispatch_to_member(req, addressed.member_id, member_data_resource_path(addressed.full_topic_path), + const std::string & serving_member = addressed.member_id.empty() ? addressed.owner_member_id : addressed.member_id; + auto dispatch = ctx.dispatch_to_member(req, serving_member, member_data_resource_path(addressed.full_topic_path), json{{"entity_id", entity_id}, {"id", topic_name}}); if (!dispatch) { return dispatch.error(); @@ -401,7 +465,28 @@ http::Result DataHandlers::list_data(const http::TypedReque auto data_access_mgr = ctx_.node()->get_data_access_manager(); auto type_introspection = data_access_mgr->get_type_introspection(); + // True for a topic every one of whose providers is a member another gateway + // runs. The owners are a list because a topic can be published by one member + // and subscribed by another, and a single local provider is enough to make + // this gateway's walk the account of the item - so it is "all", not "any". + const auto served_only_by_peers = [&aggregated, &cache](const std::string & topic_name) { + auto owners = aggregated.owners_by_topic.find(topic_name); + if (owners == aggregated.owners_by_topic.end() || owners->second.empty()) { + return false; + } + return std::all_of(owners->second.begin(), owners->second.end(), [&cache](const std::string & member_id) { + return member_is_peer_contributed(cache, member_id); + }); + }; + dto::Collection response; + // A peer's topics are held here so ownership can be settled without asking + // anyone. They are not reported from this walk while the peer is reachable - + // the gateway that owns an item is the one that reports it, and its copy + // carries the message type and the sample metadata this one cannot - so they + // are set aside and only fall back into the list below, when the fan-out + // that should have carried them did not. + std::vector retained_from_peers; for (const auto & topic : aggregated.topics) { dto::DataItem di; di.id = topic.name; @@ -416,7 +501,7 @@ http::Result DataHandlers::list_data(const http::TypedReque owners != aggregated.owners_by_topic.end() && aggregated.is_aggregated) { di.x_medkit->member_ids = owners->second; } - response.items.push_back(std::move(di)); + (served_only_by_peers(topic.name) ? retained_from_peers : response.items).push_back(std::move(di)); } // Typed fan-out for the data list. Replaces the legacy raw-JSON @@ -432,15 +517,32 @@ http::Result DataHandlers::list_data(const http::TypedReque #pragma GCC diagnostic ignored "-Wdeprecated-declarations" const auto & raw_req = req.raw_for_framework(); #pragma GCC diagnostic pop + // Two different reasons a peer's copy can be missing, and they are not the + // same answer. The caller asking for no fan-out means the peers were never + // consulted, and reporting their items anyway is what turns a + // bidirectionally peered pair into a bounce. A fan-out that ran and came + // back without them means the peer is not answering, and the tree still + // knows what it declared. + const bool fan_out_suppressed = raw_req.has_header("X-Medkit-No-Fan-Out"); auto * agg = ctx_.aggregation_manager(); auto fan_out = fan_out_collection(agg, raw_req); + // The ROS topic an item names. Empty when the item names none, which is + // what a peer's malformed item - or a plugin's data point - looks like. + const auto topic_of = [](const dto::DataItem & item) -> std::string { + if (!item.x_medkit.has_value() || !item.x_medkit->ros2.has_value()) { + return {}; + } + return item.x_medkit->ros2->topic.value_or(std::string{}); + }; + // A peer names its members as its own tree names them, and an App whose id // collided with a local one was merged under `__` - so the name - // the peer sends names the LOCAL leaf here. A merged App carries no topics, - // so this attribution is the only account of who owns a peer's item, and a - // client that builds `:` out of it addresses a member that - // does not publish that topic at all. + // the peer sends names the LOCAL leaf here. Re-emitted verbatim it + // attributes the peer's topic to a member that does not publish it, and + // every id built from that attribution - by this gateway or by a client + // reading the list - is resolved against the wrong member. + std::unordered_set topics_from_peers; for (size_t index = 0; index < fan_out.items.size(); ++index) { auto & item = fan_out.items[index]; const std::string peer_name = index < fan_out.item_peers.size() ? fan_out.item_peers[index] : std::string{}; @@ -449,9 +551,44 @@ http::Result DataHandlers::list_data(const http::TypedReque member_id = agg->local_member_id(peer_name, member_id); } } + if (auto topic = topic_of(item); !topic.empty()) { + topics_from_peers.insert(std::move(topic)); + } response.items.push_back(std::move(item)); } + // The full ROS topic path is the key, because that is what makes two copies + // one item: the wire id is that same path, and a topic is held once per ROS + // graph, so a path arriving from the owner and a path held here from its + // declaration describe the same topic. Without it both are listed, and the + // duplicate ids are then qualified into two items that address one topic. + if (!fan_out_suppressed) { + for (auto & item : retained_from_peers) { + const std::string topic = topic_of(item); + if (!topic.empty() && topics_from_peers.count(topic) > 0u) { + continue; // the owner answered for itself, which is the better copy + } + // `available` is a statement about the MEMBER, not about the fan-out: + // false means the gateway that owns the item is not answering, so a + // request for it cannot be served. A fan-out reaching this gateway with + // nothing for this topic says nothing on its own - it also never ran + // when no peer contributes this entity. The member's own reachability + // is what a request for the item will meet, and it is the same reading + // `dispatch_to_member` acts on, so the listing and the read cannot + // disagree. + if (auto owners = aggregated.owners_by_topic.find(topic); owners != aggregated.owners_by_topic.end()) { + const bool any_unreachable = + std::any_of(owners->second.begin(), owners->second.end(), [this](const std::string & member_id) { + return !ctx_.is_entity_available(member_id); + }); + if (any_unreachable) { + item.x_medkit->available = false; + } + } + response.items.push_back(std::move(item)); + } + } + // A topic path names one topic however many members publish and subscribe // to it - those merge into a single item, whose contributors are already // named in member_ids - so nothing here is qualified in the ordinary case. @@ -537,7 +674,8 @@ http::Result DataHandlers::get_data_item(const http::TypedReques } try { - auto addressed = address_data_item(ctx_.node()->get_thread_safe_cache(), entity_id, topic_name); + auto addressed = address_data_item(ctx_.node()->get_thread_safe_cache(), entity_id, topic_name, + ctx_.aggregation_manager() != nullptr); if (!addressed) { return tl::make_unexpected(addressed.error()); } @@ -719,7 +857,8 @@ http::Result DataHandlers::put_data_item(const http::TypedReques // A write addresses the same item a read does, so it resolves the same way - // including which gateway publishes it. Publishing here for a member another // gateway runs would create a publisher on a graph that member is not on. - auto addressed = address_data_item(ctx_.node()->get_thread_safe_cache(), entity_id, topic_name); + auto addressed = address_data_item(ctx_.node()->get_thread_safe_cache(), entity_id, topic_name, + ctx_.aggregation_manager() != nullptr); if (!addressed) { return tl::make_unexpected(addressed.error()); } diff --git a/src/ros2_medkit_gateway/src/http/handlers/operation_handlers.cpp b/src/ros2_medkit_gateway/src/http/handlers/operation_handlers.cpp index 7aa4a867c..abddb4d8f 100644 --- a/src/ros2_medkit_gateway/src/http/handlers/operation_handlers.cpp +++ b/src/ros2_medkit_gateway/src/http/handlers/operation_handlers.cpp @@ -18,7 +18,6 @@ #include #include #include -#include #include #include #include @@ -607,19 +606,9 @@ http::Result> OperationHandlers::list_operat // the same member for both. const std::unordered_set path_addressed = http::operation_paths_addressed_by_path(ops); - const auto contributed_by_peer = [&cache](const std::string & member_id) { - static constexpr std::string_view kPeerPrefix = "peer:"; - if (auto app = cache.get_app(member_id)) { - return app->source.rfind(kPeerPrefix, 0) == 0; - } - if (auto component = cache.get_component(member_id)) { - return component->source.rfind(kPeerPrefix, 0) == 0; - } - return false; - }; - const auto owner_is_remote = [&ops, &contributed_by_peer](const std::string & full_path) { + const auto owner_is_remote = [&ops, &cache](const std::string & full_path) { auto owner = ops.owner_by_path.find(full_path); - return owner != ops.owner_by_path.end() && contributed_by_peer(owner->second); + return owner != ops.owner_by_path.end() && member_is_peer_contributed(cache, owner->second); }; // The ROS path an item names, in the form an id carries it. Empty when the diff --git a/src/ros2_medkit_gateway/test/test_peer_client.cpp b/src/ros2_medkit_gateway/test/test_peer_client.cpp index b65598289..56421778c 100644 --- a/src/ros2_medkit_gateway/test/test_peer_client.cpp +++ b/src/ros2_medkit_gateway/test/test_peer_client.cpp @@ -15,8 +15,10 @@ #include #include +#include #include #include +#include #include "ros2_medkit_gateway/core/aggregation/entity_merger.hpp" #include "ros2_medkit_gateway/core/aggregation/peer_client.hpp" @@ -1233,3 +1235,145 @@ TEST(PeerClientAvailability, a_504_that_is_not_a_statement_about_an_entity_still auto result = client.fetch_entities(); EXPECT_FALSE(result.has_value()) << "a gateway timeout was read as a statement that an entity is unreachable"; } + +// ============================================================================= +// A peer's topics +// +// An App's topics are discovered from the ROS graph, never declared, so what the +// peer reports on its own data route is the only record of them this gateway can +// have. Without it nothing here maps a peer's topic to the member that owns it, +// and the id a single-provider topic is listed under - the bare path - is +// unroutable. +// ============================================================================= + +TEST(PeerClientTopics, a_peers_topics_are_read_into_the_merged_app) { + httplib::Server svr; + svr.Get("/api/v1/apps", [](const httplib::Request &, httplib::Response & res) { + res.set_content(R"({"items":[{"id":"nav","name":"Navigation"}]})", "application/json"); + }); + svr.Get("/api/v1/apps/nav/operations", [](const httplib::Request &, httplib::Response & res) { + res.set_content(R"({"items":[]})", "application/json"); + }); + svr.Get("/api/v1/apps/nav/data", [](const httplib::Request & req, httplib::Response & res) { + // The peer must not re-ask ITS peers, or a bidirectionally peered pair bounces. + EXPECT_EQ(req.get_header_value("X-Medkit-No-Fan-Out"), "1"); + res.set_content( + R"({"items":[ + {"id":"/nav/pose","name":"/nav/pose","x-medkit":{"ros2":{"topic":"/nav/pose","direction":"publish"}}}, + {"id":"/nav/goal","name":"/nav/goal","x-medkit":{"ros2":{"topic":"/nav/goal","direction":"subscribe"}}}, + {"id":"/nav/odom","name":"/nav/odom","x-medkit":{"ros2":{"topic":"/nav/odom","direction":"both"}}} + ]})", + "application/json"); + }); + svr.Get("/api/v1/areas", [](const httplib::Request &, httplib::Response & res) { + res.set_content(R"({"items":[]})", "application/json"); + }); + svr.Get("/api/v1/components", [](const httplib::Request &, httplib::Response & res) { + res.set_content(R"({"items":[]})", "application/json"); + }); + svr.Get("/api/v1/functions", [](const httplib::Request &, httplib::Response & res) { + res.set_content(R"({"items":[]})", "application/json"); + }); + + ScopedServer running(svr); + PeerClient client(running.url(), "peer_b", 5000); + auto result = client.fetch_entities(); + ASSERT_TRUE(result.has_value()) << result.error(); + + ASSERT_EQ(result->apps.size(), 1u); + const auto & app = result->apps[0]; + EXPECT_EQ(app.topics.publishes, (std::vector{"/nav/pose", "/nav/odom"})); + EXPECT_EQ(app.topics.subscribes, (std::vector{"/nav/goal", "/nav/odom"})); +} + +TEST(PeerClientTopics, an_item_that_does_not_describe_a_topic_is_not_recorded_as_one) { + httplib::Server svr; + svr.Get("/api/v1/apps", [](const httplib::Request &, httplib::Response & res) { + res.set_content(R"({"items":[{"id":"plc","name":"PLC bridge"}]})", "application/json"); + }); + svr.Get("/api/v1/apps/plc/operations", [](const httplib::Request &, httplib::Response & res) { + res.set_content(R"({"items":[]})", "application/json"); + }); + // A plugin's data point carries no ROS metadata at all, and a topic with no + // direction says nothing about which side the App is on. Recorded anyway, each + // would attribute a topic to a member that may not have it, and a bare id + // built on that attribution is dispatched to the wrong gateway. + svr.Get("/api/v1/apps/plc/data", [](const httplib::Request &, httplib::Response & res) { + res.set_content( + R"({"items":[ + {"id":"tank_level","name":"tank_level","category":"currentData"}, + {"id":"valve","name":"valve","x-medkit":{"source":"opcua"}}, + {"id":"pump","name":"pump","x-medkit":{"ros2":{"type":"std_msgs/msg/Float32"}}}, + {"id":"/plc/raw","name":"/plc/raw","x-medkit":{"ros2":{"topic":"/plc/raw"}}}, + {"id":"/plc/aux","name":"/plc/aux","x-medkit":{"ros2":{"topic":"/plc/aux","direction":"sideways"}}} + ]})", + "application/json"); + }); + install_empty_roots(svr); + + ScopedServer running(svr); + PeerClient client(running.url(), "peer_b", 5000); + auto result = client.fetch_entities(); + ASSERT_TRUE(result.has_value()) << result.error(); + + ASSERT_EQ(result->apps.size(), 1u); + EXPECT_TRUE(result->apps[0].topics.publishes.empty()) + << "an item that names no ROS topic was recorded as a published one"; + EXPECT_TRUE(result->apps[0].topics.subscribes.empty()) + << "an item that names no ROS topic was recorded as a subscribed one"; +} + +TEST(PeerClientTopics, a_peer_without_the_data_route_still_describes_its_apps) { + httplib::Server svr; + svr.Get("/api/v1/apps", [](const httplib::Request &, httplib::Response & res) { + res.set_content(R"({"items":[{"id":"nav","name":"Navigation"}]})", "application/json"); + }); + // A gateway old enough not to serve the data route answers 404. Aggregation + // works across that version boundary: the App is described by everything else + // the peer does offer, and the missing route is reported rather than fatal. + svr.Get("/api/v1/apps/nav/operations", [](const httplib::Request &, httplib::Response & res) { + res.set_content(R"({"items":[{"name":"calibrate","x-medkit":{"ros2":{"service":"/nav/calibrate"}}}]})", + "application/json"); + }); + install_empty_roots(svr); + + ScopedServer running(svr); + PeerClient client(running.url(), "peer_b", 5000); + auto result = client.fetch_entities(); + ASSERT_TRUE(result.has_value()) << "an absent data route discarded the peer's picture: " << result.error(); + + ASSERT_EQ(result->apps.size(), 1u); + EXPECT_EQ(result->apps[0].services.size(), 1u) << "the operations route was skipped along with the data route"; + EXPECT_TRUE(result->apps[0].topics.publishes.empty()); + EXPECT_NE(std::find(result->absent_routes.begin(), result->absent_routes.end(), "/apps/{id}/data"), + result->absent_routes.end()) + << "the missing route was not reported"; +} + +TEST(PeerClientTopics, a_data_route_that_says_not_responding_does_not_abort_the_fetch) { + httplib::Server svr; + svr.Get("/api/v1/apps", [](const httplib::Request &, httplib::Response & res) { + res.set_content(R"({"items":[{"id":"app-quiet","name":"app-quiet","x-medkit":{"available":false}}]})", + "application/json"); + }); + svr.Get("/api/v1/apps/app-quiet/operations", [](const httplib::Request &, httplib::Response & res) { + res.status = 504; + res.set_content(not_responding_body("app-quiet"), "application/json"); + }); + // Every route of a retained member answers 504 not-responding, the data route + // included. Read as a failed request it would discard the peer's whole picture. + svr.Get("/api/v1/apps/app-quiet/data", [](const httplib::Request &, httplib::Response & res) { + res.status = 504; + res.set_content(not_responding_body("app-quiet"), "application/json"); + }); + install_empty_roots(svr); + + ScopedServer running(svr); + PeerClient client(running.url(), "peer_b", 5000); + auto result = client.fetch_entities(); + ASSERT_TRUE(result.has_value()) << "one unreachable member discarded the peer's whole picture: " << result.error(); + + ASSERT_EQ(result->apps.size(), 1u); + EXPECT_FALSE(result->apps[0].available); + EXPECT_TRUE(result->apps[0].topics.publishes.empty()); +} diff --git a/src/ros2_medkit_integration_tests/test/features/test_grouping_entity_aggregation.test.py b/src/ros2_medkit_integration_tests/test/features/test_grouping_entity_aggregation.test.py index e5082ba80..91488406d 100644 --- a/src/ros2_medkit_integration_tests/test/features/test_grouping_entity_aggregation.test.py +++ b/src/ros2_medkit_integration_tests/test/features/test_grouping_entity_aggregation.test.py @@ -183,6 +183,20 @@ PRIMARY_RPM_APP = 'rpm_sensor' PEER_ACTUATOR_APP = 'brake_actuator' +# A topic that exists only on the PEER's ROS graph, and the member the merged +# tree hands a request for it to. Both peer members touch it - the sensor +# publishes it and the actuator publishes its own copy under the same path - so +# it is one topic with two providers on ONE gateway, which is where the bare id +# has to land. Its wire id stays bare: two members of the same gateway are not +# two copies to tell apart. +PEER_ONLY_TOPIC = '/chassis/brakes/pressure' +PEER_ONLY_TOPIC_APP = 'pressure_sensor' + +# The same shape on this gateway's own graph, for the half of the rule that says +# a locally owned topic keeps being served here. +LOCAL_ONLY_TOPIC = '/powertrain/engine/temperature' +LOCAL_ONLY_TOPIC_APP = 'temp_sensor' + # Declared by the calibration demo node, so BOTH `primary_calibration` and # `peer_calibration` expose it under one name. A member-qualified id is the only # thing that separates the two copies, which is what makes this the parameter @@ -466,6 +480,7 @@ def setUpClass(cls): {'pressure_sensor', 'peer_calibration', PEER_LONG_APP, COLLIDING_LEAF}, 'peer') cls._wait_until_merged() + cls._wait_for_peer_topic_ownership() @classmethod def _wait_for_apps(cls, base_url, required, label): @@ -491,6 +506,37 @@ def _wait_for_apps(cls, base_url, required, label): time.sleep(1.0) raise AssertionError(f'{label}: {required} not online within 60s') + @classmethod + def _wait_for_peer_topic_ownership(cls): + """Block until a peer's topics have reached this gateway's merged tree. + + A peer's topics are read over HTTP, one poll behind whatever the peer's + own graph looked like when it answered, so an App can be online on the + peer and merged here before the report that names its topics has + arrived. Until it does, a bare id for one of those topics resolves to no + owner and is sampled from the local graph. + + Best effort, and deliberately so: this absorbs a polling delay, it does + not assert a contract. Raising here would make every case below report + as a setUpClass error, hiding which of them the answer was actually + wrong for. + """ + deadline = time.monotonic() + 30.0 + while time.monotonic() < deadline: + try: + response = requests.get( + f'{PRIMARY_URL}/functions/{MERGED_FUNCTION}/data/' + f'{quote(PEER_ONLY_TOPIC.lstrip("/"), safe="")}', + timeout=10, + ) + if (response.status_code == 200 + and response.json().get('x-medkit', {}).get('entity_id') + == PEER_ONLY_TOPIC_APP): + return + except requests.RequestException: + pass + time.sleep(0.5) + @classmethod def _wait_until_merged(cls): """Block until the primary has merged the peer's half of the Function.""" @@ -1745,6 +1791,247 @@ def test_a_compound_id_reaches_a_local_member(self): f'a locally owned member was not served here: {body}', ) + def test_a_bare_data_id_the_list_offers_reaches_its_peer_owned_member(self): + """R4 for the id a single-provider topic is actually listed under. + + The compound form is not the only form a client is handed. A topic one + gateway provides keeps its bare id - qualification follows ambiguity - + so the bare id IS the address, and it has to reach the gateway that has + the topic. Served here it samples a graph the topic is not on and comes + back 404 topic-unavailable, with the member and its gateway both + healthy. + + The id is taken from the collection rather than written out, because the + agreement between the list and the read is what is under test. + """ + items = self._items(f'functions/{MERGED_FUNCTION}', 'data') + offered = [ + item for item in items + if item.get('x-medkit', {}).get('ros2', {}).get('topic') == PEER_ONLY_TOPIC + ] + self.assertEqual( + len(offered), 1, + f'{PEER_ONLY_TOPIC} is not offered exactly once: ' + f'{[item.get("id") for item in items]}', + ) + item_id = offered[0]['id'] + self.assertNotIn( + ':', item_id, + f'a topic only one gateway provides was qualified: {item_id!r}', + ) + + response = requests.get( + f'{PRIMARY_URL}/functions/{MERGED_FUNCTION}/data/{quote(item_id, safe="")}', + timeout=15, + ) + self.assertNotEqual( + response.status_code, 404, + f'the bare id the list offers was refused as an unknown topic: {response.text}', + ) + self.assertEqual(response.status_code, 200, response.text) + body = response.json() + self.assertEqual( + body.get('x-medkit', {}).get('status'), 'data', + f'the id the list offers read nothing: {body}', + ) + self.assertTrue(body.get('data'), 'the peer member returned an empty payload') + + # And it is THAT member's item, not merely some 200. Compared against + # the peer's own answer for the same topic, so a read that fell back to + # a local member cannot satisfy it and neither can an empty envelope. + direct = requests.get( + f'{PEER_URL}/apps/{PEER_ONLY_TOPIC_APP}/data' + f'/{quote(PEER_ONLY_TOPIC.lstrip("/"), safe="")}', + timeout=15, + ) + self.assertEqual(direct.status_code, 200, direct.text) + direct_body = direct.json() + self.assertEqual( + body.get('x-medkit', {}).get('ros2', {}).get('topic'), PEER_ONLY_TOPIC, + f'the answer names a topic the member does not publish: {body}', + ) + self.assertEqual( + body.get('x-medkit', {}).get('ros2', {}).get('type'), + direct_body.get('x-medkit', {}).get('ros2', {}).get('type'), + f'the answer is not the message the member publishes: {body}', + ) + self.assertEqual( + sorted(body['data'].keys()), sorted(direct_body['data'].keys()), + f"the payload is not shaped like the member's own: {body}", + ) + # Which gateway served it is the only thing separating a real answer + # from a plausible one: served here the entity named would be the + # aggregating Function, which is also what a local sample of a topic + # this gateway cannot see would carry. + self.assertEqual( + body.get('x-medkit', {}).get('entity_id'), PEER_ONLY_TOPIC_APP, + f'the aggregating entity answered for a member it does not run: {body}', + ) + + def test_a_peer_owned_topic_is_offered_exactly_once(self): + """The duplicate a merged App carrying topics makes possible. + + The list is assembled from two places that now both know the peer's + topics: this gateway's own walk over the merged tree, and the fan-out to + the gateway that owns them. Without a key the two copies are both + emitted, and because they then share an id each is qualified with its + member - so one topic is offered under two ids, neither of them the bare + one a client already sends. Counted per copy, not set-ified, because a + set hides exactly the duplicate this case exists to catch. + """ + # The Function and the Area, because those are the two aggregating kinds + # that reach the peer's member: the parent Component draws only from the + # apps located on itself, and the peer's are located on its + # subcomponent. + for entity_path in ( + f'functions/{MERGED_FUNCTION}', + f'areas/{MERGED_AREA}', + ): + with self.subTest(entity=entity_path): + items = self._items(entity_path, 'data') + copies = [ + item for item in items + if item.get('x-medkit', {}).get('ros2', {}).get('topic') == PEER_ONLY_TOPIC + ] + self.assertEqual( + len(copies), 1, + f'{PEER_ONLY_TOPIC} is listed {len(copies)} times: ' + f'{[item.get("id") for item in items]}', + ) + self.assertNotIn( + ':', copies[0].get('id', ''), + f'the surviving copy was qualified, so the bare id the ' + f'collection promised is gone: {copies[0]}', + ) + + def test_a_bare_data_id_of_a_local_member_is_still_served_here(self): + """R4 in the other direction: resolving owners must not export the local half. + + A topic this gateway's own member provides is on this gateway's graph, + and nothing about reading it may change. The value is asserted against + the member's own App route on THIS gateway, and the serving entity is + read as well - a dispatch that handed a local topic to a peer would + answer 404 or 504 there rather than fail visibly here. + """ + items = self._items(f'functions/{MERGED_FUNCTION}', 'data') + offered = [ + item for item in items + if item.get('x-medkit', {}).get('ros2', {}).get('topic') == LOCAL_ONLY_TOPIC + ] + self.assertEqual( + len(offered), 1, + f'{LOCAL_ONLY_TOPIC} is not offered exactly once: ' + f'{[item.get("id") for item in items]}', + ) + item_id = offered[0]['id'] + self.assertNotIn(':', item_id, f'a locally owned topic was qualified: {item_id!r}') + + response = requests.get( + f'{PRIMARY_URL}/functions/{MERGED_FUNCTION}/data/{quote(item_id, safe="")}', + timeout=15, + ) + self.assertEqual(response.status_code, 200, response.text) + body = response.json() + self.assertEqual( + body.get('x-medkit', {}).get('status'), 'data', + f'a locally owned topic read nothing: {body}', + ) + self.assertTrue(body.get('data'), 'the local member returned an empty payload') + + direct = requests.get( + f'{PRIMARY_URL}/apps/{LOCAL_ONLY_TOPIC_APP}/data' + f'/{quote(LOCAL_ONLY_TOPIC.lstrip("/"), safe="")}', + timeout=15, + ) + self.assertEqual(direct.status_code, 200, direct.text) + direct_body = direct.json() + self.assertEqual( + body.get('x-medkit', {}).get('ros2', {}).get('type'), + direct_body.get('x-medkit', {}).get('ros2', {}).get('type'), + f'the answer is not the message the local member publishes: {body}', + ) + self.assertEqual( + sorted(body['data'].keys()), sorted(direct_body['data'].keys()), + f"the payload is not shaped like the local member's own: {body}", + ) + self.assertEqual( + body.get('x-medkit', {}).get('entity_id'), MERGED_FUNCTION, + f'a locally owned topic was not served here: {body}', + ) + + def test_a_bare_data_id_naming_no_topic_is_answered_here_and_empty(self): + """R6, and the guard against resolving an owner for a name nobody owns. + + An id no member provides has no owner, so there is nothing to dispatch + it to and this gateway answers. What it answers is a metadata-only + reading with no payload - the sampler reports a topic it cannot find as + one nobody is publishing - which is what makes it distinguishable from + an id that exists, and that pair is asserted together here rather than + as a status on its own. + + The serving entity is asserted because the failure this guards is + specific: an owner resolved for a name nobody owns forwards a typo to a + peer, and the answer then comes back naming that peer's member. + """ + missing = requests.get( + f'{PRIMARY_URL}/functions/{MERGED_FUNCTION}/data/' + f'{quote("chassis/brakes/no_such_reading", safe="")}', + timeout=15, + ) + self.assertEqual(missing.status_code, 200, missing.text) + missing_body = missing.json() + self.assertEqual( + missing_body.get('x-medkit', {}).get('status'), 'metadata_only', + f'a topic no member provides reported data: {missing_body}', + ) + self.assertFalse( + missing_body.get('data'), + f'a topic no member provides came back with a payload: {missing_body}', + ) + self.assertEqual( + missing_body.get('x-medkit', {}).get('entity_id'), MERGED_FUNCTION, + f'an id nobody owns was dispatched to a member: {missing_body}', + ) + + present = requests.get( + f'{PRIMARY_URL}/functions/{MERGED_FUNCTION}/data/' + f'{quote(PEER_ONLY_TOPIC.lstrip("/"), safe="")}', + timeout=15, + ) + self.assertEqual(present.status_code, 200, present.text) + self.assertEqual( + present.json().get('x-medkit', {}).get('status'), 'data', + f'an id that exists is not distinguishable from one that does not: ' + f'{present.text}', + ) + + def test_a_suppressed_data_response_omits_the_peers_topics(self): + """The loop-suppression guard on ``/data``, for the reason it was written. + + Suppression means the peers were never asked. Reporting their topics + anyway - out of the copies this gateway holds from their last report - + is what turns a bidirectionally peered pair into a bounce, because the + header exists precisely to make one hop terminal. The peer's topic is + addressed by name rather than by counting items, so a response that + merely got shorter cannot pass this. + """ + response = requests.get( + f'{PRIMARY_URL}/functions/{MERGED_FUNCTION}/data', + headers={'X-Medkit-No-Fan-Out': '1'}, + timeout=10, + ) + self.assertEqual(response.status_code, 200, response.text) + items = response.json().get('items', []) + topics = [item.get('x-medkit', {}).get('ros2', {}).get('topic') for item in items] + self.assertNotIn( + PEER_ONLY_TOPIC, topics, + f'a suppressed response reported a peer-owned topic: {topics}', + ) + self.assertIn( + LOCAL_ONLY_TOPIC, topics, + f"suppression dropped this gateway's own topics too: {topics}", + ) + def test_a_compound_operation_id_runs_on_the_members_gateway(self): """R4 for an operation, asserted on the result rather than the status. @@ -2555,6 +2842,78 @@ def test_z6b_a_configuration_read_of_a_silent_peer_owned_member_says_not_respond self.assertEqual( body.get('parameters', {}).get('member_id'), 'peer_calibration', body) + def test_z6c_a_bare_data_id_of_a_silent_peer_owned_member_says_not_responding(self): + """R10 for the bare form of the dispatch path, for the reason z6a exists. + + A bare id resolves its owner from the tree, and the tree keeps a + declared member after its gateway stops answering - so the id still + resolves, and the answer is that the member cannot be reached. The two + wrong answers are both plausible: forwarding to a socket that is gone + gives 502, which says THIS gateway broke, and dropping the owner gives a + local sample of a topic that is not on this graph, which is a 200 with + an empty body. Both are asserted against explicitly, because a bare + `assertEqual(504)` reads the same whichever one arrives. + """ + response = requests.get( + f'{PRIMARY_URL}/functions/{MERGED_FUNCTION}/data/' + f'{quote(PEER_ONLY_TOPIC.lstrip("/"), safe="")}', + timeout=15, + ) + self.assertNotEqual( + response.status_code, 502, + f'a silent peer was forwarded to instead of answered for: {response.text}', + ) + self.assertNotEqual( + response.status_code, 200, + f'a topic on a silent gateway was sampled here and reported as read: ' + f'{response.text}', + ) + self.assertEqual(response.status_code, 504, response.text) + body = response.json() + self.assertEqual(body.get('error_code'), 'not-responding', body) + self.assertEqual( + body.get('parameters', {}).get('member_id'), PEER_ONLY_TOPIC_APP, body) + + def test_z6d_a_retained_member_keeps_the_topics_it_reported(self): + """R10 for ``/data``, and the reason the retained copy is kept at all. + + A topic the tree only knows from a peer's report is held here so that + ownership can be settled without asking anyone. While the peer answers + the owner's own copy is the one listed - it carries the message type and + the sample metadata this gateway has no way to produce - so the held copy + is emitted only once the fan-out has come back without it. Dropped + instead, the collection would lose a member's items the moment a link + went down, which is the shape change retention exists to prevent. + + Counted per copy, because "still listed" and "listed once" are different + claims and both have to hold. + """ + items = self._items(f'functions/{MERGED_FUNCTION}', 'data') + copies = [ + item for item in items + if item.get('x-medkit', {}).get('ros2', {}).get('topic') == PEER_ONLY_TOPIC + ] + self.assertEqual( + len(copies), 1, + f'a retained member forgot the topic it last reported, or reported it ' + f'twice: {[item.get("id") for item in items]}', + ) + # And the retained copy says it cannot be served, so a client can tell + # "declared, unreachable" from "publishing right now". + self.assertIs( + copies[0].get('x-medkit', {}).get('available'), False, + f'the retained topic does not report itself unavailable: {copies[0]}', + ) + local = [ + item for item in items + if item.get('x-medkit', {}).get('ros2', {}).get('topic') == LOCAL_ONLY_TOPIC + ] + self.assertEqual(len(local), 1, f'the local half vanished: {local}') + self.assertNotEqual( + local[0].get('x-medkit', {}).get('available'), False, + f'a locally owned topic was marked unavailable: {local[0]}', + ) + def test_z7_suppression_omits_the_peer_without_losing_ambiguity(self): """The loop-suppression guard, checked for the reason it was written. From b5d18f371944d43a0440d2bcbeeb0ebe72078ced Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Thu, 20 Aug 2026 21:10:51 +0200 Subject: [PATCH 22/22] test(aggregation): prove a peer that comes back is taken back Retention is well covered in one direction only. Every case that watches a peer go quiet leaves it quiet, so nothing observed the return trip, and the design doc's promise that a recovering peer is re-included rested on no test at all. A regression that kept replaying the retained copy would have poisoned the tree after a single outage with every suite still green. A peer is killed and then replaced through the same gate the trigger suite uses, so the return happens when the case asks for it rather than on a clock. The declared member's flag is watched down and back up, the runtime-discovered one disappears and is merged again, a read that answered not-responding answers with the peer's own payload, and the retained declaration does not linger beside the live copy. The two availability signals do not clear together, and the case pins that rather than papering over it: reachability clears on the first refresh after the health check passes, while is_online is the peer's own account of its graph and follows a refresh or two later. --- docs/config/aggregation.rst | 9 + .../design/aggregation.rst | 10 +- .../CMakeLists.txt | 2 + .../test/features/test_peer_recovery.test.py | 955 ++++++++++++++++++ 4 files changed, 975 insertions(+), 1 deletion(-) create mode 100644 src/ros2_medkit_integration_tests/test/features/test_peer_recovery.test.py diff --git a/docs/config/aggregation.rst b/docs/config/aggregation.rst index 246f4e491..af92836f2 100644 --- a/docs/config/aggregation.rst +++ b/docs/config/aggregation.rst @@ -465,6 +465,15 @@ What clients see then depends on the peer's health check: incomplete refresh is logged at ``WARN``. Availability is untouched - the peer can still be reached; this gateway merely failed to read all of it. +A peer that starts answering again is read again on the next refresh, and a +retained declaration is replayed only for a peer that could not be read - so +the live answer replaces the retained one rather than being merged beside it. +``x-medkit.available`` clears on that refresh, and the entities the peer only +discovered at runtime, dropped while it was silent, are merged again with it. +``x-medkit.is_online`` is the peer's own account of an App rather than a +statement about the link, so after a gateway restart it stays ``false`` until +that gateway has relinked its ROS graph. + Two statuses are read rather than treated as failures: - ``404`` on a nested collection route means the peer runs a gateway version diff --git a/src/ros2_medkit_gateway/design/aggregation.rst b/src/ros2_medkit_gateway/design/aggregation.rst index 0a89bc02a..f73d00a44 100644 --- a/src/ros2_medkit_gateway/design/aggregation.rst +++ b/src/ros2_medkit_gateway/design/aggregation.rst @@ -694,7 +694,15 @@ GETs ``/api/v1/health`` on its peer. If the health check fails, the peer is marked unhealthy and excluded from fan-out queries and entity fetching. When a peer recovers (health check succeeds again), it is automatically -re-included. +re-included: the next refresh fetches it like any other healthy peer, and since +a retained declaration is replayed only for a peer that could not be read that +cycle, the live answer replaces the retained one wholesale rather than being +merged beside it. ``x-medkit.available`` therefore clears on that same refresh, +and the entities the peer only discovered at runtime - dropped while it was +silent - are merged again with it. ``x-medkit.is_online`` is read off the wire +as the peer's own account of an App, not as a statement about the link, so an +App on a gateway that has just restarted stays ``false`` for however many +refreshes that gateway needs to relink its ROS graph, and then turns true. ``PeerClient::fetch_entities()`` reads a peer over several requests and either describes it whole or reports failure: a dead connection, a status a route has diff --git a/src/ros2_medkit_integration_tests/CMakeLists.txt b/src/ros2_medkit_integration_tests/CMakeLists.txt index a4188d658..82dd615a4 100644 --- a/src/ros2_medkit_integration_tests/CMakeLists.txt +++ b/src/ros2_medkit_integration_tests/CMakeLists.txt @@ -255,6 +255,7 @@ if(BUILD_TESTING) # four where two are used takes two away from whatever the runner could # otherwise have started alongside it. set(_TWO_GATEWAY_TESTS + test_peer_recovery test_triggers_restore_before_discovery) set(_TWO_GATEWAY_DOMAINS 2) @@ -292,6 +293,7 @@ if(BUILD_TESTING) test_graph_provider_greenwave 300 test_graph_provider_stale 300 test_graph_provider_sse 300 + test_peer_recovery 300 test_triggers_restore_before_discovery 300) # Names actually matched against a discovered test_name in the two loops # below. Checked against _MEDKIT_TEST_TIMEOUT_OVERRIDES itself after both diff --git a/src/ros2_medkit_integration_tests/test/features/test_peer_recovery.test.py b/src/ros2_medkit_integration_tests/test/features/test_peer_recovery.test.py new file mode 100644 index 000000000..7aedd42e2 --- /dev/null +++ b/src/ros2_medkit_integration_tests/test/features/test_peer_recovery.test.py @@ -0,0 +1,955 @@ +#!/usr/bin/env python3 +# Copyright 2026 bburda +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Feature test: an aggregated tree recovers when a silent peer answers again. + +Retention is what makes an outage survivable. A peer's manifest-declared +entities stay in the merged tree marked unreachable and keep the items they +last reported, while the entities that peer only discovered at runtime drop +out, because they describe a graph this gateway can no longer observe. + +Recovery is the other end of that same mechanism and it has no code of its own. +``mark_unreachable`` is one-way - it only ever sets ``available`` false, and +nothing ever sets it back. A peer that answers its health check again is simply +fetched again, and the live answer supersedes the retained declaration because +retained copies are replayed only for peers that could not be read this cycle. + +Which is precisely why it needs a case of its own. If a refresh ever preferred +the retained copy over a live one - a merge order swapped, a retained entry +never dropped - one transient outage would poison the tree for the life of the +process: every entity behind that peer frozen as unreachable, every read of one +answering ``504`` forever. Nothing else in the suite restarts a peer, so every +other case measures either a healthy pair or a peer that stays dead, and both +stay green through that regression. + +WHAT IS CHECKED, in the order the cases run + + 1 The pair merges, and a read of a peer-owned member is served by the peer. + 2 The peer is killed; a DECLARED member reports ``x-medkit.available: false``. + 3 A member the peer only discovered at runtime is gone from the merged tree. + 4 A read addressed to a peer-owned member answers ``504 not-responding``. + 5 The replacement peer answers; the declared member reports itself reachable + again - the flag is absent, which is how a reachable entity is emitted. + 6 The runtime-discovered member is merged again. + 7 The read that answered 504 answers 200, with the peer's own payload. + 8 Nothing lingers twice: the retained declaration is gone rather than sitting + beside the live copy, under its own id or a collision-renamed one. + +Case 5 is only evidence because case 2 watched the flag go false first, and +case 7 only because case 4 watched that same URL fail. Every case therefore +asserts that its predecessor recorded what it depends on, so a case that broke +earlier cannot leave a later one quietly proving nothing. + +The peer is REPLACED rather than resurrected. A gate process waits on a file +the test writes, and its exit starts a second gateway carrying the first one's +configuration on the same port and the same DDS domain. The replacement +appears when the outage has been observed, not on a clock, so the window in +which the peer is provably gone is as long as the test says it is. +""" + +import os +import signal +import tempfile +import time +import unittest +from urllib.parse import quote + +from launch import LaunchDescription +from launch.actions import ( + ExecuteProcess, + RegisterEventHandler, + SetEnvironmentVariable, + TimerAction, +) +from launch.event_handlers import OnProcessExit +import launch_testing +import launch_testing.actions +import requests +from ros2_medkit_test_utils.constants import ( + ALLOWED_EXIT_CODES, + API_BASE_PATH, + get_test_domain_id, + get_test_port, +) +from ros2_medkit_test_utils.launch_helpers import ( + create_demo_nodes, + create_gateway_node, +) + +PRIMARY_PORT = get_test_port(0) +PEER_PORT = get_test_port(1) +PRIMARY_URL = f'http://localhost:{PRIMARY_PORT}{API_BASE_PATH}' +PEER_URL = f'http://localhost:{PEER_PORT}{API_BASE_PATH}' + +PRIMARY_DOMAIN_ID = get_test_domain_id(0) +PEER_DOMAIN_ID = get_test_domain_id(1) + +# The name the aggregator files this peer under. It is also the prefix a +# collision rename would carry, which is what case 8 looks for. +PEER_NAME = 'secondary_gateway' + +MERGED_AREA = 'vehicle' +MERGED_FUNCTION = 'vehicle_health' +PARENT_COMPONENT = 'vehicle-ecu' +PEER_SUBCOMPONENT = 'brake-ecu' + +# A member on this gateway's own graph. It is the control for every case that +# asserts something about the peer's half: whatever the link does, this one is +# served here and never becomes unreachable. +LOCAL_APP = 'temp_sensor' +LOCAL_TOPIC = '/powertrain/engine/temperature' + +# The peer's DECLARED member. Its manifest entry is what outlives the link, so +# it is the entity whose availability flag can flip in both directions, and the +# one whose topic the dispatch cases address. +PEER_DECLARED_APP = 'pressure_sensor' +PEER_TOPIC = '/chassis/brakes/pressure' + +# A node the peer's manifest does NOT declare. The peer's policy is `warn`, so +# it exposes the node as a heuristic App; retention drops it when the link goes +# down, and only a completed fetch can bring it back. That makes it the half of +# the tree that cannot be faked by a stale copy. +PEER_RUNTIME_NODE = 'rpm_sensor' +PEER_RUNTIME_APP = 'rpm_sensor' + +# The gate the test opens to start the peer's replacement. +GATE_REPLACEMENT = os.path.join( + tempfile.gettempdir(), + f'test_peer_recovery_replacement_{os.getpid()}', +) + +# Detection and re-inclusion both ride the discovery refresh, 1000 ms for a +# test gateway, so both are seconds rather than tens of seconds. The budgets +# are wide enough that a loaded machine does not fail the case, and the +# measured latency is printed so a regression that pushes either towards the +# 30 s production default is visible rather than merely slow. +OUTAGE_TIMEOUT = 60.0 +RECOVERY_TIMEOUT = 90.0 +STARTUP_TIMEOUT = 90.0 + +# PIDs this test killed on purpose, so the post-shutdown exit-code check can +# tell a process the test destroyed from one that died on its own. +_KILLED_PIDS = set() + +PRIMARY_MANIFEST = f"""\ +manifest_version: "1.0" +metadata: + name: "Primary ECU" + version: "1.0.0" +config: + unmanifested_nodes: ignore +areas: + - id: {MERGED_AREA} + name: "Vehicle" +components: + - id: {PARENT_COMPONENT} + name: "Vehicle ECU" + area: {MERGED_AREA} +apps: + - id: {LOCAL_APP} + name: "Engine Temperature Sensor" + is_located_on: {PARENT_COMPONENT} + ros_binding: + node_name: temp_sensor + namespace: /powertrain/engine +functions: + - id: {MERGED_FUNCTION} + name: "Vehicle Health Monitoring" + category: monitoring + hosted_by: + - {LOCAL_APP} +""" + +PEER_MANIFEST = f"""\ +manifest_version: "1.0" +metadata: + name: "Secondary ECU" + version: "1.0.0" +config: + # The peer exposes what it did not declare, so its half of the tree has both + # origins in it. Retention keeps the declared entity and drops the other, and + # a peer that declares everything it runs cannot show the difference. + unmanifested_nodes: warn +areas: + - id: {MERGED_AREA} + name: "Vehicle" +components: + # The parent is declared on both gateways because a subcomponent may not name + # a parent absent from its own manifest - the validator rejects that as an + # error, and an errored manifest is not loaded, so the peer would contribute + # nothing at all. + - id: {PARENT_COMPONENT} + name: "Vehicle ECU" + area: {MERGED_AREA} + - id: {PEER_SUBCOMPONENT} + name: "Brake ECU" + area: {MERGED_AREA} + parent_component_id: {PARENT_COMPONENT} +apps: + - id: {PEER_DECLARED_APP} + name: "Brake Pressure Sensor" + is_located_on: {PEER_SUBCOMPONENT} + ros_binding: + node_name: pressure_sensor + namespace: /chassis/brakes +functions: + - id: {MERGED_FUNCTION} + name: "Vehicle Health Monitoring" + category: monitoring + hosted_by: + - {PEER_DECLARED_APP} +""" + + +def _write_manifest(content): + """Write manifest YAML to a temporary file and return its path.""" + fd, path = tempfile.mkstemp(suffix='.yaml', prefix='test_peer_recovery_manifest_') + with os.fdopen(fd, 'w') as handle: + handle.write(content) + return path + + +def _gate_process(name, path): + """Return a process that exits once ``path`` exists.""" + return ExecuteProcess( + cmd=['sh', '-c', f'while [ ! -e "{path}" ]; do sleep 0.2; done'], + name=name, + output='screen', + ) + + +def _open_gate(path): + """Write a gate file, releasing the process that waits on it.""" + with open(path, 'w', encoding='utf-8') as gate: + gate.write('open') + + +def _remove_gate(): + """Drop the gate file so a rerun does not inherit an open gate.""" + if os.path.exists(GATE_REPLACEMENT): + try: + os.unlink(GATE_REPLACEMENT) + except OSError: + pass + + +def _peer_gateway_params(manifest_path): + """Gateway parameters shared by the peer and by its replacement. + + The replacement is the same gateway again, not a similar one: same port, + same manifest, same DDS domain. Anything that differed here would leave the + recovery it demonstrates ambiguous. + """ + return { + 'discovery.mode': 'hybrid', + 'discovery.manifest_path': manifest_path, + 'discovery.manifest_strict_validation': False, + } + + +def generate_test_description(): + """Launch an aggregator, a peer, and the peer's gated replacement.""" + primary_manifest_path = _write_manifest(PRIMARY_MANIFEST) + peer_manifest_path = _write_manifest(PEER_MANIFEST) + + peer_domain_env = {'ROS_DOMAIN_ID': str(PEER_DOMAIN_ID)} + peer_params = _peer_gateway_params(peer_manifest_path) + + primary_gateway = create_gateway_node( + port=PRIMARY_PORT, + extra_params={ + 'discovery.mode': 'hybrid', + 'discovery.manifest_path': primary_manifest_path, + 'discovery.manifest_strict_validation': False, + 'aggregation.enabled': True, + 'aggregation.timeout_ms': 5000, + 'aggregation.announce': False, + 'aggregation.discover': False, + 'aggregation.peer_urls': [f'http://localhost:{PEER_PORT}'], + 'aggregation.peer_names': [PEER_NAME], + }, + ) + + peer_gateway = create_gateway_node( + name='secondary_gateway_node', + port=PEER_PORT, + extra_params=peer_params, + extra_env=peer_domain_env, + ) + + # Distinct ROS node name only so the two processes are separable in the + # launch output and in proc_info. The first is dead before this one starts. + replacement_gateway = create_gateway_node( + name='secondary_gateway_node_replacement', + port=PEER_PORT, + extra_params=peer_params, + extra_env=peer_domain_env, + ) + + replacement_gate = _gate_process('peer_replacement_gate', GATE_REPLACEMENT) + + delayed = TimerAction( + period=2.0, + actions=( + create_demo_nodes([LOCAL_APP], lidar_faulty=False) + + create_demo_nodes( + [PEER_DECLARED_APP, PEER_RUNTIME_NODE], + lidar_faulty=False, + extra_env=peer_domain_env, + ) + ), + ) + + launch_description = LaunchDescription([ + SetEnvironmentVariable('ROS_DOMAIN_ID', str(PRIMARY_DOMAIN_ID)), + primary_gateway, + peer_gateway, + replacement_gate, + RegisterEventHandler( + OnProcessExit(target_action=replacement_gate, on_exit=[replacement_gateway]), + ), + delayed, + launch_testing.actions.ReadyToTest(), + ]) + + return ( + launch_description, + { + 'gateway_node': primary_gateway, + 'peer_gateway': peer_gateway, + 'replacement_gateway': replacement_gateway, + }, + ) + + +def _wait_for_health(base_url, *, timeout): + """Poll ``/health`` until it answers 200, or fail.""" + deadline = time.monotonic() + timeout + last = None + while time.monotonic() < deadline: + try: + response = requests.get(f'{base_url}/health', timeout=2) + if response.status_code == 200: + return + last = response.status_code + except requests.exceptions.RequestException as exc: + last = repr(exc) + time.sleep(0.25) + raise AssertionError(f'{base_url} was not healthy within {timeout}s (last: {last})') + + +def _poll(predicate, *, timeout, interval=0.25): + """Call ``predicate`` until it answers something; return that, or None. + + ``None`` from the predicate means "not yet"; every other value is the + answer. Falsiness deliberately does not mean "not yet": a + ``requests.Response`` is falsy for any status at or above 400, so a + predicate handing back the 504 a case is waiting for would otherwise be + polled straight past until the budget ran out. + + Polling rather than sleeping is what keeps the budgets above from becoming + the thing under test: a case finishes as soon as the aggregator has caught + up, and only a gateway that never catches up spends the whole budget. + """ + deadline = time.monotonic() + timeout + while True: + value = predicate() + if value is not None: + return value + if time.monotonic() >= deadline: + return None + time.sleep(interval) + + +class PeerRecoveryTest(unittest.TestCase): + """Drives the aggregating gateway; the peer is only ever used to verify.""" + + #: True once a case has watched the declared member go unavailable. + _outage_observed = False + #: Seconds the aggregator took to notice the peer was gone. + _noticed_after_s = None + #: Seconds the aggregator took to re-include the replacement. + _recovered_after_s = None + #: True once a case has watched the peer-owned read answer 504. + _read_refused_while_down = False + #: The peer's own answer for the addressed topic, read while it was healthy. + _peer_payload_keys = None + + @classmethod + def setUpClass(cls): + """Wait until both gateways answer and the peer's half has merged. + + The merge is driven by HTTP from the peer while the local ROS graph is + still binding Apps to nodes, so a case that starts on the first + successful response can be reading a tree that is still filling in. + """ + cls.addClassCleanup(_remove_gate) + _remove_gate() + + _wait_for_health(PRIMARY_URL, timeout=STARTUP_TIMEOUT) + _wait_for_health(PEER_URL, timeout=STARTUP_TIMEOUT) + + wanted = {LOCAL_APP, PEER_DECLARED_APP, PEER_RUNTIME_APP} + merged = _poll( + lambda: True if cls._app_ids_seen_by_primary() >= wanted else None, + timeout=STARTUP_TIMEOUT, + ) + if not merged: + raise AssertionError( + f'the peer half never merged; the aggregator lists ' + f'{sorted(cls._app_ids_seen_by_primary())}' + ) + + served = _poll( + lambda: True if cls._aggregate_read_of_peer_topic().status_code == 200 else None, + timeout=STARTUP_TIMEOUT, + ) + if not served: + raise AssertionError( + 'the aggregator never served the peer-owned topic while the peer ' + 'was healthy, so the outage cases would prove nothing' + ) + + # ------------------------------------------------------------------ + # Reading the merged tree + # ------------------------------------------------------------------ + + @staticmethod + def _primary_items(collection): + """Every item of a top-level collection as the aggregator sees it.""" + response = requests.get(f'{PRIMARY_URL}/{collection}', timeout=10) + if response.status_code != 200: + return [] + return response.json().get('items', []) + + @classmethod + def _app_ids_seen_by_primary(cls): + """Return the set of App ids in the merged tree.""" + return {item.get('id') for item in cls._primary_items('apps')} + + @classmethod + def _primary_apps_named(cls, app_id): + """Every copy of one App id in the merged tree, so copies can be counted.""" + return [item for item in cls._primary_items('apps') if item.get('id') == app_id] + + @classmethod + def _primary_app(cls, app_id): + """One App as the aggregator currently sees it, or None.""" + found = cls._primary_apps_named(app_id) + return found[0] if found else None + + @staticmethod + def _primary_subcomponents(parent_id, subcomponent_id): + """Every copy of one subcomponent id under a parent Component.""" + response = requests.get( + f'{PRIMARY_URL}/components/{parent_id}/subcomponents', timeout=10) + if response.status_code != 200: + return [] + return [ + item for item in response.json().get('items', []) + if item.get('id') == subcomponent_id + ] + + @staticmethod + def _function_items(collection): + """Items of a resource collection on the merged Function.""" + response = requests.get( + f'{PRIMARY_URL}/functions/{MERGED_FUNCTION}/{collection}', timeout=15) + if response.status_code != 200: + return [] + return response.json().get('items', []) + + @staticmethod + def _aggregate_read_of_peer_topic(): + """Read the peer-owned topic through the merged Function. + + The compound form names the member, so the answer has to come from the + gateway that runs it. This one URL is used by every dispatch case here: + it is what answers 504 while the peer is down and what has to answer + with the peer's own sample once the peer is back. + """ + item_id = f'{PEER_DECLARED_APP}:{PEER_TOPIC.lstrip("/")}' + return requests.get( + f'{PRIMARY_URL}/functions/{MERGED_FUNCTION}/data/{quote(item_id, safe="")}', + timeout=15, + ) + + @staticmethod + def _peer_direct_read(): + """Read the same topic on the peer's own App route, on the peer itself.""" + return requests.get( + f'{PEER_URL}/apps/{PEER_DECLARED_APP}/data{PEER_TOPIC}', timeout=15) + + @staticmethod + def _peer_app_view(app_id): + """Return one App as the PEER itself lists it, or None.""" + response = requests.get(f'{PEER_URL}/apps', timeout=10) + if response.status_code != 200: + return None + for item in response.json().get('items', []): + if item.get('id') == app_id: + return item + return None + + @staticmethod + def _peer_source_of(app_id): + """Ask the PEER what origin it gives one of its Apps, or None. + + The aggregator overwrites ``source`` with ``peer:`` on arrival, so + the peer's own answer is the only place the manifest/runtime split is + visible. + """ + response = requests.get(f'{PEER_URL}/apps', timeout=10) + if response.status_code != 200: + return None + for item in response.json().get('items', []): + if item.get('id') == app_id: + return item.get('x-medkit', {}).get('source') + return None + + # ------------------------------------------------------------------ + # Cases + # ------------------------------------------------------------------ + + def test_01_both_gateways_answer_and_the_peer_half_is_merged(self): + """The baseline every later case is measured against. + + Two things are established here rather than assumed. The declared + member is reachable, so case 5 can claim the flag moved instead of + merely reporting where it ended up. And the peer really does call the + other member runtime-discovered, so case 3's disappearance is the rule + working rather than an entity that was never there. + """ + cls = type(self) + + declared = self._primary_app(PEER_DECLARED_APP) + self.assertIsNotNone( + declared, f'{PEER_DECLARED_APP} was not merged while the peer answered') + self.assertNotEqual( + declared.get('x-medkit', {}).get('available'), False, + f'{PEER_DECLARED_APP} was already unavailable before the peer was killed: ' + f'{declared}', + ) + self.assertEqual( + self._peer_source_of(PEER_DECLARED_APP), 'manifest', + f'the peer does not call {PEER_DECLARED_APP} manifest-declared, so nothing ' + f'here is retained and the recovery this file measures is not the one ' + f'described', + ) + + runtime_source = self._peer_source_of(PEER_RUNTIME_APP) + self.assertIsNotNone( + runtime_source, + f'the peer does not expose {PEER_RUNTIME_APP} at all, so the ' + f'runtime-discovered half of this test is missing', + ) + self.assertNotEqual( + runtime_source, 'manifest', + f'the peer calls {PEER_RUNTIME_APP} manifest-declared, so it would be ' + f'retained through the outage and case 3 would prove nothing', + ) + self.assertIsNotNone( + self._primary_app(PEER_RUNTIME_APP), + f'{PEER_RUNTIME_APP} was not merged while the peer answered', + ) + + # And the peer answers for its own topic, so case 7's success has + # something to be compared against. A merged App is not a publishing + # one yet: the sample carries no data until the peer's node is up and + # has published, so this waits for the payload rather than reading once + # and calling an empty answer a failure. + def _read_carrying_data(): + reply = self._aggregate_read_of_peer_topic() + if reply.status_code != 200: + return None + payload = reply.json() + return payload if payload.get('data') else None + + body = _poll(_read_carrying_data, timeout=60.0) + self.assertIsNotNone( + body, 'the peer-owned topic never carried data while the peer answered') + self.assertEqual( + body.get('x-medkit', {}).get('entity_id'), PEER_DECLARED_APP, + f'the aggregating entity answered for a member it does not run: {body}', + ) + cls._peer_payload_keys = sorted(body['data'].keys()) + + def test_02_the_declared_half_reports_itself_unreachable_when_the_peer_dies( + self, peer_gateway): + """The outage, and the flag whose reverse trip case 5 checks for. + + Watching the flag go false here is what makes case 5 evidence: without + it, "the entity is reachable" describes a tree that never noticed + anything happened. + """ + cls = type(self) + self.assertIsNotNone( + cls._peer_payload_keys, + 'test_01 must establish the healthy baseline before test_02 runs', + ) + + pid = peer_gateway.process_details['pid'] + _KILLED_PIDS.add(pid) + os.kill(pid, signal.SIGKILL) + + def unavailable(): + app = self._primary_app(PEER_DECLARED_APP) + if app is not None and app.get('x-medkit', {}).get('available') is False: + return app + return None + + started = time.monotonic() + observed = _poll(unavailable, timeout=OUTAGE_TIMEOUT) + self.assertIsNotNone( + observed, + f'{PEER_DECLARED_APP} never reported itself unavailable within ' + f'{OUTAGE_TIMEOUT}s of its peer being killed; the aggregator now sees ' + f'{self._primary_app(PEER_DECLARED_APP)}', + ) + cls._noticed_after_s = time.monotonic() - started + cls._outage_observed = True + + self.assertIs( + observed.get('x-medkit', {}).get('is_online'), False, + f'a retained App still claims to be running: {observed}', + ) + print(f'[recovery] aggregator noticed the peer was gone in ' + f'{cls._noticed_after_s:.1f}s') + # Detection is bounded by one discovery refresh - 1000 ms for a test + # gateway - plus the failed health check that runs inside it. Bounded + # here so a regression that pushes it towards the 30 s production + # default is a failure rather than a slower green run. + self.assertLess( + cls._noticed_after_s, 30.0, + f'took {cls._noticed_after_s:.1f}s to notice a dead peer', + ) + + local = self._primary_app(LOCAL_APP) + self.assertIsNotNone(local, f'{LOCAL_APP} is local and must not vanish') + self.assertNotEqual( + local.get('x-medkit', {}).get('available'), False, + f'a locally owned App was marked unreachable by a peer outage: {local}', + ) + + def test_03_a_runtime_discovered_member_is_gone_while_the_peer_is_down(self): + """The half that is NOT retained, so its return is not a stale replay. + + A retained copy could satisfy case 6 on its own. This case rules that + out by requiring the entity to be absent first: whatever brings it back + has to be a completed fetch from a gateway that is answering again. + """ + self.assertTrue( + self._outage_observed, + 'test_02 must observe the outage before test_03 runs', + ) + vanished = _poll( + lambda: True if self._primary_app(PEER_RUNTIME_APP) is None else None, + timeout=OUTAGE_TIMEOUT, + ) + self.assertTrue( + vanished, + f'{PEER_RUNTIME_APP} was runtime-discovered on the peer and must not ' + f'outlive the link that reported it: ' + f'{self._primary_app(PEER_RUNTIME_APP)}', + ) + + def test_04_a_peer_owned_read_says_not_responding_while_the_peer_is_down(self): + """The request that has to start working again, failing first. + + Case 7 drives this same URL. Recording the refusal here is what lets + that case claim a request recovered, rather than reporting that a + request works - which it also did before anything was killed. + """ + cls = type(self) + self.assertTrue( + self._outage_observed, + 'test_02 must observe the outage before test_04 runs', + ) + + def refused(): + answer = self._aggregate_read_of_peer_topic() + return answer if answer.status_code != 200 else None + + response = _poll(refused, timeout=OUTAGE_TIMEOUT) + if response is None: + last = self._aggregate_read_of_peer_topic() + self.fail( + f'a topic on a dead gateway was still being served as a successful ' + f'read for {OUTAGE_TIMEOUT}s: {last.text}' + ) + self.assertNotEqual( + response.status_code, 502, + f'a silent peer was forwarded to instead of answered for: {response.text}', + ) + self.assertEqual(response.status_code, 504, response.text) + body = response.json() + self.assertEqual(body.get('error_code'), 'not-responding', body) + self.assertEqual( + body.get('parameters', {}).get('member_id'), PEER_DECLARED_APP, body) + cls._read_refused_while_down = True + + def test_05_a_declared_member_is_reachable_again_once_the_peer_returns(self): + """The promise: a peer that answers again is re-included automatically. + + Nothing resets the availability flag, so the only thing that can clear + it is a completed fetch replacing the retained declaration wholesale. A + merge that preferred the retained copy would leave this entity + unreachable forever after one outage, with every other case in the + suite still green. + + A reachable entity is emitted with no ``available`` field at all, so the + x-medkit block is read rather than defaulted: "no key" is only evidence + when there is a block that could have carried one. + """ + cls = type(self) + self.assertTrue( + cls._outage_observed, + 'test_02 must watch the flag go false before test_05 can claim it moved', + ) + + _open_gate(GATE_REPLACEMENT) + _wait_for_health(PEER_URL, timeout=RECOVERY_TIMEOUT) + + def reachable(): + app = self._primary_app(PEER_DECLARED_APP) + if app is not None and app.get('x-medkit', {}).get('available') is not False: + return app + return None + + started = time.monotonic() + recovered = _poll(reachable, timeout=RECOVERY_TIMEOUT) + self.assertIsNotNone( + recovered, + f'{PEER_DECLARED_APP} was still unreachable {RECOVERY_TIMEOUT}s after its ' + f'peer started answering again: {self._primary_app(PEER_DECLARED_APP)}', + ) + cls._recovered_after_s = time.monotonic() - started + print(f'[recovery] aggregator re-included the peer in ' + f'{cls._recovered_after_s:.1f}s') + + x_medkit = recovered.get('x-medkit', {}) + self.assertTrue( + x_medkit, + f'{PEER_DECLARED_APP} carries no x-medkit block, so an absent ' + f'`available` proves nothing: {recovered}', + ) + self.assertNotIn( + 'available', x_medkit, + f'a reachable entity still emits an availability flag: {recovered}', + ) + # Re-inclusion rides the same refresh as detection did. Reported so a + # regression that pushes either towards the 30 s production default + # shows up as a number rather than as a slow test. + self.assertLess( + cls._recovered_after_s, 30.0, + f'took {cls._recovered_after_s:.1f}s to re-include a peer that answers', + ) + + # The second availability signal, waited for separately because it does + # not come back at the same moment. Reachability is settled by the + # peer's health check, but `is_online` is the peer's own account of + # whether that App is bound to a running node - and a gateway that has + # just started answering has not finished linking its ROS graph yet, so + # it truthfully reports the App as offline for a refresh or two. What + # the rule requires is that the aggregator ends up carrying what the + # peer says, rather than the false that retention wrote over it. + def online(): + app = self._primary_app(PEER_DECLARED_APP) + if app is not None and app.get('x-medkit', {}).get('is_online') is True: + return app + return None + + back_online = _poll(online, timeout=RECOVERY_TIMEOUT) + self.assertIsNotNone( + back_online, + f'{PEER_DECLARED_APP} never reported itself running again, while the peer ' + f'itself says {self._peer_app_view(PEER_DECLARED_APP)}', + ) + self.assertIs( + self._peer_app_view(PEER_DECLARED_APP).get('x-medkit', {}).get('is_online'), + True, + 'the peer does not consider its own App online, so the aggregator ' + 'agreeing with it proves nothing', + ) + + def test_06_a_runtime_discovered_member_is_merged_again(self): + """The half a retained copy cannot account for. + + This entity was dropped when the link went down, so its presence now is + a fetch that completed against a live peer - which is the difference + between a tree that recovered and a tree that is replaying what it + remembers. + """ + self.assertIsNotNone( + self._recovered_after_s, + 'test_05 must see the peer re-included before test_06 runs', + ) + merged = _poll( + lambda: self._primary_app(PEER_RUNTIME_APP), + timeout=RECOVERY_TIMEOUT, + ) + self.assertIsNotNone( + merged, + f'{PEER_RUNTIME_APP} was discovered on the peer, dropped with the link, ' + f'and never came back; the aggregator lists ' + f'{sorted(self._app_ids_seen_by_primary())}', + ) + self.assertNotEqual( + merged.get('x-medkit', {}).get('available'), False, + f'a freshly fetched entity arrived marked unreachable: {merged}', + ) + + def test_07_a_peer_owned_read_succeeds_again_and_the_peer_answered_it(self): + """The request that answered 504 answers with the member's own sample. + + Status alone cannot show this. The failure that matters is a 200 with + an empty body - a local sample of a topic this gateway cannot see - so + the answer is compared against the peer's own read of the same topic on + the peer's own route, and it has to name the member as the entity that + produced it. + """ + self.assertTrue( + self._read_refused_while_down, + 'test_04 must watch this URL fail before test_07 can claim it recovered', + ) + + def served(): + answer = self._aggregate_read_of_peer_topic() + return answer if answer.status_code == 200 else None + + response = _poll(served, timeout=RECOVERY_TIMEOUT) + self.assertIsNotNone( + response, + f'a read of {PEER_DECLARED_APP} never recovered after its peer came back; ' + f'last answer was {self._aggregate_read_of_peer_topic().text}', + ) + + body = response.json() + self.assertEqual( + body.get('x-medkit', {}).get('status'), 'data', + f'the read reported success with no data from the peer member: {body}', + ) + self.assertTrue(body.get('data'), f'the peer member returned an empty payload: {body}') + self.assertEqual( + body.get('x-medkit', {}).get('entity_id'), PEER_DECLARED_APP, + f'the aggregating entity answered for a member it does not run: {body}', + ) + self.assertEqual( + body.get('x-medkit', {}).get('ros2', {}).get('topic'), PEER_TOPIC, + f'the answer names a topic the member does not publish: {body}', + ) + + direct = self._peer_direct_read() + self.assertEqual(direct.status_code, 200, direct.text) + direct_body = direct.json() + self.assertEqual( + body.get('x-medkit', {}).get('ros2', {}).get('type'), + direct_body.get('x-medkit', {}).get('ros2', {}).get('type'), + f'the answer is not the message the member publishes: {body}', + ) + self.assertEqual( + sorted(body['data'].keys()), sorted(direct_body['data'].keys()), + f"the payload is not shaped like the member's own: {body}", + ) + self.assertEqual( + sorted(body['data'].keys()), self._peer_payload_keys, + f'the recovered payload is not shaped like the one the same read ' + f'returned before the outage: {body}', + ) + + def test_08_the_retained_declaration_does_not_linger_beside_the_live_copy(self): + """Recovery is a replacement, not an addition. + + A retained entry that is merged rather than superseded shows up either + as the same id twice or - since the merge renames a leaf two + contributors claim - as a peer-prefixed second copy. Both are counted, + because "still listed" and "listed once" are different claims and the + tree is only correct when both hold. + """ + self.assertIsNotNone( + self._recovered_after_s, + 'test_05 must see the peer re-included before test_08 runs', + ) + + copies = self._primary_apps_named(PEER_DECLARED_APP) + self.assertEqual( + len(copies), 1, + f'{PEER_DECLARED_APP} is listed {len(copies)} times after its peer came ' + f'back: {copies}', + ) + + renamed = f'{PEER_NAME}__{PEER_DECLARED_APP}' + self.assertEqual( + self._primary_apps_named(renamed), [], + f'the retained declaration was merged alongside the live copy and renamed ' + f'to {renamed}', + ) + + subcomponents = self._primary_subcomponents(PARENT_COMPONENT, PEER_SUBCOMPONENT) + self.assertEqual( + len(subcomponents), 1, + f'{PEER_SUBCOMPONENT} is listed {len(subcomponents)} times under ' + f'{PARENT_COMPONENT} after its peer came back: {subcomponents}', + ) + self.assertNotEqual( + subcomponents[0].get('x-medkit', {}).get('available'), False, + f'a re-included Component still reports itself unreachable: {subcomponents[0]}', + ) + + # And one level down: the items the retained copy carried through the + # outage are not sitting beside the ones the live peer just reported. + peer_topics = [ + item for item in self._function_items('data') + if item.get('x-medkit', {}).get('ros2', {}).get('topic') == PEER_TOPIC + ] + self.assertEqual( + len(peer_topics), 1, + f'{PEER_TOPIC} is offered {len(peer_topics)} times after its peer came ' + f'back: {[item.get("id") for item in peer_topics]}', + ) + self.assertNotEqual( + peer_topics[0].get('x-medkit', {}).get('available'), False, + f'a re-included topic still reports itself unavailable: {peer_topics[0]}', + ) + local_topics = [ + item for item in self._function_items('data') + if item.get('x-medkit', {}).get('ros2', {}).get('topic') == LOCAL_TOPIC + ] + self.assertEqual( + len(local_topics), 1, + f'the local half of the Function changed shape across the outage: ' + f'{local_topics}', + ) + + +@launch_testing.post_shutdown_test() +class TestShutdown(unittest.TestCase): + + def test_exit_codes(self, proc_info): + """Check all processes exited cleanly. + + The peer this test kills is allowed to report SIGKILL, and only that + one: it is matched by the pid the test killed, so a different process + dying that way is still a failure. + """ + for info in proc_info: + allowed = set(ALLOWED_EXIT_CODES) + if info.pid in _KILLED_PIDS: + allowed.add(-9) + self.assertIn( + info.returncode, allowed, + f'{info.process_name} exited with code {info.returncode}', + ) + _remove_gate()