Sensors fixes for state, restart configurations, death trigger emit

CHANGELOG.rst

@@ -36,6 +36,22 @@ Changed
 Added
 ~~~~~
 * added raw_string type to allow template strings to pass through variable processing (by @guzzijones12@gmail.com) #6351
+* Added a new internal ``core.st2.sensor.process_abandoned`` trigger which is emitted when the sensor
+  container permanently gives up on a sensor after exceeding the maximum number of respawn attempts.
+  This lets users define their own alerting (email, Slack, workflow, etc.) via a rule instead of a
+  sensor failing silently.
+* Added a ``SensorInstanceDB`` collection tracking each sensor's runtime health (status, hostname, pid,
+  exit code, respawn count, last updated). The health fields are merged into the ``/v1/sensors`` API
+  response and shown by ``st2 sensor list`` / ``st2 sensor get``, and can be filtered with
+  ``st2 sensor list --status=abandoned``.
+* Added a ``status`` column to ``st2 sensor list`` and the sensor runtime health fields (``status``,
+  ``hostname``, ``pid``, ``exit_code``, ``respawn_count``, ``updated_at``) to ``st2 sensor get`` output
+  in the ``st2`` CLI. (by @guzzijones12@gmail.com)
+* Added ``[sensorcontainer].max_respawn_count``, ``[sensorcontainer].respawn_delay`` and
+  ``[sensorcontainer].respawn_backoff_factor`` config options to control how many times a crashed
+  sensor is respawned before being abandoned and how long to wait between attempts. The backoff
+  factor defaults to ``1`` (a constant ``respawn_delay`` between attempts); a value greater than ``1``
+  grows the delay exponentially.
 
 3.9.0 - October 10, 2025
 ------------------------

conf/st2.conf.sample

@@ -306,8 +306,14 @@ version = 4
 [sensorcontainer]
 # location of the logging.conf file
 logging = /etc/st2/logging.sensorcontainer.conf
+# Maximum number of times to respawn a sensor after it exits with a non-zero code before giving up and firing the st2.sensor.process_abandoned trigger.
+max_respawn_count = 2
 # Provider of sensor node partition config.
 partition_provider = name:default
+# Exponential backoff multiplier applied to respawn_delay per attempt. 1 (default) means a constant delay between attempts; a value > 1 grows the delay exponentially (respawn_delay * factor ** (attempt - 1)).
+respawn_backoff_factor = 1.0
+# Base delay (in seconds) to wait between sensor respawn attempts.
+respawn_delay = 2.5
 # name of the sensor node.
 sensor_node_name = sensornode1
 # Run in a single sensor mode where parent process exits when a sensor crashes / dies. This is useful in environments where partitioning, sensor process life cycle and failover is handled by a 3rd party service such as kubernetes.

contrib/schemas/sensor.json

@@ -89,6 +89,48 @@
             "description": "Path to the metadata file relative to the pack directory.",
             "type": "string",
             "default": ""
+        },
+        "status": {
+            "description": "Current runtime status of the sensor (running, stopped, abandoned). Null if the sensor has never run.",
+            "type": [
+                "string",
+                "null"
+            ]
+        },
+        "hostname": {
+            "description": "Host of the sensor container which owns this sensor.",
+            "type": [
+                "string",
+                "null"
+            ]
+        },
+        "pid": {
+            "description": "PID of the sensor process when running.",
+            "type": [
+                "integer",
+                "null"
+            ]
+        },
+        "exit_code": {
+            "description": "Exit code of the last observed process exit.",
+            "type": [
+                "integer",
+                "null"
+            ]
+        },
+        "respawn_count": {
+            "description": "Respawn attempts for the current failure streak.",
+            "type": [
+                "integer",
+                "null"
+            ]
+        },
+        "updated_at": {
+            "description": "Timestamp when the runtime status was last updated.",
+            "type": [
+                "string",
+                "null"
+            ]
         }
     },
     "additionalProperties": false

lockfiles/st2.lock

@@ -3085,7 +3085,7 @@
           "artifacts": [
             {
               "algorithm": "sha256",
-              "hash": "491767e81c1bb11a54fb68d1a24119bdeede593a2beccca5bc09bfed36fdb35c",
+              "hash": "b9feb1769b48102061fe4fc59b2f5ad600bc2ac0b55cf12ef5fe49464ac0d230",
               "url": "git+https://github.com/StackStorm/orquesta.git"
             }
           ],

st2api/st2api/controllers/v1/sensors.py

@@ -15,17 +15,21 @@
 
 import six
 from mongoengine import ValidationError
+from mongoengine.errors import MongoEngineException
 
 from st2common import log as logging
 from st2common.persistence.sensor import SensorType
+from st2common.persistence.sensor_instance import SensorInstance
 from st2common.models.api.sensor import SensorTypeAPI
 from st2common.exceptions.apivalidation import ValueValidationException
 from st2common.validators.api.misc import validate_not_part_of_system_pack
+from st2common.util import isotime
 from st2api.controllers import resource
 from st2api.controllers.controller_transforms import transform_to_bool
 from st2common.rbac.types import PermissionType
 from st2common.rbac.backends import get_rbac_backend
 from st2common.router import abort
+from st2common.router import Response
 
 http_client = six.moves.http_client
 
@@ -40,12 +44,28 @@ class SensorTypeController(resource.ContentPackResourceController):
         "pack": "pack",
         "enabled": "enabled",
         "trigger": "trigger_types",
+        # "ref" maps to a "ref__in" filter so it accepts one or more refs. It is
+        # also used internally to constrain results when filtering by "status"
+        # (which lives in the separate SensorInstanceDB collection).
+        "ref": "ref.in",
     }
 
     filter_transform_functions = {"enabled": transform_to_bool}
 
     options = {"sort": ["pack", "name"]}
 
+    # Explicit allow-list of runtime health fields merged in from SensorInstanceDB; must stay in sync with the read-only fields on SensorTypeAPI (updated_at handled separately in _apply_health).
+    HEALTH_ATTRIBUTES = [
+        "status",
+        "hostname",
+        "pid",
+        "exit_code",
+        "respawn_count",
+    ]
+
+    # Synthetic fields merged from SensorInstanceDB (not SensorTypeDB fields); stripped from include/exclude lists before querying and re-applied in _apply_health.
+    _SYNTHETIC_HEALTH_FIELDS = set(HEALTH_ATTRIBUTES) | {"updated_at"}
+
     def get_all(
         self,
         exclude_attributes=None,
@@ -56,6 +76,45 @@ def get_all(
         requester_user=None,
         **raw_filters,
     ):
+        # "status" lives in SensorInstanceDB, not SensorTypeDB - resolve it to matching refs and constrain the query.
+        status = raw_filters.pop("status", None)
+        if status:
+            try:
+                refs = [
+                    instance.ref
+                    for instance in SensorInstance.query(
+                        status=status, only_fields=["ref"]
+                    )
+                ]
+            except MongoEngineException:
+                LOG.warning(
+                    "Failed to resolve sensor refs for status filter", exc_info=True
+                )
+                refs = []
+
+            if not refs:
+                resp = Response(json=[])
+                resp.headers["X-Total-Count"] = "0"
+                return resp
+
+            raw_filters["ref"] = refs
+
+        # Health fields are merged in post-query; strip them from include/exclude lists but keep "ref" for correlation.
+        if include_attributes:
+            include_attributes = [
+                attribute
+                for attribute in include_attributes
+                if attribute not in self._SYNTHETIC_HEALTH_FIELDS
+            ]
+            if "ref" not in include_attributes:
+                include_attributes.append("ref")
+        if exclude_attributes:
+            exclude_attributes = [
+                attribute
+                for attribute in exclude_attributes
+                if attribute not in self._SYNTHETIC_HEALTH_FIELDS
+            ]
+
         return super(SensorTypeController, self)._get_all(
             exclude_fields=exclude_attributes,
             include_fields=include_attributes,
@@ -72,6 +131,79 @@ def get_one(self, ref_or_id, requester_user):
             ref_or_id, requester_user=requester_user, permission_type=permission_type
         )
 
+    def resources_model_filter(
+        self,
+        model,
+        instances,
+        requester_user=None,
+        offset=0,
+        eop=0,
+        **from_model_kwargs,
+    ):
+        # List path (get_all): batch-fetch health records once to avoid N+1.
+        page = list(instances[offset:eop])
+        health_by_ref = self._get_health_by_ref(
+            [getattr(instance, "ref", None) for instance in page]
+        )
+
+        result = []
+        for instance in page:
+            # Call the base per-item converter directly (not self.resource_model_filter)
+            # so we don't issue a second, per-item health query.
+            item = super(SensorTypeController, self).resource_model_filter(
+                model=model,
+                instance=instance,
+                requester_user=requester_user,
+                **from_model_kwargs,
+            )
+            if item:
+                self._apply_health(item, health_by_ref.get(instance.ref))
+            result.append(item)
+        return result
+
+    def resource_model_filter(
+        self, model, instance, requester_user=None, **from_model_kwargs
+    ):
+        # Single path (get_one): convert then merge runtime health.
+        item = super(SensorTypeController, self).resource_model_filter(
+            model=model,
+            instance=instance,
+            requester_user=requester_user,
+            **from_model_kwargs,
+        )
+        if item:
+            health_by_ref = self._get_health_by_ref([getattr(instance, "ref", None)])
+            self._apply_health(item, health_by_ref.get(instance.ref))
+        return item
+
+    def _get_health_by_ref(self, refs):
+        """
+        Return a ``ref -> SensorInstanceDB`` map for the provided sensor refs.
+        """
+        refs = [ref for ref in refs if ref]
+        if not refs:
+            return {}
+
+        try:
+            instances = SensorInstance.query(ref__in=refs)
+            return {instance.ref: instance for instance in instances}
+        except MongoEngineException:
+            LOG.warning("Failed to load sensor health records", exc_info=True)
+            return {}
+
+    def _apply_health(self, item, instance_db):
+        """
+        Merge runtime health fields from a SensorInstanceDB record onto the
+        SensorTypeAPI instance. Fields default to None when no record exists.
+        """
+        for attribute in self.HEALTH_ATTRIBUTES:
+            setattr(item, attribute, getattr(instance_db, attribute, None))
+
+        updated_at = getattr(instance_db, "updated_at", None)
+        item.updated_at = (
+            isotime.format(updated_at, offset=False) if updated_at else None
+        )
+
     def put(self, sensor_type, ref_or_id, requester_user):
         # Note: Right now this function only supports updating of "enabled"
         # attribute on the SensorType model.

st2api/tests/unit/controllers/v1/test_sensortypes.py

@@ -20,6 +20,11 @@
 import st2common.bootstrap.sensorsregistrar as sensors_registrar
 from st2api.controllers.v1.sensors import SensorTypeController
 
+from st2common.constants.sensors import SENSOR_STATUS_RUNNING
+from st2common.constants.sensors import SENSOR_STATUS_ABANDONED
+from st2common.models.db.sensor_instance import SensorInstanceDB
+from st2common.persistence.sensor_instance import SensorInstance
+
 from st2tests.api import FunctionalTest
 from st2tests.api import APIControllerWithIncludeAndExcludeFilterTestCase
 
@@ -122,6 +127,107 @@ def test_get_one_doesnt_exist(self):
         resp = self.app.get("/v1/sensortypes/1", expect_errors=True)
         self.assertEqual(resp.status_int, http_client.NOT_FOUND)
 
+    def tearDown(self):
+        super(SensorTypeControllerTestCase, self).tearDown()
+        # Remove any runtime health records created by health tests so they do
+        # not leak into other tests.
+        for instance in SensorInstance.get_all():
+            SensorInstance.delete(instance)
+
+    @staticmethod
+    def _create_health_record(ref, pack, status, **kwargs):
+        instance_db = SensorInstanceDB(ref=ref, pack=pack, status=status, **kwargs)
+        return SensorInstance.add_or_update(instance_db)
+
+    def test_get_one_merges_health_fields(self):
+        ref = f"{DUMMY_PACK_1}.SampleSensor"
+        self._create_health_record(
+            ref,
+            DUMMY_PACK_1,
+            SENSOR_STATUS_RUNNING,
+            hostname="sensor-node-1",
+            pid=4321,
+            exit_code=0,
+            respawn_count=0,
+        )
+
+        resp = self.app.get(f"/v1/sensortypes/{ref}")
+        self.assertEqual(resp.status_int, http_client.OK)
+        self.assertEqual(resp.json["status"], SENSOR_STATUS_RUNNING)
+        self.assertEqual(resp.json["hostname"], "sensor-node-1")
+        self.assertEqual(resp.json["pid"], 4321)
+        self.assertEqual(resp.json["exit_code"], 0)
+        self.assertEqual(resp.json["respawn_count"], 0)
+        self.assertIsNotNone(resp.json["updated_at"])
+
+    def test_get_one_no_health_record_returns_null_fields(self):
+        # A sensor which has never run has no SensorInstanceDB record; the
+        # health fields must still be present and null.
+        resp = self.app.get(f"/v1/sensortypes/{DUMMY_PACK_1}.SampleSensor")
+        self.assertEqual(resp.status_int, http_client.OK)
+        self.assertIsNone(resp.json["status"])
+        self.assertIsNone(resp.json["hostname"])
+        self.assertIsNone(resp.json["pid"])
+        self.assertIsNone(resp.json["updated_at"])
+
+    def test_get_all_merges_health_fields(self):
+        running_ref = f"{DUMMY_PACK_1}.SampleSensor"
+        abandoned_ref = f"{DUMMY_PACK_1}.SampleSensor2"
+        self._create_health_record(
+            running_ref, DUMMY_PACK_1, SENSOR_STATUS_RUNNING, pid=100
+        )
+        self._create_health_record(
+            abandoned_ref, DUMMY_PACK_1, SENSOR_STATUS_ABANDONED, exit_code=1
+        )
+
+        resp = self.app.get("/v1/sensortypes")
+        self.assertEqual(resp.status_int, http_client.OK)
+        status_by_ref = {item["ref"]: item["status"] for item in resp.json}
+        self.assertEqual(status_by_ref[running_ref], SENSOR_STATUS_RUNNING)
+        self.assertEqual(status_by_ref[abandoned_ref], SENSOR_STATUS_ABANDONED)
+        # Sensor without a health record reports a null status.
+        self.assertIsNone(status_by_ref[f"{DUMMY_PACK_1}.SampleSensor3"])
+
+    def test_get_all_status_filter(self):
+        running_ref = f"{DUMMY_PACK_1}.SampleSensor"
+        abandoned_ref = f"{DUMMY_PACK_1}.SampleSensor2"
+        self._create_health_record(running_ref, DUMMY_PACK_1, SENSOR_STATUS_RUNNING)
+        self._create_health_record(abandoned_ref, DUMMY_PACK_1, SENSOR_STATUS_ABANDONED)
+
+        resp = self.app.get("/v1/sensortypes?status=abandoned")
+        self.assertEqual(resp.status_int, http_client.OK)
+        self.assertEqual(len(resp.json), 1)
+        self.assertEqual(resp.json[0]["ref"], abandoned_ref)
+        self.assertEqual(resp.json[0]["status"], SENSOR_STATUS_ABANDONED)
+
+    def test_get_all_status_filter_no_matches(self):
+        # No sensor is in the requested status - the result is empty.
+        resp = self.app.get("/v1/sensortypes?status=abandoned")
+        self.assertEqual(resp.status_int, http_client.OK)
+        self.assertEqual(len(resp.json), 0)
+
+    def test_get_all_include_attributes_with_health_fields(self):
+        # The CLI (st2 sensor list) requests health fields via include_attributes.
+        # They are not SensorTypeDB fields, so they must be stripped before the
+        # query (otherwise mongoengine 400s) and still merged in from the health
+        # record afterwards. This mirrors the request st2-self-check issues.
+        running_ref = f"{DUMMY_PACK_1}.SampleSensor"
+        self._create_health_record(
+            running_ref, DUMMY_PACK_1, SENSOR_STATUS_RUNNING, pid=100
+        )
+
+        resp = self.app.get(
+            "/v1/sensortypes?include_attributes=ref,pack,enabled,status,updated_at"
+        )
+        self.assertEqual(resp.status_int, http_client.OK)
+        item_by_ref = {item["ref"]: item for item in resp.json}
+        # DB-backed included field is present.
+        self.assertIn("pack", item_by_ref[running_ref])
+        # Synthetic health fields are merged in even though they were stripped
+        # from the DB query.
+        self.assertEqual(item_by_ref[running_ref]["status"], SENSOR_STATUS_RUNNING)
+        self.assertIsNotNone(item_by_ref[running_ref]["updated_at"])
+
     def test_disable_and_enable_sensor(self):
         # Verify initial state
         resp = self.app.get(f"/v1/sensortypes/{DUMMY_PACK_1}.SampleSensor")