Wire Bugsnag into the orchestrator

cloud_pipelines_backend/orchestrator_sql.py

@@ -21,6 +21,7 @@
 from . import component_structures as structures
 from .launchers import common_annotations
 from .launchers import interfaces as launcher_interfaces
+from .instrumentation import bugsnag_instrumentation
 from .instrumentation import contextual_logging
 from .instrumentation import metrics as app_metrics
 
@@ -1055,6 +1056,7 @@ def _retry(
 
 def record_system_error_exception(execution: bts.ExecutionNode, exception: Exception):
     app_metrics.execution_system_errors.add(1)
+    bugsnag_instrumentation.notify(exception=exception, execution_id=str(execution.id))
 
     if execution.extra_data is None:
         execution.extra_data = {}

orchestrator_main.py

@@ -6,6 +6,7 @@
 from sqlalchemy import orm
 
 from cloud_pipelines_backend import orchestrator_sql
+from cloud_pipelines_backend.instrumentation import bugsnag_instrumentation
 from cloud_pipelines_backend.launchers import kubernetes_launchers
 from cloud_pipelines.orchestration.storage_providers import local_storage
 
@@ -32,6 +33,7 @@ def main():
     logger.addHandler(stderr_handler)
 
     logger.info("Starting the orchestrator")
+    bugsnag_instrumentation.setup(service_name="tangle-orchestrator")
 
     DEFAULT_DATABASE_URI = "sqlite:///db.sqlite"
     database_uri = os.environ.get("DATABASE_URI", DEFAULT_DATABASE_URI)
@@ -77,7 +79,11 @@ def main():
         default_task_annotations=default_task_annotations,
         sleep_seconds_between_queue_sweeps=5.0,
     )
-    orchestrator.run_loop()
+    try:
+        orchestrator.run_loop()
+    except Exception as exc:
+        bugsnag_instrumentation.notify(exception=exc)
+        raise
 
 
 if __name__ == "__main__":