diff --git a/CHANGELOG.md b/CHANGELOG.md index a75a9bc1..9807023f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ possible include a PR number for easier tracking. ## Next +* feat(endpoints): add inodes introspection endpoint (#1273) * feat: add --replay mode for JSONL event replay without eBPF (#1010) * feat(config): configurable loaded BPF programs (#1086) * feat(output): add basic opentelemetry output (#971) diff --git a/fact/src/config/mod.rs b/fact/src/config/mod.rs index ab8e9d6a..f6e123a2 100644 --- a/fact/src/config/mod.rs +++ b/fact/src/config/mod.rs @@ -276,6 +276,7 @@ pub struct EndpointConfig { address: Option, expose_metrics: Option, health_check: Option, + introspection: Option, } impl EndpointConfig { @@ -291,6 +292,10 @@ impl EndpointConfig { if let Some(health_check) = from.health_check { self.health_check = Some(health_check); } + + if let Some(introspection) = from.introspection { + self.introspection = Some(introspection); + } } pub fn address(&self) -> SocketAddr { @@ -305,6 +310,10 @@ impl EndpointConfig { pub fn health_check(&self) -> bool { self.health_check.unwrap_or(false) } + + pub fn introspection(&self) -> bool { + self.introspection.unwrap_or(false) + } } impl TryFrom<&yaml::Hash> for EndpointConfig { @@ -340,6 +349,12 @@ impl TryFrom<&yaml::Hash> for EndpointConfig { }; endpoint.health_check = Some(hc); } + "introspection" => { + let Some(i) = v.as_bool() else { + bail!("endpoint.introspection field has incorrect type: {v:?}"); + }; + endpoint.introspection = Some(i); + } name => bail!("Invalid field 'endpoint.{name}' with value: {v:?}"), } } @@ -792,6 +807,16 @@ pub struct FactCli { #[arg(long, overrides_with = "health_check", hide(true))] no_health_check: bool, + /// Whether the introspection endpoints should be enabled + #[arg( + long, + overrides_with("no_introspection"), + env = "FACT_ENDPOINT_INTROSPECTION" + )] + introspection: bool, + #[arg(long, overrides_with = "introspection", hide(true))] + no_introspection: bool, + /// Whether to perform a pre flight check #[arg( long, @@ -880,6 +905,7 @@ impl FactCli { address: self.address, expose_metrics: resolve_bool_arg(self.expose_metrics, self.no_expose_metrics), health_check: resolve_bool_arg(self.health_check, self.no_health_check), + introspection: resolve_bool_arg(self.introspection, self.no_introspection), }, bpf: BpfConfig { ringbuf_size: self.ringbuf_size, diff --git a/fact/src/config/reloader.rs b/fact/src/config/reloader.rs index 362f337e..12b6f6cb 100644 --- a/fact/src/config/reloader.rs +++ b/fact/src/config/reloader.rs @@ -145,26 +145,39 @@ impl Reloader { res } - /// Recreate the configuration and notify of changes to any - /// subscribers. - fn reload(&mut self) { - if !self.update_cache() { - return; - } - - let new = match FactConfig::build() { - Ok(config) => config, - Err(e) => { - warn!("Configuration reloading failed: {e}"); - return; - } - }; - info!("Updated configuration: {new:#?}"); + /// Compare endpoint configurations and figure out if reloading is + /// needed. + /// + /// EndpointConfig has non-reloadable fields, its PartialEq + /// implementation still has all fields so unit tests can check + /// properly. + /// + /// The first argument is destructured here so the compiler + /// complains when new fields are added, making sure we fix this + /// with things that need to be reloaded. + fn endpoint_should_reload( + &EndpointConfig { + address: l_address, + expose_metrics: l_expose_metrics, + health_check: l_health_check, + introspection: _, + }: &EndpointConfig, + right: &EndpointConfig, + ) -> bool { + l_address != right.address + || l_expose_metrics != right.expose_metrics + || l_health_check != right.health_check + } + /// Propagate configuration changes to all subscribers that need it + fn send_updates(&self, new: &FactConfig) { self.endpoint.send_if_modified(|old| { - if *old != new.endpoint { + if Reloader::endpoint_should_reload(old, &new.endpoint) { debug!("Sending new endpoint configuration..."); - *old = new.endpoint.clone(); + *old = EndpointConfig { + introspection: old.introspection, + ..new.endpoint + }; true } else { false @@ -227,6 +240,25 @@ impl Reloader { if self.config.hotreload() != new.hotreload() { warn!("Changes to the hotreload field only take effect on startup"); } + } + + /// Recreate the configuration and notify of changes to any + /// subscribers. + fn reload(&mut self) { + if !self.update_cache() { + return; + } + + let new = match FactConfig::build() { + Ok(config) => config, + Err(e) => { + warn!("Configuration reloading failed: {e}"); + return; + } + }; + info!("Updated configuration: {new:#?}"); + + self.send_updates(&new); self.config = new; } @@ -274,3 +306,214 @@ impl From for Reloader { } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_reloading_endpoint() { + let tests = [ + (FactConfig::default(), FactConfig::default(), None), + ( + FactConfig::default(), + FactConfig { + endpoint: EndpointConfig { + address: Some(([127, 0, 0, 1], 8080).into()), + ..Default::default() + }, + ..Default::default() + }, + Some(EndpointConfig { + address: Some(([127, 0, 0, 1], 8080).into()), + ..Default::default() + }), + ), + ( + FactConfig { + endpoint: EndpointConfig { + address: Some(([0, 0, 0, 0], 9090).into()), + ..Default::default() + }, + ..Default::default() + }, + FactConfig { + endpoint: EndpointConfig { + address: Some(([127, 0, 0, 1], 8080).into()), + ..Default::default() + }, + ..Default::default() + }, + Some(EndpointConfig { + address: Some(([127, 0, 0, 1], 8080).into()), + ..Default::default() + }), + ), + ( + FactConfig::default(), + FactConfig { + endpoint: EndpointConfig { + expose_metrics: Some(true), + ..Default::default() + }, + ..Default::default() + }, + Some(EndpointConfig { + expose_metrics: Some(true), + ..Default::default() + }), + ), + ( + FactConfig { + endpoint: EndpointConfig { + expose_metrics: Some(true), + ..Default::default() + }, + ..Default::default() + }, + FactConfig { + endpoint: EndpointConfig { + expose_metrics: Some(false), + ..Default::default() + }, + ..Default::default() + }, + Some(EndpointConfig { + expose_metrics: Some(false), + ..Default::default() + }), + ), + ( + FactConfig::default(), + FactConfig { + endpoint: EndpointConfig { + health_check: Some(true), + ..Default::default() + }, + ..Default::default() + }, + Some(EndpointConfig { + health_check: Some(true), + ..Default::default() + }), + ), + ( + FactConfig { + endpoint: EndpointConfig { + health_check: Some(true), + ..Default::default() + }, + ..Default::default() + }, + FactConfig { + endpoint: EndpointConfig { + health_check: Some(false), + ..Default::default() + }, + ..Default::default() + }, + Some(EndpointConfig { + health_check: Some(false), + ..Default::default() + }), + ), + ( + FactConfig::default(), + FactConfig { + endpoint: EndpointConfig { + introspection: Some(true), + ..Default::default() + }, + ..Default::default() + }, + None, + ), + ( + FactConfig { + endpoint: EndpointConfig { + introspection: Some(true), + ..Default::default() + }, + ..Default::default() + }, + FactConfig { + endpoint: EndpointConfig { + introspection: Some(false), + ..Default::default() + }, + ..Default::default() + }, + None, + ), + ( + FactConfig::default(), + FactConfig { + endpoint: EndpointConfig { + address: Some(([127, 0, 0, 1], 8080).into()), + introspection: Some(true), + expose_metrics: Some(true), + health_check: Some(true), + }, + ..Default::default() + }, + Some(EndpointConfig { + address: Some(([127, 0, 0, 1], 8080).into()), + introspection: None, + expose_metrics: Some(true), + health_check: Some(true), + }), + ), + ( + FactConfig { + endpoint: EndpointConfig { + address: Some(([0, 0, 0, 0], 9090).into()), + introspection: Some(true), + expose_metrics: Some(true), + health_check: Some(true), + }, + ..Default::default() + }, + FactConfig { + endpoint: EndpointConfig { + address: Some(([127, 0, 0, 1], 8080).into()), + introspection: Some(false), + expose_metrics: Some(false), + health_check: Some(false), + }, + ..Default::default() + }, + Some(EndpointConfig { + address: Some(([127, 0, 0, 1], 8080).into()), + introspection: Some(true), + expose_metrics: Some(false), + health_check: Some(false), + }), + ), + ]; + + for (old, new, expected) in tests { + let reloader = Reloader::from(old); + let endpoint = reloader.endpoint(); + let assert_has_changed = |has_changed: bool| { + assert!( + has_changed, + "\ninput: {:?}\nnew: {:?}", + reloader.config().endpoint, + new.endpoint + ); + }; + + reloader.send_updates(&new); + + match expected { + Some(expected) => { + assert_has_changed(endpoint.has_changed().unwrap()); + assert_eq!(*endpoint.borrow(), expected); + } + None => { + assert_has_changed(!endpoint.has_changed().unwrap()); + } + } + } + } +} diff --git a/fact/src/config/tests.rs b/fact/src/config/tests.rs index f00bae2c..6b6abde3 100644 --- a/fact/src/config/tests.rs +++ b/fact/src/config/tests.rs @@ -297,6 +297,32 @@ fn parsing() { ..Default::default() }, ), + ( + r#" + endpoint: + introspection: true + "#, + FactConfig { + endpoint: EndpointConfig { + introspection: Some(true), + ..Default::default() + }, + ..Default::default() + }, + ), + ( + r#" + endpoint: + introspection: false + "#, + FactConfig { + endpoint: EndpointConfig { + introspection: Some(false), + ..Default::default() + }, + ..Default::default() + }, + ), ( "skip_pre_flight: true", FactConfig { @@ -477,6 +503,7 @@ fn parsing() { address: 0.0.0.0:8080 expose_metrics: true health_check: true + introspection: true skip_pre_flight: false json: false bpf: @@ -514,6 +541,7 @@ fn parsing() { address: Some(SocketAddr::from(([0, 0, 0, 0], 8080))), expose_metrics: Some(true), health_check: Some(true), + introspection: Some(true), }, skip_pre_flight: Some(false), json: Some(false), @@ -802,6 +830,13 @@ paths: "#, "endpoint.health_check field has incorrect type: Integer(4)", ), + ( + r#" + endpoint: + introspection: 4 + "#, + "endpoint.introspection field has incorrect type: Integer(4)", + ), ( r#" endpoint: @@ -1474,6 +1509,60 @@ fn update() { ..Default::default() }, ), + ( + r#" + endpoint: + introspection: true + "#, + FactConfig::default(), + FactConfig { + endpoint: EndpointConfig { + introspection: Some(true), + ..Default::default() + }, + ..Default::default() + }, + ), + ( + r#" + endpoint: + introspection: true + "#, + FactConfig { + endpoint: EndpointConfig { + introspection: Some(false), + ..Default::default() + }, + ..Default::default() + }, + FactConfig { + endpoint: EndpointConfig { + introspection: Some(true), + ..Default::default() + }, + ..Default::default() + }, + ), + ( + r#" + endpoint: + introspection: true + "#, + FactConfig { + endpoint: EndpointConfig { + introspection: Some(true), + ..Default::default() + }, + ..Default::default() + }, + FactConfig { + endpoint: EndpointConfig { + introspection: Some(true), + ..Default::default() + }, + ..Default::default() + }, + ), ( "skip_pre_flight: true", FactConfig::default(), @@ -1831,6 +1920,7 @@ fn update() { address: 127.0.0.1:8080 expose_metrics: true health_check: true + introspection: true skip_pre_flight: false json: false bpf: @@ -1863,6 +1953,7 @@ fn update() { address: Some(SocketAddr::from(([0, 0, 0, 0], 9000))), expose_metrics: Some(false), health_check: Some(false), + introspection: Some(false), }, skip_pre_flight: Some(true), json: Some(true), @@ -1901,6 +1992,7 @@ fn update() { address: Some(SocketAddr::from(([127, 0, 0, 1], 8080))), expose_metrics: Some(true), health_check: Some(true), + introspection: Some(true), }, skip_pre_flight: Some(false), json: Some(false), @@ -1952,6 +2044,7 @@ fn defaults() { ); assert!(!config.endpoint.expose_metrics()); assert!(!config.endpoint.health_check()); + assert!(!config.endpoint.introspection()); assert!(!config.skip_pre_flight()); assert!(!config.json()); assert_eq!(config.bpf.ringbuf_size(), 8192); @@ -2313,6 +2406,19 @@ fn env_vars() { ..Default::default() }, ), + ( + EnvVar { + name: "FACT_ENDPOINT_INTROSPECTION", + value: "true", + }, + FactConfig { + endpoint: EndpointConfig { + introspection: Some(true), + ..Default::default() + }, + ..Default::default() + }, + ), ( EnvVar { name: "FACT_SKIP_PRE_FLIGHT", @@ -2535,6 +2641,20 @@ fn env_vars_override_yaml() { ..Default::default() }, ), + ( + EnvVar { + name: "FACT_ENDPOINT_INTROSPECTION", + value: "true", + }, + "endpoint:\n introspection: false", + FactConfig { + endpoint: EndpointConfig { + introspection: Some(true), + ..Default::default() + }, + ..Default::default() + }, + ), ( EnvVar { name: "FACT_SKIP_PRE_FLIGHT", diff --git a/fact/src/endpoints.rs b/fact/src/endpoints.rs index cc97415a..947076b6 100644 --- a/fact/src/endpoints.rs +++ b/fact/src/endpoints.rs @@ -9,7 +9,11 @@ use hyper::{ }; use hyper_util::rt::TokioIo; use log::{info, warn}; -use tokio::{net::TcpListener, sync::watch, task::JoinHandle}; +use tokio::{ + net::TcpListener, + sync::{mpsc, oneshot, watch}, + task::JoinHandle, +}; use crate::{config::EndpointConfig, metrics::exporter::Exporter}; @@ -18,6 +22,8 @@ pub struct Server { metrics: Exporter, config: watch::Receiver, running: watch::Receiver, + + host_scanner_intro: mpsc::Sender>>, } impl Server { @@ -25,11 +31,13 @@ impl Server { metrics: Exporter, config: watch::Receiver, running: watch::Receiver, + host_scanner_intro: mpsc::Sender>>, ) -> Self { Server { metrics, config, running, + host_scanner_intro, } } @@ -95,7 +103,7 @@ impl Server { /// Check if there are active endpoints to serve. fn is_active(&self) -> bool { let config = self.config.borrow(); - config.health_check() || config.expose_metrics() + config.health_check() || config.expose_metrics() || config.introspection() } fn health_check_is_active(&self) -> bool { @@ -108,17 +116,17 @@ impl Server { fn make_response( res: StatusCode, - body: String, + body: impl Into, ) -> Result>, anyhow::Error> { Ok(Response::builder() .status(res) - .body(Full::new(Bytes::from(body))) + .body(Full::new(body.into())) .unwrap()) } fn handle_metrics(&self) -> Result>, anyhow::Error> { if !self.metrics_is_active() { - return Server::make_response(StatusCode::SERVICE_UNAVAILABLE, String::new()); + return Server::make_response(StatusCode::SERVICE_UNAVAILABLE, ""); } self.metrics.encode().map(|buf| { @@ -139,7 +147,29 @@ impl Server { } else { StatusCode::SERVICE_UNAVAILABLE }; - Server::make_response(res, String::new()) + Server::make_response(res, "") + } + + async fn handle_inodes(&self) -> anyhow::Result>> { + if !self.config.borrow().introspection() { + return Server::make_response(StatusCode::SERVICE_UNAVAILABLE, ""); + } + + let (tx, rx) = oneshot::channel(); + if let Err(e) = self.host_scanner_intro.send(tx).await { + return Server::make_response(StatusCode::INTERNAL_SERVER_ERROR, e.to_string()); + } + match rx.await { + Ok(Ok(b)) => Response::builder() + .header( + hyper::header::CONTENT_TYPE, + "application/json; charset=utf-8", + ) + .body(Full::new(Bytes::from(b))) + .map_err(anyhow::Error::new), + Ok(Err(e)) => Server::make_response(StatusCode::INTERNAL_SERVER_ERROR, e.to_string()), + Err(e) => Server::make_response(StatusCode::INTERNAL_SERVER_ERROR, e.to_string()), + } } } @@ -154,7 +184,8 @@ impl Service> for Server { match (req.method(), req.uri().path()) { (&Method::GET, "/metrics") => s.handle_metrics(), (&Method::GET, "/health_check") => s.handle_health_check(), - _ => Server::make_response(StatusCode::NOT_FOUND, String::new()), + (&Method::GET, "/inodes") => s.handle_inodes().await, + _ => Server::make_response(StatusCode::NOT_FOUND, ""), } }) } diff --git a/fact/src/host_scanner.rs b/fact/src/host_scanner.rs index 9e46eef1..04dff52a 100644 --- a/fact/src/host_scanner.rs +++ b/fact/src/host_scanner.rs @@ -20,7 +20,9 @@ use std::{ cell::RefCell, + collections::HashMap, io, + ops::{Deref, DerefMut}, os::linux::fs::MetadataExt, path::{Path, PathBuf}, sync::Arc, @@ -35,8 +37,9 @@ use aya::{ use fact_ebpf::{inode_key_t, inode_value_t, monitored_t}; use globset::{Glob, GlobSet, GlobSetBuilder}; use log::{debug, info, warn}; +use serde::{Serialize, ser::SerializeMap}; use tokio::{ - sync::{Notify, mpsc, watch}, + sync::{Notify, mpsc, oneshot, watch}, task::JoinSet, }; @@ -47,15 +50,55 @@ use crate::{ metrics::host_scanner::{HostScannerMetrics, ScanLabels}, }; +struct InodeMap(HashMap); + +impl InodeMap { + fn new() -> Self { + InodeMap(HashMap::new()) + } +} + +impl Deref for InodeMap { + type Target = HashMap; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for InodeMap { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl Serialize for InodeMap { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + let mut map = serializer.serialize_map(Some(self.len()))?; + for (k, v) in &self.0 { + // In order to be able to serialize InodeMap to JSON, we use + // a string key. This enables us to send this type over HTTP + // as part of the "/inodes" introspection endpoint, while + // keeping the existing inode_key_t Serialize implementation. + map.serialize_entry(&format!("{}:{}", k.dev, k.inode), v)?; + } + map.end() + } +} + pub struct HostScanner { kernel_inode_map: RefCell>, - inode_map: RefCell>, + inode_map: RefCell, paths: watch::Receiver>, scan_interval: watch::Receiver, rx: mpsc::Receiver, tx: mpsc::Sender, + introspection: mpsc::Receiver>>, metrics: HostScannerMetrics, @@ -69,9 +112,10 @@ impl HostScanner { paths: watch::Receiver>, scan_interval: watch::Receiver, metrics: HostScannerMetrics, + introspection: mpsc::Receiver>>, ) -> anyhow::Result<(Self, mpsc::Receiver)> { let kernel_inode_map = RefCell::new(bpf.take_inode_map()?); - let inode_map = RefCell::new(std::collections::HashMap::new()); + let inode_map = RefCell::new(InodeMap::new()); let (tx, output) = mpsc::channel(100); let paths_globset = HostScanner::build_globset(paths.borrow().as_slice())?; @@ -82,6 +126,7 @@ impl HostScanner { scan_interval, rx, tx, + introspection, metrics, paths_globset, }; @@ -473,6 +518,16 @@ You can increase this limit with: warn!("Failed to send event: {e}"); } }, + req = self.introspection.recv() => { + let Some(req) = req else { + continue; + }; + + let resp = serde_json::to_string(&*self.inode_map.borrow()); + if let Err(e) = req.send(resp) { + warn!("Failed to reply introspection query: {e:?}"); + } + } _ = scan_trigger.notified() => self.scan()?, _ = self.paths.changed() => { self.paths_globset = HostScanner::build_globset(self.paths.borrow().as_slice())?; diff --git a/fact/src/lib.rs b/fact/src/lib.rs index 463c0c8f..d9f78fb1 100644 --- a/fact/src/lib.rs +++ b/fact/src/lib.rs @@ -9,7 +9,7 @@ use metrics::exporter::Exporter; use rate_limiter::RateLimiter; use tokio::{ signal::unix::{SignalKind, signal}, - sync::{mpsc, watch}, + sync::{mpsc, oneshot, watch}, task::JoinSet, time::timeout, }; @@ -99,12 +99,14 @@ pub async fn run(config: FactConfig) -> anyhow::Result<()> { let config_trigger = reloader.get_trigger(); let mut task_set = JoinSet::new(); let metrics_userspace = Metrics::new(); + let (host_scanner_intro_tx, host_scanner_intro_rx) = mpsc::channel(10); let (metrics_kernelspace, rx) = setup_input( &mut task_set, &reloader, &metrics_userspace, running_pipeline_rx, + host_scanner_intro_rx, )?; let (rate_limiter, rx) = RateLimiter::new( rx, @@ -123,7 +125,13 @@ pub async fn run(config: FactConfig) -> anyhow::Result<()> { rate_limiter.start(&mut task_set); let exporter = Exporter::new(&metrics_userspace, metrics_kernelspace); - endpoints::Server::new(exporter, reloader.endpoint(), running_helpers.subscribe()).start(); + endpoints::Server::new( + exporter, + reloader.endpoint(), + running_helpers.subscribe(), + host_scanner_intro_tx, + ) + .start(); reloader.start(running_helpers.subscribe()); let mut sigterm = signal(SignalKind::terminate())?; @@ -162,6 +170,7 @@ fn setup_input( reloader: &config::reloader::Reloader, metrics: &Metrics, running: watch::Receiver, + hs_introspection: mpsc::Receiver>>, ) -> anyhow::Result<(Option, mpsc::Receiver)> { match reloader.config().replay() { Some(replay_file) => { @@ -176,7 +185,7 @@ fn setup_input( debug!("Skipping pre-flight checks"); } - bpf_input(task_set, reloader, running, metrics) + bpf_input(task_set, reloader, running, metrics, hs_introspection) } } } @@ -186,6 +195,7 @@ fn bpf_input( reloader: &config::reloader::Reloader, running: watch::Receiver, metrics_userspace: &Metrics, + hs_introspection: mpsc::Receiver>>, ) -> anyhow::Result<(Option, mpsc::Receiver)> { let (mut bpf, rx) = Bpf::new( reloader.paths(), @@ -201,6 +211,7 @@ fn bpf_input( reloader.paths(), reloader.scan_interval(), metrics_userspace.host_scanner.clone(), + hs_introspection, )?; bpf.start(task_set);