diff --git a/crates/openshell-supervisor-network/Cargo.toml b/crates/openshell-supervisor-network/Cargo.toml index 1449276f4f..979097f92b 100644 --- a/crates/openshell-supervisor-network/Cargo.toml +++ b/crates/openshell-supervisor-network/Cargo.toml @@ -59,8 +59,10 @@ tokio-tungstenite = { workspace = true } futures = { workspace = true } tracing-subscriber = { workspace = true } -[target.'cfg(unix)'.dev-dependencies] +[target.'cfg(unix)'.dependencies] libc = "0.2" +[target.'cfg(unix)'.dev-dependencies] + [lints] workspace = true diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index ab2313b890..aa9cf00c6c 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -265,9 +265,13 @@ impl ProxyHandle { } } + let mut consecutive_resource_errors: u32 = 0; + let mut consecutive_unknown_errors: u32 = 0; loop { match listener.accept().await { Ok((stream, _addr)) => { + consecutive_resource_errors = 0; + consecutive_unknown_errors = 0; let opa = opa_engine.clone(); let cache = identity_cache.clone(); let spid = entrypoint_pid.clone(); @@ -314,14 +318,24 @@ impl ProxyHandle { }); } Err(err) => { + let outcome = handle_accept_error( + &err, + &mut consecutive_resource_errors, + &mut consecutive_unknown_errors, + ); + let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) .activity(ActivityId::Fail) - .severity(SeverityId::Low) + .severity(outcome.severity) .status(StatusId::Failure) - .message(format!("Proxy accept error: {err}")) + .message(outcome.message) .build(); ocsf_emit!(event); - break; + + match outcome.backoff { + Some(backoff) => tokio::time::sleep(backoff).await, + None => break, + } } } } @@ -351,6 +365,139 @@ fn emit_activity(tx: &Option, denied: bool, deny_group: &'static } } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum AcceptErrorClass { + Transient, + Terminal, + Unknown, +} + +#[cfg(unix)] +fn classify_accept_error(err: &std::io::Error) -> AcceptErrorClass { + match err.raw_os_error() { + Some( + libc::EMFILE + | libc::ENFILE + | libc::ENOBUFS + | libc::ENOMEM + | libc::ECONNABORTED + | libc::ECONNRESET + | libc::EINTR + | libc::ENETDOWN + | libc::EPROTO + | libc::ENOPROTOOPT + | libc::EHOSTDOWN + | libc::EHOSTUNREACH + | libc::EOPNOTSUPP + | libc::ENETUNREACH + | libc::ENOSR + | libc::ESOCKTNOSUPPORT + | libc::EPROTONOSUPPORT + | libc::ETIMEDOUT, + ) => AcceptErrorClass::Transient, + #[cfg(target_os = "linux")] + Some(libc::ENONET) => AcceptErrorClass::Transient, + Some(libc::EBADF | libc::EINVAL | libc::ENOTSOCK) => AcceptErrorClass::Terminal, + _ => AcceptErrorClass::Unknown, + } +} + +#[cfg(not(unix))] +fn classify_accept_error(_err: &std::io::Error) -> AcceptErrorClass { + AcceptErrorClass::Unknown +} + +#[cfg(unix)] +fn is_resource_pressure_error(err: &std::io::Error) -> bool { + matches!( + err.raw_os_error(), + Some(libc::EMFILE | libc::ENFILE | libc::ENOBUFS | libc::ENOMEM | libc::ENOSR) + ) +} + +#[cfg(not(unix))] +fn is_resource_pressure_error(_err: &std::io::Error) -> bool { + false +} + +const ACCEPT_BACKOFF_BASE_MS: u64 = 100; +const ACCEPT_BACKOFF_MAX_MS: u64 = 5_000; +const MAX_CONSECUTIVE_UNKNOWN_ERRORS: u32 = 5; + +fn accept_backoff(consecutive_errors: u32) -> std::time::Duration { + let exponent = consecutive_errors.saturating_sub(1).min(7); + let ms = ACCEPT_BACKOFF_BASE_MS + .saturating_mul(1u64 << exponent) + .min(ACCEPT_BACKOFF_MAX_MS); + std::time::Duration::from_millis(ms) +} + +struct AcceptErrorOutcome { + severity: SeverityId, + message: String, + backoff: Option, +} + +fn handle_accept_error( + err: &std::io::Error, + consecutive_resource_errors: &mut u32, + consecutive_unknown_errors: &mut u32, +) -> AcceptErrorOutcome { + let class = classify_accept_error(err); + + match class { + AcceptErrorClass::Terminal => AcceptErrorOutcome { + severity: SeverityId::High, + message: format!("Proxy accept error (terminal, exiting): {err}"), + backoff: None, + }, + AcceptErrorClass::Unknown => { + *consecutive_unknown_errors = consecutive_unknown_errors.saturating_add(1); + if *consecutive_unknown_errors > MAX_CONSECUTIVE_UNKNOWN_ERRORS { + AcceptErrorOutcome { + severity: SeverityId::High, + message: format!( + "Proxy accept error (exceeded {MAX_CONSECUTIVE_UNKNOWN_ERRORS} retries, exiting): {err}" + ), + backoff: None, + } + } else { + let backoff = accept_backoff(*consecutive_unknown_errors); + AcceptErrorOutcome { + severity: SeverityId::Medium, + message: format!( + "Proxy accept error (retry {}/{MAX_CONSECUTIVE_UNKNOWN_ERRORS} in {}ms): {err}", + *consecutive_unknown_errors, + backoff.as_millis(), + ), + backoff: Some(backoff), + } + } + } + AcceptErrorClass::Transient => { + *consecutive_unknown_errors = 0; + if is_resource_pressure_error(err) { + *consecutive_resource_errors = consecutive_resource_errors.saturating_add(1); + let backoff = accept_backoff(*consecutive_resource_errors); + AcceptErrorOutcome { + severity: SeverityId::Medium, + message: format!( + "Proxy accept error (retrying in {}ms): {err}", + backoff.as_millis(), + ), + backoff: Some(backoff), + } + } else { + AcceptErrorOutcome { + severity: SeverityId::Low, + message: format!("Proxy accept error (retrying in 100ms): {err}"), + backoff: Some(std::time::Duration::from_millis(100)), + } + } + } + } +} + fn l7_inspection_active(l7_route: Option<&L7RouteSnapshot>) -> bool { l7_route.is_some_and(|route| !route.configs.is_empty()) } @@ -9827,4 +9974,334 @@ network_policies: } } } + + #[test] + fn accept_backoff_exponential_progression() { + let ms = |n| accept_backoff(n).as_millis(); + assert_eq!(ms(1), 100); + assert_eq!(ms(2), 200); + assert_eq!(ms(3), 400); + assert_eq!(ms(4), 800); + assert_eq!(ms(5), 1_600); + assert_eq!(ms(6), 3_200); + assert_eq!(ms(7), 5_000); // 6400 capped to 5000 + assert_eq!(ms(8), 5_000); // stays at cap + } + + #[test] + fn accept_backoff_zero_consecutive_errors() { + assert_eq!(accept_backoff(0).as_millis(), 100); + } + + #[test] + fn accept_backoff_saturates_at_cap() { + assert_eq!(accept_backoff(100).as_millis(), 5_000); + assert_eq!(accept_backoff(u32::MAX).as_millis(), 5_000); + } + + #[cfg(unix)] + #[test] + fn is_resource_pressure_detects_emfile() { + let err = std::io::Error::from_raw_os_error(libc::EMFILE); + assert!(is_resource_pressure_error(&err)); + } + + #[cfg(unix)] + #[test] + fn is_resource_pressure_detects_enfile() { + let err = std::io::Error::from_raw_os_error(libc::ENFILE); + assert!(is_resource_pressure_error(&err)); + } + + #[cfg(unix)] + #[test] + fn is_resource_pressure_detects_memory_pressure() { + assert!(is_resource_pressure_error( + &std::io::Error::from_raw_os_error(libc::ENOBUFS) + )); + assert!(is_resource_pressure_error( + &std::io::Error::from_raw_os_error(libc::ENOMEM) + )); + assert!(is_resource_pressure_error( + &std::io::Error::from_raw_os_error(libc::ENOSR) + )); + } + + #[cfg(unix)] + #[test] + fn is_resource_pressure_rejects_other_errors() { + let err = std::io::Error::from_raw_os_error(libc::ECONNABORTED); + assert!(!is_resource_pressure_error(&err)); + } + + #[cfg(unix)] + #[test] + fn classify_accept_error_fd_exhaustion_is_transient() { + assert_eq!( + classify_accept_error(&std::io::Error::from_raw_os_error(libc::EMFILE)), + AcceptErrorClass::Transient, + ); + assert_eq!( + classify_accept_error(&std::io::Error::from_raw_os_error(libc::ENFILE)), + AcceptErrorClass::Transient, + ); + } + + #[cfg(unix)] + #[test] + fn classify_accept_error_connection_errors_are_transient() { + assert_eq!( + classify_accept_error(&std::io::Error::from_raw_os_error(libc::ECONNABORTED)), + AcceptErrorClass::Transient, + ); + assert_eq!( + classify_accept_error(&std::io::Error::from_raw_os_error(libc::ECONNRESET)), + AcceptErrorClass::Transient, + ); + assert_eq!( + classify_accept_error(&std::io::Error::from_raw_os_error(libc::EINTR)), + AcceptErrorClass::Transient, + ); + } + + #[cfg(unix)] + #[test] + fn classify_accept_error_broken_listener_is_terminal() { + assert_eq!( + classify_accept_error(&std::io::Error::from_raw_os_error(libc::EBADF)), + AcceptErrorClass::Terminal, + ); + assert_eq!( + classify_accept_error(&std::io::Error::from_raw_os_error(libc::EINVAL)), + AcceptErrorClass::Terminal, + ); + assert_eq!( + classify_accept_error(&std::io::Error::from_raw_os_error(libc::ENOTSOCK)), + AcceptErrorClass::Terminal, + ); + } + + #[cfg(unix)] + #[test] + fn classify_accept_error_unrecognized_errno_is_unknown() { + assert_eq!( + classify_accept_error(&std::io::Error::from_raw_os_error(libc::EPERM)), + AcceptErrorClass::Unknown, + ); + } + + #[cfg(unix)] + #[test] + fn handle_accept_error_terminal_exits_immediately() { + let mut res = 0; + let mut unk = 0; + let err = std::io::Error::from_raw_os_error(libc::EBADF); + let outcome = handle_accept_error(&err, &mut res, &mut unk); + assert!(outcome.backoff.is_none()); + assert_eq!(outcome.severity, SeverityId::High); + } + + #[cfg(unix)] + #[test] + fn handle_accept_error_transient_retries_indefinitely() { + let mut res = 0; + let mut unk = 0; + let err = std::io::Error::from_raw_os_error(libc::EMFILE); + for i in 1..=20 { + let outcome = handle_accept_error(&err, &mut res, &mut unk); + assert!(outcome.backoff.is_some(), "should retry on attempt {i}"); + assert_eq!(outcome.severity, SeverityId::Medium); + } + assert_eq!(res, 20); + } + + #[cfg(unix)] + #[test] + fn handle_accept_error_unknown_exits_after_limit() { + let mut res = 0; + let mut unk = 0; + let err = std::io::Error::from_raw_os_error(libc::EPERM); + for i in 1..=MAX_CONSECUTIVE_UNKNOWN_ERRORS { + let outcome = handle_accept_error(&err, &mut res, &mut unk); + assert!( + outcome.backoff.is_some(), + "should retry on attempt {i}/{MAX_CONSECUTIVE_UNKNOWN_ERRORS}", + ); + assert_eq!(outcome.severity, SeverityId::Medium); + } + let outcome = handle_accept_error(&err, &mut res, &mut unk); + assert!( + outcome.backoff.is_none(), + "should exit after limit exceeded" + ); + assert_eq!(outcome.severity, SeverityId::High); + } + + #[cfg(unix)] + #[test] + fn handle_accept_error_transient_resets_unknown_counter() { + let mut res = 0; + let mut unk = 0; + let unknown_err = std::io::Error::from_raw_os_error(libc::EPERM); + let transient_err = std::io::Error::from_raw_os_error(libc::ECONNABORTED); + + // Accumulate unknowns up to the limit. + for _ in 1..=MAX_CONSECUTIVE_UNKNOWN_ERRORS { + handle_accept_error(&unknown_err, &mut res, &mut unk); + } + assert_eq!(unk, MAX_CONSECUTIVE_UNKNOWN_ERRORS); + + // A transient error resets the unknown counter. + let outcome = handle_accept_error(&transient_err, &mut res, &mut unk); + assert!(outcome.backoff.is_some()); + assert_eq!(unk, 0); + + // Unknown errors can retry again from zero. + let outcome = handle_accept_error(&unknown_err, &mut res, &mut unk); + assert!(outcome.backoff.is_some()); + assert_eq!(unk, 1); + } + + #[cfg(unix)] + #[test] + fn handle_accept_error_fd_exhaustion_uses_exponential_backoff() { + let mut res = 0; + let mut unk = 0; + let err = std::io::Error::from_raw_os_error(libc::EMFILE); + + let b1 = handle_accept_error(&err, &mut res, &mut unk) + .backoff + .unwrap(); + let b2 = handle_accept_error(&err, &mut res, &mut unk) + .backoff + .unwrap(); + let b3 = handle_accept_error(&err, &mut res, &mut unk) + .backoff + .unwrap(); + + assert_eq!(b1.as_millis(), 100); + assert_eq!(b2.as_millis(), 200); + assert_eq!(b3.as_millis(), 400); + } + + #[cfg(unix)] + #[test] + fn classify_accept_error_network_errors_are_transient() { + for errno in [ + libc::ENETDOWN, + libc::EPROTO, + libc::ENOPROTOOPT, + libc::EHOSTDOWN, + libc::EHOSTUNREACH, + libc::EOPNOTSUPP, + libc::ENETUNREACH, + libc::ESOCKTNOSUPPORT, + libc::EPROTONOSUPPORT, + libc::ETIMEDOUT, + ] { + assert_eq!( + classify_accept_error(&std::io::Error::from_raw_os_error(errno)), + AcceptErrorClass::Transient, + "errno {errno} should be transient", + ); + } + } + + #[cfg(target_os = "linux")] + #[test] + fn classify_accept_error_enonet_is_transient() { + assert_eq!( + classify_accept_error(&std::io::Error::from_raw_os_error(libc::ENONET)), + AcceptErrorClass::Transient, + ); + } + + #[cfg(unix)] + #[test] + fn classify_accept_error_resource_pressure_is_transient() { + assert_eq!( + classify_accept_error(&std::io::Error::from_raw_os_error(libc::ENOBUFS)), + AcceptErrorClass::Transient, + ); + assert_eq!( + classify_accept_error(&std::io::Error::from_raw_os_error(libc::ENOMEM)), + AcceptErrorClass::Transient, + ); + assert_eq!( + classify_accept_error(&std::io::Error::from_raw_os_error(libc::ENOSR)), + AcceptErrorClass::Transient, + ); + } + + #[cfg(unix)] + #[test] + fn handle_accept_error_non_resource_transient_uses_fixed_backoff() { + let mut res = 0; + let mut unk = 0; + let err = std::io::Error::from_raw_os_error(libc::ECONNABORTED); + + let o1 = handle_accept_error(&err, &mut res, &mut unk); + let o2 = handle_accept_error(&err, &mut res, &mut unk); + + assert_eq!(o1.severity, SeverityId::Low); + assert_eq!(o1.backoff.unwrap().as_millis(), 100); + assert_eq!(o2.backoff.unwrap().as_millis(), 100); + assert_eq!(res, 0); + } + + #[cfg(unix)] + #[test] + fn handle_accept_error_unknown_uses_exponential_backoff() { + let mut res = 0; + let mut unk = 0; + let err = std::io::Error::from_raw_os_error(libc::EPERM); + + let b1 = handle_accept_error(&err, &mut res, &mut unk) + .backoff + .unwrap(); + let b2 = handle_accept_error(&err, &mut res, &mut unk) + .backoff + .unwrap(); + let b3 = handle_accept_error(&err, &mut res, &mut unk) + .backoff + .unwrap(); + + assert_eq!(b1.as_millis(), 100); + assert_eq!(b2.as_millis(), 200); + assert_eq!(b3.as_millis(), 400); + } + + #[cfg(unix)] + #[test] + fn handle_accept_error_resource_counter_persists_across_mixed_transient() { + let mut res = 0; + let mut unk = 0; + let resource_err = std::io::Error::from_raw_os_error(libc::EMFILE); + let transient_err = std::io::Error::from_raw_os_error(libc::ECONNABORTED); + + let o1 = handle_accept_error(&resource_err, &mut res, &mut unk); + assert_eq!(res, 1); + assert_eq!(o1.backoff.unwrap().as_millis(), 100); + + let o2 = handle_accept_error(&transient_err, &mut res, &mut unk); + assert_eq!(res, 1); + assert_eq!(o2.backoff.unwrap().as_millis(), 100); + + let o3 = handle_accept_error(&resource_err, &mut res, &mut unk); + assert_eq!(res, 2); + assert_eq!(o3.backoff.unwrap().as_millis(), 200); + } + + #[cfg(unix)] + #[test] + fn handle_accept_error_terminal_leaves_counters_unchanged() { + let mut res = 3; + let mut unk = 2; + let err = std::io::Error::from_raw_os_error(libc::EBADF); + + let outcome = handle_accept_error(&err, &mut res, &mut unk); + assert!(outcome.backoff.is_none()); + assert_eq!(res, 3); + assert_eq!(unk, 2); + } } diff --git a/crates/openshell-supervisor-network/tests/accept_fd_exhaustion.rs b/crates/openshell-supervisor-network/tests/accept_fd_exhaustion.rs new file mode 100644 index 0000000000..3fd09f9c89 --- /dev/null +++ b/crates/openshell-supervisor-network/tests/accept_fd_exhaustion.rs @@ -0,0 +1,113 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Subprocess-isolated integration test: lowers `RLIMIT_NOFILE`, induces +//! EMFILE on `accept()`, releases file descriptors, and verifies the +//! listener recovers and accepts a subsequent connection. +//! +//! Uses blocking I/O in the child to surface EMFILE reliably across +//! platforms (tokio's async accept may swallow EMFILE on kqueue-based +//! systems). The proxy's retry logic is validated by the unit tests for +//! `handle_accept_error`; this test validates the OS-level precondition +//! that recovery is possible after FD exhaustion clears. +//! +//! Runs in a child process so the test runner's process-wide resource +//! limits are not affected. + +#![cfg(unix)] +#![allow(unsafe_code, reason = "setrlimit requires unsafe")] + +use std::env; +use std::io::Read; +use std::io::Write; +use std::process::Command; + +const CHILD_SENTINEL: &str = "__ACCEPT_FD_EXHAUSTION_CHILD"; + +#[test] +fn accept_recovers_after_fd_exhaustion() { + let exe = env::current_exe().expect("current_exe"); + let output = Command::new(exe) + .env(CHILD_SENTINEL, "1") + .arg("--test-threads=1") + .arg("accept_fd_exhaustion_child") + .output() + .expect("spawn child"); + + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + output.status.success(), + "child failed (exit {}):\nstdout: {stdout}\nstderr: {stderr}", + output.status, + ); +} + +#[test] +fn accept_fd_exhaustion_child() { + if env::var(CHILD_SENTINEL).is_err() { + return; + } + + let limit = libc::rlimit { + rlim_cur: 32, + rlim_max: 32, + }; + assert_eq!( + unsafe { libc::setrlimit(libc::RLIMIT_NOFILE, std::ptr::from_ref(&limit)) }, + 0, + ); + + let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind"); + let addr = listener.local_addr().unwrap(); + + // Place a connection in the kernel backlog before exhausting FDs. + let backlog_conn = std::net::TcpStream::connect(addr).expect("backlog connect"); + + // Exhaust remaining FDs. + #[allow(clippy::collection_is_never_read)] + let mut held_fds = Vec::new(); + while let Ok(f) = std::fs::File::open("/dev/null") { + held_fds.push(f); + } + + // accept() should fail with EMFILE: there's a connection in the + // backlog but no FD available for the accepted socket. + let first = listener.accept(); + match first { + Err(ref e) if e.raw_os_error() == Some(libc::EMFILE) => {} + Err(ref e) => panic!("expected EMFILE, got: {e}"), + Ok(_) => { + panic!("EMFILE not triggered: OS found a spare FD slot despite RLIMIT_NOFILE=32"); + } + } + + // Release FDs and close the backlog connection. + drop(held_fds); + drop(backlog_conn); + + // Make a fresh connection now that FDs are available. + let client = std::net::TcpStream::connect(addr).expect("connect after FD release"); + + // Accept should succeed, proving that an accept loop retrying on + // EMFILE (like the proxy does) will recover once FDs are available. + // On Linux the closed backlog connection may still be queued ahead + // of the fresh client (EMFILE fires before dequeue), so use a read + // timeout to detect and drain it. + let (accepted, _peer) = listener + .accept() + .expect("accept should succeed after FD release"); + + // Verify the connection is functional. + (&client).write_all(b"ping").expect("write"); + accepted + .set_read_timeout(Some(std::time::Duration::from_secs(1))) + .expect("set timeout"); + let mut buf = [0u8; 4]; + if (&accepted).read_exact(&mut buf).is_err() { + // Got the stale backlog socket; accept the fresh connection. + let (fresh, _) = listener.accept().expect("accept fresh connection"); + (&fresh).read_exact(&mut buf).expect("read"); + } + assert_eq!(&buf, b"ping"); +}