diff --git a/docs/src/design/orchestrator/orchestrator-model.md b/docs/src/design/orchestrator/orchestrator-model.md index 3d39187de..4ade85eb2 100644 --- a/docs/src/design/orchestrator/orchestrator-model.md +++ b/docs/src/design/orchestrator/orchestrator-model.md @@ -65,12 +65,17 @@ verification. ``` Required — attempt recovery; re-walk the chain after `Restored`; latch - `Locked` if the retry cap is reached + `Locked` once recovery is exhausted Isolable — hold the component in reset; advance past it; continue the walk Cascading — same as Isolable, and additionally hold in reset any component whose `depends_on` names this one ``` +Recovery is *exhausted* either when the retry cap is reached or when the +platform reports `RecoveryUnavailable` — both run the same policy branch, so a +platform that is out of recovery images gates an `Isolable` component rather +than halting the platform. + ### `RegionId` An opaque `u8` that groups components into a *recovery region*. Components @@ -361,7 +366,10 @@ names **what** must happen to **which** component; the platform decides **how**. For example, `Effect::RecoverComponent(id)` says only "recover this component" — whether that resolves to a golden-image restore, an A/B slot swap, a streamed image, or a vendor-specific scheme is a platform/configuration decision, never -encoded in the core. +encoded in the core. The platform reports the outcome as an event: `Restored` +once it has swapped in an untried image, `RecoveryUnavailable` when it has none +left. Both are verdicts; `EffectError` stays reserved for a genuine actuation +fault, which fails closed to `Locked` regardless of policy. The core never reads flash, never checks signatures, never observes reset lines. It only emits descriptions. The complete split: diff --git a/services/orchestrator/sm/src/lib.rs b/services/orchestrator/sm/src/lib.rs index 07f9f4d3c..c6ff4e2c4 100644 --- a/services/orchestrator/sm/src/lib.rs +++ b/services/orchestrator/sm/src/lib.rs @@ -424,6 +424,42 @@ impl Rot { } } + /// Whether a `VerificationPassed(id)` may release `id`. Two conditions, and + /// both release sites check them through here so they cannot drift apart: + /// + /// - `id` is the component currently under verification (`chain[cursor]`, + /// whose `VerifyFirmware` was just emitted). A verdict for any other id is + /// stale or out-of-turn (cf. INV9 for `ComponentReady`). + /// - `id` is not gated. Gating does not move the cursor off the component, + /// so a component isolated *while its verification was in flight* is still + /// `chain[cursor]` when that verdict lands — and a stale pass must not + /// undo the `AssertReset` that just held it (INV8). + fn may_release(&self, id: ComponentId) -> bool { + self.chain.get(self.cursor as usize).map(|(c, _)| c) == Some(&id) && !self.is_gated(id) + } + + /// Recovery of `target` has run out of options. Gate it per policy: + /// `Isolable`/`Cascading` are skipped and the walk continues, while + /// `Required` (or an unknown id) reports the component and latches + /// [`State::Locked`]. Both exhaustion paths — the retry cap + /// ([`Event::Restored`] past `max_retry`) and the platform's + /// [`Event::RecoveryUnavailable`] — share this so they cannot diverge. + fn exhaust_recovery(&mut self, ctx: &mut Sink, target: ComponentId) -> Outcome { + match self.gate_by_policy(ctx, target) { + Gating::Gated => { + self.clear_retry(target); + Outcome::Transition(State::PreSupervision) + } + // The report precedes the internal `Emit`, so it is actuated before + // the machine moves toward `Locked`. + Gating::NotGated => { + ctx.emit(Effect::ReportRecoveryFailed(target)); + ctx.emit(Effect::Emit(Event::RecoveryFailed)); + Outcome::Handled + } + } + } + /// Shared `CorruptionDetected` handling, called from both `PreSupervision` /// (directly) and `SupervisingPlatform` (via its superstate handler). /// Delegates the policy interpretation to [`gate_by_policy`](Self::gate_by_policy) @@ -498,13 +534,9 @@ impl Rot { // Cursor walk via Outcome::Handled — a self-transition would reset cursor. State::PreSupervision => match event { Event::VerificationPassed(id) => { - // Only the component currently under verification - // (`chain[cursor]`, whose `VerifyFirmware` was just emitted) - // may be released. A verdict for any other id is stale or - // out-of-turn and is dropped, so a misordered or hostile - // report cannot release an unverified component (cf. INV9 - // for `ComponentReady`). - if self.chain.get(self.cursor as usize).map(|(c, _)| c) != Some(id) { + // A misordered, stale, or hostile verdict cannot release an + // unverified or already-isolated component. + if !self.may_release(*id) { return Outcome::Handled; } // The component passed its check — it has recovered, so its @@ -596,11 +628,8 @@ impl Rot { } } Event::VerificationPassed(id) => { - // Only the component currently under verification - // (`chain[cursor]`) may be released; a verdict for any other - // id is stale or out-of-turn and is dropped (cf. INV9 for - // `ComponentReady`). - if self.chain.get(self.cursor as usize).map(|(c, _)| c) != Some(id) { + // Same guard as the `PreSupervision` release site. + if !self.may_release(*id) { return Outcome::Handled; } // The component passed its check — it has recovered, so its @@ -707,26 +736,19 @@ impl Rot { if attempts < self.max_retry { Outcome::Transition(State::PreSupervision) } else { - // Retries exhausted: gate via the same `gate_by_policy` - // the runtime-corruption path uses, so the two can never - // disagree. Gated → continue the walk; NotGated - // (Required/unknown) → lock down. - match self.gate_by_policy(ctx, failed) { - Gating::Gated => { - self.clear_retry(failed); - Outcome::Transition(State::PreSupervision) - } - // `Required`, or an unknown/missing id: report the - // component that forced the halt, then lock down. - // The report precedes the internal `Emit`, so it is - // actuated before the machine moves toward `Locked`. - Gating::NotGated => { - ctx.emit(Effect::ReportRecoveryFailed(failed)); - ctx.emit(Effect::Emit(Event::RecoveryFailed)); - Outcome::Handled - } - } + self.exhaust_recovery(ctx, failed) + } + } + Event::RecoveryUnavailable(id) => { + // Same target guard as `Restored`: a verdict for a + // component other than the one under recovery is dropped. + if *id != failed { + return Outcome::Handled; } + // No `bump_retry`: the platform is authoritative about being + // out of sources, so this short-circuits the cap rather than + // waiting for it to run out on non-progressing restores. + self.exhaust_recovery(ctx, failed) } Event::RecoveryFailed => Outcome::Transition(State::Locked), _ => Outcome::Super, @@ -918,6 +940,13 @@ pub struct EffectError; /// `VerifyFirmware` covers code that cannot run or rewrite its own flash /// between the check and the release. A reset that merely pulses would let a /// component resume before verification and void that guarantee. +/// - **`EffectError` means actuation failed, not "policy says no".** It is +/// fail-closed: it latches [`State::Locked`] from any state. In particular, +/// an [`Effect::RecoverComponent`] with no untried recovery source left is +/// *not* an error — the driver returns `Ok` and reports +/// [`Event::RecoveryUnavailable`], which routes through [`FailurePolicy`]. +/// Returning `Err` there locks the whole platform down for a component the +/// board may have classified `Isolable`. /// - **A failed [`Effect::LatchLockdown`] is a hard fault.** Lockdown is the top /// of the escalation ladder — the core has nothing stronger to emit and /// will *believe* it is `Locked`. The driver must treat that failure as diff --git a/services/orchestrator/sm/src/model.rs b/services/orchestrator/sm/src/model.rs index f9212695c..200ae3a03 100644 --- a/services/orchestrator/sm/src/model.rs +++ b/services/orchestrator/sm/src/model.rs @@ -43,9 +43,11 @@ pub enum ComponentKind { Passive, } -/// Recovery-failure classification: what the machine does once a required -/// component's restore attempts are **exhausted** (its per-component retry -/// count reaches `max_retry`). Every verification or corruption failure enters +/// Recovery-failure classification: what the machine does once a component's +/// recovery is **exhausted** — either its per-component retry count reaches +/// `max_retry`, or the platform reports [`Event::RecoveryUnavailable`] because +/// it has no remaining recovery source. Every verification or corruption +/// failure enters /// [`State::Recovering`] and is retried first, regardless of this /// classification — CSA's "recover first" principle. This value is consulted /// only after retries are exhausted. @@ -213,6 +215,11 @@ pub enum Event { CorruptionDetected(ComponentId), /// This component has been restored from its configured recovery source. Restored(ComponentId), + /// The platform driver is out of recovery sources for this component. + /// Routed through [`FailurePolicy`] (gate, not the fail-closed + /// [`Event::EffectFailed`] lockdown). Authoritative: exhausts recovery at + /// once, bypassing the retry cap. + RecoveryUnavailable(ComponentId), /// A required component's recovery was exhausted. RecoveryFailed, /// The platform driver's boot-progress watchdog fired: `id` did not report its @@ -261,6 +268,7 @@ impl Event { | Event::BootConfirmed(id) | Event::CorruptionDetected(id) | Event::Restored(id) + | Event::RecoveryUnavailable(id) | Event::Timeout(id) => Some(*id), Event::PowerGood(_) | Event::AttestationChallenge @@ -307,6 +315,14 @@ pub enum Effect { /// per configuration policy. The core only names the component to /// recover; it does not encode how recovery is performed. /// + /// Two-phase: this starts recovery, and the driver later reports the + /// verdict as an event — [`Event::Restored`] once it has swapped in an + /// untried image, or [`Event::RecoveryUnavailable`] when it has none left. + /// "Out of images" is a verdict, not an actuation failure: returning + /// `EffectError` for it would latch [`State::Locked`] regardless of the + /// component's [`FailurePolicy`]. Reserve `EffectError` for a genuine swap + /// fault (bus error, hardware fault). + /// /// `attempt` is this component's consecutive-recovery count (0 on the first /// attempt), taken straight from the core's own retry counter — the same /// value the retry cap is measured against. It rides on the effect so the diff --git a/services/orchestrator/sm/src/tests.rs b/services/orchestrator/sm/src/tests.rs index 2eab2a71e..92bf05b41 100644 --- a/services/orchestrator/sm/src/tests.rs +++ b/services/orchestrator/sm/src/tests.rs @@ -1031,6 +1031,196 @@ fn required_exhaustion_reports_before_lockdown() { ); } +/// The platform reporting "no recovery source left" for an `Isolable` +/// component gates it exactly like count-driven exhaustion does — the walk +/// continues and the platform boots degraded rather than locking down. +#[test] +fn isolable_recovery_unavailable_skips() { + let (effects, state) = drive( + chain(&[ + (C0, ComponentAttrs::passive_required()), + (C1, ComponentAttrs::passive_isolable()), + ]), + &[ + BOOT, + Event::VerificationPassed(C0), + Event::VerificationFailed(C1), // → Recovering(C1) + Event::RecoveryUnavailable(C1), // platform: no image left + Event::VerificationPassed(C0), // re-walk from the top + ], + ); + assert_eq!(state, State::Ready); + assert!(effects.contains(&Effect::RecoverComponent { id: C1, attempt: 0 })); + assert!(effects.contains(&Effect::AssertReset(C1))); + assert!(effects.contains(&Effect::ReportIsolated(C1))); + assert!(!effects.contains(&Effect::ReleaseReset(C1))); + assert!(!effects.contains(&Effect::LatchLockdown)); +} + +/// The same platform signal on a `Required` component is *not* graceful: it +/// names the component and latches, exactly as count-driven exhaustion does. +#[test] +fn required_recovery_unavailable_locks() { + let (effects, state) = drive( + passive_required(&[C0]), + &[ + BOOT, + Event::VerificationFailed(C0), // → Recovering(C0) + Event::RecoveryUnavailable(C0), + ], + ); + assert_eq!(state, State::Locked); + let report = effects + .iter() + .position(|e| *e == Effect::ReportRecoveryFailed(C0)) + .expect("the component that forced the halt is reported"); + let latch = effects + .iter() + .position(|e| *e == Effect::LatchLockdown) + .expect("the platform latches"); + assert!( + report < latch, + "the report must be actuated before the latch" + ); +} + +/// A `Cascading` component the platform can no longer restore takes its +/// transitive dependents down with it, exactly as count-driven exhaustion does. +#[test] +fn cascading_recovery_unavailable_cascades() { + let (effects, state) = drive( + chain(&[ + (C0, ComponentAttrs::passive_required()), + (C1, ComponentAttrs::passive_cascading()), + (C2, ComponentAttrs::passive_required().with_depends_on(C1)), + ]), + &[ + BOOT, + Event::VerificationPassed(C0), + Event::VerificationFailed(C1), + Event::RecoveryUnavailable(C1), + Event::VerificationPassed(C0), + ], + ); + assert_eq!(state, State::Ready); + assert!(effects.contains(&Effect::AssertReset(C1))); + assert!(effects.contains(&Effect::AssertReset(C2))); + assert!(effects.contains(&Effect::ReportIsolated(C1))); + assert!(effects.contains(&Effect::ReportIsolated(C2))); + assert!(!effects.contains(&Effect::LatchLockdown)); +} + +/// The platform's verdict is authoritative: one `RecoveryUnavailable` exhausts +/// recovery immediately, without burning the retry budget on restores the +/// platform has already said it cannot perform. +#[test] +fn recovery_unavailable_short_circuits_retry_budget() { + let mut c = heapless::Vec::<(ComponentId, ComponentAttrs), CAPACITY>::new(); + c.push((C0, ComponentAttrs::passive_required())) + .expect("fits"); + c.push((C1, ComponentAttrs::passive_isolable())) + .expect("fits"); + let mut orch = Orchestrator::::new(c.try_into().expect("valid chain"), 200); + let mut effects = Vec::new(); + for ev in [ + BOOT, + Event::VerificationPassed(C0), + Event::VerificationFailed(C1), + Event::RecoveryUnavailable(C1), + Event::VerificationPassed(C0), + ] { + orch.dispatch_with(ev, |e| { + effects.push(e); + Ok(None) + }); + } + assert_eq!(orch.state(), State::Ready); + assert_eq!( + effects + .iter() + .filter(|e| matches!(e, Effect::RecoverComponent { id, .. } if *id == C1)) + .count(), + 1, + "the cap was never consulted, so recovery is attempted exactly once", + ); + assert!(effects.contains(&Effect::ReportIsolated(C1))); +} + +/// Same target guard as `Restored`: a verdict naming a component other than the +/// one under recovery is not credited to this episode. +#[test] +fn recovery_unavailable_other_component_dropped() { + let (effects, state) = drive( + chain(&[ + (C0, ComponentAttrs::passive_required()), + (C1, ComponentAttrs::passive_isolable()), + (C2, ComponentAttrs::passive_isolable()), + ]), + &[ + BOOT, + Event::VerificationPassed(C0), + Event::VerificationFailed(C1), // → Recovering(C1) + Event::RecoveryUnavailable(C2), + ], + ); + assert_eq!(state, State::Recovering(C1)); + assert!(!effects.contains(&Effect::AssertReset(C2))); + assert!(!effects.contains(&Effect::ReportIsolated(C2))); +} + +/// The new variant names a component, so it must be listed in +/// `Event::component_id` — that is the single enumeration the dispatch boundary +/// consults to drop off-chain ids before any handler runs. Asserted directly: +/// end to end the `Recovering` target guard would mask an unlisted variant, so +/// only this can catch the omission. +#[test] +fn recovery_unavailable_names_its_component() { + assert_eq!(Event::RecoveryUnavailable(C3).component_id(), Some(C3)); +} + +/// INV8: an isolated component is never released, however good the verdict. +/// Gating a component does not move the cursor off it, so the in-flight +/// `VerifyFirmware` issued before the corruption report can still be answered +/// — and that stale pass must not undo the `AssertReset` that just held it. +#[test] +fn stale_verdict_never_releases_an_isolated_component() { + let (effects, _state) = drive( + chain(&[ + (C0, ComponentAttrs::passive_required()), + (C1, ComponentAttrs::passive_isolable()), + ]), + &[ + BOOT, + Event::VerificationPassed(C0), // → VerifyFirmware(C1) in flight + Event::CorruptionDetected(C1), // → AssertReset(C1), isolated + Event::VerificationPassed(C1), // the in-flight verdict, now stale + ], + ); + assert!(effects.contains(&Effect::AssertReset(C1))); + assert!(!effects.contains(&Effect::ReleaseReset(C1))); +} + +/// Same hole on the supervised release site: a component isolated while the +/// walk is parked in `AwaitingReady` must not be released by the verdict for +/// the verification that was already in flight. +#[test] +fn stale_verdict_never_releases_an_isolated_component_in_awaiting_ready() { + let (effects, _state) = drive( + chain(&[ + (C0, ComponentAttrs::active_required()), + (C1, ComponentAttrs::passive_isolable()), + ]), + &[ + BOOT, + Event::VerificationPassed(C0), // → AwaitingReady(C0); VerifyFirmware(C1) in flight + Event::CorruptionDetected(C1), // → AssertReset(C1), isolated + Event::VerificationPassed(C1), // the in-flight verdict, now stale + ], + ); + assert!(effects.contains(&Effect::AssertReset(C1))); + assert!(!effects.contains(&Effect::ReleaseReset(C1))); +} + /// A component that recovers within its retry budget is not degraded, so /// nothing is reported — reports mark components taken *out of service*, not /// every transient failure. @@ -1884,7 +2074,7 @@ impl SplitMix64 { /// Build one random event over the given id palette. Id-less events ignore it. fn random_event(rng: &mut SplitMix64, ids: &[ComponentId]) -> Event { let id = ids[rng.below(ids.len() as u32) as usize]; - match rng.below(15) { + match rng.below(16) { 0 => Event::VerificationPassed(id), 1 => Event::VerificationFailed(id), 2 => Event::ComponentReady(id), @@ -1899,6 +2089,7 @@ fn random_event(rng: &mut SplitMix64, ids: &[ComponentId]) -> Event { 11 => Event::UpdateRejected, 12 => Event::RecoveryFailed, 13 => Event::CommitTimeout, + 14 => Event::RecoveryUnavailable(id), _ => Event::EffectFailed, } }