From 92c22a2c67c667444f9d6155bd67150fb14673a0 Mon Sep 17 00:00:00 2001 From: japabu Date: Tue, 13 Oct 2026 07:21:55 +0200 Subject: [PATCH 1/5] fileserver: a FAT entry that grows its file is a cache flush after its chain and data The cache writes its dirty blocks lowest first, and a FAT volume's directory clusters lie below the data it allocates later. So a sync put the grown entry, the chain and the data in one flush, and the disk took the entry before the clusters it names: a machine stopped there mounts a file whose appended bytes are whatever those clusters held. A disk with a volatile write cache keeps no order between writes until a flush either. `level` now flushes the cache before it writes an entry that reaches more than the disk holds (`File::needs_reconcile`), so the chain and the data are durable before the entry is in the cache. The module test the step-10 exit owes holds it: an append and its sync, stopped after each block write in turn, mount and read back either the last whole sync's bytes or the append's, never anything else. Without the flush it is red at block write 9 of 13: a 15,288-byte file that is neither the 3,000 bytes the last sync held nor the append. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015xhaydXCVRNscoccHYFUWy --- system/fileserver/src/fat.rs | 89 +++++++++++++++++++++++++++++++++++- 1 file changed, 88 insertions(+), 1 deletion(-) diff --git a/system/fileserver/src/fat.rs b/system/fileserver/src/fat.rs index 7d797d6923..c95e7ca08b 100644 --- a/system/fileserver/src/fat.rs +++ b/system/fileserver/src/fat.rs @@ -5,7 +5,10 @@ //! Between them the volume holds clusters its entry does not reach, which //! fatgen103 refuses, so every open file is brought level — its entry written, //! its chain reconciled — at its last close, at an fsync and at a sync, as -//! the kernel's adapter did. +//! the kernel's adapter did. **The two are a cache flush apart**: the cache +//! writes lowest block first and the disk keeps no order until a flush, so an +//! entry put into the cache beside its chain and data would reach the disk +//! before the clusters it names. //! //! **A create makes the directories on its way**, as the kernel's did: a log //! file's path is its own, and FAT has the directories to carry it. A @@ -181,6 +184,11 @@ impl FatVolume { if open.gone { return Ok(()); } + // The chain and the data on the disk before the entry that reaches + // them is in the cache. + if open.file.needs_reconcile() { + self.cache.flush().map_err(|_| logged("the chain", &open.path, Error::Io))?; + } self.fs.flush_meta(&mut open.file, time).map_err(|e| logged("the entry", &open.path, e))?; if open.file.needs_reconcile() { self.fs.reconcile(&mut open.file, time).map_err(|e| logged("a reconcile", &open.path, e))?; @@ -724,6 +732,85 @@ mod tests { assert_eq!(v.sync(), Ok(Vec::new())); } + /// A disk that keeps every block it is written, in order, so a test can + /// stop it after any one of them. + struct Traced { + ram: Ram, + writes: Rc)>>>, + } + + impl Disk for Traced { + fn blocks(&self) -> u64 { + self.ram.blocks() + } + fn read(&mut self, first: u64, out: &mut [u8]) -> Result<(), DiskError> { + self.ram.read(first, out) + } + fn write(&mut self, first: u64, data: &[u8]) -> Result<(), DiskError> { + let mut writes = self.writes.borrow_mut(); + for (i, block) in data.chunks_exact(BLOCK).enumerate() { + writes.push((first + i as u64, block.to_vec())); + } + self.ram.write(first, data) + } + fn flush(&mut self) -> Result<(), DiskError> { + self.ram.flush() + } + } + + /// An append and its sync, stopped after each block write in turn, leave + /// a volume that mounts and holds the file as the last whole sync left it + /// or as the append made it: never an entry naming bytes its chain does + /// not reach or its clusters do not hold yet. + #[test] + fn an_append_stopped_after_any_block_write_reads_back_the_last_sync() { + const CREATE: OpenHow = OpenHow { create: true, create_new: false, truncate: false }; + const OPEN: OpenHow = OpenHow { create: false, create_new: false, truncate: false }; + let clock = || 1_717_245_296 * NANOS_PER_SEC; + let spec = spec_volume::fixture(); + let blocks = spec.bytes.len().div_ceil(BLOCK); + let mut image = spec.bytes.clone(); + image.resize(blocks * BLOCK, 0); + let disk = |trace: &[(u64, Vec)]| { + let mut ram = Ram::new(blocks as u64); + ram.write(0, &image).unwrap(); + for (block, data) in trace { + ram.write(*block, data).unwrap(); + } + ram + }; + let writes = Rc::new(std::cell::RefCell::new(Vec::new())); + let mut v = FatVolume::mount(Traced { ram: disk(&[]), writes: Rc::clone(&writes) }, true, clock).unwrap(); + let held: Vec = (0..3000u32).map(|i| (i % 251 + 1) as u8).collect(); + let appended: Vec = (0..3 * BLOCK as u32).map(|i| (i % 241 + 2) as u8).collect(); + let a = v.open("a.txt", CREATE).unwrap(); + v.write(a, 0, &held).unwrap(); + assert_eq!(v.sync(), Ok(Vec::new())); + let synced = writes.borrow().len(); + v.write(a, held.len() as u64, &appended).unwrap(); + assert_eq!(v.sync(), Ok(Vec::new())); + let trace = writes.borrow().clone(); + let whole = [&held[..], &appended[..]].concat(); + + for stop in synced..=trace.len() { + let mut after = FatVolume::mount(disk(&trace[..stop]), false, clock) + .unwrap_or_else(|e| panic!("stopped after block write {stop} of {}: {e}", trace.len())); + let n = after.open("a.txt", OPEN).unwrap(); + let size = after.lstat("a.txt").unwrap().size as usize; + let mut out = vec![0u8; size]; + let read = after.read(n, 0, &mut crate::volume::Buf(&mut out)); + assert_eq!(read, Ok(size), "stopped after block write {stop} of {}: a {size}-byte file", trace.len()); + assert!( + out == held || out == whole, + "stopped after block write {stop} of {}: the file is {size} bytes and neither the {} the last \ + sync held nor the {} the append made", + trace.len(), + held.len(), + whole.len() + ); + } + } + /// What the driver says of a volume that stopped answering is `Io`, which /// no caller takes for a name that is not there. #[test] From 671e5c73679eba4640a515a1628496058cf9ccde Mon Sep 17 00:00:00 2001 From: japabu Date: Tue, 13 Oct 2026 07:46:54 +0200 Subject: [PATCH 2/5] diskserver: an open's answer is told only once its ends are made, in one function a host test reopens a session through A client that opens its session again sends the page its killed server left, and looks at it the moment it hears. So the server's stores of its two cursors come before its answer. That order spanned `Service::open`, which made the ends, and `handshake`, which sent the answer, and both stand on syscalls, so no host test reached it. `answer` now makes the ends and only then calls the closure that tells the client; `handshake` passes it the `MSG_OPENED` send. Its test leaves a used page, a request asked, taken and answered and the server gone, makes the client's ends again as `Session::reconnect` does, and looks at the page from inside the answer: room for every request and nothing to read. With the answer told before `Opened::over`, it reads `Err(HeadPastTail)`. Closes issues/nothing-reopens-a-diskserver-session-over-a-used-page.md: the test is the one its exit and the track's step-10 exit name. What the client glue does with such a page is still read off the code, and is issues/nothing-runs-a-block-sessions-client-glue-on-a-host.md. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015xhaydXCVRNscoccHYFUWy --- ...s-a-diskserver-session-over-a-used-page.md | 36 --------- ...-a-block-sessions-client-glue-on-a-host.md | 23 ++++++ ...-small-interrupts-post-and-threads-wait.md | 12 +-- system/diskserver/src/main.rs | 74 +++++++++++++++---- 4 files changed, 88 insertions(+), 57 deletions(-) delete mode 100644 issues/nothing-reopens-a-diskserver-session-over-a-used-page.md create mode 100644 issues/nothing-runs-a-block-sessions-client-glue-on-a-host.md diff --git a/issues/nothing-reopens-a-diskserver-session-over-a-used-page.md b/issues/nothing-reopens-a-diskserver-session-over-a-used-page.md deleted file mode 100644 index c03cc56c4e..0000000000 --- a/issues/nothing-reopens-a-diskserver-session-over-a-used-page.md +++ /dev/null @@ -1,36 +0,0 @@ ---- -status: open -kind: tooling -opened: 2026-10-02 ---- - -# Nothing reopens a blockd session over a used page - -A nightly went red on `blockd_io: FAIL /AFTER.BIN after the restart: Io` (run -36753172688, guest 7). blockd answered an open and only then made its ends of -the session page. A client that reconnects sends the page its last server -wrote, and one that looks before the new server's ends are made has no room to -ask (`Violation::HeadPastTail`) and hears the dead session's completion -(`Violation::Tag`). The order is a type now: `wire::Opened::over` makes a -server's ends and its answer together. - -That was fixed by reading. Nothing reproduced the `Io` before it, and nothing -that runs reopens a session over a used page: - -- blockd's host test reaches `Service::place` and `Service::listing`, never - `Service::open`; -- `toyos-blockring`'s model test holds what the transport answers over a page - a crashed session left, and is green with blockd's order reverted; -- no host runs the client's glue (`Session::pump`, `drain` and `wait` in - `userland/blockd/src/session.rs`), so that `HeadPastTail` and `Tag` end the - session and the caller's call is `Io` is read off it, not run; -- `520c0d129` cut `blockd_survives_its_death`, the guest test that killed - blockd under its client, and no guest test or metal row ends blockd. - -**Exit**: the restart test that Stage D of -`issues/the-guest-suite-runs-only-what-no-cheaper-tier-reaches.md` owes -for `blockd_survives_its_death`, at the tier that stage builds it on: red with -that `Io` under a mutation that moves blockd's two stores after its answer, -and green without it. - -Owner: the orchestrator, which dispatches Stage D. diff --git a/issues/nothing-runs-a-block-sessions-client-glue-on-a-host.md b/issues/nothing-runs-a-block-sessions-client-glue-on-a-host.md new file mode 100644 index 0000000000..53ab5debf1 --- /dev/null +++ b/issues/nothing-runs-a-block-sessions-client-glue-on-a-host.md @@ -0,0 +1,23 @@ +--- +status: open +kind: tooling +opened: 2026-10-13 +--- + +# Nothing runs a block session's client glue on a host + +diskserver's +`a_session_opened_over_a_killed_servers_page_is_told_after_its_ends_are_made` +holds the server's half of a reopen: a client told its session is open finds +the new server's ends made. What the client does with a page that breaks the +protocol is read off the code and not run. `Session::pump`, `drain` and +`wait` (`system/diskserver/src/session.rs`) hold a `Region`, a `Connection` +and a `Poller`, so no host reaches them: that a `Violation::HeadPastTail` or +`Violation::Tag` ends the session, and that `disk::Served` answers its caller +`Gone`, is unexercised. + +**Exit**: a host test drives the client's ring half over a page whose server +cursor is past what the client published, and the session ends with the +caller's call refused; red with that check in `Session::drain` removed. + +Owner: the orchestrator. diff --git a/issues/the-kernel-is-small-interrupts-post-and-threads-wait.md b/issues/the-kernel-is-small-interrupts-post-and-threads-wait.md index f5309e3018..469efdbef2 100644 --- a/issues/the-kernel-is-small-interrupts-post-and-threads-wait.md +++ b/issues/the-kernel-is-small-interrupts-post-and-threads-wait.md @@ -67,8 +67,8 @@ Beside that order, each waiting on something outside it: `every_request_is_answered_exactly_once`, `what_a_flush_calls_durable_is_on_the_medium` and `what_was_on_the_wire_at_the_end_is_refused_and_what_was_not_waits` for - the transport, by the restart test - `issues/nothing-reopens-a-diskserver-session-over-a-used-page.md` owes + the transport, by diskserver's + `a_session_opened_over_a_killed_servers_page_is_told_after_its_ends_are_made` for diskserver's part, a session reopened over the page a killed server left, red under the mutation that moves the server's stores after its answer, by fileserver's @@ -196,10 +196,10 @@ Beside that order, each waiting on something outside it: same controller delivering every report, red on a usbd that waits on the stick before it reads the keyboard's ring. `/log`'s, what a killed usbd had on the wire: stage 3's `toyos-blockring` tests for the - transport, and for usbd's session open the reopen test - `issues/nothing-reopens-a-diskserver-session-over-a-used-page.md` - owes, red under the mutation that moves usbd's stores after its - answer; with one test this step owes in + transport, and for usbd's session open diskserver's + `a_session_opened_over_a_killed_servers_page_is_told_after_its_ends_are_made`, + moved with the open, red under the mutation that moves usbd's stores + after its answer; with one test this step owes in `system/fileserver/src/fat.rs`: a disk stopped after each block write in turn of an append and its sync mounts at every stop and reads back all the last whole sync held, red on a sync that writes a growing diff --git a/system/diskserver/src/main.rs b/system/diskserver/src/main.rs index 4978fc349f..bdab9b5b84 100644 --- a/system/diskserver/src/main.rs +++ b/system/diskserver/src/main.rs @@ -51,6 +51,7 @@ use std::collections::BTreeMap; +use std::sync::atomic::AtomicU32; use std::time::{Duration, Instant}; use diskserver::region::Region; @@ -63,7 +64,7 @@ use toyos_abi::part::{PartGuid, GUID_TEXT_LEN}; use toyos_abi::syscall::{DEV_PREFIX, MAX_BADGE, SyscallError}; use toyos_blockhold::Holds; use toyos_blockring::entry::{Completion, Op}; -use toyos_blockring::layout::{ServerRings, DEPTH}; +use toyos_blockring::layout::{ServerRings, DEPTH, RING_WORDS}; use toyos_blockring::server::{ServerSession, Taken}; use toyos_blockring::wire::{self, Grant, Opened, Refusal}; use toyos_blockring::{BLOCK_BYTES, PORT}; @@ -308,8 +309,7 @@ struct Service { /// A session decided on and not yet told to its client. struct Opening { region: Region, - rings: ServerRings, - opened: Opened, + blocks: u64, device_addr: u64, first: u64, writes: bool, @@ -358,10 +358,7 @@ impl Service { let (first, blocks) = self.place(guid, grant)?; let writes = grant.expect("placed through a grant").writes; match self.ctrl.up().claim().dma_map(region.handle()) { - Ok(mapping) => { - let (rings, opened) = Opened::over(region.words(), blocks, guid); - Ok(Opening { region, rings, opened, device_addr: mapping.device_addr, first, writes }) - } + Ok(mapping) => Ok(Opening { region, blocks, device_addr: mapping.device_addr, first, writes }), Err(why) => { self.holds.release(first); Err(match why { @@ -372,8 +369,9 @@ impl Service { } } - /// The client heard its session is open: from here it is served. - fn admit(&mut self, opening: Opening, conn: Connection) -> u64 { + /// The client heard its session is open, over the ends [`answer`] made: + /// from here it is served. + fn admit(&mut self, opening: Opening, rings: ServerRings, opened: Opened, conn: Connection) -> u64 { let id = self.next_id; self.next_id += 1; self.sessions.insert( @@ -382,9 +380,9 @@ impl Service { conn, region: opening.region, device_addr: opening.device_addr, - rings: opening.rings, - state: ServerSession::new(opening.first, opening.opened.blocks(), opening.writes), - unique: opening.opened.unique(), + rings, + state: ServerSession::new(opening.first, opened.blocks(), opening.writes), + unique: opened.unique(), closing: false, requests: 0, posted: false, @@ -727,6 +725,21 @@ fn serve(service: &mut Service, acceptor: &toyos::port::Acceptor) -> ! { } } +/// A server's ends of the session `page` it was sent, and the answer that +/// `tell` sends its client, **told only once the ends are made**: a client +/// looks at the page the moment it hears, and one that opens a session again +/// sends the cursors its last server left on it. +fn answer( + page: &[AtomicU32; RING_WORDS], + blocks: u64, + unique: [u8; 16], + tell: impl FnOnce(&Opened) -> R, +) -> (ServerRings, Opened, R) { + let (rings, opened) = Opened::over(page, blocks, unique); + let told = tell(&opened); + (rings, opened, told) +} + /// The grant `conn`, accepted from `acceptor`, was minted with; `None` for a /// connection through an unbadged connector or with bytes no minter stamps. fn grant(acceptor: &toyos::port::Acceptor, conn: &Connection) -> Option { @@ -772,12 +785,15 @@ fn handshake(service: &mut Service, acceptor: &toyos::port::Acceptor, p: Pending refuse(&p.conn, why); } Ok(opening) => { - let (opened, writes) = (opening.opened, opening.writes); - if p.conn.try_send_bytes(wire::MSG_OPENED, &opened.encode()).is_err() { + let (rings, opened, sent) = answer(opening.region.words(), opening.blocks, guid, |opened| { + p.conn.try_send_bytes(wire::MSG_OPENED, &opened.encode()) + }); + if sent.is_err() { service.abandon(opening); return; } - let id = service.admit(opening, p.conn); + let writes = opening.writes; + let id = service.admit(opening, rings, opened, p.conn); let access = if writes { "" } else { ", read-only" }; println!("diskserver: session {id} opened {} ({} blocks{access})", guid_text(guid), opened.blocks()); } @@ -869,4 +885,32 @@ mod tests { assert_eq!(listed(Ok(&parts), unique([9; 16])).map(decoded), Ok(vec![])); assert_eq!(listed(Ok(&parts), None), Err(Refusal::NotGranted)); } + + /// A client that opens its session again sends the page a killed server + /// left, its own two cursors made again (`Session::reconnect`): the moment + /// it is told the session is open, every request has room and nothing of + /// the dead session is there to read. + #[test] + fn a_session_opened_over_a_killed_servers_page_is_told_after_its_ends_are_made() { + use toyos_blockring::layout::client; + use toyos_blockring::{Request, Status}; + + let page: [AtomicU32; RING_WORDS] = core::array::from_fn(|_| AtomicU32::new(0)); + let mut first = client(&page); + let (mut killed, _, ()) = answer(&page, 100, LOG, |_| ()); + assert_eq!(first.0.push(&page, Request { op: Op::Flush, tag: 1 }.encode()), Ok(true)); + first.0.publish(&page); + assert!(killed.0.pop(&page).expect("a published request").is_some()); + killed.0.release(&page); + let answered = Completion { tag: 1, status: Status::Ok }.encode(); + assert_eq!(killed.1.push(&page, answered), Ok(true)); + killed.1.publish(&page); + + let mut again = client(&page); + let (_, _, (room, stale)) = + answer(&page, 100, LOG, |_| (again.0.space(&page), again.1.pop(&page).map(|c| c.is_some()))); + + assert_eq!(room, Ok(DEPTH)); + assert_eq!(stale, Ok(false)); + } } From 7978cdecfc6c10a6157cace34ed1e420af1d32da Mon Sep 17 00:00:00 2001 From: japabu Date: Tue, 13 Oct 2026 09:07:24 +0200 Subject: [PATCH 3/5] diskserver: an open's answer is made inline at its send, and the reopen issue keeps the client's half `answer` and its host test go (ruling 6139586864, from review 6139576035's BLOCKER 4): `Opened::over` already makes a server's ends of the session page and its answer together, and nothing but `Opened::over` or the client's `decode` makes an `Opened`, so the order the test held is a type's. `handshake` now calls `Opened::over` at the `MSG_OPENED` send; `Service::open` no longer makes the ends before the claim is told. `issues/nothing-reopens-a-diskserver-session-over-a-used-page.md` stays open, narrowed to the client's half: a reopen over a used page ends the session with `Io` by `drain`'s `Tag` refusal, held by a client host test red with that refusal removed. The client-glue issue this branch had filed folds into it. The submission-ring hang the review found (`Session::pump` stops on `HeadPastTail` and nothing ends the session, so `wait` blocks in `poller.wait(1, u64::MAX, ..)`) is filed as its own defect, `issues/a-block-session-waits-forever-on-a-server-past-its-submission-ring.md`. The kernel-is-small track's two citations name the type and the issue. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015xhaydXCVRNscoccHYFUWy --- ...er-on-a-server-past-its-submission-ring.md | 24 ++++++++ ...s-a-diskserver-session-over-a-used-page.md | 30 ++++++++++ ...-a-block-sessions-client-glue-on-a-host.md | 23 -------- ...-small-interrupts-post-and-threads-wait.md | 17 +++--- system/diskserver/src/main.rs | 56 ++----------------- 5 files changed, 67 insertions(+), 83 deletions(-) create mode 100644 issues/a-block-session-waits-forever-on-a-server-past-its-submission-ring.md create mode 100644 issues/nothing-reopens-a-diskserver-session-over-a-used-page.md delete mode 100644 issues/nothing-runs-a-block-sessions-client-glue-on-a-host.md diff --git a/issues/a-block-session-waits-forever-on-a-server-past-its-submission-ring.md b/issues/a-block-session-waits-forever-on-a-server-past-its-submission-ring.md new file mode 100644 index 0000000000..d2153d672e --- /dev/null +++ b/issues/a-block-session-waits-forever-on-a-server-past-its-submission-ring.md @@ -0,0 +1,24 @@ +--- +status: open +kind: defect +opened: 2026-10-13 +--- + +# A block session waits forever on a server past its submission ring + +A diskserver client trusts its server's cursor on the submission ring. Where +the server's consumer index is past what the client published, +`Session::pump` (`system/diskserver/src/session.rs`) meets the transport's +`Violation::HeadPastTail` from `space` and only stops pushing, on the comment +that "the next read of the connection is where that is acted on". Nothing +acts on it: `drain` reads only the completion ring, so with no completion to +refuse `wait` blocks in `poller.wait(1, u64::MAX, ..)` until the server hangs +up. A server that keeps its connection and writes that index holds every +caller of the session for good: an unbounded wait on an untrusted peer, where +the producer's violation should end the session as `drain`'s do. + +**Exit**: a submission-ring violation ends the session, so the caller's call +is refused; held by a host test of the client's ring half over a page whose +server cursor is past what the client published, red with that end removed. + +Owner: the orchestrator. diff --git a/issues/nothing-reopens-a-diskserver-session-over-a-used-page.md b/issues/nothing-reopens-a-diskserver-session-over-a-used-page.md new file mode 100644 index 0000000000..897ff1d5c0 --- /dev/null +++ b/issues/nothing-reopens-a-diskserver-session-over-a-used-page.md @@ -0,0 +1,30 @@ +--- +status: open +kind: tooling +opened: 2026-10-02 +--- + +# Nothing reopens a diskserver session over a used page + +A nightly went red on `blockd_io: FAIL /AFTER.BIN after the restart: Io` (run +36753172688, guest 7). blockd answered an open and only then made its ends of +the session page. A client that reconnects sends the page its last server +wrote, and one that looks before the new server's ends are made hears the dead +session's completion. The server's half is a type now: `wire::Opened::over` +makes a server's ends and its answer together, and diskserver sends the answer +it returns. + +The client's half is read off the code and not run. `Session::pump`, `drain` +and `wait` (`system/diskserver/src/session.rs`) hold a `Region`, a +`Connection` and a `Poller`, so no host reaches them. A page whose completion +ring holds the dead session's answer ends the session only through `drain`'s +refusal of a completion for no ticket in flight (`Violation::Tag`, through +`client.complete`); `disk::Served` then answers its caller `Gone`, which a file +server's caller hears as `Io`. That nothing else ends it is +`issues/a-block-session-waits-forever-on-a-server-past-its-submission-ring.md`. + +**Exit**: a host test reopens a client's session over a page a killed server +left its answer on, and the session ends and the caller's call is `Io`, by +`drain`'s `Tag` refusal: red with that refusal removed, and green with it. + +Owner: the orchestrator. diff --git a/issues/nothing-runs-a-block-sessions-client-glue-on-a-host.md b/issues/nothing-runs-a-block-sessions-client-glue-on-a-host.md deleted file mode 100644 index 53ab5debf1..0000000000 --- a/issues/nothing-runs-a-block-sessions-client-glue-on-a-host.md +++ /dev/null @@ -1,23 +0,0 @@ ---- -status: open -kind: tooling -opened: 2026-10-13 ---- - -# Nothing runs a block session's client glue on a host - -diskserver's -`a_session_opened_over_a_killed_servers_page_is_told_after_its_ends_are_made` -holds the server's half of a reopen: a client told its session is open finds -the new server's ends made. What the client does with a page that breaks the -protocol is read off the code and not run. `Session::pump`, `drain` and -`wait` (`system/diskserver/src/session.rs`) hold a `Region`, a `Connection` -and a `Poller`, so no host reaches them: that a `Violation::HeadPastTail` or -`Violation::Tag` ends the session, and that `disk::Served` answers its caller -`Gone`, is unexercised. - -**Exit**: a host test drives the client's ring half over a page whose server -cursor is past what the client published, and the session ends with the -caller's call refused; red with that check in `Session::drain` removed. - -Owner: the orchestrator. diff --git a/issues/the-kernel-is-small-interrupts-post-and-threads-wait.md b/issues/the-kernel-is-small-interrupts-post-and-threads-wait.md index d8a3b22bbe..524b799505 100644 --- a/issues/the-kernel-is-small-interrupts-post-and-threads-wait.md +++ b/issues/the-kernel-is-small-interrupts-post-and-threads-wait.md @@ -67,11 +67,10 @@ Beside that order, each waiting on something outside it: `every_request_is_answered_exactly_once`, `what_a_flush_calls_durable_is_on_the_medium` and `what_was_on_the_wire_at_the_end_is_refused_and_what_was_not_waits` for - the transport, by diskserver's - `a_session_opened_over_a_killed_servers_page_is_told_after_its_ends_are_made` - for diskserver's part, a session reopened over the page a killed server - left, red under the mutation that moves the server's stores after its - answer, by fileserver's + the transport, for diskserver's part by `wire::Opened::over`, whose type + makes a server's ends before its answer, and by the client test + `issues/nothing-reopens-a-diskserver-session-over-a-used-page.md` owes, a + session reopened over the page a killed server left, by fileserver's `a_directory_rename_is_whole_under_one_name_wherever_the_server_is_killed` for DATA, and for LOG by the FAT test step 10 owes. **Partly done** (#525, #536): diskserver drives NVMe, and the kernel's NVMe driver is @@ -208,10 +207,10 @@ Beside that order, each waiting on something outside it: same controller delivering every report, red on a usbd that waits on the stick before it reads the keyboard's ring. `/log`'s, what a killed usbd had on the wire: stage 3's `toyos-blockring` tests for the - transport, and for usbd's session open diskserver's - `a_session_opened_over_a_killed_servers_page_is_told_after_its_ends_are_made`, - moved with the open, red under the mutation that moves usbd's stores - after its answer; with one test this step owes in + transport, and for usbd's session open `wire::Opened::over`, moved with + the open, and the client test + `issues/nothing-reopens-a-diskserver-session-over-a-used-page.md` + owes; with one test this step owes in `system/fileserver/src/fat.rs`: a disk stopped after each block write in turn of an append and its sync mounts at every stop and reads back all the last whole sync held, red on a sync that writes a growing diff --git a/system/diskserver/src/main.rs b/system/diskserver/src/main.rs index bdab9b5b84..a95d854a6b 100644 --- a/system/diskserver/src/main.rs +++ b/system/diskserver/src/main.rs @@ -51,7 +51,6 @@ use std::collections::BTreeMap; -use std::sync::atomic::AtomicU32; use std::time::{Duration, Instant}; use diskserver::region::Region; @@ -64,7 +63,7 @@ use toyos_abi::part::{PartGuid, GUID_TEXT_LEN}; use toyos_abi::syscall::{DEV_PREFIX, MAX_BADGE, SyscallError}; use toyos_blockhold::Holds; use toyos_blockring::entry::{Completion, Op}; -use toyos_blockring::layout::{ServerRings, DEPTH, RING_WORDS}; +use toyos_blockring::layout::{ServerRings, DEPTH}; use toyos_blockring::server::{ServerSession, Taken}; use toyos_blockring::wire::{self, Grant, Opened, Refusal}; use toyos_blockring::{BLOCK_BYTES, PORT}; @@ -369,8 +368,8 @@ impl Service { } } - /// The client heard its session is open, over the ends [`answer`] made: - /// from here it is served. + /// The client heard its session is open, over the ends `Opened::over` + /// made: from here it is served. fn admit(&mut self, opening: Opening, rings: ServerRings, opened: Opened, conn: Connection) -> u64 { let id = self.next_id; self.next_id += 1; @@ -725,21 +724,6 @@ fn serve(service: &mut Service, acceptor: &toyos::port::Acceptor) -> ! { } } -/// A server's ends of the session `page` it was sent, and the answer that -/// `tell` sends its client, **told only once the ends are made**: a client -/// looks at the page the moment it hears, and one that opens a session again -/// sends the cursors its last server left on it. -fn answer( - page: &[AtomicU32; RING_WORDS], - blocks: u64, - unique: [u8; 16], - tell: impl FnOnce(&Opened) -> R, -) -> (ServerRings, Opened, R) { - let (rings, opened) = Opened::over(page, blocks, unique); - let told = tell(&opened); - (rings, opened, told) -} - /// The grant `conn`, accepted from `acceptor`, was minted with; `None` for a /// connection through an unbadged connector or with bytes no minter stamps. fn grant(acceptor: &toyos::port::Acceptor, conn: &Connection) -> Option { @@ -785,10 +769,8 @@ fn handshake(service: &mut Service, acceptor: &toyos::port::Acceptor, p: Pending refuse(&p.conn, why); } Ok(opening) => { - let (rings, opened, sent) = answer(opening.region.words(), opening.blocks, guid, |opened| { - p.conn.try_send_bytes(wire::MSG_OPENED, &opened.encode()) - }); - if sent.is_err() { + let (rings, opened) = Opened::over(opening.region.words(), opening.blocks, guid); + if p.conn.try_send_bytes(wire::MSG_OPENED, &opened.encode()).is_err() { service.abandon(opening); return; } @@ -885,32 +867,4 @@ mod tests { assert_eq!(listed(Ok(&parts), unique([9; 16])).map(decoded), Ok(vec![])); assert_eq!(listed(Ok(&parts), None), Err(Refusal::NotGranted)); } - - /// A client that opens its session again sends the page a killed server - /// left, its own two cursors made again (`Session::reconnect`): the moment - /// it is told the session is open, every request has room and nothing of - /// the dead session is there to read. - #[test] - fn a_session_opened_over_a_killed_servers_page_is_told_after_its_ends_are_made() { - use toyos_blockring::layout::client; - use toyos_blockring::{Request, Status}; - - let page: [AtomicU32; RING_WORDS] = core::array::from_fn(|_| AtomicU32::new(0)); - let mut first = client(&page); - let (mut killed, _, ()) = answer(&page, 100, LOG, |_| ()); - assert_eq!(first.0.push(&page, Request { op: Op::Flush, tag: 1 }.encode()), Ok(true)); - first.0.publish(&page); - assert!(killed.0.pop(&page).expect("a published request").is_some()); - killed.0.release(&page); - let answered = Completion { tag: 1, status: Status::Ok }.encode(); - assert_eq!(killed.1.push(&page, answered), Ok(true)); - killed.1.publish(&page); - - let mut again = client(&page); - let (_, _, (room, stale)) = - answer(&page, 100, LOG, |_| (again.0.space(&page), again.1.pop(&page).map(|c| c.is_some()))); - - assert_eq!(room, Ok(DEPTH)); - assert_eq!(stale, Ok(false)); - } } From 17ea52983ae5f01dfa8f5b7de19d663d55e1076a Mon Sep 17 00:00:00 2001 From: japabu Date: Tue, 13 Oct 2026 09:07:25 +0200 Subject: [PATCH 4/5] fileserver: the FAT append test stops inside a flush epoch, and fatgen103's checker judges each stop Review 6139576035's NOTEs 1 and 2, taken by ruling 6139586864. `Traced` keeps the blocks it is written per flush epoch, and the test stops with every earlier epoch on the disk and any subset of one epoch's writes: the disk keeps no order between flushes, so a prefix of an in-order trace was a model of the cache's order, not the disk's. 70 stops on this append and its sync. Each stop's image is judged by `toyos-fat32-check`, written from fatgen103 and sharing no code with `toyos-fat32`, before the driver mounts it. Allowed by name: `LostChain`, `FsInfoFreeCount` and `FatMirror`, which the review named, and `ChainTooLong` on `/a.txt` at the last sync's 3000 bytes alone. That last one is the window the module header names between a growing file's chain and its entry, and the one `toyos-fat32`'s own `a_stop_at_any_write_leaves_only_the_named_windows` admits; without it the fixed code reds at its first stop past the chain ("DIR_FileSize is 3000 bytes, which needs 6 clusters, and the chain holds 30"), which is the order this branch chose and not a defect. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015xhaydXCVRNscoccHYFUWy --- Cargo.lock | 1 + system/fileserver/Cargo.toml | 5 ++ system/fileserver/src/fat.rs | 138 +++++++++++++++++++++++------------ 3 files changed, 98 insertions(+), 46 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 126652cbce..3285301c15 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1197,6 +1197,7 @@ dependencies = [ "toyos-abi", "toyos-blockring", "toyos-fat32", + "toyos-fat32-check", "toyos-gpt", "toyos-manifest", ] diff --git a/system/fileserver/Cargo.toml b/system/fileserver/Cargo.toml index 0f04c1d813..e5d5ad66fc 100644 --- a/system/fileserver/Cargo.toml +++ b/system/fileserver/Cargo.toml @@ -24,5 +24,10 @@ toyos-fat32 = { path = "../../lib/fat32" } bcachefs = { path = "../../lib/bcachefs" } diskserver = { path = "../diskserver" } +[dev-dependencies] +# The volumes the FAT role leaves, judged by the checker that shares no code +# with the driver it wrote them with. +toyos-fat32-check = { path = "../../lib/fat32/check" } + [package.metadata.toyos.host] exempt.owns = "ToyOS's storage roles, served from the partitions ToyOS claims for it" diff --git a/system/fileserver/src/fat.rs b/system/fileserver/src/fat.rs index c95e7ca08b..da3fe1328e 100644 --- a/system/fileserver/src/fat.rs +++ b/system/fileserver/src/fat.rs @@ -732,82 +732,128 @@ mod tests { assert_eq!(v.sync(), Ok(Vec::new())); } - /// A disk that keeps every block it is written, in order, so a test can - /// stop it after any one of them. + /// A disk whose bytes are the volume's, whole in memory. + struct Image(Vec); + + impl Disk for Image { + fn blocks(&self) -> u64 { + (self.0.len() / BLOCK) as u64 + } + fn read(&mut self, first: u64, out: &mut [u8]) -> Result<(), DiskError> { + let at = first as usize * BLOCK; + out.copy_from_slice(&self.0[at..at + out.len()]); + Ok(()) + } + fn write(&mut self, first: u64, data: &[u8]) -> Result<(), DiskError> { + let at = first as usize * BLOCK; + self.0[at..at + data.len()].copy_from_slice(data); + Ok(()) + } + fn flush(&mut self) -> Result<(), DiskError> { + Ok(()) + } + } + + /// The blocks a disk was written, in order, one epoch to each flush: the + /// disk keeps no order among one epoch's writes, and has all of an epoch + /// before any of the next. + type Epochs = Rc)>>>>; + + /// A disk that keeps every block it is written in its flush epoch, so a + /// test can stop it with any of one epoch's writes on it. struct Traced { - ram: Ram, - writes: Rc)>>>, + image: Image, + epochs: Epochs, } impl Disk for Traced { fn blocks(&self) -> u64 { - self.ram.blocks() + self.image.blocks() } fn read(&mut self, first: u64, out: &mut [u8]) -> Result<(), DiskError> { - self.ram.read(first, out) + self.image.read(first, out) } fn write(&mut self, first: u64, data: &[u8]) -> Result<(), DiskError> { - let mut writes = self.writes.borrow_mut(); + let mut epochs = self.epochs.borrow_mut(); + let epoch = epochs.last_mut().expect("an epoch is always open"); for (i, block) in data.chunks_exact(BLOCK).enumerate() { - writes.push((first + i as u64, block.to_vec())); + epoch.push((first + i as u64, block.to_vec())); } - self.ram.write(first, data) + self.image.write(first, data) } fn flush(&mut self) -> Result<(), DiskError> { - self.ram.flush() + self.epochs.borrow_mut().push(Vec::new()); + self.image.flush() + } + } + + fn written<'a>(image: &mut [u8], writes: impl IntoIterator)>) { + for (block, data) in writes { + let at = *block as usize * BLOCK; + image[at..at + BLOCK].copy_from_slice(data); } } - /// An append and its sync, stopped after each block write in turn, leave - /// a volume that mounts and holds the file as the last whole sync left it - /// or as the append made it: never an entry naming bytes its chain does - /// not reach or its clusters do not hold yet. + /// An append and its sync, stopped with every flush epoch before one on + /// the disk and any of that one's writes, leave a volume fatgen103's + /// checker refuses only for clusters no file's size reaches, and that mounts and + /// holds the file as the last whole sync left it or as the append made + /// it: never an entry naming bytes its chain does not reach or its + /// clusters do not hold yet. #[test] - fn an_append_stopped_after_any_block_write_reads_back_the_last_sync() { + fn an_append_stopped_with_any_writes_of_a_flush_on_the_disk_reads_back_the_last_sync() { + use toyos_fat32_check::{check, describe, Complaint}; const CREATE: OpenHow = OpenHow { create: true, create_new: false, truncate: false }; const OPEN: OpenHow = OpenHow { create: false, create_new: false, truncate: false }; let clock = || 1_717_245_296 * NANOS_PER_SEC; let spec = spec_volume::fixture(); - let blocks = spec.bytes.len().div_ceil(BLOCK); let mut image = spec.bytes.clone(); - image.resize(blocks * BLOCK, 0); - let disk = |trace: &[(u64, Vec)]| { - let mut ram = Ram::new(blocks as u64); - ram.write(0, &image).unwrap(); - for (block, data) in trace { - ram.write(*block, data).unwrap(); - } - ram - }; - let writes = Rc::new(std::cell::RefCell::new(Vec::new())); - let mut v = FatVolume::mount(Traced { ram: disk(&[]), writes: Rc::clone(&writes) }, true, clock).unwrap(); + image.resize(spec.bytes.len().div_ceil(BLOCK) * BLOCK, 0); + let epochs: Epochs = Rc::new(std::cell::RefCell::new(vec![Vec::new()])); + let mut v = FatVolume::mount(Traced { image: Image(image.clone()), epochs: Rc::clone(&epochs) }, true, clock) + .unwrap(); let held: Vec = (0..3000u32).map(|i| (i % 251 + 1) as u8).collect(); let appended: Vec = (0..3 * BLOCK as u32).map(|i| (i % 241 + 2) as u8).collect(); let a = v.open("a.txt", CREATE).unwrap(); v.write(a, 0, &held).unwrap(); assert_eq!(v.sync(), Ok(Vec::new())); - let synced = writes.borrow().len(); + let synced = epochs.borrow().len() - 1; v.write(a, held.len() as u64, &appended).unwrap(); assert_eq!(v.sync(), Ok(Vec::new())); - let trace = writes.borrow().clone(); + let epochs = epochs.borrow().clone(); let whole = [&held[..], &appended[..]].concat(); - - for stop in synced..=trace.len() { - let mut after = FatVolume::mount(disk(&trace[..stop]), false, clock) - .unwrap_or_else(|e| panic!("stopped after block write {stop} of {}: {e}", trace.len())); - let n = after.open("a.txt", OPEN).unwrap(); - let size = after.lstat("a.txt").unwrap().size as usize; - let mut out = vec![0u8; size]; - let read = after.read(n, 0, &mut crate::volume::Buf(&mut out)); - assert_eq!(read, Ok(size), "stopped after block write {stop} of {}: a {size}-byte file", trace.len()); - assert!( - out == held || out == whole, - "stopped after block write {stop} of {}: the file is {size} bytes and neither the {} the last \ - sync held nor the {} the append made", - trace.len(), - held.len(), - whole.len() - ); + written(&mut image, epochs[..synced].iter().flatten()); + + for (e, epoch) in epochs.iter().enumerate().skip(synced) { + for kept in 0..1u32 << epoch.len() { + let stop = format!("stopped in flush epoch {e} of {} with writes {kept:#b} of its {}", epochs.len(), epoch.len()); + let mut bytes = image.clone(); + written(&mut bytes, epoch.iter().enumerate().filter(|(i, _)| kept >> i & 1 == 1).map(|(_, w)| w)); + // Clusters no entry reaches yet, and the last sync's entry over + // the append's chain: the window between the chain and its entry. + let refused: Vec<_> = check(&bytes) + .into_iter() + .filter(|c| match c { + Complaint::LostChain { .. } | Complaint::FsInfoFreeCount { .. } | Complaint::FatMirror { .. } => false, + Complaint::ChainTooLong { path, size, .. } => !(path == "/a.txt" && *size == held.len() as u64), + _ => true, + }) + .collect(); + assert!(refused.is_empty(), "{stop}: {}", describe(&refused)); + let mut after = FatVolume::mount(Image(bytes), false, clock).unwrap_or_else(|e| panic!("{stop}: {e}")); + let n = after.open("a.txt", OPEN).unwrap(); + let size = after.lstat("a.txt").unwrap().size as usize; + let mut out = vec![0u8; size]; + let read = after.read(n, 0, &mut crate::volume::Buf(&mut out)); + assert_eq!(read, Ok(size), "{stop}: a {size}-byte file"); + assert!( + out == held || out == whole, + "{stop}: the file is {size} bytes and neither the {} the last sync held nor the {} the append made", + held.len(), + whole.len() + ); + } + written(&mut image, epoch); } } From c7201d3757e506593b417406017d1452a344bf5f Mon Sep 17 00:00:00 2001 From: japabu Date: Tue, 13 Oct 2026 09:21:58 +0200 Subject: [PATCH 5/5] fileserver: the append test holds its final stop to a clean check and the append's bytes The stop with every flush epoch on the disk is the sync whole. The ChainTooLong allowance, the window between the chain's epoch and the entry's, and the last sync's bytes are admitted only before it; at it, fatgen103's checker refuses nothing and the file reads as the append made it. Before, the allowance held at every stop, so a sync that returned Ok and never made the append's size durable passed. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015xhaydXCVRNscoccHYFUWy --- system/fileserver/src/fat.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/system/fileserver/src/fat.rs b/system/fileserver/src/fat.rs index da3fe1328e..42a8a39be8 100644 --- a/system/fileserver/src/fat.rs +++ b/system/fileserver/src/fat.rs @@ -799,7 +799,8 @@ mod tests { /// checker refuses only for clusters no file's size reaches, and that mounts and /// holds the file as the last whole sync left it or as the append made /// it: never an entry naming bytes its chain does not reach or its - /// clusters do not hold yet. + /// clusters do not hold yet. With every epoch on the disk the checker + /// refuses nothing and the file is the append's. #[test] fn an_append_stopped_with_any_writes_of_a_flush_on_the_disk_reads_back_the_last_sync() { use toyos_fat32_check::{check, describe, Complaint}; @@ -829,11 +830,14 @@ mod tests { let stop = format!("stopped in flush epoch {e} of {} with writes {kept:#b} of its {}", epochs.len(), epoch.len()); let mut bytes = image.clone(); written(&mut bytes, epoch.iter().enumerate().filter(|(i, _)| kept >> i & 1 == 1).map(|(_, w)| w)); + let synced_whole = kept == (1 << epoch.len()) - 1 && epochs[e + 1..].iter().all(Vec::is_empty); // Clusters no entry reaches yet, and the last sync's entry over - // the append's chain: the window between the chain and its entry. + // the append's chain: the window between the chain and its + // entry, closed once the sync is whole on the disk. let refused: Vec<_> = check(&bytes) .into_iter() .filter(|c| match c { + _ if synced_whole => true, Complaint::LostChain { .. } | Complaint::FsInfoFreeCount { .. } | Complaint::FatMirror { .. } => false, Complaint::ChainTooLong { path, size, .. } => !(path == "/a.txt" && *size == held.len() as u64), _ => true, @@ -847,7 +851,7 @@ mod tests { let read = after.read(n, 0, &mut crate::volume::Buf(&mut out)); assert_eq!(read, Ok(size), "{stop}: a {size}-byte file"); assert!( - out == held || out == whole, + out == whole || out == held && !synced_whole, "{stop}: the file is {size} bytes and neither the {} the last sync held nor the {} the append made", held.len(), whole.len()