From cf9f601cd99435c9be03702c3c52bc8abd4b0386 Mon Sep 17 00:00:00 2001 From: Pedro Werneck Date: Wed, 21 Oct 2026 17:38:29 -0300 Subject: [PATCH 01/36] feat: deliver owner changes from per-peer views The DeliveryTracker records, for each peer, the hash of each file that the peer holds. Each sync sends a peer each readable file that its view does not have. It also sends a delete for each file that the peer can no longer read. A view changes only after the outbox write succeeds. Thus a peer that the owner approves later receives the existing files, and a peer that loses read access receives a delete. The tracker replaces _read_perm_cache and the pending deliveries. --- docs/connections.md | 2 +- syft/sync/connections/connection_router.py | 8 +- syft/sync/syftbox_manager.py | 7 +- syft/sync/sync/caches/datasite_owner_cache.py | 28 +- syft/sync/sync/constants.py | 1 - syft/sync/sync/datasite_owner_syncer.py | 312 ++++++------------ syft/sync/sync/delivery_tracker.py | 161 ++++++--- tests/unit/test_delivery_tracker.py | 248 ++++++++++++++ tests/unit/test_permission_file_case.py | 47 --- tests/unit/test_persisted_cache.py | 6 +- tests/unit/test_transport_contracts.py | 9 +- 11 files changed, 489 insertions(+), 340 deletions(-) create mode 100644 tests/unit/test_delivery_tracker.py delete mode 100644 tests/unit/test_permission_file_case.py diff --git a/docs/connections.md b/docs/connections.md index 49493ab1fb3..0e18fc66beb 100644 --- a/docs/connections.md +++ b/docs/connections.md @@ -62,7 +62,7 @@ A pair encrypts when one side is `required`. An `off` and a `required` client co A client takes its policy at login. The key file records a `required` policy, so later logins (and syft-bg) stay `required`. A process that started with another policy stops at its next sync with an error to restart it. Files at rest written before the switch stay plaintext; new ones are encrypted. -While a peer's policy or keys are not known yet, a change for it waits, with a warning, and a later sync sends it. An owner's change stays pending on disk. A scientist's change waits in memory, and is lost if the process exits. +While a peer's policy or keys are not known yet, a change for it waits, with a warning, and a later sync sends it. An owner's change is not lost: the owner keeps a record of what each peer holds, and that record changes only after a send. A scientist's change waits in memory, and is lost if the process exits. A pair that encrypts refuses plaintext. The receiver drops a plaintext message with a warning: the owner archives it unread, and a scientist skips it. diff --git a/syft/sync/connections/connection_router.py b/syft/sync/connections/connection_router.py index d797f28ff9b..9ec1864e57b 100644 --- a/syft/sync/connections/connection_router.py +++ b/syft/sync/connections/connection_router.py @@ -31,7 +31,7 @@ from syft.sync.peers.peer_store import PeerStore from syft.sync.transport.naming import PEERS_META_KEY from syft.sync.transport.protocols import Transport -from syft.sync.transport.types import MessageKind +from syft.sync.transport.types import MessageKind, SendStatus from syft.sync.utils.print_utils import ( print_peer_added_to_platform, print_peer_adding_to_platform, @@ -225,13 +225,15 @@ def owner_get_next_proposed_filechange_message( def owner_write_event_messages_to_outbox( self, recipient_email: str, events_message: FileChangeEventsMessage - ): + ) -> SendStatus: self._refresh_unknown_peer(recipient_email) events_message = self._downgrade_for_peer(events_message, recipient_email) data = events_message.as_compressed_data() data = self.peer_store.encrypt_if_needed(recipient_email, data) fname = events_message.message_filepath.as_string() - self.transport.p2p.send(recipient_email, fname, data, MessageKind.FILE_EVENTS) + return self.transport.p2p.send( + recipient_email, fname, data, MessageKind.FILE_EVENTS + ) def watcher_get_events_messages( self, peer_email: str, since_timestamp: float | None diff --git a/syft/sync/syftbox_manager.py b/syft/sync/syftbox_manager.py index 48a3a3d2a03..67d33bb46ff 100644 --- a/syft/sync/syftbox_manager.py +++ b/syft/sync/syftbox_manager.py @@ -1029,12 +1029,7 @@ def sync( compatible_emails = self.peer_manager.without_encryption_conflicts( compatible_emails ) - self.datasite_owner_syncer.sync( - compatible_emails, - deferred_peer_emails=[ - e for e in peer_emails if e not in compatible_emails - ], - ) + self.datasite_owner_syncer.sync(compatible_emails) if auto_compact: for peer_email in compatible_emails: self.datasite_owner_syncer.compact_outbox_if_needed( diff --git a/syft/sync/sync/caches/datasite_owner_cache.py b/syft/sync/sync/caches/datasite_owner_cache.py index c968abec178..eb65c04155f 100644 --- a/syft/sync/sync/caches/datasite_owner_cache.py +++ b/syft/sync/sync/caches/datasite_owner_cache.py @@ -18,10 +18,12 @@ ) from syft.sync.utils.syftbox_utils import get_event_hash_from_content from syft.sync.checkpoints.checkpoint import Checkpoint -from syft.sync.sync.constants import ( - CACHE_DIR, - OWNER_FILE_HASHES_FILENAME, - OWNER_PENDING_DELIVERIES_FILENAME, +from syft.sync.sync.constants import CACHE_DIR, OWNER_FILE_HASHES_FILENAME +from syft.sync.sync.delivery_tracker import ( + DELIVERY_VIEWS_FILENAME, + DeliveryTracker, + LocalDeliveryStateStore, + MemoryDeliveryStateStore, ) @@ -62,9 +64,10 @@ class DataSiteOwnerEventCache(BaseModelCallbackMixin): # Wired in pre-init validator: persisted-to-disk when syftbox_folder is set, # plain in-memory PersistedDict otherwise. file_hashes: PersistedDict = Field(default_factory=PersistedDict) - # peer email -> {path in datasite: name of the last message that holds it}. - # A path stays here until the upload of that message to the peer succeeds. - pending_deliveries: PersistedDict = Field(default_factory=PersistedDict) + # What each peer holds. Kept next to file_hashes when syftbox_folder is set. + deliveries: DeliveryTracker = Field( + default_factory=lambda: DeliveryTracker(MemoryDeliveryStateStore()) + ) email: str # Full path to collections (datasets) folder collections_folder: Path | None = None @@ -85,11 +88,13 @@ def _build_file_hashes(cls, data): key_serializer=str, key_deserializer=Path, ) - if isinstance(data, dict) and "pending_deliveries" not in data: + if isinstance(data, dict) and "deliveries" not in data: folder = data.get("syftbox_folder") if folder is not None: - data["pending_deliveries"] = PersistedDict( - path=Path(folder) / CACHE_DIR / OWNER_PENDING_DELIVERIES_FILENAME + data["deliveries"] = DeliveryTracker( + LocalDeliveryStateStore( + Path(folder) / CACHE_DIR / DELIVERY_VIEWS_FILENAME + ) ) return data @@ -102,7 +107,7 @@ def from_config(cls, config: DataSiteOwnerEventCacheConfig): email=config.email, syftbox_folder=config.syftbox_folder, file_hashes=PersistedDict(), - pending_deliveries=PersistedDict(), + deliveries=DeliveryTracker(MemoryDeliveryStateStore()), collections_folder=config.collections_folder, collection_variants=config.collection_variants, ) @@ -310,6 +315,7 @@ def clear_cache(self): self.events_messages_connection.clear_cache() self.file_connection.clear_cache() self.file_hashes.clear() + self.deliveries.clear() def has_conflict(self, proposed_event: ProposedFileChange) -> bool: if proposed_event.path_in_datasite not in self.file_hashes: diff --git a/syft/sync/sync/constants.py b/syft/sync/sync/constants.py index 8d195bfad55..2e8508efb4f 100644 --- a/syft/sync/sync/constants.py +++ b/syft/sync/sync/constants.py @@ -1,5 +1,4 @@ CACHE_DIR = ".cache" OWNER_FILE_HASHES_FILENAME = "owner_file_hashes.json" -OWNER_PENDING_DELIVERIES_FILENAME = "owner_pending_deliveries.json" ROLLING_STATE_FILENAME = "rolling_state.json" diff --git a/syft/sync/sync/datasite_owner_syncer.py b/syft/sync/sync/datasite_owner_syncer.py index 1d27bc7814b..3283098d099 100644 --- a/syft/sync/sync/datasite_owner_syncer.py +++ b/syft/sync/sync/datasite_owner_syncer.py @@ -43,6 +43,7 @@ ) from syft_perms import SyftPermContext from syft.sync.sync.constants import CACHE_DIR, ROLLING_STATE_FILENAME +from syft.sync.transport.types import SendStatus from syft.sync.utils.syftbox_utils import ( create_event_timestamp, get_event_hash_from_content, @@ -141,11 +142,6 @@ class DatasiteOwnerSyncer(BaseModelCallbackMixin): ) # Cache of collections shared with "any" - list of (tag, content_hash) tuples _any_shared_collections: List[tuple] = PrivateAttr(default_factory=list) - # Cache of read permissions per file path → frozenset of peer emails - _read_perm_cache: dict[str, frozenset[str]] = PrivateAttr(default_factory=dict) - # Approved peers that fail the version gate in this sync. Their messages - # stay pending and are not uploaded. - _deferred_peer_emails: frozenset[str] = PrivateAttr(default_factory=frozenset) # In-memory rolling state for tracking events since last checkpoint _rolling_state: RollingState | None = PrivateAttr(default=None) @@ -204,32 +200,18 @@ def _save_rolling_state(self) -> None: tmp.write_text(self._rolling_state.model_dump_json()) tmp.rename(path) - def sync( - self, - peer_emails: list[str], - recompute_hashes: bool = True, - deferred_peer_emails: tuple[str, ...] | list[str] = (), - ): - """Push local changes and pull proposed changes from ``peer_emails``. - - ``deferred_peer_emails`` are approved peers that fail the version gate. - Changes they can read stay pending, and go out in a later sync where - the peer is in ``peer_emails``. - """ + def sync(self, peer_emails: list[str], recompute_hashes: bool = True): + """Pull proposed changes from ``peer_emails``, then send each of them + what it does not hold yet.""" if self.persist_owner_state: self._load_rolling_state() - self._deferred_peer_emails = frozenset(deferred_peer_emails) try: if not self.initial_sync_done: self.pull_initial_state() - self._queue_pending_deliveries(peer_emails) if recompute_hashes: - self.process_local_changes( - recipients=[*peer_emails, *deferred_peer_emails] - ) + self._record_local_changes() - # first, pull existing state for peer_email in peer_emails: proposed_count = 0 while True: @@ -246,8 +228,8 @@ def sync( f"from {peer_email}" ) self.process_syftbox_events_queue() + self.deliver(peer_emails) finally: - self._deferred_peer_emails = frozenset() if self.persist_owner_state: self._save_rolling_state() @@ -587,130 +569,48 @@ def _download_file_with_new_connection(self, file_id: str) -> bytes: connection = self.connection_router.connection_for_parallel_download() return connection.watcher_download_collection_file(file_id) - def _get_readers(self, path: str, recipients: list[str]) -> frozenset[str]: - """Return the set of recipients that have read access to the given path.""" - return frozenset(r for r in recipients if self.check_read_permissions(r, path)) + def _create_current_state_event(self, path: str) -> FileChangeEvent: + """An event that sets ``path`` on a peer to its current local state. - def _get_paths_under_perm_file(self, perm_path: str) -> list[str]: - """Find all files under the perm file's parent directory on the filesystem.""" - datasite = self.syftbox_folder / self.email - parent_dir = datasite / str(Path(perm_path).parent) - if not parent_dir.exists(): - return [] - collections_rel = self.event_cache.collections_relative_path() - paths = [] - for file_path in parent_dir.rglob("*"): - if file_path.is_file(): - rel = str(file_path.relative_to(datasite)) - if is_normal_syncable_path(rel, collections_path=collections_rel): - paths.append(rel) - return paths - - def _create_resend_event(self, path: str) -> "FileChangeEvent | None": - """Create a FileChangeEvent for an existing file to resend to new readers.""" + A file missing from the cache gives a deletion. A read error raises. + """ try: content = self.event_cache.file_connection.read_file(path) - except Exception: - return None - if content is None: - return None - from syft.sync.utils.syftbox_utils import ( - get_event_hash_from_content, - create_event_timestamp, - ) - + except (KeyError, FileNotFoundError): + # The in-memory cache raises KeyError, the filesystem cache + # FileNotFoundError. + content = None timestamp = create_event_timestamp() return FileChangeEvent( id=uuid4(), path_in_datasite=Path(path), content=content, - old_hash=None, - new_hash=get_event_hash_from_content(content), + new_hash=None if content is None else get_event_hash_from_content(content), submitted_timestamp=timestamp, timestamp=timestamp, datasite_email=self.email, - is_deleted=False, + is_deleted=content is None, ) - def _create_current_state_event(self, path: str) -> FileChangeEvent: - """An event that sets ``path`` on a peer to its current local state. - - A missing file gives a deletion. A read error raises. - """ - content = None - if (self.syftbox_folder / self.email / path).exists(): - content = self.event_cache.file_connection.read_file(path) + def _delete_event(self, path: str) -> FileChangeEvent: + """An event that removes ``path`` from a peer.""" timestamp = create_event_timestamp() return FileChangeEvent( id=uuid4(), path_in_datasite=Path(path), - content=content, - new_hash=None if content is None else get_event_hash_from_content(content), + content=None, + new_hash=None, submitted_timestamp=timestamp, timestamp=timestamp, datasite_email=self.email, - is_deleted=content is None, - ) - - def _queue_pending_deliveries(self, peer_emails: list[str]) -> None: - """Queue the current state of each pending path for each peer.""" - pending = self.event_cache.pending_deliveries - for peer_email in peer_emails: - paths = [ - path - for path in pending.get(peer_email) or {} - if self.check_read_permissions(peer_email, path) - ] - if paths: - msg = FileChangeEventsMessage( - events=[self._create_current_state_event(p) for p in paths] - ) - self._queue_for_outbox(peer_email, msg) - name = msg.message_filepath.as_string() - else: - name = None - # Drop the paths that this resend does not hold. The peer can no - # longer read them. - pending.update_entry( - peer_email, - lambda old: {p: m for p, m in (old or {}).items() if m == name} or None, - ) - - def _queue_for_outbox(self, recipient: str, msg: FileChangeEventsMessage) -> None: - """Queue ``msg`` for ``recipient``. Its paths stay pending until the upload succeeds.""" - name = msg.message_filepath.as_string() - paths = {str(e.path_in_datasite): name for e in msg.events} - self.event_cache.pending_deliveries.update_entry( - recipient, lambda old: {**(old or {}), **paths} - ) - self.outbox_queue.put((recipient, msg)) - - def _clear_delivered(self, recipient: str, msg: FileChangeEventsMessage) -> None: - """Remove the paths that ``msg`` delivered, unless a later message holds them.""" - name = msg.message_filepath.as_string() - self.event_cache.pending_deliveries.update_entry( - recipient, - lambda old: {p: m for p, m in (old or {}).items() if m != name} or None, + is_deleted=True, ) - def process_local_changes(self, recipients: list[str]): + def _record_local_changes(self) -> None: + """Find the local file changes and add them to the owner's event log.""" file_change_events_message = self.event_cache.process_local_file_changes() if file_change_events_message is None: return - - perm_events, data_events = self._split_events(file_change_events_message.events) - events_by_recipient: dict[str, list[FileChangeEvent]] = { - r: [] for r in recipients - } - data_event_sent_to: dict[str, set[str]] = {} - - self._route_data_events( - data_events, recipients, events_by_recipient, data_event_sent_to - ) - self._route_perm_events( - perm_events, recipients, events_by_recipient, data_event_sent_to - ) - self._queue_events_for_outbox(events_by_recipient) if self.persist_owner_state: # Record local events in the owner's log and feed them into rolling # state so they flow through incremental/full checkpoints. @@ -718,77 +618,39 @@ def process_local_changes(self, recipients: list[str]): self._add_events_to_rolling_state(file_change_events_message) self.process_syftbox_events_queue() - def _split_events( - self, events: list[FileChangeEvent] - ) -> tuple[list[FileChangeEvent], list[FileChangeEvent]]: - """Split events into (permission_events, data_events).""" - from syft_permissions.spec.ruleset import PERMISSION_FILE_NAME - - perm_events = [] - data_events = [] - for event in events: - # Compare the last path part without case, as the ACL gate does. - if event.path_in_datasite.name.casefold() == PERMISSION_FILE_NAME: - perm_events.append(event) - else: - data_events.append(event) - return perm_events, data_events + def process_local_changes(self, recipients: list[str]): + """Record the local file changes, then deliver to ``recipients``.""" + self._record_local_changes() + self.deliver(recipients) - def _route_data_events( - self, - data_events: list[FileChangeEvent], - recipients: list[str], - events_by_recipient: dict[str, list[FileChangeEvent]], - data_event_sent_to: dict[str, set[str]], - ): - """Route data events to recipients who have read access.""" - for event in data_events: - path_str = str(event.path_in_datasite) - readers = self._get_readers(path_str, recipients) - self._read_perm_cache[path_str] = readers - data_event_sent_to[path_str] = set(readers) - for reader in readers: - events_by_recipient[reader].append(event) - - def _route_perm_events( - self, - perm_events: list[FileChangeEvent], - recipients: list[str], - events_by_recipient: dict[str, list[FileChangeEvent]], - data_event_sent_to: dict[str, set[str]], - ): - """Share perm files with readers and resend affected files to newly-permitted readers.""" - for event in perm_events: - path_str = str(event.path_in_datasite) - perm_readers = self._get_readers(path_str, recipients) - for reader in perm_readers: - events_by_recipient[reader].append(event) - - affected_paths = self._get_paths_under_perm_file(path_str) - for affected_path in affected_paths: - new_readers = self._get_readers(affected_path, recipients) - old_readers = self._read_perm_cache.get(affected_path, frozenset()) - newly_permitted = new_readers - old_readers - self._read_perm_cache[affected_path] = new_readers - - if not newly_permitted: - continue - already_sent = data_event_sent_to.get(affected_path, set()) - needs_resend = newly_permitted - already_sent - if needs_resend: - resend_event = self._create_resend_event(affected_path) - if resend_event is not None: - for reader in needs_resend: - events_by_recipient[reader].append(resend_event) - - def _queue_events_for_outbox( - self, events_by_recipient: dict[str, list[FileChangeEvent]] - ): - """Wrap events into messages per recipient and queue for outbox.""" - for recipient, events in events_by_recipient.items(): + def _syncable_hashes(self) -> dict[str, str]: + """The current hash of every file that a peer can get, by path.""" + collections_rel = self.event_cache.collections_relative_path() + return { + str(path): file_hash + for path, file_hash in self.event_cache.file_hashes.items() + if is_normal_syncable_path(str(path), collections_path=collections_rel) + } + + def _may_read(self, peer: str, path: str) -> bool: + """``check_read_permissions`` without a reload: the caller reloads once.""" + return self.perm_context.open(path).has_read_access(peer) + + def deliver(self, peer_emails: list[str]) -> None: + """Send each peer the files its view lacks, and deletes for the files it + may no longer have.""" + deliveries = self.event_cache.deliveries + deliveries.confirm(self.connection_router.transport.p2p.delivered()) + self.perm_context._reload() + hashes = self._syncable_hashes() + for peer in peer_emails: + readable = {p: h for p, h in hashes.items() if self._may_read(peer, p)} + plan = deliveries.plan(peer, readable) + events = [self._create_current_state_event(p) for p in plan.send] + events += [self._delete_event(p) for p in plan.delete] if events: - msg = FileChangeEventsMessage(events=events) - self._queue_for_outbox(recipient, msg) + self.outbox_queue.put((peer, FileChangeEventsMessage(events=events))) + self.process_syftbox_events_queue() def pull_and_process_next_proposed_filechange( self, sender_email: str, raise_on_none=True @@ -872,7 +734,7 @@ def queue_event_for_syftbox( self.syftbox_events_queue.put(file_change_events_message) for recipient in recipients: - self._queue_for_outbox(recipient, file_change_events_message) + self.outbox_queue.put((recipient, file_change_events_message)) def process_syftbox_events_queue(self): # TODO: make this atomic @@ -881,39 +743,55 @@ def process_syftbox_events_queue(self): self.connection_router.owner_write_events_message_to_syftbox( file_change_events_message ) - # A message that waits is not cleared: its paths stay pending, and a - # later sync sends their current state. Later messages to the same + # A message that is not sent leaves the view as it is, so the plan of a + # later sync holds its changes again. Later messages to the same # recipient wait too, so the order stays the same. + if self.outbox_queue.empty(): + return + self.perm_context._reload() waiting_for = set() while not self.outbox_queue.empty(): recipient, message = self.outbox_queue.get() - if recipient in self._deferred_peer_emails or recipient in waiting_for: + if recipient in waiting_for: continue - if self._send_outbox_message(recipient, message): - self._clear_delivered(recipient, message) - else: + try: + status = self.connection_router.owner_write_event_messages_to_outbox( + recipient, message + ) + except (PeerEncryptionPendingError, EncryptionPolicyConflictError) as e: + if not self.connection_router.peer_store.should_hold(recipient): + warnings.warn( + f"Dropped an outbox message for {recipient}: not a peer" + ) + continue + warnings.warn(f"Changes for {recipient} wait for a later sync: {e}") waiting_for.add(recipient) + else: + self._record_sent(recipient, message, status) - def _send_outbox_message( - self, recipient: str, message: FileChangeEventsMessage - ) -> bool: - """Write one outbox message. + def _record_sent( + self, recipient: str, message: FileChangeEventsMessage, status: SendStatus + ) -> None: + """Add the changes of a sent message to ``recipient``'s view. - Returns False when the message waits for encryption with a peer, or - for a policy conflict to end. Returns True when it is written, or - dropped because ``recipient`` is not a peer. + A delete always counts. A file counts only if it is syncable and + ``recipient`` may read it, so a later permission change sends a delete + for it. Other files, such as a private dataset sent to an enclave, are + not in the view. """ - try: - self.connection_router.owner_write_event_messages_to_outbox( - recipient, message - ) - except (PeerEncryptionPendingError, EncryptionPolicyConflictError) as e: - if not self.connection_router.peer_store.should_hold(recipient): - warnings.warn(f"Dropped an outbox message for {recipient}: not a peer") - return True - warnings.warn(f"Changes for {recipient} wait for a later sync: {e}") - return False - return True + collections_rel = self.event_cache.collections_relative_path() + changes = {} + for event in message.events: + path = str(event.path_in_datasite) + if event.is_deleted: + changes[path] = None + elif is_normal_syncable_path( + path, collections_path=collections_rel + ) and self._may_read(recipient, path): + changes[path] = event.new_hash + self.event_cache.deliveries.sent( + recipient, message.message_filepath.as_string(), status, changes + ) def _download_outbox_message_with_new_connection( self, file_id: str diff --git a/syft/sync/sync/delivery_tracker.py b/syft/sync/sync/delivery_tracker.py index 963bdd8479a..1810e0519d6 100644 --- a/syft/sync/sync/delivery_tracker.py +++ b/syft/sync/sync/delivery_tracker.py @@ -1,7 +1,5 @@ """What each peer holds, so the data owner can resend, delete and catch up. -STUB: the names and contracts are agreed; the bodies are to be written. - The tracker knows nothing about the transport or where its record is kept. It works on paths and content hashes, takes the outcome of each send as a ``SendStatus``, and keeps its record through a ``DeliveryStateStore``. @@ -16,8 +14,8 @@ in the view, but no longer readable -> delete in the view, but the file is gone -> delete -A peer with no view, because it is new or its record was lost, gets a -``PeerReset``: its view starts empty, so it is sent every file it may read. +A peer with no view, because it is new or its record was lost, starts with an +empty view (a ``PeerReset``), so it is sent every file it may read. Recording a send ---------------- @@ -28,36 +26,19 @@ only when ``P2PConnection.delivered()`` reports that ``msg_id`` and ``confirm`` is called. A receipt is not enough; only the peer's ack counts. -A view therefore never changes before a delivery is confirmed. - -Pending deliveries ------------------- -Changes queued for a peer but not yet sent, for example because the upload -failed or the peer was skipped by the version gate, are kept per peer by the -owner cache's pending deliveries (PR #9563). The tracker does not duplicate -them. - -Storage -------- -``DeliveryState`` is the whole record, versioned. A ``DeliveryStateStore`` -loads and saves it: a local file next to the owner's file hashes, and with -``persist_owner_state`` also Drive, inside the rolling state and checkpoints, -so an enclave that wipes its local state at every boot does not reset every -peer. - -Sends not routed by permission, such as the echo of a submitter's own change -or a private dataset shared with an enclave, are recorded only where the -recipient may read the file, so a later permission change never deletes them. - -Open decisions: -- A peer rejected, then approved again: reset its view and resend (proposed). -- Accepted job submissions: echo only to the submitter, as today (proposed), or - also send to every peer who may read the file. +A view therefore never changes before a delivery is confirmed. A change that is +not sent, because the upload failed, the peer was skipped, or a process +stopped before the send, stays in the plan of the next sync. """ +import os from abc import ABC, abstractmethod -from typing import Iterable, Mapping +from contextlib import contextmanager, nullcontext +from pathlib import Path +from typing import ContextManager, Iterable, Mapping +from uuid import uuid4 +import portalocker from pydantic import BaseModel, ConfigDict, Field from syft.sync.transport.types import SendStatus @@ -102,38 +83,117 @@ class DeliveryStateStore(ABC): @abstractmethod def load(self) -> DeliveryState: - """The stored state, or an empty one.""" + """The stored state, or an empty one. + + Raises: + ValueError: when a later client wrote the state. + """ @abstractmethod def save(self, state: DeliveryState) -> None: """Store ``state``, replacing what was there.""" + def locked(self) -> ContextManager[None]: + """Keep other writers out from a ``load`` to the ``save`` that follows.""" + return nullcontext() + + +class MemoryDeliveryStateStore(DeliveryStateStore): + """A state kept in this process only.""" + + def __init__(self): + self._state = DeliveryState() + + def load(self) -> DeliveryState: + return self._state.model_copy(deep=True) + + def save(self, state: DeliveryState) -> None: + self._state = state.model_copy(deep=True) + + +class LocalDeliveryStateStore(DeliveryStateStore): + """A state kept in a JSON file, shared by the processes of one datasite.""" + + def __init__(self, path: Path): + self.path = path + + def load(self) -> DeliveryState: + if not self.path.exists(): + return DeliveryState() + state = DeliveryState.model_validate_json(self.path.read_text()) + if state.version > DELIVERY_STATE_VERSION: + raise ValueError( + f"{self.path} has version {state.version}, and this client reads " + f"up to version {DELIVERY_STATE_VERSION}." + ) + return state + + @contextmanager + def locked(self): + self.path.parent.mkdir(parents=True, exist_ok=True) + with open(self.path.with_suffix(self.path.suffix + ".lock"), "a") as f: + portalocker.lock(f, portalocker.LOCK_EX) + try: + yield + finally: + portalocker.unlock(f) + + def save(self, state: DeliveryState) -> None: + self.path.parent.mkdir(parents=True, exist_ok=True) + tmp = self.path.with_suffix(f".tmp.{os.getpid()}.{uuid4().hex}") + try: + tmp.write_text(state.model_dump_json()) + tmp.replace(self.path) + finally: + tmp.unlink(missing_ok=True) + + +def _apply(view: dict[str, str], changes: Mapping[str, str | None]) -> None: + for path, new_hash in changes.items(): + if new_hash is None: + view.pop(path, None) + else: + view[path] = new_hash + class DeliveryTracker: - """Per-peer views of confirmed file versions, for the data owner.""" + """Per-peer views of confirmed file versions, for the data owner. + + Each call reads the state from the store. Each change reads and writes it + back under the store lock, so the processes of one datasite see the same + views and do not lose each other's changes. + """ def __init__(self, store: DeliveryStateStore): self.store = store def has_view(self, peer: str) -> bool: """Whether ``peer`` has a view. False means a ``PeerReset`` is due.""" - raise NotImplementedError + return peer in self.store.load().views def reset(self, peer: str) -> None: """Start ``peer``'s view empty (a ``PeerReset``), so the next plan sends it every file it may read.""" - raise NotImplementedError + with self.store.locked(): + state = self.store.load() + state.views[peer] = {} + self.store.save(state) + + def clear(self) -> None: + """Forget every view and every queued send.""" + with self.store.locked(): + self.store.save(DeliveryState()) - def plan( - self, peer: str, readable: Mapping[str, str], paths: Iterable[str] - ) -> DeliveryPlan: - """What to send and delete for ``peer`` among ``paths``. + def plan(self, peer: str, readable: Mapping[str, str]) -> DeliveryPlan: + """What to send and delete for ``peer``. ``readable`` maps every path ``peer`` may read now to its current hash. - ``paths`` are the dirty paths; for a peer with no view, every readable - path. """ - raise NotImplementedError + view = self.store.load().views.get(peer, {}) + return DeliveryPlan( + send=sorted(p for p, h in readable.items() if view.get(p) != h), + delete=sorted(p for p in view if p not in readable), + ) def sent( self, @@ -146,8 +206,23 @@ def sent( ``DELIVERED`` updates the view now; ``QUEUED`` waits for ``confirm``. """ - raise NotImplementedError + with self.store.locked(): + state = self.store.load() + if status is SendStatus.DELIVERED: + _apply(state.views.setdefault(peer, {}), changes) + else: + assert status is SendStatus.QUEUED, status + state.queued[msg_id] = QueuedSend(peer=peer, changes=dict(changes)) + self.store.save(state) def confirm(self, msg_ids: Iterable[str]) -> None: """Apply the queued sends that ``P2PConnection.delivered()`` reported.""" - raise NotImplementedError + msg_ids = list(msg_ids) + if not msg_ids: + return + with self.store.locked(): + state = self.store.load() + for msg_id in msg_ids: + queued = state.queued.pop(msg_id) + _apply(state.views.setdefault(queued.peer, {}), queued.changes) + self.store.save(state) diff --git a/tests/unit/test_delivery_tracker.py b/tests/unit/test_delivery_tracker.py new file mode 100644 index 00000000000..1a1a6783467 --- /dev/null +++ b/tests/unit/test_delivery_tracker.py @@ -0,0 +1,248 @@ +"""The data owner sends each peer what it lacks, and deletes what it may no +longer have, from a per-peer record of what the peer holds.""" + +import threading + +import pytest + +from syft.sync.connections.connection_router import ConnectionRouter +from syft.sync.sync.constants import CACHE_DIR +from syft.sync.sync.delivery_tracker import ( + DELIVERY_STATE_VERSION, + DELIVERY_VIEWS_FILENAME, + DeliveryState, + DeliveryTracker, + LocalDeliveryStateStore, + MemoryDeliveryStateStore, +) +from syft.sync.syftbox_manager import SyftboxManager +from syft.sync.transport.types import SendStatus + +PEER = "ds@example.com" + + +def _tracker_holding(view: dict[str, str]) -> DeliveryTracker: + tracker = DeliveryTracker(MemoryDeliveryStateStore()) + tracker.sent(PEER, "m0", SendStatus.DELIVERED, view) + return tracker + + +def test_plan_sends_changed_files_deletes_unreadable(): + tracker = _tracker_holding({"same": "h1", "old": "h1", "revoked": "h1"}) + plan = tracker.plan(PEER, {"same": "h1", "old": "h2", "new": "h3"}) + assert plan.send == ["new", "old"] + assert plan.delete == ["revoked"] + + +def test_peer_without_view_gets_all_readable_files(): + tracker = DeliveryTracker(MemoryDeliveryStateStore()) + assert not tracker.has_view(PEER) + assert tracker.plan(PEER, {"a": "h1", "b": "h2"}).send == ["a", "b"] + + +def test_reset_resends_everything(): + tracker = _tracker_holding({"a": "h1"}) + tracker.reset(PEER) + assert tracker.has_view(PEER) + assert tracker.plan(PEER, {"a": "h1"}).send == ["a"] + + +def test_delivered_send_updates_view(): + tracker = _tracker_holding({"a": "h1", "b": "h1"}) + tracker.sent(PEER, "m1", SendStatus.DELIVERED, {"a": "h2", "b": None}) + assert tracker.plan(PEER, {"a": "h2"}).send == [] + assert tracker.plan(PEER, {"a": "h2"}).delete == [] + + +def test_queued_send_updates_view_on_confirm(): + tracker = DeliveryTracker(MemoryDeliveryStateStore()) + tracker.sent(PEER, "m1", SendStatus.QUEUED, {"a": "h1"}) + assert tracker.plan(PEER, {"a": "h1"}).send == ["a"] + tracker.confirm(["m1"]) + assert tracker.plan(PEER, {"a": "h1"}).send == [] + + +def test_confirm_unknown_message_raises(): + tracker = DeliveryTracker(MemoryDeliveryStateStore()) + with pytest.raises(KeyError): + tracker.confirm(["never-sent"]) + + +def test_local_store_persists_state_refuses_later_version(tmp_path): + path = tmp_path / DELIVERY_VIEWS_FILENAME + DeliveryTracker(LocalDeliveryStateStore(path)).sent( + PEER, "m1", SendStatus.DELIVERED, {"a": "h1"} + ) + assert ( + DeliveryTracker(LocalDeliveryStateStore(path)).plan(PEER, {"a": "h1"}).send + == [] + ) + + path.write_text(DeliveryState(version=DELIVERY_STATE_VERSION + 1).model_dump_json()) + with pytest.raises(ValueError, match="version"): + LocalDeliveryStateStore(path).load() + + +def test_overlapping_writers_keep_both_views(tmp_path, monkeypatch): + path = tmp_path / DELIVERY_VIEWS_FILENAME + first_loaded = threading.Event() + go = threading.Event() + load = LocalDeliveryStateStore.load + + def load_then_wait_in_first_writer(self): + state = load(self) + if threading.current_thread().name == "first": + first_loaded.set() + go.wait(timeout=5) + return state + + monkeypatch.setattr(LocalDeliveryStateStore, "load", load_then_wait_in_first_writer) + + def send(peer: str): + tracker = DeliveryTracker(LocalDeliveryStateStore(path)) + tracker.sent(peer, f"m-{peer}", SendStatus.DELIVERED, {"a": "h1"}) + + first = threading.Thread(target=send, args=("one@x.com",), name="first") + second = threading.Thread(target=send, args=("two@x.com",), name="second") + first.start() + first_loaded.wait(timeout=5) + second.start() + # Without the lock, the second writer saves now, and the first writer then + # overwrites it with the state it read before. + second.join(timeout=0.5) + go.set() + first.join() + second.join() + + tracker = DeliveryTracker(LocalDeliveryStateStore(path)) + assert tracker.has_view("one@x.com") + assert tracker.has_view("two@x.com") + + +# --- The owner syncer --------------------------------------------------------- + + +def _pair(**kwargs): + return SyftboxManager.pair_with_mock_drive_service_connection( + use_in_memory_cache=False, **kwargs + ) + + +def _write(do_manager, path: str, content: str): + file = do_manager.syftbox_folder / do_manager.email / path + file.parent.mkdir(parents=True, exist_ok=True) + file.write_text(content) + + +def _received(ds_manager, do_manager, path: str): + return ds_manager.syftbox_folder / do_manager.email / path + + +def _outbox_paths(ds_manager, do_manager) -> list[str]: + messages = ds_manager._connection_router.watcher_get_events_messages( + do_manager.email, None + ) + return [str(e.path_in_datasite) for m in messages for e in m.events] + + +def test_peer_approved_later_gets_existing_files(): + ds_manager, do_manager = _pair(add_peers=False) + do_manager.datasite_owner_syncer.perm_context.open("shared/").grant_read_access( + ds_manager.email + ) + _write(do_manager, "shared/data.txt", "existing") + do_manager.sync(auto_checkpoint=False) + + ds_manager.add_peer(do_manager.email, sync=False) + do_manager.load_peers() + do_manager.approve_peer_request(ds_manager.email) + ds_manager.load_peers() + do_manager.sync(auto_checkpoint=False) + ds_manager.sync() + + assert ( + _received(ds_manager, do_manager, "shared/data.txt").read_text() == "existing" + ) + + +def test_in_memory_owner_sends_file_content(): + ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection() + syncer = do_manager.datasite_owner_syncer + syncer.perm_context.open("shared/").grant_read_access(ds_manager.email) + syncer.event_cache.file_connection.write_file("shared/data.txt", "hello") + + do_manager.sync(auto_checkpoint=False) + do_manager.sync(auto_checkpoint=False) + messages = ds_manager._connection_router.watcher_get_events_messages( + do_manager.email, None + ) + sent = [ + (e.is_deleted, e.content) + for m in messages + for e in m.events + if str(e.path_in_datasite) == "shared/data.txt" + ] + assert sent == [(False, "hello")] + + +@pytest.mark.parametrize("persist_owner_state", [True, False]) +def test_revoked_peer_gets_delete(persist_owner_state): + ds_manager, do_manager = _pair(persist_owner_state=persist_owner_state) + folder = do_manager.datasite_owner_syncer.perm_context.open("shared/") + folder.grant_read_access(ds_manager.email) + _write(do_manager, "shared/data.txt", "secret") + do_manager.sync(auto_checkpoint=False) + ds_manager.sync() + received = _received(ds_manager, do_manager, "shared/data.txt") + assert received.exists() + + folder.revoke_read_access(ds_manager.email) + do_manager.sync(auto_checkpoint=False) + ds_manager.sync() + assert not received.exists(), "The DS kept a file it may no longer read." + + +def test_restarted_owner_sends_nothing_new(): + ds_manager, do_manager = _pair() + do_manager.datasite_owner_syncer.perm_context.open("shared/").grant_read_access( + ds_manager.email + ) + _write(do_manager, "shared/data.txt", "one") + do_manager.sync(auto_checkpoint=False) + sent_before = _outbox_paths(ds_manager, do_manager) + + do_manager._copy().sync(auto_checkpoint=False) + assert _outbox_paths(ds_manager, do_manager) == sent_before + + +def test_lost_views_resend_everything(): + ds_manager, do_manager = _pair() + do_manager.datasite_owner_syncer.perm_context.open("shared/").grant_read_access( + ds_manager.email + ) + _write(do_manager, "shared/data.txt", "one") + do_manager.sync(auto_checkpoint=False) + (do_manager.syftbox_folder / CACHE_DIR / DELIVERY_VIEWS_FILENAME).unlink() + + do_manager.sync(auto_checkpoint=False) + assert _outbox_paths(ds_manager, do_manager).count("shared/data.txt") == 2 + + +def test_failed_outbox_write_resent_next_sync(monkeypatch): + ds_manager, do_manager = _pair() + do_manager.datasite_owner_syncer.perm_context.open("shared/").grant_read_access( + ds_manager.email + ) + _write(do_manager, "shared/data.txt", "one") + + def fail_upload(*args, **kwargs): + raise ConnectionError("upload failed") + + with monkeypatch.context() as m: + m.setattr(ConnectionRouter, "owner_write_event_messages_to_outbox", fail_upload) + with pytest.raises(ConnectionError): + do_manager.sync(auto_checkpoint=False) + + do_manager.sync(auto_checkpoint=False) + ds_manager.sync() + assert _received(ds_manager, do_manager, "shared/data.txt").read_text() == "one" diff --git a/tests/unit/test_permission_file_case.py b/tests/unit/test_permission_file_case.py deleted file mode 100644 index 8cb8920074d..00000000000 --- a/tests/unit/test_permission_file_case.py +++ /dev/null @@ -1,47 +0,0 @@ -"""A permission file keeps its meaning whatever case its name has. - -macOS and Windows keep the case of a name but match it without case, so a -write to SYFT.PUB.YAML replaces the content of syft.pub.yaml. The event split -must classify that write as a permission event, or the DO sends it as ordinary -data and never recomputes who may read the files it governs. -""" - -import uuid - -from syft.sync.events.file_change_event import FileChangeEvent -from syft.sync.sync.datasite_owner_syncer import DatasiteOwnerSyncer - - -def _event(path: str) -> FileChangeEvent: - return FileChangeEvent( - id=uuid.uuid4(), - path_in_datasite=path, - datasite_email="owner@test.com", - content="", - submitted_timestamp=0.0, - timestamp=0.0, - ) - - -def _split(paths: list[str]) -> tuple[list[str], list[str]]: - """Split by path. _split_events reads no instance state.""" - perm, data = DatasiteOwnerSyncer._split_events(None, [_event(p) for p in paths]) - return ( - [str(e.path_in_datasite) for e in perm], - [str(e.path_in_datasite) for e in data], - ) - - -def test_permission_events_are_split_by_name_without_case(): - """A name that only ends with the constant stays a data event.""" - perm, data = _split( - [ - "SYFT.PUB.YAML", - "sub/Syft.Pub.Yaml", - "syft.pub.yaml", - "notsyft.pub.yaml", - "sub/data.csv", - ] - ) - assert perm == ["SYFT.PUB.YAML", "sub/Syft.Pub.Yaml", "syft.pub.yaml"] - assert data == ["notsyft.pub.yaml", "sub/data.csv"] diff --git a/tests/unit/test_persisted_cache.py b/tests/unit/test_persisted_cache.py index ab1b31658b6..026bc83846f 100644 --- a/tests/unit/test_persisted_cache.py +++ b/tests/unit/test_persisted_cache.py @@ -201,8 +201,8 @@ def test_peer_loses_file_deleted_while_other_process_skipped_it(monkeypatch): assert not received.exists(), "The DS kept a file that the DO deleted." -def test_read_error_on_pending_file_raises_and_keeps_peer_copy(monkeypatch): - """A pending file that cannot be read must not reach the peer as a deletion.""" +def test_read_error_on_unsent_file_raises_and_keeps_peer_copy(monkeypatch): + """A changed file that cannot be read must not reach the peer as a deletion.""" ds_manager, do_manager = _pair_with_unsent_do_file() do_manager.sync(auto_checkpoint=False) ds_manager.sync() @@ -220,7 +220,7 @@ def test_read_error_on_pending_file_raises_and_keeps_peer_copy(monkeypatch): api_file.write_text("print('api v2')") do_manager_b.sync(auto_checkpoint=False) - # The first read is the resend of the pending path. + # The first read is the send of the changed file. read_file = FSFileConnection.read_file calls = [] diff --git a/tests/unit/test_transport_contracts.py b/tests/unit/test_transport_contracts.py index dbb84239d10..b34e77e2eed 100644 --- a/tests/unit/test_transport_contracts.py +++ b/tests/unit/test_transport_contracts.py @@ -1,5 +1,5 @@ """The transport contracts: the abstract jobs, their helper methods, the -message types, the naming rules and the DeliveryTracker stub. No backend +message types, the naming rules and the DeliveryTracker state. No backend implements them yet.""" from types import SimpleNamespace @@ -13,7 +13,6 @@ DELIVERY_STATE_VERSION, DeliveryState, DeliveryStateStore, - DeliveryTracker, ) from syft.sync.transport import ( Collections, @@ -122,12 +121,6 @@ def test_message_names_carry_their_send_time(name, expected): assert timestamp_from_message_name(name) == expected -def test_delivery_tracker_is_a_stub_until_implemented(): - tracker = DeliveryTracker(store=Mock(spec=DeliveryStateStore)) - with pytest.raises(NotImplementedError): - tracker.has_view("ds@example.com") - - def test_a_delivery_state_store_needs_a_backend(): with pytest.raises(TypeError): DeliveryStateStore() From de64cc5b1ac7216c6fd782ddc7da9cc7c6acd413 Mon Sep 17 00:00:00 2001 From: Pedro Werneck Date: Thu, 22 Oct 2026 09:40:02 -0300 Subject: [PATCH 02/36] perf: make each owner delivery check cheaper The owner checks read access for each file and each peer at every sync. SyftPermContext.can_read() asks the ACL service directly, so a check does not touch the filesystem. DeliveryTracker.plan_all() loads the views once for all peers, not once for each peer. --- .../src/syft_perms/syftperm_context.py | 11 +++++++++- packages/syft-perms/tests/test_syft_perm.py | 8 ++++++++ syft/sync/sync/datasite_owner_syncer.py | 12 +++++++---- syft/sync/sync/delivery_tracker.py | 20 ++++++++++++++----- tests/unit/test_delivery_tracker.py | 13 ++++++++++++ 5 files changed, 54 insertions(+), 10 deletions(-) diff --git a/packages/syft-perms/src/syft_perms/syftperm_context.py b/packages/syft-perms/src/syft_perms/syftperm_context.py index 571ef691bbc..9b86a2b5595 100644 --- a/packages/syft-perms/src/syft_perms/syftperm_context.py +++ b/packages/syft-perms/src/syft_perms/syftperm_context.py @@ -1,6 +1,6 @@ from pathlib import Path -from syft_permissions import ACLService +from syft_permissions import ACLRequest, ACLService, AccessLevel, User from syft_perms.browser import FilesBrowser from syft_perms.file import SyftFile @@ -31,6 +31,15 @@ def open(self, path: str | Path) -> SyftFile | SyftFolder: return SyftFolder(abs_path, self) return SyftFile(abs_path, self) + def can_read(self, path: str, user: str) -> bool: + """Whether ``user`` may read the file at ``path``, relative to the datasite. + + Unlike ``open(path).has_read_access(user)``, this does not touch the + filesystem. + """ + request = ACLRequest(path=path, level=AccessLevel.READ, user=User(id=user)) + return self.service.can_access(request) + @property def files(self) -> FilesBrowser: return FilesBrowser(self, kind="files") diff --git a/packages/syft-perms/tests/test_syft_perm.py b/packages/syft-perms/tests/test_syft_perm.py index 0af81818e13..69afbd14cb8 100644 --- a/packages/syft-perms/tests/test_syft_perm.py +++ b/packages/syft-perms/tests/test_syft_perm.py @@ -161,6 +161,14 @@ def test_no_permission_denies(sp): assert not file.has_admin_access(USER) +def test_can_read_matches_has_read_access(sp): + sp.open("shared/").grant_read_access(USER) + assert sp.can_read("shared/data.csv", USER) + assert not sp.can_read("shared/data.csv", USER2) + assert sp.can_read("other.csv", OWNER) + assert not sp.can_read("other.csv", USER) + + def test_wildcard_grant(sp, datasite): yaml_data = { "rules": [ diff --git a/syft/sync/sync/datasite_owner_syncer.py b/syft/sync/sync/datasite_owner_syncer.py index 3283098d099..8ada51ed62e 100644 --- a/syft/sync/sync/datasite_owner_syncer.py +++ b/syft/sync/sync/datasite_owner_syncer.py @@ -634,7 +634,7 @@ def _syncable_hashes(self) -> dict[str, str]: def _may_read(self, peer: str, path: str) -> bool: """``check_read_permissions`` without a reload: the caller reloads once.""" - return self.perm_context.open(path).has_read_access(peer) + return self.perm_context.can_read(path, peer) def deliver(self, peer_emails: list[str]) -> None: """Send each peer the files its view lacks, and deletes for the files it @@ -643,9 +643,13 @@ def deliver(self, peer_emails: list[str]) -> None: deliveries.confirm(self.connection_router.transport.p2p.delivered()) self.perm_context._reload() hashes = self._syncable_hashes() - for peer in peer_emails: - readable = {p: h for p, h in hashes.items() if self._may_read(peer, p)} - plan = deliveries.plan(peer, readable) + plans = deliveries.plan_all( + { + peer: {p: h for p, h in hashes.items() if self._may_read(peer, p)} + for peer in peer_emails + } + ) + for peer, plan in plans.items(): events = [self._create_current_state_event(p) for p in plan.send] events += [self._delete_event(p) for p in plan.delete] if events: diff --git a/syft/sync/sync/delivery_tracker.py b/syft/sync/sync/delivery_tracker.py index 1810e0519d6..6cc33c41954 100644 --- a/syft/sync/sync/delivery_tracker.py +++ b/syft/sync/sync/delivery_tracker.py @@ -189,11 +189,21 @@ def plan(self, peer: str, readable: Mapping[str, str]) -> DeliveryPlan: ``readable`` maps every path ``peer`` may read now to its current hash. """ - view = self.store.load().views.get(peer, {}) - return DeliveryPlan( - send=sorted(p for p, h in readable.items() if view.get(p) != h), - delete=sorted(p for p in view if p not in readable), - ) + return self.plan_all({peer: readable})[peer] + + def plan_all( + self, readable_by_peer: Mapping[str, Mapping[str, str]] + ) -> dict[str, DeliveryPlan]: + """``plan`` for each peer in ``readable_by_peer``, from one load.""" + views = self.store.load().views + plans = {} + for peer, readable in readable_by_peer.items(): + view = views.get(peer, {}) + plans[peer] = DeliveryPlan( + send=sorted(p for p, h in readable.items() if view.get(p) != h), + delete=sorted(p for p in view if p not in readable), + ) + return plans def sent( self, diff --git a/tests/unit/test_delivery_tracker.py b/tests/unit/test_delivery_tracker.py index 1a1a6783467..f8def779999 100644 --- a/tests/unit/test_delivery_tracker.py +++ b/tests/unit/test_delivery_tracker.py @@ -62,6 +62,19 @@ def test_queued_send_updates_view_on_confirm(): assert tracker.plan(PEER, {"a": "h1"}).send == [] +def test_plan_all_loads_state_once(monkeypatch): + tracker = _tracker_holding({"a": "h1"}) + loads = [] + load = MemoryDeliveryStateStore.load + monkeypatch.setattr( + MemoryDeliveryStateStore, "load", lambda self: loads.append(1) or load(self) + ) + plans = tracker.plan_all({PEER: {"a": "h1"}, "new@x.com": {"a": "h1"}}) + assert plans[PEER].send == [] + assert plans["new@x.com"].send == ["a"] + assert len(loads) == 1 + + def test_confirm_unknown_message_raises(): tracker = DeliveryTracker(MemoryDeliveryStateStore()) with pytest.raises(KeyError): From 90de0c8be1be2eaef66bd33b3e8d5e2b0a2ae0e8 Mon Sep 17 00:00:00 2001 From: Pedro Werneck Date: Thu, 22 Oct 2026 11:48:39 -0300 Subject: [PATCH 03/36] refactor: remove unused delivery code and unclear names Delete the functions that production does not call: check_read_permissions, process_local_changes, DeliveryTracker.has_view and DeliveryTracker.reset. DeliveryTracker.plan now plans all peers from one load. Rename _delete_event to _deletion_event. _record_local_changes writes the event log only with persist_owner_state. The docstring now states two rules. A change that the owner accepts from one peer goes to every peer that may read it. The views are kept on local disk only. Delete the tests of removed code and the tests that only pin internals. Add a test for accepted changes. --- packages/syft-perms/tests/test_syft_perm.py | 2 +- syft/sync/sync/datasite_owner_syncer.py | 47 ++++++--------- syft/sync/sync/delivery_tracker.py | 40 ++++++------- tests/unit/test_delivery_tracker.py | 50 +++++----------- tests/unit/test_sync_manager.py | 63 +++++++++++++-------- 5 files changed, 89 insertions(+), 113 deletions(-) diff --git a/packages/syft-perms/tests/test_syft_perm.py b/packages/syft-perms/tests/test_syft_perm.py index 69afbd14cb8..875ee80819b 100644 --- a/packages/syft-perms/tests/test_syft_perm.py +++ b/packages/syft-perms/tests/test_syft_perm.py @@ -161,7 +161,7 @@ def test_no_permission_denies(sp): assert not file.has_admin_access(USER) -def test_can_read_matches_has_read_access(sp): +def test_can_read_follows_grants(sp): sp.open("shared/").grant_read_access(USER) assert sp.can_read("shared/data.csv", USER) assert not sp.can_read("shared/data.csv", USER2) diff --git a/syft/sync/sync/datasite_owner_syncer.py b/syft/sync/sync/datasite_owner_syncer.py index 8ada51ed62e..523d787b3ac 100644 --- a/syft/sync/sync/datasite_owner_syncer.py +++ b/syft/sync/sync/datasite_owner_syncer.py @@ -579,20 +579,19 @@ def _create_current_state_event(self, path: str) -> FileChangeEvent: except (KeyError, FileNotFoundError): # The in-memory cache raises KeyError, the filesystem cache # FileNotFoundError. - content = None + return self._deletion_event(path) timestamp = create_event_timestamp() return FileChangeEvent( id=uuid4(), path_in_datasite=Path(path), content=content, - new_hash=None if content is None else get_event_hash_from_content(content), + new_hash=get_event_hash_from_content(content), submitted_timestamp=timestamp, timestamp=timestamp, datasite_email=self.email, - is_deleted=content is None, ) - def _delete_event(self, path: str) -> FileChangeEvent: + def _deletion_event(self, path: str) -> FileChangeEvent: """An event that removes ``path`` from a peer.""" timestamp = create_event_timestamp() return FileChangeEvent( @@ -607,22 +606,17 @@ def _delete_event(self, path: str) -> FileChangeEvent: ) def _record_local_changes(self) -> None: - """Find the local file changes and add them to the owner's event log.""" + """Update the file hashes from the local files. With + ``persist_owner_state``, also write the changes to the owner's event log.""" file_change_events_message = self.event_cache.process_local_file_changes() - if file_change_events_message is None: + if file_change_events_message is None or not self.persist_owner_state: return - if self.persist_owner_state: - # Record local events in the owner's log and feed them into rolling - # state so they flow through incremental/full checkpoints. - self.syftbox_events_queue.put(file_change_events_message) - self._add_events_to_rolling_state(file_change_events_message) + # Feed the events into rolling state so they flow through + # incremental/full checkpoints. + self.syftbox_events_queue.put(file_change_events_message) + self._add_events_to_rolling_state(file_change_events_message) self.process_syftbox_events_queue() - def process_local_changes(self, recipients: list[str]): - """Record the local file changes, then deliver to ``recipients``.""" - self._record_local_changes() - self.deliver(recipients) - def _syncable_hashes(self) -> dict[str, str]: """The current hash of every file that a peer can get, by path.""" collections_rel = self.event_cache.collections_relative_path() @@ -632,10 +626,6 @@ def _syncable_hashes(self) -> dict[str, str]: if is_normal_syncable_path(str(path), collections_path=collections_rel) } - def _may_read(self, peer: str, path: str) -> bool: - """``check_read_permissions`` without a reload: the caller reloads once.""" - return self.perm_context.can_read(path, peer) - def deliver(self, peer_emails: list[str]) -> None: """Send each peer the files its view lacks, and deletes for the files it may no longer have.""" @@ -643,15 +633,19 @@ def deliver(self, peer_emails: list[str]) -> None: deliveries.confirm(self.connection_router.transport.p2p.delivered()) self.perm_context._reload() hashes = self._syncable_hashes() - plans = deliveries.plan_all( + plans = deliveries.plan( { - peer: {p: h for p, h in hashes.items() if self._may_read(peer, p)} + peer: { + p: h + for p, h in hashes.items() + if self.perm_context.can_read(p, peer) + } for peer in peer_emails } ) for peer, plan in plans.items(): events = [self._create_current_state_event(p) for p in plan.send] - events += [self._delete_event(p) for p in plan.delete] + events += [self._deletion_event(p) for p in plan.delete] if events: self.outbox_queue.put((peer, FileChangeEventsMessage(events=events))) self.process_syftbox_events_queue() @@ -691,11 +685,6 @@ def check_write_permission(self, sender_email: str, path: str) -> bool: self.perm_context._reload() return self.perm_context.open(path).has_write_access(sender_email) - def check_read_permissions(self, recipient_email: str, path: str) -> bool: - """Check if recipient has read access to the given path.""" - self.perm_context._reload() - return self.perm_context.open(path).has_read_access(recipient_email) - def handle_proposed_filechange_events_message( self, sender_email: str, proposed_events_message: ProposedFileChangesMessage ): @@ -791,7 +780,7 @@ def _record_sent( changes[path] = None elif is_normal_syncable_path( path, collections_path=collections_rel - ) and self._may_read(recipient, path): + ) and self.perm_context.can_read(path, recipient): changes[path] = event.new_hash self.event_cache.deliveries.sent( recipient, message.message_filepath.as_string(), status, changes diff --git a/syft/sync/sync/delivery_tracker.py b/syft/sync/sync/delivery_tracker.py index 6cc33c41954..1529929c9b5 100644 --- a/syft/sync/sync/delivery_tracker.py +++ b/syft/sync/sync/delivery_tracker.py @@ -14,8 +14,11 @@ in the view, but no longer readable -> delete in the view, but the file is gone -> delete +Every readable file counts, whoever wrote it. Thus a change that the owner +accepts from one peer also goes to every other peer that may read it. + A peer with no view, because it is new or its record was lost, starts with an -empty view (a ``PeerReset``), so it is sent every file it may read. +empty view, so it is sent every file it may read. Recording a send ---------------- @@ -29,6 +32,12 @@ A view therefore never changes before a delivery is confirmed. A change that is not sent, because the upload failed, the peer was skipped, or a process stopped before the send, stays in the plan of the next sync. + +Storage +------- +The views are kept on the owner's local disk only, also with +``persist_owner_state``. An owner that loses its local state sends every peer +every file it may read once again. """ import os @@ -131,7 +140,7 @@ def load(self) -> DeliveryState: @contextmanager def locked(self): self.path.parent.mkdir(parents=True, exist_ok=True) - with open(self.path.with_suffix(self.path.suffix + ".lock"), "a") as f: + with open(self.path.with_suffix(f"{self.path.suffix}.lock"), "a") as f: portalocker.lock(f, portalocker.LOCK_EX) try: yield @@ -167,34 +176,19 @@ class DeliveryTracker: def __init__(self, store: DeliveryStateStore): self.store = store - def has_view(self, peer: str) -> bool: - """Whether ``peer`` has a view. False means a ``PeerReset`` is due.""" - return peer in self.store.load().views - - def reset(self, peer: str) -> None: - """Start ``peer``'s view empty (a ``PeerReset``), so the next plan sends - it every file it may read.""" - with self.store.locked(): - state = self.store.load() - state.views[peer] = {} - self.store.save(state) - def clear(self) -> None: """Forget every view and every queued send.""" with self.store.locked(): self.store.save(DeliveryState()) - def plan(self, peer: str, readable: Mapping[str, str]) -> DeliveryPlan: - """What to send and delete for ``peer``. - - ``readable`` maps every path ``peer`` may read now to its current hash. - """ - return self.plan_all({peer: readable})[peer] - - def plan_all( + def plan( self, readable_by_peer: Mapping[str, Mapping[str, str]] ) -> dict[str, DeliveryPlan]: - """``plan`` for each peer in ``readable_by_peer``, from one load.""" + """What to send and delete for each peer in ``readable_by_peer``. + + ``readable_by_peer`` maps each peer to every path it may read now, with + the path's current hash. + """ views = self.store.load().views plans = {} for peer, readable in readable_by_peer.items(): diff --git a/tests/unit/test_delivery_tracker.py b/tests/unit/test_delivery_tracker.py index f8def779999..7a899b94a10 100644 --- a/tests/unit/test_delivery_tracker.py +++ b/tests/unit/test_delivery_tracker.py @@ -29,56 +29,29 @@ def _tracker_holding(view: dict[str, str]) -> DeliveryTracker: def test_plan_sends_changed_files_deletes_unreadable(): tracker = _tracker_holding({"same": "h1", "old": "h1", "revoked": "h1"}) - plan = tracker.plan(PEER, {"same": "h1", "old": "h2", "new": "h3"}) + plan = tracker.plan({PEER: {"same": "h1", "old": "h2", "new": "h3"}})[PEER] assert plan.send == ["new", "old"] assert plan.delete == ["revoked"] def test_peer_without_view_gets_all_readable_files(): tracker = DeliveryTracker(MemoryDeliveryStateStore()) - assert not tracker.has_view(PEER) - assert tracker.plan(PEER, {"a": "h1", "b": "h2"}).send == ["a", "b"] - - -def test_reset_resends_everything(): - tracker = _tracker_holding({"a": "h1"}) - tracker.reset(PEER) - assert tracker.has_view(PEER) - assert tracker.plan(PEER, {"a": "h1"}).send == ["a"] + assert tracker.plan({PEER: {"a": "h1", "b": "h2"}})[PEER].send == ["a", "b"] def test_delivered_send_updates_view(): tracker = _tracker_holding({"a": "h1", "b": "h1"}) tracker.sent(PEER, "m1", SendStatus.DELIVERED, {"a": "h2", "b": None}) - assert tracker.plan(PEER, {"a": "h2"}).send == [] - assert tracker.plan(PEER, {"a": "h2"}).delete == [] + assert tracker.plan({PEER: {"a": "h2"}})[PEER].send == [] + assert tracker.plan({PEER: {"a": "h2"}})[PEER].delete == [] def test_queued_send_updates_view_on_confirm(): tracker = DeliveryTracker(MemoryDeliveryStateStore()) tracker.sent(PEER, "m1", SendStatus.QUEUED, {"a": "h1"}) - assert tracker.plan(PEER, {"a": "h1"}).send == ["a"] + assert tracker.plan({PEER: {"a": "h1"}})[PEER].send == ["a"] tracker.confirm(["m1"]) - assert tracker.plan(PEER, {"a": "h1"}).send == [] - - -def test_plan_all_loads_state_once(monkeypatch): - tracker = _tracker_holding({"a": "h1"}) - loads = [] - load = MemoryDeliveryStateStore.load - monkeypatch.setattr( - MemoryDeliveryStateStore, "load", lambda self: loads.append(1) or load(self) - ) - plans = tracker.plan_all({PEER: {"a": "h1"}, "new@x.com": {"a": "h1"}}) - assert plans[PEER].send == [] - assert plans["new@x.com"].send == ["a"] - assert len(loads) == 1 - - -def test_confirm_unknown_message_raises(): - tracker = DeliveryTracker(MemoryDeliveryStateStore()) - with pytest.raises(KeyError): - tracker.confirm(["never-sent"]) + assert tracker.plan({PEER: {"a": "h1"}})[PEER].send == [] def test_local_store_persists_state_refuses_later_version(tmp_path): @@ -87,7 +60,9 @@ def test_local_store_persists_state_refuses_later_version(tmp_path): PEER, "m1", SendStatus.DELIVERED, {"a": "h1"} ) assert ( - DeliveryTracker(LocalDeliveryStateStore(path)).plan(PEER, {"a": "h1"}).send + DeliveryTracker(LocalDeliveryStateStore(path)) + .plan({PEER: {"a": "h1"}})[PEER] + .send == [] ) @@ -127,9 +102,10 @@ def send(peer: str): first.join() second.join() - tracker = DeliveryTracker(LocalDeliveryStateStore(path)) - assert tracker.has_view("one@x.com") - assert tracker.has_view("two@x.com") + plans = DeliveryTracker(LocalDeliveryStateStore(path)).plan( + {"one@x.com": {"a": "h1"}, "two@x.com": {"a": "h1"}} + ) + assert plans["one@x.com"].send == plans["two@x.com"].send == [] # --- The owner syncer --------------------------------------------------------- diff --git a/tests/unit/test_sync_manager.py b/tests/unit/test_sync_manager.py index f97b2b8a22d..3393caa04d4 100644 --- a/tests/unit/test_sync_manager.py +++ b/tests/unit/test_sync_manager.py @@ -453,6 +453,18 @@ def _add_known_peer(do_manager, email: str) -> None: do_manager.peer_manager.peer_store.add_peer(Peer(email=email, version=version)) +def _peer_router(ds_manager, do_manager, email: str) -> ConnectionRouter: + """Make ``email`` a second peer of the DO, and return its router.""" + conn = GDriveConnection.from_service( + email, ds_manager._connection_router.connections[0].drive_service + ) + conn.add_peer(do_manager.email) + _add_known_peer(do_manager, email) + return ConnectionRouter( + connections=[conn], peer_store=_store_knowing(email, do_manager.email) + ) + + def test_job_files_sync_to_submitter_only(): """Test that job files only sync to the peer who submitted the job. @@ -502,16 +514,8 @@ def test_job_files_sync_to_submitter_only(): # Process local changes with both recipients recipients = [submitter_email, non_submitter_email] - submitter_conn = GDriveConnection.from_service( - submitter_email, ds_manager._connection_router.connections[0].drive_service - ) - submitter_conn.add_peer(do_manager.email) - _add_known_peer(do_manager, submitter_email) - submitter_router = ConnectionRouter( - connections=[submitter_conn], - peer_store=_store_knowing(submitter_email, do_manager.email), - ) - do_manager.datasite_owner_syncer.process_local_changes(recipients) + submitter_router = _peer_router(ds_manager, do_manager, submitter_email) + do_manager.datasite_owner_syncer.sync(recipients) messages_for_non_submitter = ( ds_manager._connection_router.watcher_get_events_messages( @@ -591,7 +595,7 @@ def test_job_file_beside_outputs_does_not_sync_to_submitter(): assert (review_dir / "outputs" / "result.txt").exists() review_rel = review_dir.relative_to(do_manager.syftbox_folder / do_manager.email) - do_manager.datasite_owner_syncer.process_local_changes([ds_email]) + do_manager.datasite_owner_syncer.sync([ds_email]) paths = [ str(event.path_in_datasite) @@ -704,16 +708,7 @@ def test_permission_change_triggers_resend(): datasite_dir_do = do_manager.syftbox_folder / do_manager.email - # Set up a second peer connection router for peer B - peer_b_conn = GDriveConnection.from_service( - peer_b_email, ds_manager._connection_router.connections[0].drive_service - ) - peer_b_conn.add_peer(do_manager.email) - _add_known_peer(do_manager, peer_b_email) - peer_b_router = ConnectionRouter( - connections=[peer_b_conn], - peer_store=_store_knowing(peer_b_email, do_manager.email), - ) + peer_b_router = _peer_router(ds_manager, do_manager, peer_b_email) # Grant only peer A read access to project/ ctx = do_manager.datasite_owner_syncer.perm_context @@ -726,7 +721,7 @@ def test_permission_change_triggers_resend(): # First sync: only peer A should receive the file recipients = [peer_a_email, peer_b_email] - do_manager.datasite_owner_syncer.process_local_changes(recipients) + do_manager.datasite_owner_syncer.sync(recipients) messages_for_a = ds_manager._connection_router.watcher_get_events_messages( do_manager.email, None @@ -749,7 +744,7 @@ def test_permission_change_triggers_resend(): ctx.open("project/").grant_read_access(peer_b_email) # Second sync: peer B should receive data.txt via resend - do_manager.datasite_owner_syncer.process_local_changes(recipients) + do_manager.datasite_owner_syncer.sync(recipients) messages_for_b_after = peer_b_router.watcher_get_events_messages( do_manager.email, None @@ -763,6 +758,28 @@ def test_permission_change_triggers_resend(): ) +def test_accepted_change_reaches_other_readers(): + """A change that the DO accepts from peer A also goes to peer B, who may read it.""" + ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + use_in_memory_cache=False, + ) + peer_b_email = "peer_b@example.com" + peer_b_router = _peer_router(ds_manager, do_manager, peer_b_email) + shared = do_manager.datasite_owner_syncer.perm_context.open("shared/") + shared.grant_write_access(ds_manager.email) + shared.grant_read_access(peer_b_email) + + ds_manager._send_file_change(f"{do_manager.email}/shared/proposal.txt", "from A") + do_manager.datasite_owner_syncer.sync([ds_manager.email, peer_b_email]) + + paths_for_b = [ + str(e.path_in_datasite) + for m in peer_b_router.watcher_get_events_messages(do_manager.email, None) + for e in m.events + ] + assert "shared/proposal.txt" in paths_for_b + + def test_default_collections_folder_picks_shareable_spec_regardless_of_order(): """The collections folder is chosen by owner_only, not by position.""" from syft.sync.syftbox_manager import default_collections_folder From 9baa0122fc96266477749c16c7aad9388fe806c1 Mon Sep 17 00:00:00 2001 From: Pedro Werneck Date: Thu, 22 Oct 2026 16:11:46 -0300 Subject: [PATCH 04/36] fix: send owner changes before pulling proposals An outdated proposal raises on every sync and stays in the inbox. The owner sent its changes only after the pull, so such a proposal stopped every delivery to every peer. Now sync() sends first, as before the delivery tracker. A change that the owner accepts from one peer reaches the other readers in the next sync. --- syft/sync/sync/datasite_owner_syncer.py | 10 +++++++--- tests/unit/test_delivery_tracker.py | 23 +++++++++++++++++++++++ tests/unit/test_sync_manager.py | 6 +++++- 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/syft/sync/sync/datasite_owner_syncer.py b/syft/sync/sync/datasite_owner_syncer.py index 523d787b3ac..3f48555a7c0 100644 --- a/syft/sync/sync/datasite_owner_syncer.py +++ b/syft/sync/sync/datasite_owner_syncer.py @@ -201,8 +201,12 @@ def _save_rolling_state(self) -> None: tmp.rename(path) def sync(self, peer_emails: list[str], recompute_hashes: bool = True): - """Pull proposed changes from ``peer_emails``, then send each of them - what it does not hold yet.""" + """Send each of ``peer_emails`` what it does not hold yet, then pull + their proposed changes. + + The send comes first, so a proposal that fails does not hold back the + owner's own changes. + """ if self.persist_owner_state: self._load_rolling_state() try: @@ -211,6 +215,7 @@ def sync(self, peer_emails: list[str], recompute_hashes: bool = True): if recompute_hashes: self._record_local_changes() + self.deliver(peer_emails) for peer_email in peer_emails: proposed_count = 0 @@ -228,7 +233,6 @@ def sync(self, peer_emails: list[str], recompute_hashes: bool = True): f"from {peer_email}" ) self.process_syftbox_events_queue() - self.deliver(peer_emails) finally: if self.persist_owner_state: self._save_rolling_state() diff --git a/tests/unit/test_delivery_tracker.py b/tests/unit/test_delivery_tracker.py index 7a899b94a10..d92122f12e5 100644 --- a/tests/unit/test_delivery_tracker.py +++ b/tests/unit/test_delivery_tracker.py @@ -7,6 +7,9 @@ from syft.sync.connections.connection_router import ConnectionRouter from syft.sync.sync.constants import CACHE_DIR +from syft.sync.sync.caches.datasite_owner_cache import ( + ProposedEventFileOutdatedException, +) from syft.sync.sync.delivery_tracker import ( DELIVERY_STATE_VERSION, DELIVERY_VIEWS_FILENAME, @@ -217,6 +220,26 @@ def test_lost_views_resend_everything(): assert _outbox_paths(ds_manager, do_manager).count("shared/data.txt") == 2 +def test_outdated_proposal_does_not_block_owner_changes(): + ds_manager, do_manager = _pair() + shared = do_manager.datasite_owner_syncer.perm_context.open("shared/") + shared.grant_write_access(ds_manager.email) + _write(do_manager, "shared/a.txt", "v1") + do_manager.sync(auto_checkpoint=False) + ds_manager.sync() + + # The DS edits a.txt from v1 while the DO edits it too, so the + # proposal is outdated and its handling raises on every sync. + ds_manager._send_file_change(f"{do_manager.email}/shared/a.txt", "ds edit") + _write(do_manager, "shared/a.txt", "do edit") + _write(do_manager, "shared/b.txt", "new") + with pytest.raises(ProposedEventFileOutdatedException): + do_manager.sync(auto_checkpoint=False) + + ds_manager.sync() + assert _received(ds_manager, do_manager, "shared/b.txt").read_text() == "new" + + def test_failed_outbox_write_resent_next_sync(monkeypatch): ds_manager, do_manager = _pair() do_manager.datasite_owner_syncer.perm_context.open("shared/").grant_read_access( diff --git a/tests/unit/test_sync_manager.py b/tests/unit/test_sync_manager.py index 3393caa04d4..40150f5f590 100644 --- a/tests/unit/test_sync_manager.py +++ b/tests/unit/test_sync_manager.py @@ -759,7 +759,8 @@ def test_permission_change_triggers_resend(): def test_accepted_change_reaches_other_readers(): - """A change that the DO accepts from peer A also goes to peer B, who may read it.""" + """A change that the DO accepts from peer A goes to peer B, who may read it, + in the next DO sync.""" ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( use_in_memory_cache=False, ) @@ -770,6 +771,9 @@ def test_accepted_change_reaches_other_readers(): shared.grant_read_access(peer_b_email) ds_manager._send_file_change(f"{do_manager.email}/shared/proposal.txt", "from A") + # The first sync accepts the change. The second one sends it to peer B, + # because a sync sends before it pulls. + do_manager.datasite_owner_syncer.sync([ds_manager.email, peer_b_email]) do_manager.datasite_owner_syncer.sync([ds_manager.email, peer_b_email]) paths_for_b = [ From d40f7e8bbc7ecbd1841a617fffd0cd4ad3c70c73 Mon Sep 17 00:00:00 2001 From: Pedro Werneck Date: Thu, 22 Oct 2026 16:11:47 -0300 Subject: [PATCH 05/36] refactor: move the encryption hold out of the outbox loop _hold_or_drop() warns about a message that cannot go to a peer now, and tells the loop whether the message waits or is dropped. --- syft/sync/sync/datasite_owner_syncer.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/syft/sync/sync/datasite_owner_syncer.py b/syft/sync/sync/datasite_owner_syncer.py index 3f48555a7c0..e567bbc9819 100644 --- a/syft/sync/sync/datasite_owner_syncer.py +++ b/syft/sync/sync/datasite_owner_syncer.py @@ -756,16 +756,23 @@ def process_syftbox_events_queue(self): recipient, message ) except (PeerEncryptionPendingError, EncryptionPolicyConflictError) as e: - if not self.connection_router.peer_store.should_hold(recipient): - warnings.warn( - f"Dropped an outbox message for {recipient}: not a peer" - ) - continue - warnings.warn(f"Changes for {recipient} wait for a later sync: {e}") - waiting_for.add(recipient) + if self._hold_or_drop(recipient, e): + waiting_for.add(recipient) else: self._record_sent(recipient, message, status) + def _hold_or_drop(self, recipient: str, error: Exception) -> bool: + """Warn about an outbox message that cannot go to ``recipient`` now. + + Returns True when the message waits for a later sync, and False when it + is dropped because ``recipient`` is not a peer. + """ + if not self.connection_router.peer_store.should_hold(recipient): + warnings.warn(f"Dropped an outbox message for {recipient}: not a peer") + return False + warnings.warn(f"Changes for {recipient} wait for a later sync: {error}") + return True + def _record_sent( self, recipient: str, message: FileChangeEventsMessage, status: SendStatus ) -> None: From 8142b073db51d8cf790b64082684363779c311a1 Mon Sep 17 00:00:00 2001 From: Pedro Werneck Date: Thu, 22 Oct 2026 16:38:41 -0300 Subject: [PATCH 06/36] feat: seed views on upgrade and split large deliveries An owner with no views state starts each current peer's view with every file it may read, so the upgrade sends nothing extra. The paths that the earlier client still had to send stay out of the view, so they are sent. The seed runs before the local changes are recorded, so a file changed while the owner was offline is still sent. deliver() splits a peer's events into messages of about 50 MB of file content at most. --- syft/sync/sync/caches/datasite_owner_cache.py | 18 ++++- syft/sync/sync/constants.py | 2 + syft/sync/sync/datasite_owner_syncer.py | 73 ++++++++++++++--- syft/sync/sync/delivery_tracker.py | 44 +++++++++-- tests/unit/test_delivery_tracker.py | 79 +++++++++++++++++-- 5 files changed, 188 insertions(+), 28 deletions(-) diff --git a/syft/sync/sync/caches/datasite_owner_cache.py b/syft/sync/sync/caches/datasite_owner_cache.py index eb65c04155f..92e8e4e8395 100644 --- a/syft/sync/sync/caches/datasite_owner_cache.py +++ b/syft/sync/sync/caches/datasite_owner_cache.py @@ -18,7 +18,11 @@ ) from syft.sync.utils.syftbox_utils import get_event_hash_from_content from syft.sync.checkpoints.checkpoint import Checkpoint -from syft.sync.sync.constants import CACHE_DIR, OWNER_FILE_HASHES_FILENAME +from syft.sync.sync.constants import ( + CACHE_DIR, + OWNER_FILE_HASHES_FILENAME, + OWNER_PENDING_DELIVERIES_FILENAME, +) from syft.sync.sync.delivery_tracker import ( DELIVERY_VIEWS_FILENAME, DeliveryTracker, @@ -317,6 +321,18 @@ def clear_cache(self): self.file_hashes.clear() self.deliveries.clear() + def legacy_pending_paths(self) -> dict[str, set[str]]: + """The paths that a client before the delivery tracker still had to send, + per peer. Empty for an in-memory cache.""" + if self.syftbox_folder is None: + return {} + pending = PersistedDict( + path=Path(self.syftbox_folder) + / CACHE_DIR + / OWNER_PENDING_DELIVERIES_FILENAME + ) + return {peer: set(paths) for peer, paths in pending.items()} + def has_conflict(self, proposed_event: ProposedFileChange) -> bool: if proposed_event.path_in_datasite not in self.file_hashes: if proposed_event.old_hash is None: diff --git a/syft/sync/sync/constants.py b/syft/sync/sync/constants.py index 2e8508efb4f..1972c14723b 100644 --- a/syft/sync/sync/constants.py +++ b/syft/sync/sync/constants.py @@ -1,4 +1,6 @@ CACHE_DIR = ".cache" OWNER_FILE_HASHES_FILENAME = "owner_file_hashes.json" +# Written by clients before the delivery tracker. Read only to seed the views. +OWNER_PENDING_DELIVERIES_FILENAME = "owner_pending_deliveries.json" ROLLING_STATE_FILENAME = "rolling_state.json" diff --git a/syft/sync/sync/datasite_owner_syncer.py b/syft/sync/sync/datasite_owner_syncer.py index e567bbc9819..d1d0d88f7d4 100644 --- a/syft/sync/sync/datasite_owner_syncer.py +++ b/syft/sync/sync/datasite_owner_syncer.py @@ -58,6 +58,10 @@ # Default: upload rolling state to GDrive after this many events DEFAULT_ROLLING_STATE_UPLOAD_THRESHOLD = 1 +# A delivery message holds about this much file content at most. A larger +# catch-up goes out in more than one message. +MAX_DELIVERY_MESSAGE_CONTENT = 50 * 1024 * 1024 + # Minimum number of message files in a peer's outbox before # compact_outbox_if_needed() will merge them into a single message. MIN_MESSAGES_COMPACT = 20 @@ -113,6 +117,25 @@ class DatasiteOwnerSyncerConfig(BaseModel): connection_configs: List[ConnectionConfig] = [] +def _batches_by_content_size( + events: list[FileChangeEvent], limit: int +) -> list[list[FileChangeEvent]]: + """Split ``events`` in order into lists of at most ``limit`` content bytes. + + An event larger than ``limit`` gets a list of its own. + """ + batches: list[list[FileChangeEvent]] = [] + size = 0 + for event in events: + event_size = len(event.content or b"") + if not batches or size + event_size > limit: + batches.append([]) + size = 0 + batches[-1].append(event) + size += event_size + return batches + + class DatasiteOwnerSyncer(BaseModelCallbackMixin): """Responsible for downloading files and checking permissions""" @@ -212,6 +235,9 @@ def sync(self, peer_emails: list[str], recompute_hashes: bool = True): try: if not self.initial_sync_done: self.pull_initial_state() + # Before the local changes are recorded, so that a file changed + # while the owner was offline is not seeded as held. + self._seed_views_on_upgrade(peer_emails) if recompute_hashes: self._record_local_changes() @@ -630,28 +656,49 @@ def _syncable_hashes(self) -> dict[str, str]: if is_normal_syncable_path(str(path), collections_path=collections_rel) } + def _readable_hashes(self, peer_emails: list[str]) -> dict[str, dict[str, str]]: + """For each peer, the current hash of every syncable file it may read. + + The caller reloads the permissions first. + """ + hashes = self._syncable_hashes() + return { + peer: { + p: h for p, h in hashes.items() if self.perm_context.can_read(p, peer) + } + for peer in peer_emails + } + + def _seed_views_on_upgrade(self, peer_emails: list[str]) -> None: + """Start the views of an owner that kept none before. + + Each peer is taken to hold every file it may read now, except the paths + that the earlier client still had to send it. + """ + deliveries = self.event_cache.deliveries + if deliveries.has_state(): + return + self.perm_context._reload() + pending = self.event_cache.legacy_pending_paths() + deliveries.seed( + { + peer: {p: h for p, h in files.items() if p not in pending.get(peer, ())} + for peer, files in self._readable_hashes(peer_emails).items() + } + ) + def deliver(self, peer_emails: list[str]) -> None: """Send each peer the files its view lacks, and deletes for the files it may no longer have.""" deliveries = self.event_cache.deliveries deliveries.confirm(self.connection_router.transport.p2p.delivered()) self.perm_context._reload() - hashes = self._syncable_hashes() - plans = deliveries.plan( - { - peer: { - p: h - for p, h in hashes.items() - if self.perm_context.can_read(p, peer) - } - for peer in peer_emails - } - ) + plans = deliveries.plan(self._readable_hashes(peer_emails)) for peer, plan in plans.items(): events = [self._create_current_state_event(p) for p in plan.send] events += [self._deletion_event(p) for p in plan.delete] - if events: - self.outbox_queue.put((peer, FileChangeEventsMessage(events=events))) + for batch in _batches_by_content_size(events, MAX_DELIVERY_MESSAGE_CONTENT): + self.outbox_queue.put((peer, FileChangeEventsMessage(events=batch))) self.process_syftbox_events_queue() def pull_and_process_next_proposed_filechange( diff --git a/syft/sync/sync/delivery_tracker.py b/syft/sync/sync/delivery_tracker.py index 1529929c9b5..04de66f6e6e 100644 --- a/syft/sync/sync/delivery_tracker.py +++ b/syft/sync/sync/delivery_tracker.py @@ -17,8 +17,15 @@ Every readable file counts, whoever wrote it. Thus a change that the owner accepts from one peer also goes to every other peer that may read it. -A peer with no view, because it is new or its record was lost, starts with an -empty view, so it is sent every file it may read. +A new peer starts with an empty view, so it is sent every file it may read. A +peer that the owner rejects and then approves again keeps its view, and gets +only what changed. + +An owner that kept no views before, because it ran an earlier client or lost +its views file, has file hashes but no state. ``seed`` then starts each current +peer's view with every file it may read now, so the upgrade sends nothing +extra. The paths that the earlier client still had to send are left out, so +they are sent. Recording a send ---------------- @@ -36,8 +43,10 @@ Storage ------- The views are kept on the owner's local disk only, also with -``persist_owner_state``. An owner that loses its local state sends every peer -every file it may read once again. +``persist_owner_state``. An owner that loses its local state, file hashes +included, sends every peer every file it may read once again. With +``persist_owner_state``, the file hashes come back from Drive, so ``seed`` +starts the views from them. """ import os @@ -102,6 +111,10 @@ def load(self) -> DeliveryState: def save(self, state: DeliveryState) -> None: """Store ``state``, replacing what was there.""" + @abstractmethod + def has_state(self) -> bool: + """Whether a state was ever saved.""" + def locked(self) -> ContextManager[None]: """Keep other writers out from a ``load`` to the ``save`` that follows.""" return nullcontext() @@ -111,14 +124,17 @@ class MemoryDeliveryStateStore(DeliveryStateStore): """A state kept in this process only.""" def __init__(self): - self._state = DeliveryState() + self._state: DeliveryState | None = None def load(self) -> DeliveryState: - return self._state.model_copy(deep=True) + return (self._state or DeliveryState()).model_copy(deep=True) def save(self, state: DeliveryState) -> None: self._state = state.model_copy(deep=True) + def has_state(self) -> bool: + return self._state is not None + class LocalDeliveryStateStore(DeliveryStateStore): """A state kept in a JSON file, shared by the processes of one datasite.""" @@ -137,6 +153,9 @@ def load(self) -> DeliveryState: ) return state + def has_state(self) -> bool: + return self.path.exists() + @contextmanager def locked(self): self.path.parent.mkdir(parents=True, exist_ok=True) @@ -181,6 +200,19 @@ def clear(self) -> None: with self.store.locked(): self.store.save(DeliveryState()) + def has_state(self) -> bool: + """Whether the store holds a state. False until the first ``seed``.""" + return self.store.has_state() + + def seed(self, readable_by_peer: Mapping[str, Mapping[str, str]]) -> None: + """Start each view with the files in ``readable_by_peer``, if the store + holds no state yet. Otherwise do nothing.""" + with self.store.locked(): + if self.store.has_state(): + return + views = {peer: dict(files) for peer, files in readable_by_peer.items()} + self.store.save(DeliveryState(views=views)) + def plan( self, readable_by_peer: Mapping[str, Mapping[str, str]] ) -> dict[str, DeliveryPlan]: diff --git a/tests/unit/test_delivery_tracker.py b/tests/unit/test_delivery_tracker.py index d92122f12e5..b910222354e 100644 --- a/tests/unit/test_delivery_tracker.py +++ b/tests/unit/test_delivery_tracker.py @@ -2,11 +2,14 @@ longer have, from a per-peer record of what the peer holds.""" import threading +from pathlib import Path import pytest from syft.sync.connections.connection_router import ConnectionRouter -from syft.sync.sync.constants import CACHE_DIR +from syft.sync.sync import datasite_owner_syncer +from syft.sync.sync.caches.persisted_dict import PersistedDict +from syft.sync.sync.constants import CACHE_DIR, OWNER_PENDING_DELIVERIES_FILENAME from syft.sync.sync.caches.datasite_owner_cache import ( ProposedEventFileOutdatedException, ) @@ -49,6 +52,13 @@ def test_delivered_send_updates_view(): assert tracker.plan({PEER: {"a": "h2"}})[PEER].delete == [] +def test_seed_starts_views_only_without_state(): + tracker = DeliveryTracker(MemoryDeliveryStateStore()) + tracker.seed({PEER: {"a": "h1"}}) + tracker.seed({PEER: {}}) + assert tracker.plan({PEER: {"a": "h1"}})[PEER].send == [] + + def test_queued_send_updates_view_on_confirm(): tracker = DeliveryTracker(MemoryDeliveryStateStore()) tracker.sent(PEER, "m1", SendStatus.QUEUED, {"a": "h1"}) @@ -157,14 +167,17 @@ def test_peer_approved_later_gets_existing_files(): ) -def test_in_memory_owner_sends_file_content(): +def test_in_memory_owner_sends_content_then_delete_on_revoke(): ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection() syncer = do_manager.datasite_owner_syncer - syncer.perm_context.open("shared/").grant_read_access(ds_manager.email) + folder = syncer.perm_context.open("shared/") + folder.grant_read_access(ds_manager.email) syncer.event_cache.file_connection.write_file("shared/data.txt", "hello") do_manager.sync(auto_checkpoint=False) do_manager.sync(auto_checkpoint=False) + folder.revoke_read_access(ds_manager.email) + do_manager.sync(auto_checkpoint=False) messages = ds_manager._connection_router.watcher_get_events_messages( do_manager.email, None ) @@ -174,7 +187,7 @@ def test_in_memory_owner_sends_file_content(): for e in m.events if str(e.path_in_datasite) == "shared/data.txt" ] - assert sent == [(False, "hello")] + assert sent == [(False, "hello"), (True, None)] @pytest.mark.parametrize("persist_owner_state", [True, False]) @@ -207,17 +220,67 @@ def test_restarted_owner_sends_nothing_new(): assert _outbox_paths(ds_manager, do_manager) == sent_before -def test_lost_views_resend_everything(): +def test_upgrade_sends_only_unsent_changes(): ds_manager, do_manager = _pair() do_manager.datasite_owner_syncer.perm_context.open("shared/").grant_read_access( ds_manager.email ) - _write(do_manager, "shared/data.txt", "one") + _write(do_manager, "shared/held.txt", "one") + _write(do_manager, "shared/pending.txt", "one") do_manager.sync(auto_checkpoint=False) - (do_manager.syftbox_folder / CACHE_DIR / DELIVERY_VIEWS_FILENAME).unlink() + + # Make the cache look like one from a client before the delivery tracker: + # no views, and one path that the client still had to send. + cache_dir = do_manager.syftbox_folder / CACHE_DIR + (cache_dir / DELIVERY_VIEWS_FILENAME).unlink() + pending = PersistedDict(path=cache_dir / OWNER_PENDING_DELIVERIES_FILENAME) + pending[ds_manager.email] = {"shared/pending.txt": "msg"} + _write(do_manager, "shared/offline.txt", "new") + + do_manager.sync(auto_checkpoint=False) + sent = _outbox_paths(ds_manager, do_manager) + assert sent.count("shared/held.txt") == 1 + assert sent.count("shared/pending.txt") == 2 + assert sent.count("shared/offline.txt") == 1 + + +def test_private_dataset_sent_to_peer_never_revoked(): + ds_manager, do_manager = _pair() + syncer = do_manager.datasite_owner_syncer + _write(do_manager, "private/data.csv", "rows") + message = syncer.event_cache.create_events_for_files( + {Path("private/data.csv"): b"rows"} + ) + syncer.queue_event_for_syftbox( + recipients=[ds_manager.email], file_change_events_message=message + ) + syncer.process_syftbox_events_queue() do_manager.sync(auto_checkpoint=False) - assert _outbox_paths(ds_manager, do_manager).count("shared/data.txt") == 2 + do_manager.sync(auto_checkpoint=False) + messages = ds_manager._connection_router.watcher_get_events_messages( + do_manager.email, None + ) + events = [ + (str(e.path_in_datasite), e.is_deleted) for m in messages for e in m.events + ] + assert events == [("private/data.csv", False)] + + +def test_large_catch_up_split_into_messages(monkeypatch): + monkeypatch.setattr(datasite_owner_syncer, "MAX_DELIVERY_MESSAGE_CONTENT", 6) + ds_manager, do_manager = _pair() + do_manager.datasite_owner_syncer.perm_context.open("shared/").grant_read_access( + ds_manager.email + ) + for name in ("a", "b", "c"): + _write(do_manager, f"shared/{name}.txt", "abc") + + do_manager.sync(auto_checkpoint=False) + messages = ds_manager._connection_router.watcher_get_events_messages( + do_manager.email, None + ) + assert sorted(len(m.events) for m in messages) == [1, 2] def test_outdated_proposal_does_not_block_owner_changes(): From e69dea1cedd3a2234398c749df6dc1b23c8bfa1e Mon Sep 17 00:00:00 2001 From: Pedro Werneck Date: Thu, 22 Oct 2026 17:08:21 -0300 Subject: [PATCH 07/36] fix: seed every approved peer once and drop unsent plan messages The seed now runs once, by a flag in the state, so an early send does not skip it. It covers every approved peer, also one that the sync skips. A path that the earlier client still had to send gets a marker hash, so the next plan sends the file or its delete. A send that raises no longer leaves the rest of the plan queued for a later send that checks neither the views nor the permissions. Outbox compaction keeps the message size cap. --- syft/sync/sync/caches/datasite_owner_cache.py | 20 +- syft/sync/sync/datasite_owner_syncer.py | 104 ++++++---- syft/sync/sync/delivery_tracker.py | 54 ++--- tests/unit/test_delivery_tracker.py | 196 ++++++++++++------ 4 files changed, 240 insertions(+), 134 deletions(-) diff --git a/syft/sync/sync/caches/datasite_owner_cache.py b/syft/sync/sync/caches/datasite_owner_cache.py index 92e8e4e8395..198f0a6d486 100644 --- a/syft/sync/sync/caches/datasite_owner_cache.py +++ b/syft/sync/sync/caches/datasite_owner_cache.py @@ -323,15 +323,19 @@ def clear_cache(self): def legacy_pending_paths(self) -> dict[str, set[str]]: """The paths that a client before the delivery tracker still had to send, - per peer. Empty for an in-memory cache.""" - if self.syftbox_folder is None: + per peer. + + Empty when the views are not kept on disk, or when that client left no + file. + """ + store = self.deliveries.store + if not isinstance(store, LocalDeliveryStateStore): return {} - pending = PersistedDict( - path=Path(self.syftbox_folder) - / CACHE_DIR - / OWNER_PENDING_DELIVERIES_FILENAME - ) - return {peer: set(paths) for peer, paths in pending.items()} + # The earlier client kept its file next to the views file. + path = store.path.parent / OWNER_PENDING_DELIVERIES_FILENAME + if not path.exists(): + return {} + return {peer: set(paths) for peer, paths in PersistedDict(path=path).items()} def has_conflict(self, proposed_event: ProposedFileChange) -> bool: if proposed_event.path_in_datasite not in self.file_hashes: diff --git a/syft/sync/sync/datasite_owner_syncer.py b/syft/sync/sync/datasite_owner_syncer.py index d1d0d88f7d4..2b6b1124b91 100644 --- a/syft/sync/sync/datasite_owner_syncer.py +++ b/syft/sync/sync/datasite_owner_syncer.py @@ -7,7 +7,7 @@ from pydantic import ConfigDict, Field, BaseModel, PrivateAttr from concurrent.futures import ThreadPoolExecutor from queue import Queue -from typing import List, NamedTuple, Tuple +from typing import List, Mapping, NamedTuple, Tuple from syft.sync.events.file_change_event import ( FileChangeEventsMessage, FileChangeEventsMessageFileName, @@ -43,6 +43,7 @@ ) from syft_perms import SyftPermContext from syft.sync.sync.constants import CACHE_DIR, ROLLING_STATE_FILENAME +from syft.sync.sync.delivery_tracker import UNSENT_HASH from syft.sync.transport.types import SendStatus from syft.sync.utils.syftbox_utils import ( create_event_timestamp, @@ -165,6 +166,8 @@ class DatasiteOwnerSyncer(BaseModelCallbackMixin): ) # Cache of collections shared with "any" - list of (tag, content_hash) tuples _any_shared_collections: List[tuple] = PrivateAttr(default_factory=list) + # Names of the outbox messages that `deliver` queued and that are not sent yet + _planned_message_names: set[str] = PrivateAttr(default_factory=set) # In-memory rolling state for tracking events since last checkpoint _rolling_state: RollingState | None = PrivateAttr(default=None) @@ -237,7 +240,7 @@ def sync(self, peer_emails: list[str], recompute_hashes: bool = True): self.pull_initial_state() # Before the local changes are recorded, so that a file changed # while the owner was offline is not seeded as held. - self._seed_views_on_upgrade(peer_emails) + self._seed_views_if_missing(peer_emails) if recompute_hashes: self._record_local_changes() @@ -661,29 +664,37 @@ def _readable_hashes(self, peer_emails: list[str]) -> dict[str, dict[str, str]]: The caller reloads the permissions first. """ - hashes = self._syncable_hashes() + syncable = self._syncable_hashes() return { peer: { - p: h for p, h in hashes.items() if self.perm_context.can_read(p, peer) + p: h for p, h in syncable.items() if self._may_hold(peer, p, syncable) } for peer in peer_emails } - def _seed_views_on_upgrade(self, peer_emails: list[str]) -> None: - """Start the views of an owner that kept none before. + def _may_hold(self, peer: str, path: str, syncable: Mapping[str, str]) -> bool: + """Whether a view of ``peer`` may hold ``path``: a file in ``syncable`` + that ``peer`` may read.""" + return path in syncable and self.perm_context.can_read(path, peer) - Each peer is taken to hold every file it may read now, except the paths - that the earlier client still had to send it. + def _seed_views_if_missing(self, peer_emails: list[str]) -> None: + """Seed the views one time, for an owner whose state was never seeded. + + Each approved peer, and each peer in ``peer_emails``, is taken to hold + every file it may read now. A path that an earlier client still had to + send gets ``UNSENT_HASH``, so the next plan sends or deletes it. """ deliveries = self.event_cache.deliveries - if deliveries.has_state(): + if deliveries.is_seeded(): return self.perm_context._reload() + approved = [p.email for p in self.connection_router.peer_store.approved_peers] pending = self.event_cache.legacy_pending_paths() + readable = self._readable_hashes(sorted({*approved, *peer_emails})) deliveries.seed( { - peer: {p: h for p, h in files.items() if p not in pending.get(peer, ())} - for peer, files in self._readable_hashes(peer_emails).items() + peer: {**files, **dict.fromkeys(pending.get(peer, ()), UNSENT_HASH)} + for peer, files in readable.items() } ) @@ -698,7 +709,9 @@ def deliver(self, peer_emails: list[str]) -> None: events = [self._create_current_state_event(p) for p in plan.send] events += [self._deletion_event(p) for p in plan.delete] for batch in _batches_by_content_size(events, MAX_DELIVERY_MESSAGE_CONTENT): - self.outbox_queue.put((peer, FileChangeEventsMessage(events=batch))) + message = FileChangeEventsMessage(events=batch) + self._planned_message_names.add(message.message_filepath.as_string()) + self.outbox_queue.put((peer, message)) self.process_syftbox_events_queue() def pull_and_process_next_proposed_filechange( @@ -794,21 +807,42 @@ def process_syftbox_events_queue(self): return self.perm_context._reload() waiting_for = set() + try: + while not self.outbox_queue.empty(): + recipient, message = self.outbox_queue.get() + if recipient in waiting_for: + continue + try: + status = ( + self.connection_router.owner_write_event_messages_to_outbox( + recipient, message + ) + ) + except (PeerEncryptionPendingError, EncryptionPolicyConflictError) as e: + if self._warn_and_should_hold(recipient, e): + waiting_for.add(recipient) + else: + self._record_sent(recipient, message, status) + finally: + self._drop_planned_messages() + + def _drop_planned_messages(self) -> None: + """Remove from the outbox queue the messages that ``deliver`` queued. + + A send that raised leaves later messages in the queue. A later send of + them would not check the views or the permissions again. ``deliver`` + plans them again from the views instead. + """ + kept = [] while not self.outbox_queue.empty(): recipient, message = self.outbox_queue.get() - if recipient in waiting_for: - continue - try: - status = self.connection_router.owner_write_event_messages_to_outbox( - recipient, message - ) - except (PeerEncryptionPendingError, EncryptionPolicyConflictError) as e: - if self._hold_or_drop(recipient, e): - waiting_for.add(recipient) - else: - self._record_sent(recipient, message, status) + if message.message_filepath.as_string() not in self._planned_message_names: + kept.append((recipient, message)) + for item in kept: + self.outbox_queue.put(item) + self._planned_message_names.clear() - def _hold_or_drop(self, recipient: str, error: Exception) -> bool: + def _warn_and_should_hold(self, recipient: str, error: Exception) -> bool: """Warn about an outbox message that cannot go to ``recipient`` now. Returns True when the message waits for a later sync, and False when it @@ -830,15 +864,13 @@ def _record_sent( for it. Other files, such as a private dataset sent to an enclave, are not in the view. """ - collections_rel = self.event_cache.collections_relative_path() + syncable = self._syncable_hashes() changes = {} for event in message.events: path = str(event.path_in_datasite) if event.is_deleted: changes[path] = None - elif is_normal_syncable_path( - path, collections_path=collections_rel - ) and self.perm_context.can_read(path, recipient): + elif self._may_hold(recipient, path, syncable): changes[path] = event.new_hash self.event_cache.deliveries.sent( recipient, message.message_filepath.as_string(), status, changes @@ -916,14 +948,16 @@ def compact_outbox_if_needed( for msg in messages: merged_events.extend(msg.events) - merged = FileChangeEventsMessage(events=merged_events) - - # Write the merged message FIRST. A DS that pulls between this and + # Write the merged messages FIRST. A DS that pulls between this and # the delete sees `merged ∪ some originals`; its file_hashes cache - # makes duplicate events no-op writes. - self.connection_router.owner_write_event_messages_to_outbox( - recipient_email, merged - ) + # makes duplicate events no-op writes. The size cap of a delivery + # message holds here too. + for batch in _batches_by_content_size( + merged_events, MAX_DELIVERY_MESSAGE_CONTENT + ): + self.connection_router.owner_write_event_messages_to_outbox( + recipient_email, FileChangeEventsMessage(events=batch) + ) conn.delete_multiple_files_by_ids(file_ids) return len(parsed) diff --git a/syft/sync/sync/delivery_tracker.py b/syft/sync/sync/delivery_tracker.py index 04de66f6e6e..7bc11a0eb21 100644 --- a/syft/sync/sync/delivery_tracker.py +++ b/syft/sync/sync/delivery_tracker.py @@ -21,11 +21,12 @@ peer that the owner rejects and then approves again keeps its view, and gets only what changed. -An owner that kept no views before, because it ran an earlier client or lost -its views file, has file hashes but no state. ``seed`` then starts each current -peer's view with every file it may read now, so the upgrade sends nothing -extra. The paths that the earlier client still had to send are left out, so -they are sent. +An owner whose state was never seeded, because it ran an earlier client or +lost its views file, has file hashes but no views. ``seed`` then starts the +view of each approved peer with every file it may read now, so the upgrade +sends nothing extra. A path that the earlier client still had to send gets +``UNSENT_HASH``, so the next plan sends it, or deletes it when it is gone. A +view that a send already recorded is kept. Recording a send ---------------- @@ -65,6 +66,9 @@ # Format of DeliveryState. Raise it when the layout changes, and keep a read # path for every earlier version. DELIVERY_STATE_VERSION = 1 +# A view hash that matches no file, for a path that the peer may hold in any +# version. The next plan sends the path, or deletes it. +UNSENT_HASH = "" class QueuedSend(BaseModel): @@ -85,6 +89,8 @@ class DeliveryState(BaseModel): views: dict[str, dict[str, str]] = Field(default_factory=dict) # msg_id -> the send it confirms queued: dict[str, QueuedSend] = Field(default_factory=dict) + # Whether ``DeliveryTracker.seed`` ran on this state. + seeded: bool = False class DeliveryPlan(BaseModel): @@ -111,10 +117,6 @@ def load(self) -> DeliveryState: def save(self, state: DeliveryState) -> None: """Store ``state``, replacing what was there.""" - @abstractmethod - def has_state(self) -> bool: - """Whether a state was ever saved.""" - def locked(self) -> ContextManager[None]: """Keep other writers out from a ``load`` to the ``save`` that follows.""" return nullcontext() @@ -124,17 +126,14 @@ class MemoryDeliveryStateStore(DeliveryStateStore): """A state kept in this process only.""" def __init__(self): - self._state: DeliveryState | None = None + self._state = DeliveryState() def load(self) -> DeliveryState: - return (self._state or DeliveryState()).model_copy(deep=True) + return self._state.model_copy(deep=True) def save(self, state: DeliveryState) -> None: self._state = state.model_copy(deep=True) - def has_state(self) -> bool: - return self._state is not None - class LocalDeliveryStateStore(DeliveryStateStore): """A state kept in a JSON file, shared by the processes of one datasite.""" @@ -153,9 +152,6 @@ def load(self) -> DeliveryState: ) return state - def has_state(self) -> bool: - return self.path.exists() - @contextmanager def locked(self): self.path.parent.mkdir(parents=True, exist_ok=True) @@ -200,18 +196,24 @@ def clear(self) -> None: with self.store.locked(): self.store.save(DeliveryState()) - def has_state(self) -> bool: - """Whether the store holds a state. False until the first ``seed``.""" - return self.store.has_state() + def is_seeded(self) -> bool: + """Whether ``seed`` ran on the stored state.""" + return self.store.load().seeded - def seed(self, readable_by_peer: Mapping[str, Mapping[str, str]]) -> None: - """Start each view with the files in ``readable_by_peer``, if the store - holds no state yet. Otherwise do nothing.""" + def seed(self, held_by_peer: Mapping[str, Mapping[str, str]]) -> None: + """Add the files in ``held_by_peer`` to each peer's view, one time. + + A path that the view already holds keeps its hash. After the first + call, later calls do nothing. + """ with self.store.locked(): - if self.store.has_state(): + state = self.store.load() + if state.seeded: return - views = {peer: dict(files) for peer, files in readable_by_peer.items()} - self.store.save(DeliveryState(views=views)) + for peer, files in held_by_peer.items(): + state.views[peer] = {**files, **state.views.get(peer, {})} + state.seeded = True + self.store.save(state) def plan( self, readable_by_peer: Mapping[str, Mapping[str, str]] diff --git a/tests/unit/test_delivery_tracker.py b/tests/unit/test_delivery_tracker.py index b910222354e..32759ccbf9c 100644 --- a/tests/unit/test_delivery_tracker.py +++ b/tests/unit/test_delivery_tracker.py @@ -40,11 +40,6 @@ def test_plan_sends_changed_files_deletes_unreadable(): assert plan.delete == ["revoked"] -def test_peer_without_view_gets_all_readable_files(): - tracker = DeliveryTracker(MemoryDeliveryStateStore()) - assert tracker.plan({PEER: {"a": "h1", "b": "h2"}})[PEER].send == ["a", "b"] - - def test_delivered_send_updates_view(): tracker = _tracker_holding({"a": "h1", "b": "h1"}) tracker.sent(PEER, "m1", SendStatus.DELIVERED, {"a": "h2", "b": None}) @@ -52,11 +47,11 @@ def test_delivered_send_updates_view(): assert tracker.plan({PEER: {"a": "h2"}})[PEER].delete == [] -def test_seed_starts_views_only_without_state(): - tracker = DeliveryTracker(MemoryDeliveryStateStore()) - tracker.seed({PEER: {"a": "h1"}}) +def test_seed_runs_once_and_keeps_recorded_views(): + tracker = _tracker_holding({"b": "h9"}) + tracker.seed({PEER: {"a": "h1", "b": "h1"}}) tracker.seed({PEER: {}}) - assert tracker.plan({PEER: {"a": "h1"}})[PEER].send == [] + assert tracker.plan({PEER: {"a": "h1", "b": "h9"}})[PEER].send == [] def test_queued_send_updates_view_on_confirm(): @@ -147,6 +142,42 @@ def _outbox_paths(ds_manager, do_manager) -> list[str]: return [str(e.path_in_datasite) for m in messages for e in m.events] +def _sent_events(ds_manager, do_manager, path: str) -> list[tuple[bool, object]]: + """(is_deleted, content) of each event for ``path`` in the DS's outbox.""" + messages = ds_manager._connection_router.watcher_get_events_messages( + do_manager.email, None + ) + return [ + (e.is_deleted, e.content) + for m in messages + for e in m.events + if str(e.path_in_datasite) == path + ] + + +def _simulate_earlier_client(do_manager, pending: dict[str, list[str]]) -> None: + """Make the DO cache look like one from a client before the delivery + tracker: no views, and ``pending`` paths it still had to send.""" + cache_dir = do_manager.syftbox_folder / CACHE_DIR + (cache_dir / DELIVERY_VIEWS_FILENAME).unlink() + legacy = PersistedDict(path=cache_dir / OWNER_PENDING_DELIVERIES_FILENAME) + for peer, paths in pending.items(): + legacy[peer] = dict.fromkeys(paths, "msg") + + +def _push_private_file(do_manager, ds_manager) -> None: + """Send a file outside the plan, as a private dataset share does.""" + syncer = do_manager.datasite_owner_syncer + _write(do_manager, "private/data.csv", "rows") + message = syncer.event_cache.create_events_for_files( + {Path("private/data.csv"): b"rows"} + ) + syncer.queue_event_for_syftbox( + recipients=[ds_manager.email], file_change_events_message=message + ) + syncer.process_syftbox_events_queue() + + def test_peer_approved_later_gets_existing_files(): ds_manager, do_manager = _pair(add_peers=False) do_manager.datasite_owner_syncer.perm_context.open("shared/").grant_read_access( @@ -167,44 +198,29 @@ def test_peer_approved_later_gets_existing_files(): ) -def test_in_memory_owner_sends_content_then_delete_on_revoke(): - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection() +@pytest.mark.parametrize( + "use_in_memory_cache, persist_owner_state", + [(False, True), (False, False), (True, False)], +) +def test_revoked_peer_gets_content_then_delete( + use_in_memory_cache, persist_owner_state +): + ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + use_in_memory_cache=use_in_memory_cache, persist_owner_state=persist_owner_state + ) syncer = do_manager.datasite_owner_syncer folder = syncer.perm_context.open("shared/") folder.grant_read_access(ds_manager.email) - syncer.event_cache.file_connection.write_file("shared/data.txt", "hello") + syncer.event_cache.file_connection.write_file("shared/data.txt", "secret") do_manager.sync(auto_checkpoint=False) do_manager.sync(auto_checkpoint=False) folder.revoke_read_access(ds_manager.email) do_manager.sync(auto_checkpoint=False) - messages = ds_manager._connection_router.watcher_get_events_messages( - do_manager.email, None - ) - sent = [ - (e.is_deleted, e.content) - for m in messages - for e in m.events - if str(e.path_in_datasite) == "shared/data.txt" + assert _sent_events(ds_manager, do_manager, "shared/data.txt") == [ + (False, "secret"), + (True, None), ] - assert sent == [(False, "hello"), (True, None)] - - -@pytest.mark.parametrize("persist_owner_state", [True, False]) -def test_revoked_peer_gets_delete(persist_owner_state): - ds_manager, do_manager = _pair(persist_owner_state=persist_owner_state) - folder = do_manager.datasite_owner_syncer.perm_context.open("shared/") - folder.grant_read_access(ds_manager.email) - _write(do_manager, "shared/data.txt", "secret") - do_manager.sync(auto_checkpoint=False) - ds_manager.sync() - received = _received(ds_manager, do_manager, "shared/data.txt") - assert received.exists() - - folder.revoke_read_access(ds_manager.email) - do_manager.sync(auto_checkpoint=False) - ds_manager.sync() - assert not received.exists(), "The DS kept a file it may no longer read." def test_restarted_owner_sends_nothing_new(): @@ -229,12 +245,7 @@ def test_upgrade_sends_only_unsent_changes(): _write(do_manager, "shared/pending.txt", "one") do_manager.sync(auto_checkpoint=False) - # Make the cache look like one from a client before the delivery tracker: - # no views, and one path that the client still had to send. - cache_dir = do_manager.syftbox_folder / CACHE_DIR - (cache_dir / DELIVERY_VIEWS_FILENAME).unlink() - pending = PersistedDict(path=cache_dir / OWNER_PENDING_DELIVERIES_FILENAME) - pending[ds_manager.email] = {"shared/pending.txt": "msg"} + _simulate_earlier_client(do_manager, {ds_manager.email: ["shared/pending.txt"]}) _write(do_manager, "shared/offline.txt", "new") do_manager.sync(auto_checkpoint=False) @@ -244,27 +255,61 @@ def test_upgrade_sends_only_unsent_changes(): assert sent.count("shared/offline.txt") == 1 -def test_private_dataset_sent_to_peer_never_revoked(): +def test_upgrade_sends_pending_delete(): ds_manager, do_manager = _pair() - syncer = do_manager.datasite_owner_syncer - _write(do_manager, "private/data.csv", "rows") - message = syncer.event_cache.create_events_for_files( - {Path("private/data.csv"): b"rows"} - ) - syncer.queue_event_for_syftbox( - recipients=[ds_manager.email], file_change_events_message=message + do_manager.datasite_owner_syncer.perm_context.open("shared/").grant_read_access( + ds_manager.email ) - syncer.process_syftbox_events_queue() + _write(do_manager, "shared/gone.txt", "one") + do_manager.sync(auto_checkpoint=False) + + # The earlier client recorded the delete, but did not send it. + (do_manager.syftbox_folder / do_manager.email / "shared/gone.txt").unlink() + do_manager.datasite_owner_syncer.event_cache.process_local_file_changes() + _simulate_earlier_client(do_manager, {ds_manager.email: ["shared/gone.txt"]}) do_manager.sync(auto_checkpoint=False) + assert _sent_events(ds_manager, do_manager, "shared/gone.txt")[-1] == (True, None) + + +def test_upgrade_seeds_after_send_outside_plan(): + ds_manager, do_manager = _pair() + do_manager.datasite_owner_syncer.perm_context.open("shared/").grant_read_access( + ds_manager.email + ) + _write(do_manager, "shared/held.txt", "one") do_manager.sync(auto_checkpoint=False) - messages = ds_manager._connection_router.watcher_get_events_messages( - do_manager.email, None + _simulate_earlier_client(do_manager, {}) + + _push_private_file(do_manager, ds_manager) + do_manager.sync(auto_checkpoint=False) + assert _outbox_paths(ds_manager, do_manager).count("shared/held.txt") == 1 + + +def test_upgrade_seeds_peer_skipped_in_first_sync(): + ds_manager, do_manager = _pair() + do_manager.datasite_owner_syncer.perm_context.open("shared/").grant_read_access( + ds_manager.email ) - events = [ - (str(e.path_in_datasite), e.is_deleted) for m in messages for e in m.events + _write(do_manager, "shared/held.txt", "one") + do_manager.sync(auto_checkpoint=False) + _simulate_earlier_client(do_manager, {}) + + # The version gate skips the DS in the first sync after the upgrade. + do_manager.datasite_owner_syncer.sync([]) + do_manager.sync(auto_checkpoint=False) + assert _outbox_paths(ds_manager, do_manager).count("shared/held.txt") == 1 + + +def test_private_dataset_sent_to_peer_never_revoked(): + ds_manager, do_manager = _pair() + _push_private_file(do_manager, ds_manager) + + do_manager.sync(auto_checkpoint=False) + do_manager.sync(auto_checkpoint=False) + assert _sent_events(ds_manager, do_manager, "private/data.csv") == [ + (False, b"rows") ] - assert events == [("private/data.csv", False)] def test_large_catch_up_split_into_messages(monkeypatch): @@ -303,12 +348,13 @@ def test_outdated_proposal_does_not_block_owner_changes(): assert _received(ds_manager, do_manager, "shared/b.txt").read_text() == "new" -def test_failed_outbox_write_resent_next_sync(monkeypatch): +def test_failed_send_drops_rest_of_plan(monkeypatch): + monkeypatch.setattr(datasite_owner_syncer, "MAX_DELIVERY_MESSAGE_CONTENT", 3) ds_manager, do_manager = _pair() - do_manager.datasite_owner_syncer.perm_context.open("shared/").grant_read_access( - ds_manager.email - ) - _write(do_manager, "shared/data.txt", "one") + folder = do_manager.datasite_owner_syncer.perm_context.open("shared/") + folder.grant_read_access(ds_manager.email) + _write(do_manager, "shared/a.txt", "abc") + _write(do_manager, "shared/b.txt", "abc") def fail_upload(*args, **kwargs): raise ConnectionError("upload failed") @@ -318,6 +364,26 @@ def fail_upload(*args, **kwargs): with pytest.raises(ConnectionError): do_manager.sync(auto_checkpoint=False) + # The batch after the failed one must not go out after the revoke. + folder.revoke_read_access(ds_manager.email) do_manager.sync(auto_checkpoint=False) - ds_manager.sync() - assert _received(ds_manager, do_manager, "shared/data.txt").read_text() == "one" + assert _outbox_paths(ds_manager, do_manager) == [] + + +def test_compaction_keeps_message_size_cap(monkeypatch): + monkeypatch.setattr(datasite_owner_syncer, "MAX_DELIVERY_MESSAGE_CONTENT", 3) + ds_manager, do_manager = _pair() + do_manager.datasite_owner_syncer.perm_context.open("shared/").grant_read_access( + ds_manager.email + ) + for name in ("a", "b", "c", "d"): + _write(do_manager, f"shared/{name}.txt", "abc") + do_manager.sync(auto_checkpoint=False) + + monkeypatch.setattr(datasite_owner_syncer, "MAX_DELIVERY_MESSAGE_CONTENT", 6) + syncer = do_manager.datasite_owner_syncer + assert syncer.compact_outbox_if_needed(ds_manager.email, min_messages=2) == 4 + messages = ds_manager._connection_router.watcher_get_events_messages( + do_manager.email, None + ) + assert [len(m.events) for m in messages] == [2, 2] From 9022f9b1f1c07fa9fd4ce574b2775431125133bd Mon Sep 17 00:00:00 2001 From: Pedro Werneck Date: Thu, 22 Oct 2026 18:36:50 -0300 Subject: [PATCH 08/36] refactor: move the outbox send loop into its own method _send_outbox_messages() sends the queued outbox messages, so process_syftbox_events_queue() stays short. Rename _apply() to _apply_changes_to_view(). --- syft/sync/sync/datasite_owner_syncer.py | 15 ++++++++++----- syft/sync/sync/delivery_tracker.py | 10 +++++++--- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/syft/sync/sync/datasite_owner_syncer.py b/syft/sync/sync/datasite_owner_syncer.py index 2b6b1124b91..ff0fc0dcf99 100644 --- a/syft/sync/sync/datasite_owner_syncer.py +++ b/syft/sync/sync/datasite_owner_syncer.py @@ -800,11 +800,16 @@ def process_syftbox_events_queue(self): self.connection_router.owner_write_events_message_to_syftbox( file_change_events_message ) - # A message that is not sent leaves the view as it is, so the plan of a - # later sync holds its changes again. Later messages to the same - # recipient wait too, so the order stays the same. - if self.outbox_queue.empty(): - return + if not self.outbox_queue.empty(): + self._send_outbox_messages() + + def _send_outbox_messages(self) -> None: + """Send each message in the outbox queue, and record it in its peer's view. + + A message that is not sent leaves the view as it is, so the plan of a + later sync holds its changes again. Later messages to the same + recipient wait too, so the order stays the same. + """ self.perm_context._reload() waiting_for = set() try: diff --git a/syft/sync/sync/delivery_tracker.py b/syft/sync/sync/delivery_tracker.py index 7bc11a0eb21..cda3e4c9dab 100644 --- a/syft/sync/sync/delivery_tracker.py +++ b/syft/sync/sync/delivery_tracker.py @@ -172,7 +172,9 @@ def save(self, state: DeliveryState) -> None: tmp.unlink(missing_ok=True) -def _apply(view: dict[str, str], changes: Mapping[str, str | None]) -> None: +def _apply_changes_to_view( + view: dict[str, str], changes: Mapping[str, str | None] +) -> None: for path, new_hash in changes.items(): if new_hash is None: view.pop(path, None) @@ -247,7 +249,7 @@ def sent( with self.store.locked(): state = self.store.load() if status is SendStatus.DELIVERED: - _apply(state.views.setdefault(peer, {}), changes) + _apply_changes_to_view(state.views.setdefault(peer, {}), changes) else: assert status is SendStatus.QUEUED, status state.queued[msg_id] = QueuedSend(peer=peer, changes=dict(changes)) @@ -262,5 +264,7 @@ def confirm(self, msg_ids: Iterable[str]) -> None: state = self.store.load() for msg_id in msg_ids: queued = state.queued.pop(msg_id) - _apply(state.views.setdefault(queued.peer, {}), queued.changes) + _apply_changes_to_view( + state.views.setdefault(queued.peer, {}), queued.changes + ) self.store.save(state) From ea2452826d24545f2a45b4cbf8a0327afe80cb52 Mon Sep 17 00:00:00 2001 From: Pedro Werneck Date: Thu, 22 Oct 2026 19:09:45 -0300 Subject: [PATCH 09/36] fix: keep pending private shares and held pushes The seed marks only syncable pending paths as unsent. A pending private dataset share stays out of the view, so the next plan does not delete it. A held outbox message that deliver() did not queue goes back on the queue, ahead of the messages not reached yet. A later flush sends it, as the warning says. --- syft/sync/sync/datasite_owner_syncer.py | 48 ++++++++++++++++--------- tests/unit/test_delivery_tracker.py | 32 +++++++++++++++++ 2 files changed, 64 insertions(+), 16 deletions(-) diff --git a/syft/sync/sync/datasite_owner_syncer.py b/syft/sync/sync/datasite_owner_syncer.py index ff0fc0dcf99..80f2bbe8658 100644 --- a/syft/sync/sync/datasite_owner_syncer.py +++ b/syft/sync/sync/datasite_owner_syncer.py @@ -650,13 +650,17 @@ def _record_local_changes(self) -> None: self._add_events_to_rolling_state(file_change_events_message) self.process_syftbox_events_queue() + def _is_syncable(self, path: str) -> bool: + """Whether a peer can get ``path`` by a delivery plan.""" + collections_rel = self.event_cache.collections_relative_path() + return is_normal_syncable_path(path, collections_path=collections_rel) + def _syncable_hashes(self) -> dict[str, str]: """The current hash of every file that a peer can get, by path.""" - collections_rel = self.event_cache.collections_relative_path() return { str(path): file_hash for path, file_hash in self.event_cache.file_hashes.items() - if is_normal_syncable_path(str(path), collections_path=collections_rel) + if self._is_syncable(str(path)) } def _readable_hashes(self, peer_emails: list[str]) -> dict[str, dict[str, str]]: @@ -681,8 +685,10 @@ def _seed_views_if_missing(self, peer_emails: list[str]) -> None: """Seed the views one time, for an owner whose state was never seeded. Each approved peer, and each peer in ``peer_emails``, is taken to hold - every file it may read now. A path that an earlier client still had to - send gets ``UNSENT_HASH``, so the next plan sends or deletes it. + every file it may read now. A syncable path that an earlier client + still had to send gets ``UNSENT_HASH``, so the next plan sends or + deletes it. Other pending paths, such as a private dataset share, stay + out of the view: a plan never sends them, so it must not delete them. """ deliveries = self.event_cache.deliveries if deliveries.is_seeded(): @@ -693,7 +699,14 @@ def _seed_views_if_missing(self, peer_emails: list[str]) -> None: readable = self._readable_hashes(sorted({*approved, *peer_emails})) deliveries.seed( { - peer: {**files, **dict.fromkeys(pending.get(peer, ()), UNSENT_HASH)} + peer: { + **files, + **{ + p: UNSENT_HASH + for p in pending.get(peer, ()) + if self._is_syncable(p) + }, + } for peer, files in readable.items() } ) @@ -812,10 +825,12 @@ def _send_outbox_messages(self) -> None: """ self.perm_context._reload() waiting_for = set() + held = [] try: while not self.outbox_queue.empty(): recipient, message = self.outbox_queue.get() if recipient in waiting_for: + held.append((recipient, message)) continue try: status = ( @@ -826,25 +841,26 @@ def _send_outbox_messages(self) -> None: except (PeerEncryptionPendingError, EncryptionPolicyConflictError) as e: if self._warn_and_should_hold(recipient, e): waiting_for.add(recipient) + held.append((recipient, message)) else: self._record_sent(recipient, message, status) finally: - self._drop_planned_messages() + self._requeue_unplanned_messages(held) - def _drop_planned_messages(self) -> None: - """Remove from the outbox queue the messages that ``deliver`` queued. + def _requeue_unplanned_messages(self, held: list) -> None: + """Put back the unsent messages that ``deliver`` did not queue. - A send that raised leaves later messages in the queue. A later send of - them would not check the views or the permissions again. ``deliver`` - plans them again from the views instead. + The ``held`` messages go first, so each peer keeps its order. A + message that ``deliver`` queued is dropped: a later send would not + check the views or the permissions again, so ``deliver`` plans it + again from the views instead. """ - kept = [] + unsent = [*held] while not self.outbox_queue.empty(): - recipient, message = self.outbox_queue.get() + unsent.append(self.outbox_queue.get()) + for recipient, message in unsent: if message.message_filepath.as_string() not in self._planned_message_names: - kept.append((recipient, message)) - for item in kept: - self.outbox_queue.put(item) + self.outbox_queue.put((recipient, message)) self._planned_message_names.clear() def _warn_and_should_hold(self, recipient: str, error: Exception) -> bool: diff --git a/tests/unit/test_delivery_tracker.py b/tests/unit/test_delivery_tracker.py index 32759ccbf9c..49b88a84efc 100644 --- a/tests/unit/test_delivery_tracker.py +++ b/tests/unit/test_delivery_tracker.py @@ -7,6 +7,7 @@ import pytest from syft.sync.connections.connection_router import ConnectionRouter +from syft.sync.encryption_policy import PeerEncryptionPendingError from syft.sync.sync import datasite_owner_syncer from syft.sync.sync.caches.persisted_dict import PersistedDict from syft.sync.sync.constants import CACHE_DIR, OWNER_PENDING_DELIVERIES_FILENAME @@ -301,6 +302,37 @@ def test_upgrade_seeds_peer_skipped_in_first_sync(): assert _outbox_paths(ds_manager, do_manager).count("shared/held.txt") == 1 +def test_upgrade_keeps_pending_private_share(): + ds_manager, do_manager = _pair() + _push_private_file(do_manager, ds_manager) + do_manager.sync(auto_checkpoint=False) + _simulate_earlier_client(do_manager, {ds_manager.email: ["private/data.csv"]}) + + do_manager.sync(auto_checkpoint=False) + assert _sent_events(ds_manager, do_manager, "private/data.csv") == [ + (False, b"rows") + ] + + +def test_held_send_outside_plan_goes_out_later(monkeypatch): + ds_manager, do_manager = _pair() + + def encryption_pending(*args, **kwargs): + raise PeerEncryptionPendingError("keys not known yet") + + with monkeypatch.context() as m: + m.setattr( + ConnectionRouter, "owner_write_event_messages_to_outbox", encryption_pending + ) + with pytest.warns(UserWarning, match="wait for a later sync"): + _push_private_file(do_manager, ds_manager) + + do_manager.datasite_owner_syncer.process_syftbox_events_queue() + assert _sent_events(ds_manager, do_manager, "private/data.csv") == [ + (False, b"rows") + ] + + def test_private_dataset_sent_to_peer_never_revoked(): ds_manager, do_manager = _pair() _push_private_file(do_manager, ds_manager) From 2cc75d5534dbab21e2fdf82329d75ad743238cc3 Mon Sep 17 00:00:00 2001 From: Pedro Werneck Date: Thu, 22 Oct 2026 19:31:04 -0300 Subject: [PATCH 10/36] fix: queue a delivery plan only after all its messages are built A read error in deliver() left the messages of earlier peers in the queue, and a later flush sent them without a check of the views. One locked_file() and one write_text_atomic() in persisted_dict.py now serve PersistedDict, the views store and the key file. The docs no longer say that every owner send survives a restart. --- docs/connections.md | 2 +- syft/sync/peers/peer_store.py | 29 ++------ syft/sync/sync/caches/datasite_owner_cache.py | 7 +- syft/sync/sync/caches/persisted_dict.py | 55 +++++++++------ syft/sync/sync/datasite_owner_syncer.py | 69 +++++++++++-------- syft/sync/sync/delivery_tracker.py | 27 ++------ tests/unit/test_delivery_tracker.py | 29 ++++++++ 7 files changed, 119 insertions(+), 99 deletions(-) diff --git a/docs/connections.md b/docs/connections.md index 0e18fc66beb..6742b5ddeb3 100644 --- a/docs/connections.md +++ b/docs/connections.md @@ -62,7 +62,7 @@ A pair encrypts when one side is `required`. An `off` and a `required` client co A client takes its policy at login. The key file records a `required` policy, so later logins (and syft-bg) stay `required`. A process that started with another policy stops at its next sync with an error to restart it. Files at rest written before the switch stay plaintext; new ones are encrypted. -While a peer's policy or keys are not known yet, a change for it waits, with a warning, and a later sync sends it. An owner's change is not lost: the owner keeps a record of what each peer holds, and that record changes only after a send. A scientist's change waits in memory, and is lost if the process exits. +While a peer's policy or keys are not known yet, a change for it waits, with a warning, and a later sync sends it. An owner's change to a file that the peer may read is not lost: the owner keeps a record of what each peer holds, and that record changes only after a send. Other owner sends, such as a private dataset share or an enclave result, wait in memory, and are lost if the process exits. A scientist's change waits in memory, and is lost if the process exits. A pair that encrypts refuses plaintext. The receiver drops a plaintext message with a warning: the owner archives it unread, and a scientist skips it. diff --git a/syft/sync/peers/peer_store.py b/syft/sync/peers/peer_store.py index 582f39583dd..6a0d5193884 100644 --- a/syft/sync/peers/peer_store.py +++ b/syft/sync/peers/peer_store.py @@ -9,11 +9,9 @@ import os import tempfile import warnings -from contextlib import contextmanager from pathlib import Path -from typing import Iterator, List, Optional +from typing import List, Optional -import portalocker import syft_crypto_python as syc from pydantic import BaseModel, PrivateAttr @@ -37,6 +35,7 @@ parse_and_validate_bundle, ) from syft.sync.peers.peer import Peer, PeerState +from syft.sync.sync.caches.persisted_dict import locked_file logger = logging.getLogger(__name__) @@ -447,7 +446,7 @@ def ensure_keys(self) -> None: self.generate_keys() return unsaved_pins: list[tuple[str, dict]] = [] - with _locked_key_file(self._keys_path): + with locked_file(self._keys_path): if self._keys_path.exists(): unsaved_pins = self._take_key_file(self.load_keys(self._keys_path)) else: @@ -661,7 +660,7 @@ def _record_required_policy(self) -> None: """Record a required policy in the key file, keeping what else it holds.""" if self._keys_path is None or not self.use_encryption: return - with _locked_key_file(self._keys_path): + with locked_file(self._keys_path): data = self._read_key_file() or self._key_file_data() if data.get(ENCRYPTION_POLICY_KEY) == EncryptionPolicy.REQUIRED.value: return @@ -672,7 +671,7 @@ def _record_required_policy(self) -> None: def save_keys(self, path: Path) -> None: data = self._key_file_data() path = Path(path) - with _locked_key_file(path): + with locked_file(path): _write_key_file(path, data) self._keys_path = path @@ -717,7 +716,7 @@ def _persist_peer_bundle(self, peer_email: str, bundle: dict) -> None: """ if self._keys_path is None or self._private_keys is None: return - with _locked_key_file(self._keys_path): + with locked_file(self._keys_path): data = self._read_key_file() or self._key_file_data() _write_key_file(self._keys_path, _with_pin(data, peer_email, bundle)) @@ -806,22 +805,6 @@ def _warn_key_file_holds_data(keys_path: Path) -> None: ) -@contextmanager -def _locked_key_file(path: Path) -> Iterator[None]: - """Hold an exclusive cross-process lock on the key file at ``path``. - - The lock is on a sibling ``.lock`` file, as in ``PersistedDict``, so the - key file itself can be replaced while the lock is held. - """ - path.parent.mkdir(parents=True, exist_ok=True) - with open(path.with_suffix(f"{path.suffix}.lock"), "a") as f: - portalocker.lock(f, portalocker.LOCK_EX) - try: - yield - finally: - portalocker.unlock(f) - - def _with_pin(data: dict, peer_email: str, bundle: dict) -> dict: """``data`` from the key file, with ``bundle`` recorded as the pin of ``peer_email``.""" bundles = data.get(PEER_BUNDLES_KEY) diff --git a/syft/sync/sync/caches/datasite_owner_cache.py b/syft/sync/sync/caches/datasite_owner_cache.py index 198f0a6d486..31269c6aee3 100644 --- a/syft/sync/sync/caches/datasite_owner_cache.py +++ b/syft/sync/sync/caches/datasite_owner_cache.py @@ -466,9 +466,10 @@ def apply_checkpoint(self, checkpoint: Checkpoint, write_files: bool = True): self.file_hashes.clear() # Restore from checkpoint - # Always write to file_connection to maintain consistent state for - # process_local_changes comparison. This is needed even when write_files=False - # because file_connection may be in-memory and serves as the source of truth. + # Always write to file_connection to maintain consistent state for the + # comparison in process_local_file_changes. This is needed even when + # write_files=False, because file_connection may be in-memory and serves + # as the source of truth. for file_entry in checkpoint.files: path = Path(file_entry.path) self.file_hashes[path] = file_entry.hash diff --git a/syft/sync/sync/caches/persisted_dict.py b/syft/sync/sync/caches/persisted_dict.py index af22ddf5735..a811bef0cdd 100644 --- a/syft/sync/sync/caches/persisted_dict.py +++ b/syft/sync/sync/caches/persisted_dict.py @@ -26,7 +26,7 @@ import os from contextlib import contextmanager from pathlib import Path -from typing import Any, Callable +from typing import Any, Callable, Iterator from uuid import uuid4 import portalocker @@ -37,6 +37,36 @@ PERSISTED_DICT_VERSION = 1 +@contextmanager +def locked_file(path: Path, exclusive: bool = True) -> Iterator[None]: + """Hold a cross-process lock on ``path``, exclusive or shared. + + The lock is on a sibling ``.lock`` file, so ``path`` itself can be + replaced while the lock is held. + """ + path.parent.mkdir(parents=True, exist_ok=True) + flag = portalocker.LOCK_EX if exclusive else portalocker.LOCK_SH + with open(path.with_suffix(f"{path.suffix}.lock"), "a") as f: + portalocker.lock(f, flag) + try: + yield + finally: + portalocker.unlock(f) + + +def write_text_atomic(path: Path, text: str) -> None: + """Replace the content of ``path`` with ``text`` in one step.""" + path.parent.mkdir(parents=True, exist_ok=True) + # Per-process unique tmp path: even with the file lock, this guards + # against any path where two writers share a tmp filename. + tmp = path.with_suffix(f".tmp.{os.getpid()}.{uuid4().hex}") + try: + tmp.write_text(text) + tmp.replace(path) + finally: + tmp.unlink(missing_ok=True) + + class PersistedDict(dict): """Dict that persists to a JSON file. With path=None it's a plain in-memory dict.""" @@ -55,22 +85,13 @@ def __init__( # --- locking primitives --- - def _lock_path(self) -> Path: - return self._path.with_suffix(self._path.suffix + ".lock") - @contextmanager def _locked(self, exclusive: bool): if self._path is None: yield return - self._path.parent.mkdir(parents=True, exist_ok=True) - flag = portalocker.LOCK_EX if exclusive else portalocker.LOCK_SH - with open(self._lock_path(), "a") as f: - portalocker.lock(f, flag) - try: - yield - finally: - portalocker.unlock(f) + with locked_file(self._path, exclusive): + yield def exclusive_lock(self): """Acquire a cross-process exclusive lock on this dict's file. @@ -125,19 +146,11 @@ def _entries_of(data: Any) -> dict: def _write_to_file(self) -> None: if self._path is None: return - self._path.parent.mkdir(parents=True, exist_ok=True) - # Per-process unique tmp path: even with the file lock, this guards - # against any path where two writers share a tmp filename. - tmp = self._path.with_suffix(f".tmp.{os.getpid()}.{uuid4().hex}") serialized = { "version": PERSISTED_DICT_VERSION, "entries": {self._key_serializer(k): v for k, v in super().items()}, } - try: - tmp.write_text(json.dumps(serialized)) - tmp.replace(self._path) - finally: - tmp.unlink(missing_ok=True) + write_text_atomic(self._path, json.dumps(serialized)) # --- I/O with locking --- diff --git a/syft/sync/sync/datasite_owner_syncer.py b/syft/sync/sync/datasite_owner_syncer.py index 80f2bbe8658..8ddbe73b871 100644 --- a/syft/sync/sync/datasite_owner_syncer.py +++ b/syft/sync/sync/datasite_owner_syncer.py @@ -43,7 +43,7 @@ ) from syft_perms import SyftPermContext from syft.sync.sync.constants import CACHE_DIR, ROLLING_STATE_FILENAME -from syft.sync.sync.delivery_tracker import UNSENT_HASH +from syft.sync.sync.delivery_tracker import UNSENT_HASH, DeliveryPlan from syft.sync.transport.types import SendStatus from syft.sync.utils.syftbox_utils import ( create_event_timestamp, @@ -684,32 +684,30 @@ def _may_hold(self, peer: str, path: str, syncable: Mapping[str, str]) -> bool: def _seed_views_if_missing(self, peer_emails: list[str]) -> None: """Seed the views one time, for an owner whose state was never seeded. - Each approved peer, and each peer in ``peer_emails``, is taken to hold - every file it may read now. A syncable path that an earlier client - still had to send gets ``UNSENT_HASH``, so the next plan sends or - deletes it. Other pending paths, such as a private dataset share, stay - out of the view: a plan never sends them, so it must not delete them. + The seed covers each approved peer, and each peer in ``peer_emails``. """ deliveries = self.event_cache.deliveries if deliveries.is_seeded(): return self.perm_context._reload() approved = [p.email for p in self.connection_router.peer_store.approved_peers] + deliveries.seed(self._held_before_seed(sorted({*approved, *peer_emails}))) + + def _held_before_seed(self, peers: list[str]) -> dict[str, dict[str, str]]: + """What each of ``peers`` is taken to hold when the views are seeded. + + Each peer holds every file it may read now. A syncable path that an + earlier client still had to send gets ``UNSENT_HASH``, so the next plan + sends or deletes it. Other pending paths, such as a private dataset + share, stay out of the view: a plan never sends them, so it must not + delete them. The caller reloads the permissions first. + """ pending = self.event_cache.legacy_pending_paths() - readable = self._readable_hashes(sorted({*approved, *peer_emails})) - deliveries.seed( - { - peer: { - **files, - **{ - p: UNSENT_HASH - for p in pending.get(peer, ()) - if self._is_syncable(p) - }, - } - for peer, files in readable.items() - } - ) + held = self._readable_hashes(peers) + for peer, files in held.items(): + unsent = [p for p in pending.get(peer, ()) if self._is_syncable(p)] + files.update(dict.fromkeys(unsent, UNSENT_HASH)) + return held def deliver(self, peer_emails: list[str]) -> None: """Send each peer the files its view lacks, and deletes for the files it @@ -718,15 +716,25 @@ def deliver(self, peer_emails: list[str]) -> None: deliveries.confirm(self.connection_router.transport.p2p.delivered()) self.perm_context._reload() plans = deliveries.plan(self._readable_hashes(peer_emails)) - for peer, plan in plans.items(): - events = [self._create_current_state_event(p) for p in plan.send] - events += [self._deletion_event(p) for p in plan.delete] - for batch in _batches_by_content_size(events, MAX_DELIVERY_MESSAGE_CONTENT): - message = FileChangeEventsMessage(events=batch) - self._planned_message_names.add(message.message_filepath.as_string()) - self.outbox_queue.put((peer, message)) + # Build every message before the first one is queued, so a read error + # leaves no message of this plan in the queue. + messages = [ + (peer, FileChangeEventsMessage(events=batch)) + for peer, plan in plans.items() + for batch in _batches_by_content_size( + self._plan_events(plan), MAX_DELIVERY_MESSAGE_CONTENT + ) + ] + for peer, message in messages: + self._planned_message_names.add(message.message_filepath.as_string()) + self.outbox_queue.put((peer, message)) self.process_syftbox_events_queue() + def _plan_events(self, plan: DeliveryPlan) -> list[FileChangeEvent]: + """The events that carry out ``plan``: its sends, then its deletes.""" + events = [self._create_current_state_event(p) for p in plan.send] + return events + [self._deletion_event(p) for p in plan.delete] + def pull_and_process_next_proposed_filechange( self, sender_email: str, raise_on_none=True ) -> ProposedFileChangesMessage | None: @@ -910,10 +918,11 @@ def compact_outbox_if_needed( recipient_email: str, min_messages: int = MIN_MESSAGES_COMPACT, ) -> int: - """Merge accumulated event-message files in a recipient's outbox into one. + """Merge accumulated event-message files in a recipient's outbox into fewer. - Compaction downloads them, concatenates their events into one fresh message, and - deletes the originals. + Compaction downloads them, concatenates their events, writes them again + as messages of at most ``MAX_DELIVERY_MESSAGE_CONTENT`` of file content, + and deletes the originals. Returns the number of source messages compacted (0 if below threshold, if no outbox folder, or if peer encryption is on or the peer policy is diff --git a/syft/sync/sync/delivery_tracker.py b/syft/sync/sync/delivery_tracker.py index cda3e4c9dab..8e2b2ee5afa 100644 --- a/syft/sync/sync/delivery_tracker.py +++ b/syft/sync/sync/delivery_tracker.py @@ -50,16 +50,14 @@ starts the views from them. """ -import os from abc import ABC, abstractmethod -from contextlib import contextmanager, nullcontext +from contextlib import nullcontext from pathlib import Path from typing import ContextManager, Iterable, Mapping -from uuid import uuid4 -import portalocker from pydantic import BaseModel, ConfigDict, Field +from syft.sync.sync.caches.persisted_dict import locked_file, write_text_atomic from syft.sync.transport.types import SendStatus DELIVERY_VIEWS_FILENAME = "owner_delivery_views.json" @@ -103,7 +101,7 @@ class DeliveryPlan(BaseModel): class DeliveryStateStore(ABC): - """Where a ``DeliveryState`` is kept: a local file, Drive, or memory.""" + """Where a ``DeliveryState`` is kept: a local file or memory.""" @abstractmethod def load(self) -> DeliveryState: @@ -152,24 +150,11 @@ def load(self) -> DeliveryState: ) return state - @contextmanager - def locked(self): - self.path.parent.mkdir(parents=True, exist_ok=True) - with open(self.path.with_suffix(f"{self.path.suffix}.lock"), "a") as f: - portalocker.lock(f, portalocker.LOCK_EX) - try: - yield - finally: - portalocker.unlock(f) + def locked(self) -> ContextManager[None]: + return locked_file(self.path) def save(self, state: DeliveryState) -> None: - self.path.parent.mkdir(parents=True, exist_ok=True) - tmp = self.path.with_suffix(f".tmp.{os.getpid()}.{uuid4().hex}") - try: - tmp.write_text(state.model_dump_json()) - tmp.replace(self.path) - finally: - tmp.unlink(missing_ok=True) + write_text_atomic(self.path, state.model_dump_json()) def _apply_changes_to_view( diff --git a/tests/unit/test_delivery_tracker.py b/tests/unit/test_delivery_tracker.py index 49b88a84efc..1ea2bfaa576 100644 --- a/tests/unit/test_delivery_tracker.py +++ b/tests/unit/test_delivery_tracker.py @@ -402,6 +402,35 @@ def fail_upload(*args, **kwargs): assert _outbox_paths(ds_manager, do_manager) == [] +def test_read_error_leaves_no_planned_message(monkeypatch): + ds_manager, do_manager = _pair() + syncer = do_manager.datasite_owner_syncer + shared = syncer.perm_context.open("shared/") + shared.grant_read_access(ds_manager.email) + shared.grant_read_access("peer_b@example.com") + _write(do_manager, "shared/data.txt", "one") + syncer._record_local_changes() + + # The plan reads the file once per peer. The read for the second peer fails. + reads = [] + create_event = datasite_owner_syncer.DatasiteOwnerSyncer._create_current_state_event + + def fail_second_read(self, path): + reads.append(path) + if len(reads) == 2: + raise PermissionError(path) + return create_event(self, path) + + monkeypatch.setattr( + datasite_owner_syncer.DatasiteOwnerSyncer, + "_create_current_state_event", + fail_second_read, + ) + with pytest.raises(PermissionError): + syncer.deliver([ds_manager.email, "peer_b@example.com"]) + assert syncer.outbox_queue.empty() + + def test_compaction_keeps_message_size_cap(monkeypatch): monkeypatch.setattr(datasite_owner_syncer, "MAX_DELIVERY_MESSAGE_CONTENT", 3) ds_manager, do_manager = _pair() From 7b89481c84f51d1451fa5c6b260b1d62c7c688ae Mon Sep 17 00:00:00 2001 From: Pedro Werneck Date: Thu, 22 Oct 2026 19:50:00 -0300 Subject: [PATCH 11/36] fix: keep the delete flag of an accepted proposal FileChangeEvent.from_proposed_filechange() dropped is_deleted, so the owner turned an accepted deletion into a write with no content, and the write raised. --- syft/sync/events/file_change_event.py | 1 + .../unit/test_file_change_events_serialization.py | 12 ++++++++++++ 2 files changed, 13 insertions(+) diff --git a/syft/sync/events/file_change_event.py b/syft/sync/events/file_change_event.py index 0ce4151030e..13ee2644621 100644 --- a/syft/sync/events/file_change_event.py +++ b/syft/sync/events/file_change_event.py @@ -127,6 +127,7 @@ def from_proposed_filechange( submitted_timestamp=proposed_filechange.submitted_timestamp, timestamp=create_event_timestamp(), datasite_email=proposed_filechange.datasite_email, + is_deleted=proposed_filechange.is_deleted, ) def __hash__(self) -> int: diff --git a/tests/migrations/unit/test_file_change_events_serialization.py b/tests/migrations/unit/test_file_change_events_serialization.py index fca62d4e252..751e6542a8f 100644 --- a/tests/migrations/unit/test_file_change_events_serialization.py +++ b/tests/migrations/unit/test_file_change_events_serialization.py @@ -95,3 +95,15 @@ def test_from_proposed_filechange_carries_identity_free_items(): assert event.new_hash == proposed.new_hash # Items carry no identity fields on the wire; only envelopes do. assert "canonical_name" not in json.loads(event.model_dump_json()) + + +def test_from_proposed_filechange_keeps_delete(): + proposed = ProposedFileChange( + path_in_datasite="data/file.txt", + old_hash="h1", + datasite_email="do@test.org", + is_deleted=True, + ) + event = FileChangeEvent.from_proposed_filechange(proposed) + assert event.is_deleted + assert event.content is None From d688ba0f12bff79fea999f3d73d60c7946ef0de4 Mon Sep 17 00:00:00 2001 From: Pedro Werneck Date: Thu, 22 Oct 2026 19:50:01 -0300 Subject: [PATCH 12/36] feat: follow the ticket on re-approval and accepted changes When the owner rejects a peer, its view is dropped, so a later approval sends it every file it may read. A change that the owner accepts from a peer goes only to that peer, while the file keeps the state that the change gave it. A delete after a revoke still goes out. --- syft/sync/syftbox_manager.py | 11 ++- syft/sync/sync/datasite_owner_syncer.py | 51 +++++++++++- syft/sync/sync/delivery_tracker.py | 44 +++++++++-- tests/unit/test_delivery_tracker.py | 26 ++++++ tests/unit/test_sync_manager.py | 101 +++++++++++++++++++++--- 5 files changed, 214 insertions(+), 19 deletions(-) diff --git a/syft/sync/syftbox_manager.py b/syft/sync/syftbox_manager.py index 67d33bb46ff..7cee9198a33 100644 --- a/syft/sync/syftbox_manager.py +++ b/syft/sync/syftbox_manager.py @@ -1085,8 +1085,17 @@ def _emit_peers_loaded(self) -> None: self._emit("peers_loaded") def reject_peer_request(self, email_or_peer: str | Peer): - """Reject a pending peer request. Delegates to PeerManager.""" + """Reject a pending peer request. Delegates to PeerManager. + + The DO drops the peer's delivery view, so a later approval sends the + peer every file it may read. + """ self.peer_manager.reject_peer_request(email_or_peer) + if self.has_do_role: + peer_email = ( + email_or_peer if isinstance(email_or_peer, str) else email_or_peer.email + ) + self.datasite_owner_syncer.event_cache.deliveries.reset(peer_email) def validate_peer(self, peer_email: str) -> Peer: """Load the peers, then return ``peer_email`` if it is approved. diff --git a/syft/sync/sync/datasite_owner_syncer.py b/syft/sync/sync/datasite_owner_syncer.py index 8ddbe73b871..826e9b8df04 100644 --- a/syft/sync/sync/datasite_owner_syncer.py +++ b/syft/sync/sync/datasite_owner_syncer.py @@ -43,7 +43,11 @@ ) from syft_perms import SyftPermContext from syft.sync.sync.constants import CACHE_DIR, ROLLING_STATE_FILENAME -from syft.sync.sync.delivery_tracker import UNSENT_HASH, DeliveryPlan +from syft.sync.sync.delivery_tracker import ( + UNSENT_HASH, + AcceptedChange, + DeliveryPlan, +) from syft.sync.transport.types import SendStatus from syft.sync.utils.syftbox_utils import ( create_event_timestamp, @@ -716,6 +720,12 @@ def deliver(self, peer_emails: list[str]) -> None: deliveries.confirm(self.connection_router.transport.p2p.delivered()) self.perm_context._reload() plans = deliveries.plan(self._readable_hashes(peer_emails)) + accepted = deliveries.accepted_changes() + syncable = self._syncable_hashes() + plans = { + peer: self._skip_accepted_from_others(peer, plan, accepted, syncable) + for peer, plan in plans.items() + } # Build every message before the first one is queued, so a read error # leaves no message of this plan in the queue. messages = [ @@ -730,6 +740,38 @@ def deliver(self, peer_emails: list[str]) -> None: self.outbox_queue.put((peer, message)) self.process_syftbox_events_queue() + def _skip_accepted_from_others( + self, + peer: str, + plan: DeliveryPlan, + accepted: Mapping[str, AcceptedChange], + syncable: Mapping[str, str], + ) -> DeliveryPlan: + """``plan`` without the changes that the owner accepted from another peer. + + An accepted file goes only to its submitter while ``syncable`` still has + the hash that the change gave it. An accepted delete goes only to its + submitter while the file stays gone. A delete after a revoke still goes + out. + """ + + def from_other(path: str) -> bool: + change = accepted.get(path) + return ( + change is not None + and change.peer != peer + and change.hash == syncable.get(path) + ) + + return DeliveryPlan( + send=[p for p in plan.send if not from_other(p)], + delete=[ + p + for p in plan.delete + if not (from_other(p) and accepted[p].hash is None) + ], + ) + def _plan_events(self, plan: DeliveryPlan) -> list[FileChangeEvent]: """The events that carry out ``plan``: its sends, then its deletes.""" events = [self._create_current_state_event(p) for p in plan.send] @@ -795,6 +837,13 @@ def handle_proposed_filechange_events_message( filtered_message ) if accepted_events_message is not None: + self.event_cache.deliveries.accept( + sender_email, + { + str(e.path_in_datasite): None if e.is_deleted else e.new_hash + for e in accepted_events_message.events + }, + ) self.queue_event_for_syftbox( recipients=[sender_email], file_change_events_message=accepted_events_message, diff --git a/syft/sync/sync/delivery_tracker.py b/syft/sync/sync/delivery_tracker.py index 8e2b2ee5afa..9db35227ee0 100644 --- a/syft/sync/sync/delivery_tracker.py +++ b/syft/sync/sync/delivery_tracker.py @@ -14,12 +14,14 @@ in the view, but no longer readable -> delete in the view, but the file is gone -> delete -Every readable file counts, whoever wrote it. Thus a change that the owner -accepts from one peer also goes to every other peer that may read it. +A change that the owner accepts from a peer goes only to that peer. +``accepted`` records it, and the owner leaves it out of the plan of every other +peer while the file stays in that state. When the owner changes the file +again, every peer that may read it gets it. -A new peer starts with an empty view, so it is sent every file it may read. A -peer that the owner rejects and then approves again keeps its view, and gets -only what changed. +A new peer starts with an empty view, so it is sent every file it may read. +When the owner rejects a peer, ``reset`` drops its view. Thus a peer that the +owner rejects and then approves again is sent every file it may read. An owner whose state was never seeded, because it ran an earlier client or lost its views file, has file hashes but no views. ``seed`` then starts the @@ -79,6 +81,16 @@ class QueuedSend(BaseModel): changes: dict[str, str | None] +class AcceptedChange(BaseModel): + """A change that the owner accepted from ``peer``. + + ``hash`` is the new hash of the file, or None for a delete. + """ + + peer: str + hash: str | None + + class DeliveryState(BaseModel): """Everything the tracker keeps: views, and sends awaiting confirmation.""" @@ -89,6 +101,8 @@ class DeliveryState(BaseModel): queued: dict[str, QueuedSend] = Field(default_factory=dict) # Whether ``DeliveryTracker.seed`` ran on this state. seeded: bool = False + # path -> the last change that the owner accepted from a peer for it + accepted: dict[str, AcceptedChange] = Field(default_factory=dict) class DeliveryPlan(BaseModel): @@ -183,6 +197,26 @@ def clear(self) -> None: with self.store.locked(): self.store.save(DeliveryState()) + def reset(self, peer: str) -> None: + """Drop ``peer``'s view, so the next plan sends it every file it may read.""" + with self.store.locked(): + state = self.store.load() + state.views.pop(peer, None) + self.store.save(state) + + def accept(self, peer: str, changes: Mapping[str, str | None]) -> None: + """Record ``changes`` (path -> new hash, None for a delete) as accepted + from ``peer``.""" + with self.store.locked(): + state = self.store.load() + for path, new_hash in changes.items(): + state.accepted[path] = AcceptedChange(peer=peer, hash=new_hash) + self.store.save(state) + + def accepted_changes(self) -> dict[str, AcceptedChange]: + """The last change that the owner accepted from a peer, by path.""" + return self.store.load().accepted + def is_seeded(self) -> bool: """Whether ``seed`` ran on the stored state.""" return self.store.load().seeded diff --git a/tests/unit/test_delivery_tracker.py b/tests/unit/test_delivery_tracker.py index 1ea2bfaa576..7e7d95b065e 100644 --- a/tests/unit/test_delivery_tracker.py +++ b/tests/unit/test_delivery_tracker.py @@ -8,6 +8,7 @@ from syft.sync.connections.connection_router import ConnectionRouter from syft.sync.encryption_policy import PeerEncryptionPendingError +from syft.sync.peers.peer import PeerState from syft.sync.sync import datasite_owner_syncer from syft.sync.sync.caches.persisted_dict import PersistedDict from syft.sync.sync.constants import CACHE_DIR, OWNER_PENDING_DELIVERIES_FILENAME @@ -55,6 +56,12 @@ def test_seed_runs_once_and_keeps_recorded_views(): assert tracker.plan({PEER: {"a": "h1", "b": "h9"}})[PEER].send == [] +def test_reset_drops_view(): + tracker = _tracker_holding({"a": "h1"}) + tracker.reset(PEER) + assert tracker.plan({PEER: {"a": "h1"}})[PEER].send == ["a"] + + def test_queued_send_updates_view_on_confirm(): tracker = DeliveryTracker(MemoryDeliveryStateStore()) tracker.sent(PEER, "m1", SendStatus.QUEUED, {"a": "h1"}) @@ -224,6 +231,25 @@ def test_revoked_peer_gets_content_then_delete( ] +def test_rejected_then_approved_peer_gets_every_file(): + ds_manager, do_manager = _pair() + do_manager.datasite_owner_syncer.perm_context.open("shared/").grant_read_access( + ds_manager.email + ) + _write(do_manager, "shared/data.txt", "one") + do_manager.sync(auto_checkpoint=False) + + # The DS asks again. The DO rejects it, and then approves a later request. + peer = do_manager.peer_manager.get_cached_peer(ds_manager.email) + peer.state = PeerState.REQUESTED_BY_PEER + do_manager.reject_peer_request(ds_manager.email) + peer.state = PeerState.REQUESTED_BY_PEER + do_manager.approve_peer_request(ds_manager.email) + + do_manager.sync(auto_checkpoint=False) + assert _outbox_paths(ds_manager, do_manager).count("shared/data.txt") == 2 + + def test_restarted_owner_sends_nothing_new(): ds_manager, do_manager = _pair() do_manager.datasite_owner_syncer.perm_context.open("shared/").grant_read_access( diff --git a/tests/unit/test_sync_manager.py b/tests/unit/test_sync_manager.py index 40150f5f590..0c5e9a8fb3d 100644 --- a/tests/unit/test_sync_manager.py +++ b/tests/unit/test_sync_manager.py @@ -758,9 +758,18 @@ def test_permission_change_triggers_resend(): ) -def test_accepted_change_reaches_other_readers(): - """A change that the DO accepts from peer A goes to peer B, who may read it, - in the next DO sync.""" +def _events_for(router: ConnectionRouter, do_manager) -> list[tuple[str, bool]]: + """(path, is_deleted) of each event in the DO's outbox for ``router``'s peer.""" + return [ + (str(e.path_in_datasite), e.is_deleted) + for m in router.watcher_get_events_messages(do_manager.email, None) + for e in m.events + ] + + +def test_accepted_change_goes_only_to_submitter(): + """A change that the DO accepts from peer A goes only to peer A. Peer B, who + may read the file, gets it when the DO changes the file.""" ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( use_in_memory_cache=False, ) @@ -769,19 +778,87 @@ def test_accepted_change_reaches_other_readers(): shared = do_manager.datasite_owner_syncer.perm_context.open("shared/") shared.grant_write_access(ds_manager.email) shared.grant_read_access(peer_b_email) + peers = [ds_manager.email, peer_b_email] ds_manager._send_file_change(f"{do_manager.email}/shared/proposal.txt", "from A") - # The first sync accepts the change. The second one sends it to peer B, - # because a sync sends before it pulls. - do_manager.datasite_owner_syncer.sync([ds_manager.email, peer_b_email]) - do_manager.datasite_owner_syncer.sync([ds_manager.email, peer_b_email]) + do_manager.datasite_owner_syncer.sync(peers) + do_manager.datasite_owner_syncer.sync(peers) + assert ("shared/proposal.txt", False) in _events_for( + ds_manager._connection_router, do_manager + ) + assert ("shared/proposal.txt", False) not in _events_for(peer_b_router, do_manager) - paths_for_b = [ - str(e.path_in_datasite) - for m in peer_b_router.watcher_get_events_messages(do_manager.email, None) - for e in m.events + proposal = do_manager.syftbox_folder / do_manager.email / "shared/proposal.txt" + proposal.write_text("from the DO") + do_manager.datasite_owner_syncer.sync(peers) + assert ("shared/proposal.txt", False) in _events_for(peer_b_router, do_manager) + + +def test_revoke_after_accepted_change_sends_delete(): + """Peer B holds the DO's version of a file. After the DO accepts peer A's + change to it and peer B loses read access, peer B gets a delete.""" + ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + use_in_memory_cache=False, + ) + peer_b_email = "peer_b@example.com" + peer_b_router = _peer_router(ds_manager, do_manager, peer_b_email) + syncer = do_manager.datasite_owner_syncer + shared = syncer.perm_context.open("shared/") + shared.grant_write_access(ds_manager.email) + shared.grant_read_access(peer_b_email) + peers = [ds_manager.email, peer_b_email] + data = do_manager.syftbox_folder / do_manager.email / "shared/data.txt" + data.parent.mkdir(parents=True, exist_ok=True) + data.write_text("one") + syncer.sync(peers) + ds_manager.sync() + + ds_manager._send_file_change(f"{do_manager.email}/shared/data.txt", "from A") + syncer.sync(peers) + shared.revoke_read_access(peer_b_email) + syncer.sync(peers) + assert _events_for(peer_b_router, do_manager) == [ + ("shared/data.txt", False), + ("shared/data.txt", True), ] - assert "shared/proposal.txt" in paths_for_b + + +def test_accepted_delete_goes_only_to_submitter(): + """A delete that the DO accepts from peer A goes only to peer A. Peer B keeps + its copy.""" + ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + use_in_memory_cache=False, + ) + peer_b_email = "peer_b@example.com" + peer_b_router = _peer_router(ds_manager, do_manager, peer_b_email) + syncer = do_manager.datasite_owner_syncer + shared = syncer.perm_context.open("shared/") + shared.grant_write_access(ds_manager.email) + shared.grant_read_access(peer_b_email) + peers = [ds_manager.email, peer_b_email] + data = do_manager.syftbox_folder / do_manager.email / "shared/data.txt" + data.parent.mkdir(parents=True, exist_ok=True) + data.write_text("one") + syncer.sync(peers) + + proposal = ProposedFileChangesMessage( + sender_email=ds_manager.email, + proposed_file_changes=[ + ProposedFileChange( + path_in_datasite="shared/data.txt", + old_hash=syncer.event_cache.file_hashes[Path("shared/data.txt")], + datasite_email=do_manager.email, + is_deleted=True, + ) + ], + ) + syncer.handle_proposed_filechange_events_message(ds_manager.email, proposal) + syncer.process_syftbox_events_queue() + syncer.sync(peers) + assert ("shared/data.txt", True) in _events_for( + ds_manager._connection_router, do_manager + ) + assert _events_for(peer_b_router, do_manager) == [("shared/data.txt", False)] def test_default_collections_folder_picks_shareable_spec_regardless_of_order(): From 003294fc5ed5220f4b576abb77ecc2d5bca08ad7 Mon Sep 17 00:00:00 2001 From: Pedro Werneck Date: Thu, 22 Oct 2026 21:07:35 -0300 Subject: [PATCH 13/36] fix: send deletes after re-approval and let later readers catch up When the DO rejects a peer, reset() now marks each path in its view as unsent. A later approval then sends every readable file, and a delete for each file that the peer may no longer have. An accepted change is withheld only from the peers that may read the file when the DO accepts it. A peer that may read the file later gets it. --- syft/sync/syftbox_manager.py | 4 +-- syft/sync/sync/datasite_owner_syncer.py | 42 ++++++++++++++++++------- syft/sync/sync/delivery_tracker.py | 39 ++++++++++++++--------- tests/unit/test_delivery_tracker.py | 24 +++++++++++--- tests/unit/test_sync_manager.py | 24 ++++++++++++++ 5 files changed, 101 insertions(+), 32 deletions(-) diff --git a/syft/sync/syftbox_manager.py b/syft/sync/syftbox_manager.py index 7cee9198a33..26f13315e2c 100644 --- a/syft/sync/syftbox_manager.py +++ b/syft/sync/syftbox_manager.py @@ -1087,8 +1087,8 @@ def _emit_peers_loaded(self) -> None: def reject_peer_request(self, email_or_peer: str | Peer): """Reject a pending peer request. Delegates to PeerManager. - The DO drops the peer's delivery view, so a later approval sends the - peer every file it may read. + The DO resets the peer's delivery view, so a later approval sends the + peer every file it may read, and deletes the files it may no longer have. """ self.peer_manager.reject_peer_request(email_or_peer) if self.has_do_role: diff --git a/syft/sync/sync/datasite_owner_syncer.py b/syft/sync/sync/datasite_owner_syncer.py index 826e9b8df04..4b71718f768 100644 --- a/syft/sync/sync/datasite_owner_syncer.py +++ b/syft/sync/sync/datasite_owner_syncer.py @@ -747,19 +747,18 @@ def _skip_accepted_from_others( accepted: Mapping[str, AcceptedChange], syncable: Mapping[str, str], ) -> DeliveryPlan: - """``plan`` without the changes that the owner accepted from another peer. + """``plan`` without the accepted changes that are withheld from ``peer``. - An accepted file goes only to its submitter while ``syncable`` still has - the hash that the change gave it. An accepted delete goes only to its - submitter while the file stays gone. A delete after a revoke still goes - out. + An accepted file is withheld while ``syncable`` still has the hash that + the change gave it. An accepted delete is withheld while the file stays + gone. A delete after a revoke still goes out. """ def from_other(path: str) -> bool: change = accepted.get(path) return ( change is not None - and change.peer != peer + and peer in change.withheld_from and change.hash == syncable.get(path) ) @@ -838,11 +837,7 @@ def handle_proposed_filechange_events_message( ) if accepted_events_message is not None: self.event_cache.deliveries.accept( - sender_email, - { - str(e.path_in_datasite): None if e.is_deleted else e.new_hash - for e in accepted_events_message.events - }, + self._accepted_changes(sender_email, accepted_events_message) ) self.queue_event_for_syftbox( recipients=[sender_email], @@ -851,6 +846,31 @@ def handle_proposed_filechange_events_message( if self.persist_owner_state: self._add_events_to_rolling_state(accepted_events_message) + def _accepted_changes( + self, sender_email: str, message: FileChangeEventsMessage + ) -> dict[str, AcceptedChange]: + """The changes of ``message``, accepted from ``sender_email``, by path. + + Each change is withheld from the other approved peers that may read its + file now. + """ + self.perm_context._reload() + others = [ + p.email + for p in self.connection_router.peer_store.approved_peers + if p.email != sender_email + ] + changes = {} + for event in message.events: + path = str(event.path_in_datasite) + changes[path] = AcceptedChange( + hash=None if event.is_deleted else event.new_hash, + withheld_from=[ + p for p in others if self.perm_context.can_read(path, p) + ], + ) + return changes + def queue_event_for_syftbox( self, recipients: list[str], file_change_events_message: FileChangeEventsMessage ): diff --git a/syft/sync/sync/delivery_tracker.py b/syft/sync/sync/delivery_tracker.py index 9db35227ee0..e341952e420 100644 --- a/syft/sync/sync/delivery_tracker.py +++ b/syft/sync/sync/delivery_tracker.py @@ -15,13 +15,16 @@ in the view, but the file is gone -> delete A change that the owner accepts from a peer goes only to that peer. -``accepted`` records it, and the owner leaves it out of the plan of every other -peer while the file stays in that state. When the owner changes the file -again, every peer that may read it gets it. +``accept`` records it, with the other peers that may read the file at that +time. The owner leaves the change out of their plans while the file stays in +that state. A peer that joins later, or that may read the file only later, gets +it. When the owner changes the file again, every peer that may read it gets it. A new peer starts with an empty view, so it is sent every file it may read. -When the owner rejects a peer, ``reset`` drops its view. Thus a peer that the -owner rejects and then approves again is sent every file it may read. +When the owner rejects a peer, ``reset`` sets each path in its view to +``UNSENT_HASH``, and stops withholding accepted changes from it. Thus a peer +that the owner rejects and then approves again is sent every file it may read, +and a delete for each file that it may no longer have. An owner whose state was never seeded, because it ran an earlier client or lost its views file, has file hashes but no views. ``seed`` then starts the @@ -82,13 +85,15 @@ class QueuedSend(BaseModel): class AcceptedChange(BaseModel): - """A change that the owner accepted from ``peer``. + """A change that the owner accepted from a peer. ``hash`` is the new hash of the file, or None for a delete. + ``withheld_from`` holds the other peers that could read the file when the + owner accepted the change. """ - peer: str hash: str | None + withheld_from: list[str] = Field(default_factory=list) class DeliveryState(BaseModel): @@ -198,19 +203,25 @@ def clear(self) -> None: self.store.save(DeliveryState()) def reset(self, peer: str) -> None: - """Drop ``peer``'s view, so the next plan sends it every file it may read.""" + """Set each path in ``peer``'s view to ``UNSENT_HASH``. + + The next plan sends ``peer`` every file it may read, and deletes the + paths of its view that it may no longer have. + """ with self.store.locked(): state = self.store.load() - state.views.pop(peer, None) + view = state.views.get(peer, {}) + state.views[peer] = dict.fromkeys(view, UNSENT_HASH) + for change in state.accepted.values(): + if peer in change.withheld_from: + change.withheld_from.remove(peer) self.store.save(state) - def accept(self, peer: str, changes: Mapping[str, str | None]) -> None: - """Record ``changes`` (path -> new hash, None for a delete) as accepted - from ``peer``.""" + def accept(self, changes: Mapping[str, AcceptedChange]) -> None: + """Record ``changes``, by path, as the last change accepted from a peer.""" with self.store.locked(): state = self.store.load() - for path, new_hash in changes.items(): - state.accepted[path] = AcceptedChange(peer=peer, hash=new_hash) + state.accepted.update(changes) self.store.save(state) def accepted_changes(self) -> dict[str, AcceptedChange]: diff --git a/tests/unit/test_delivery_tracker.py b/tests/unit/test_delivery_tracker.py index 7e7d95b065e..65332489772 100644 --- a/tests/unit/test_delivery_tracker.py +++ b/tests/unit/test_delivery_tracker.py @@ -17,6 +17,7 @@ ) from syft.sync.sync.delivery_tracker import ( DELIVERY_STATE_VERSION, + AcceptedChange, DELIVERY_VIEWS_FILENAME, DeliveryState, DeliveryTracker, @@ -56,10 +57,19 @@ def test_seed_runs_once_and_keeps_recorded_views(): assert tracker.plan({PEER: {"a": "h1", "b": "h9"}})[PEER].send == [] -def test_reset_drops_view(): - tracker = _tracker_holding({"a": "h1"}) +def test_reset_resends_readable_deletes_gone(): + tracker = _tracker_holding({"a": "h1", "gone": "h1"}) tracker.reset(PEER) - assert tracker.plan({PEER: {"a": "h1"}})[PEER].send == ["a"] + plan = tracker.plan({PEER: {"a": "h1"}})[PEER] + assert plan.send == ["a"] + assert plan.delete == ["gone"] + + +def test_reset_stops_withholding_accepted_changes(): + tracker = DeliveryTracker(MemoryDeliveryStateStore()) + tracker.accept({"a": AcceptedChange(hash="h1", withheld_from=[PEER, "b@x.com"])}) + tracker.reset(PEER) + assert tracker.accepted_changes()["a"].withheld_from == ["b@x.com"] def test_queued_send_updates_view_on_confirm(): @@ -231,23 +241,27 @@ def test_revoked_peer_gets_content_then_delete( ] -def test_rejected_then_approved_peer_gets_every_file(): +def test_rejected_then_approved_peer_gets_every_file_and_deletes(): ds_manager, do_manager = _pair() do_manager.datasite_owner_syncer.perm_context.open("shared/").grant_read_access( ds_manager.email ) _write(do_manager, "shared/data.txt", "one") + _write(do_manager, "shared/gone.txt", "one") do_manager.sync(auto_checkpoint=False) - # The DS asks again. The DO rejects it, and then approves a later request. + # The DS asks again. The DO rejects it, deletes a file, and then approves a + # later request. peer = do_manager.peer_manager.get_cached_peer(ds_manager.email) peer.state = PeerState.REQUESTED_BY_PEER do_manager.reject_peer_request(ds_manager.email) + (do_manager.syftbox_folder / do_manager.email / "shared/gone.txt").unlink() peer.state = PeerState.REQUESTED_BY_PEER do_manager.approve_peer_request(ds_manager.email) do_manager.sync(auto_checkpoint=False) assert _outbox_paths(ds_manager, do_manager).count("shared/data.txt") == 2 + assert _sent_events(ds_manager, do_manager, "shared/gone.txt")[-1] == (True, None) def test_restarted_owner_sends_nothing_new(): diff --git a/tests/unit/test_sync_manager.py b/tests/unit/test_sync_manager.py index 0c5e9a8fb3d..28bd235bffa 100644 --- a/tests/unit/test_sync_manager.py +++ b/tests/unit/test_sync_manager.py @@ -794,6 +794,30 @@ def test_accepted_change_goes_only_to_submitter(): assert ("shared/proposal.txt", False) in _events_for(peer_b_router, do_manager) +def test_reader_granted_after_accept_gets_accepted_file(): + """Peer C may read the folder only after the DO accepts peer A's change, so + peer C gets the file. Peer B, who could read it then, does not.""" + ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + use_in_memory_cache=False, + ) + peer_b_email = "peer_b@example.com" + peer_c_email = "peer_c@example.com" + peer_b_router = _peer_router(ds_manager, do_manager, peer_b_email) + peer_c_router = _peer_router(ds_manager, do_manager, peer_c_email) + syncer = do_manager.datasite_owner_syncer + shared = syncer.perm_context.open("shared/") + shared.grant_write_access(ds_manager.email) + shared.grant_read_access(peer_b_email) + peers = [ds_manager.email, peer_b_email, peer_c_email] + + ds_manager._send_file_change(f"{do_manager.email}/shared/proposal.txt", "from A") + syncer.sync(peers) + shared.grant_read_access(peer_c_email) + syncer.sync(peers) + assert ("shared/proposal.txt", False) in _events_for(peer_c_router, do_manager) + assert ("shared/proposal.txt", False) not in _events_for(peer_b_router, do_manager) + + def test_revoke_after_accepted_change_sends_delete(): """Peer B holds the DO's version of a file. After the DO accepts peer A's change to it and peer B loses read access, peer B gets a delete.""" From 42cc22f823392c00f3ca2d41659e9fa35757db09 Mon Sep 17 00:00:00 2001 From: Pedro Werneck Date: Mon, 26 Oct 2026 17:04:42 -0300 Subject: [PATCH 14/36] docs: fix outdated delivery state and contract test docstrings --- syft/sync/sync/delivery_tracker.py | 2 +- tests/unit/test_transport_contracts.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/syft/sync/sync/delivery_tracker.py b/syft/sync/sync/delivery_tracker.py index e341952e420..643cbb3467b 100644 --- a/syft/sync/sync/delivery_tracker.py +++ b/syft/sync/sync/delivery_tracker.py @@ -97,7 +97,7 @@ class AcceptedChange(BaseModel): class DeliveryState(BaseModel): - """Everything the tracker keeps: views, and sends awaiting confirmation.""" + """Everything the tracker keeps.""" version: int = DELIVERY_STATE_VERSION # peer -> {path: hash} diff --git a/tests/unit/test_transport_contracts.py b/tests/unit/test_transport_contracts.py index b34e77e2eed..598999f3547 100644 --- a/tests/unit/test_transport_contracts.py +++ b/tests/unit/test_transport_contracts.py @@ -1,6 +1,5 @@ """The transport contracts: the abstract jobs, their helper methods, the -message types, the naming rules and the DeliveryTracker state. No backend -implements them yet.""" +message types, the naming rules and the DeliveryTracker state.""" from types import SimpleNamespace from unittest.mock import Mock From 155707d23635e047d32802e26a059746b8a7a6ed Mon Sep 17 00:00:00 2001 From: Pedro Werneck Date: Mon, 26 Oct 2026 17:15:39 -0300 Subject: [PATCH 15/36] fix: stop failed sends and accepted changes from blocking deliveries A failed owner send is dropped and raises, so it no longer blocks every later sync. An accepted change is no longer withheld from a peer that loses read access, gets a delete, or reads a file that the DO changed again. --- docs/connections.md | 2 +- syft/sync/sync/datasite_owner_syncer.py | 25 ++++-- syft/sync/sync/delivery_tracker.py | 29 ++++++- tests/unit/test_delivery_tracker.py | 25 +++++- tests/unit/test_sync_manager.py | 100 ++++++++++++++++++++++++ 5 files changed, 164 insertions(+), 17 deletions(-) diff --git a/docs/connections.md b/docs/connections.md index 88c4dc7f8d2..64e81552e36 100644 --- a/docs/connections.md +++ b/docs/connections.md @@ -61,7 +61,7 @@ A pair encrypts when one side is `required`. A SyftBox has one policy, kept in `//private/syft_settings.json`. The first client to start on a new SyftBox writes it: the `encryption` it logged in with, or `optional`. A later client, the notebook or syft-bg, logs in without `encryption` and gets the policy on disk. A client that asks for another policy raises `EncryptionPolicyMismatchError`, and so does a running client at its next sync when the file changed under it. Files at rest are read with the current policy, so to change the policy, delete the SyftBox (`client.delete_syftbox()`) and log in again. `syft-bg init --encryption ...` sets the policy the same way as a login. -Nothing is sent to a peer until its policy is known, that is, until it accepts and shares its version file. A scientist's change for a peer whose policy or bundle is not known is dropped, and the send raises. An owner's send to that peer raises. An owner's change to a file that the peer may read is not lost: the owner keeps a record of what each peer holds, and that record changes only after a send, so a later sync sends it. Other owner sends, such as a private dataset share or an enclave result, wait in memory, and are lost if the process exits. An owner keeps a message it cannot read yet in the inbox, with a warning. +Nothing is sent to a peer until its policy is known, that is, until it accepts and shares its version file. A scientist's change for a peer whose policy or bundle is not known is dropped, and the send raises. An owner's send to that peer raises. An owner's change to a file that the peer may read is not lost: the owner keeps a record of what each peer holds, and that record changes only after a send, so a later sync sends it. Other owner sends, such as a private dataset share or an enclave result, are not kept. A send that fails raises, and the send must be made again. An owner keeps a message it cannot read yet in the inbox, with a warning. The format of a message must match the pair: plaintext in a pair that encrypts is refused, and so is an envelope in a pair that does not. The owner archives such a message unread, and a scientist skips it, each with a warning. diff --git a/syft/sync/sync/datasite_owner_syncer.py b/syft/sync/sync/datasite_owner_syncer.py index 5df6bb0ccfd..69a6cf8620e 100644 --- a/syft/sync/sync/datasite_owner_syncer.py +++ b/syft/sync/sync/datasite_owner_syncer.py @@ -647,7 +647,12 @@ def _record_local_changes(self) -> None: """Update the file hashes from the local files. With ``persist_owner_state``, also write the changes to the owner's event log.""" file_change_events_message = self.event_cache.process_local_file_changes() - if file_change_events_message is None or not self.persist_owner_state: + if file_change_events_message is None: + return + self.event_cache.deliveries.drop_accepted( + str(e.path_in_datasite) for e in file_change_events_message.events + ) + if not self.persist_owner_state: return # Feed the events into rolling state so they flow through # incremental/full checkpoints. @@ -752,7 +757,8 @@ def _skip_accepted_from_others( An accepted file is withheld while ``syncable`` still has the hash that the change gave it. An accepted delete is withheld while the file stays - gone. A delete after a revoke still goes out. + gone and ``peer`` may still read its path. A delete after a revoke + still goes out. """ def from_other(path: str) -> bool: @@ -768,7 +774,11 @@ def from_other(path: str) -> bool: delete=[ p for p in plan.delete - if not (from_other(p) and accepted[p].hash is None) + if not ( + from_other(p) + and accepted[p].hash is None + and self.perm_context.can_read(p, peer) + ) ], ) @@ -897,9 +907,9 @@ def process_syftbox_events_queue(self): def _send_outbox_messages(self) -> None: """Send each message in the outbox queue, and record it in its peer's view. - A send that cannot go raises, and the messages from it on are not sent. - Their views stay as they are, so the plan of a later sync holds their - changes again. + A send that cannot go raises. Its message is dropped, so it cannot block + a later sync, and the messages after it are not sent. Their views stay + as they are, so the plan of a later sync holds their changes again. """ self.perm_context._reload() unsent = deque() @@ -907,11 +917,10 @@ def _send_outbox_messages(self) -> None: unsent.append(self.outbox_queue.get()) try: while unsent: - recipient, message = unsent[0] + recipient, message = unsent.popleft() status = self.connection_router.owner_write_event_messages_to_outbox( recipient, message ) - unsent.popleft() self._record_sent(recipient, message, status) finally: self._requeue_unplanned_messages(unsent) diff --git a/syft/sync/sync/delivery_tracker.py b/syft/sync/sync/delivery_tracker.py index 643cbb3467b..56099156c85 100644 --- a/syft/sync/sync/delivery_tracker.py +++ b/syft/sync/sync/delivery_tracker.py @@ -186,6 +186,21 @@ def _apply_changes_to_view( view[path] = new_hash +def _record_delivery( + state: DeliveryState, peer: str, changes: Mapping[str, str | None] +) -> None: + """Apply ``changes`` to ``peer``'s view. + + After a delete, ``peer`` no longer holds the version from before an accepted + change, so that change is no longer withheld from it. + """ + _apply_changes_to_view(state.views.setdefault(peer, {}), changes) + for path, new_hash in changes.items(): + change = state.accepted.get(path) + if new_hash is None and change is not None and peer in change.withheld_from: + change.withheld_from.remove(peer) + + class DeliveryTracker: """Per-peer views of confirmed file versions, for the data owner. @@ -224,6 +239,14 @@ def accept(self, changes: Mapping[str, AcceptedChange]) -> None: state.accepted.update(changes) self.store.save(state) + def drop_accepted(self, paths: Iterable[str]) -> None: + """Forget the accepted changes of ``paths``, which the owner changed.""" + with self.store.locked(): + state = self.store.load() + for path in paths: + state.accepted.pop(path, None) + self.store.save(state) + def accepted_changes(self) -> dict[str, AcceptedChange]: """The last change that the owner accepted from a peer, by path.""" return self.store.load().accepted @@ -279,7 +302,7 @@ def sent( with self.store.locked(): state = self.store.load() if status is SendStatus.DELIVERED: - _apply_changes_to_view(state.views.setdefault(peer, {}), changes) + _record_delivery(state, peer, changes) else: assert status is SendStatus.QUEUED, status state.queued[msg_id] = QueuedSend(peer=peer, changes=dict(changes)) @@ -294,7 +317,5 @@ def confirm(self, msg_ids: Iterable[str]) -> None: state = self.store.load() for msg_id in msg_ids: queued = state.queued.pop(msg_id) - _apply_changes_to_view( - state.views.setdefault(queued.peer, {}), queued.changes - ) + _record_delivery(state, queued.peer, queued.changes) self.store.save(state) diff --git a/tests/unit/test_delivery_tracker.py b/tests/unit/test_delivery_tracker.py index ddf33fe493d..151554dfc41 100644 --- a/tests/unit/test_delivery_tracker.py +++ b/tests/unit/test_delivery_tracker.py @@ -354,7 +354,7 @@ def test_upgrade_keeps_pending_private_share(): ] -def test_held_send_outside_plan_goes_out_later(monkeypatch): +def test_failed_send_outside_plan_dropped(monkeypatch): ds_manager, do_manager = _pair() def peer_not_ready(self, recipient, *args, **kwargs): @@ -368,9 +368,26 @@ def peer_not_ready(self, recipient, *args, **kwargs): _push_private_file(do_manager, ds_manager) do_manager.datasite_owner_syncer.process_syftbox_events_queue() - assert _sent_events(ds_manager, do_manager, "private/data.csv") == [ - (False, b"rows") - ] + assert _sent_events(ds_manager, do_manager, "private/data.csv") == [] + + +def test_failed_send_outside_plan_does_not_block_later_syncs(): + ds_manager, do_manager = _pair() + syncer = do_manager.datasite_owner_syncer + _write(do_manager, "private/data.csv", "rows") + message = syncer.event_cache.create_events_for_files( + {Path("private/data.csv"): b"rows"} + ) + syncer.queue_event_for_syftbox( + recipients=["stranger@example.com"], file_change_events_message=message + ) + with pytest.raises(SyftPeerNotReadyError): + syncer.process_syftbox_events_queue() + + syncer.perm_context.open("shared/").grant_read_access(ds_manager.email) + _write(do_manager, "shared/data.txt", "one") + do_manager.sync(auto_checkpoint=False) + assert _outbox_paths(ds_manager, do_manager) == ["shared/data.txt"] def test_private_dataset_sent_to_peer_never_revoked(): diff --git a/tests/unit/test_sync_manager.py b/tests/unit/test_sync_manager.py index 184861e0613..b6de19984fd 100644 --- a/tests/unit/test_sync_manager.py +++ b/tests/unit/test_sync_manager.py @@ -893,6 +893,106 @@ def test_accepted_delete_goes_only_to_submitter(): assert _events_for(peer_b_router, do_manager) == [("shared/data.txt", False)] +def _accept_setup(ds_manager, do_manager, peer_b_email: str): + """Peer A may write ``shared/``, peer B may read it, and both hold + ``shared/data.txt`` with "one". Returns the syncer, the folder and the peers.""" + syncer = do_manager.datasite_owner_syncer + shared = syncer.perm_context.open("shared/") + shared.grant_write_access(ds_manager.email) + shared.grant_read_access(peer_b_email) + peers = [ds_manager.email, peer_b_email] + data = do_manager.syftbox_folder / do_manager.email / "shared/data.txt" + data.parent.mkdir(parents=True, exist_ok=True) + data.write_text("one") + syncer.sync(peers) + ds_manager.sync() + return syncer, shared, peers + + +def _contents_for(router: ConnectionRouter, do_manager, path: str) -> list: + """The content of each event for ``path`` in the DO's outbox for ``router``'s + peer, None for a delete.""" + return [ + e.content + for m in router.watcher_get_events_messages(do_manager.email, None) + for e in m.events + if str(e.path_in_datasite) == path + ] + + +def test_revoke_after_accepted_delete_sends_delete(): + ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + use_in_memory_cache=False, + ) + peer_b_email = "peer_b@example.com" + peer_b_router = _peer_router(ds_manager, do_manager, peer_b_email) + syncer, shared, peers = _accept_setup(ds_manager, do_manager, peer_b_email) + + proposal = ProposedFileChangesMessage( + sender_email=ds_manager.email, + proposed_file_changes=[ + ProposedFileChange( + path_in_datasite="shared/data.txt", + old_hash=syncer.event_cache.file_hashes[Path("shared/data.txt")], + datasite_email=do_manager.email, + is_deleted=True, + ) + ], + ) + syncer.handle_proposed_filechange_events_message(ds_manager.email, proposal) + syncer.process_syftbox_events_queue() + syncer.sync(peers) + shared.revoke_read_access(peer_b_email) + syncer.sync(peers) + assert _events_for(peer_b_router, do_manager) == [ + ("shared/data.txt", False), + ("shared/data.txt", True), + ] + + +def test_owner_change_back_to_accepted_content_reaches_readers(): + ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + use_in_memory_cache=False, + ) + peer_b_email = "peer_b@example.com" + peer_b_router = _peer_router(ds_manager, do_manager, peer_b_email) + syncer, _, peers = _accept_setup(ds_manager, do_manager, peer_b_email) + data = do_manager.syftbox_folder / do_manager.email / "shared/data.txt" + + ds_manager._send_file_change(f"{do_manager.email}/shared/data.txt", "from A") + syncer.sync(peers) + data.write_text("two") + syncer.sync(peers) + data.write_text("from A") + syncer.sync(peers) + assert _contents_for(peer_b_router, do_manager, "shared/data.txt") == [ + "one", + "two", + "from A", + ] + + +def test_reader_granted_again_after_revoke_gets_accepted_file(): + ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + use_in_memory_cache=False, + ) + peer_b_email = "peer_b@example.com" + peer_b_router = _peer_router(ds_manager, do_manager, peer_b_email) + syncer, shared, peers = _accept_setup(ds_manager, do_manager, peer_b_email) + + ds_manager._send_file_change(f"{do_manager.email}/shared/data.txt", "from A") + syncer.sync(peers) + shared.revoke_read_access(peer_b_email) + syncer.sync(peers) + shared.grant_read_access(peer_b_email) + syncer.sync(peers) + assert _contents_for(peer_b_router, do_manager, "shared/data.txt") == [ + "one", + None, + "from A", + ] + + def test_default_collections_folder_picks_shareable_spec_regardless_of_order(): """The collections folder is chosen by owner_only, not by position.""" from syft.sync.syftbox_manager import default_collections_folder From d80b1979a09991a4d74843ce4b11958f35517bac Mon Sep 17 00:00:00 2001 From: Pedro Werneck Date: Mon, 26 Oct 2026 18:00:00 -0300 Subject: [PATCH 16/36] fix: let a peer that is not ready wait without blocking the others A planned send to a peer that is not ready now warns and skips that peer, so the other peers still get their changes. An accepted change is kept only while it is withheld from a peer, so the views file no longer grows with each accepted job. --- docs/connections.md | 2 +- syft/sync/sync/datasite_owner_syncer.py | 29 +++++++++++++++----- syft/sync/sync/delivery_tracker.py | 26 ++++++++++++++---- tests/unit/test_delivery_tracker.py | 35 +++++++++++++++++++++++++ tests/unit/test_encryption_policy.py | 4 +-- 5 files changed, 82 insertions(+), 14 deletions(-) diff --git a/docs/connections.md b/docs/connections.md index 64e81552e36..0feede8fe27 100644 --- a/docs/connections.md +++ b/docs/connections.md @@ -61,7 +61,7 @@ A pair encrypts when one side is `required`. A SyftBox has one policy, kept in `//private/syft_settings.json`. The first client to start on a new SyftBox writes it: the `encryption` it logged in with, or `optional`. A later client, the notebook or syft-bg, logs in without `encryption` and gets the policy on disk. A client that asks for another policy raises `EncryptionPolicyMismatchError`, and so does a running client at its next sync when the file changed under it. Files at rest are read with the current policy, so to change the policy, delete the SyftBox (`client.delete_syftbox()`) and log in again. `syft-bg init --encryption ...` sets the policy the same way as a login. -Nothing is sent to a peer until its policy is known, that is, until it accepts and shares its version file. A scientist's change for a peer whose policy or bundle is not known is dropped, and the send raises. An owner's send to that peer raises. An owner's change to a file that the peer may read is not lost: the owner keeps a record of what each peer holds, and that record changes only after a send, so a later sync sends it. Other owner sends, such as a private dataset share or an enclave result, are not kept. A send that fails raises, and the send must be made again. An owner keeps a message it cannot read yet in the inbox, with a warning. +Nothing is sent to a peer until its policy is known, that is, until it accepts and shares its version file. A scientist's change for a peer whose policy or bundle is not known is dropped, and the send raises. An owner's change for that peer waits, with a warning, and a later sync sends it. Another owner send to that peer raises. An owner's change to a file that the peer may read is not lost: the owner keeps a record of what each peer holds, and that record changes only after a send, so a later sync sends it. Other owner sends, such as a private dataset share or an enclave result, are not kept. A send that fails raises, and the send must be made again. An owner keeps a message it cannot read yet in the inbox, with a warning. The format of a message must match the pair: plaintext in a pair that encrypts is refused, and so is an envelope in a pair that does not. The owner archives such a message unread, and a scientist skips it, each with a warning. diff --git a/syft/sync/sync/datasite_owner_syncer.py b/syft/sync/sync/datasite_owner_syncer.py index 69a6cf8620e..61d4c17ab27 100644 --- a/syft/sync/sync/datasite_owner_syncer.py +++ b/syft/sync/sync/datasite_owner_syncer.py @@ -907,21 +907,38 @@ def process_syftbox_events_queue(self): def _send_outbox_messages(self) -> None: """Send each message in the outbox queue, and record it in its peer's view. - A send that cannot go raises. Its message is dropped, so it cannot block - a later sync, and the messages after it are not sent. Their views stay - as they are, so the plan of a later sync holds their changes again. + A planned send to a peer that is not ready warns, and the rest of that + peer's plan is skipped. Its view stays as it is, so a later plan holds + the changes again. Any other send that cannot go raises, and its + message is dropped. The planned messages after it are dropped too, and + the other messages after it go back on the queue. """ self.perm_context._reload() unsent = deque() while not self.outbox_queue.empty(): unsent.append(self.outbox_queue.get()) + not_ready = set() try: while unsent: recipient, message = unsent.popleft() - status = self.connection_router.owner_write_event_messages_to_outbox( - recipient, message + planned = ( + message.message_filepath.as_string() in self._planned_message_names ) - self._record_sent(recipient, message, status) + if planned and recipient in not_ready: + continue + try: + status = ( + self.connection_router.owner_write_event_messages_to_outbox( + recipient, message + ) + ) + except SyftPeerNotReadyError as e: + if not planned: + raise + not_ready.add(recipient) + warnings.warn(f"Changes for {recipient} wait for a later sync: {e}") + else: + self._record_sent(recipient, message, status) finally: self._requeue_unplanned_messages(unsent) diff --git a/syft/sync/sync/delivery_tracker.py b/syft/sync/sync/delivery_tracker.py index 56099156c85..ba51ad901f1 100644 --- a/syft/sync/sync/delivery_tracker.py +++ b/syft/sync/sync/delivery_tracker.py @@ -198,7 +198,16 @@ def _record_delivery( for path, new_hash in changes.items(): change = state.accepted.get(path) if new_hash is None and change is not None and peer in change.withheld_from: - change.withheld_from.remove(peer) + _stop_withholding(state, path, peer) + + +def _stop_withholding(state: DeliveryState, path: str, peer: str) -> None: + """Remove ``peer`` from the accepted change of ``path``, and drop the change + when it is withheld from nobody.""" + change = state.accepted[path] + change.withheld_from.remove(peer) + if not change.withheld_from: + del state.accepted[path] class DeliveryTracker: @@ -227,16 +236,23 @@ def reset(self, peer: str) -> None: state = self.store.load() view = state.views.get(peer, {}) state.views[peer] = dict.fromkeys(view, UNSENT_HASH) - for change in state.accepted.values(): + for path, change in list(state.accepted.items()): if peer in change.withheld_from: - change.withheld_from.remove(peer) + _stop_withholding(state, path, peer) self.store.save(state) def accept(self, changes: Mapping[str, AcceptedChange]) -> None: - """Record ``changes``, by path, as the last change accepted from a peer.""" + """Record ``changes``, by path, as the last change accepted from a peer. + + A change withheld from nobody is not kept. + """ with self.store.locked(): state = self.store.load() - state.accepted.update(changes) + for path, change in changes.items(): + if change.withheld_from: + state.accepted[path] = change + else: + state.accepted.pop(path, None) self.store.save(state) def drop_accepted(self, paths: Iterable[str]) -> None: diff --git a/tests/unit/test_delivery_tracker.py b/tests/unit/test_delivery_tracker.py index 151554dfc41..9cf8dbca417 100644 --- a/tests/unit/test_delivery_tracker.py +++ b/tests/unit/test_delivery_tracker.py @@ -72,6 +72,21 @@ def test_reset_stops_withholding_accepted_changes(): assert tracker.accepted_changes()["a"].withheld_from == ["b@x.com"] +def test_accepted_change_kept_only_while_withheld(): + tracker = DeliveryTracker(MemoryDeliveryStateStore()) + tracker.accept( + { + "a": AcceptedChange(hash="h1", withheld_from=[]), + "b": AcceptedChange(hash="h1", withheld_from=[PEER]), + "c": AcceptedChange(hash="h1", withheld_from=["c@x.com"]), + } + ) + assert sorted(tracker.accepted_changes()) == ["b", "c"] + tracker.sent(PEER, "m1", SendStatus.DELIVERED, {"b": None}) + tracker.reset("c@x.com") + assert tracker.accepted_changes() == {} + + def test_queued_send_updates_view_on_confirm(): tracker = DeliveryTracker(MemoryDeliveryStateStore()) tracker.sent(PEER, "m1", SendStatus.QUEUED, {"a": "h1"}) @@ -371,6 +386,26 @@ def peer_not_ready(self, recipient, *args, **kwargs): assert _sent_events(ds_manager, do_manager, "private/data.csv") == [] +def test_peer_not_ready_does_not_block_other_peers(): + ds_manager, do_manager = _pair() + syncer = do_manager.datasite_owner_syncer + not_ready = "stranger@example.com" + shared = syncer.perm_context.open("shared/") + shared.grant_read_access(not_ready) + shared.grant_read_access(ds_manager.email) + _write(do_manager, "shared/data.txt", "one") + + syncer._record_local_changes() + with pytest.warns(UserWarning, match=not_ready): + syncer.deliver([not_ready, ds_manager.email]) + assert _outbox_paths(ds_manager, do_manager) == ["shared/data.txt"] + # The skipped peer's view did not change, so its next plan sends the file. + readable = syncer._readable_hashes([not_ready]) + assert syncer.event_cache.deliveries.plan(readable)[not_ready].send == [ + "shared/data.txt" + ] + + def test_failed_send_outside_plan_does_not_block_later_syncs(): ds_manager, do_manager = _pair() syncer = do_manager.datasite_owner_syncer diff --git a/tests/unit/test_encryption_policy.py b/tests/unit/test_encryption_policy.py index 56cd8736859..14a1e5cf826 100644 --- a/tests/unit/test_encryption_policy.py +++ b/tests/unit/test_encryption_policy.py @@ -548,7 +548,7 @@ def send_message(self, recipient, message): def test_unsent_owner_change_stays_pending(monkeypatch): - """The send raises, and the next sync sends the change.""" + """The send warns, and the next sync sends the change.""" ds_manager, do_manager = _pair_with_unsent_do_file() def fail(self, recipient, message): @@ -556,7 +556,7 @@ def fail(self, recipient, message): with monkeypatch.context() as m: m.setattr(ConnectionRouter, "owner_write_event_messages_to_outbox", fail) - with pytest.raises(SyftPeerNotReadyError): + with pytest.warns(UserWarning, match="wait for a later sync"): do_manager.sync(auto_checkpoint=False) _assert_ds_receives_after_notebook_sync(ds_manager, do_manager) From a5eb5136c31b0cfa18a613f6de0b0b6d0bc75e13 Mon Sep 17 00:00:00 2001 From: Pedro Werneck Date: Mon, 26 Oct 2026 18:06:12 -0300 Subject: [PATCH 17/36] perf: load the permissions once per owner sync The DO sync loads the permission files once, and the seed, the plan, the send records and the accepted changes read that load. The plan and the send records thus see the same permissions. --- syft/sync/sync/datasite_owner_syncer.py | 10 ++++------ tests/unit/test_delivery_tracker.py | 20 ++++++++++++++++++++ 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/syft/sync/sync/datasite_owner_syncer.py b/syft/sync/sync/datasite_owner_syncer.py index 61d4c17ab27..48e41f566b1 100644 --- a/syft/sync/sync/datasite_owner_syncer.py +++ b/syft/sync/sync/datasite_owner_syncer.py @@ -240,6 +240,8 @@ def sync(self, peer_emails: list[str], recompute_hashes: bool = True): try: if not self.initial_sync_done: self.pull_initial_state() + # The one permission load of this pass. Each step below reads it. + self.perm_context._reload() # Before the local changes are recorded, so that a file changed # while the owner was offline is not seeded as held. self._seed_views_if_missing(peer_emails) @@ -699,7 +701,6 @@ def _seed_views_if_missing(self, peer_emails: list[str]) -> None: deliveries = self.event_cache.deliveries if deliveries.is_seeded(): return - self.perm_context._reload() approved = [p.email for p in self.connection_router.peer_store.approved_peers] deliveries.seed(self._held_before_seed(sorted({*approved, *peer_emails}))) @@ -721,10 +722,9 @@ def _held_before_seed(self, peers: list[str]) -> dict[str, dict[str, str]]: def deliver(self, peer_emails: list[str]) -> None: """Send each peer the files its view lacks, and deletes for the files it - may no longer have.""" + may no longer have. The caller loads the permissions first.""" deliveries = self.event_cache.deliveries deliveries.confirm(self.connection_router.transport.p2p.delivered()) - self.perm_context._reload() plans = deliveries.plan(self._readable_hashes(peer_emails)) accepted = deliveries.accepted_changes() syncable = self._syncable_hashes() @@ -863,9 +863,8 @@ def _accepted_changes( """The changes of ``message``, accepted from ``sender_email``, by path. Each change is withheld from the other approved peers that may read its - file now. + file now. The caller loads the permissions first. """ - self.perm_context._reload() others = [ p.email for p in self.connection_router.peer_store.approved_peers @@ -913,7 +912,6 @@ def _send_outbox_messages(self) -> None: message is dropped. The planned messages after it are dropped too, and the other messages after it go back on the queue. """ - self.perm_context._reload() unsent = deque() while not self.outbox_queue.empty(): unsent.append(self.outbox_queue.get()) diff --git a/tests/unit/test_delivery_tracker.py b/tests/unit/test_delivery_tracker.py index 9cf8dbca417..355f1b2547e 100644 --- a/tests/unit/test_delivery_tracker.py +++ b/tests/unit/test_delivery_tracker.py @@ -26,6 +26,7 @@ ) from syft.sync.syftbox_manager import SyftboxManager from syft.sync.transport.types import SendStatus +from syft_perms import SyftPermContext PEER = "ds@example.com" @@ -436,6 +437,25 @@ def test_private_dataset_sent_to_peer_never_revoked(): ] +def test_sync_reloads_permissions_once(monkeypatch): + ds_manager, do_manager = _pair() + do_manager.datasite_owner_syncer.perm_context.open("shared/").grant_read_access( + ds_manager.email + ) + _write(do_manager, "shared/data.txt", "one") + reload = SyftPermContext._reload + reloads = [] + + def counting_reload(self): + reloads.append(self) + reload(self) + + monkeypatch.setattr(SyftPermContext, "_reload", counting_reload) + do_manager.sync(auto_checkpoint=False) + assert _outbox_paths(ds_manager, do_manager) == ["shared/data.txt"] + assert len(reloads) == 1 + + def test_large_catch_up_split_into_messages(monkeypatch): monkeypatch.setattr(datasite_owner_syncer, "MAX_DELIVERY_MESSAGE_CONTENT", 6) ds_manager, do_manager = _pair() From 082ef4e499c840e5d882a1011a7a73ef62780f41 Mon Sep 17 00:00:00 2001 From: Pedro Werneck Date: Mon, 26 Oct 2026 19:03:31 -0300 Subject: [PATCH 18/36] refactor: streamline message handling in DatasiteOwnerSyncer and update tests --- syft/sync/sync/datasite_owner_syncer.py | 112 +++++++++++------------- tests/unit/test_encryption_policy.py | 5 +- 2 files changed, 55 insertions(+), 62 deletions(-) diff --git a/syft/sync/sync/datasite_owner_syncer.py b/syft/sync/sync/datasite_owner_syncer.py index 48e41f566b1..4a3b0b9f58b 100644 --- a/syft/sync/sync/datasite_owner_syncer.py +++ b/syft/sync/sync/datasite_owner_syncer.py @@ -1,45 +1,45 @@ import logging import warnings +from collections import deque +from concurrent.futures import ThreadPoolExecutor +from functools import partial from pathlib import Path +from queue import Queue +from typing import List, Mapping, NamedTuple, Tuple from uuid import uuid4 -from functools import partial -from pydantic import ConfigDict, Field, BaseModel, PrivateAttr -from concurrent.futures import ThreadPoolExecutor -from collections import deque -from queue import Queue -from typing import Iterable, List, Mapping, NamedTuple, Tuple -from syft.sync.events.file_change_event import ( - FileChangeEventsMessage, - FileChangeEventsMessageFileName, - FileChangeEvent, -) -from syft.sync.connections.base_connection import ( - ConnectionConfig, - FileCollection, -) -from syft.sync.sync.caches.datasite_owner_cache import ( - DataSiteOwnerEventCacheConfig, -) -from syft.sync.connections.connection_router import ConnectionRouter -from syft.sync.peers.exceptions import SyftPeerNotReadyError -from syft.sync.sync.caches.datasite_owner_cache import DataSiteOwnerEventCache -from syft.sync.sync.collection_spec import CollectionSyncSpec +from pydantic import BaseModel, ConfigDict, Field, PrivateAttr +from syft_perms import SyftPermContext + from syft.sync.callback_mixin import BaseModelCallbackMixin -from syft.sync.messages.proposed_filechange import ProposedFileChangesMessage -from syft.sync.utils.path_filters import is_normal_syncable_path from syft.sync.checkpoints.checkpoint import ( + DEFAULT_COMPACTING_THRESHOLD, Checkpoint, CheckpointFile, IncrementalCheckpoint, compact_incremental_checkpoints, - DEFAULT_COMPACTING_THRESHOLD, ) from syft.sync.checkpoints.rolling_state import ( RollingState, raise_for_later_version, ) -from syft_perms import SyftPermContext +from syft.sync.connections.base_connection import ( + ConnectionConfig, + FileCollection, +) +from syft.sync.connections.connection_router import ConnectionRouter +from syft.sync.events.file_change_event import ( + FileChangeEvent, + FileChangeEventsMessage, + FileChangeEventsMessageFileName, +) +from syft.sync.messages.proposed_filechange import ProposedFileChangesMessage +from syft.sync.peers.exceptions import SyftPeerNotReadyError +from syft.sync.sync.caches.datasite_owner_cache import ( + DataSiteOwnerEventCache, + DataSiteOwnerEventCacheConfig, +) +from syft.sync.sync.collection_spec import CollectionSyncSpec from syft.sync.sync.constants import CACHE_DIR, ROLLING_STATE_FILENAME from syft.sync.sync.delivery_tracker import ( UNSENT_HASH, @@ -47,6 +47,7 @@ DeliveryPlan, ) from syft.sync.transport.types import SendStatus +from syft.sync.utils.path_filters import is_normal_syncable_path from syft.sync.utils.syftbox_utils import ( create_event_timestamp, get_event_hash_from_content, @@ -120,6 +121,13 @@ class DatasiteOwnerSyncerConfig(BaseModel): connection_configs: List[ConnectionConfig] = [] +class OutgoingMessage(NamedTuple): + recipient: str + message: FileChangeEventsMessage + # Whether `deliver` queued the message from the views of `recipient`. + planned: bool + + def _batches_by_content_size( events: list[FileChangeEvent], limit: int ) -> list[list[FileChangeEvent]]: @@ -159,17 +167,13 @@ class DatasiteOwnerSyncer(BaseModelCallbackMixin): collection_specs: List[CollectionSyncSpec] = [] syftbox_events_queue: Queue[FileChangeEventsMessage] = Field(default_factory=Queue) - outbox_queue: Queue[Tuple[str, FileChangeEventsMessage]] = Field( - default_factory=Queue - ) + outbox_queue: Queue[OutgoingMessage] = Field(default_factory=Queue) _executor: ThreadPoolExecutor = PrivateAttr( default_factory=lambda: ThreadPoolExecutor(max_workers=10) ) # Cache of collections shared with "any" - list of (tag, content_hash) tuples _any_shared_collections: List[tuple] = PrivateAttr(default_factory=list) - # Names of the outbox messages that `deliver` queued and that are not sent yet - _planned_message_names: set[str] = PrivateAttr(default_factory=set) # In-memory rolling state for tracking events since last checkpoint _rolling_state: RollingState | None = PrivateAttr(default=None) @@ -735,15 +739,14 @@ def deliver(self, peer_emails: list[str]) -> None: # Build every message before the first one is queued, so a read error # leaves no message of this plan in the queue. messages = [ - (peer, FileChangeEventsMessage(events=batch)) + OutgoingMessage(peer, FileChangeEventsMessage(events=batch), planned=True) for peer, plan in plans.items() for batch in _batches_by_content_size( self._plan_events(plan), MAX_DELIVERY_MESSAGE_CONTENT ) ] - for peer, message in messages: - self._planned_message_names.add(message.message_filepath.as_string()) - self.outbox_queue.put((peer, message)) + for message in messages: + self.outbox_queue.put(message) self.process_syftbox_events_queue() def _skip_accepted_from_others( @@ -891,7 +894,9 @@ def queue_event_for_syftbox( self.syftbox_events_queue.put(file_change_events_message) for recipient in recipients: - self.outbox_queue.put((recipient, file_change_events_message)) + self.outbox_queue.put( + OutgoingMessage(recipient, file_change_events_message, planned=False) + ) def process_syftbox_events_queue(self): # TODO: make this atomic @@ -904,13 +909,11 @@ def process_syftbox_events_queue(self): self._send_outbox_messages() def _send_outbox_messages(self) -> None: - """Send each message in the outbox queue, and record it in its peer's view. + """Send the queued messages, and record each sent one in its peer's view. - A planned send to a peer that is not ready warns, and the rest of that - peer's plan is skipped. Its view stays as it is, so a later plan holds - the changes again. Any other send that cannot go raises, and its - message is dropped. The planned messages after it are dropped too, and - the other messages after it go back on the queue. + A peer that is not ready gets none of its planned messages. They wait + for the next plan, with a warning. Any other failed send raises. Only + the unplanned messages after it go back on the queue. """ unsent = deque() while not self.outbox_queue.empty(): @@ -918,10 +921,7 @@ def _send_outbox_messages(self) -> None: not_ready = set() try: while unsent: - recipient, message = unsent.popleft() - planned = ( - message.message_filepath.as_string() in self._planned_message_names - ) + recipient, message, planned = unsent.popleft() if planned and recipient in not_ready: continue try: @@ -938,21 +938,11 @@ def _send_outbox_messages(self) -> None: else: self._record_sent(recipient, message, status) finally: - self._requeue_unplanned_messages(unsent) - - def _requeue_unplanned_messages( - self, unsent: Iterable[Tuple[str, FileChangeEventsMessage]] - ) -> None: - """Put back, in order, the ``unsent`` messages that ``deliver`` did not queue. - - A message that ``deliver`` queued is dropped: a later send would not - check the views or the permissions again, so ``deliver`` plans it - again from the views instead. - """ - for recipient, message in unsent: - if message.message_filepath.as_string() not in self._planned_message_names: - self.outbox_queue.put((recipient, message)) - self._planned_message_names.clear() + # A later send of a planned message would not check the views or the + # permissions again, so `deliver` plans it again instead. + for entry in unsent: + if not entry.planned: + self.outbox_queue.put(entry) def _record_sent( self, recipient: str, message: FileChangeEventsMessage, status: SendStatus diff --git a/tests/unit/test_encryption_policy.py b/tests/unit/test_encryption_policy.py index 14a1e5cf826..295e5e3ebc8 100644 --- a/tests/unit/test_encryption_policy.py +++ b/tests/unit/test_encryption_policy.py @@ -27,6 +27,7 @@ resolve_encryption_policy, ) from syft.sync.syftbox_manager import SyftboxManager +from syft.sync.sync.datasite_owner_syncer import OutgoingMessage from syft.sync.version.peer_info import PeerInfo from syft.sync.version.peer_manager import PeerManager, PeerManagerConfig from syft.sync.version.version_info import VersionInfo @@ -433,7 +434,9 @@ def test_owner_send_to_non_peer_raises(): ) syncer = do_manager.datasite_owner_syncer message = FileChangeEventsMessage(events=[]) - syncer.outbox_queue.put(("former@example.com", message)) + syncer.outbox_queue.put( + OutgoingMessage("former@example.com", message, planned=False) + ) with pytest.raises(SyftPeerNotReadyError, match="not known"): syncer.process_syftbox_events_queue() From f16d32c7bf0543d8466ddfd9909fd96c44c26f2c Mon Sep 17 00:00:00 2001 From: Koen van der Veen Date: Tue, 27 Oct 2026 12:43:22 +0100 Subject: [PATCH 19/36] refactor: clearer names and named intermediate values in delivery tracking --- syft/sync/sync/datasite_owner_syncer.py | 25 ++++++++++++++++--------- syft/sync/sync/delivery_tracker.py | 7 ++++--- tests/unit/test_delivery_tracker.py | 14 +++++++------- 3 files changed, 27 insertions(+), 19 deletions(-) diff --git a/syft/sync/sync/datasite_owner_syncer.py b/syft/sync/sync/datasite_owner_syncer.py index 4a3b0b9f58b..93233ffd684 100644 --- a/syft/sync/sync/datasite_owner_syncer.py +++ b/syft/sync/sync/datasite_owner_syncer.py @@ -655,9 +655,10 @@ def _record_local_changes(self) -> None: file_change_events_message = self.event_cache.process_local_file_changes() if file_change_events_message is None: return - self.event_cache.deliveries.drop_accepted( + changed_paths = [ str(e.path_in_datasite) for e in file_change_events_message.events - ) + ] + self.event_cache.deliveries.drop_accepted(changed_paths) if not self.persist_owner_state: return # Feed the events into rolling state so they flow through @@ -679,7 +680,9 @@ def _syncable_hashes(self) -> dict[str, str]: if self._is_syncable(str(path)) } - def _readable_hashes(self, peer_emails: list[str]) -> dict[str, dict[str, str]]: + def _readable_files_by_peer( + self, peer_emails: list[str] + ) -> dict[str, dict[str, str]]: """For each peer, the current hash of every syncable file it may read. The caller reloads the permissions first. @@ -706,9 +709,12 @@ def _seed_views_if_missing(self, peer_emails: list[str]) -> None: if deliveries.is_seeded(): return approved = [p.email for p in self.connection_router.peer_store.approved_peers] - deliveries.seed(self._held_before_seed(sorted({*approved, *peer_emails}))) + peers_to_seed = sorted({*approved, *peer_emails}) + deliveries.seed(self._files_held_by_peer_at_seed(peers_to_seed)) - def _held_before_seed(self, peers: list[str]) -> dict[str, dict[str, str]]: + def _files_held_by_peer_at_seed( + self, peers: list[str] + ) -> dict[str, dict[str, str]]: """What each of ``peers`` is taken to hold when the views are seeded. Each peer holds every file it may read now. A syncable path that an @@ -718,10 +724,11 @@ def _held_before_seed(self, peers: list[str]) -> dict[str, dict[str, str]]: delete them. The caller reloads the permissions first. """ pending = self.event_cache.legacy_pending_paths() - held = self._readable_hashes(peers) + held = self._readable_files_by_peer(peers) for peer, files in held.items(): unsent = [p for p in pending.get(peer, ()) if self._is_syncable(p)] - files.update(dict.fromkeys(unsent, UNSENT_HASH)) + unsent_files = dict.fromkeys(unsent, UNSENT_HASH) + files.update(unsent_files) return held def deliver(self, peer_emails: list[str]) -> None: @@ -729,7 +736,7 @@ def deliver(self, peer_emails: list[str]) -> None: may no longer have. The caller loads the permissions first.""" deliveries = self.event_cache.deliveries deliveries.confirm(self.connection_router.transport.p2p.delivered()) - plans = deliveries.plan(self._readable_hashes(peer_emails)) + plans = deliveries.plan(self._readable_files_by_peer(peer_emails)) accepted = deliveries.accepted_changes() syncable = self._syncable_hashes() plans = { @@ -962,7 +969,7 @@ def _record_sent( changes[path] = None elif self._may_hold(recipient, path, syncable): changes[path] = event.new_hash - self.event_cache.deliveries.sent( + self.event_cache.deliveries.record_send( recipient, message.message_filepath.as_string(), status, changes ) diff --git a/syft/sync/sync/delivery_tracker.py b/syft/sync/sync/delivery_tracker.py index ba51ad901f1..6538b9c16ff 100644 --- a/syft/sync/sync/delivery_tracker.py +++ b/syft/sync/sync/delivery_tracker.py @@ -35,7 +35,7 @@ Recording a send ---------------- -``sent`` takes the ``SendStatus`` from ``P2PConnection.send``: +``record_send`` takes the ``SendStatus`` from ``P2PConnection.send``: - ``DELIVERED``: the view changes at once. - ``QUEUED``: the changes are held under their ``msg_id``, and the view changes @@ -222,7 +222,8 @@ def __init__(self, store: DeliveryStateStore): self.store = store def clear(self) -> None: - """Forget every view and every queued send.""" + """Forget the whole state: views, queued sends, accepted changes and the + seeded flag.""" with self.store.locked(): self.store.save(DeliveryState()) @@ -304,7 +305,7 @@ def plan( ) return plans - def sent( + def record_send( self, peer: str, msg_id: str, diff --git a/tests/unit/test_delivery_tracker.py b/tests/unit/test_delivery_tracker.py index 355f1b2547e..bb9c5a89232 100644 --- a/tests/unit/test_delivery_tracker.py +++ b/tests/unit/test_delivery_tracker.py @@ -33,7 +33,7 @@ def _tracker_holding(view: dict[str, str]) -> DeliveryTracker: tracker = DeliveryTracker(MemoryDeliveryStateStore()) - tracker.sent(PEER, "m0", SendStatus.DELIVERED, view) + tracker.record_send(PEER, "m0", SendStatus.DELIVERED, view) return tracker @@ -46,7 +46,7 @@ def test_plan_sends_changed_files_deletes_unreadable(): def test_delivered_send_updates_view(): tracker = _tracker_holding({"a": "h1", "b": "h1"}) - tracker.sent(PEER, "m1", SendStatus.DELIVERED, {"a": "h2", "b": None}) + tracker.record_send(PEER, "m1", SendStatus.DELIVERED, {"a": "h2", "b": None}) assert tracker.plan({PEER: {"a": "h2"}})[PEER].send == [] assert tracker.plan({PEER: {"a": "h2"}})[PEER].delete == [] @@ -83,14 +83,14 @@ def test_accepted_change_kept_only_while_withheld(): } ) assert sorted(tracker.accepted_changes()) == ["b", "c"] - tracker.sent(PEER, "m1", SendStatus.DELIVERED, {"b": None}) + tracker.record_send(PEER, "m1", SendStatus.DELIVERED, {"b": None}) tracker.reset("c@x.com") assert tracker.accepted_changes() == {} def test_queued_send_updates_view_on_confirm(): tracker = DeliveryTracker(MemoryDeliveryStateStore()) - tracker.sent(PEER, "m1", SendStatus.QUEUED, {"a": "h1"}) + tracker.record_send(PEER, "m1", SendStatus.QUEUED, {"a": "h1"}) assert tracker.plan({PEER: {"a": "h1"}})[PEER].send == ["a"] tracker.confirm(["m1"]) assert tracker.plan({PEER: {"a": "h1"}})[PEER].send == [] @@ -98,7 +98,7 @@ def test_queued_send_updates_view_on_confirm(): def test_local_store_persists_state_refuses_later_version(tmp_path): path = tmp_path / DELIVERY_VIEWS_FILENAME - DeliveryTracker(LocalDeliveryStateStore(path)).sent( + DeliveryTracker(LocalDeliveryStateStore(path)).record_send( PEER, "m1", SendStatus.DELIVERED, {"a": "h1"} ) assert ( @@ -130,7 +130,7 @@ def load_then_wait_in_first_writer(self): def send(peer: str): tracker = DeliveryTracker(LocalDeliveryStateStore(path)) - tracker.sent(peer, f"m-{peer}", SendStatus.DELIVERED, {"a": "h1"}) + tracker.record_send(peer, f"m-{peer}", SendStatus.DELIVERED, {"a": "h1"}) first = threading.Thread(target=send, args=("one@x.com",), name="first") second = threading.Thread(target=send, args=("two@x.com",), name="second") @@ -401,7 +401,7 @@ def test_peer_not_ready_does_not_block_other_peers(): syncer.deliver([not_ready, ds_manager.email]) assert _outbox_paths(ds_manager, do_manager) == ["shared/data.txt"] # The skipped peer's view did not change, so its next plan sends the file. - readable = syncer._readable_hashes([not_ready]) + readable = syncer._readable_files_by_peer([not_ready]) assert syncer.event_cache.deliveries.plan(readable)[not_ready].send == [ "shared/data.txt" ] From d1f15b01fa6547804db00969e4252b905a071e7f Mon Sep 17 00:00:00 2001 From: Koen van der Veen Date: Tue, 27 Oct 2026 12:51:02 +0100 Subject: [PATCH 20/36] refactor: rename deliver to deliver_file_changes --- syft/sync/sync/datasite_owner_syncer.py | 8 ++++---- tests/unit/test_delivery_tracker.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/syft/sync/sync/datasite_owner_syncer.py b/syft/sync/sync/datasite_owner_syncer.py index 93233ffd684..c1c6d0fdfa8 100644 --- a/syft/sync/sync/datasite_owner_syncer.py +++ b/syft/sync/sync/datasite_owner_syncer.py @@ -124,7 +124,7 @@ class DatasiteOwnerSyncerConfig(BaseModel): class OutgoingMessage(NamedTuple): recipient: str message: FileChangeEventsMessage - # Whether `deliver` queued the message from the views of `recipient`. + # Whether `deliver_file_changes` queued the message from the views of `recipient`. planned: bool @@ -252,7 +252,7 @@ def sync(self, peer_emails: list[str], recompute_hashes: bool = True): if recompute_hashes: self._record_local_changes() - self.deliver(peer_emails) + self.deliver_file_changes(peer_emails) for peer_email in peer_emails: proposed_count = 0 @@ -731,7 +731,7 @@ def _files_held_by_peer_at_seed( files.update(unsent_files) return held - def deliver(self, peer_emails: list[str]) -> None: + def deliver_file_changes(self, peer_emails: list[str]) -> None: """Send each peer the files its view lacks, and deletes for the files it may no longer have. The caller loads the permissions first.""" deliveries = self.event_cache.deliveries @@ -946,7 +946,7 @@ def _send_outbox_messages(self) -> None: self._record_sent(recipient, message, status) finally: # A later send of a planned message would not check the views or the - # permissions again, so `deliver` plans it again instead. + # permissions again, so `deliver_file_changes` plans it again instead. for entry in unsent: if not entry.planned: self.outbox_queue.put(entry) diff --git a/tests/unit/test_delivery_tracker.py b/tests/unit/test_delivery_tracker.py index bb9c5a89232..384096d640b 100644 --- a/tests/unit/test_delivery_tracker.py +++ b/tests/unit/test_delivery_tracker.py @@ -398,7 +398,7 @@ def test_peer_not_ready_does_not_block_other_peers(): syncer._record_local_changes() with pytest.warns(UserWarning, match=not_ready): - syncer.deliver([not_ready, ds_manager.email]) + syncer.deliver_file_changes([not_ready, ds_manager.email]) assert _outbox_paths(ds_manager, do_manager) == ["shared/data.txt"] # The skipped peer's view did not change, so its next plan sends the file. readable = syncer._readable_files_by_peer([not_ready]) @@ -539,7 +539,7 @@ def fail_second_read(self, path): fail_second_read, ) with pytest.raises(PermissionError): - syncer.deliver([ds_manager.email, "peer_b@example.com"]) + syncer.deliver_file_changes([ds_manager.email, "peer_b@example.com"]) assert syncer.outbox_queue.empty() From f6ff3d028540958db6a726504434cacae7c4ed38 Mon Sep 17 00:00:00 2001 From: Koen van der Veen Date: Tue, 27 Oct 2026 13:10:04 +0100 Subject: [PATCH 21/36] refactor: rename _create_current_state_event to _create_current_filestate_event --- syft/sync/sync/datasite_owner_syncer.py | 4 ++-- tests/unit/test_delivery_tracker.py | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/syft/sync/sync/datasite_owner_syncer.py b/syft/sync/sync/datasite_owner_syncer.py index c1c6d0fdfa8..e07d4aab186 100644 --- a/syft/sync/sync/datasite_owner_syncer.py +++ b/syft/sync/sync/datasite_owner_syncer.py @@ -613,7 +613,7 @@ def _download_file_with_new_connection(self, file_id: str) -> bytes: connection = self.connection_router.connection_for_parallel_download() return connection.watcher_download_collection_file(file_id) - def _create_current_state_event(self, path: str) -> FileChangeEvent: + def _create_current_filestate_event(self, path: str) -> FileChangeEvent: """An event that sets ``path`` on a peer to its current local state. A file missing from the cache gives a deletion. A read error raises. @@ -794,7 +794,7 @@ def from_other(path: str) -> bool: def _plan_events(self, plan: DeliveryPlan) -> list[FileChangeEvent]: """The events that carry out ``plan``: its sends, then its deletes.""" - events = [self._create_current_state_event(p) for p in plan.send] + events = [self._create_current_filestate_event(p) for p in plan.send] return events + [self._deletion_event(p) for p in plan.delete] def pull_and_process_next_proposed_filechange( diff --git a/tests/unit/test_delivery_tracker.py b/tests/unit/test_delivery_tracker.py index 384096d640b..aded9264213 100644 --- a/tests/unit/test_delivery_tracker.py +++ b/tests/unit/test_delivery_tracker.py @@ -525,7 +525,9 @@ def test_read_error_leaves_no_planned_message(monkeypatch): # The plan reads the file once per peer. The read for the second peer fails. reads = [] - create_event = datasite_owner_syncer.DatasiteOwnerSyncer._create_current_state_event + create_event = ( + datasite_owner_syncer.DatasiteOwnerSyncer._create_current_filestate_event + ) def fail_second_read(self, path): reads.append(path) @@ -535,7 +537,7 @@ def fail_second_read(self, path): monkeypatch.setattr( datasite_owner_syncer.DatasiteOwnerSyncer, - "_create_current_state_event", + "_create_current_filestate_event", fail_second_read, ) with pytest.raises(PermissionError): From a4825bcc886638de7d761a366a554d8eab85fc21 Mon Sep 17 00:00:00 2001 From: Koen van der Veen Date: Tue, 27 Oct 2026 13:12:24 +0100 Subject: [PATCH 22/36] refactor: rename confirm to confirm_deliveries --- syft/sync/sync/datasite_owner_syncer.py | 2 +- syft/sync/sync/delivery_tracker.py | 6 +++--- tests/unit/test_delivery_tracker.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/syft/sync/sync/datasite_owner_syncer.py b/syft/sync/sync/datasite_owner_syncer.py index e07d4aab186..8ceed20fc84 100644 --- a/syft/sync/sync/datasite_owner_syncer.py +++ b/syft/sync/sync/datasite_owner_syncer.py @@ -735,7 +735,7 @@ def deliver_file_changes(self, peer_emails: list[str]) -> None: """Send each peer the files its view lacks, and deletes for the files it may no longer have. The caller loads the permissions first.""" deliveries = self.event_cache.deliveries - deliveries.confirm(self.connection_router.transport.p2p.delivered()) + deliveries.confirm_deliveries(self.connection_router.transport.p2p.delivered()) plans = deliveries.plan(self._readable_files_by_peer(peer_emails)) accepted = deliveries.accepted_changes() syncable = self._syncable_hashes() diff --git a/syft/sync/sync/delivery_tracker.py b/syft/sync/sync/delivery_tracker.py index 6538b9c16ff..36da1860238 100644 --- a/syft/sync/sync/delivery_tracker.py +++ b/syft/sync/sync/delivery_tracker.py @@ -40,7 +40,7 @@ - ``DELIVERED``: the view changes at once. - ``QUEUED``: the changes are held under their ``msg_id``, and the view changes only when ``P2PConnection.delivered()`` reports that ``msg_id`` and - ``confirm`` is called. A receipt is not enough; only the peer's ack counts. + ``confirm_deliveries`` is called. A receipt is not enough; only the peer's ack counts. A view therefore never changes before a delivery is confirmed. A change that is not sent, because the upload failed, the peer was skipped, or a process @@ -314,7 +314,7 @@ def record_send( ) -> None: """Record a send of ``changes`` (path -> new hash, None for a delete). - ``DELIVERED`` updates the view now; ``QUEUED`` waits for ``confirm``. + ``DELIVERED`` updates the view now; ``QUEUED`` waits for ``confirm_deliveries``. """ with self.store.locked(): state = self.store.load() @@ -325,7 +325,7 @@ def record_send( state.queued[msg_id] = QueuedSend(peer=peer, changes=dict(changes)) self.store.save(state) - def confirm(self, msg_ids: Iterable[str]) -> None: + def confirm_deliveries(self, msg_ids: Iterable[str]) -> None: """Apply the queued sends that ``P2PConnection.delivered()`` reported.""" msg_ids = list(msg_ids) if not msg_ids: diff --git a/tests/unit/test_delivery_tracker.py b/tests/unit/test_delivery_tracker.py index aded9264213..1c97f7c7482 100644 --- a/tests/unit/test_delivery_tracker.py +++ b/tests/unit/test_delivery_tracker.py @@ -92,7 +92,7 @@ def test_queued_send_updates_view_on_confirm(): tracker = DeliveryTracker(MemoryDeliveryStateStore()) tracker.record_send(PEER, "m1", SendStatus.QUEUED, {"a": "h1"}) assert tracker.plan({PEER: {"a": "h1"}})[PEER].send == ["a"] - tracker.confirm(["m1"]) + tracker.confirm_deliveries(["m1"]) assert tracker.plan({PEER: {"a": "h1"}})[PEER].send == [] From 3ee33120b6d00b52d29553a639fff9d33a966e58 Mon Sep 17 00:00:00 2001 From: Koen van der Veen Date: Tue, 27 Oct 2026 13:15:34 +0100 Subject: [PATCH 23/36] docs: say what maps to what for nested delivery dicts --- syft/sync/sync/caches/datasite_owner_cache.py | 3 ++- syft/sync/sync/datasite_owner_syncer.py | 7 +++++-- syft/sync/sync/delivery_tracker.py | 10 ++++++---- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/syft/sync/sync/caches/datasite_owner_cache.py b/syft/sync/sync/caches/datasite_owner_cache.py index 31269c6aee3..a384b5ad723 100644 --- a/syft/sync/sync/caches/datasite_owner_cache.py +++ b/syft/sync/sync/caches/datasite_owner_cache.py @@ -325,7 +325,7 @@ def legacy_pending_paths(self) -> dict[str, set[str]]: """The paths that a client before the delivery tracker still had to send, per peer. - Empty when the views are not kept on disk, or when that client left no + Maps each peer email to the set of paths. Empty when the views are not kept on disk, or when that client left no file. """ store = self.deliveries.store @@ -335,6 +335,7 @@ def legacy_pending_paths(self) -> dict[str, set[str]]: path = store.path.parent / OWNER_PENDING_DELIVERIES_FILENAME if not path.exists(): return {} + # peer email -> {path} return {peer: set(paths) for peer, paths in PersistedDict(path=path).items()} def has_conflict(self, proposed_event: ProposedFileChange) -> bool: diff --git a/syft/sync/sync/datasite_owner_syncer.py b/syft/sync/sync/datasite_owner_syncer.py index 8ceed20fc84..dcd98cf5d86 100644 --- a/syft/sync/sync/datasite_owner_syncer.py +++ b/syft/sync/sync/datasite_owner_syncer.py @@ -685,9 +685,11 @@ def _readable_files_by_peer( ) -> dict[str, dict[str, str]]: """For each peer, the current hash of every syncable file it may read. - The caller reloads the permissions first. + Maps each peer email to ``{path: hash}``. The caller reloads the + permissions first. """ syncable = self._syncable_hashes() + # peer email -> {path: hash} return { peer: { p: h for p, h in syncable.items() if self._may_hold(peer, p, syncable) @@ -717,13 +719,14 @@ def _files_held_by_peer_at_seed( ) -> dict[str, dict[str, str]]: """What each of ``peers`` is taken to hold when the views are seeded. - Each peer holds every file it may read now. A syncable path that an + Maps each peer email to ``{path: hash}``. Each peer holds every file it may read now. A syncable path that an earlier client still had to send gets ``UNSENT_HASH``, so the next plan sends or deletes it. Other pending paths, such as a private dataset share, stay out of the view: a plan never sends them, so it must not delete them. The caller reloads the permissions first. """ pending = self.event_cache.legacy_pending_paths() + # peer email -> {path: hash} held = self._readable_files_by_peer(peers) for peer, files in held.items(): unsent = [p for p in pending.get(peer, ()) if self._is_syncable(p)] diff --git a/syft/sync/sync/delivery_tracker.py b/syft/sync/sync/delivery_tracker.py index 36da1860238..9ce59d31557 100644 --- a/syft/sync/sync/delivery_tracker.py +++ b/syft/sync/sync/delivery_tracker.py @@ -100,7 +100,7 @@ class DeliveryState(BaseModel): """Everything the tracker keeps.""" version: int = DELIVERY_STATE_VERSION - # peer -> {path: hash} + # peer email -> {path: hash} views: dict[str, dict[str, str]] = Field(default_factory=dict) # msg_id -> the send it confirms queued: dict[str, QueuedSend] = Field(default_factory=dict) @@ -275,13 +275,14 @@ def is_seeded(self) -> bool: def seed(self, held_by_peer: Mapping[str, Mapping[str, str]]) -> None: """Add the files in ``held_by_peer`` to each peer's view, one time. - A path that the view already holds keeps its hash. After the first + ``held_by_peer`` maps each peer email to ``{path: hash}``. A path that the view already holds keeps its hash. After the first call, later calls do nothing. """ with self.store.locked(): state = self.store.load() if state.seeded: return + # held_by_peer: peer email -> {path: hash} for peer, files in held_by_peer.items(): state.views[peer] = {**files, **state.views.get(peer, {})} state.seeded = True @@ -292,11 +293,12 @@ def plan( ) -> dict[str, DeliveryPlan]: """What to send and delete for each peer in ``readable_by_peer``. - ``readable_by_peer`` maps each peer to every path it may read now, with - the path's current hash. + ``readable_by_peer`` maps each peer email to ``{path: hash}``: every + path it may read now, with the path's current hash. """ views = self.store.load().views plans = {} + # readable_by_peer: peer email -> {path: hash} for peer, readable in readable_by_peer.items(): view = views.get(peer, {}) plans[peer] = DeliveryPlan( From 2b8cfbd29e0e5f4de7e7cd78f901275ab429a622 Mon Sep 17 00:00:00 2001 From: Koen van der Veen Date: Tue, 27 Oct 2026 13:57:37 +0100 Subject: [PATCH 24/36] refactor: stop reading the pre-tracker pending deliveries file Seeding still runs when the views file is missing, for an owner that wipes its local state but restores its file hashes from Drive. --- syft/sync/sync/caches/datasite_owner_cache.py | 18 ------ syft/sync/sync/constants.py | 2 - syft/sync/sync/datasite_owner_syncer.py | 28 ++------- syft/sync/sync/delivery_tracker.py | 10 ++-- tests/unit/test_delivery_tracker.py | 59 ++++--------------- 5 files changed, 19 insertions(+), 98 deletions(-) diff --git a/syft/sync/sync/caches/datasite_owner_cache.py b/syft/sync/sync/caches/datasite_owner_cache.py index a384b5ad723..b735c397fe6 100644 --- a/syft/sync/sync/caches/datasite_owner_cache.py +++ b/syft/sync/sync/caches/datasite_owner_cache.py @@ -21,7 +21,6 @@ from syft.sync.sync.constants import ( CACHE_DIR, OWNER_FILE_HASHES_FILENAME, - OWNER_PENDING_DELIVERIES_FILENAME, ) from syft.sync.sync.delivery_tracker import ( DELIVERY_VIEWS_FILENAME, @@ -321,23 +320,6 @@ def clear_cache(self): self.file_hashes.clear() self.deliveries.clear() - def legacy_pending_paths(self) -> dict[str, set[str]]: - """The paths that a client before the delivery tracker still had to send, - per peer. - - Maps each peer email to the set of paths. Empty when the views are not kept on disk, or when that client left no - file. - """ - store = self.deliveries.store - if not isinstance(store, LocalDeliveryStateStore): - return {} - # The earlier client kept its file next to the views file. - path = store.path.parent / OWNER_PENDING_DELIVERIES_FILENAME - if not path.exists(): - return {} - # peer email -> {path} - return {peer: set(paths) for peer, paths in PersistedDict(path=path).items()} - def has_conflict(self, proposed_event: ProposedFileChange) -> bool: if proposed_event.path_in_datasite not in self.file_hashes: if proposed_event.old_hash is None: diff --git a/syft/sync/sync/constants.py b/syft/sync/sync/constants.py index 1972c14723b..2e8508efb4f 100644 --- a/syft/sync/sync/constants.py +++ b/syft/sync/sync/constants.py @@ -1,6 +1,4 @@ CACHE_DIR = ".cache" OWNER_FILE_HASHES_FILENAME = "owner_file_hashes.json" -# Written by clients before the delivery tracker. Read only to seed the views. -OWNER_PENDING_DELIVERIES_FILENAME = "owner_pending_deliveries.json" ROLLING_STATE_FILENAME = "rolling_state.json" diff --git a/syft/sync/sync/datasite_owner_syncer.py b/syft/sync/sync/datasite_owner_syncer.py index dcd98cf5d86..d8405b24e2c 100644 --- a/syft/sync/sync/datasite_owner_syncer.py +++ b/syft/sync/sync/datasite_owner_syncer.py @@ -42,7 +42,6 @@ from syft.sync.sync.collection_spec import CollectionSyncSpec from syft.sync.sync.constants import CACHE_DIR, ROLLING_STATE_FILENAME from syft.sync.sync.delivery_tracker import ( - UNSENT_HASH, AcceptedChange, DeliveryPlan, ) @@ -703,36 +702,17 @@ def _may_hold(self, peer: str, path: str, syncable: Mapping[str, str]) -> bool: return path in syncable and self.perm_context.can_read(path, peer) def _seed_views_if_missing(self, peer_emails: list[str]) -> None: - """Seed the views one time, for an owner whose state was never seeded. + """Seed the views one time, for an owner that has file hashes but no views. - The seed covers each approved peer, and each peer in ``peer_emails``. + Each approved peer, and each peer in ``peer_emails``, is taken to hold + every file it may read now. The caller reloads the permissions first. """ deliveries = self.event_cache.deliveries if deliveries.is_seeded(): return approved = [p.email for p in self.connection_router.peer_store.approved_peers] peers_to_seed = sorted({*approved, *peer_emails}) - deliveries.seed(self._files_held_by_peer_at_seed(peers_to_seed)) - - def _files_held_by_peer_at_seed( - self, peers: list[str] - ) -> dict[str, dict[str, str]]: - """What each of ``peers`` is taken to hold when the views are seeded. - - Maps each peer email to ``{path: hash}``. Each peer holds every file it may read now. A syncable path that an - earlier client still had to send gets ``UNSENT_HASH``, so the next plan - sends or deletes it. Other pending paths, such as a private dataset - share, stay out of the view: a plan never sends them, so it must not - delete them. The caller reloads the permissions first. - """ - pending = self.event_cache.legacy_pending_paths() - # peer email -> {path: hash} - held = self._readable_files_by_peer(peers) - for peer, files in held.items(): - unsent = [p for p in pending.get(peer, ()) if self._is_syncable(p)] - unsent_files = dict.fromkeys(unsent, UNSENT_HASH) - files.update(unsent_files) - return held + deliveries.seed(self._readable_files_by_peer(peers_to_seed)) def deliver_file_changes(self, peer_emails: list[str]) -> None: """Send each peer the files its view lacks, and deletes for the files it diff --git a/syft/sync/sync/delivery_tracker.py b/syft/sync/sync/delivery_tracker.py index 9ce59d31557..2f0e7bab0c7 100644 --- a/syft/sync/sync/delivery_tracker.py +++ b/syft/sync/sync/delivery_tracker.py @@ -26,12 +26,10 @@ that the owner rejects and then approves again is sent every file it may read, and a delete for each file that it may no longer have. -An owner whose state was never seeded, because it ran an earlier client or -lost its views file, has file hashes but no views. ``seed`` then starts the -view of each approved peer with every file it may read now, so the upgrade -sends nothing extra. A path that the earlier client still had to send gets -``UNSENT_HASH``, so the next plan sends it, or deletes it when it is gone. A -view that a send already recorded is kept. +An owner that lost its views file, for example an enclave that wipes its local +state at every boot, has file hashes but no views. ``seed`` then starts the +view of each approved peer with every file it may read now, so the owner does +not send every file again. A view that a send already recorded is kept. Recording a send ---------------- diff --git a/tests/unit/test_delivery_tracker.py b/tests/unit/test_delivery_tracker.py index 1c97f7c7482..abc9bcdb05c 100644 --- a/tests/unit/test_delivery_tracker.py +++ b/tests/unit/test_delivery_tracker.py @@ -10,8 +10,7 @@ from syft.sync.peers.exceptions import SyftPeerNotReadyError from syft.sync.peers.peer import PeerState from syft.sync.sync import datasite_owner_syncer -from syft.sync.sync.caches.persisted_dict import PersistedDict -from syft.sync.sync.constants import CACHE_DIR, OWNER_PENDING_DELIVERIES_FILENAME +from syft.sync.sync.constants import CACHE_DIR from syft.sync.sync.caches.datasite_owner_cache import ( ProposedEventFileOutdatedException, ) @@ -189,14 +188,9 @@ def _sent_events(ds_manager, do_manager, path: str) -> list[tuple[bool, object]] ] -def _simulate_earlier_client(do_manager, pending: dict[str, list[str]]) -> None: - """Make the DO cache look like one from a client before the delivery - tracker: no views, and ``pending`` paths it still had to send.""" - cache_dir = do_manager.syftbox_folder / CACHE_DIR - (cache_dir / DELIVERY_VIEWS_FILENAME).unlink() - legacy = PersistedDict(path=cache_dir / OWNER_PENDING_DELIVERIES_FILENAME) - for peer, paths in pending.items(): - legacy[peer] = dict.fromkeys(paths, "msg") +def _lose_views(do_manager) -> None: + """Delete the DO's views file, keeping its file hashes.""" + (do_manager.syftbox_folder / CACHE_DIR / DELIVERY_VIEWS_FILENAME).unlink() def _push_private_file(do_manager, ds_manager) -> None: @@ -293,83 +287,52 @@ def test_restarted_owner_sends_nothing_new(): assert _outbox_paths(ds_manager, do_manager) == sent_before -def test_upgrade_sends_only_unsent_changes(): +def test_lost_views_sends_only_offline_changes(): ds_manager, do_manager = _pair() do_manager.datasite_owner_syncer.perm_context.open("shared/").grant_read_access( ds_manager.email ) _write(do_manager, "shared/held.txt", "one") - _write(do_manager, "shared/pending.txt", "one") do_manager.sync(auto_checkpoint=False) - _simulate_earlier_client(do_manager, {ds_manager.email: ["shared/pending.txt"]}) + _lose_views(do_manager) _write(do_manager, "shared/offline.txt", "new") do_manager.sync(auto_checkpoint=False) sent = _outbox_paths(ds_manager, do_manager) assert sent.count("shared/held.txt") == 1 - assert sent.count("shared/pending.txt") == 2 assert sent.count("shared/offline.txt") == 1 -def test_upgrade_sends_pending_delete(): - ds_manager, do_manager = _pair() - do_manager.datasite_owner_syncer.perm_context.open("shared/").grant_read_access( - ds_manager.email - ) - _write(do_manager, "shared/gone.txt", "one") - do_manager.sync(auto_checkpoint=False) - - # The earlier client recorded the delete, but did not send it. - (do_manager.syftbox_folder / do_manager.email / "shared/gone.txt").unlink() - do_manager.datasite_owner_syncer.event_cache.process_local_file_changes() - _simulate_earlier_client(do_manager, {ds_manager.email: ["shared/gone.txt"]}) - - do_manager.sync(auto_checkpoint=False) - assert _sent_events(ds_manager, do_manager, "shared/gone.txt")[-1] == (True, None) - - -def test_upgrade_seeds_after_send_outside_plan(): +def test_lost_views_seeded_after_send_outside_plan(): ds_manager, do_manager = _pair() do_manager.datasite_owner_syncer.perm_context.open("shared/").grant_read_access( ds_manager.email ) _write(do_manager, "shared/held.txt", "one") do_manager.sync(auto_checkpoint=False) - _simulate_earlier_client(do_manager, {}) + _lose_views(do_manager) _push_private_file(do_manager, ds_manager) do_manager.sync(auto_checkpoint=False) assert _outbox_paths(ds_manager, do_manager).count("shared/held.txt") == 1 -def test_upgrade_seeds_peer_skipped_in_first_sync(): +def test_lost_views_seeds_peer_skipped_in_first_sync(): ds_manager, do_manager = _pair() do_manager.datasite_owner_syncer.perm_context.open("shared/").grant_read_access( ds_manager.email ) _write(do_manager, "shared/held.txt", "one") do_manager.sync(auto_checkpoint=False) - _simulate_earlier_client(do_manager, {}) + _lose_views(do_manager) - # The version gate skips the DS in the first sync after the upgrade. + # The version gate skips the DS in the first sync after the views are lost. do_manager.datasite_owner_syncer.sync([]) do_manager.sync(auto_checkpoint=False) assert _outbox_paths(ds_manager, do_manager).count("shared/held.txt") == 1 -def test_upgrade_keeps_pending_private_share(): - ds_manager, do_manager = _pair() - _push_private_file(do_manager, ds_manager) - do_manager.sync(auto_checkpoint=False) - _simulate_earlier_client(do_manager, {ds_manager.email: ["private/data.csv"]}) - - do_manager.sync(auto_checkpoint=False) - assert _sent_events(ds_manager, do_manager, "private/data.csv") == [ - (False, b"rows") - ] - - def test_failed_send_outside_plan_dropped(monkeypatch): ds_manager, do_manager = _pair() From fd4b7cfcac8f1047e460ffffa6366be67bf1efc3 Mon Sep 17 00:00:00 2001 From: Koen van der Veen Date: Tue, 27 Oct 2026 14:30:32 +0100 Subject: [PATCH 25/36] refactor: send accepted changes to every reader An accepted change now reaches every peer that may read the file at the next plan, like any other change, so the accepted-change record and its withholding are gone. --- syft/sync/sync/datasite_owner_syncer.py | 74 ---------- syft/sync/sync/delivery_tracker.py | 74 +--------- tests/unit/test_delivery_tracker.py | 23 --- tests/unit/test_sync_manager.py | 184 ++++-------------------- 4 files changed, 33 insertions(+), 322 deletions(-) diff --git a/syft/sync/sync/datasite_owner_syncer.py b/syft/sync/sync/datasite_owner_syncer.py index d8405b24e2c..757da779ca9 100644 --- a/syft/sync/sync/datasite_owner_syncer.py +++ b/syft/sync/sync/datasite_owner_syncer.py @@ -42,7 +42,6 @@ from syft.sync.sync.collection_spec import CollectionSyncSpec from syft.sync.sync.constants import CACHE_DIR, ROLLING_STATE_FILENAME from syft.sync.sync.delivery_tracker import ( - AcceptedChange, DeliveryPlan, ) from syft.sync.transport.types import SendStatus @@ -654,10 +653,6 @@ def _record_local_changes(self) -> None: file_change_events_message = self.event_cache.process_local_file_changes() if file_change_events_message is None: return - changed_paths = [ - str(e.path_in_datasite) for e in file_change_events_message.events - ] - self.event_cache.deliveries.drop_accepted(changed_paths) if not self.persist_owner_state: return # Feed the events into rolling state so they flow through @@ -720,12 +715,6 @@ def deliver_file_changes(self, peer_emails: list[str]) -> None: deliveries = self.event_cache.deliveries deliveries.confirm_deliveries(self.connection_router.transport.p2p.delivered()) plans = deliveries.plan(self._readable_files_by_peer(peer_emails)) - accepted = deliveries.accepted_changes() - syncable = self._syncable_hashes() - plans = { - peer: self._skip_accepted_from_others(peer, plan, accepted, syncable) - for peer, plan in plans.items() - } # Build every message before the first one is queued, so a read error # leaves no message of this plan in the queue. messages = [ @@ -739,42 +728,6 @@ def deliver_file_changes(self, peer_emails: list[str]) -> None: self.outbox_queue.put(message) self.process_syftbox_events_queue() - def _skip_accepted_from_others( - self, - peer: str, - plan: DeliveryPlan, - accepted: Mapping[str, AcceptedChange], - syncable: Mapping[str, str], - ) -> DeliveryPlan: - """``plan`` without the accepted changes that are withheld from ``peer``. - - An accepted file is withheld while ``syncable`` still has the hash that - the change gave it. An accepted delete is withheld while the file stays - gone and ``peer`` may still read its path. A delete after a revoke - still goes out. - """ - - def from_other(path: str) -> bool: - change = accepted.get(path) - return ( - change is not None - and peer in change.withheld_from - and change.hash == syncable.get(path) - ) - - return DeliveryPlan( - send=[p for p in plan.send if not from_other(p)], - delete=[ - p - for p in plan.delete - if not ( - from_other(p) - and accepted[p].hash is None - and self.perm_context.can_read(p, peer) - ) - ], - ) - def _plan_events(self, plan: DeliveryPlan) -> list[FileChangeEvent]: """The events that carry out ``plan``: its sends, then its deletes.""" events = [self._create_current_filestate_event(p) for p in plan.send] @@ -840,9 +793,6 @@ def handle_proposed_filechange_events_message( filtered_message ) if accepted_events_message is not None: - self.event_cache.deliveries.accept( - self._accepted_changes(sender_email, accepted_events_message) - ) self.queue_event_for_syftbox( recipients=[sender_email], file_change_events_message=accepted_events_message, @@ -850,30 +800,6 @@ def handle_proposed_filechange_events_message( if self.persist_owner_state: self._add_events_to_rolling_state(accepted_events_message) - def _accepted_changes( - self, sender_email: str, message: FileChangeEventsMessage - ) -> dict[str, AcceptedChange]: - """The changes of ``message``, accepted from ``sender_email``, by path. - - Each change is withheld from the other approved peers that may read its - file now. The caller loads the permissions first. - """ - others = [ - p.email - for p in self.connection_router.peer_store.approved_peers - if p.email != sender_email - ] - changes = {} - for event in message.events: - path = str(event.path_in_datasite) - changes[path] = AcceptedChange( - hash=None if event.is_deleted else event.new_hash, - withheld_from=[ - p for p in others if self.perm_context.can_read(path, p) - ], - ) - return changes - def queue_event_for_syftbox( self, recipients: list[str], file_change_events_message: FileChangeEventsMessage ): diff --git a/syft/sync/sync/delivery_tracker.py b/syft/sync/sync/delivery_tracker.py index 2f0e7bab0c7..5f1492aeefa 100644 --- a/syft/sync/sync/delivery_tracker.py +++ b/syft/sync/sync/delivery_tracker.py @@ -14,15 +14,12 @@ in the view, but no longer readable -> delete in the view, but the file is gone -> delete -A change that the owner accepts from a peer goes only to that peer. -``accept`` records it, with the other peers that may read the file at that -time. The owner leaves the change out of their plans while the file stays in -that state. A peer that joins later, or that may read the file only later, gets -it. When the owner changes the file again, every peer that may read it gets it. +A change that the owner accepts from a peer goes to that peer at once. Every +other peer that may read the file gets it in its next plan, as with any change. A new peer starts with an empty view, so it is sent every file it may read. When the owner rejects a peer, ``reset`` sets each path in its view to -``UNSENT_HASH``, and stops withholding accepted changes from it. Thus a peer +``UNSENT_HASH``. Thus a peer that the owner rejects and then approves again is sent every file it may read, and a delete for each file that it may no longer have. @@ -82,18 +79,6 @@ class QueuedSend(BaseModel): changes: dict[str, str | None] -class AcceptedChange(BaseModel): - """A change that the owner accepted from a peer. - - ``hash`` is the new hash of the file, or None for a delete. - ``withheld_from`` holds the other peers that could read the file when the - owner accepted the change. - """ - - hash: str | None - withheld_from: list[str] = Field(default_factory=list) - - class DeliveryState(BaseModel): """Everything the tracker keeps.""" @@ -104,8 +89,6 @@ class DeliveryState(BaseModel): queued: dict[str, QueuedSend] = Field(default_factory=dict) # Whether ``DeliveryTracker.seed`` ran on this state. seeded: bool = False - # path -> the last change that the owner accepted from a peer for it - accepted: dict[str, AcceptedChange] = Field(default_factory=dict) class DeliveryPlan(BaseModel): @@ -187,25 +170,8 @@ def _apply_changes_to_view( def _record_delivery( state: DeliveryState, peer: str, changes: Mapping[str, str | None] ) -> None: - """Apply ``changes`` to ``peer``'s view. - - After a delete, ``peer`` no longer holds the version from before an accepted - change, so that change is no longer withheld from it. - """ + """Apply ``changes`` to ``peer``'s view.""" _apply_changes_to_view(state.views.setdefault(peer, {}), changes) - for path, new_hash in changes.items(): - change = state.accepted.get(path) - if new_hash is None and change is not None and peer in change.withheld_from: - _stop_withholding(state, path, peer) - - -def _stop_withholding(state: DeliveryState, path: str, peer: str) -> None: - """Remove ``peer`` from the accepted change of ``path``, and drop the change - when it is withheld from nobody.""" - change = state.accepted[path] - change.withheld_from.remove(peer) - if not change.withheld_from: - del state.accepted[path] class DeliveryTracker: @@ -220,8 +186,7 @@ def __init__(self, store: DeliveryStateStore): self.store = store def clear(self) -> None: - """Forget the whole state: views, queued sends, accepted changes and the - seeded flag.""" + """Forget the whole state: views, queued sends and the seeded flag.""" with self.store.locked(): self.store.save(DeliveryState()) @@ -235,37 +200,8 @@ def reset(self, peer: str) -> None: state = self.store.load() view = state.views.get(peer, {}) state.views[peer] = dict.fromkeys(view, UNSENT_HASH) - for path, change in list(state.accepted.items()): - if peer in change.withheld_from: - _stop_withholding(state, path, peer) self.store.save(state) - def accept(self, changes: Mapping[str, AcceptedChange]) -> None: - """Record ``changes``, by path, as the last change accepted from a peer. - - A change withheld from nobody is not kept. - """ - with self.store.locked(): - state = self.store.load() - for path, change in changes.items(): - if change.withheld_from: - state.accepted[path] = change - else: - state.accepted.pop(path, None) - self.store.save(state) - - def drop_accepted(self, paths: Iterable[str]) -> None: - """Forget the accepted changes of ``paths``, which the owner changed.""" - with self.store.locked(): - state = self.store.load() - for path in paths: - state.accepted.pop(path, None) - self.store.save(state) - - def accepted_changes(self) -> dict[str, AcceptedChange]: - """The last change that the owner accepted from a peer, by path.""" - return self.store.load().accepted - def is_seeded(self) -> bool: """Whether ``seed`` ran on the stored state.""" return self.store.load().seeded diff --git a/tests/unit/test_delivery_tracker.py b/tests/unit/test_delivery_tracker.py index abc9bcdb05c..79156d2956d 100644 --- a/tests/unit/test_delivery_tracker.py +++ b/tests/unit/test_delivery_tracker.py @@ -16,7 +16,6 @@ ) from syft.sync.sync.delivery_tracker import ( DELIVERY_STATE_VERSION, - AcceptedChange, DELIVERY_VIEWS_FILENAME, DeliveryState, DeliveryTracker, @@ -65,28 +64,6 @@ def test_reset_resends_readable_deletes_gone(): assert plan.delete == ["gone"] -def test_reset_stops_withholding_accepted_changes(): - tracker = DeliveryTracker(MemoryDeliveryStateStore()) - tracker.accept({"a": AcceptedChange(hash="h1", withheld_from=[PEER, "b@x.com"])}) - tracker.reset(PEER) - assert tracker.accepted_changes()["a"].withheld_from == ["b@x.com"] - - -def test_accepted_change_kept_only_while_withheld(): - tracker = DeliveryTracker(MemoryDeliveryStateStore()) - tracker.accept( - { - "a": AcceptedChange(hash="h1", withheld_from=[]), - "b": AcceptedChange(hash="h1", withheld_from=[PEER]), - "c": AcceptedChange(hash="h1", withheld_from=["c@x.com"]), - } - ) - assert sorted(tracker.accepted_changes()) == ["b", "c"] - tracker.record_send(PEER, "m1", SendStatus.DELIVERED, {"b": None}) - tracker.reset("c@x.com") - assert tracker.accepted_changes() == {} - - def test_queued_send_updates_view_on_confirm(): tracker = DeliveryTracker(MemoryDeliveryStateStore()) tracker.record_send(PEER, "m1", SendStatus.QUEUED, {"a": "h1"}) diff --git a/tests/unit/test_sync_manager.py b/tests/unit/test_sync_manager.py index b6de19984fd..aab37c58eea 100644 --- a/tests/unit/test_sync_manager.py +++ b/tests/unit/test_sync_manager.py @@ -775,103 +775,72 @@ def _events_for(router: ConnectionRouter, do_manager) -> list[tuple[str, bool]]: ] -def test_accepted_change_goes_only_to_submitter(): - """A change that the DO accepts from peer A goes only to peer A. Peer B, who - may read the file, gets it when the DO changes the file.""" - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( - use_in_memory_cache=False, - ) - peer_b_email = "peer_b@example.com" - peer_b_router = _peer_router(ds_manager, do_manager, peer_b_email) - shared = do_manager.datasite_owner_syncer.perm_context.open("shared/") +def _accept_setup(ds_manager, do_manager, peer_b_email: str): + """Peer A may write ``shared/``, peer B may read it, and both hold + ``shared/data.txt`` with "one". Returns the syncer, the folder and the peers.""" + syncer = do_manager.datasite_owner_syncer + shared = syncer.perm_context.open("shared/") shared.grant_write_access(ds_manager.email) shared.grant_read_access(peer_b_email) peers = [ds_manager.email, peer_b_email] - - ds_manager._send_file_change(f"{do_manager.email}/shared/proposal.txt", "from A") - do_manager.datasite_owner_syncer.sync(peers) - do_manager.datasite_owner_syncer.sync(peers) - assert ("shared/proposal.txt", False) in _events_for( - ds_manager._connection_router, do_manager - ) - assert ("shared/proposal.txt", False) not in _events_for(peer_b_router, do_manager) - - proposal = do_manager.syftbox_folder / do_manager.email / "shared/proposal.txt" - proposal.write_text("from the DO") - do_manager.datasite_owner_syncer.sync(peers) - assert ("shared/proposal.txt", False) in _events_for(peer_b_router, do_manager) + data = do_manager.syftbox_folder / do_manager.email / "shared/data.txt" + data.parent.mkdir(parents=True, exist_ok=True) + data.write_text("one") + syncer.sync(peers) + ds_manager.sync() + return syncer, shared, peers -def test_reader_granted_after_accept_gets_accepted_file(): - """Peer C may read the folder only after the DO accepts peer A's change, so - peer C gets the file. Peer B, who could read it then, does not.""" +def test_accepted_change_reaches_every_reader(): + """A change that the DO accepts from peer A goes to peer A, and to peer B, who + may read the file.""" ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( use_in_memory_cache=False, ) peer_b_email = "peer_b@example.com" - peer_c_email = "peer_c@example.com" peer_b_router = _peer_router(ds_manager, do_manager, peer_b_email) - peer_c_router = _peer_router(ds_manager, do_manager, peer_c_email) - syncer = do_manager.datasite_owner_syncer - shared = syncer.perm_context.open("shared/") - shared.grant_write_access(ds_manager.email) - shared.grant_read_access(peer_b_email) - peers = [ds_manager.email, peer_b_email, peer_c_email] + syncer, _, peers = _accept_setup(ds_manager, do_manager, peer_b_email) ds_manager._send_file_change(f"{do_manager.email}/shared/proposal.txt", "from A") syncer.sync(peers) - shared.grant_read_access(peer_c_email) syncer.sync(peers) - assert ("shared/proposal.txt", False) in _events_for(peer_c_router, do_manager) - assert ("shared/proposal.txt", False) not in _events_for(peer_b_router, do_manager) + assert ("shared/proposal.txt", False) in _events_for( + ds_manager._connection_router, do_manager + ) + assert ("shared/proposal.txt", False) in _events_for(peer_b_router, do_manager) def test_revoke_after_accepted_change_sends_delete(): - """Peer B holds the DO's version of a file. After the DO accepts peer A's - change to it and peer B loses read access, peer B gets a delete.""" + """Peer B gets peer A's accepted change to a file, and a delete once it loses + read access.""" ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( use_in_memory_cache=False, ) peer_b_email = "peer_b@example.com" peer_b_router = _peer_router(ds_manager, do_manager, peer_b_email) - syncer = do_manager.datasite_owner_syncer - shared = syncer.perm_context.open("shared/") - shared.grant_write_access(ds_manager.email) - shared.grant_read_access(peer_b_email) - peers = [ds_manager.email, peer_b_email] - data = do_manager.syftbox_folder / do_manager.email / "shared/data.txt" - data.parent.mkdir(parents=True, exist_ok=True) - data.write_text("one") - syncer.sync(peers) - ds_manager.sync() + syncer, shared, peers = _accept_setup(ds_manager, do_manager, peer_b_email) ds_manager._send_file_change(f"{do_manager.email}/shared/data.txt", "from A") syncer.sync(peers) + syncer.sync(peers) shared.revoke_read_access(peer_b_email) syncer.sync(peers) assert _events_for(peer_b_router, do_manager) == [ + ("shared/data.txt", False), ("shared/data.txt", False), ("shared/data.txt", True), ] -def test_accepted_delete_goes_only_to_submitter(): - """A delete that the DO accepts from peer A goes only to peer A. Peer B keeps - its copy.""" +def test_accepted_delete_reaches_every_reader(): + """A delete that the DO accepts from peer A goes to peer A, and to peer B, who + may read the file.""" ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( use_in_memory_cache=False, ) peer_b_email = "peer_b@example.com" peer_b_router = _peer_router(ds_manager, do_manager, peer_b_email) - syncer = do_manager.datasite_owner_syncer - shared = syncer.perm_context.open("shared/") - shared.grant_write_access(ds_manager.email) - shared.grant_read_access(peer_b_email) - peers = [ds_manager.email, peer_b_email] - data = do_manager.syftbox_folder / do_manager.email / "shared/data.txt" - data.parent.mkdir(parents=True, exist_ok=True) - data.write_text("one") - syncer.sync(peers) + syncer, _, peers = _accept_setup(ds_manager, do_manager, peer_b_email) proposal = ProposedFileChangesMessage( sender_email=ds_manager.email, @@ -890,109 +859,12 @@ def test_accepted_delete_goes_only_to_submitter(): assert ("shared/data.txt", True) in _events_for( ds_manager._connection_router, do_manager ) - assert _events_for(peer_b_router, do_manager) == [("shared/data.txt", False)] - - -def _accept_setup(ds_manager, do_manager, peer_b_email: str): - """Peer A may write ``shared/``, peer B may read it, and both hold - ``shared/data.txt`` with "one". Returns the syncer, the folder and the peers.""" - syncer = do_manager.datasite_owner_syncer - shared = syncer.perm_context.open("shared/") - shared.grant_write_access(ds_manager.email) - shared.grant_read_access(peer_b_email) - peers = [ds_manager.email, peer_b_email] - data = do_manager.syftbox_folder / do_manager.email / "shared/data.txt" - data.parent.mkdir(parents=True, exist_ok=True) - data.write_text("one") - syncer.sync(peers) - ds_manager.sync() - return syncer, shared, peers - - -def _contents_for(router: ConnectionRouter, do_manager, path: str) -> list: - """The content of each event for ``path`` in the DO's outbox for ``router``'s - peer, None for a delete.""" - return [ - e.content - for m in router.watcher_get_events_messages(do_manager.email, None) - for e in m.events - if str(e.path_in_datasite) == path - ] - - -def test_revoke_after_accepted_delete_sends_delete(): - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( - use_in_memory_cache=False, - ) - peer_b_email = "peer_b@example.com" - peer_b_router = _peer_router(ds_manager, do_manager, peer_b_email) - syncer, shared, peers = _accept_setup(ds_manager, do_manager, peer_b_email) - - proposal = ProposedFileChangesMessage( - sender_email=ds_manager.email, - proposed_file_changes=[ - ProposedFileChange( - path_in_datasite="shared/data.txt", - old_hash=syncer.event_cache.file_hashes[Path("shared/data.txt")], - datasite_email=do_manager.email, - is_deleted=True, - ) - ], - ) - syncer.handle_proposed_filechange_events_message(ds_manager.email, proposal) - syncer.process_syftbox_events_queue() - syncer.sync(peers) - shared.revoke_read_access(peer_b_email) - syncer.sync(peers) assert _events_for(peer_b_router, do_manager) == [ ("shared/data.txt", False), ("shared/data.txt", True), ] -def test_owner_change_back_to_accepted_content_reaches_readers(): - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( - use_in_memory_cache=False, - ) - peer_b_email = "peer_b@example.com" - peer_b_router = _peer_router(ds_manager, do_manager, peer_b_email) - syncer, _, peers = _accept_setup(ds_manager, do_manager, peer_b_email) - data = do_manager.syftbox_folder / do_manager.email / "shared/data.txt" - - ds_manager._send_file_change(f"{do_manager.email}/shared/data.txt", "from A") - syncer.sync(peers) - data.write_text("two") - syncer.sync(peers) - data.write_text("from A") - syncer.sync(peers) - assert _contents_for(peer_b_router, do_manager, "shared/data.txt") == [ - "one", - "two", - "from A", - ] - - -def test_reader_granted_again_after_revoke_gets_accepted_file(): - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( - use_in_memory_cache=False, - ) - peer_b_email = "peer_b@example.com" - peer_b_router = _peer_router(ds_manager, do_manager, peer_b_email) - syncer, shared, peers = _accept_setup(ds_manager, do_manager, peer_b_email) - - ds_manager._send_file_change(f"{do_manager.email}/shared/data.txt", "from A") - syncer.sync(peers) - shared.revoke_read_access(peer_b_email) - syncer.sync(peers) - shared.grant_read_access(peer_b_email) - syncer.sync(peers) - assert _contents_for(peer_b_router, do_manager, "shared/data.txt") == [ - "one", - None, - "from A", - ] - - def test_default_collections_folder_picks_shareable_spec_regardless_of_order(): """The collections folder is chosen by owner_only, not by position.""" from syft.sync.syftbox_manager import default_collections_folder From 0c19872ee0e3884cc7950cd06dffa068e1a2b340 Mon Sep 17 00:00:00 2001 From: Koen van der Veen Date: Tue, 27 Oct 2026 15:48:32 +0100 Subject: [PATCH 26/36] fix: let one failed outbox send not block the others Every queued message is tried. The first error that is not a not-ready peer raises after the rest went out. --- syft/sync/sync/datasite_owner_syncer.py | 56 ++++++++++++------------- tests/unit/test_delivery_tracker.py | 25 +++++++++++ 2 files changed, 51 insertions(+), 30 deletions(-) diff --git a/syft/sync/sync/datasite_owner_syncer.py b/syft/sync/sync/datasite_owner_syncer.py index 757da779ca9..fae81538e8a 100644 --- a/syft/sync/sync/datasite_owner_syncer.py +++ b/syft/sync/sync/datasite_owner_syncer.py @@ -1,6 +1,5 @@ import logging import warnings -from collections import deque from concurrent.futures import ThreadPoolExecutor from functools import partial from pathlib import Path @@ -827,38 +826,35 @@ def process_syftbox_events_queue(self): def _send_outbox_messages(self) -> None: """Send the queued messages, and record each sent one in its peer's view. - A peer that is not ready gets none of its planned messages. They wait - for the next plan, with a warning. Any other failed send raises. Only - the unplanned messages after it go back on the queue. + A failed send does not stop the others. A peer that is not ready gets + none of its planned messages. They wait for the next plan, with a + warning. Any other failed message is dropped, and the first such error + raises after every other message went out. A failed planned message is + planned again by `deliver_file_changes`, which checks the views and the + permissions again. """ - unsent = deque() - while not self.outbox_queue.empty(): - unsent.append(self.outbox_queue.get()) not_ready = set() - try: - while unsent: - recipient, message, planned = unsent.popleft() - if planned and recipient in not_ready: + errors = [] + while not self.outbox_queue.empty(): + recipient, message, planned = self.outbox_queue.get() + if planned and recipient in not_ready: + continue + try: + status = self.connection_router.owner_write_event_messages_to_outbox( + recipient, message + ) + except SyftPeerNotReadyError as e: + if not planned: + errors.append(e) continue - try: - status = ( - self.connection_router.owner_write_event_messages_to_outbox( - recipient, message - ) - ) - except SyftPeerNotReadyError as e: - if not planned: - raise - not_ready.add(recipient) - warnings.warn(f"Changes for {recipient} wait for a later sync: {e}") - else: - self._record_sent(recipient, message, status) - finally: - # A later send of a planned message would not check the views or the - # permissions again, so `deliver_file_changes` plans it again instead. - for entry in unsent: - if not entry.planned: - self.outbox_queue.put(entry) + not_ready.add(recipient) + warnings.warn(f"Changes for {recipient} wait for a later sync: {e}") + except Exception as e: + errors.append(e) + else: + self._record_sent(recipient, message, status) + if errors: + raise errors[0] def _record_sent( self, recipient: str, message: FileChangeEventsMessage, status: SendStatus diff --git a/tests/unit/test_delivery_tracker.py b/tests/unit/test_delivery_tracker.py index 79156d2956d..33afaac5fdd 100644 --- a/tests/unit/test_delivery_tracker.py +++ b/tests/unit/test_delivery_tracker.py @@ -347,6 +347,31 @@ def test_peer_not_ready_does_not_block_other_peers(): ] +def test_failed_send_does_not_block_other_peers(monkeypatch): + ds_manager, do_manager = _pair() + syncer = do_manager.datasite_owner_syncer + broken = "broken@example.com" + shared = syncer.perm_context.open("shared/") + shared.grant_read_access(broken) + shared.grant_read_access(ds_manager.email) + _write(do_manager, "shared/data.txt", "one") + syncer._record_local_changes() + + send = ConnectionRouter.owner_write_event_messages_to_outbox + + def fail_for_broken(self, recipient, message): + if recipient == broken: + raise ConnectionError("upload failed") + return send(self, recipient, message) + + monkeypatch.setattr( + ConnectionRouter, "owner_write_event_messages_to_outbox", fail_for_broken + ) + with pytest.raises(ConnectionError): + syncer.deliver_file_changes([broken, ds_manager.email]) + assert _outbox_paths(ds_manager, do_manager) == ["shared/data.txt"] + + def test_failed_send_outside_plan_does_not_block_later_syncs(): ds_manager, do_manager = _pair() syncer = do_manager.datasite_owner_syncer From aacd11efe590e0b1c73fb31c539f2f9b260d9070 Mon Sep 17 00:00:00 2001 From: Koen van der Veen Date: Tue, 27 Oct 2026 16:11:27 +0100 Subject: [PATCH 27/36] refactor: one send path for accepted and owner changes sync() records the owner's changes, pulls the proposals, and then delivers once. An accepted change reaches every reader through the plan, the submitter included. Outbox messages carry no planned flag: a failed send is tried again in the same pass, logged, and dropped with an error after three tries. A failed proposal no longer holds back the send; its error raises after it. --- syft/sync/sync/datasite_owner_syncer.py | 136 ++++++++++++++---------- tests/unit/test_delivery_tracker.py | 22 ++-- tests/unit/test_encryption_policy.py | 10 +- tests/unit/test_persisted_cache.py | 3 +- 4 files changed, 96 insertions(+), 75 deletions(-) diff --git a/syft/sync/sync/datasite_owner_syncer.py b/syft/sync/sync/datasite_owner_syncer.py index fae81538e8a..3cdbbe4072d 100644 --- a/syft/sync/sync/datasite_owner_syncer.py +++ b/syft/sync/sync/datasite_owner_syncer.py @@ -1,4 +1,5 @@ import logging +from collections import deque import warnings from concurrent.futures import ThreadPoolExecutor from functools import partial @@ -67,6 +68,10 @@ # compact_outbox_if_needed() will merge them into a single message. MIN_MESSAGES_COMPACT = 20 +# How often the owner tries an outbox message in one send pass before it gives +# up and logs an error. A file that a view tracks is planned again next sync. +MAX_SEND_ATTEMPTS = 3 + class _CheckpointRestore(NamedTuple): """What restoring from checkpoints recovered. @@ -121,8 +126,8 @@ class DatasiteOwnerSyncerConfig(BaseModel): class OutgoingMessage(NamedTuple): recipient: str message: FileChangeEventsMessage - # Whether `deliver_file_changes` queued the message from the views of `recipient`. - planned: bool + # Failed sends of this message so far. + attempts: int = 0 def _batches_by_content_size( @@ -230,11 +235,13 @@ def _save_rolling_state(self) -> None: tmp.rename(path) def sync(self, peer_emails: list[str], recompute_hashes: bool = True): - """Send each of ``peer_emails`` what it does not hold yet, then pull - their proposed changes. + """Record the owner's changes, pull the proposed changes of + ``peer_emails``, then send each of them what it does not hold yet, in + one pass. - The send comes first, so a proposal that fails does not hold back the - owner's own changes. + Accepted proposals and the owner's own changes go out the same way: to + every peer that may read the file. A proposal that fails does not hold + back the send; its error raises after it. """ if self.persist_owner_state: self._load_rolling_state() @@ -246,34 +253,41 @@ def sync(self, peer_emails: list[str], recompute_hashes: bool = True): # Before the local changes are recorded, so that a file changed # while the owner was offline is not seeded as held. self._seed_views_if_missing(peer_emails) - + # Before the proposals, so that a proposal based on an older + # version of a file the owner changed is refused as outdated. if recompute_hashes: self._record_local_changes() + errors = [e for peer in peer_emails if (e := self._pull_proposals(peer))] self.deliver_file_changes(peer_emails) - - for peer_email in peer_emails: - proposed_count = 0 - try: - while True: - msg = self.pull_and_process_next_proposed_filechange( - peer_email, raise_on_none=False - ) - if msg is None: - # no new message, we are done - break - proposed_count += len(msg.proposed_file_changes) - except SyftPeerNotReadyError as e: - warnings.warn(f"Messages from {peer_email} stay in the inbox: {e}") - if proposed_count: - print( - f"Received {proposed_count} proposed change(s) " - f"from {peer_email}" - ) - self.process_syftbox_events_queue() + for error in errors[1:]: + logger.error(f"Proposed changes failed: {error!r}") + if errors: + raise errors[0] finally: if self.persist_owner_state: self._save_rolling_state() + def _pull_proposals(self, peer_email: str) -> Exception | None: + """Apply every proposed change waiting from ``peer_email``. + + Returns the error of a proposal that failed, so that it does not hold + back the other peers or the owner's changes. + """ + proposed_count = 0 + try: + while msg := self.pull_and_process_next_proposed_filechange( + peer_email, raise_on_none=False + ): + proposed_count += len(msg.proposed_file_changes) + except SyftPeerNotReadyError as e: + warnings.warn(f"Messages from {peer_email} stay in the inbox: {e}") + except Exception as e: + return e + finally: + if proposed_count: + print(f"Received {proposed_count} proposed change(s) from {peer_email}") + return None + def download_events_message_by_id_with_connection( self, events_message_id: str ) -> FileChangeEventsMessage: @@ -717,7 +731,7 @@ def deliver_file_changes(self, peer_emails: list[str]) -> None: # Build every message before the first one is queued, so a read error # leaves no message of this plan in the queue. messages = [ - OutgoingMessage(peer, FileChangeEventsMessage(events=batch), planned=True) + OutgoingMessage(peer, FileChangeEventsMessage(events=batch)) for peer, plan in plans.items() for batch in _batches_by_content_size( self._plan_events(plan), MAX_DELIVERY_MESSAGE_CONTENT @@ -791,13 +805,11 @@ def handle_proposed_filechange_events_message( accepted_events_message = self.event_cache.process_proposed_events_message( filtered_message ) - if accepted_events_message is not None: - self.queue_event_for_syftbox( - recipients=[sender_email], - file_change_events_message=accepted_events_message, - ) - if self.persist_owner_state: - self._add_events_to_rolling_state(accepted_events_message) + # The next plan sends the accepted change to every peer that may read + # the file, the submitter included. + if accepted_events_message is not None and self.persist_owner_state: + self.syftbox_events_queue.put(accepted_events_message) + self._add_events_to_rolling_state(accepted_events_message) def queue_event_for_syftbox( self, recipients: list[str], file_change_events_message: FileChangeEventsMessage @@ -810,7 +822,7 @@ def queue_event_for_syftbox( for recipient in recipients: self.outbox_queue.put( - OutgoingMessage(recipient, file_change_events_message, planned=False) + OutgoingMessage(recipient, file_change_events_message) ) def process_syftbox_events_queue(self): @@ -826,35 +838,47 @@ def process_syftbox_events_queue(self): def _send_outbox_messages(self) -> None: """Send the queued messages, and record each sent one in its peer's view. - A failed send does not stop the others. A peer that is not ready gets - none of its planned messages. They wait for the next plan, with a - warning. Any other failed message is dropped, and the first such error - raises after every other message went out. A failed planned message is - planned again by `deliver_file_changes`, which checks the views and the - permissions again. + Nothing raises, and a failed send does not stop the others. A failed + message goes to the back of this pass, and after ``MAX_SEND_ATTEMPTS`` + tries it is dropped with an error in the log. A peer that is not ready + gets none of its messages in this pass, with a warning. A file that a + view tracks is planned again at the next sync. """ - not_ready = set() - errors = [] + pending = deque() while not self.outbox_queue.empty(): - recipient, message, planned = self.outbox_queue.get() - if planned and recipient in not_ready: + pending.append(self.outbox_queue.get()) + not_ready = set() + while pending: + entry = pending.popleft() + if entry.recipient in not_ready: continue try: status = self.connection_router.owner_write_event_messages_to_outbox( - recipient, message + entry.recipient, entry.message ) except SyftPeerNotReadyError as e: - if not planned: - errors.append(e) - continue - not_ready.add(recipient) - warnings.warn(f"Changes for {recipient} wait for a later sync: {e}") + not_ready.add(entry.recipient) + warnings.warn(f"Messages for {entry.recipient} are not sent: {e}") except Exception as e: - errors.append(e) + pending.extend(self._retry_or_give_up(entry, e)) else: - self._record_sent(recipient, message, status) - if errors: - raise errors[0] + self._record_sent(entry.recipient, entry.message, status) + + def _retry_or_give_up( + self, entry: OutgoingMessage, error: Exception + ) -> list[OutgoingMessage]: + """The entry to try again for a failed send, if any try is left.""" + name = entry.message.message_filepath.as_string() + attempts = entry.attempts + 1 + if attempts >= MAX_SEND_ATTEMPTS: + logger.error( + f"Gave up sending {name} to {entry.recipient} after {attempts} tries: {error!r}" + ) + return [] + logger.info( + f"Send of {name} to {entry.recipient} failed, trying again: {error!r}" + ) + return [entry._replace(attempts=attempts)] def _record_sent( self, recipient: str, message: FileChangeEventsMessage, status: SendStatus diff --git a/tests/unit/test_delivery_tracker.py b/tests/unit/test_delivery_tracker.py index 33afaac5fdd..c1bd75b2c34 100644 --- a/tests/unit/test_delivery_tracker.py +++ b/tests/unit/test_delivery_tracker.py @@ -310,7 +310,7 @@ def test_lost_views_seeds_peer_skipped_in_first_sync(): assert _outbox_paths(ds_manager, do_manager).count("shared/held.txt") == 1 -def test_failed_send_outside_plan_dropped(monkeypatch): +def test_send_to_peer_not_ready_dropped(monkeypatch): ds_manager, do_manager = _pair() def peer_not_ready(self, recipient, *args, **kwargs): @@ -320,7 +320,7 @@ def peer_not_ready(self, recipient, *args, **kwargs): m.setattr( ConnectionRouter, "owner_write_event_messages_to_outbox", peer_not_ready ) - with pytest.raises(SyftPeerNotReadyError): + with pytest.warns(UserWarning, match="are not sent"): _push_private_file(do_manager, ds_manager) do_manager.datasite_owner_syncer.process_syftbox_events_queue() @@ -347,7 +347,7 @@ def test_peer_not_ready_does_not_block_other_peers(): ] -def test_failed_send_does_not_block_other_peers(monkeypatch): +def test_failed_send_retried_then_logged_without_blocking_others(monkeypatch, caplog): ds_manager, do_manager = _pair() syncer = do_manager.datasite_owner_syncer broken = "broken@example.com" @@ -367,12 +367,13 @@ def fail_for_broken(self, recipient, message): monkeypatch.setattr( ConnectionRouter, "owner_write_event_messages_to_outbox", fail_for_broken ) - with pytest.raises(ConnectionError): - syncer.deliver_file_changes([broken, ds_manager.email]) + syncer.deliver_file_changes([broken, ds_manager.email]) assert _outbox_paths(ds_manager, do_manager) == ["shared/data.txt"] + tries = [r for r in caplog.records if broken in r.getMessage()] + assert [r.levelname for r in tries] == ["INFO", "INFO", "ERROR"] -def test_failed_send_outside_plan_does_not_block_later_syncs(): +def test_send_to_stranger_does_not_block_later_syncs(): ds_manager, do_manager = _pair() syncer = do_manager.datasite_owner_syncer _write(do_manager, "private/data.csv", "rows") @@ -382,7 +383,7 @@ def test_failed_send_outside_plan_does_not_block_later_syncs(): syncer.queue_event_for_syftbox( recipients=["stranger@example.com"], file_change_events_message=message ) - with pytest.raises(SyftPeerNotReadyError): + with pytest.warns(UserWarning, match="stranger@example.com"): syncer.process_syftbox_events_queue() syncer.perm_context.open("shared/").grant_read_access(ds_manager.email) @@ -457,7 +458,7 @@ def test_outdated_proposal_does_not_block_owner_changes(): assert _received(ds_manager, do_manager, "shared/b.txt").read_text() == "new" -def test_failed_send_drops_rest_of_plan(monkeypatch): +def test_failed_send_not_retried_after_revoke(monkeypatch): monkeypatch.setattr(datasite_owner_syncer, "MAX_DELIVERY_MESSAGE_CONTENT", 3) ds_manager, do_manager = _pair() folder = do_manager.datasite_owner_syncer.perm_context.open("shared/") @@ -470,10 +471,9 @@ def fail_upload(*args, **kwargs): with monkeypatch.context() as m: m.setattr(ConnectionRouter, "owner_write_event_messages_to_outbox", fail_upload) - with pytest.raises(ConnectionError): - do_manager.sync(auto_checkpoint=False) + do_manager.sync(auto_checkpoint=False) - # The batch after the failed one must not go out after the revoke. + # The failed messages must not go out after the revoke. folder.revoke_read_access(ds_manager.email) do_manager.sync(auto_checkpoint=False) assert _outbox_paths(ds_manager, do_manager) == [] diff --git a/tests/unit/test_encryption_policy.py b/tests/unit/test_encryption_policy.py index 295e5e3ebc8..85cb0471eb1 100644 --- a/tests/unit/test_encryption_policy.py +++ b/tests/unit/test_encryption_policy.py @@ -428,16 +428,14 @@ def test_negotiated_encryption_flow(ds_policy, do_policy): assert len(_ds_cached_contents(ds_manager)) > 0 -def test_owner_send_to_non_peer_raises(): +def test_owner_send_to_non_peer_warns(): _, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( do_encryption_policy=REQUIRED, ) syncer = do_manager.datasite_owner_syncer message = FileChangeEventsMessage(events=[]) - syncer.outbox_queue.put( - OutgoingMessage("former@example.com", message, planned=False) - ) - with pytest.raises(SyftPeerNotReadyError, match="not known"): + syncer.outbox_queue.put(OutgoingMessage("former@example.com", message)) + with pytest.warns(UserWarning, match="not known"): syncer.process_syftbox_events_queue() @@ -559,7 +557,7 @@ def fail(self, recipient, message): with monkeypatch.context() as m: m.setattr(ConnectionRouter, "owner_write_event_messages_to_outbox", fail) - with pytest.warns(UserWarning, match="wait for a later sync"): + with pytest.warns(UserWarning, match="are not sent"): do_manager.sync(auto_checkpoint=False) _assert_ds_receives_after_notebook_sync(ds_manager, do_manager) diff --git a/tests/unit/test_persisted_cache.py b/tests/unit/test_persisted_cache.py index b2c1b4f5f56..06e68f74323 100644 --- a/tests/unit/test_persisted_cache.py +++ b/tests/unit/test_persisted_cache.py @@ -171,8 +171,7 @@ def fail_upload(*args, **kwargs): # ConnectionRouter is a pydantic model, so patch the class for B's sync only. with monkeypatch.context() as m: m.setattr(ConnectionRouter, "owner_write_event_messages_to_outbox", fail_upload) - with pytest.raises(ConnectionError): - do_manager_b.sync(auto_checkpoint=False) + do_manager_b.sync(auto_checkpoint=False) _assert_ds_receives_after_notebook_sync(ds_manager, do_manager) From 0b21f329870cd46d3d7bf8127d1b3543e824af8d Mon Sep 17 00:00:00 2001 From: Koen van der Veen Date: Tue, 27 Oct 2026 16:18:06 +0100 Subject: [PATCH 28/36] refactor: log a failed proposal as an error instead of raising Each failed proposal is logged as soon as it fails, and the sync goes on. --- syft/sync/sync/datasite_owner_syncer.py | 23 +++++++++-------------- tests/unit/test_delivery_tracker.py | 11 +++++++---- tests/unit/test_encryption.py | 11 +++++++---- 3 files changed, 23 insertions(+), 22 deletions(-) diff --git a/syft/sync/sync/datasite_owner_syncer.py b/syft/sync/sync/datasite_owner_syncer.py index 3cdbbe4072d..acd4f22d754 100644 --- a/syft/sync/sync/datasite_owner_syncer.py +++ b/syft/sync/sync/datasite_owner_syncer.py @@ -240,8 +240,8 @@ def sync(self, peer_emails: list[str], recompute_hashes: bool = True): one pass. Accepted proposals and the owner's own changes go out the same way: to - every peer that may read the file. A proposal that fails does not hold - back the send; its error raises after it. + every peer that may read the file. A proposal that fails is logged as + an error and does not hold back the send. """ if self.persist_owner_state: self._load_rolling_state() @@ -257,20 +257,17 @@ def sync(self, peer_emails: list[str], recompute_hashes: bool = True): # version of a file the owner changed is refused as outdated. if recompute_hashes: self._record_local_changes() - errors = [e for peer in peer_emails if (e := self._pull_proposals(peer))] + for peer_email in peer_emails: + self._pull_proposals(peer_email) self.deliver_file_changes(peer_emails) - for error in errors[1:]: - logger.error(f"Proposed changes failed: {error!r}") - if errors: - raise errors[0] finally: if self.persist_owner_state: self._save_rolling_state() - def _pull_proposals(self, peer_email: str) -> Exception | None: + def _pull_proposals(self, peer_email: str) -> None: """Apply every proposed change waiting from ``peer_email``. - Returns the error of a proposal that failed, so that it does not hold + A proposal that fails is logged as an error, so that it does not hold back the other peers or the owner's changes. """ proposed_count = 0 @@ -282,11 +279,9 @@ def _pull_proposals(self, peer_email: str) -> Exception | None: except SyftPeerNotReadyError as e: warnings.warn(f"Messages from {peer_email} stay in the inbox: {e}") except Exception as e: - return e - finally: - if proposed_count: - print(f"Received {proposed_count} proposed change(s) from {peer_email}") - return None + logger.error(f"Proposed changes from {peer_email} failed: {e!r}") + if proposed_count: + print(f"Received {proposed_count} proposed change(s) from {peer_email}") def download_events_message_by_id_with_connection( self, events_message_id: str diff --git a/tests/unit/test_delivery_tracker.py b/tests/unit/test_delivery_tracker.py index c1bd75b2c34..67ffca8505f 100644 --- a/tests/unit/test_delivery_tracker.py +++ b/tests/unit/test_delivery_tracker.py @@ -438,7 +438,7 @@ def test_large_catch_up_split_into_messages(monkeypatch): assert sorted(len(m.events) for m in messages) == [1, 2] -def test_outdated_proposal_does_not_block_owner_changes(): +def test_outdated_proposal_logged_without_blocking_owner_changes(caplog): ds_manager, do_manager = _pair() shared = do_manager.datasite_owner_syncer.perm_context.open("shared/") shared.grant_write_access(ds_manager.email) @@ -447,12 +447,15 @@ def test_outdated_proposal_does_not_block_owner_changes(): ds_manager.sync() # The DS edits a.txt from v1 while the DO edits it too, so the - # proposal is outdated and its handling raises on every sync. + # proposal is outdated and its handling fails on every sync. ds_manager._send_file_change(f"{do_manager.email}/shared/a.txt", "ds edit") _write(do_manager, "shared/a.txt", "do edit") _write(do_manager, "shared/b.txt", "new") - with pytest.raises(ProposedEventFileOutdatedException): - do_manager.sync(auto_checkpoint=False) + do_manager.sync(auto_checkpoint=False) + assert ProposedEventFileOutdatedException.__name__ in caplog.text + assert ( + do_manager.syftbox_folder / do_manager.email / "shared/a.txt" + ).read_text() == "do edit" ds_manager.sync() assert _received(ds_manager, do_manager, "shared/b.txt").read_text() == "new" diff --git a/tests/unit/test_encryption.py b/tests/unit/test_encryption.py index 21db1ae4f89..51567c8ef2a 100644 --- a/tests/unit/test_encryption.py +++ b/tests/unit/test_encryption.py @@ -610,7 +610,7 @@ def test_verified_encrypted_message_flow(): assert len(matching) > 0 -def test_tampered_inbox_message_raises(): +def test_tampered_inbox_message_logs_error(caplog): """Tampered message in mock inbox fails signature verification on sync.""" ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( encryption=True, @@ -633,6 +633,9 @@ def test_tampered_inbox_message_raises(): tampered[-1] ^= 0xFF mock_file.content = bytes(tampered) - # DO sync should fail due to signature verification - with pytest.raises(Exception): - do_manager.sync() + # DO sync logs the failed signature verification + do_manager.sync() + assert any( + r.levelname == "ERROR" and ds_manager.email in r.getMessage() + for r in caplog.records + ) From bd34a08ceceecfba7538b0a6b972f468f2c58460 Mon Sep 17 00:00:00 2001 From: Koen van der Veen Date: Tue, 27 Oct 2026 16:48:56 +0100 Subject: [PATCH 29/36] refactor: rename deliveries to delivery_tracker and flush once per sync _record_local_changes only queues the owner's log, and deliver_file_changes only queues the outbox; sync() flushes both at its end, the log first. The tracker drops queued sends and confirm_deliveries until a transport needs them. --- syft/sync/syftbox_manager.py | 2 +- syft/sync/sync/caches/datasite_owner_cache.py | 10 +-- syft/sync/sync/datasite_owner_syncer.py | 36 +++++------ syft/sync/sync/delivery_tracker.py | 61 +++---------------- tests/unit/test_delivery_tracker.py | 23 +++---- tests/unit/test_transport_contracts.py | 1 - 6 files changed, 37 insertions(+), 96 deletions(-) diff --git a/syft/sync/syftbox_manager.py b/syft/sync/syftbox_manager.py index 56699a4ca14..58f030801b8 100644 --- a/syft/sync/syftbox_manager.py +++ b/syft/sync/syftbox_manager.py @@ -1028,7 +1028,7 @@ def reject_peer_request(self, email_or_peer: str | Peer): peer_email = ( email_or_peer if isinstance(email_or_peer, str) else email_or_peer.email ) - self.datasite_owner_syncer.event_cache.deliveries.reset(peer_email) + self.datasite_owner_syncer.event_cache.delivery_tracker.reset(peer_email) def validate_peer(self, peer_email: str) -> Peer: """Load the peers, then return ``peer_email`` if it is approved. diff --git a/syft/sync/sync/caches/datasite_owner_cache.py b/syft/sync/sync/caches/datasite_owner_cache.py index b735c397fe6..cc06dc00345 100644 --- a/syft/sync/sync/caches/datasite_owner_cache.py +++ b/syft/sync/sync/caches/datasite_owner_cache.py @@ -68,7 +68,7 @@ class DataSiteOwnerEventCache(BaseModelCallbackMixin): # plain in-memory PersistedDict otherwise. file_hashes: PersistedDict = Field(default_factory=PersistedDict) # What each peer holds. Kept next to file_hashes when syftbox_folder is set. - deliveries: DeliveryTracker = Field( + delivery_tracker: DeliveryTracker = Field( default_factory=lambda: DeliveryTracker(MemoryDeliveryStateStore()) ) email: str @@ -91,10 +91,10 @@ def _build_file_hashes(cls, data): key_serializer=str, key_deserializer=Path, ) - if isinstance(data, dict) and "deliveries" not in data: + if isinstance(data, dict) and "delivery_tracker" not in data: folder = data.get("syftbox_folder") if folder is not None: - data["deliveries"] = DeliveryTracker( + data["delivery_tracker"] = DeliveryTracker( LocalDeliveryStateStore( Path(folder) / CACHE_DIR / DELIVERY_VIEWS_FILENAME ) @@ -110,7 +110,7 @@ def from_config(cls, config: DataSiteOwnerEventCacheConfig): email=config.email, syftbox_folder=config.syftbox_folder, file_hashes=PersistedDict(), - deliveries=DeliveryTracker(MemoryDeliveryStateStore()), + delivery_tracker=DeliveryTracker(MemoryDeliveryStateStore()), collections_folder=config.collections_folder, collection_variants=config.collection_variants, ) @@ -318,7 +318,7 @@ def clear_cache(self): self.events_messages_connection.clear_cache() self.file_connection.clear_cache() self.file_hashes.clear() - self.deliveries.clear() + self.delivery_tracker.clear() def has_conflict(self, proposed_event: ProposedFileChange) -> bool: if proposed_event.path_in_datasite not in self.file_hashes: diff --git a/syft/sync/sync/datasite_owner_syncer.py b/syft/sync/sync/datasite_owner_syncer.py index acd4f22d754..47e5fb58cc2 100644 --- a/syft/sync/sync/datasite_owner_syncer.py +++ b/syft/sync/sync/datasite_owner_syncer.py @@ -44,7 +44,6 @@ from syft.sync.sync.delivery_tracker import ( DeliveryPlan, ) -from syft.sync.transport.types import SendStatus from syft.sync.utils.path_filters import is_normal_syncable_path from syft.sync.utils.syftbox_utils import ( create_event_timestamp, @@ -260,6 +259,8 @@ def sync(self, peer_emails: list[str], recompute_hashes: bool = True): for peer_email in peer_emails: self._pull_proposals(peer_email) self.deliver_file_changes(peer_emails) + # The one flush of this pass: the owner's log first, then the outbox. + self.process_syftbox_events_queue() finally: if self.persist_owner_state: self._save_rolling_state() @@ -657,7 +658,8 @@ def _deletion_event(self, path: str) -> FileChangeEvent: def _record_local_changes(self) -> None: """Update the file hashes from the local files. With - ``persist_owner_state``, also write the changes to the owner's event log.""" + ``persist_owner_state``, also queue the changes for the owner's event + log; the caller flushes it.""" file_change_events_message = self.event_cache.process_local_file_changes() if file_change_events_message is None: return @@ -667,7 +669,6 @@ def _record_local_changes(self) -> None: # incremental/full checkpoints. self.syftbox_events_queue.put(file_change_events_message) self._add_events_to_rolling_state(file_change_events_message) - self.process_syftbox_events_queue() def _is_syncable(self, path: str) -> bool: """Whether a peer can get ``path`` by a delivery plan.""" @@ -710,19 +711,19 @@ def _seed_views_if_missing(self, peer_emails: list[str]) -> None: Each approved peer, and each peer in ``peer_emails``, is taken to hold every file it may read now. The caller reloads the permissions first. """ - deliveries = self.event_cache.deliveries - if deliveries.is_seeded(): + delivery_tracker = self.event_cache.delivery_tracker + if delivery_tracker.is_seeded(): return approved = [p.email for p in self.connection_router.peer_store.approved_peers] peers_to_seed = sorted({*approved, *peer_emails}) - deliveries.seed(self._readable_files_by_peer(peers_to_seed)) + delivery_tracker.seed(self._readable_files_by_peer(peers_to_seed)) def deliver_file_changes(self, peer_emails: list[str]) -> None: - """Send each peer the files its view lacks, and deletes for the files it - may no longer have. The caller loads the permissions first.""" - deliveries = self.event_cache.deliveries - deliveries.confirm_deliveries(self.connection_router.transport.p2p.delivered()) - plans = deliveries.plan(self._readable_files_by_peer(peer_emails)) + """Queue for each peer the files its view lacks, and deletes for the files + it may no longer have. The caller loads the permissions first, and sends + the queue with ``process_syftbox_events_queue``.""" + delivery_tracker = self.event_cache.delivery_tracker + plans = delivery_tracker.plan(self._readable_files_by_peer(peer_emails)) # Build every message before the first one is queued, so a read error # leaves no message of this plan in the queue. messages = [ @@ -734,7 +735,6 @@ def deliver_file_changes(self, peer_emails: list[str]) -> None: ] for message in messages: self.outbox_queue.put(message) - self.process_syftbox_events_queue() def _plan_events(self, plan: DeliveryPlan) -> list[FileChangeEvent]: """The events that carry out ``plan``: its sends, then its deletes.""" @@ -848,7 +848,7 @@ def _send_outbox_messages(self) -> None: if entry.recipient in not_ready: continue try: - status = self.connection_router.owner_write_event_messages_to_outbox( + self.connection_router.owner_write_event_messages_to_outbox( entry.recipient, entry.message ) except SyftPeerNotReadyError as e: @@ -857,7 +857,7 @@ def _send_outbox_messages(self) -> None: except Exception as e: pending.extend(self._retry_or_give_up(entry, e)) else: - self._record_sent(entry.recipient, entry.message, status) + self._record_sent(entry.recipient, entry.message) def _retry_or_give_up( self, entry: OutgoingMessage, error: Exception @@ -875,9 +875,7 @@ def _retry_or_give_up( ) return [entry._replace(attempts=attempts)] - def _record_sent( - self, recipient: str, message: FileChangeEventsMessage, status: SendStatus - ) -> None: + def _record_sent(self, recipient: str, message: FileChangeEventsMessage) -> None: """Add the changes of a sent message to ``recipient``'s view. A delete always counts. A file counts only if it is syncable and @@ -893,9 +891,7 @@ def _record_sent( changes[path] = None elif self._may_hold(recipient, path, syncable): changes[path] = event.new_hash - self.event_cache.deliveries.record_send( - recipient, message.message_filepath.as_string(), status, changes - ) + self.event_cache.delivery_tracker.record_send(recipient, changes) def _download_outbox_message_with_new_connection( self, file_id: str diff --git a/syft/sync/sync/delivery_tracker.py b/syft/sync/sync/delivery_tracker.py index 5f1492aeefa..1abde3a7485 100644 --- a/syft/sync/sync/delivery_tracker.py +++ b/syft/sync/sync/delivery_tracker.py @@ -1,8 +1,8 @@ """What each peer holds, so the data owner can resend, delete and catch up. The tracker knows nothing about the transport or where its record is kept. It -works on paths and content hashes, takes the outcome of each send as a -``SendStatus``, and keeps its record through a ``DeliveryStateStore``. +works on paths and content hashes, is told of each send that returned, and +keeps its record through a ``DeliveryStateStore``. Views ----- @@ -30,14 +30,7 @@ Recording a send ---------------- -``record_send`` takes the ``SendStatus`` from ``P2PConnection.send``: - -- ``DELIVERED``: the view changes at once. -- ``QUEUED``: the changes are held under their ``msg_id``, and the view changes - only when ``P2PConnection.delivered()`` reports that ``msg_id`` and - ``confirm_deliveries`` is called. A receipt is not enough; only the peer's ack counts. - -A view therefore never changes before a delivery is confirmed. A change that is +``record_send`` changes the view once a send returned. A change that is not sent, because the upload failed, the peer was skipped, or a process stopped before the send, stays in the plan of the next sync. @@ -53,12 +46,11 @@ from abc import ABC, abstractmethod from contextlib import nullcontext from pathlib import Path -from typing import ContextManager, Iterable, Mapping +from typing import ContextManager, Mapping from pydantic import BaseModel, ConfigDict, Field from syft.sync.sync.caches.persisted_dict import locked_file, write_text_atomic -from syft.sync.transport.types import SendStatus DELIVERY_VIEWS_FILENAME = "owner_delivery_views.json" # Format of DeliveryState. Raise it when the layout changes, and keep a read @@ -69,24 +61,12 @@ UNSENT_HASH = "" -class QueuedSend(BaseModel): - """Changes sent to ``peer`` that wait for confirmation. - - ``changes`` maps each path to its new hash, or None for a delete. - """ - - peer: str - changes: dict[str, str | None] - - class DeliveryState(BaseModel): """Everything the tracker keeps.""" version: int = DELIVERY_STATE_VERSION # peer email -> {path: hash} views: dict[str, dict[str, str]] = Field(default_factory=dict) - # msg_id -> the send it confirms - queued: dict[str, QueuedSend] = Field(default_factory=dict) # Whether ``DeliveryTracker.seed`` ran on this state. seeded: bool = False @@ -186,7 +166,7 @@ def __init__(self, store: DeliveryStateStore): self.store = store def clear(self) -> None: - """Forget the whole state: views, queued sends and the seeded flag.""" + """Forget the whole state: views and the seeded flag.""" with self.store.locked(): self.store.save(DeliveryState()) @@ -241,34 +221,9 @@ def plan( ) return plans - def record_send( - self, - peer: str, - msg_id: str, - status: SendStatus, - changes: Mapping[str, str | None], - ) -> None: - """Record a send of ``changes`` (path -> new hash, None for a delete). - - ``DELIVERED`` updates the view now; ``QUEUED`` waits for ``confirm_deliveries``. - """ - with self.store.locked(): - state = self.store.load() - if status is SendStatus.DELIVERED: - _record_delivery(state, peer, changes) - else: - assert status is SendStatus.QUEUED, status - state.queued[msg_id] = QueuedSend(peer=peer, changes=dict(changes)) - self.store.save(state) - - def confirm_deliveries(self, msg_ids: Iterable[str]) -> None: - """Apply the queued sends that ``P2PConnection.delivered()`` reported.""" - msg_ids = list(msg_ids) - if not msg_ids: - return + def record_send(self, peer: str, changes: Mapping[str, str | None]) -> None: + """Record a send of ``changes`` (path -> new hash, None for a delete).""" with self.store.locked(): state = self.store.load() - for msg_id in msg_ids: - queued = state.queued.pop(msg_id) - _record_delivery(state, queued.peer, queued.changes) + _record_delivery(state, peer, changes) self.store.save(state) diff --git a/tests/unit/test_delivery_tracker.py b/tests/unit/test_delivery_tracker.py index 67ffca8505f..82f511044dc 100644 --- a/tests/unit/test_delivery_tracker.py +++ b/tests/unit/test_delivery_tracker.py @@ -23,7 +23,6 @@ MemoryDeliveryStateStore, ) from syft.sync.syftbox_manager import SyftboxManager -from syft.sync.transport.types import SendStatus from syft_perms import SyftPermContext PEER = "ds@example.com" @@ -31,7 +30,7 @@ def _tracker_holding(view: dict[str, str]) -> DeliveryTracker: tracker = DeliveryTracker(MemoryDeliveryStateStore()) - tracker.record_send(PEER, "m0", SendStatus.DELIVERED, view) + tracker.record_send(PEER, view) return tracker @@ -44,7 +43,7 @@ def test_plan_sends_changed_files_deletes_unreadable(): def test_delivered_send_updates_view(): tracker = _tracker_holding({"a": "h1", "b": "h1"}) - tracker.record_send(PEER, "m1", SendStatus.DELIVERED, {"a": "h2", "b": None}) + tracker.record_send(PEER, {"a": "h2", "b": None}) assert tracker.plan({PEER: {"a": "h2"}})[PEER].send == [] assert tracker.plan({PEER: {"a": "h2"}})[PEER].delete == [] @@ -64,19 +63,9 @@ def test_reset_resends_readable_deletes_gone(): assert plan.delete == ["gone"] -def test_queued_send_updates_view_on_confirm(): - tracker = DeliveryTracker(MemoryDeliveryStateStore()) - tracker.record_send(PEER, "m1", SendStatus.QUEUED, {"a": "h1"}) - assert tracker.plan({PEER: {"a": "h1"}})[PEER].send == ["a"] - tracker.confirm_deliveries(["m1"]) - assert tracker.plan({PEER: {"a": "h1"}})[PEER].send == [] - - def test_local_store_persists_state_refuses_later_version(tmp_path): path = tmp_path / DELIVERY_VIEWS_FILENAME - DeliveryTracker(LocalDeliveryStateStore(path)).record_send( - PEER, "m1", SendStatus.DELIVERED, {"a": "h1"} - ) + DeliveryTracker(LocalDeliveryStateStore(path)).record_send(PEER, {"a": "h1"}) assert ( DeliveryTracker(LocalDeliveryStateStore(path)) .plan({PEER: {"a": "h1"}})[PEER] @@ -106,7 +95,7 @@ def load_then_wait_in_first_writer(self): def send(peer: str): tracker = DeliveryTracker(LocalDeliveryStateStore(path)) - tracker.record_send(peer, f"m-{peer}", SendStatus.DELIVERED, {"a": "h1"}) + tracker.record_send(peer, {"a": "h1"}) first = threading.Thread(target=send, args=("one@x.com",), name="first") second = threading.Thread(target=send, args=("two@x.com",), name="second") @@ -339,10 +328,11 @@ def test_peer_not_ready_does_not_block_other_peers(): syncer._record_local_changes() with pytest.warns(UserWarning, match=not_ready): syncer.deliver_file_changes([not_ready, ds_manager.email]) + syncer.process_syftbox_events_queue() assert _outbox_paths(ds_manager, do_manager) == ["shared/data.txt"] # The skipped peer's view did not change, so its next plan sends the file. readable = syncer._readable_files_by_peer([not_ready]) - assert syncer.event_cache.deliveries.plan(readable)[not_ready].send == [ + assert syncer.event_cache.delivery_tracker.plan(readable)[not_ready].send == [ "shared/data.txt" ] @@ -368,6 +358,7 @@ def fail_for_broken(self, recipient, message): ConnectionRouter, "owner_write_event_messages_to_outbox", fail_for_broken ) syncer.deliver_file_changes([broken, ds_manager.email]) + syncer.process_syftbox_events_queue() assert _outbox_paths(ds_manager, do_manager) == ["shared/data.txt"] tries = [r for r in caplog.records if broken in r.getMessage()] assert [r.levelname for r in tries] == ["INFO", "INFO", "ERROR"] diff --git a/tests/unit/test_transport_contracts.py b/tests/unit/test_transport_contracts.py index 598999f3547..467fb3aed46 100644 --- a/tests/unit/test_transport_contracts.py +++ b/tests/unit/test_transport_contracts.py @@ -129,4 +129,3 @@ def test_an_empty_delivery_state_is_versioned(): state = DeliveryState() assert state.version == DELIVERY_STATE_VERSION assert state.views == {} - assert state.queued == {} From 79e8b7eef0a98c1089891b3f39f44fcdeca63845 Mon Sep 17 00:00:00 2001 From: Koen van der Veen Date: Tue, 27 Oct 2026 16:54:24 +0100 Subject: [PATCH 30/36] refactor: write the owner's log before delivering to peers sync() flushes the owner's event log, then deliver_file_changes plans and sends. --- syft/sync/sync/datasite_owner_syncer.py | 10 +++++----- tests/unit/test_delivery_tracker.py | 2 -- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/syft/sync/sync/datasite_owner_syncer.py b/syft/sync/sync/datasite_owner_syncer.py index 47e5fb58cc2..da2fd0a24e6 100644 --- a/syft/sync/sync/datasite_owner_syncer.py +++ b/syft/sync/sync/datasite_owner_syncer.py @@ -258,9 +258,9 @@ def sync(self, peer_emails: list[str], recompute_hashes: bool = True): self._record_local_changes() for peer_email in peer_emails: self._pull_proposals(peer_email) - self.deliver_file_changes(peer_emails) - # The one flush of this pass: the owner's log first, then the outbox. + # The owner's log goes out before any peer gets a change. self.process_syftbox_events_queue() + self.deliver_file_changes(peer_emails) finally: if self.persist_owner_state: self._save_rolling_state() @@ -719,9 +719,8 @@ def _seed_views_if_missing(self, peer_emails: list[str]) -> None: delivery_tracker.seed(self._readable_files_by_peer(peers_to_seed)) def deliver_file_changes(self, peer_emails: list[str]) -> None: - """Queue for each peer the files its view lacks, and deletes for the files - it may no longer have. The caller loads the permissions first, and sends - the queue with ``process_syftbox_events_queue``.""" + """Send each peer the files its view lacks, and deletes for the files it + may no longer have. The caller loads the permissions first.""" delivery_tracker = self.event_cache.delivery_tracker plans = delivery_tracker.plan(self._readable_files_by_peer(peer_emails)) # Build every message before the first one is queued, so a read error @@ -735,6 +734,7 @@ def deliver_file_changes(self, peer_emails: list[str]) -> None: ] for message in messages: self.outbox_queue.put(message) + self._send_outbox_messages() def _plan_events(self, plan: DeliveryPlan) -> list[FileChangeEvent]: """The events that carry out ``plan``: its sends, then its deletes.""" diff --git a/tests/unit/test_delivery_tracker.py b/tests/unit/test_delivery_tracker.py index 82f511044dc..01bc3350e04 100644 --- a/tests/unit/test_delivery_tracker.py +++ b/tests/unit/test_delivery_tracker.py @@ -328,7 +328,6 @@ def test_peer_not_ready_does_not_block_other_peers(): syncer._record_local_changes() with pytest.warns(UserWarning, match=not_ready): syncer.deliver_file_changes([not_ready, ds_manager.email]) - syncer.process_syftbox_events_queue() assert _outbox_paths(ds_manager, do_manager) == ["shared/data.txt"] # The skipped peer's view did not change, so its next plan sends the file. readable = syncer._readable_files_by_peer([not_ready]) @@ -358,7 +357,6 @@ def fail_for_broken(self, recipient, message): ConnectionRouter, "owner_write_event_messages_to_outbox", fail_for_broken ) syncer.deliver_file_changes([broken, ds_manager.email]) - syncer.process_syftbox_events_queue() assert _outbox_paths(ds_manager, do_manager) == ["shared/data.txt"] tries = [r for r in caplog.records if broken in r.getMessage()] assert [r.levelname for r in tries] == ["INFO", "INFO", "ERROR"] From 6e96061b6697ce2389f5847e158cb025fe712c37 Mon Sep 17 00:00:00 2001 From: Koen van der Veen Date: Tue, 27 Oct 2026 18:03:52 +0100 Subject: [PATCH 31/36] refactor: one path for the owner's persisted state Every change goes into the rolling state, and into the owner's event log only when the rolling-state upload threshold is above 1. The rolling state is uploaded and saved locally at the flush; a failed upload is logged and tried again. A change before any rolling state exists raises. fetch_owner_state() reads checkpoints, rolling state and the event log in restore order, and both the restore and _get_all_accepted_events_do() use it. --- syft/sync/syftbox_manager.py | 6 +- syft/sync/sync/datasite_owner_syncer.py | 276 ++++++++++++++---------- tests/unit/test_checkpoints.py | 9 +- tests/unit/test_delivery_tracker.py | 5 +- tests/unit/test_drive_connections.py | 7 +- tests/unit/test_persisted_cache.py | 13 +- tests/unit/test_sync_manager.py | 1 + 7 files changed, 187 insertions(+), 130 deletions(-) diff --git a/syft/sync/syftbox_manager.py b/syft/sync/syftbox_manager.py index 58f030801b8..8235d03369e 100644 --- a/syft/sync/syftbox_manager.py +++ b/syft/sync/syftbox_manager.py @@ -1102,8 +1102,10 @@ def _add_connection(self, connection: SyftboxPlatformConnection): def _send_file_change(self, path: str | Path, content: str): self.file_writer.write_file(path, content) - def _get_all_accepted_events_do(self) -> List[FileChangeEvent]: - return self.datasite_owner_syncer.connection_router.owner_get_all_accepted_events_messages() + def _get_all_accepted_events_do(self) -> List[FileChangeEventsMessage]: + """The events of the owner's state on the backend since the last full + checkpoint, as a restore reads them.""" + return self.datasite_owner_syncer.fetch_owner_state().events_messages() @property def _connection_router(self) -> ConnectionRouter: diff --git a/syft/sync/sync/datasite_owner_syncer.py b/syft/sync/sync/datasite_owner_syncer.py index da2fd0a24e6..e53c2fa748a 100644 --- a/syft/sync/sync/datasite_owner_syncer.py +++ b/syft/sync/sync/datasite_owner_syncer.py @@ -5,7 +5,7 @@ from functools import partial from pathlib import Path from queue import Queue -from typing import List, Mapping, NamedTuple, Tuple +from typing import List, Mapping, NamedTuple from uuid import uuid4 from pydantic import BaseModel, ConfigDict, Field, PrivateAttr @@ -56,7 +56,9 @@ # Default threshold for creating incremental checkpoint from rolling state DEFAULT_CHECKPOINT_EVENT_THRESHOLD = 50 -# Default: upload rolling state to GDrive after this many events +# Upload the rolling state at a flush once this many events were added since +# the last upload. At 1 the rolling state on Drive holds every change after each +# flush, so the owner's event log is not written. DEFAULT_ROLLING_STATE_UPLOAD_THRESHOLD = 1 # A delivery message holds about this much file content at most. A larger @@ -72,17 +74,48 @@ MAX_SEND_ATTEMPTS = 3 -class _CheckpointRestore(NamedTuple): - """What restoring from checkpoints recovered. +class OwnerStateOnBackend(NamedTuple): + """The owner's persisted state on the backend, in the order a restore + applies it. - ``found_checkpoint`` is not the same as ``events_since_timestamp is not - None``: a full checkpoint may carry no event cursor, and in that case we - must NOT fall back to downloading every event. + ``log_messages`` are the event-log messages newer than the checkpoints and + the rolling state. When there is no checkpoint and no rolling state, they + are the whole log, and ``log_is_whole`` is True. A full checkpoint that + carries no event cursor gets no log messages: it is the whole state. """ - events_since_timestamp: float | None - events: List[FileChangeEvent] - found_checkpoint: bool + full_checkpoint: Checkpoint | None + incremental_checkpoints: List[IncrementalCheckpoint] + # None when the backend has no rolling state, or an empty one. + rolling_state: RollingState | None + log_messages: List[FileChangeEventsMessage] + log_is_whole: bool + + def events_messages(self) -> List[FileChangeEventsMessage]: + """Every event since the full checkpoint, in order.""" + messages = [ + FileChangeEventsMessage(events=checkpoint.events) + for checkpoint in self.incremental_checkpoints + ] + if self.rolling_state is not None: + messages.append(FileChangeEventsMessage(events=self.rolling_state.events)) + return messages + self.log_messages + + +def _cursor_after( + full_checkpoint: Checkpoint | None, + incrementals: List[IncrementalCheckpoint], + rolling_state: RollingState | None = None, +) -> float | None: + """The timestamp of the last event that the checkpoints and the rolling + state hold. The event log is read after it.""" + since = full_checkpoint.last_event_timestamp if full_checkpoint else None + for checkpoint in incrementals: + for event in checkpoint.events: + since = _later_timestamp(since, event.timestamp) + if rolling_state is not None: + since = _later_timestamp(since, rolling_state.last_event_timestamp) + return since def _later_timestamp(current: float | None, candidate: float | None) -> float | None: @@ -324,109 +357,98 @@ def pull_initial_state(self): def _restore_owner_state(self) -> None: """Rebuild the cache from the owner's persisted state on the backend.""" - restore = self._restore_from_checkpoints() - - if restore.events_since_timestamp is not None: - self._download_events_since(restore.events_since_timestamp) - elif not restore.found_checkpoint: - print("No checkpoints found, downloading all events...") - self._download_all_events() + state = self.fetch_owner_state( + whole_log_since=self.event_cache.latest_cached_timestamp + ) + self._apply_full_checkpoint(state.full_checkpoint) + events = self._apply_incremental_checkpoints(state.incremental_checkpoints) + events += self._apply_rolling_state(state) + self._apply_log_messages(state) # The restore above only populates file_hashes and file_connection, not # events_messages_connection, which get_cached_events() reads from. - if restore.events and not self.event_cache.get_cached_events(): - self._write_events_to_messages_cache(restore.events) + if events and not self.event_cache.get_cached_events(): + self._write_events_to_messages_cache(events) - def _restore_from_checkpoints(self) -> _CheckpointRestore: - """Apply the full checkpoint, then the incrementals, then the rolling state. + def fetch_owner_state( + self, whole_log_since: float | None = None + ) -> OwnerStateOnBackend: + """Download the owner's persisted state: the full checkpoint, the + incrementals, the rolling state, and the event-log messages after them. - Each step restores cache state and moves the event cursor forward. + ``whole_log_since`` limits the log when it is read whole, because + there is no checkpoint and no rolling state. """ - full_checkpoint = self.connection_router.get_latest_checkpoint() - since = self._apply_full_checkpoint(full_checkpoint) - - incremental_cps = self.connection_router.get_all_incremental_checkpoints() - since, events = self._apply_incremental_checkpoints(incremental_cps, since) - - since, rolling_events = self._apply_rolling_state(since) - return _CheckpointRestore( - events_since_timestamp=since, - events=events + rolling_events, - found_checkpoint=full_checkpoint is not None or bool(incremental_cps), + router = self.connection_router + full_checkpoint = router.get_latest_checkpoint() + incrementals = router.get_all_incremental_checkpoints() + rolling_state = router.get_rolling_state() + if rolling_state is not None and rolling_state.event_count == 0: + rolling_state = None + since = _cursor_after(full_checkpoint, incrementals, rolling_state) + log_is_whole = since is None and full_checkpoint is None and not incrementals + if since is not None: + log_messages = router.get_events_messages_since_timestamp(since) + elif log_is_whole: + log_messages = self.get_all_accepted_events_messages(whole_log_since) + else: + log_messages = [] + return OwnerStateOnBackend( + full_checkpoint, incrementals, rolling_state, log_messages, log_is_whole ) - def _apply_full_checkpoint(self, checkpoint: Checkpoint | None) -> float | None: - """Restore a full snapshot. Returns the event cursor it carries.""" + def _apply_full_checkpoint(self, checkpoint: Checkpoint | None) -> None: + """Restore a full snapshot.""" if checkpoint is None: - return None + return print(f"Found full checkpoint with {len(checkpoint.files)} files, restoring...") self.event_cache.apply_checkpoint(checkpoint, write_files=self.write_files) - return checkpoint.last_event_timestamp def _apply_incremental_checkpoints( - self, - checkpoints: List[IncrementalCheckpoint], - since: float | None, - ) -> Tuple[float | None, List[FileChangeEvent]]: - """Apply incrementals in order, advancing the cursor past their events.""" + self, checkpoints: List[IncrementalCheckpoint] + ) -> List[FileChangeEvent]: + """Apply incrementals in order. Returns their events.""" if not checkpoints: - return since, [] - + return [] print(f"Found {len(checkpoints)} incremental checkpoints, applying...") events: List[FileChangeEvent] = [] for inc_cp in checkpoints: self._apply_incremental_checkpoint_to_cache(inc_cp) events.extend(inc_cp.events) - for event in inc_cp.events: - since = _later_timestamp(since, event.timestamp) - return since, events - - def _apply_rolling_state( - self, since: float | None - ) -> Tuple[float | None, List[FileChangeEvent]]: - """Apply the rolling state, or start an empty one when there is none.""" - rolling_state = self.connection_router.get_rolling_state() - if rolling_state is None or rolling_state.event_count == 0: + return events + + def _apply_rolling_state(self, state: OwnerStateOnBackend) -> List[FileChangeEvent]: + """Apply the rolling state, or start an empty one when there is none. + Returns its events.""" + rolling_state = state.rolling_state + if rolling_state is None: + since = _cursor_after(state.full_checkpoint, state.incremental_checkpoints) self._rolling_state = RollingState( - email=self.email, - base_checkpoint_timestamp=since or 0.0, + email=self.email, base_checkpoint_timestamp=since or 0.0 ) - return since, [] - + return [] print( f"Found rolling state with {rolling_state.event_count} events, applying..." ) self._apply_rolling_state_to_cache(rolling_state) self._rolling_state = rolling_state - return ( - _later_timestamp(since, rolling_state.last_event_timestamp), - rolling_state.events, - ) - - def _download_events_since(self, timestamp: float) -> None: - """Download and cache every events message newer than `timestamp`.""" - events_messages = self.connection_router.get_events_messages_since_timestamp( - timestamp - ) - if not events_messages: - return + return rolling_state.events - print( - f"Downloading {len(events_messages)} events since " - "checkpoint/rolling state..." - ) - for events_message in events_messages: - self.event_cache.add_events_message_to_local_cache(events_message) - self._add_events_to_rolling_state(events_message) - - def _download_all_events(self) -> None: - """Download every accepted events message - the no-checkpoint path.""" - since_timestamp = self.event_cache.latest_cached_timestamp - events_messages_list: List[FileChangeEventsMessage] = ( - self.get_all_accepted_events_messages(since_timestamp=since_timestamp) - ) - for events_message in events_messages_list: + def _apply_log_messages(self, state: OwnerStateOnBackend) -> None: + """Cache the event-log messages after the checkpoints and the rolling + state. Only those that follow a checkpoint or a rolling state go into + the rolling state.""" + if state.log_is_whole: + print("No checkpoints found, downloading all events...") + elif state.log_messages: + print( + f"Downloading {len(state.log_messages)} events since " + "checkpoint/rolling state..." + ) + for events_message in state.log_messages: self.event_cache.add_events_message_to_local_cache(events_message) + if not state.log_is_whole: + self._add_events_to_rolling_state(events_message) def _apply_incremental_checkpoint_to_cache( self, checkpoint: IncrementalCheckpoint @@ -657,18 +679,11 @@ def _deletion_event(self, path: str) -> FileChangeEvent: ) def _record_local_changes(self) -> None: - """Update the file hashes from the local files. With - ``persist_owner_state``, also queue the changes for the owner's event - log; the caller flushes it.""" + """Update the file hashes from the local files, and add the changes to + the owner's persisted state. The caller flushes it.""" file_change_events_message = self.event_cache.process_local_file_changes() - if file_change_events_message is None: - return - if not self.persist_owner_state: - return - # Feed the events into rolling state so they flow through - # incremental/full checkpoints. - self.syftbox_events_queue.put(file_change_events_message) - self._add_events_to_rolling_state(file_change_events_message) + if file_change_events_message is not None: + self._add_to_owner_state(file_change_events_message) def _is_syncable(self, path: str) -> bool: """Whether a peer can get ``path`` by a delivery plan.""" @@ -802,31 +817,54 @@ def handle_proposed_filechange_events_message( ) # The next plan sends the accepted change to every peer that may read # the file, the submitter included. - if accepted_events_message is not None and self.persist_owner_state: - self.syftbox_events_queue.put(accepted_events_message) - self._add_events_to_rolling_state(accepted_events_message) + if accepted_events_message is not None: + self._add_to_owner_state(accepted_events_message) def queue_event_for_syftbox( self, recipients: list[str], file_change_events_message: FileChangeEventsMessage ): - # The syftbox queue is the owner's own append-only log, read back only - # to restore this datasite. The outbox is how peers get the change, so - # it is queued either way. - if self.persist_owner_state: - self.syftbox_events_queue.put(file_change_events_message) - + self._add_to_owner_state(file_change_events_message) for recipient in recipients: self.outbox_queue.put( OutgoingMessage(recipient, file_change_events_message) ) + def _add_to_owner_state(self, events_message: FileChangeEventsMessage) -> None: + """Add ``events_message`` to the owner's persisted state, read back only + to restore this datasite. + + It goes into the rolling state, and into the owner's event log unless + the rolling state holds every change after each flush. A rolling state + that is not loaded yet is loaded from disk. + + Raises: + RuntimeError: when there is no rolling state on disk either, because + this datasite never synced here. + """ + if not self.persist_owner_state: + return + if self._rolling_state is None: + self._load_rolling_state() + if self._rolling_state is None: + raise RuntimeError( + "Sync this datasite before changing it: its owner state is not loaded." + ) + if DEFAULT_ROLLING_STATE_UPLOAD_THRESHOLD > 1: + self.syftbox_events_queue.put(events_message) + self._add_events_to_rolling_state(events_message) + def process_syftbox_events_queue(self): + """Flush the owner's persisted state, then the outbox.""" # TODO: make this atomic while not self.syftbox_events_queue.empty(): file_change_events_message = self.syftbox_events_queue.get() self.connection_router.owner_write_events_message_to_syftbox( file_change_events_message ) + if self.persist_owner_state: + self._upload_rolling_state_if_due() + # Also outside sync(), whose start reloads the rolling state from disk. + self._save_rolling_state() if not self.outbox_queue.empty(): self._send_outbox_messages() @@ -1011,26 +1049,28 @@ def _apply_rolling_state_to_cache(self, rolling_state: RollingState) -> None: ) def _add_events_to_rolling_state( - self, - events_message: FileChangeEventsMessage, - upload_threshold: int = DEFAULT_ROLLING_STATE_UPLOAD_THRESHOLD, + self, events_message: FileChangeEventsMessage ) -> None: - """ - Add events to the in-memory rolling state and upload if threshold is reached. - - Args: - events_message: The events to add. - upload_threshold: Upload to GDrive after this many events added. - """ + """Add events to the in-memory rolling state. The next flush uploads it.""" if self._rolling_state is None: return - self._rolling_state.add_events_message(events_message) self._events_since_rolling_state_upload += len(events_message.events) - # Upload if threshold reached - if self._events_since_rolling_state_upload >= upload_threshold: + def _upload_rolling_state_if_due(self) -> None: + """Upload the rolling state once enough events were added since the last + upload. A failed upload is logged, and the next flush tries again.""" + if ( + self._events_since_rolling_state_upload + < DEFAULT_ROLLING_STATE_UPLOAD_THRESHOLD + ): + return + try: self._upload_rolling_state() + except Exception as e: + logger.error( + f"Rolling state upload failed, the next sync tries again: {e!r}" + ) def _upload_rolling_state(self) -> None: """Upload the in-memory rolling state to GDrive.""" diff --git a/tests/unit/test_checkpoints.py b/tests/unit/test_checkpoints.py index ca561bc60a3..6f54a66cf50 100644 --- a/tests/unit/test_checkpoints.py +++ b/tests/unit/test_checkpoints.py @@ -12,6 +12,7 @@ import pytest +from syft.sync.sync import datasite_owner_syncer from syft.sync.checkpoints.checkpoint import ( Checkpoint, IncrementalCheckpoint, @@ -143,8 +144,12 @@ def test_checkpoint_restore_on_sync(): assert len(do_cache.file_hashes) == 2 -def test_checkpoint_events_since(): +def test_checkpoint_events_since(monkeypatch): """Test getting events since checkpoint timestamp.""" + # At 1 the event log is not written. + monkeypatch.setattr( + datasite_owner_syncer, "DEFAULT_ROLLING_STATE_UPLOAD_THRESHOLD", 2 + ) ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( use_in_memory_cache=True, ) @@ -732,6 +737,6 @@ def test_owner_state_is_written_by_default(): do_manager.sync(auto_checkpoint=False) router = do_manager._connection_router - assert router.owner_get_all_accepted_event_file_ids() != [] + assert do_manager._get_all_accepted_events_do() != [] assert router.get_rolling_state() is not None assert do_manager.datasite_owner_syncer._rolling_state is not None diff --git a/tests/unit/test_delivery_tracker.py b/tests/unit/test_delivery_tracker.py index 01bc3350e04..358266eff6c 100644 --- a/tests/unit/test_delivery_tracker.py +++ b/tests/unit/test_delivery_tracker.py @@ -119,9 +119,12 @@ def send(peer: str): def _pair(**kwargs): - return SyftboxManager.pair_with_mock_drive_service_connection( + """A DS and DO pair. The DO synced once, so its owner state is loaded.""" + ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( use_in_memory_cache=False, **kwargs ) + do_manager.sync(auto_checkpoint=False) + return ds_manager, do_manager def _write(do_manager, path: str, content: str): diff --git a/tests/unit/test_drive_connections.py b/tests/unit/test_drive_connections.py index eabfbd7135c..bd735ed3898 100644 --- a/tests/unit/test_drive_connections.py +++ b/tests/unit/test_drive_connections.py @@ -11,6 +11,7 @@ import pytest import syft.sync.connections.connection_router as connection_router_module +from syft.sync.sync import datasite_owner_syncer from syft.sync.connections.connection_router import ConnectionRouter from syft.sync.connections.drive.gdrive_transport import CollectionFolder from syft.sync.connections.drive.mock_drive_service import pair_with_mock_service @@ -320,9 +321,13 @@ def test_the_router_holds_no_drive_specific_code(): # -- restoring the owner's state -------------------------------------------------- -def test_restoring_from_the_event_log_keeps_the_newest_version(): +def test_restoring_from_the_event_log_keeps_the_newest_version(monkeypatch): """A fresh client that restores from the event log alone replays the messages oldest first, so a file changed twice ends at its last version.""" + # At 1 the event log is not written. + monkeypatch.setattr( + datasite_owner_syncer, "DEFAULT_ROLLING_STATE_UPLOAD_THRESHOLD", 2 + ) _, do = SyftboxManager.pair_with_mock_drive_service_connection( use_in_memory_cache=True ) diff --git a/tests/unit/test_persisted_cache.py b/tests/unit/test_persisted_cache.py index 06e68f74323..a6a462225c2 100644 --- a/tests/unit/test_persisted_cache.py +++ b/tests/unit/test_persisted_cache.py @@ -15,6 +15,11 @@ ) +def _owner_event_count(do_manager) -> int: + """The events in the owner's state on the backend.""" + return sum(len(m.events) for m in do_manager._get_all_accepted_events_do()) + + def test_no_duplicate_events_across_processes(): """Two managers sharing a syftbox_folder must not create duplicate events for the same file change. @@ -40,17 +45,13 @@ def test_no_duplicate_events_across_processes(): # Process A syncs — detects changes, uploads events, saves cache do_manager.sync(auto_checkpoint=False) - events_after_a = len( - do_manager.datasite_owner_syncer.connection_router.owner_get_all_accepted_event_file_ids() - ) + events_after_a = _owner_event_count(do_manager) assert events_after_a > 0 # Process B (syft-bg) syncs — must NOT re-create the same events do_manager_b = do_manager._copy() do_manager_b.sync(auto_checkpoint=False) - events_after_b = len( - do_manager_b.datasite_owner_syncer.connection_router.owner_get_all_accepted_event_file_ids() - ) + events_after_b = _owner_event_count(do_manager_b) # Verify cache files were created cache_dir = do_manager.syftbox_folder / CACHE_DIR diff --git a/tests/unit/test_sync_manager.py b/tests/unit/test_sync_manager.py index aab37c58eea..20a33da8c5d 100644 --- a/tests/unit/test_sync_manager.py +++ b/tests/unit/test_sync_manager.py @@ -53,6 +53,7 @@ def test_valid_and_invalid_proposed_filechange_event(): do_manager.datasite_owner_syncer.perm_context.open(".").grant_write_access( ds_manager.email ) + do_manager.sync() ds_email = ds_manager.email do_email = do_manager.email From dc29e6ce887e0bdc543c3a69cb14f57b05d43c1f Mon Sep 17 00:00:00 2001 From: Koen van der Veen Date: Tue, 27 Oct 2026 18:07:19 +0100 Subject: [PATCH 32/36] docs: give a correct example of an owner that seeds its views An enclave keeps no owner state by default, so it never restores its file hashes from Drive. --- syft/sync/sync/delivery_tracker.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/syft/sync/sync/delivery_tracker.py b/syft/sync/sync/delivery_tracker.py index 1abde3a7485..45c447d0f89 100644 --- a/syft/sync/sync/delivery_tracker.py +++ b/syft/sync/sync/delivery_tracker.py @@ -23,10 +23,11 @@ that the owner rejects and then approves again is sent every file it may read, and a delete for each file that it may no longer have. -An owner that lost its views file, for example an enclave that wipes its local -state at every boot, has file hashes but no views. ``seed`` then starts the -view of each approved peer with every file it may read now, so the owner does -not send every file again. A view that a send already recorded is kept. +An owner that lost its views file but kept or restored its file hashes, for +example one with ``persist_owner_state`` on a new machine, has file hashes but +no views. ``seed`` then starts the view of each approved peer with every file +it may read now, so the owner does not send every file again. A view that a +send already recorded is kept. Recording a send ---------------- From ae8f5bbb14ee73fbf40dd53bd359ddb1e6523da9 Mon Sep 17 00:00:00 2001 From: Koen van der Veen Date: Tue, 27 Oct 2026 18:19:03 +0100 Subject: [PATCH 33/36] refactor: save the owner state and send to peers in separate methods --- .../syft-enclave/src/syft_enclaves/client.py | 9 +++------ packages/syft-rds/src/syft_rds/client.py | 3 +-- syft/sync/sync/datasite_owner_syncer.py | 16 +++++++++------- tests/unit/test_delivery_tracker.py | 12 +++++------- tests/unit/test_encryption_policy.py | 2 +- tests/unit/test_sync_manager.py | 2 +- 6 files changed, 20 insertions(+), 24 deletions(-) diff --git a/packages/syft-enclave/src/syft_enclaves/client.py b/packages/syft-enclave/src/syft_enclaves/client.py index 8d7613a7fa0..954d56c73b3 100644 --- a/packages/syft-enclave/src/syft_enclaves/client.py +++ b/packages/syft-enclave/src/syft_enclaves/client.py @@ -595,11 +595,10 @@ def _push_files_to_recipients( events_message = syncer.event_cache.create_events_for_files( files_by_datasite_path ) - syncer.queue_event_for_syftbox( + syncer.send_events_to_peers( recipients=recipients, file_change_events_message=events_message, ) - syncer.process_syftbox_events_queue() def _push_approval_file(self, approval_file: Path) -> None: """Push a changed approval file to the enclave.""" @@ -704,11 +703,10 @@ def _forward_job_to_dos(self, job_dir: Path, do_emails: list[str]): events_message = syncer.event_cache.create_events_for_files( files_by_datasite_path ) - syncer.queue_event_for_syftbox( + syncer.send_events_to_peers( recipients=do_emails, file_change_events_message=events_message, ) - syncer.process_syftbox_events_queue() def _forward_approval_files_to_dos(self, review_dir: Path, do_emails: list[str]): """Forward each DO's approval state file to them individually.""" @@ -722,11 +720,10 @@ def _forward_approval_files_to_dos(self, review_dir: Path, do_emails: list[str]) events_message = syncer.event_cache.create_events_for_files( files_by_datasite_path ) - syncer.queue_event_for_syftbox( + syncer.send_events_to_peers( recipients=[do_email], file_change_events_message=events_message, ) - syncer.process_syftbox_events_queue() def _get_files_in_dir(self, directory: Path) -> dict[Path, bytes]: """Read all files under directory, keyed by path relative to the datasite root.""" diff --git a/packages/syft-rds/src/syft_rds/client.py b/packages/syft-rds/src/syft_rds/client.py index f994babcc4f..50a3bf528fd 100644 --- a/packages/syft-rds/src/syft_rds/client.py +++ b/packages/syft-rds/src/syft_rds/client.py @@ -1249,11 +1249,10 @@ def share_private_dataset(self, tag: str, enclave_email: str): events_message = self.sync_engine.datasite_owner_syncer.event_cache.create_events_for_files( files ) - self.sync_engine.datasite_owner_syncer.queue_event_for_syftbox( + self.sync_engine.datasite_owner_syncer.send_events_to_peers( recipients=[enclave_email], file_change_events_message=events_message, ) - self.sync_engine.datasite_owner_syncer.process_syftbox_events_queue() def _private_share_protocol_version(self, tag: str, peer_email: str) -> str: """The protocol version of the copy to ship privately to this peer. diff --git a/syft/sync/sync/datasite_owner_syncer.py b/syft/sync/sync/datasite_owner_syncer.py index e53c2fa748a..ede2e59ccc0 100644 --- a/syft/sync/sync/datasite_owner_syncer.py +++ b/syft/sync/sync/datasite_owner_syncer.py @@ -292,7 +292,7 @@ def sync(self, peer_emails: list[str], recompute_hashes: bool = True): for peer_email in peer_emails: self._pull_proposals(peer_email) # The owner's log goes out before any peer gets a change. - self.process_syftbox_events_queue() + self.save_owner_state() self.deliver_file_changes(peer_emails) finally: if self.persist_owner_state: @@ -820,14 +820,18 @@ def handle_proposed_filechange_events_message( if accepted_events_message is not None: self._add_to_owner_state(accepted_events_message) - def queue_event_for_syftbox( + def send_events_to_peers( self, recipients: list[str], file_change_events_message: FileChangeEventsMessage - ): + ) -> None: + """Add ``file_change_events_message`` to the owner state, save that + state, then send the message to each of ``recipients``.""" self._add_to_owner_state(file_change_events_message) + self.save_owner_state() for recipient in recipients: self.outbox_queue.put( OutgoingMessage(recipient, file_change_events_message) ) + self._send_outbox_messages() def _add_to_owner_state(self, events_message: FileChangeEventsMessage) -> None: """Add ``events_message`` to the owner's persisted state, read back only @@ -853,8 +857,8 @@ def _add_to_owner_state(self, events_message: FileChangeEventsMessage) -> None: self.syftbox_events_queue.put(events_message) self._add_events_to_rolling_state(events_message) - def process_syftbox_events_queue(self): - """Flush the owner's persisted state, then the outbox.""" + def save_owner_state(self): + """Write the owner's event log, and upload and save its rolling state.""" # TODO: make this atomic while not self.syftbox_events_queue.empty(): file_change_events_message = self.syftbox_events_queue.get() @@ -865,8 +869,6 @@ def process_syftbox_events_queue(self): self._upload_rolling_state_if_due() # Also outside sync(), whose start reloads the rolling state from disk. self._save_rolling_state() - if not self.outbox_queue.empty(): - self._send_outbox_messages() def _send_outbox_messages(self) -> None: """Send the queued messages, and record each sent one in its peer's view. diff --git a/tests/unit/test_delivery_tracker.py b/tests/unit/test_delivery_tracker.py index 358266eff6c..1fd9a28c138 100644 --- a/tests/unit/test_delivery_tracker.py +++ b/tests/unit/test_delivery_tracker.py @@ -169,10 +169,9 @@ def _push_private_file(do_manager, ds_manager) -> None: message = syncer.event_cache.create_events_for_files( {Path("private/data.csv"): b"rows"} ) - syncer.queue_event_for_syftbox( + syncer.send_events_to_peers( recipients=[ds_manager.email], file_change_events_message=message ) - syncer.process_syftbox_events_queue() def test_peer_approved_later_gets_existing_files(): @@ -315,7 +314,7 @@ def peer_not_ready(self, recipient, *args, **kwargs): with pytest.warns(UserWarning, match="are not sent"): _push_private_file(do_manager, ds_manager) - do_manager.datasite_owner_syncer.process_syftbox_events_queue() + do_manager.datasite_owner_syncer._send_outbox_messages() assert _sent_events(ds_manager, do_manager, "private/data.csv") == [] @@ -372,11 +371,10 @@ def test_send_to_stranger_does_not_block_later_syncs(): message = syncer.event_cache.create_events_for_files( {Path("private/data.csv"): b"rows"} ) - syncer.queue_event_for_syftbox( - recipients=["stranger@example.com"], file_change_events_message=message - ) with pytest.warns(UserWarning, match="stranger@example.com"): - syncer.process_syftbox_events_queue() + syncer.send_events_to_peers( + recipients=["stranger@example.com"], file_change_events_message=message + ) syncer.perm_context.open("shared/").grant_read_access(ds_manager.email) _write(do_manager, "shared/data.txt", "one") diff --git a/tests/unit/test_encryption_policy.py b/tests/unit/test_encryption_policy.py index 85cb0471eb1..dad4165ab32 100644 --- a/tests/unit/test_encryption_policy.py +++ b/tests/unit/test_encryption_policy.py @@ -436,7 +436,7 @@ def test_owner_send_to_non_peer_warns(): message = FileChangeEventsMessage(events=[]) syncer.outbox_queue.put(OutgoingMessage("former@example.com", message)) with pytest.warns(UserWarning, match="not known"): - syncer.process_syftbox_events_queue() + syncer._send_outbox_messages() def test_owner_leaves_envelope_from_peer_without_bundle_in_inbox(): diff --git a/tests/unit/test_sync_manager.py b/tests/unit/test_sync_manager.py index 20a33da8c5d..ff516214a4f 100644 --- a/tests/unit/test_sync_manager.py +++ b/tests/unit/test_sync_manager.py @@ -855,7 +855,7 @@ def test_accepted_delete_reaches_every_reader(): ], ) syncer.handle_proposed_filechange_events_message(ds_manager.email, proposal) - syncer.process_syftbox_events_queue() + syncer.save_owner_state() syncer.sync(peers) assert ("shared/data.txt", True) in _events_for( ds_manager._connection_router, do_manager From d05d19418acde1a6f5018dae68912c5beb5299f1 Mon Sep 17 00:00:00 2001 From: Koen van der Veen Date: Tue, 27 Oct 2026 18:22:41 +0100 Subject: [PATCH 34/36] refactor: rename send_events_to_peers to process_events_for_own_state_and_peers --- packages/syft-enclave/src/syft_enclaves/client.py | 6 +++--- packages/syft-rds/src/syft_rds/client.py | 2 +- syft/sync/sync/datasite_owner_syncer.py | 2 +- tests/unit/test_delivery_tracker.py | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/syft-enclave/src/syft_enclaves/client.py b/packages/syft-enclave/src/syft_enclaves/client.py index 954d56c73b3..99a1772f900 100644 --- a/packages/syft-enclave/src/syft_enclaves/client.py +++ b/packages/syft-enclave/src/syft_enclaves/client.py @@ -595,7 +595,7 @@ def _push_files_to_recipients( events_message = syncer.event_cache.create_events_for_files( files_by_datasite_path ) - syncer.send_events_to_peers( + syncer.process_events_for_own_state_and_peers( recipients=recipients, file_change_events_message=events_message, ) @@ -703,7 +703,7 @@ def _forward_job_to_dos(self, job_dir: Path, do_emails: list[str]): events_message = syncer.event_cache.create_events_for_files( files_by_datasite_path ) - syncer.send_events_to_peers( + syncer.process_events_for_own_state_and_peers( recipients=do_emails, file_change_events_message=events_message, ) @@ -720,7 +720,7 @@ def _forward_approval_files_to_dos(self, review_dir: Path, do_emails: list[str]) events_message = syncer.event_cache.create_events_for_files( files_by_datasite_path ) - syncer.send_events_to_peers( + syncer.process_events_for_own_state_and_peers( recipients=[do_email], file_change_events_message=events_message, ) diff --git a/packages/syft-rds/src/syft_rds/client.py b/packages/syft-rds/src/syft_rds/client.py index 50a3bf528fd..0066807958d 100644 --- a/packages/syft-rds/src/syft_rds/client.py +++ b/packages/syft-rds/src/syft_rds/client.py @@ -1249,7 +1249,7 @@ def share_private_dataset(self, tag: str, enclave_email: str): events_message = self.sync_engine.datasite_owner_syncer.event_cache.create_events_for_files( files ) - self.sync_engine.datasite_owner_syncer.send_events_to_peers( + self.sync_engine.datasite_owner_syncer.process_events_for_own_state_and_peers( recipients=[enclave_email], file_change_events_message=events_message, ) diff --git a/syft/sync/sync/datasite_owner_syncer.py b/syft/sync/sync/datasite_owner_syncer.py index ede2e59ccc0..378e6292fd1 100644 --- a/syft/sync/sync/datasite_owner_syncer.py +++ b/syft/sync/sync/datasite_owner_syncer.py @@ -820,7 +820,7 @@ def handle_proposed_filechange_events_message( if accepted_events_message is not None: self._add_to_owner_state(accepted_events_message) - def send_events_to_peers( + def process_events_for_own_state_and_peers( self, recipients: list[str], file_change_events_message: FileChangeEventsMessage ) -> None: """Add ``file_change_events_message`` to the owner state, save that diff --git a/tests/unit/test_delivery_tracker.py b/tests/unit/test_delivery_tracker.py index 1fd9a28c138..273c08ccc4b 100644 --- a/tests/unit/test_delivery_tracker.py +++ b/tests/unit/test_delivery_tracker.py @@ -169,7 +169,7 @@ def _push_private_file(do_manager, ds_manager) -> None: message = syncer.event_cache.create_events_for_files( {Path("private/data.csv"): b"rows"} ) - syncer.send_events_to_peers( + syncer.process_events_for_own_state_and_peers( recipients=[ds_manager.email], file_change_events_message=message ) @@ -372,7 +372,7 @@ def test_send_to_stranger_does_not_block_later_syncs(): {Path("private/data.csv"): b"rows"} ) with pytest.warns(UserWarning, match="stranger@example.com"): - syncer.send_events_to_peers( + syncer.process_events_for_own_state_and_peers( recipients=["stranger@example.com"], file_change_events_message=message ) From 81d25243bef684558660c01a1735f406d21d3a8e Mon Sep 17 00:00:00 2001 From: Koen van der Veen Date: Tue, 27 Oct 2026 18:23:51 +0100 Subject: [PATCH 35/36] refactor: rename _add_to_owner_state to _queue_for_owner_state --- syft/sync/sync/datasite_owner_syncer.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/syft/sync/sync/datasite_owner_syncer.py b/syft/sync/sync/datasite_owner_syncer.py index 378e6292fd1..2ef758926af 100644 --- a/syft/sync/sync/datasite_owner_syncer.py +++ b/syft/sync/sync/datasite_owner_syncer.py @@ -683,7 +683,7 @@ def _record_local_changes(self) -> None: the owner's persisted state. The caller flushes it.""" file_change_events_message = self.event_cache.process_local_file_changes() if file_change_events_message is not None: - self._add_to_owner_state(file_change_events_message) + self._queue_for_owner_state(file_change_events_message) def _is_syncable(self, path: str) -> bool: """Whether a peer can get ``path`` by a delivery plan.""" @@ -818,14 +818,14 @@ def handle_proposed_filechange_events_message( # The next plan sends the accepted change to every peer that may read # the file, the submitter included. if accepted_events_message is not None: - self._add_to_owner_state(accepted_events_message) + self._queue_for_owner_state(accepted_events_message) def process_events_for_own_state_and_peers( self, recipients: list[str], file_change_events_message: FileChangeEventsMessage ) -> None: """Add ``file_change_events_message`` to the owner state, save that state, then send the message to each of ``recipients``.""" - self._add_to_owner_state(file_change_events_message) + self._queue_for_owner_state(file_change_events_message) self.save_owner_state() for recipient in recipients: self.outbox_queue.put( @@ -833,7 +833,7 @@ def process_events_for_own_state_and_peers( ) self._send_outbox_messages() - def _add_to_owner_state(self, events_message: FileChangeEventsMessage) -> None: + def _queue_for_owner_state(self, events_message: FileChangeEventsMessage) -> None: """Add ``events_message`` to the owner's persisted state, read back only to restore this datasite. From 0b2cdf42bdf247bc23f6f03ac907c13fe0c130e0 Mon Sep 17 00:00:00 2001 From: Koen van der Veen Date: Tue, 27 Oct 2026 18:41:21 +0100 Subject: [PATCH 36/36] refactor: apply quickfix findings 2-9 (unused router method, names, docs) --- docs/connections.md | 2 +- syft/sync/connections/connection_router.py | 9 -------- syft/sync/sync/datasite_owner_syncer.py | 26 +++++++++++++--------- syft/sync/sync/delivery_tracker.py | 18 ++++++--------- 4 files changed, 23 insertions(+), 32 deletions(-) diff --git a/docs/connections.md b/docs/connections.md index 0feede8fe27..b1cc561aa70 100644 --- a/docs/connections.md +++ b/docs/connections.md @@ -61,7 +61,7 @@ A pair encrypts when one side is `required`. A SyftBox has one policy, kept in `//private/syft_settings.json`. The first client to start on a new SyftBox writes it: the `encryption` it logged in with, or `optional`. A later client, the notebook or syft-bg, logs in without `encryption` and gets the policy on disk. A client that asks for another policy raises `EncryptionPolicyMismatchError`, and so does a running client at its next sync when the file changed under it. Files at rest are read with the current policy, so to change the policy, delete the SyftBox (`client.delete_syftbox()`) and log in again. `syft-bg init --encryption ...` sets the policy the same way as a login. -Nothing is sent to a peer until its policy is known, that is, until it accepts and shares its version file. A scientist's change for a peer whose policy or bundle is not known is dropped, and the send raises. An owner's change for that peer waits, with a warning, and a later sync sends it. Another owner send to that peer raises. An owner's change to a file that the peer may read is not lost: the owner keeps a record of what each peer holds, and that record changes only after a send, so a later sync sends it. Other owner sends, such as a private dataset share or an enclave result, are not kept. A send that fails raises, and the send must be made again. An owner keeps a message it cannot read yet in the inbox, with a warning. +Nothing is sent to a peer until its policy is known, that is, until it accepts and shares its version file. A scientist's change for a peer whose policy or bundle is not known is dropped, and the send raises. An owner's change for that peer waits, with a warning, and a later sync sends it. Other owner sends to that peer are skipped with a warning. An owner's change to a file that the peer may read is not lost: the owner keeps a record of what each peer holds, and that record changes only after a send, so a later sync sends it. Other owner sends, such as a private dataset share or an enclave result, are not kept. A failed send is tried up to three times in the same sync, then logged as an error. An owner keeps a message it cannot read yet in the inbox, with a warning. The format of a message must match the pair: plaintext in a pair that encrypts is refused, and so is an envelope in a pair that does not. The owner archives such a message unread, and a scientist skips it, each with a warning. diff --git a/syft/sync/connections/connection_router.py b/syft/sync/connections/connection_router.py index 685972bea4b..2ccf05718cd 100644 --- a/syft/sync/connections/connection_router.py +++ b/syft/sync/connections/connection_router.py @@ -251,15 +251,6 @@ def owner_write_events_message_to_syftbox( filename = events_message.message_filepath.as_string() self.transport.owner_store.append_event(filename, data) - def owner_get_all_accepted_events_messages(self) -> List[FileChangeEventsMessage]: - raw_list = self.transport.owner_store.fetch_all_events() - return [ - FileChangeEventsMessage.from_compressed_data( - self.peer_store.decrypt_and_verify_for_self_if_needed(raw) - ) - for raw in raw_list - ] - def download_events_message_by_id( self, events_message_id: str ) -> FileChangeEventsMessage: diff --git a/syft/sync/sync/datasite_owner_syncer.py b/syft/sync/sync/datasite_owner_syncer.py index 2ef758926af..8e01312751b 100644 --- a/syft/sync/sync/datasite_owner_syncer.py +++ b/syft/sync/sync/datasite_owner_syncer.py @@ -56,9 +56,9 @@ # Default threshold for creating incremental checkpoint from rolling state DEFAULT_CHECKPOINT_EVENT_THRESHOLD = 50 -# Upload the rolling state at a flush once this many events were added since -# the last upload. At 1 the rolling state on Drive holds every change after each -# flush, so the owner's event log is not written. +# save_owner_state() uploads the rolling state once this many events were added +# since the last upload. At 1 the rolling state on Drive holds every change after +# each save_owner_state(), so the owner's event log is not written. DEFAULT_ROLLING_STATE_UPLOAD_THRESHOLD = 1 # A delivery message holds about this much file content at most. A larger @@ -102,7 +102,7 @@ def events_messages(self) -> List[FileChangeEventsMessage]: return messages + self.log_messages -def _cursor_after( +def _last_persisted_event_timestamp( full_checkpoint: Checkpoint | None, incrementals: List[IncrementalCheckpoint], rolling_state: RollingState | None = None, @@ -385,7 +385,9 @@ def fetch_owner_state( rolling_state = router.get_rolling_state() if rolling_state is not None and rolling_state.event_count == 0: rolling_state = None - since = _cursor_after(full_checkpoint, incrementals, rolling_state) + since = _last_persisted_event_timestamp( + full_checkpoint, incrementals, rolling_state + ) log_is_whole = since is None and full_checkpoint is None and not incrementals if since is not None: log_messages = router.get_events_messages_since_timestamp(since) @@ -422,7 +424,9 @@ def _apply_rolling_state(self, state: OwnerStateOnBackend) -> List[FileChangeEve Returns its events.""" rolling_state = state.rolling_state if rolling_state is None: - since = _cursor_after(state.full_checkpoint, state.incremental_checkpoints) + since = _last_persisted_event_timestamp( + state.full_checkpoint, state.incremental_checkpoints + ) self._rolling_state = RollingState( email=self.email, base_checkpoint_timestamp=since or 0.0 ) @@ -680,7 +684,7 @@ def _deletion_event(self, path: str) -> FileChangeEvent: def _record_local_changes(self) -> None: """Update the file hashes from the local files, and add the changes to - the owner's persisted state. The caller flushes it.""" + the owner's persisted state. The caller then calls ``save_owner_state``.""" file_change_events_message = self.event_cache.process_local_file_changes() if file_change_events_message is not None: self._queue_for_owner_state(file_change_events_message) @@ -710,7 +714,7 @@ def _readable_files_by_peer( # peer email -> {path: hash} return { peer: { - p: h for p, h in syncable.items() if self._may_hold(peer, p, syncable) + p: h for p, h in syncable.items() if self.perm_context.can_read(p, peer) } for peer in peer_emails } @@ -838,7 +842,7 @@ def _queue_for_owner_state(self, events_message: FileChangeEventsMessage) -> Non to restore this datasite. It goes into the rolling state, and into the owner's event log unless - the rolling state holds every change after each flush. A rolling state + the rolling state holds every change after each ``save_owner_state``. A rolling state that is not loaded yet is loaded from disk. Raises: @@ -1053,7 +1057,7 @@ def _apply_rolling_state_to_cache(self, rolling_state: RollingState) -> None: def _add_events_to_rolling_state( self, events_message: FileChangeEventsMessage ) -> None: - """Add events to the in-memory rolling state. The next flush uploads it.""" + """Add events to the in-memory rolling state. The next ``save_owner_state`` uploads it.""" if self._rolling_state is None: return self._rolling_state.add_events_message(events_message) @@ -1061,7 +1065,7 @@ def _add_events_to_rolling_state( def _upload_rolling_state_if_due(self) -> None: """Upload the rolling state once enough events were added since the last - upload. A failed upload is logged, and the next flush tries again.""" + upload. A failed upload is logged, and the next ``save_owner_state`` tries again.""" if ( self._events_since_rolling_state_upload < DEFAULT_ROLLING_STATE_UPLOAD_THRESHOLD diff --git a/syft/sync/sync/delivery_tracker.py b/syft/sync/sync/delivery_tracker.py index 45c447d0f89..0a14ff946b2 100644 --- a/syft/sync/sync/delivery_tracker.py +++ b/syft/sync/sync/delivery_tracker.py @@ -14,8 +14,8 @@ in the view, but no longer readable -> delete in the view, but the file is gone -> delete -A change that the owner accepts from a peer goes to that peer at once. Every -other peer that may read the file gets it in its next plan, as with any change. +A change that the owner accepts from a peer goes, like any change, to every +peer that may read the file, that peer included. A new peer starts with an empty view, so it is sent every file it may read. When the owner rejects a peer, ``reset`` sets each path in its view to @@ -141,6 +141,8 @@ def save(self, state: DeliveryState) -> None: def _apply_changes_to_view( view: dict[str, str], changes: Mapping[str, str | None] ) -> None: + """Apply ``changes`` (path -> new hash, None for a delete) to ``view`` + (path -> hash).""" for path, new_hash in changes.items(): if new_hash is None: view.pop(path, None) @@ -148,15 +150,8 @@ def _apply_changes_to_view( view[path] = new_hash -def _record_delivery( - state: DeliveryState, peer: str, changes: Mapping[str, str | None] -) -> None: - """Apply ``changes`` to ``peer``'s view.""" - _apply_changes_to_view(state.views.setdefault(peer, {}), changes) - - class DeliveryTracker: - """Per-peer views of confirmed file versions, for the data owner. + """Per-peer views of the file versions sent to each peer, for the data owner. Each call reads the state from the store. Each change reads and writes it back under the store lock, so the processes of one datasite see the same @@ -226,5 +221,6 @@ def record_send(self, peer: str, changes: Mapping[str, str | None]) -> None: """Record a send of ``changes`` (path -> new hash, None for a delete).""" with self.store.locked(): state = self.store.load() - _record_delivery(state, peer, changes) + peer_view = state.views.setdefault(peer, {}) + _apply_changes_to_view(peer_view, changes) self.store.save(state)