From 93636aaceb45dd88bfc9d014d65429f385d259f6 Mon Sep 17 00:00:00 2001 From: Blaine Jester Date: Thu, 8 Oct 2026 13:32:15 -0700 Subject: [PATCH 1/4] Add outcome resolution to deserialization task Co-Authored-By: Claude Opus 5.5 (1M context) --- morango/sync/stream/deserialize.py | 42 +++++++++++++- .../tests/sync/stream/test_deserialize.py | 56 +++++++++++++++++++ 2 files changed, 97 insertions(+), 1 deletion(-) diff --git a/morango/sync/stream/deserialize.py b/morango/sync/stream/deserialize.py index 38121987..2f423397 100644 --- a/morango/sync/stream/deserialize.py +++ b/morango/sync/stream/deserialize.py @@ -12,16 +12,30 @@ from morango.sync.stream.source import SourceTask +class DeserializeOutcome: + """The resolved action for a `DeserializeTask`, which the sink applies""" + + SAVE = "save" + DELETE = "delete" + PROPAGATE_DELETE = "propagate_delete" + PROPAGATE_HARD_DELETE = "propagate_hard_delete" + ERROR = "error" + + class DeserializeTask(SourceTask): """Carrier class for providing context through the deserialization pipeline.""" - __slots__ = ("store", "app_model", "fk_cache", "errors") + __slots__ = ("store", "app_model", "fk_cache", "errors", "_propagated_deletion") + + HARD_DELETION = "hard" + SOFT_DELETION = "soft" def __init__(self, store: Store, fk_cache: Dict): self.store = store self.fk_cache: Dict = fk_cache self.app_model: Optional[SyncableModel] = None self.errors: List[Exception] = [] + self._propagated_deletion: Optional[str] = None @property def id(self) -> str: @@ -35,9 +49,35 @@ def model(self) -> Type[SyncableModel]: def has_errors(self) -> bool: return len(self.errors) > 0 + @property + def outcome(self) -> str: + """ + The resolved `DeserializeOutcome` for this task, in order of precedence + :raises AssertionError: if the pipeline has not resolved an outcome + """ + if self.has_errors: + return DeserializeOutcome.ERROR + if self.store.deleted: + return DeserializeOutcome.DELETE + if self._propagated_deletion == self.HARD_DELETION: + return DeserializeOutcome.PROPAGATE_HARD_DELETE + if self._propagated_deletion == self.SOFT_DELETION: + return DeserializeOutcome.PROPAGATE_DELETE + if self.app_model is not None: + return DeserializeOutcome.SAVE + raise AssertionError("DeserializeTask has no resolved outcome") + + def self_referential_fk(self) -> Optional[str]: + """Return the attname of the self-referential FK on the task's model, or ``None``.""" + return syncable_models.get_self_referential_fk(self.model) + def set_app_model(self, app_model: Optional[SyncableModel]) -> None: self.app_model = app_model + def set_propagated_deletion(self, hard: bool) -> None: + """Mark this task's record as deleted because a record it references was deleted""" + self._propagated_deletion = self.HARD_DELETION if hard else self.SOFT_DELETION + def add_error(self, error: Exception) -> None: self.errors.append(error) diff --git a/tests/testapp/tests/sync/stream/test_deserialize.py b/tests/testapp/tests/sync/stream/test_deserialize.py index 41f7c870..5c75fe23 100644 --- a/tests/testapp/tests/sync/stream/test_deserialize.py +++ b/tests/testapp/tests/sync/stream/test_deserialize.py @@ -7,6 +7,7 @@ from morango.models.certificates import Filter from morango.models.core import Store from morango.models.core import SyncableModel +from morango.sync.stream.deserialize import DeserializeOutcome from morango.sync.stream.deserialize import DeserializeTask from morango.sync.stream.deserialize import StoreModelSource @@ -18,6 +19,7 @@ def setUp(self): self.store = mock.Mock(spec_set=Store) self.store.profile = "test" self.store.model_name = "testmodel" + self.store.deleted = False self.task = DeserializeTask(self.store, {}) @mock.patch("morango.sync.stream.deserialize.syncable_models.get_model") @@ -38,6 +40,60 @@ def test_set_app_model(self): self.task.set_app_model(app_model) self.assertEqual(self.task.app_model, app_model) + @mock.patch("morango.sync.stream.deserialize.syncable_models.get_self_referential_fk") + @mock.patch("morango.sync.stream.deserialize.syncable_models.get_model") + def test_self_referential_fk(self, mock_get_model, mock_get_self_referential_fk): + model = mock.Mock(spec_set=SyncableModel) + mock_get_model.return_value = model + mock_get_self_referential_fk.return_value = "parent_id" + + self.assertEqual(self.task.self_referential_fk(), "parent_id") + mock_get_self_referential_fk.assert_called_once_with(model) + + def test_outcome__save(self): + self.task.set_app_model(mock.Mock(spec_set=SyncableModel)) + self.assertEqual(self.task.outcome, DeserializeOutcome.SAVE) + + def test_outcome__delete(self): + self.store.deleted = True + self.assertEqual(self.task.outcome, DeserializeOutcome.DELETE) + + def test_outcome__propagate_delete(self): + self.task.set_propagated_deletion(hard=False) + self.assertEqual(self.task.outcome, DeserializeOutcome.PROPAGATE_DELETE) + + def test_outcome__propagate_hard_delete(self): + self.task.set_propagated_deletion(hard=True) + self.assertEqual(self.task.outcome, DeserializeOutcome.PROPAGATE_HARD_DELETE) + + def test_outcome__error(self): + self.task.add_error(ValueError("bad data")) + self.assertEqual(self.task.outcome, DeserializeOutcome.ERROR) + + def test_outcome__error_precedes_delete(self): + self.store.deleted = True + self.task.add_error(ValueError("bad data")) + self.assertEqual(self.task.outcome, DeserializeOutcome.ERROR) + + def test_outcome__error_precedes_propagated_deletion(self): + self.task.set_propagated_deletion(hard=True) + self.task.add_error(ValueError("bad data")) + self.assertEqual(self.task.outcome, DeserializeOutcome.ERROR) + + def test_outcome__delete_precedes_propagated_deletion(self): + self.store.deleted = True + self.task.set_propagated_deletion(hard=True) + self.assertEqual(self.task.outcome, DeserializeOutcome.DELETE) + + def test_outcome__propagated_deletion_precedes_save(self): + self.task.set_app_model(mock.Mock(spec_set=SyncableModel)) + self.task.set_propagated_deletion(hard=False) + self.assertEqual(self.task.outcome, DeserializeOutcome.PROPAGATE_DELETE) + + def test_outcome__unresolved(self): + with self.assertRaises(AssertionError): + _ = self.task.outcome + class StoreModelSourcePrefixConditionsTestCase(SimpleTestCase): """`prefix_conditions` is pure ordering logic over the sync filter and touches no ORM""" From 12b3880007f09f936a0f1ca3e64893e0db52faab Mon Sep 17 00:00:00 2001 From: Blaine Jester Date: Thu, 8 Oct 2026 13:32:16 -0700 Subject: [PATCH 2/4] Add flag to toggle parent dereference in test Facility validation Co-Authored-By: Claude Opus 5.5 (1M context) --- tests/testapp/facility_profile/models.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/testapp/facility_profile/models.py b/tests/testapp/facility_profile/models.py index 3bf3cbef..37457471 100644 --- a/tests/testapp/facility_profile/models.py +++ b/tests/testapp/facility_profile/models.py @@ -24,6 +24,10 @@ class SyncableUserModelManager(SyncableModelManager, UserManager): class Facility(FacilityDataSyncableModel): morango_model_name = "facility" + # when True, clean_fields loads the parent object, which raises ObjectDoesNotExist if the + # parent row is not in the app table; used to test non-validation error handling + clean_dereferences_parent = True + name = models.CharField(max_length=100) now_date = models.DateTimeField(default=timezone.now) parent = models.ForeignKey( @@ -45,8 +49,9 @@ def calculate_partition(self, *args, **kwargs): return "{id}".format(id=self.ID_PLACEHOLDER) def clean_fields(self, *args, **kwargs): - # reference parent here just to trigger a non-validation error to make sure we handle it - _ = self.parent + if self.clean_dereferences_parent: + # reference parent here just to trigger a non-validation error to make sure we handle it + _ = self.parent super(Facility, self).clean_fields(*args, **kwargs) From b608f2059b18ad9925fd4904cc761e1dde27d667 Mon Sep 17 00:00:00 2001 From: Blaine Jester Date: Thu, 8 Oct 2026 13:41:34 -0700 Subject: [PATCH 3/4] Add transforms for deserializing and validating store records Co-Authored-By: Claude Opus 5.5 (1M context) --- morango/sync/stream/deserialize.py | 228 ++++++++ .../tests/sync/stream/test_deserialize.py | 485 ++++++++++++++++++ 2 files changed, 713 insertions(+) diff --git a/morango/sync/stream/deserialize.py b/morango/sync/stream/deserialize.py index 2f423397..c8414408 100644 --- a/morango/sync/stream/deserialize.py +++ b/morango/sync/stream/deserialize.py @@ -1,16 +1,38 @@ +import json +from collections import namedtuple from typing import Dict from typing import Generator from typing import List from typing import Optional +from typing import Set +from typing import Tuple from typing import Type +from django.core import exceptions +from django.db.models import ForeignKey +from django.db.utils import IntegrityError + +from morango.errors import MorangoDirtyParent +from morango.errors import MorangoMissingParent from morango.models.certificates import Filter from morango.models.core import Store from morango.models.core import SyncableModel from morango.registry import syncable_models +from morango.sync.stream.core import Transform from morango.sync.stream.source import MorangoSource from morango.sync.stream.source import SourceTask +DIRTY_PARENT_MESSAGE = "Parent is dirty; could not deserialize." +MISSING_PARENT_MESSAGE = "Parent does not exist in Store; could not deserialize." + +# the store state of an FK target, used to resolve validation failures +_TargetState = namedtuple("_TargetState", ["deleted", "hard_deleted", "dirty_bit"]) + + +def _fk_cache_key(model: Type[SyncableModel], record_id: str) -> str: + """The key `SyncableModel.cached_clean_fields` uses for a record in the `fk_cache`""" + return "{id}_{db_table}".format(id=record_id, db_table=model._meta.db_table) + class DeserializeOutcome: """The resolved action for a `DeserializeTask`, which the sink applies""" @@ -132,3 +154,209 @@ def stream_for_filter( for store_model in qs.iterator(): yield DeserializeTask(store_model, self.fk_cache) + + +class AppModelDeserialize(Transform[DeserializeTask]): + """ + Builds the unsaved app model from each task's serialized store data. Deleted store records + pass through untouched, since deleting the app model is left to the sink. + """ + + def __init__(self, sync_filter: Optional[Filter] = None): + self.sync_filter = sync_filter + + def transform(self, task: DeserializeTask) -> DeserializeTask: + if task.store.deleted: + return task + + try: + app_model = task.model.deserialize( + json.loads(task.store.serialized), sync_filter=self.sync_filter + ) + except (ValueError, exceptions.ValidationError) as e: + # `json.JSONDecodeError` is a `ValueError` + task.add_error(e) + return task + + app_model._morango_source_id = task.store.source_id + app_model._morango_partition = task.store.partition + app_model._morango_dirty_bit = False + task.set_app_model(app_model) + return task + + +class AppModelValidate(Transform[DeserializeTask]): + """ + Validates each task's app model and resolves its outcome. The source streams FK targets ahead + of the records that reference them, so a target validated earlier in the run is already in + the shared `fk_cache` even though the sink may not have written it yet. A cache miss falls back + to looking up the target in the app tables. + + Must be instantiated per pipeline run, since it tracks which records failed or were deleted + during the run, so that the records referencing them can be resolved accordingly. + + Deletions spread to referencing records as in legacy deserialization. For models without a + self-referential FK, a reference to any deleted record spreads its deletion. For models with + one, only a validation failure of `ObjectDoesNotExist` spreads it. + """ + + def __init__(self, sync_filter: Optional[Filter] = None): + self.sync_filter = sync_filter + self.failed_ids: Set[str] = set() + # maps the ID of a deleted record to whether the deletion is a hard deletion + self.deleted_ids: Dict[str, bool] = {} + + def transform(self, task: DeserializeTask) -> DeserializeTask: + if task.has_errors: + self.failed_ids.add(task.id) + return task + + if task.store.deleted: + self.deleted_ids[task.id] = task.store.hard_deleted + return task + + fk_references = self._fk_references(task.app_model) + + for field, target_id in fk_references: + if target_id in self.deleted_ids: + if task.self_referential_fk() is not None: + # the sink has not deleted the target's app row yet, so validation can't + # report the missing target; fail as legacy does once the row is gone + self._fail(task, self._deleted_target_error(field, target_id)) + else: + self._propagate_deletion(task, hard=self.deleted_ids[target_id]) + return task + + for field, target_id in fk_references: + if target_id in self.failed_ids: + self._fail(task, self._failed_target_error(task, field, target_id)) + return task + + try: + task.app_model.cached_clean_fields(task.fk_cache, sync_filter=self.sync_filter) + except (exceptions.ValidationError, exceptions.ObjectDoesNotExist, ValueError) as e: + self._classify_failure(task, fk_references, e) + return task + + @staticmethod + def _fk_references(app_model: SyncableModel) -> List[Tuple[ForeignKey, str]]: + """The FK fields on the app model that hold a value, paired with that value""" + references = [] + for field in app_model._meta.fields: + if not isinstance(field, ForeignKey): + continue + target_id = getattr(app_model, field.attname) + if target_id is not None: + references.append((field, target_id)) + return references + + @staticmethod + def _failed_target_error(task: DeserializeTask, field: ForeignKey, target_id: str) -> Exception: + if field.attname == task.self_referential_fk(): + return MorangoDirtyParent(DIRTY_PARENT_MESSAGE) + return exceptions.ValidationError( + "{} with id {} failed to deserialize".format( + field.related_model._meta.verbose_name, target_id + ) + ) + + @staticmethod + def _deleted_target_error(field: ForeignKey, target_id: str) -> Exception: + return exceptions.ValidationError( + "{} with id {} was deleted".format(field.related_model._meta.verbose_name, target_id) + ) + + def _classify_failure( + self, + task: DeserializeTask, + fk_references: List[Tuple[ForeignKey, str]], + error: Exception, + ) -> None: + """ + Resolves a validation failure using the store records of the FK targets. Only runs on + failure, so it costs at most one query per failed record and none per valid record. + """ + targets = self._lookup_targets(fk_references) + + # a reference to a deleted record propagates the deletion, instead of failing + if task.self_referential_fk() is None or isinstance(error, exceptions.ObjectDoesNotExist): + for _, target_id in fk_references: + target = targets.get(target_id) + if target is not None and (target.deleted or target.hard_deleted): + self._propagate_deletion(task, hard=target.hard_deleted) + return + + self._fail(task, self._failure_error(task, fk_references, targets, error)) + + @staticmethod + def _lookup_targets( + fk_references: List[Tuple[ForeignKey, str]], + ) -> Dict[str, _TargetState]: + """The store state of each FK target that has a store record, keyed by the target ID""" + if not fk_references: + return {} + target_ids = [target_id for _, target_id in fk_references] + return { + target_id: _TargetState(deleted, hard_deleted, dirty_bit) + for target_id, deleted, hard_deleted, dirty_bit in Store.objects.filter( + id__in=target_ids + ).values_list("id", "deleted", "hard_deleted", "dirty_bit") + } + + def _failure_error( + self, + task: DeserializeTask, + fk_references: List[Tuple[ForeignKey, str]], + targets: Dict[str, _TargetState], + error: Exception, + ) -> Exception: + """ + The error to record for a failure that did not propagate a deletion. Problems with the + parent take precedence, then missing targets, otherwise the original validation error. + """ + self_ref_fk = task.self_referential_fk() + for field, target_id in fk_references: + if field.attname != self_ref_fk: + continue + if target_id not in targets: + return MorangoMissingParent(MISSING_PARENT_MESSAGE) + if targets[target_id].dirty_bit and self._parent_not_written( + task, field, target_id, error + ): + return MorangoDirtyParent(DIRTY_PARENT_MESSAGE) + + for field, target_id in fk_references: + if target_id not in targets: + return IntegrityError( + "{from_model}.{from_field} references non-existent {to_model} instance with " + "id '{to_pk}'".format( + from_model=task.model.__name__, + from_field=field.attname, + to_model=field.related_model._meta.verbose_name, + to_pk=target_id, + ) + ) + + return error + + @staticmethod + def _parent_not_written( + task: DeserializeTask, field: ForeignKey, parent_id: str, error: Exception + ) -> bool: + """ + Whether a failure is explained by a dirty parent's app row not being written. The parent + is in `fk_cache` when its FK resolved, either because it was accepted earlier in the run or + because an older app row exists, and then only loading the parent object can fail on it. + Any other error is the record's own. + """ + return _fk_cache_key(field.related_model, parent_id) not in task.fk_cache or isinstance( + error, exceptions.ObjectDoesNotExist + ) + + def _propagate_deletion(self, task: DeserializeTask, hard: bool) -> None: + task.set_propagated_deletion(hard) + self.deleted_ids[task.id] = hard + + def _fail(self, task: DeserializeTask, error: Exception) -> None: + task.add_error(error) + self.failed_ids.add(task.id) diff --git a/tests/testapp/tests/sync/stream/test_deserialize.py b/tests/testapp/tests/sync/stream/test_deserialize.py index 5c75fe23..491929f8 100644 --- a/tests/testapp/tests/sync/stream/test_deserialize.py +++ b/tests/testapp/tests/sync/stream/test_deserialize.py @@ -1,12 +1,22 @@ import uuid import mock +from django.core.exceptions import ValidationError +from django.core.serializers.json import DjangoJSONEncoder +from django.db.utils import IntegrityError from django.test import SimpleTestCase from django.test import TestCase +from facility_profile.models import ConditionalLog +from facility_profile.models import Facility +from facility_profile.models import SummaryLog +from morango.errors import MorangoDirtyParent +from morango.errors import MorangoMissingParent from morango.models.certificates import Filter from morango.models.core import Store from morango.models.core import SyncableModel +from morango.sync.stream.deserialize import AppModelDeserialize +from morango.sync.stream.deserialize import AppModelValidate from morango.sync.stream.deserialize import DeserializeOutcome from morango.sync.stream.deserialize import DeserializeTask from morango.sync.stream.deserialize import StoreModelSource @@ -95,6 +105,481 @@ def test_outcome__unresolved(self): _ = self.task.outcome +def _facility_store(facility_id=None, parent_id=None, **kwargs): + """An unsaved `Store` record holding a serialized `Facility`""" + facility_id = facility_id or uuid.uuid4().hex + facility = Facility(id=facility_id, name="Facility {}".format(facility_id), parent_id=parent_id) + store_kwargs = dict( + id=facility_id, + profile="facilitydata", + model_name="facility", + partition=facility_id, + source_id=facility.name, + serialized=DjangoJSONEncoder().encode(facility.serialize()), + last_saved_instance=uuid.uuid4().hex, + last_saved_counter=1, + ) + store_kwargs.update(kwargs) + return Store(**store_kwargs) + + +class AppModelDeserializeTestCase(SimpleTestCase): + def setUp(self): + self.store = _facility_store() + self.task = DeserializeTask(self.store, {}) + + def test_transform__deserializes_app_model(self): + task = AppModelDeserialize().transform(self.task) + + self.assertIs(task, self.task) + self.assertIsInstance(task.app_model, Facility) + self.assertEqual(task.app_model.id, self.store.id) + self.assertEqual(task.outcome, DeserializeOutcome.SAVE) + + def test_transform__sets_morango_fields_from_store(self): + task = AppModelDeserialize().transform(self.task) + + self.assertEqual(task.app_model._morango_source_id, self.store.source_id) + self.assertEqual(task.app_model._morango_partition, self.store.partition) + self.assertFalse(task.app_model._morango_dirty_bit) + + def test_transform__passes_sync_filter(self): + sync_filter = Filter(self.store.partition) + with mock.patch.object(Facility, "deserialize", wraps=Facility.deserialize) as deserialize: + AppModelDeserialize(sync_filter=sync_filter).transform(self.task) + + deserialize.assert_called_once_with(mock.ANY, sync_filter=sync_filter) + + def test_transform__deleted_store(self): + """Deleting the app model is the sink's job, so the deleted record passes through as-is""" + self.store.deleted = True + with mock.patch.object(Facility, "deserialize") as deserialize: + task = AppModelDeserialize().transform(self.task) + + deserialize.assert_not_called() + self.assertIsNone(task.app_model) + self.assertEqual(task.outcome, DeserializeOutcome.DELETE) + + def test_transform__invalid_json(self): + self.store.serialized = "{bad" + task = AppModelDeserialize().transform(self.task) + + self.assertIsNone(task.app_model) + self.assertIsInstance(task.errors[0], ValueError) + self.assertEqual(task.outcome, DeserializeOutcome.ERROR) + + def test_transform__validation_error(self): + error = ValidationError("bad value") + with mock.patch.object(Facility, "deserialize", side_effect=error): + task = AppModelDeserialize().transform(self.task) + + self.assertIsNone(task.app_model) + self.assertEqual(task.errors, [error]) + + def test_transform__unexpected_error_propagates(self): + """Only per-record data errors are recorded; anything else is a defect and must surface""" + with mock.patch.object(Facility, "deserialize", side_effect=RuntimeError("defect")): + with self.assertRaises(RuntimeError): + AppModelDeserialize().transform(self.task) + + +def _validate_task(app_model, fk_cache=None, **store_kwargs): + """A task carrying an already deserialized `app_model`, as `AppModelDeserialize` leaves it""" + store = Store( + id=app_model.id, + profile=app_model.morango_profile, + model_name=app_model.morango_model_name, + **store_kwargs, + ) + task = DeserializeTask(store, fk_cache if fk_cache is not None else {}) + task.set_app_model(app_model) + return task + + +class AppModelValidateTestCase(SimpleTestCase): + """ + Covers the in-run checks that run ahead of model validation. Model validation itself is + patched out, so these never reach the database. + """ + + def setUp(self): + self.validate = AppModelValidate() + self.parent_id = uuid.uuid4().hex + self.user_id = uuid.uuid4().hex + + patcher = mock.patch.object(SyncableModel, "cached_clean_fields") + self.cached_clean_fields = patcher.start() + self.addCleanup(patcher.stop) + + def _facility_task(self, parent_id=None): + return _validate_task(Facility(id=uuid.uuid4().hex, parent_id=parent_id)) + + def _summary_log_task(self): + return _validate_task(SummaryLog(id=uuid.uuid4().hex, user_id=self.user_id)) + + def test_transform__valid(self): + task = self._facility_task(parent_id=self.parent_id) + + self.assertIs(self.validate.transform(task), task) + self.assertEqual(task.outcome, DeserializeOutcome.SAVE) + + def test_transform__validates_with_fk_cache_and_sync_filter(self): + sync_filter = Filter("abc") + task = self._facility_task(parent_id=self.parent_id) + + AppModelValidate(sync_filter=sync_filter).transform(task) + + self.cached_clean_fields.assert_called_once_with(task.fk_cache, sync_filter=sync_filter) + + def test_transform__errored_task_recorded_as_failed(self): + task = self._facility_task() + task.add_error(ValueError("bad data")) + + self.validate.transform(task) + + self.cached_clean_fields.assert_not_called() + self.assertIn(task.id, self.validate.failed_ids) + self.assertEqual(task.outcome, DeserializeOutcome.ERROR) + + def test_transform__deleted_task_recorded_as_deleted(self): + task = _validate_task(Facility(id=uuid.uuid4().hex), deleted=True) + + self.validate.transform(task) + + self.cached_clean_fields.assert_not_called() + self.assertEqual(self.validate.deleted_ids, {task.id: False}) + self.assertEqual(task.outcome, DeserializeOutcome.DELETE) + + def test_transform__hard_deleted_task_recorded_as_hard_deleted(self): + task = _validate_task(Facility(id=uuid.uuid4().hex), deleted=True, hard_deleted=True) + + self.validate.transform(task) + + self.assertEqual(self.validate.deleted_ids, {task.id: True}) + + def test_transform__target_deleted_in_run(self): + """ + The sink may not have deleted the target's app row yet, so validation could wrongly pass + """ + self.validate.deleted_ids[self.user_id] = False + task = self._summary_log_task() + + self.validate.transform(task) + + self.cached_clean_fields.assert_not_called() + self.assertEqual(task.outcome, DeserializeOutcome.PROPAGATE_DELETE) + self.assertEqual(self.validate.deleted_ids[task.id], False) + + def test_transform__target_hard_deleted_in_run(self): + self.validate.deleted_ids[self.user_id] = True + task = self._summary_log_task() + + self.validate.transform(task) + + self.assertEqual(task.outcome, DeserializeOutcome.PROPAGATE_HARD_DELETE) + self.assertEqual(self.validate.deleted_ids[task.id], True) + + def test_transform__parent_deleted_in_run(self): + """Models with a self-referential FK fail instead, as legacy does once the row is gone""" + self.validate.deleted_ids[self.parent_id] = True + task = self._facility_task(parent_id=self.parent_id) + + self.validate.transform(task) + + self.cached_clean_fields.assert_not_called() + self.assertIsInstance(task.errors[0], ValidationError) + self.assertIn(self.parent_id, str(task.errors[0])) + self.assertIn(task.id, self.validate.failed_ids) + self.assertNotIn(task.id, self.validate.deleted_ids) + + def test_transform__parent_deleted_in_run__grandchildren_fail(self): + self.validate.deleted_ids[self.parent_id] = False + child = self._facility_task(parent_id=self.parent_id) + grandchild = self._facility_task(parent_id=child.id) + + self.validate.transform(child) + self.validate.transform(grandchild) + + self.assertIsInstance(grandchild.errors[0], MorangoDirtyParent) + + def test_transform__parent_failed_in_run(self): + self.validate.failed_ids.add(self.parent_id) + task = self._facility_task(parent_id=self.parent_id) + + self.validate.transform(task) + + self.cached_clean_fields.assert_not_called() + self.assertIsInstance(task.errors[0], MorangoDirtyParent) + self.assertEqual(str(task.errors[0]), "Parent is dirty; could not deserialize.") + self.assertIn(task.id, self.validate.failed_ids) + + def test_transform__failure_propagates_to_grandchildren(self): + self.validate.failed_ids.add(self.parent_id) + child = self._facility_task(parent_id=self.parent_id) + grandchild = self._facility_task(parent_id=child.id) + + self.validate.transform(child) + self.validate.transform(grandchild) + + self.assertIsInstance(grandchild.errors[0], MorangoDirtyParent) + + def test_transform__target_failed_in_run(self): + self.validate.failed_ids.add(self.user_id) + task = self._summary_log_task() + + self.validate.transform(task) + + self.assertIsInstance(task.errors[0], ValidationError) + self.assertIn(self.user_id, str(task.errors[0])) + self.assertIn(task.id, self.validate.failed_ids) + + def test_transform__deleted_target_precedes_failed_target(self): + task = _validate_task( + ConditionalLog(id=uuid.uuid4().hex, facility_id=self.parent_id, user_id=self.user_id) + ) + self.validate.failed_ids.add(self.parent_id) + self.validate.deleted_ids[self.user_id] = False + + self.validate.transform(task) + + self.assertEqual(task.outcome, DeserializeOutcome.PROPAGATE_DELETE) + + def test_transform__null_fk_ignored(self): + """A null FK has no target, so the in-run checks must not match it""" + self.validate.failed_ids.add(None) + task = self._facility_task(parent_id=None) + + self.validate.transform(task) + + self.assertEqual(task.outcome, DeserializeOutcome.SAVE) + + +class AppModelValidateClassifyFailureTestCase(TestCase): + """ + When model validation fails, the FK targets' store records determine how the failure is + resolved and reported + """ + + def setUp(self): + self.validate = AppModelValidate() + self.parent_id = uuid.uuid4().hex + self.user_id = uuid.uuid4().hex + self.error = ValidationError("invalid") + + patcher = mock.patch.object(SyncableModel, "cached_clean_fields", side_effect=self.error) + self.cached_clean_fields = patcher.start() + self.addCleanup(patcher.stop) + + def _target_store(self, target_id, model_name, **kwargs): + return StoreFactory( + id=target_id, + profile="facilitydata", + model_name=model_name, + partition=target_id, + serialized="{}", + last_saved_instance=uuid.uuid4().hex, + last_saved_counter=1, + **kwargs, + ) + + def _facility_task(self): + return _validate_task(Facility(id=uuid.uuid4().hex, parent_id=self.parent_id)) + + def _summary_log_task(self): + return _validate_task(SummaryLog(id=uuid.uuid4().hex, user_id=self.user_id)) + + def test_target_deleted(self): + """The target was deleted in an earlier run, so its app row is already gone""" + self._target_store(self.user_id, "user", deleted=True) + task = self._summary_log_task() + + self.validate.transform(task) + + self.assertEqual(task.outcome, DeserializeOutcome.PROPAGATE_DELETE) + self.assertEqual(self.validate.deleted_ids, {task.id: False}) + self.assertNotIn(task.id, self.validate.failed_ids) + + def test_parent_deleted__object_does_not_exist(self): + """Models with a self-referential FK only propagate the deletion on this error""" + self.cached_clean_fields.side_effect = Facility.DoesNotExist() + self._target_store(self.parent_id, "facility", deleted=True) + task = self._facility_task() + + self.validate.transform(task) + + self.assertEqual(task.outcome, DeserializeOutcome.PROPAGATE_DELETE) + self.assertEqual(self.validate.deleted_ids, {task.id: False}) + + def test_parent_deleted__validation_error(self): + self._target_store(self.parent_id, "facility", deleted=True, hard_deleted=True) + task = self._facility_task() + + self.validate.transform(task) + + self.assertEqual(task.errors, [self.error]) + self.assertIn(task.id, self.validate.failed_ids) + self.assertEqual(self.validate.deleted_ids, {}) + + def test_target_hard_deleted(self): + self._target_store(self.user_id, "user", deleted=True, hard_deleted=True) + task = self._summary_log_task() + + self.validate.transform(task) + + self.assertEqual(task.outcome, DeserializeOutcome.PROPAGATE_HARD_DELETE) + self.assertEqual(self.validate.deleted_ids, {task.id: True}) + + def test_target_hard_deleted_only(self): + """Serialization's hard deletion only sets `hard_deleted`, which still counts as deleted""" + self._target_store(self.user_id, "user", hard_deleted=True) + task = self._summary_log_task() + + self.validate.transform(task) + + self.assertEqual(task.outcome, DeserializeOutcome.PROPAGATE_HARD_DELETE) + + def test_target_deleted_precedes_missing_target(self): + """The hard flag comes from the deleted target, regardless of the other targets""" + self._target_store(self.user_id, "user", deleted=True, hard_deleted=True) + task = _validate_task( + ConditionalLog(id=uuid.uuid4().hex, facility_id=self.parent_id, user_id=self.user_id) + ) + + self.validate.transform(task) + + self.assertEqual(task.outcome, DeserializeOutcome.PROPAGATE_HARD_DELETE) + + def test_parent_missing(self): + task = self._facility_task() + + self.validate.transform(task) + + self.assertIsInstance(task.errors[0], MorangoMissingParent) + self.assertEqual( + str(task.errors[0]), "Parent does not exist in Store; could not deserialize." + ) + self.assertIn(task.id, self.validate.failed_ids) + + def test_parent_dirty(self): + """The parent is dirty but was not deserialized in this run, e.g. it was skipped""" + self._target_store(self.parent_id, "facility", dirty_bit=True) + task = self._facility_task() + + self.validate.transform(task) + + self.assertIsInstance(task.errors[0], MorangoDirtyParent) + self.assertEqual(str(task.errors[0]), "Parent is dirty; could not deserialize.") + self.assertIn(task.id, self.validate.failed_ids) + + def _resolved_parent_task(self): + """A task whose parent FK resolved during validation, so the parent is in `fk_cache`""" + task = self._facility_task() + task.fk_cache["{}_{}".format(self.parent_id, Facility._meta.db_table)] = 1 + return task + + def test_parent_resolved(self): + """ + The parent's store record is dirty, but its FK resolved, either because it was accepted + earlier in the run and the sink has not written it yet, or because an older app row exists. + So the error is the record's own. + """ + self._target_store(self.parent_id, "facility", dirty_bit=True) + task = self._resolved_parent_task() + + self.validate.transform(task) + + self.assertEqual(task.errors, [self.error]) + + def test_parent_resolved__object_does_not_exist(self): + """Loading the parent fails because the sink has not written it yet""" + self.cached_clean_fields.side_effect = Facility.DoesNotExist() + self._target_store(self.parent_id, "facility", dirty_bit=True) + task = self._resolved_parent_task() + + self.validate.transform(task) + + self.assertIsInstance(task.errors[0], MorangoDirtyParent) + + def test_target_missing(self): + task = self._summary_log_task() + + self.validate.transform(task) + + self.assertIsInstance(task.errors[0], IntegrityError) + self.assertEqual( + str(task.errors[0]), + "SummaryLog.user_id references non-existent my user instance with id '{}'".format( + self.user_id + ), + ) + self.assertIn(task.id, self.validate.failed_ids) + + def test_parent_clean(self): + """With nothing wrong with the target, the failure is the record's own""" + self._target_store(self.parent_id, "facility", dirty_bit=False) + task = self._facility_task() + + self.validate.transform(task) + + self.assertEqual(task.errors, [self.error]) + self.assertIn(task.id, self.validate.failed_ids) + + def test_target_dirty(self): + """Only the self-referential parent is checked for dirtiness, matching legacy behavior""" + self._target_store(self.user_id, "user", dirty_bit=True) + task = self._summary_log_task() + + self.validate.transform(task) + + self.assertEqual(task.errors, [self.error]) + + def test_no_fk_references(self): + task = _validate_task(Facility(id=uuid.uuid4().hex)) + + with self.assertNumQueries(0): + self.validate.transform(task) + + self.assertEqual(task.errors, [self.error]) + + def test_object_does_not_exist(self): + """e.g. `clean_fields` loading a related object whose row does not exist""" + self.cached_clean_fields.side_effect = Facility.DoesNotExist() + task = self._facility_task() + + self.validate.transform(task) + + self.assertIsInstance(task.errors[0], MorangoMissingParent) + + def test_value_error(self): + """e.g. `clean_fields` failing to coerce bad synced data""" + error = ValueError("bad data") + self.cached_clean_fields.side_effect = error + self._target_store(self.parent_id, "facility", dirty_bit=False) + task = self._facility_task() + + self.validate.transform(task) + + self.assertEqual(task.errors, [error]) + self.assertIn(task.id, self.validate.failed_ids) + + def test_failure_queries_store_once(self): + task = _validate_task( + ConditionalLog(id=uuid.uuid4().hex, facility_id=self.parent_id, user_id=self.user_id) + ) + + with self.assertNumQueries(1): + self.validate.transform(task) + + def test_success_queries_store_never(self): + self.cached_clean_fields.side_effect = None + task = self._facility_task() + + with self.assertNumQueries(0): + self.validate.transform(task) + + self.assertEqual(task.outcome, DeserializeOutcome.SAVE) + + class StoreModelSourcePrefixConditionsTestCase(SimpleTestCase): """`prefix_conditions` is pure ordering logic over the sync filter and touches no ORM""" From dadac46d5caf555fba1d85db3e27b198145a08b4 Mon Sep 17 00:00:00 2001 From: Blaine Jester Date: Thu, 8 Oct 2026 14:57:53 -0700 Subject: [PATCH 4/4] Add integration and legacy parity tests for deserialize transforms Co-Authored-By: Claude Opus 5.5 (1M context) --- .../tests/sync/stream/test_deserialize.py | 397 ++++++++++++++++++ 1 file changed, 397 insertions(+) diff --git a/tests/testapp/tests/sync/stream/test_deserialize.py b/tests/testapp/tests/sync/stream/test_deserialize.py index 491929f8..37b36e8c 100644 --- a/tests/testapp/tests/sync/stream/test_deserialize.py +++ b/tests/testapp/tests/sync/stream/test_deserialize.py @@ -1,25 +1,34 @@ +import json import uuid import mock from django.core.exceptions import ValidationError from django.core.serializers.json import DjangoJSONEncoder +from django.db import connection from django.db.utils import IntegrityError from django.test import SimpleTestCase from django.test import TestCase +from django.test.utils import CaptureQueriesContext from facility_profile.models import ConditionalLog from facility_profile.models import Facility +from facility_profile.models import MyUser from facility_profile.models import SummaryLog from morango.errors import MorangoDirtyParent from morango.errors import MorangoMissingParent from morango.models.certificates import Filter +from morango.models.core import DeletedModels +from morango.models.core import HardDeletedModels from morango.models.core import Store from morango.models.core import SyncableModel +from morango.sync.operations import _deserialize_from_store +from morango.sync.stream.core import Sink from morango.sync.stream.deserialize import AppModelDeserialize from morango.sync.stream.deserialize import AppModelValidate from morango.sync.stream.deserialize import DeserializeOutcome from morango.sync.stream.deserialize import DeserializeTask from morango.sync.stream.deserialize import StoreModelSource +from morango.utils import exception_path from ...helpers import StoreFactory @@ -756,3 +765,391 @@ def test_stream__ignores_other_profiles(self): ) self.assertEqual(self._stream_ids(), [included.id]) + + +class CollectingSink(Sink[DeserializeTask]): + """Collects each task's resolved outcome and first error type, keyed by the task's ID""" + + def __init__(self): + self.results = {} + + def consume(self, task: DeserializeTask) -> None: + error_type = type(task.errors[0]) if task.errors else None + self.results[task.id] = (task.outcome, error_type) + + +class DeserializeFixturesMixin: + """Builds store records for the `facilitydata` profile and runs them through the transforms""" + + profile = "facilitydata" + + def _store(self, app_model, dirty_bit=True, **kwargs): + store_kwargs = dict( + id=app_model.id, + profile=self.profile, + model_name=app_model.morango_model_name, + partition=app_model.id, + source_id=uuid.uuid4().hex, + serialized=DjangoJSONEncoder().encode(app_model.serialize()), + last_saved_instance=uuid.uuid4().hex, + last_saved_counter=1, + dirty_bit=dirty_bit, + ) + store_kwargs.update(kwargs) + return StoreFactory(**store_kwargs) + + def _facility(self, parent=None, order=None, name="facility", **kwargs): + facility = Facility( + id=uuid.uuid4().hex, + name=name, + parent_id=parent.id if parent else None, + ) + if order is None and parent is None: + order = 0 + return self._store( + facility, _self_ref_fk=facility.parent_id or "", _self_ref_order=order, **kwargs + ) + + def _user(self, **kwargs): + user = MyUser(id=uuid.uuid4().hex, username=uuid.uuid4().hex[:20], password="password") + return self._store(user, **kwargs) + + def _summary_log(self, user_id, **kwargs): + return self._store(SummaryLog(id=uuid.uuid4().hex, user_id=user_id), **kwargs) + + def _run(self, sync_filter=None, skip_errored=False): + sink = CollectingSink() + ( + StoreModelSource(self.profile, sync_filter=sync_filter, skip_errored=skip_errored) + .pipe(AppModelDeserialize(sync_filter=sync_filter)) + .pipe(AppModelValidate(sync_filter=sync_filter)) + .end(sink) + ) + return sink.results + + def _run_both_modes(self, **run_kwargs): + """Runs once for each `Facility.clean_dereferences_parent` mode, returning both results""" + results = {} + for dereferences_parent in (False, True): + with mock.patch.object(Facility, "clean_dereferences_parent", dereferences_parent): + results[dereferences_parent] = self._run(**run_kwargs) + return results + + def _dirty_parent_with_app_row(self, **kwargs): + """A parent whose store record is dirty, but which has an older app row""" + parent = Facility.objects.create(name="parent") + self._store(parent, _self_ref_order=0, **kwargs) + return parent + + +class DeserializeTransformsIntegrationTestCase(DeserializeFixturesMixin, TestCase): + """ + Streams real store records through the source and both transforms, without a writing sink, + so app rows only exist where a test creates them. This mirrors the sink lag, where records + accepted earlier in the run have not been written yet. + """ + + def test_self_ref_chain_in_run(self): + root = self._facility() + child = self._facility(parent=root, order=1) + grandchild = self._facility(parent=child, order=2) + + with mock.patch.object(Facility, "clean_dereferences_parent", False): + results = self._run() + + self.assertEqual(results[root.id], (DeserializeOutcome.SAVE, None)) + self.assertEqual(results[child.id], (DeserializeOutcome.SAVE, None)) + self.assertEqual(results[grandchild.id], (DeserializeOutcome.SAVE, None)) + self.assertEqual(Facility.objects.count(), 0) + + def test_self_ref_chain_in_run__dereferences_parent(self): + """Loading the parent fails, because the sink has not written it yet""" + root = self._facility() + child = self._facility(parent=root, order=1) + grandchild = self._facility(parent=child, order=2) + + with mock.patch.object(Facility, "clean_dereferences_parent", True): + results = self._run() + + self.assertEqual(results[root.id], (DeserializeOutcome.SAVE, None)) + self.assertEqual(results[child.id], (DeserializeOutcome.ERROR, MorangoDirtyParent)) + self.assertEqual(results[grandchild.id], (DeserializeOutcome.ERROR, MorangoDirtyParent)) + + def test_own_error_with_parent_accepted_in_run(self): + """The parent's store record is still dirty, but the child fails on its own field""" + root = self._facility() + child = self._facility(parent=root, order=1, name="x" * 200) + + with mock.patch.object(Facility, "clean_dereferences_parent", False): + results = self._run() + + self.assertEqual(results[root.id], (DeserializeOutcome.SAVE, None)) + self.assertEqual(results[child.id], (DeserializeOutcome.ERROR, ValidationError)) + + def test_cross_model_chain_in_run(self): + user = self._user() + log = self._summary_log(user.id) + + results = self._run() + + self.assertEqual(results[user.id], (DeserializeOutcome.SAVE, None)) + self.assertEqual(results[log.id], (DeserializeOutcome.SAVE, None)) + + def test_parent_not_in_run(self): + """The common case, where the parent is unchanged and found through the app tables""" + parent = Facility.objects.create(name="parent") + self._store(parent, dirty_bit=False, _self_ref_order=0) + child = self._facility(parent=parent, order=1) + + for mode, results in self._run_both_modes().items(): + with self.subTest(dereferences_parent=mode): + self.assertEqual(results[child.id], (DeserializeOutcome.SAVE, None)) + + def test_dirty_parent_not_in_run__skip_errored(self): + """ + Unlike legacy, the child validates against the parent's older app row, so it is saved. A + sibling that fails its own validation keeps its own error. Legacy never deserializes + children of a parent whose store record is dirty, and records `MorangoMissingParent` when + the parent is left out of the run. + """ + parent = self._dirty_parent_with_app_row(deserialization_error="failed earlier") + child = self._facility(parent=parent, order=1) + invalid_child = self._facility(parent=parent, order=1, name="x" * 200) + + for mode, results in self._run_both_modes(skip_errored=True).items(): + with self.subTest(dereferences_parent=mode): + self.assertNotIn(parent.id, results) + self.assertEqual(results[child.id], (DeserializeOutcome.SAVE, None)) + self.assertEqual( + results[invalid_child.id], (DeserializeOutcome.ERROR, ValidationError) + ) + + def test_dirty_parent_not_in_run__filtered(self): + """Same as with `skip_errored`, when the sync filter leaves out the parent's partition""" + parent = self._dirty_parent_with_app_row() + child = self._facility(parent=parent, order=1) + + for mode, results in self._run_both_modes(sync_filter=Filter(child.partition)).items(): + with self.subTest(dereferences_parent=mode): + self.assertNotIn(parent.id, results) + self.assertEqual(results[child.id], (DeserializeOutcome.SAVE, None)) + + def test_parent_failed_in_run(self): + root = self._facility(serialized="{bad") + child = self._facility(parent=root, order=1) + grandchild = self._facility(parent=child, order=2) + + for mode, results in self._run_both_modes().items(): + with self.subTest(dereferences_parent=mode): + self.assertEqual(results[root.id], (DeserializeOutcome.ERROR, json.JSONDecodeError)) + self.assertEqual(results[child.id], (DeserializeOutcome.ERROR, MorangoDirtyParent)) + self.assertEqual( + results[grandchild.id], (DeserializeOutcome.ERROR, MorangoDirtyParent) + ) + + def test_parent_missing(self): + missing_parent = Facility(id=uuid.uuid4().hex, name="missing") + child = self._facility(parent=missing_parent) + + for mode, results in self._run_both_modes().items(): + with self.subTest(dereferences_parent=mode): + self.assertEqual( + results[child.id], (DeserializeOutcome.ERROR, MorangoMissingParent) + ) + + def test_parent_deleted_in_run(self): + """ + Models with a self-referential FK don't propagate the deletion. When the parent object is + loaded, this differs from legacy, which propagates it once the sink has deleted the parent. + """ + root = self._facility(deleted=True) + child = self._facility(parent=root, order=1) + grandchild = self._facility(parent=child, order=2) + + for mode, results in self._run_both_modes().items(): + with self.subTest(dereferences_parent=mode): + self.assertEqual(results[root.id], (DeserializeOutcome.DELETE, None)) + self.assertEqual(results[child.id], (DeserializeOutcome.ERROR, ValidationError)) + self.assertEqual( + results[grandchild.id], (DeserializeOutcome.ERROR, MorangoDirtyParent) + ) + + def test_parent_deleted_earlier(self): + root = self._facility(dirty_bit=False, deleted=True) + child = self._facility(parent=root, order=1) + + results = self._run_both_modes() + + self.assertEqual(results[False][child.id], (DeserializeOutcome.ERROR, ValidationError)) + self.assertEqual(results[True][child.id], (DeserializeOutcome.PROPAGATE_DELETE, None)) + + def test_parent_hard_deleted_earlier(self): + root = self._facility(dirty_bit=False, deleted=True, hard_deleted=True) + child = self._facility(parent=root, order=1) + + results = self._run_both_modes() + + self.assertEqual(results[False][child.id], (DeserializeOutcome.ERROR, ValidationError)) + self.assertEqual(results[True][child.id], (DeserializeOutcome.PROPAGATE_HARD_DELETE, None)) + + def test_target_deleted_in_run(self): + user = self._user(deleted=True) + log = self._summary_log(user.id) + + results = self._run() + + self.assertEqual(results[user.id], (DeserializeOutcome.DELETE, None)) + self.assertEqual(results[log.id], (DeserializeOutcome.PROPAGATE_DELETE, None)) + + def test_target_deleted_earlier(self): + user = self._user(dirty_bit=False, deleted=True, hard_deleted=True) + log = self._summary_log(user.id) + + results = self._run() + + self.assertEqual(results[log.id], (DeserializeOutcome.PROPAGATE_HARD_DELETE, None)) + + def test_target_missing(self): + log = self._summary_log(uuid.uuid4().hex) + + results = self._run() + + self.assertEqual(results[log.id], (DeserializeOutcome.ERROR, IntegrityError)) + + def test_transforms_only_read(self): + root = self._facility() + self._facility(parent=root, order=1) + self._facility(serialized="{bad") + self._facility(deleted=True) + self._summary_log(uuid.uuid4().hex) + + with CaptureQueriesContext(connection) as context: + self._run() + + statements = [query["sql"].strip().upper() for query in context.captured_queries] + self.assertTrue(statements) + for statement in statements: + # postgres wraps iterated reads in a server-side cursor (`DECLARE ... FOR SELECT`), so + # check for writes rather than requiring every statement to start with `SELECT` + self.assertFalse( + statement.startswith(("INSERT", "UPDATE", "DELETE")), + statement, + ) + + +class LegacyParityTestCase(DeserializeFixturesMixin, TestCase): + """ + Characterizes legacy `_deserialize_from_store` errors and propagated deletions against the + transforms, on the same store records. The transforms don't write, so both can run against the + same database state. Runs with the default `Facility.clean_dereferences_parent`, which is the + legacy test behavior, unless a test patches it. + """ + + def _legacy_result(self, store_id): + """The legacy result of a store record that did not save, in the transforms' terms""" + if HardDeletedModels.objects.filter(id=store_id).exists(): + return DeserializeOutcome.PROPAGATE_HARD_DELETE, None + if DeletedModels.objects.filter(id=store_id).exists(): + return DeserializeOutcome.PROPAGATE_DELETE, None + exception = Store.objects.get(id=store_id).deserialization_exception + self.assertIsNotNone(exception) + return DeserializeOutcome.ERROR, exception + + def _assert_parity(self, *store_ids): + results = self._run() + _deserialize_from_store(self.profile) + + for store_id in store_ids: + with self.subTest(store_id=store_id): + outcome, error_type = results[store_id] + error_path = exception_path(error_type) if error_type else None + self.assertEqual((outcome, error_path), self._legacy_result(store_id)) + + def test_parent_failed(self): + root = self._facility(serialized="{bad") + child = self._facility(parent=root, order=1) + grandchild = self._facility(parent=child, order=2) + + self._assert_parity(root.id, child.id, grandchild.id) + + def test_parent_missing(self): + missing_parent = Facility(id=uuid.uuid4().hex, name="missing") + child = self._facility(parent=missing_parent) + + self._assert_parity(child.id) + + def test_target_missing(self): + log = self._summary_log(uuid.uuid4().hex) + + self._assert_parity(log.id) + + @mock.patch.object(Facility, "clean_dereferences_parent", False) + def test_own_error_with_parent_accepted(self): + root = self._facility() + child = self._facility(parent=root, order=1, name="x" * 200) + + self._assert_parity(child.id) + + def test_target_deleted_in_run(self): + user = self._user(deleted=True) + log = self._summary_log(user.id) + + self._assert_parity(log.id) + + def test_target_deleted_earlier(self): + user = self._user(dirty_bit=False, deleted=True) + log = self._summary_log(user.id) + + self._assert_parity(log.id) + + def test_target_hard_deleted_earlier(self): + user = self._user(dirty_bit=False, deleted=True, hard_deleted=True) + log = self._summary_log(user.id) + + self._assert_parity(log.id) + + def test_parent_deleted_earlier(self): + root = self._facility(dirty_bit=False, deleted=True) + child = self._facility(parent=root, order=1) + + self._assert_parity(child.id) + + def test_parent_hard_deleted_earlier(self): + root = self._facility(dirty_bit=False, deleted=True, hard_deleted=True) + child = self._facility(parent=root, order=1) + + self._assert_parity(child.id) + + @mock.patch.object(Facility, "clean_dereferences_parent", False) + def test_parent_deleted_earlier__parent_not_loaded(self): + root = self._facility(dirty_bit=False, deleted=True) + child = self._facility(parent=root, order=1) + + self._assert_parity(child.id) + + def test_dirty_parent_not_in_run__differs(self): + """ + Characterizes an intentional difference: legacy never deserializes children of a parent + whose store record is dirty. The transforms save the child, which validates against the + parent's older app row. + """ + parent = Facility.objects.create(name="parent") + self._store(parent, _self_ref_order=0, deserialization_error="failed earlier") + child = self._facility(parent=parent, order=1) + + results = self._run(skip_errored=True) + _deserialize_from_store(self.profile, skip_erroring=True) + + self.assertEqual(results[child.id], (DeserializeOutcome.SAVE, None)) + self.assertEqual( + self._legacy_result(child.id), + (DeserializeOutcome.ERROR, exception_path(MorangoMissingParent)), + ) + + @mock.patch.object(Facility, "clean_dereferences_parent", False) + def test_parent_deleted_in_run__parent_not_loaded(self): + root = self._facility(deleted=True) + child = self._facility(parent=root, order=1) + grandchild = self._facility(parent=child, order=2) + + self._assert_parity(child.id, grandchild.id)