From 91effb888c4aa2295dfaefbd3abb1fd48b481dd3 Mon Sep 17 00:00:00 2001 From: Valentijn Scholten Date: Sat, 13 Jun 2026 09:53:25 +0200 Subject: [PATCH 01/12] feat(importers): add async-wait import execution mode Introduce a third import/reimport post-processing execution mode and make it configurable per request and via the user profile. - async (default): dispatch post-processing to the background, respond immediately - async_wait (new): dispatch to the background but wait for deduplication to finish before responding, so scan_added notifications and the returned statistics reflect the deduplicated state - sync: run post-processing inline in the web process Replace the legacy UserContactInfo.block_execution boolean with an import_execution_mode CharField; a data migration maps block_execution=True to the 'sync' mode. wants_block_execution() now derives from the sync mode, preserving global foreground-execution semantics for all async tasks. Add a request field import_execution_mode (ImportScanSerializer/ ReImportScanSerializer) resolved against the profile (request override wins), and a deduplication_complete boolean in the import/reimport response. The post_process_findings_batch task now stores its result (ignore_result=False) so async_wait can join on it via AsyncResult.get(); the join is bounded by the new DD_IMPORT_ASYNC_WAIT_TIMEOUT setting. --- dojo/api_v2/serializers.py | 41 ++++- dojo/celery_dispatch.py | 7 +- ...9_usercontactinfo_import_execution_mode.py | 16 ++ ..._remove_usercontactinfo_block_execution.py | 28 ++++ dojo/decorators.py | 6 +- dojo/finding/helper.py | 8 +- dojo/fixtures/defect_dojo_sample_data.json | 3 - .../defect_dojo_sample_data_locations.json | 3 - dojo/fixtures/dojo_testdata.json | 3 - dojo/fixtures/dojo_testdata_locations.json | 3 - dojo/forms.py | 2 +- dojo/importers/base_importer.py | 75 +++++++++ dojo/importers/default_importer.py | 10 +- dojo/importers/default_reimporter.py | 10 +- dojo/importers/options.py | 23 +++ dojo/middleware.py | 4 +- dojo/models.py | 59 ++++++- dojo/settings/settings.dist.py | 4 + dojo/templates/dojo/view_user.html | 8 +- dojo/templates_classic/dojo/view_user.html | 8 +- tests/base_test_class.py | 26 ++-- unittests/test_async_delete.py | 8 +- unittests/test_cascade_delete.py | 2 +- unittests/test_celery_dispatch_force_async.py | 2 +- unittests/test_deduplication_logic.py | 2 +- unittests/test_duplication_loops.py | 2 +- .../test_false_positive_history_logic.py | 2 +- unittests/test_finding_helper.py | 2 +- unittests/test_finding_model.py | 2 +- unittests/test_import_execution_mode.py | 147 ++++++++++++++++++ unittests/test_import_reimport.py | 4 +- unittests/test_importers_deduplication.py | 2 +- unittests/test_importers_performance.py | 14 +- unittests/test_jira_config_engagement.py | 2 +- unittests/test_jira_config_engagement_epic.py | 2 +- unittests/test_jira_import_and_pushing_api.py | 2 +- unittests/test_notifications.py | 10 +- .../test_prepare_duplicates_for_delete.py | 2 +- unittests/test_product_grading.py | 2 +- unittests/test_reimport_prefetch.py | 2 +- unittests/test_tag_inheritance.py | 4 +- unittests/test_tags.py | 4 +- unittests/test_watson_async_search_index.py | 2 +- 43 files changed, 473 insertions(+), 95 deletions(-) create mode 100644 dojo/db_migrations/0269_usercontactinfo_import_execution_mode.py create mode 100644 dojo/db_migrations/0270_remove_usercontactinfo_block_execution.py create mode 100644 unittests/test_import_execution_mode.py diff --git a/dojo/api_v2/serializers.py b/dojo/api_v2/serializers.py index dc15ac6c2dc..a47ce022d11 100644 --- a/dojo/api_v2/serializers.py +++ b/dojo/api_v2/serializers.py @@ -42,6 +42,7 @@ from dojo.location.models import Location, LocationFindingReference from dojo.models import ( IMPORT_ACTIONS, + IMPORT_EXECUTION_MODE_CHOICES, SEVERITIES, SEVERITY_CHOICES, STATS_FIELDS, @@ -1868,6 +1869,16 @@ class CommonImportScanSerializer(serializers.Serializer): allow_null=True, default=None, queryset=User.objects.all(), ) push_to_jira = serializers.BooleanField(default=False) + import_execution_mode = serializers.ChoiceField( + required=False, + allow_null=True, + choices=IMPORT_EXECUTION_MODE_CHOICES, + help_text="Override how import post-processing (deduplication, jira push, grading, ...) is executed for " + "this request. 'async' dispatches post-processing to the background and responds immediately (default). " + "'async_wait' dispatches to the background but waits for deduplication to finish before responding, so " + "notifications and the returned statistics reflect the deduplicated state. 'sync' runs everything inline. " + "If omitted, falls back to the user's profile setting (import_execution_mode).", + ) environment = serializers.CharField(required=False) build_id = serializers.CharField( required=False, help_text="ID of the build that was scanned.", @@ -1913,6 +1924,14 @@ class CommonImportScanSerializer(serializers.Serializer): help_text=_("Also referred to as 'Organization' ID."), ) statistics = ImportStatisticsSerializer(read_only=True, required=False) + deduplication_complete = serializers.BooleanField( + read_only=True, + required=False, + help_text="Whether deduplication had finished by the time this response was produced. " + "True for 'sync' and for 'async_wait' when deduplication completed within the timeout; " + "False for 'async' (deduplication is still running in the background) or when an " + "'async_wait' import timed out waiting for it.", + ) pro = serializers.ListField(read_only=True, required=False) apply_tags_to_findings = serializers.BooleanField( help_text="If set to True, the tags will be applied to the findings", @@ -1971,6 +1990,7 @@ def process_scan( data["product_id"] = test.engagement.product.id data["product_type_id"] = test.engagement.product.prod_type.id data["statistics"] = {"after": test.statistics} + data["deduplication_complete"] = importer.deduplication_complete duration = time.perf_counter() - start_time LargeScanSizeProductAnnouncement(response_data=data, duration=duration) ScanTypeProductAnnouncement(response_data=data, scan_type=context.get("scan_type")) @@ -2069,6 +2089,14 @@ def setup_common_context(self, data: dict) -> dict: if eng_end_date: context["target_end"] = context.get("engagement_end_date") + # Resolve the effective import execution mode: request override (if any) + # takes precedence over the user's profile setting, otherwise default async. + request = self.context.get("request") + user = getattr(request, "user", None) + context["import_execution_mode"] = Dojo_User.resolve_import_execution_mode( + user, data.get("import_execution_mode"), + ) + return context @@ -2242,11 +2270,11 @@ def process_scan( try: logger.debug(f"process_scan called with context: {context}") start_time = time.perf_counter() + processor = None if test := context.get("test"): statistics_before = test.statistics - context["test"], _, _, _, _, _, test_import = self.get_reimporter( - **context, - ).process_scan( + processor = self.get_reimporter(**context) + context["test"], _, _, _, _, _, test_import = processor.process_scan( context.pop("scan", None), ) if test_import: @@ -2258,9 +2286,10 @@ def process_scan( # Do not close old findings when creating a brand new test: there are no # existing findings to compare against, and close_old_findings would # incorrectly close findings from other tests in the same scope. - context["test"], _, _, _, _, _, _ = self.get_importer( + processor = self.get_importer( **{**context, "close_old_findings": False}, - ).process_scan( + ) + context["test"], _, _, _, _, _, _ = processor.process_scan( context.pop("scan", None), ) else: @@ -2279,6 +2308,8 @@ def process_scan( if statistics_delta: data["statistics"]["delta"] = statistics_delta data["statistics"]["after"] = test.statistics + if processor is not None: + data["deduplication_complete"] = processor.deduplication_complete duration = time.perf_counter() - start_time LargeScanSizeProductAnnouncement(response_data=data, duration=duration) ScanTypeProductAnnouncement(response_data=data, scan_type=context.get("scan_type")) diff --git a/dojo/celery_dispatch.py b/dojo/celery_dispatch.py index 38a3b4fa06c..bd552eab7ea 100644 --- a/dojo/celery_dispatch.py +++ b/dojo/celery_dispatch.py @@ -61,10 +61,11 @@ def dojo_dispatch_task(task_or_sig: _SupportsSi | _SupportsApplyAsync | Signatur - Inject `async_user_id` if missing. - Capture and inject pghistory context if available. - - Respect `force_sync=True` (foreground execution) and user `block_execution`. + - Respect `force_sync=True` (foreground execution) and the user's + synchronous import_execution_mode. - Respect `force_async=True` (background execution even when the caller - would otherwise run synchronously, e.g. user has `block_execution`). - `force_async` wins over `force_sync` and `block_execution`. + would otherwise run synchronously, e.g. user on the synchronous mode). + `force_async` wins over `force_sync` and the user's mode. - Support `countdown=` for async dispatch. Returns: diff --git a/dojo/db_migrations/0269_usercontactinfo_import_execution_mode.py b/dojo/db_migrations/0269_usercontactinfo_import_execution_mode.py new file mode 100644 index 00000000000..220f0240b1a --- /dev/null +++ b/dojo/db_migrations/0269_usercontactinfo_import_execution_mode.py @@ -0,0 +1,16 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('dojo', '0268_release_authorization_to_pro'), + ] + + operations = [ + migrations.AddField( + model_name='usercontactinfo', + name='import_execution_mode', + field=models.CharField(blank=True, choices=[('async', 'Async (do not wait)'), ('async_wait', 'Async, wait for deduplication'), ('sync', 'Synchronous (block)')], help_text="Controls how import/reimport post-processing is executed. 'Async' returns immediately (default). 'Async, wait for deduplication' runs post-processing in the background but waits for deduplication to finish before responding, so notifications and statistics are accurate. 'Synchronous' runs everything inline (and blocks all async tasks in the foreground for this user, like the old 'block execution' flag). Can be overridden per request.", max_length=20, null=True), + ), + ] diff --git a/dojo/db_migrations/0270_remove_usercontactinfo_block_execution.py b/dojo/db_migrations/0270_remove_usercontactinfo_block_execution.py new file mode 100644 index 00000000000..56bba7e2886 --- /dev/null +++ b/dojo/db_migrations/0270_remove_usercontactinfo_block_execution.py @@ -0,0 +1,28 @@ +from django.db import migrations + + +def block_execution_to_sync_mode(apps, schema_editor): + """Map the legacy block_execution=True flag to the synchronous import execution mode.""" + UserContactInfo = apps.get_model("dojo", "UserContactInfo") + UserContactInfo.objects.filter(block_execution=True).update(import_execution_mode="sync") + + +def sync_mode_to_block_execution(apps, schema_editor): + """Reverse: restore block_execution=True for users on the synchronous mode.""" + UserContactInfo = apps.get_model("dojo", "UserContactInfo") + UserContactInfo.objects.filter(import_execution_mode="sync").update(block_execution=True) + + +class Migration(migrations.Migration): + + dependencies = [ + ('dojo', '0269_usercontactinfo_import_execution_mode'), + ] + + operations = [ + migrations.RunPython(block_execution_to_sync_mode, sync_mode_to_block_execution), + migrations.RemoveField( + model_name='usercontactinfo', + name='block_execution', + ), + ] diff --git a/dojo/decorators.py b/dojo/decorators.py index cbe33de732d..7e533368f77 100644 --- a/dojo/decorators.py +++ b/dojo/decorators.py @@ -76,15 +76,15 @@ def we_want_async(*args, func=None, **kwargs): return True if Dojo_User.wants_block_execution(user): - logger.debug("dojo_async_task %s: running task in the foreground as block_execution is set to True for %s", func, user) + logger.debug("dojo_async_task %s: running task in the foreground as import_execution_mode is 'sync' for %s", func, user) return False - logger.debug("dojo_async_task %s: running task in the background as user has not set block_execution to True for %s", func, user) + logger.debug("dojo_async_task %s: running task in the background as import_execution_mode is not 'sync' for %s", func, user) return True # Defect Dojo performs all tasks asynchrnonously using celery -# *unless* the user initiating the task has set block_execution to True in their usercontactinfo profile +# *unless* the user initiating the task has set import_execution_mode to 'sync' in their usercontactinfo profile def dojo_async_task(func=None, *, signature=False): def decorator(func): @wraps(func) diff --git a/dojo/finding/helper.py b/dojo/finding/helper.py index 7808fdd7cf5..d89c9908c91 100644 --- a/dojo/finding/helper.py +++ b/dojo/finding/helper.py @@ -456,7 +456,13 @@ def post_process_finding_save_internal(finding, dedupe_option=True, rules_option jira_services.push(finding.finding_group) -@app.task +# ignore_result=False so the 'async_wait' import execution mode can join on the +# dispatched batch via AsyncResult.get() even when CELERY_TASK_IGNORE_RESULT=True. +# NOTE: this override may not be strictly necessary — in the past .get()/await +# appears to have worked with the global CELERY_TASK_IGNORE_RESULT=True and a +# Redis broker. Needs verification against a real broker/worker setup; if join +# works without it, this override can be removed to avoid storing extra results. +@app.task(ignore_result=False) def post_process_findings_batch( finding_ids, *args, diff --git a/dojo/fixtures/defect_dojo_sample_data.json b/dojo/fixtures/defect_dojo_sample_data.json index 7923a91dd38..bd28299fb27 100644 --- a/dojo/fixtures/defect_dojo_sample_data.json +++ b/dojo/fixtures/defect_dojo_sample_data.json @@ -687,7 +687,6 @@ }, { "fields": { - "block_execution": false, "cell_number": "", "force_password_reset": false, "github_username": null, @@ -705,7 +704,6 @@ }, { "fields": { - "block_execution": false, "cell_number": "", "force_password_reset": false, "github_username": null, @@ -723,7 +721,6 @@ }, { "fields": { - "block_execution": false, "cell_number": "", "force_password_reset": false, "github_username": null, diff --git a/dojo/fixtures/defect_dojo_sample_data_locations.json b/dojo/fixtures/defect_dojo_sample_data_locations.json index 0b08e888e03..3f7f04208f9 100644 --- a/dojo/fixtures/defect_dojo_sample_data_locations.json +++ b/dojo/fixtures/defect_dojo_sample_data_locations.json @@ -687,7 +687,6 @@ }, { "fields": { - "block_execution": false, "cell_number": "", "force_password_reset": false, "github_username": null, @@ -707,7 +706,6 @@ }, { "fields": { - "block_execution": false, "cell_number": "", "force_password_reset": false, "github_username": null, @@ -727,7 +725,6 @@ }, { "fields": { - "block_execution": false, "cell_number": "", "force_password_reset": false, "github_username": null, diff --git a/dojo/fixtures/dojo_testdata.json b/dojo/fixtures/dojo_testdata.json index 50707a2d2bf..b1471612a4c 100644 --- a/dojo/fixtures/dojo_testdata.json +++ b/dojo/fixtures/dojo_testdata.json @@ -277,7 +277,6 @@ "title": null, "twitter_username": "#admin", "user": 1, - "block_execution": false, "github_username": null } }, @@ -291,7 +290,6 @@ "title": null, "twitter_username": null, "user": 2, - "block_execution": false, "github_username": null } }, @@ -305,7 +303,6 @@ "title": null, "twitter_username": null, "user": 3, - "block_execution": false, "github_username": null } }, diff --git a/dojo/fixtures/dojo_testdata_locations.json b/dojo/fixtures/dojo_testdata_locations.json index 3d4eb06ff9b..99e9e875953 100644 --- a/dojo/fixtures/dojo_testdata_locations.json +++ b/dojo/fixtures/dojo_testdata_locations.json @@ -277,7 +277,6 @@ "title": null, "twitter_username": "#admin", "user": 1, - "block_execution": false, "github_username": null } }, @@ -291,7 +290,6 @@ "title": null, "twitter_username": null, "user": 2, - "block_execution": false, "github_username": null } }, @@ -305,7 +303,6 @@ "title": null, "twitter_username": null, "user": 3, - "block_execution": false, "github_username": null } }, diff --git a/dojo/forms.py b/dojo/forms.py index e33d7ef51c4..4f09388a866 100644 --- a/dojo/forms.py +++ b/dojo/forms.py @@ -2393,7 +2393,7 @@ class Meta: # Swap order: password_last_reset before token_last_reset field_order = [ "title", "phone_number", "cell_number", "twitter_username", "github_username", - "slack_username", "ui_use_tailwind", "block_execution", "force_password_reset", "reset_api_token", + "slack_username", "ui_use_tailwind", "import_execution_mode", "force_password_reset", "reset_api_token", "password_last_reset", "token_last_reset", ] diff --git a/dojo/importers/base_importer.py b/dojo/importers/base_importer.py index d87524185fe..b5daa972bbe 100644 --- a/dojo/importers/base_importer.py +++ b/dojo/importers/base_importer.py @@ -19,6 +19,8 @@ # Import History States IMPORT_CLOSED_FINDING, IMPORT_CREATED_FINDING, + IMPORT_EXECUTION_MODE_ASYNC_WAIT, + IMPORT_EXECUTION_MODE_SYNC, IMPORT_REACTIVATED_FINDING, IMPORT_UNTOUCHED_FINDING, # Finding Severities @@ -77,6 +79,79 @@ def __init__( and will raise a `NotImplemented` exception """ ImporterOptions.__init__(self, *args, **kwargs) + # Handles for async post-processing tasks to await in 'async_wait' mode. + # Set after ImporterOptions.__init__ so it stays out of field_names + # (and the compress/decompress cycle used for async dispatch). + self.post_processing_results = [] + # Whether deduplication is known to be finished by the time the response + # is built. True for 'sync' (ran inline) and for 'async_wait' when all + # batches completed within the timeout; False for 'async' (dispatched, + # not awaited) or when an 'async_wait' join timed out/errored. + self.deduplication_complete = False + + def post_processing_dispatch_kwargs(self, **kwargs): + """ + Translate the resolved import execution mode into the force flags that + dojo_dispatch_task understands: + - SYNC: run inline in the web process (force_sync). + - ASYNC_WAIT: guarantee background dispatch (force_async) so we get a + handle to await, regardless of the user's profile mode. + - ASYNC (default): preserve historical behavior, honoring any externally + supplied force_sync and the user's sync mode via we_want_async. + """ + if self.import_execution_mode == IMPORT_EXECUTION_MODE_SYNC: + return {"force_sync": True} + if self.import_execution_mode == IMPORT_EXECUTION_MODE_ASYNC_WAIT: + return {"force_async": True} + return {"force_sync": kwargs.get("force_sync", False)} + + def record_post_processing_result(self, result): + """ + Remember an async post-processing dispatch handle so it can be awaited + later when running in the 'async_wait' execution mode. No-op for the + other modes (no handle is recorded by the caller). + """ + if not hasattr(self, "post_processing_results"): + self.post_processing_results = [] + if result is not None: + self.post_processing_results.append(result) + + def wait_for_post_processing(self): + """ + Block until the deduplication (and other batch) post-processing tasks + dispatched during this import have finished, so notifications and the + returned statistics reflect the deduplicated state. + + Only relevant in the 'async_wait' execution mode; bounded by + settings.IMPORT_ASYNC_WAIT_TIMEOUT so a stuck/missing worker degrades + to the historical (respond-anyway) behavior instead of hanging. + """ + if self.import_execution_mode == IMPORT_EXECUTION_MODE_SYNC: + # Batches ran inline during process_findings, so dedup is already done. + self.deduplication_complete = True + return + if self.import_execution_mode != IMPORT_EXECUTION_MODE_ASYNC_WAIT: + # 'async': post-processing was dispatched but is not awaited. + self.deduplication_complete = False + return + results = getattr(self, "post_processing_results", None) or [] + if not results: + # Nothing was dispatched (e.g. empty import) — dedup is trivially done. + self.deduplication_complete = True + return + timeout = getattr(settings, "IMPORT_ASYNC_WAIT_TIMEOUT", 120) + logger.debug("async_wait: waiting for %d post-processing task(s) (timeout=%ss)", len(results), timeout) + success = True + for result in results: + if result is None or not hasattr(result, "get"): + continue + try: + result.get(timeout=timeout, propagate=False) + except Exception as e: + logger.warning("async_wait: error/timeout while waiting for post-processing task: %s", e) + success = False + self.deduplication_complete = success + self.post_processing_results = [] def check_child_implementation_exception(self): """ diff --git a/dojo/importers/default_importer.py b/dojo/importers/default_importer.py index 3a920577d2d..c4865083989 100644 --- a/dojo/importers/default_importer.py +++ b/dojo/importers/default_importer.py @@ -12,6 +12,7 @@ from dojo.importers.options import ImporterOptions from dojo.jira import services as jira_services from dojo.models import ( + IMPORT_EXECUTION_MODE_ASYNC_WAIT, Engagement, Finding, Test, @@ -146,6 +147,9 @@ def process_scan( url=reverse("view_test", args=(self.test.id,)), url_api=reverse("test-detail", args=(self.test.id,)), ) + # In 'async_wait' mode, block until background deduplication has finished + # so notifications and statistics reflect the deduplicated state. + self.wait_for_post_processing() updated_count = len(new_findings) + len(closed_findings) self.notify_scan_added( self.test, @@ -290,7 +294,7 @@ def _process_findings_internal( batch_finding_ids.clear() logger.debug("process_findings: dispatching batch with push_to_jira=%s (batch_size=%d, is_final=%s)", push_to_jira, len(finding_ids_batch), is_final_finding) - dojo_dispatch_task( + result = dojo_dispatch_task( finding_helper.post_process_findings_batch, finding_ids_batch, dedupe_option=True, @@ -298,8 +302,10 @@ def _process_findings_internal( product_grading_option=True, issue_updater_option=True, push_to_jira=push_to_jira, - force_sync=kwargs.get("force_sync", False), + **self.post_processing_dispatch_kwargs(**kwargs), ) + if self.import_execution_mode == IMPORT_EXECUTION_MODE_ASYNC_WAIT: + self.record_post_processing_result(result) # No chord: tasks are dispatched immediately above per batch diff --git a/dojo/importers/default_reimporter.py b/dojo/importers/default_reimporter.py index aa33c6153b0..a32fa022af4 100644 --- a/dojo/importers/default_reimporter.py +++ b/dojo/importers/default_reimporter.py @@ -18,6 +18,7 @@ from dojo.importers.options import ImporterOptions from dojo.jira import services as jira_services from dojo.models import ( + IMPORT_EXECUTION_MODE_ASYNC_WAIT, Development_Environment, Finding, Notes, @@ -141,6 +142,9 @@ def process_scan( ) # Send out som notifications to the user logger.debug("REIMPORT_SCAN: Generating notifications") + # In 'async_wait' mode, block until background deduplication has finished + # so notifications and statistics reflect the deduplicated state. + self.wait_for_post_processing() updated_count = ( len(closed_findings) + len(reactivated_findings) + len(new_findings) ) @@ -455,7 +459,7 @@ def _process_findings_internal( batch_findings.clear() finding_ids_batch = list(batch_finding_ids) batch_finding_ids.clear() - dojo_dispatch_task( + result = dojo_dispatch_task( finding_helper.post_process_findings_batch, finding_ids_batch, dedupe_option=True, @@ -464,8 +468,10 @@ def _process_findings_internal( issue_updater_option=True, push_to_jira=push_to_jira, jira_instance_id=getattr(self.jira_instance, "id", None), - force_sync=kwargs.get("force_sync", False), + **self.post_processing_dispatch_kwargs(**kwargs), ) + if self.import_execution_mode == IMPORT_EXECUTION_MODE_ASYNC_WAIT: + self.record_post_processing_result(result) # No chord: tasks are dispatched immediately above per batch diff --git a/dojo/importers/options.py b/dojo/importers/options.py index 02cecff1113..3817a96fe08 100644 --- a/dojo/importers/options.py +++ b/dojo/importers/options.py @@ -12,6 +12,9 @@ from dojo.jira.services import get_instance as get_jira_instance from dojo.models import ( + IMPORT_EXECUTION_MODE_ASYNC, + IMPORT_EXECUTION_MODE_SYNC, + IMPORT_EXECUTION_MODES, Development_Environment, Dojo_User, Endpoint, @@ -68,6 +71,7 @@ def load_base_options( self.engagement: Engagement | None = self.validate_engagement(*args, **kwargs) self.environment: Development_Environment | None = self.validate_environment(*args, **kwargs) self.group_by: str = self.validate_group_by(*args, **kwargs) + self.import_execution_mode: str = self.validate_import_execution_mode(*args, **kwargs) self.import_type: str = self.validate_import_type(*args, **kwargs) self.lead: Dojo_User | None = self.validate_lead(*args, **kwargs) self.minimum_severity: str = self.validate_minimum_severity(*args, **kwargs) @@ -345,6 +349,25 @@ def validate_do_not_reactivate( **kwargs, ) + def validate_import_execution_mode( + self, + *args: list, + **kwargs: dict, + ) -> str: + mode = self.validate( + "import_execution_mode", + expected_types=[str], + required=False, + default=IMPORT_EXECUTION_MODE_ASYNC, + **kwargs, + ) + if mode not in IMPORT_EXECUTION_MODES: + mode = IMPORT_EXECUTION_MODE_ASYNC + # An explicit force_sync from a non-serializer caller still wins. + if kwargs.get("force_sync"): + mode = IMPORT_EXECUTION_MODE_SYNC + return mode + def validate_commit_hash( self, *args: list, diff --git a/dojo/middleware.py b/dojo/middleware.py index a576244312a..73e0ed601dd 100644 --- a/dojo/middleware.py +++ b/dojo/middleware.py @@ -281,8 +281,8 @@ def _drain_search_context_to_async(objects, source): for model_name, pk_list in model_groups.items(): batches = [pk_list[i:i + batch_size] for i in range(0, len(pk_list), batch_size)] # force_async=True keeps indexing off the request path even for users - # with block_execution=True — index updates are slow and never need - # to be synchronous from the user's perspective. + # on the synchronous import_execution_mode — index updates are slow and + # never need to be synchronous from the user's perspective. for i, batch in enumerate(batches, 1): logger.debug(f"{source}: Triggering batch {i}/{len(batches)} for {model_name}: {len(batch)} instances") dojo_dispatch_task(update_watson_search_index_for_model, model_name, batch, force_async=True) diff --git a/dojo/models.py b/dojo/models.py index a41f5640889..01abb6bcace 100644 --- a/dojo/models.py +++ b/dojo/models.py @@ -199,6 +199,28 @@ def __str__(self): User = get_user_model() +# Import post-processing execution modes. +# - ASYNC: post-processing (dedup, jira, grading, ...) runs in the background; +# the API responds immediately (default, historical behavior). +# - ASYNC_WAIT: post-processing is dispatched to the background as usual, but the +# request waits for the deduplication batches to finish before responding, so +# notifications and the returned statistics reflect the deduplicated state. +# - SYNC: post-processing runs inline in the web process (legacy block_execution). +IMPORT_EXECUTION_MODE_ASYNC = "async" +IMPORT_EXECUTION_MODE_ASYNC_WAIT = "async_wait" +IMPORT_EXECUTION_MODE_SYNC = "sync" +IMPORT_EXECUTION_MODES = ( + IMPORT_EXECUTION_MODE_ASYNC, + IMPORT_EXECUTION_MODE_ASYNC_WAIT, + IMPORT_EXECUTION_MODE_SYNC, +) +IMPORT_EXECUTION_MODE_CHOICES = ( + (IMPORT_EXECUTION_MODE_ASYNC, _("Async (do not wait)")), + (IMPORT_EXECUTION_MODE_ASYNC_WAIT, _("Async, wait for deduplication")), + (IMPORT_EXECUTION_MODE_SYNC, _("Synchronous (block)")), +) + + # proxy class for convenience and UI class Dojo_User(User): class Meta: @@ -213,8 +235,25 @@ def __str__(self): @staticmethod def wants_block_execution(user): - # this return False if there is no user, i.e. in celery processes, unittests, etc. - return hasattr(user, "usercontactinfo") and user.usercontactinfo.block_execution + # this returns False if there is no user, i.e. in celery processes, unittests, etc. + # The synchronous import execution mode is the successor of the old block_execution + # flag and governs whether async tasks run in the foreground for this user. + return hasattr(user, "usercontactinfo") and user.usercontactinfo.import_execution_mode == IMPORT_EXECUTION_MODE_SYNC + + @staticmethod + def resolve_import_execution_mode(user, override=None): + """ + Resolve the effective import post-processing execution mode. + + Priority: explicit request override > user profile setting > default async. + Returns one of IMPORT_EXECUTION_MODE_ASYNC / _ASYNC_WAIT / _SYNC. + """ + if override in IMPORT_EXECUTION_MODES: + return override + info = getattr(user, "usercontactinfo", None) + if info is not None and info.import_execution_mode in IMPORT_EXECUTION_MODES: + return info.import_execution_mode + return IMPORT_EXECUTION_MODE_ASYNC @staticmethod def force_password_reset(user): @@ -255,7 +294,21 @@ class UserContactInfo(models.Model): github_username = models.CharField(blank=True, null=True, max_length=150) slack_username = models.CharField(blank=True, null=True, max_length=150, help_text=_("Email address associated with your slack account"), verbose_name=_("Slack Email Address")) slack_user_id = models.CharField(blank=True, null=True, max_length=25) - block_execution = models.BooleanField(default=False, help_text=_("Instead of async deduping a finding the findings will be deduped synchronously and will 'block' the user until completion.")) + import_execution_mode = models.CharField( + max_length=20, + choices=IMPORT_EXECUTION_MODE_CHOICES, + null=True, + blank=True, + help_text=_( + "Controls how import/reimport post-processing is executed. " + "'Async' returns immediately (default). 'Async, wait for deduplication' " + "runs post-processing in the background but waits for deduplication to " + "finish before responding, so notifications and statistics are accurate. " + "'Synchronous' runs everything inline (and blocks all async tasks in the " + "foreground for this user, like the old 'block execution' flag). Can be " + "overridden per request.", + ), + ) force_password_reset = models.BooleanField(default=False, help_text=_("Forces this user to reset their password on next login.")) ui_use_tailwind = models.BooleanField(default=False, verbose_name=_("Use new UI (beta)"), help_text=_("Opt in to the new Tailwind-based UI. Leave off for the classic UI.")) token_last_reset = models.DateTimeField(null=True, blank=True, help_text=_("Timestamp of the most recent API token reset for this user.")) diff --git a/dojo/settings/settings.dist.py b/dojo/settings/settings.dist.py index abe31dec9f9..2c680693477 100644 --- a/dojo/settings/settings.dist.py +++ b/dojo/settings/settings.dist.py @@ -96,6 +96,9 @@ DD_CELERY_BROKER_PARAMS=(str, ""), DD_CELERY_BROKER_TRANSPORT_OPTIONS=(str, ""), DD_CELERY_TASK_IGNORE_RESULT=(bool, True), + # Max seconds the 'async_wait' import execution mode will wait for background + # deduplication/post-processing to finish before responding anyway. + DD_IMPORT_ASYNC_WAIT_TIMEOUT=(int, 120), DD_CELERY_RESULT_BACKEND=(str, "django-db"), DD_CELERY_RESULT_EXPIRES=(int, 86400), DD_CELERY_BEAT_SCHEDULE_FILENAME=(str, root("dojo.celery.beat.db")), @@ -855,6 +858,7 @@ def generate_url(scheme, double_slashes, user, password, host, port, path, param params=env("DD_CELERY_BROKER_PARAMS"), ) CELERY_TASK_IGNORE_RESULT = env("DD_CELERY_TASK_IGNORE_RESULT") +IMPORT_ASYNC_WAIT_TIMEOUT = env("DD_IMPORT_ASYNC_WAIT_TIMEOUT") CELERY_RESULT_BACKEND = env("DD_CELERY_RESULT_BACKEND") CELERY_TIMEZONE = TIME_ZONE CELERY_RESULT_EXPIRES = env("DD_CELERY_RESULT_EXPIRES") diff --git a/dojo/templates/dojo/view_user.html b/dojo/templates/dojo/view_user.html index 4f1d0a3b04d..47542577c0d 100644 --- a/dojo/templates/dojo/view_user.html +++ b/dojo/templates/dojo/view_user.html @@ -279,13 +279,9 @@

- {% trans "Block execution" %} + {% trans "Import execution mode" %} - {% if user.usercontactinfo.block_execution %} - - {% else %} - - {% endif %} + {{ user.usercontactinfo.get_import_execution_mode_display|default:_("Async (do not wait)") }} diff --git a/dojo/templates_classic/dojo/view_user.html b/dojo/templates_classic/dojo/view_user.html index e1fe9917722..4d6ac25220f 100644 --- a/dojo/templates_classic/dojo/view_user.html +++ b/dojo/templates_classic/dojo/view_user.html @@ -279,13 +279,9 @@

- {% trans "Block execution" %} + {% trans "Import execution mode" %} - {% if user.usercontactinfo.block_execution %} - - {% else %} - - {% endif %} + {{ user.usercontactinfo.get_import_execution_mode_display|default:_("Async (do not wait)") }} diff --git a/tests/base_test_class.py b/tests/base_test_class.py index d9043a43fd5..2798d2f2a58 100644 --- a/tests/base_test_class.py +++ b/tests/base_test_class.py @@ -10,7 +10,7 @@ from selenium.webdriver.chrome.options import Options from selenium.webdriver.common.by import By from selenium.webdriver.support import expected_conditions -from selenium.webdriver.support.ui import WebDriverWait +from selenium.webdriver.support.ui import Select, WebDriverWait # import time logging.basicConfig( @@ -329,22 +329,24 @@ def enable_github(self): return self.enable_system_setting("id_enable_github") def set_block_execution(self, *, block_execution=True): - # we set the admin user (ourselves) to have block_execution checked - # this will force dedupe to happen synchronously, among other things like notifications, rules, ... - logger.info("setting block execution to: %s", block_execution) + # we set the admin user (ourselves) to the synchronous import execution mode + # (the successor of the old block_execution flag) when block_execution=True. + # This forces dedupe to happen synchronously, among other things like + # notifications, rules, ... Otherwise we select the default async mode. + target_mode = "sync" if block_execution else "async" + logger.info("setting import execution mode to: %s", target_mode) driver = self.driver driver.get(self.base_url + "profile") - if ( - driver.find_element(By.ID, "id_block_execution").is_selected() - != block_execution - ): - driver.find_element(By.XPATH, '//*[@id="id_block_execution"]').click() + select = Select(driver.find_element(By.ID, "id_import_execution_mode")) + if select.first_selected_option.get_attribute("value") != target_mode: + select.select_by_value(target_mode) # save settings driver.find_element(By.CSS_SELECTOR, "input.btn.btn-primary").click() - # check if it's enabled after reload + # check if it's applied after reload + select = Select(driver.find_element(By.ID, "id_import_execution_mode")) self.assertEqual( - driver.find_element(By.ID, "id_block_execution").is_selected(), - block_execution, + select.first_selected_option.get_attribute("value"), + target_mode, ) return driver diff --git a/unittests/test_async_delete.py b/unittests/test_async_delete.py index 7aa8f0769a1..ae0105791e7 100644 --- a/unittests/test_async_delete.py +++ b/unittests/test_async_delete.py @@ -27,21 +27,21 @@ class TestAsyncDelete(DojoTestCase): """ Test async_delete functionality with dojo_dispatch_task kwargs injection. - These tests use block_execution=True and crum.impersonate to run tasks synchronously, + These tests use import_execution_mode="sync" and crum.impersonate to run tasks synchronously, which allows errors to surface immediately rather than being lost in background workers. """ def setUp(self): - """Set up test user with block_execution=True and disable unneeded features.""" + """Set up test user with import_execution_mode="sync" and disable unneeded features.""" super().setUp() - # Create test user with block_execution=True to run tasks synchronously + # Create test user with import_execution_mode="sync" to run tasks synchronously self.testuser = User.objects.create( username="test_async_delete_user", is_staff=True, is_superuser=True, ) - UserContactInfo.objects.create(user=self.testuser, block_execution=True) + UserContactInfo.objects.create(user=self.testuser, import_execution_mode="sync") # Log in as the test user (for API client) self.client.force_login(self.testuser) diff --git a/unittests/test_cascade_delete.py b/unittests/test_cascade_delete.py index f24667e44f8..8f8a4244610 100644 --- a/unittests/test_cascade_delete.py +++ b/unittests/test_cascade_delete.py @@ -37,7 +37,7 @@ def setUp(self): is_staff=True, is_superuser=True, ) - UserContactInfo.objects.create(user=self.testuser, block_execution=True) + UserContactInfo.objects.create(user=self.testuser, import_execution_mode="sync") self.system_settings(enable_deduplication=False) self.system_settings(enable_product_grade=False) diff --git a/unittests/test_celery_dispatch_force_async.py b/unittests/test_celery_dispatch_force_async.py index 71170e1bb09..7cf523d78d2 100644 --- a/unittests/test_celery_dispatch_force_async.py +++ b/unittests/test_celery_dispatch_force_async.py @@ -3,7 +3,7 @@ `force_async=True` is for callers (e.g. the watson async indexer middleware) that should always run their celery task in the background even when the -current user has `block_execution=True` or the caller passes `force_sync=True`. +current user has `import_execution_mode="sync"` or the caller passes `force_sync=True`. """ from unittest.mock import patch diff --git a/unittests/test_deduplication_logic.py b/unittests/test_deduplication_logic.py index fd6b5d2847c..e1f5e08d05e 100644 --- a/unittests/test_deduplication_logic.py +++ b/unittests/test_deduplication_logic.py @@ -162,7 +162,7 @@ class TestDuplicationLogic(DojoTestCase): def run(self, result=None): testuser = User.objects.get(username="admin") - testuser.usercontactinfo.block_execution = True + testuser.usercontactinfo.import_execution_mode = "sync" testuser.save() # unit tests are running without any user, which will result in actions like dedupe happening in the celery process diff --git a/unittests/test_duplication_loops.py b/unittests/test_duplication_loops.py index a3a70cf5c2b..fabeb376de0 100644 --- a/unittests/test_duplication_loops.py +++ b/unittests/test_duplication_loops.py @@ -19,7 +19,7 @@ class TestDuplicationLoops(DojoTestCase): def run(self, result=None): testuser = User.objects.get(username="admin") - testuser.usercontactinfo.block_execution = True + testuser.usercontactinfo.import_execution_mode = "sync" testuser.save() # unit tests are running without any user, which will result in actions like dedupe happening in the celery process diff --git a/unittests/test_false_positive_history_logic.py b/unittests/test_false_positive_history_logic.py index 8748239bedd..a599bda74ef 100644 --- a/unittests/test_false_positive_history_logic.py +++ b/unittests/test_false_positive_history_logic.py @@ -131,7 +131,7 @@ class TestFalsePositiveHistoryLogic(DojoTestCase): def run(self, result=None): testuser = User.objects.get(username="admin") - testuser.usercontactinfo.block_execution = True + testuser.usercontactinfo.import_execution_mode = "sync" testuser.save() # Unit tests are running without any user, which will result in actions like dedupe happening in the celery process diff --git a/unittests/test_finding_helper.py b/unittests/test_finding_helper.py index fa6fd2d9ea5..51c7602409a 100644 --- a/unittests/test_finding_helper.py +++ b/unittests/test_finding_helper.py @@ -253,7 +253,7 @@ def setUp(self): super().setUp() self.system_settings(enable_jira=True) self.testuser = User.objects.get(username="admin") - self.testuser.usercontactinfo.block_execution = True + self.testuser.usercontactinfo.import_execution_mode = "sync" self.testuser.usercontactinfo.save() token = Token.objects.get(user=self.testuser) self.client = APIClient() diff --git a/unittests/test_finding_model.py b/unittests/test_finding_model.py index 78ee40693f2..d3344234f40 100644 --- a/unittests/test_finding_model.py +++ b/unittests/test_finding_model.py @@ -507,7 +507,7 @@ class TestFindingSLAExpiration(DojoTestCase): def run(self, result=None): testuser = User.objects.get(username="admin") - testuser.usercontactinfo.block_execution = True + testuser.usercontactinfo.import_execution_mode = "sync" testuser.save() # unit tests are running without any user, which will result in actions like dedupe happening in the celery process diff --git a/unittests/test_import_execution_mode.py b/unittests/test_import_execution_mode.py new file mode 100644 index 00000000000..568439f7ff6 --- /dev/null +++ b/unittests/test_import_execution_mode.py @@ -0,0 +1,147 @@ +from django.test import override_settings + +from dojo.importers.default_importer import DefaultImporter +from dojo.models import ( + IMPORT_EXECUTION_MODE_ASYNC, + IMPORT_EXECUTION_MODE_ASYNC_WAIT, + IMPORT_EXECUTION_MODE_SYNC, + Development_Environment, + Dojo_User, + Engagement, + UserContactInfo, +) + +from .dojo_test_case import DojoAPITestCase, DojoTestCase, get_unit_tests_path + + +class ImportExecutionModeResolverTest(DojoTestCase): + + """resolve_import_execution_mode: request override > profile > default.""" + + fixtures = ["dojo_testdata.json"] + + def setUp(self): + self.user = Dojo_User.objects.get(username="admin") + UserContactInfo.objects.filter(user=self.user).delete() + + def _set_profile(self, *, mode=None): + UserContactInfo.objects.update_or_create( + user=self.user, + defaults={"import_execution_mode": mode}, + ) + self.user.refresh_from_db() + + def test_default_is_async(self): + self.assertEqual(IMPORT_EXECUTION_MODE_ASYNC, Dojo_User.resolve_import_execution_mode(self.user)) + + def test_request_override_wins_over_profile(self): + self._set_profile(mode=IMPORT_EXECUTION_MODE_SYNC) + self.assertEqual( + IMPORT_EXECUTION_MODE_ASYNC_WAIT, + Dojo_User.resolve_import_execution_mode(self.user, IMPORT_EXECUTION_MODE_ASYNC_WAIT), + ) + + def test_profile_mode_used_when_no_override(self): + self._set_profile(mode=IMPORT_EXECUTION_MODE_ASYNC_WAIT) + self.assertEqual(IMPORT_EXECUTION_MODE_ASYNC_WAIT, Dojo_User.resolve_import_execution_mode(self.user)) + + def test_empty_profile_falls_back_to_async(self): + self._set_profile(mode=None) + self.assertEqual(IMPORT_EXECUTION_MODE_ASYNC, Dojo_User.resolve_import_execution_mode(self.user)) + + def test_invalid_override_ignored(self): + self.assertEqual(IMPORT_EXECUTION_MODE_ASYNC, Dojo_User.resolve_import_execution_mode(self.user, "garbage")) + + def test_no_user(self): + self.assertEqual(IMPORT_EXECUTION_MODE_ASYNC, Dojo_User.resolve_import_execution_mode(None)) + + def test_wants_block_execution_only_for_sync_mode(self): + self._set_profile(mode=IMPORT_EXECUTION_MODE_SYNC) + self.assertTrue(Dojo_User.wants_block_execution(self.user)) + self._set_profile(mode=IMPORT_EXECUTION_MODE_ASYNC_WAIT) + self.assertFalse(Dojo_User.wants_block_execution(self.user)) + self._set_profile(mode=None) + self.assertFalse(Dojo_User.wants_block_execution(self.user)) + + +class ImporterDispatchKwargsTest(DojoTestCase): + + """import_execution_mode -> dojo_dispatch_task force flags.""" + + fixtures = ["dojo_testdata.json"] + + def _importer(self, mode, **extra): + return DefaultImporter( + scan_type="ZAP Scan", + engagement=Engagement.objects.first(), + environment=Development_Environment.objects.first(), + import_execution_mode=mode, + **extra, + ) + + def test_sync_mode_forces_sync(self): + self.assertEqual({"force_sync": True}, self._importer(IMPORT_EXECUTION_MODE_SYNC).post_processing_dispatch_kwargs()) + + def test_async_wait_mode_forces_async(self): + self.assertEqual({"force_async": True}, self._importer(IMPORT_EXECUTION_MODE_ASYNC_WAIT).post_processing_dispatch_kwargs()) + + def test_async_mode_preserves_external_force_sync(self): + importer = self._importer(IMPORT_EXECUTION_MODE_ASYNC) + self.assertEqual({"force_sync": False}, importer.post_processing_dispatch_kwargs()) + self.assertEqual({"force_sync": True}, importer.post_processing_dispatch_kwargs(force_sync=True)) + + def test_invalid_mode_defaults_to_async(self): + self.assertEqual(IMPORT_EXECUTION_MODE_ASYNC, self._importer("nonsense").import_execution_mode) + + def test_external_force_sync_promotes_to_sync_mode(self): + importer = self._importer(IMPORT_EXECUTION_MODE_ASYNC, force_sync=True) + self.assertEqual(IMPORT_EXECUTION_MODE_SYNC, importer.import_execution_mode) + + +@override_settings(CELERY_TASK_ALWAYS_EAGER=True) +class ImportExecutionModeAPITest(DojoAPITestCase): + + """ + End-to-end: the import endpoints accept and honor import_execution_mode. + + CELERY_TASK_ALWAYS_EAGER runs dispatched tasks inline against the test DB, so + 'async_wait' can actually join its deduplication batch (a real broker/worker + runs against a different DB and could never see the test transaction's data). + """ + + fixtures = ["dojo_testdata.json"] + + def setUp(self): + super().setUp() + self.login_as_admin() + + def _payload(self, mode): + return { + "minimum_severity": "Low", + "scan_type": "ZAP Scan", + "engagement": 1, + "import_execution_mode": mode, + } + + def test_import_async_wait_returns_statistics(self): + with (get_unit_tests_path() / "scans/zap/0_zap_sample.xml").open(encoding="utf-8") as testfile: + payload = self._payload(IMPORT_EXECUTION_MODE_ASYNC_WAIT) + payload["file"] = testfile + result = self.import_scan(payload, 201) + self.assertIn("statistics", result) + self.assertIn("after", result["statistics"]) + # async_wait joins deduplication, so it must report completion + self.assertTrue(result["deduplication_complete"]) + + def test_import_async_does_not_await_deduplication(self): + with (get_unit_tests_path() / "scans/zap/0_zap_sample.xml").open(encoding="utf-8") as testfile: + payload = self._payload(IMPORT_EXECUTION_MODE_ASYNC) + payload["file"] = testfile + result = self.import_scan(payload, 201) + self.assertFalse(result["deduplication_complete"]) + + def test_import_rejects_invalid_mode(self): + with (get_unit_tests_path() / "scans/zap/0_zap_sample.xml").open(encoding="utf-8") as testfile: + payload = self._payload("not-a-mode") + payload["file"] = testfile + self.import_scan(payload, 400) diff --git a/unittests/test_import_reimport.py b/unittests/test_import_reimport.py index 417fe0ea9ea..26c70f2a52e 100644 --- a/unittests/test_import_reimport.py +++ b/unittests/test_import_reimport.py @@ -2323,7 +2323,7 @@ def __init__(self, *args, **kwargs): def setUp(self): testuser = User.objects.get(username="admin") - testuser.usercontactinfo.block_execution = True + testuser.usercontactinfo.import_execution_mode = "sync" testuser.usercontactinfo.save() self.login_as_admin() # self.url = reverse(self.viewname + '-list') @@ -3122,7 +3122,7 @@ def __init__(self, *args, **kwargs): def setUp(self): # still using the API to verify results testuser = User.objects.get(username="admin") - testuser.usercontactinfo.block_execution = True + testuser.usercontactinfo.import_execution_mode = "sync" testuser.usercontactinfo.save() self.login_as_admin() # self.url = reverse(self.viewname + '-list') diff --git a/unittests/test_importers_deduplication.py b/unittests/test_importers_deduplication.py index 7c1359dcc36..0bd2a71568c 100644 --- a/unittests/test_importers_deduplication.py +++ b/unittests/test_importers_deduplication.py @@ -37,7 +37,7 @@ def setUp(self): testuser.is_superuser = True testuser.is_staff = True testuser.save() - UserContactInfo.objects.create(user=testuser, block_execution=True) + UserContactInfo.objects.create(user=testuser, import_execution_mode="sync") # Authenticate API client as admin for import endpoints self.login_as_admin() diff --git a/unittests/test_importers_performance.py b/unittests/test_importers_performance.py index ce9133e4a6b..0736d4c4f55 100644 --- a/unittests/test_importers_performance.py +++ b/unittests/test_importers_performance.py @@ -67,7 +67,7 @@ def setUp(self): super().setUp() testuser, _ = User.objects.get_or_create(username="admin") - UserContactInfo.objects.update_or_create(user=testuser, defaults={"block_execution": False}) + UserContactInfo.objects.update_or_create(user=testuser, defaults={"import_execution_mode": None}) self.system_settings(enable_product_grade=False) self.system_settings(enable_github=False) @@ -363,7 +363,7 @@ def test_import_reimport_reimport_performance_pghistory_no_async(self): configure_pghistory_triggers() testuser = User.objects.get(username="admin") - testuser.usercontactinfo.block_execution = True + testuser.usercontactinfo.import_execution_mode = "sync" testuser.usercontactinfo.save() self._import_reimport_performance( @@ -387,7 +387,7 @@ def test_import_reimport_reimport_performance_pghistory_no_async_with_product_gr configure_pghistory_triggers() testuser = User.objects.get(username="admin") - testuser.usercontactinfo.block_execution = True + testuser.usercontactinfo.import_execution_mode = "sync" testuser.usercontactinfo.save() self.system_settings(enable_product_grade=True) @@ -541,7 +541,7 @@ def test_deduplication_performance_pghistory_no_async(self): self.system_settings(enable_deduplication=True) testuser = User.objects.get(username="admin") - testuser.usercontactinfo.block_execution = True + testuser.usercontactinfo.import_execution_mode = "sync" testuser.usercontactinfo.save() self._deduplication_performance( @@ -653,7 +653,7 @@ def test_import_reimport_reimport_performance_pghistory_no_async(self): configure_pghistory_triggers() testuser = User.objects.get(username="admin") - testuser.usercontactinfo.block_execution = True + testuser.usercontactinfo.import_execution_mode = "sync" testuser.usercontactinfo.save() self._import_reimport_performance( @@ -677,7 +677,7 @@ def test_import_reimport_reimport_performance_pghistory_no_async_with_product_gr configure_pghistory_triggers() testuser = User.objects.get(username="admin") - testuser.usercontactinfo.block_execution = True + testuser.usercontactinfo.import_execution_mode = "sync" testuser.usercontactinfo.save() self.system_settings(enable_product_grade=True) @@ -805,7 +805,7 @@ def test_deduplication_performance_pghistory_no_async(self): self.system_settings(enable_deduplication=True) testuser = User.objects.get(username="admin") - testuser.usercontactinfo.block_execution = True + testuser.usercontactinfo.import_execution_mode = "sync" testuser.usercontactinfo.save() self._deduplication_performance( diff --git a/unittests/test_jira_config_engagement.py b/unittests/test_jira_config_engagement.py index aaabfdddc5e..7b8beb6e8ea 100644 --- a/unittests/test_jira_config_engagement.py +++ b/unittests/test_jira_config_engagement.py @@ -252,7 +252,7 @@ def setUp(self): self.system_settings(enable_jira=True) self.user = self.get_test_admin() self.client.force_login(self.user) - self.user.usercontactinfo.block_execution = True + self.user.usercontactinfo.import_execution_mode = "sync" self.user.usercontactinfo.save() # product 3 has no jira project config, double check to make sure someone didn't molest the fixture # running this in __init__ throws database access denied error diff --git a/unittests/test_jira_config_engagement_epic.py b/unittests/test_jira_config_engagement_epic.py index 614a10fbe57..6f314b9748c 100644 --- a/unittests/test_jira_config_engagement_epic.py +++ b/unittests/test_jira_config_engagement_epic.py @@ -39,7 +39,7 @@ def setUp(self): self.system_settings(enable_jira=True) self.user = self.get_test_admin() self.client.force_login(self.user) - self.user.usercontactinfo.block_execution = True + self.user.usercontactinfo.import_execution_mode = "sync" self.user.usercontactinfo.save() # product 3 has no jira project config, double check to make sure someone didn't molest the fixture # running this in __init__ throws database access denied error diff --git a/unittests/test_jira_import_and_pushing_api.py b/unittests/test_jira_import_and_pushing_api.py index eb3f0692dbc..7d472633363 100644 --- a/unittests/test_jira_import_and_pushing_api.py +++ b/unittests/test_jira_import_and_pushing_api.py @@ -73,7 +73,7 @@ def setUp(self): self.system_settings(enable_jira=True) self.system_settings(enable_webhooks_notifications=True) self.testuser = User.objects.get(username="admin") - self.testuser.usercontactinfo.block_execution = True + self.testuser.usercontactinfo.import_execution_mode = "sync" self.testuser.usercontactinfo.save() token = Token.objects.get(user=self.testuser) self.client = APIClient() diff --git a/unittests/test_notifications.py b/unittests/test_notifications.py index 28921fdf961..06627273ce8 100644 --- a/unittests/test_notifications.py +++ b/unittests/test_notifications.py @@ -641,9 +641,9 @@ class TestAsyncNotificationTaskBody(DojoTestCase): def run(self, result=None): # Same sync pattern used by TestNotificationWebhooks: run under an impersonated user - # with block_execution=True so downstream dojo_dispatch_task calls execute inline. + # with import_execution_mode="sync" so downstream dojo_dispatch_task calls execute inline. testuser = User.objects.get(username="admin") - testuser.usercontactinfo.block_execution = True + testuser.usercontactinfo.import_execution_mode = "sync" testuser.save() with impersonate(testuser): super().run(result) @@ -677,8 +677,8 @@ def test_async_task_returns_silently_on_missing_instance(self, mock_process): @patch("dojo.notifications.helper.create_notification") def test_dispatch_respects_block_execution(self, mock_create): - """With block_execution=True on the impersonated user, the post_save signal runs the task body inline.""" - # The run() wrapper impersonates admin with block_execution=True, so + """With import_execution_mode="sync" on the impersonated user, the post_save signal runs the task body inline.""" + # The run() wrapper impersonates admin with import_execution_mode="sync", so # dojo_dispatch_task takes the sync branch and the task body calls create_notification # (module-level helper inside async_create_notification) synchronously. prod = Product.objects.first() @@ -705,7 +705,7 @@ def run(self, result=None): if getattr(self, "__unittest_skip__", False): return super().run(result) testuser = User.objects.get(username="admin") - testuser.usercontactinfo.block_execution = True + testuser.usercontactinfo.import_execution_mode = "sync" testuser.save() # unit tests are running without any user, which will result in actions like dedupe happening in the celery process diff --git a/unittests/test_prepare_duplicates_for_delete.py b/unittests/test_prepare_duplicates_for_delete.py index 78d612dfdec..f879446f874 100644 --- a/unittests/test_prepare_duplicates_for_delete.py +++ b/unittests/test_prepare_duplicates_for_delete.py @@ -45,7 +45,7 @@ def setUp(self): is_staff=True, is_superuser=True, ) - UserContactInfo.objects.create(user=self.testuser, block_execution=True) + UserContactInfo.objects.create(user=self.testuser, import_execution_mode="sync") self.system_settings(enable_deduplication=False) self.system_settings(enable_product_grade=False) diff --git a/unittests/test_product_grading.py b/unittests/test_product_grading.py index 5f4680c3315..4f929233b81 100644 --- a/unittests/test_product_grading.py +++ b/unittests/test_product_grading.py @@ -12,7 +12,7 @@ class ProductGradeTest(DojoTestCase): def run(self, result=None): testuser = User.objects.get(username="admin") - testuser.usercontactinfo.block_execution = True + testuser.usercontactinfo.import_execution_mode = "sync" testuser.save() # unit tests are running without any user, which will result in actions like dedupe happening in the celery process diff --git a/unittests/test_reimport_prefetch.py b/unittests/test_reimport_prefetch.py index cc5b2f40e1e..fb6bff558c2 100644 --- a/unittests/test_reimport_prefetch.py +++ b/unittests/test_reimport_prefetch.py @@ -72,7 +72,7 @@ class ReimportDuplicateFindingsTestBase(DojoTestCase): def setUp(self): super().setUp() testuser, _ = User.objects.get_or_create(username="admin") - UserContactInfo.objects.get_or_create(user=testuser, defaults={"block_execution": True}) + UserContactInfo.objects.get_or_create(user=testuser, defaults={"import_execution_mode": "sync"}) self.system_settings(enable_deduplication=True) self.system_settings(enable_product_grade=False) diff --git a/unittests/test_tag_inheritance.py b/unittests/test_tag_inheritance.py index 964094fa8ed..f28745b1921 100644 --- a/unittests/test_tag_inheritance.py +++ b/unittests/test_tag_inheritance.py @@ -674,7 +674,7 @@ class InheritedTagsImportTestAPI(DojoAPITestCase, InheritedTagsImportMixin): def setUp(self): super().setUp() testuser = User.objects.get(username="admin") - testuser.usercontactinfo.block_execution = True + testuser.usercontactinfo.import_execution_mode = "sync" testuser.usercontactinfo.save() self.login_as_admin() settings.SECURE_SSL_REDIRECT = False @@ -692,7 +692,7 @@ class InheritedTagsImportTestUI(DojoAPITestCase, InheritedTagsImportMixin): def setUp(self): super().setUp() testuser = User.objects.get(username="admin") - testuser.usercontactinfo.block_execution = True + testuser.usercontactinfo.import_execution_mode = "sync" testuser.usercontactinfo.save() self.login_as_admin() settings.SECURE_SSL_REDIRECT = False diff --git a/unittests/test_tags.py b/unittests/test_tags.py index ea460f247bf..533d9d356b1 100644 --- a/unittests/test_tags.py +++ b/unittests/test_tags.py @@ -387,7 +387,7 @@ def setUp(self): super().setUp() settings.SECURE_SSL_REDIRECT = False testuser = User.objects.get(username="admin") - testuser.usercontactinfo.block_execution = True + testuser.usercontactinfo.import_execution_mode = "sync" testuser.usercontactinfo.save() self.login_as_admin() TagImportMixin.setUp(self) @@ -404,7 +404,7 @@ def setUp(self): super().setUp() settings.SECURE_SSL_REDIRECT = False testuser = User.objects.get(username="admin") - testuser.usercontactinfo.block_execution = True + testuser.usercontactinfo.import_execution_mode = "sync" testuser.usercontactinfo.save() self.login_as_admin() self.client_ui = Client() diff --git a/unittests/test_watson_async_search_index.py b/unittests/test_watson_async_search_index.py index 8edba606ac7..fe54d61c0f8 100644 --- a/unittests/test_watson_async_search_index.py +++ b/unittests/test_watson_async_search_index.py @@ -20,7 +20,7 @@ def setUp(self): super().setUp() self.testuser = User.objects.create(username="admin", is_staff=True, is_superuser=True) - UserContactInfo.objects.create(user=self.testuser, block_execution=True) + UserContactInfo.objects.create(user=self.testuser, import_execution_mode="sync") self.system_settings(enable_product_grade=False) self.system_settings(enable_github=False) From 8e16f481ccab7150c50e03131445ca174b92b904 Mon Sep 17 00:00:00 2001 From: Valentijn Scholten Date: Sun, 14 Jun 2026 10:30:54 +0200 Subject: [PATCH 02/12] feat(importers): make scan_added notifications dedup-aware MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit notify_scan_added used the importer's in-memory finding lists, whose duplicate flag is stale because deduplication runs on separately-fetched instances. Once deduplication has completed (sync or async_wait, signalled by deduplication_complete), refresh the duplicate flag from the database and split each action list into "real" and duplicate findings: - findings_new -> net-new (excludes deduplicated) - findings_new_duplicate / findings_reactivated_duplicate / findings_untouched_duplicate - finding_count -> recomputed to exclude new/reactivated duplicates (so an all-duplicate import sends scan_added_empty) In plain async mode (dedup not awaited) the lists are left untouched, preserving historical behavior. Mail and webhook scan_added templates render the new duplicate sections. Note: import/reimport response statistics are already dedup-accurate once awaited — Test.statistics / Test_Import.statistics query Finding and annotate the duplicate count directly; only the notification used stale in-memory data. --- dojo/importers/base_importer.py | 44 +++++++++++ .../notifications/mail/scan_added.tpl | 33 ++++++++ .../notifications/webhooks/scan_added.tpl | 8 +- unittests/test_import_execution_mode.py | 76 +++++++++++++++++++ 4 files changed, 160 insertions(+), 1 deletion(-) diff --git a/dojo/importers/base_importer.py b/dojo/importers/base_importer.py index b5daa972bbe..fc45afb33b2 100644 --- a/dojo/importers/base_importer.py +++ b/dojo/importers/base_importer.py @@ -939,10 +939,49 @@ def notify_scan_added( new_findings = [] logger.debug("Scan added notifications") + # When deduplication has finished (synchronous mode, or async_wait after the + # join), the in-memory findings still carry their pre-dedup duplicate=False + # flag because deduplication runs on separately-fetched instances. Refresh the + # flag from the database and split each list into "real" and duplicate findings + # so the notification reflects post-dedup reality instead of counting/listing + # deduplicated findings as brand new. In plain async mode dedup has not run yet, + # so we leave the lists untouched (best-effort, historical behavior). + findings_new_duplicate: list[Finding] = [] + findings_reactivated_duplicate: list[Finding] = [] + findings_untouched_duplicate: list[Finding] = [] + if getattr(self, "deduplication_complete", False): + all_ids = [f.id for f in (*new_findings, *findings_reactivated, *findings_untouched)] + duplicate_ids = set() + if all_ids: + duplicate_ids = set( + Finding.objects.filter(id__in=all_ids, duplicate=True).values_list("id", flat=True), + ) + + def _split(findings): + kept, duplicates = [], [] + for finding in findings: + if finding.id in duplicate_ids: + # refresh the in-memory flag so any template logic is correct + finding.duplicate = True + duplicates.append(finding) + else: + kept.append(finding) + return kept, duplicates + + new_findings, findings_new_duplicate = _split(new_findings) + findings_reactivated, findings_reactivated_duplicate = _split(findings_reactivated) + findings_untouched, findings_untouched_duplicate = _split(findings_untouched) + # Recompute the headline count to exclude findings that turned out to be + # duplicates of an existing finding (they are not genuinely new activity). + updated_count = len(new_findings) + len(findings_reactivated) + len(findings_mitigated) + new_findings = sorted(new_findings, key=lambda x: x.numerical_severity) findings_mitigated = sorted(findings_mitigated, key=lambda x: x.numerical_severity) findings_reactivated = sorted(findings_reactivated, key=lambda x: x.numerical_severity) findings_untouched = sorted(findings_untouched, key=lambda x: x.numerical_severity) + findings_new_duplicate = sorted(findings_new_duplicate, key=lambda x: x.numerical_severity) + findings_reactivated_duplicate = sorted(findings_reactivated_duplicate, key=lambda x: x.numerical_severity) + findings_untouched_duplicate = sorted(findings_untouched_duplicate, key=lambda x: x.numerical_severity) title = ( f"Created/Updated {updated_count} findings for {test.engagement.product}: {test.engagement.name}: {test}" @@ -959,6 +998,11 @@ def notify_scan_added( engagement=test.engagement, product=test.engagement.product, findings_untouched=findings_untouched, + # Findings deduplicated during post-processing, split by their import action. + # Populated only once deduplication has completed (sync / async_wait). + findings_new_duplicate=findings_new_duplicate, + findings_reactivated_duplicate=findings_reactivated_duplicate, + findings_untouched_duplicate=findings_untouched_duplicate, url=reverse("view_test", args=(test.id,)), url_api=reverse("test-detail", args=(test.id,)), ) diff --git a/dojo/templates_classic/notifications/mail/scan_added.tpl b/dojo/templates_classic/notifications/mail/scan_added.tpl index 263585246e0..567d80ee47e 100644 --- a/dojo/templates_classic/notifications/mail/scan_added.tpl +++ b/dojo/templates_classic/notifications/mail/scan_added.tpl @@ -26,6 +26,17 @@ {% endfor %}

+ {% if findings_new_duplicate %} +

+

+ {% blocktranslate %}New findings detected as duplicates{% endblocktranslate %} ({{ findings_new_duplicate | length }})
+ {% for finding in findings_new_duplicate %} + {% url 'view_finding' finding.id as finding_url %} + {{ finding.title }} ({{ finding.severity }})
+ {% endfor %} +
+

+ {% endif %}

{% blocktranslate %}Reactivated findings{% endblocktranslate %} ({{ findings_reactivated | length }})
@@ -37,6 +48,17 @@ {% endfor %}

+ {% if findings_reactivated_duplicate %} +

+

+ {% blocktranslate %}Reactivated findings detected as duplicates{% endblocktranslate %} ({{ findings_reactivated_duplicate | length }})
+ {% for finding in findings_reactivated_duplicate %} + {% url 'view_finding' finding.id as finding_url %} + {{ finding.title }} ({{ finding.severity }})
+ {% endfor %} +
+

+ {% endif %}

{% blocktranslate %}Closed findings{% endblocktranslate %} ({{ findings_mitigated | length }})
@@ -59,6 +81,17 @@ {% endfor %}

+ {% if findings_untouched_duplicate %} +

+

+ {% blocktranslate %}Existing findings detected as duplicates{% endblocktranslate %} ({{ findings_untouched_duplicate | length }})
+ {% for finding in findings_untouched_duplicate %} + {% url 'view_finding' finding.id as finding_url %} + {{ finding.title }} ({{ finding.severity }})
+ {% endfor %} +
+

+ {% endif %}

{% trans "Kind regards" %},

diff --git a/dojo/templates_classic/notifications/webhooks/scan_added.tpl b/dojo/templates_classic/notifications/webhooks/scan_added.tpl index b42096bfba2..0f68a72eb12 100644 --- a/dojo/templates_classic/notifications/webhooks/scan_added.tpl +++ b/dojo/templates_classic/notifications/webhooks/scan_added.tpl @@ -8,5 +8,11 @@ findings: {% include 'notifications/webhooks/subtemplates/findings_list.tpl' with findings=findings_reactivated %} mitigated: {% include 'notifications/webhooks/subtemplates/findings_list.tpl' with findings=findings_mitigated %} - untouched: + untouched: {% include 'notifications/webhooks/subtemplates/findings_list.tpl' with findings=findings_untouched %} + new_duplicate: +{% include 'notifications/webhooks/subtemplates/findings_list.tpl' with findings=findings_new_duplicate %} + reactivated_duplicate: +{% include 'notifications/webhooks/subtemplates/findings_list.tpl' with findings=findings_reactivated_duplicate %} + untouched_duplicate: +{% include 'notifications/webhooks/subtemplates/findings_list.tpl' with findings=findings_untouched_duplicate %} diff --git a/unittests/test_import_execution_mode.py b/unittests/test_import_execution_mode.py index 568439f7ff6..a9232907df5 100644 --- a/unittests/test_import_execution_mode.py +++ b/unittests/test_import_execution_mode.py @@ -1,3 +1,5 @@ +from unittest.mock import patch + from django.test import override_settings from dojo.importers.default_importer import DefaultImporter @@ -8,6 +10,8 @@ Development_Environment, Dojo_User, Engagement, + Finding, + Test, UserContactInfo, ) @@ -145,3 +149,75 @@ def test_import_rejects_invalid_mode(self): payload = self._payload("not-a-mode") payload["file"] = testfile self.import_scan(payload, 400) + + +class NotificationDeduplicationRefreshTest(DojoTestCase): + + """notify_scan_added refreshes duplicate status from the DB once dedup is complete.""" + + fixtures = ["dojo_testdata.json"] + + def _importer(self): + test = Test.objects.first() + importer = DefaultImporter( + scan_type="ZAP Scan", + engagement=test.engagement, + environment=Development_Environment.objects.first(), + ) + return importer, test + + @patch("dojo.importers.base_importer.create_notification") + def test_deduplicated_new_findings_excluded_when_complete(self, mock_notify): + importer, test = self._importer() + importer.deduplication_complete = True + + real = Finding(test=test, title="real finding", severity="High") + real.save() + dupe = Finding(test=test, title="dupe finding", severity="High") + dupe.save() + # Simulate background deduplication having flagged the second finding. + Finding.objects.filter(pk=dupe.pk).update(duplicate=True) + + importer.notify_scan_added(test, updated_count=2, new_findings=[real, dupe]) + + kwargs = mock_notify.call_args.kwargs + self.assertEqual([f.id for f in kwargs["findings_new"]], [real.id]) + self.assertEqual([f.id for f in kwargs["findings_new_duplicate"]], [dupe.id]) + # headline count excludes the deduplicated finding + self.assertEqual(kwargs["finding_count"], 1) + self.assertEqual(kwargs["event"], "scan_added") + + @patch("dojo.importers.base_importer.create_notification") + def test_async_mode_does_not_refresh(self, mock_notify): + importer, test = self._importer() + importer.deduplication_complete = False # plain async: dedup not awaited + + dupe = Finding(test=test, title="async dupe", severity="High") + dupe.save() + Finding.objects.filter(pk=dupe.pk).update(duplicate=True) + + importer.notify_scan_added(test, updated_count=1, new_findings=[dupe]) + + kwargs = mock_notify.call_args.kwargs + # historical behavior: duplicate still listed/counted as new + self.assertEqual([f.id for f in kwargs["findings_new"]], [dupe.id]) + self.assertEqual(kwargs["findings_new_duplicate"], []) + self.assertEqual(kwargs["finding_count"], 1) + + @patch("dojo.importers.base_importer.create_notification") + def test_all_new_findings_duplicate_yields_empty_event(self, mock_notify): + importer, test = self._importer() + importer.deduplication_complete = True + + dupe = Finding(test=test, title="only dupe", severity="Low") + dupe.save() + Finding.objects.filter(pk=dupe.pk).update(duplicate=True) + + importer.notify_scan_added(test, updated_count=1, new_findings=[dupe]) + + kwargs = mock_notify.call_args.kwargs + self.assertEqual(kwargs["findings_new"], []) + self.assertEqual([f.id for f in kwargs["findings_new_duplicate"]], [dupe.id]) + self.assertEqual(kwargs["finding_count"], 0) + # net-new is zero -> empty scan notification + self.assertEqual(kwargs["event"], "scan_added_empty") From f3ba2df0d79393aa34137a9b3f0b3d9d89ff22b0 Mon Sep 17 00:00:00 2001 From: Valentijn Scholten Date: Sun, 14 Jun 2026 14:47:37 +0200 Subject: [PATCH 03/12] refactor(importers): await dedup before all import notifications Move wait_for_post_processing() ahead of the test_added notification dispatch (not just notify_scan_added) so both notifications sent during a fresh import are emitted after deduplication has completed in async_wait mode. The reimporter already orders the await before its single notification. --- dojo/importers/default_importer.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/dojo/importers/default_importer.py b/dojo/importers/default_importer.py index c4865083989..5259aa7c336 100644 --- a/dojo/importers/default_importer.py +++ b/dojo/importers/default_importer.py @@ -135,6 +135,10 @@ def process_scan( new_findings=new_findings, closed_findings=closed_findings, ) + # In 'async_wait' mode, block until background deduplication has finished + # so notifications and statistics reflect the deduplicated state. + self.wait_for_post_processing() + # Send out some notifications to the user logger.debug("IMPORT_SCAN: Generating notifications") dojo_dispatch_task( @@ -147,9 +151,6 @@ def process_scan( url=reverse("view_test", args=(self.test.id,)), url_api=reverse("test-detail", args=(self.test.id,)), ) - # In 'async_wait' mode, block until background deduplication has finished - # so notifications and statistics reflect the deduplicated state. - self.wait_for_post_processing() updated_count = len(new_findings) + len(closed_findings) self.notify_scan_added( self.test, From 7f764c1eddd6d2bb1d3e9a17fa651829f485e03f Mon Sep 17 00:00:00 2001 From: Valentijn Scholten Date: Sun, 14 Jun 2026 14:53:44 +0200 Subject: [PATCH 04/12] refactor: rename import_execution_mode to deduplication_execution_mode The mode governs whether the request waits for deduplication, so name it for that. Rename the UserContactInfo field, the request/serializer field, the ImporterOptions attribute and validator, the resolver, and the IMPORT_EXECUTION_MODE_* constants to DEDUPLICATION_EXECUTION_MODE_*. Add a RenameField migration (0271) rather than mutating the add/remove migrations introduced earlier on this branch. --- dojo/api_v2/serializers.py | 12 ++-- dojo/celery_dispatch.py | 2 +- ...ame_import_execution_mode_deduplication.py | 16 ++++++ dojo/decorators.py | 6 +- dojo/forms.py | 2 +- dojo/importers/base_importer.py | 12 ++-- dojo/importers/default_importer.py | 4 +- dojo/importers/default_reimporter.py | 4 +- dojo/importers/options.py | 20 +++---- dojo/middleware.py | 2 +- dojo/models.py | 40 ++++++------- dojo/templates/dojo/view_user.html | 2 +- dojo/templates_classic/dojo/view_user.html | 2 +- unittests/test_async_delete.py | 8 +-- unittests/test_cascade_delete.py | 2 +- unittests/test_celery_dispatch_force_async.py | 2 +- unittests/test_deduplication_logic.py | 2 +- unittests/test_duplication_loops.py | 2 +- .../test_false_positive_history_logic.py | 2 +- unittests/test_finding_helper.py | 2 +- unittests/test_finding_model.py | 2 +- unittests/test_import_execution_mode.py | 56 +++++++++---------- unittests/test_import_reimport.py | 4 +- unittests/test_importers_deduplication.py | 2 +- unittests/test_importers_performance.py | 14 ++--- unittests/test_jira_config_engagement.py | 2 +- unittests/test_jira_config_engagement_epic.py | 2 +- unittests/test_jira_import_and_pushing_api.py | 2 +- unittests/test_notifications.py | 10 ++-- .../test_prepare_duplicates_for_delete.py | 2 +- unittests/test_product_grading.py | 2 +- unittests/test_reimport_prefetch.py | 2 +- unittests/test_tag_inheritance.py | 4 +- unittests/test_tags.py | 4 +- unittests/test_watson_async_search_index.py | 2 +- 35 files changed, 135 insertions(+), 119 deletions(-) create mode 100644 dojo/db_migrations/0271_rename_import_execution_mode_deduplication.py diff --git a/dojo/api_v2/serializers.py b/dojo/api_v2/serializers.py index a47ce022d11..42b2912f760 100644 --- a/dojo/api_v2/serializers.py +++ b/dojo/api_v2/serializers.py @@ -41,8 +41,8 @@ from dojo.jira import services as jira_services from dojo.location.models import Location, LocationFindingReference from dojo.models import ( + DEDUPLICATION_EXECUTION_MODE_CHOICES, IMPORT_ACTIONS, - IMPORT_EXECUTION_MODE_CHOICES, SEVERITIES, SEVERITY_CHOICES, STATS_FIELDS, @@ -1869,15 +1869,15 @@ class CommonImportScanSerializer(serializers.Serializer): allow_null=True, default=None, queryset=User.objects.all(), ) push_to_jira = serializers.BooleanField(default=False) - import_execution_mode = serializers.ChoiceField( + deduplication_execution_mode = serializers.ChoiceField( required=False, allow_null=True, - choices=IMPORT_EXECUTION_MODE_CHOICES, + choices=DEDUPLICATION_EXECUTION_MODE_CHOICES, help_text="Override how import post-processing (deduplication, jira push, grading, ...) is executed for " "this request. 'async' dispatches post-processing to the background and responds immediately (default). " "'async_wait' dispatches to the background but waits for deduplication to finish before responding, so " "notifications and the returned statistics reflect the deduplicated state. 'sync' runs everything inline. " - "If omitted, falls back to the user's profile setting (import_execution_mode).", + "If omitted, falls back to the user's profile setting (deduplication_execution_mode).", ) environment = serializers.CharField(required=False) build_id = serializers.CharField( @@ -2093,8 +2093,8 @@ def setup_common_context(self, data: dict) -> dict: # takes precedence over the user's profile setting, otherwise default async. request = self.context.get("request") user = getattr(request, "user", None) - context["import_execution_mode"] = Dojo_User.resolve_import_execution_mode( - user, data.get("import_execution_mode"), + context["deduplication_execution_mode"] = Dojo_User.resolve_deduplication_execution_mode( + user, data.get("deduplication_execution_mode"), ) return context diff --git a/dojo/celery_dispatch.py b/dojo/celery_dispatch.py index bd552eab7ea..3171291d5ee 100644 --- a/dojo/celery_dispatch.py +++ b/dojo/celery_dispatch.py @@ -62,7 +62,7 @@ def dojo_dispatch_task(task_or_sig: _SupportsSi | _SupportsApplyAsync | Signatur - Inject `async_user_id` if missing. - Capture and inject pghistory context if available. - Respect `force_sync=True` (foreground execution) and the user's - synchronous import_execution_mode. + synchronous deduplication_execution_mode. - Respect `force_async=True` (background execution even when the caller would otherwise run synchronously, e.g. user on the synchronous mode). `force_async` wins over `force_sync` and the user's mode. diff --git a/dojo/db_migrations/0271_rename_import_execution_mode_deduplication.py b/dojo/db_migrations/0271_rename_import_execution_mode_deduplication.py new file mode 100644 index 00000000000..51812be16d3 --- /dev/null +++ b/dojo/db_migrations/0271_rename_import_execution_mode_deduplication.py @@ -0,0 +1,16 @@ +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('dojo', '0270_remove_usercontactinfo_block_execution'), + ] + + operations = [ + migrations.RenameField( + model_name='usercontactinfo', + old_name='import_execution_mode', + new_name='deduplication_execution_mode', + ), + ] diff --git a/dojo/decorators.py b/dojo/decorators.py index 7e533368f77..d938c6e2661 100644 --- a/dojo/decorators.py +++ b/dojo/decorators.py @@ -76,15 +76,15 @@ def we_want_async(*args, func=None, **kwargs): return True if Dojo_User.wants_block_execution(user): - logger.debug("dojo_async_task %s: running task in the foreground as import_execution_mode is 'sync' for %s", func, user) + logger.debug("dojo_async_task %s: running task in the foreground as deduplication_execution_mode is 'sync' for %s", func, user) return False - logger.debug("dojo_async_task %s: running task in the background as import_execution_mode is not 'sync' for %s", func, user) + logger.debug("dojo_async_task %s: running task in the background as deduplication_execution_mode is not 'sync' for %s", func, user) return True # Defect Dojo performs all tasks asynchrnonously using celery -# *unless* the user initiating the task has set import_execution_mode to 'sync' in their usercontactinfo profile +# *unless* the user initiating the task has set deduplication_execution_mode to 'sync' in their usercontactinfo profile def dojo_async_task(func=None, *, signature=False): def decorator(func): @wraps(func) diff --git a/dojo/forms.py b/dojo/forms.py index 4f09388a866..07ef3c16c68 100644 --- a/dojo/forms.py +++ b/dojo/forms.py @@ -2393,7 +2393,7 @@ class Meta: # Swap order: password_last_reset before token_last_reset field_order = [ "title", "phone_number", "cell_number", "twitter_username", "github_username", - "slack_username", "ui_use_tailwind", "import_execution_mode", "force_password_reset", "reset_api_token", + "slack_username", "ui_use_tailwind", "deduplication_execution_mode", "force_password_reset", "reset_api_token", "password_last_reset", "token_last_reset", ] diff --git a/dojo/importers/base_importer.py b/dojo/importers/base_importer.py index fc45afb33b2..ab2f205b9f9 100644 --- a/dojo/importers/base_importer.py +++ b/dojo/importers/base_importer.py @@ -16,11 +16,11 @@ from dojo.jira.services import is_keep_in_sync from dojo.location.models import Location from dojo.models import ( + DEDUPLICATION_EXECUTION_MODE_ASYNC_WAIT, + DEDUPLICATION_EXECUTION_MODE_SYNC, # Import History States IMPORT_CLOSED_FINDING, IMPORT_CREATED_FINDING, - IMPORT_EXECUTION_MODE_ASYNC_WAIT, - IMPORT_EXECUTION_MODE_SYNC, IMPORT_REACTIVATED_FINDING, IMPORT_UNTOUCHED_FINDING, # Finding Severities @@ -99,9 +99,9 @@ def post_processing_dispatch_kwargs(self, **kwargs): - ASYNC (default): preserve historical behavior, honoring any externally supplied force_sync and the user's sync mode via we_want_async. """ - if self.import_execution_mode == IMPORT_EXECUTION_MODE_SYNC: + if self.deduplication_execution_mode == DEDUPLICATION_EXECUTION_MODE_SYNC: return {"force_sync": True} - if self.import_execution_mode == IMPORT_EXECUTION_MODE_ASYNC_WAIT: + if self.deduplication_execution_mode == DEDUPLICATION_EXECUTION_MODE_ASYNC_WAIT: return {"force_async": True} return {"force_sync": kwargs.get("force_sync", False)} @@ -126,11 +126,11 @@ def wait_for_post_processing(self): settings.IMPORT_ASYNC_WAIT_TIMEOUT so a stuck/missing worker degrades to the historical (respond-anyway) behavior instead of hanging. """ - if self.import_execution_mode == IMPORT_EXECUTION_MODE_SYNC: + if self.deduplication_execution_mode == DEDUPLICATION_EXECUTION_MODE_SYNC: # Batches ran inline during process_findings, so dedup is already done. self.deduplication_complete = True return - if self.import_execution_mode != IMPORT_EXECUTION_MODE_ASYNC_WAIT: + if self.deduplication_execution_mode != DEDUPLICATION_EXECUTION_MODE_ASYNC_WAIT: # 'async': post-processing was dispatched but is not awaited. self.deduplication_complete = False return diff --git a/dojo/importers/default_importer.py b/dojo/importers/default_importer.py index 5259aa7c336..203d919ce67 100644 --- a/dojo/importers/default_importer.py +++ b/dojo/importers/default_importer.py @@ -12,7 +12,7 @@ from dojo.importers.options import ImporterOptions from dojo.jira import services as jira_services from dojo.models import ( - IMPORT_EXECUTION_MODE_ASYNC_WAIT, + DEDUPLICATION_EXECUTION_MODE_ASYNC_WAIT, Engagement, Finding, Test, @@ -305,7 +305,7 @@ def _process_findings_internal( push_to_jira=push_to_jira, **self.post_processing_dispatch_kwargs(**kwargs), ) - if self.import_execution_mode == IMPORT_EXECUTION_MODE_ASYNC_WAIT: + if self.deduplication_execution_mode == DEDUPLICATION_EXECUTION_MODE_ASYNC_WAIT: self.record_post_processing_result(result) # No chord: tasks are dispatched immediately above per batch diff --git a/dojo/importers/default_reimporter.py b/dojo/importers/default_reimporter.py index a32fa022af4..83ee9386945 100644 --- a/dojo/importers/default_reimporter.py +++ b/dojo/importers/default_reimporter.py @@ -18,7 +18,7 @@ from dojo.importers.options import ImporterOptions from dojo.jira import services as jira_services from dojo.models import ( - IMPORT_EXECUTION_MODE_ASYNC_WAIT, + DEDUPLICATION_EXECUTION_MODE_ASYNC_WAIT, Development_Environment, Finding, Notes, @@ -470,7 +470,7 @@ def _process_findings_internal( jira_instance_id=getattr(self.jira_instance, "id", None), **self.post_processing_dispatch_kwargs(**kwargs), ) - if self.import_execution_mode == IMPORT_EXECUTION_MODE_ASYNC_WAIT: + if self.deduplication_execution_mode == DEDUPLICATION_EXECUTION_MODE_ASYNC_WAIT: self.record_post_processing_result(result) # No chord: tasks are dispatched immediately above per batch diff --git a/dojo/importers/options.py b/dojo/importers/options.py index 3817a96fe08..d25f1a68b29 100644 --- a/dojo/importers/options.py +++ b/dojo/importers/options.py @@ -12,9 +12,9 @@ from dojo.jira.services import get_instance as get_jira_instance from dojo.models import ( - IMPORT_EXECUTION_MODE_ASYNC, - IMPORT_EXECUTION_MODE_SYNC, - IMPORT_EXECUTION_MODES, + DEDUPLICATION_EXECUTION_MODE_ASYNC, + DEDUPLICATION_EXECUTION_MODE_SYNC, + DEDUPLICATION_EXECUTION_MODES, Development_Environment, Dojo_User, Endpoint, @@ -71,7 +71,7 @@ def load_base_options( self.engagement: Engagement | None = self.validate_engagement(*args, **kwargs) self.environment: Development_Environment | None = self.validate_environment(*args, **kwargs) self.group_by: str = self.validate_group_by(*args, **kwargs) - self.import_execution_mode: str = self.validate_import_execution_mode(*args, **kwargs) + self.deduplication_execution_mode: str = self.validate_deduplication_execution_mode(*args, **kwargs) self.import_type: str = self.validate_import_type(*args, **kwargs) self.lead: Dojo_User | None = self.validate_lead(*args, **kwargs) self.minimum_severity: str = self.validate_minimum_severity(*args, **kwargs) @@ -349,23 +349,23 @@ def validate_do_not_reactivate( **kwargs, ) - def validate_import_execution_mode( + def validate_deduplication_execution_mode( self, *args: list, **kwargs: dict, ) -> str: mode = self.validate( - "import_execution_mode", + "deduplication_execution_mode", expected_types=[str], required=False, - default=IMPORT_EXECUTION_MODE_ASYNC, + default=DEDUPLICATION_EXECUTION_MODE_ASYNC, **kwargs, ) - if mode not in IMPORT_EXECUTION_MODES: - mode = IMPORT_EXECUTION_MODE_ASYNC + if mode not in DEDUPLICATION_EXECUTION_MODES: + mode = DEDUPLICATION_EXECUTION_MODE_ASYNC # An explicit force_sync from a non-serializer caller still wins. if kwargs.get("force_sync"): - mode = IMPORT_EXECUTION_MODE_SYNC + mode = DEDUPLICATION_EXECUTION_MODE_SYNC return mode def validate_commit_hash( diff --git a/dojo/middleware.py b/dojo/middleware.py index 73e0ed601dd..d9cdd626b1e 100644 --- a/dojo/middleware.py +++ b/dojo/middleware.py @@ -281,7 +281,7 @@ def _drain_search_context_to_async(objects, source): for model_name, pk_list in model_groups.items(): batches = [pk_list[i:i + batch_size] for i in range(0, len(pk_list), batch_size)] # force_async=True keeps indexing off the request path even for users - # on the synchronous import_execution_mode — index updates are slow and + # on the synchronous deduplication_execution_mode — index updates are slow and # never need to be synchronous from the user's perspective. for i, batch in enumerate(batches, 1): logger.debug(f"{source}: Triggering batch {i}/{len(batches)} for {model_name}: {len(batch)} instances") diff --git a/dojo/models.py b/dojo/models.py index 01abb6bcace..f3c9e4f7f9b 100644 --- a/dojo/models.py +++ b/dojo/models.py @@ -206,18 +206,18 @@ def __str__(self): # request waits for the deduplication batches to finish before responding, so # notifications and the returned statistics reflect the deduplicated state. # - SYNC: post-processing runs inline in the web process (legacy block_execution). -IMPORT_EXECUTION_MODE_ASYNC = "async" -IMPORT_EXECUTION_MODE_ASYNC_WAIT = "async_wait" -IMPORT_EXECUTION_MODE_SYNC = "sync" -IMPORT_EXECUTION_MODES = ( - IMPORT_EXECUTION_MODE_ASYNC, - IMPORT_EXECUTION_MODE_ASYNC_WAIT, - IMPORT_EXECUTION_MODE_SYNC, +DEDUPLICATION_EXECUTION_MODE_ASYNC = "async" +DEDUPLICATION_EXECUTION_MODE_ASYNC_WAIT = "async_wait" +DEDUPLICATION_EXECUTION_MODE_SYNC = "sync" +DEDUPLICATION_EXECUTION_MODES = ( + DEDUPLICATION_EXECUTION_MODE_ASYNC, + DEDUPLICATION_EXECUTION_MODE_ASYNC_WAIT, + DEDUPLICATION_EXECUTION_MODE_SYNC, ) -IMPORT_EXECUTION_MODE_CHOICES = ( - (IMPORT_EXECUTION_MODE_ASYNC, _("Async (do not wait)")), - (IMPORT_EXECUTION_MODE_ASYNC_WAIT, _("Async, wait for deduplication")), - (IMPORT_EXECUTION_MODE_SYNC, _("Synchronous (block)")), +DEDUPLICATION_EXECUTION_MODE_CHOICES = ( + (DEDUPLICATION_EXECUTION_MODE_ASYNC, _("Async (do not wait)")), + (DEDUPLICATION_EXECUTION_MODE_ASYNC_WAIT, _("Async, wait for deduplication")), + (DEDUPLICATION_EXECUTION_MODE_SYNC, _("Synchronous (block)")), ) @@ -238,22 +238,22 @@ def wants_block_execution(user): # this returns False if there is no user, i.e. in celery processes, unittests, etc. # The synchronous import execution mode is the successor of the old block_execution # flag and governs whether async tasks run in the foreground for this user. - return hasattr(user, "usercontactinfo") and user.usercontactinfo.import_execution_mode == IMPORT_EXECUTION_MODE_SYNC + return hasattr(user, "usercontactinfo") and user.usercontactinfo.deduplication_execution_mode == DEDUPLICATION_EXECUTION_MODE_SYNC @staticmethod - def resolve_import_execution_mode(user, override=None): + def resolve_deduplication_execution_mode(user, override=None): """ Resolve the effective import post-processing execution mode. Priority: explicit request override > user profile setting > default async. - Returns one of IMPORT_EXECUTION_MODE_ASYNC / _ASYNC_WAIT / _SYNC. + Returns one of DEDUPLICATION_EXECUTION_MODE_ASYNC / _ASYNC_WAIT / _SYNC. """ - if override in IMPORT_EXECUTION_MODES: + if override in DEDUPLICATION_EXECUTION_MODES: return override info = getattr(user, "usercontactinfo", None) - if info is not None and info.import_execution_mode in IMPORT_EXECUTION_MODES: - return info.import_execution_mode - return IMPORT_EXECUTION_MODE_ASYNC + if info is not None and info.deduplication_execution_mode in DEDUPLICATION_EXECUTION_MODES: + return info.deduplication_execution_mode + return DEDUPLICATION_EXECUTION_MODE_ASYNC @staticmethod def force_password_reset(user): @@ -294,9 +294,9 @@ class UserContactInfo(models.Model): github_username = models.CharField(blank=True, null=True, max_length=150) slack_username = models.CharField(blank=True, null=True, max_length=150, help_text=_("Email address associated with your slack account"), verbose_name=_("Slack Email Address")) slack_user_id = models.CharField(blank=True, null=True, max_length=25) - import_execution_mode = models.CharField( + deduplication_execution_mode = models.CharField( max_length=20, - choices=IMPORT_EXECUTION_MODE_CHOICES, + choices=DEDUPLICATION_EXECUTION_MODE_CHOICES, null=True, blank=True, help_text=_( diff --git a/dojo/templates/dojo/view_user.html b/dojo/templates/dojo/view_user.html index 47542577c0d..acc31927a48 100644 --- a/dojo/templates/dojo/view_user.html +++ b/dojo/templates/dojo/view_user.html @@ -281,7 +281,7 @@

{% trans "Import execution mode" %} - {{ user.usercontactinfo.get_import_execution_mode_display|default:_("Async (do not wait)") }} + {{ user.usercontactinfo.get_deduplication_execution_mode_display|default:_("Async (do not wait)") }} diff --git a/dojo/templates_classic/dojo/view_user.html b/dojo/templates_classic/dojo/view_user.html index 4d6ac25220f..238694e1c5f 100644 --- a/dojo/templates_classic/dojo/view_user.html +++ b/dojo/templates_classic/dojo/view_user.html @@ -281,7 +281,7 @@

{% trans "Import execution mode" %} - {{ user.usercontactinfo.get_import_execution_mode_display|default:_("Async (do not wait)") }} + {{ user.usercontactinfo.get_deduplication_execution_mode_display|default:_("Async (do not wait)") }} diff --git a/unittests/test_async_delete.py b/unittests/test_async_delete.py index ae0105791e7..b7c2b5413a4 100644 --- a/unittests/test_async_delete.py +++ b/unittests/test_async_delete.py @@ -27,21 +27,21 @@ class TestAsyncDelete(DojoTestCase): """ Test async_delete functionality with dojo_dispatch_task kwargs injection. - These tests use import_execution_mode="sync" and crum.impersonate to run tasks synchronously, + These tests use deduplication_execution_mode="sync" and crum.impersonate to run tasks synchronously, which allows errors to surface immediately rather than being lost in background workers. """ def setUp(self): - """Set up test user with import_execution_mode="sync" and disable unneeded features.""" + """Set up test user with deduplication_execution_mode="sync" and disable unneeded features.""" super().setUp() - # Create test user with import_execution_mode="sync" to run tasks synchronously + # Create test user with deduplication_execution_mode="sync" to run tasks synchronously self.testuser = User.objects.create( username="test_async_delete_user", is_staff=True, is_superuser=True, ) - UserContactInfo.objects.create(user=self.testuser, import_execution_mode="sync") + UserContactInfo.objects.create(user=self.testuser, deduplication_execution_mode="sync") # Log in as the test user (for API client) self.client.force_login(self.testuser) diff --git a/unittests/test_cascade_delete.py b/unittests/test_cascade_delete.py index 8f8a4244610..ab86c942b43 100644 --- a/unittests/test_cascade_delete.py +++ b/unittests/test_cascade_delete.py @@ -37,7 +37,7 @@ def setUp(self): is_staff=True, is_superuser=True, ) - UserContactInfo.objects.create(user=self.testuser, import_execution_mode="sync") + UserContactInfo.objects.create(user=self.testuser, deduplication_execution_mode="sync") self.system_settings(enable_deduplication=False) self.system_settings(enable_product_grade=False) diff --git a/unittests/test_celery_dispatch_force_async.py b/unittests/test_celery_dispatch_force_async.py index 7cf523d78d2..d0479a02d7b 100644 --- a/unittests/test_celery_dispatch_force_async.py +++ b/unittests/test_celery_dispatch_force_async.py @@ -3,7 +3,7 @@ `force_async=True` is for callers (e.g. the watson async indexer middleware) that should always run their celery task in the background even when the -current user has `import_execution_mode="sync"` or the caller passes `force_sync=True`. +current user has `deduplication_execution_mode="sync"` or the caller passes `force_sync=True`. """ from unittest.mock import patch diff --git a/unittests/test_deduplication_logic.py b/unittests/test_deduplication_logic.py index e1f5e08d05e..1c8ed8773ab 100644 --- a/unittests/test_deduplication_logic.py +++ b/unittests/test_deduplication_logic.py @@ -162,7 +162,7 @@ class TestDuplicationLogic(DojoTestCase): def run(self, result=None): testuser = User.objects.get(username="admin") - testuser.usercontactinfo.import_execution_mode = "sync" + testuser.usercontactinfo.deduplication_execution_mode = "sync" testuser.save() # unit tests are running without any user, which will result in actions like dedupe happening in the celery process diff --git a/unittests/test_duplication_loops.py b/unittests/test_duplication_loops.py index fabeb376de0..4c4616dafc4 100644 --- a/unittests/test_duplication_loops.py +++ b/unittests/test_duplication_loops.py @@ -19,7 +19,7 @@ class TestDuplicationLoops(DojoTestCase): def run(self, result=None): testuser = User.objects.get(username="admin") - testuser.usercontactinfo.import_execution_mode = "sync" + testuser.usercontactinfo.deduplication_execution_mode = "sync" testuser.save() # unit tests are running without any user, which will result in actions like dedupe happening in the celery process diff --git a/unittests/test_false_positive_history_logic.py b/unittests/test_false_positive_history_logic.py index a599bda74ef..c81563f9eb0 100644 --- a/unittests/test_false_positive_history_logic.py +++ b/unittests/test_false_positive_history_logic.py @@ -131,7 +131,7 @@ class TestFalsePositiveHistoryLogic(DojoTestCase): def run(self, result=None): testuser = User.objects.get(username="admin") - testuser.usercontactinfo.import_execution_mode = "sync" + testuser.usercontactinfo.deduplication_execution_mode = "sync" testuser.save() # Unit tests are running without any user, which will result in actions like dedupe happening in the celery process diff --git a/unittests/test_finding_helper.py b/unittests/test_finding_helper.py index 51c7602409a..0f78154cb9a 100644 --- a/unittests/test_finding_helper.py +++ b/unittests/test_finding_helper.py @@ -253,7 +253,7 @@ def setUp(self): super().setUp() self.system_settings(enable_jira=True) self.testuser = User.objects.get(username="admin") - self.testuser.usercontactinfo.import_execution_mode = "sync" + self.testuser.usercontactinfo.deduplication_execution_mode = "sync" self.testuser.usercontactinfo.save() token = Token.objects.get(user=self.testuser) self.client = APIClient() diff --git a/unittests/test_finding_model.py b/unittests/test_finding_model.py index d3344234f40..2b3a626d9e6 100644 --- a/unittests/test_finding_model.py +++ b/unittests/test_finding_model.py @@ -507,7 +507,7 @@ class TestFindingSLAExpiration(DojoTestCase): def run(self, result=None): testuser = User.objects.get(username="admin") - testuser.usercontactinfo.import_execution_mode = "sync" + testuser.usercontactinfo.deduplication_execution_mode = "sync" testuser.save() # unit tests are running without any user, which will result in actions like dedupe happening in the celery process diff --git a/unittests/test_import_execution_mode.py b/unittests/test_import_execution_mode.py index a9232907df5..8c2bd4080ad 100644 --- a/unittests/test_import_execution_mode.py +++ b/unittests/test_import_execution_mode.py @@ -4,9 +4,9 @@ from dojo.importers.default_importer import DefaultImporter from dojo.models import ( - IMPORT_EXECUTION_MODE_ASYNC, - IMPORT_EXECUTION_MODE_ASYNC_WAIT, - IMPORT_EXECUTION_MODE_SYNC, + DEDUPLICATION_EXECUTION_MODE_ASYNC, + DEDUPLICATION_EXECUTION_MODE_ASYNC_WAIT, + DEDUPLICATION_EXECUTION_MODE_SYNC, Development_Environment, Dojo_User, Engagement, @@ -20,7 +20,7 @@ class ImportExecutionModeResolverTest(DojoTestCase): - """resolve_import_execution_mode: request override > profile > default.""" + """resolve_deduplication_execution_mode: request override > profile > default.""" fixtures = ["dojo_testdata.json"] @@ -31,38 +31,38 @@ def setUp(self): def _set_profile(self, *, mode=None): UserContactInfo.objects.update_or_create( user=self.user, - defaults={"import_execution_mode": mode}, + defaults={"deduplication_execution_mode": mode}, ) self.user.refresh_from_db() def test_default_is_async(self): - self.assertEqual(IMPORT_EXECUTION_MODE_ASYNC, Dojo_User.resolve_import_execution_mode(self.user)) + self.assertEqual(DEDUPLICATION_EXECUTION_MODE_ASYNC, Dojo_User.resolve_deduplication_execution_mode(self.user)) def test_request_override_wins_over_profile(self): - self._set_profile(mode=IMPORT_EXECUTION_MODE_SYNC) + self._set_profile(mode=DEDUPLICATION_EXECUTION_MODE_SYNC) self.assertEqual( - IMPORT_EXECUTION_MODE_ASYNC_WAIT, - Dojo_User.resolve_import_execution_mode(self.user, IMPORT_EXECUTION_MODE_ASYNC_WAIT), + DEDUPLICATION_EXECUTION_MODE_ASYNC_WAIT, + Dojo_User.resolve_deduplication_execution_mode(self.user, DEDUPLICATION_EXECUTION_MODE_ASYNC_WAIT), ) def test_profile_mode_used_when_no_override(self): - self._set_profile(mode=IMPORT_EXECUTION_MODE_ASYNC_WAIT) - self.assertEqual(IMPORT_EXECUTION_MODE_ASYNC_WAIT, Dojo_User.resolve_import_execution_mode(self.user)) + self._set_profile(mode=DEDUPLICATION_EXECUTION_MODE_ASYNC_WAIT) + self.assertEqual(DEDUPLICATION_EXECUTION_MODE_ASYNC_WAIT, Dojo_User.resolve_deduplication_execution_mode(self.user)) def test_empty_profile_falls_back_to_async(self): self._set_profile(mode=None) - self.assertEqual(IMPORT_EXECUTION_MODE_ASYNC, Dojo_User.resolve_import_execution_mode(self.user)) + self.assertEqual(DEDUPLICATION_EXECUTION_MODE_ASYNC, Dojo_User.resolve_deduplication_execution_mode(self.user)) def test_invalid_override_ignored(self): - self.assertEqual(IMPORT_EXECUTION_MODE_ASYNC, Dojo_User.resolve_import_execution_mode(self.user, "garbage")) + self.assertEqual(DEDUPLICATION_EXECUTION_MODE_ASYNC, Dojo_User.resolve_deduplication_execution_mode(self.user, "garbage")) def test_no_user(self): - self.assertEqual(IMPORT_EXECUTION_MODE_ASYNC, Dojo_User.resolve_import_execution_mode(None)) + self.assertEqual(DEDUPLICATION_EXECUTION_MODE_ASYNC, Dojo_User.resolve_deduplication_execution_mode(None)) def test_wants_block_execution_only_for_sync_mode(self): - self._set_profile(mode=IMPORT_EXECUTION_MODE_SYNC) + self._set_profile(mode=DEDUPLICATION_EXECUTION_MODE_SYNC) self.assertTrue(Dojo_User.wants_block_execution(self.user)) - self._set_profile(mode=IMPORT_EXECUTION_MODE_ASYNC_WAIT) + self._set_profile(mode=DEDUPLICATION_EXECUTION_MODE_ASYNC_WAIT) self.assertFalse(Dojo_User.wants_block_execution(self.user)) self._set_profile(mode=None) self.assertFalse(Dojo_User.wants_block_execution(self.user)) @@ -70,7 +70,7 @@ def test_wants_block_execution_only_for_sync_mode(self): class ImporterDispatchKwargsTest(DojoTestCase): - """import_execution_mode -> dojo_dispatch_task force flags.""" + """deduplication_execution_mode -> dojo_dispatch_task force flags.""" fixtures = ["dojo_testdata.json"] @@ -79,34 +79,34 @@ def _importer(self, mode, **extra): scan_type="ZAP Scan", engagement=Engagement.objects.first(), environment=Development_Environment.objects.first(), - import_execution_mode=mode, + deduplication_execution_mode=mode, **extra, ) def test_sync_mode_forces_sync(self): - self.assertEqual({"force_sync": True}, self._importer(IMPORT_EXECUTION_MODE_SYNC).post_processing_dispatch_kwargs()) + self.assertEqual({"force_sync": True}, self._importer(DEDUPLICATION_EXECUTION_MODE_SYNC).post_processing_dispatch_kwargs()) def test_async_wait_mode_forces_async(self): - self.assertEqual({"force_async": True}, self._importer(IMPORT_EXECUTION_MODE_ASYNC_WAIT).post_processing_dispatch_kwargs()) + self.assertEqual({"force_async": True}, self._importer(DEDUPLICATION_EXECUTION_MODE_ASYNC_WAIT).post_processing_dispatch_kwargs()) def test_async_mode_preserves_external_force_sync(self): - importer = self._importer(IMPORT_EXECUTION_MODE_ASYNC) + importer = self._importer(DEDUPLICATION_EXECUTION_MODE_ASYNC) self.assertEqual({"force_sync": False}, importer.post_processing_dispatch_kwargs()) self.assertEqual({"force_sync": True}, importer.post_processing_dispatch_kwargs(force_sync=True)) def test_invalid_mode_defaults_to_async(self): - self.assertEqual(IMPORT_EXECUTION_MODE_ASYNC, self._importer("nonsense").import_execution_mode) + self.assertEqual(DEDUPLICATION_EXECUTION_MODE_ASYNC, self._importer("nonsense").deduplication_execution_mode) def test_external_force_sync_promotes_to_sync_mode(self): - importer = self._importer(IMPORT_EXECUTION_MODE_ASYNC, force_sync=True) - self.assertEqual(IMPORT_EXECUTION_MODE_SYNC, importer.import_execution_mode) + importer = self._importer(DEDUPLICATION_EXECUTION_MODE_ASYNC, force_sync=True) + self.assertEqual(DEDUPLICATION_EXECUTION_MODE_SYNC, importer.deduplication_execution_mode) @override_settings(CELERY_TASK_ALWAYS_EAGER=True) class ImportExecutionModeAPITest(DojoAPITestCase): """ - End-to-end: the import endpoints accept and honor import_execution_mode. + End-to-end: the import endpoints accept and honor deduplication_execution_mode. CELERY_TASK_ALWAYS_EAGER runs dispatched tasks inline against the test DB, so 'async_wait' can actually join its deduplication batch (a real broker/worker @@ -124,12 +124,12 @@ def _payload(self, mode): "minimum_severity": "Low", "scan_type": "ZAP Scan", "engagement": 1, - "import_execution_mode": mode, + "deduplication_execution_mode": mode, } def test_import_async_wait_returns_statistics(self): with (get_unit_tests_path() / "scans/zap/0_zap_sample.xml").open(encoding="utf-8") as testfile: - payload = self._payload(IMPORT_EXECUTION_MODE_ASYNC_WAIT) + payload = self._payload(DEDUPLICATION_EXECUTION_MODE_ASYNC_WAIT) payload["file"] = testfile result = self.import_scan(payload, 201) self.assertIn("statistics", result) @@ -139,7 +139,7 @@ def test_import_async_wait_returns_statistics(self): def test_import_async_does_not_await_deduplication(self): with (get_unit_tests_path() / "scans/zap/0_zap_sample.xml").open(encoding="utf-8") as testfile: - payload = self._payload(IMPORT_EXECUTION_MODE_ASYNC) + payload = self._payload(DEDUPLICATION_EXECUTION_MODE_ASYNC) payload["file"] = testfile result = self.import_scan(payload, 201) self.assertFalse(result["deduplication_complete"]) diff --git a/unittests/test_import_reimport.py b/unittests/test_import_reimport.py index 26c70f2a52e..486d4e4eef2 100644 --- a/unittests/test_import_reimport.py +++ b/unittests/test_import_reimport.py @@ -2323,7 +2323,7 @@ def __init__(self, *args, **kwargs): def setUp(self): testuser = User.objects.get(username="admin") - testuser.usercontactinfo.import_execution_mode = "sync" + testuser.usercontactinfo.deduplication_execution_mode = "sync" testuser.usercontactinfo.save() self.login_as_admin() # self.url = reverse(self.viewname + '-list') @@ -3122,7 +3122,7 @@ def __init__(self, *args, **kwargs): def setUp(self): # still using the API to verify results testuser = User.objects.get(username="admin") - testuser.usercontactinfo.import_execution_mode = "sync" + testuser.usercontactinfo.deduplication_execution_mode = "sync" testuser.usercontactinfo.save() self.login_as_admin() # self.url = reverse(self.viewname + '-list') diff --git a/unittests/test_importers_deduplication.py b/unittests/test_importers_deduplication.py index 0bd2a71568c..f4c692c7101 100644 --- a/unittests/test_importers_deduplication.py +++ b/unittests/test_importers_deduplication.py @@ -37,7 +37,7 @@ def setUp(self): testuser.is_superuser = True testuser.is_staff = True testuser.save() - UserContactInfo.objects.create(user=testuser, import_execution_mode="sync") + UserContactInfo.objects.create(user=testuser, deduplication_execution_mode="sync") # Authenticate API client as admin for import endpoints self.login_as_admin() diff --git a/unittests/test_importers_performance.py b/unittests/test_importers_performance.py index 0736d4c4f55..8b10b2f8a02 100644 --- a/unittests/test_importers_performance.py +++ b/unittests/test_importers_performance.py @@ -67,7 +67,7 @@ def setUp(self): super().setUp() testuser, _ = User.objects.get_or_create(username="admin") - UserContactInfo.objects.update_or_create(user=testuser, defaults={"import_execution_mode": None}) + UserContactInfo.objects.update_or_create(user=testuser, defaults={"deduplication_execution_mode": None}) self.system_settings(enable_product_grade=False) self.system_settings(enable_github=False) @@ -363,7 +363,7 @@ def test_import_reimport_reimport_performance_pghistory_no_async(self): configure_pghistory_triggers() testuser = User.objects.get(username="admin") - testuser.usercontactinfo.import_execution_mode = "sync" + testuser.usercontactinfo.deduplication_execution_mode = "sync" testuser.usercontactinfo.save() self._import_reimport_performance( @@ -387,7 +387,7 @@ def test_import_reimport_reimport_performance_pghistory_no_async_with_product_gr configure_pghistory_triggers() testuser = User.objects.get(username="admin") - testuser.usercontactinfo.import_execution_mode = "sync" + testuser.usercontactinfo.deduplication_execution_mode = "sync" testuser.usercontactinfo.save() self.system_settings(enable_product_grade=True) @@ -541,7 +541,7 @@ def test_deduplication_performance_pghistory_no_async(self): self.system_settings(enable_deduplication=True) testuser = User.objects.get(username="admin") - testuser.usercontactinfo.import_execution_mode = "sync" + testuser.usercontactinfo.deduplication_execution_mode = "sync" testuser.usercontactinfo.save() self._deduplication_performance( @@ -653,7 +653,7 @@ def test_import_reimport_reimport_performance_pghistory_no_async(self): configure_pghistory_triggers() testuser = User.objects.get(username="admin") - testuser.usercontactinfo.import_execution_mode = "sync" + testuser.usercontactinfo.deduplication_execution_mode = "sync" testuser.usercontactinfo.save() self._import_reimport_performance( @@ -677,7 +677,7 @@ def test_import_reimport_reimport_performance_pghistory_no_async_with_product_gr configure_pghistory_triggers() testuser = User.objects.get(username="admin") - testuser.usercontactinfo.import_execution_mode = "sync" + testuser.usercontactinfo.deduplication_execution_mode = "sync" testuser.usercontactinfo.save() self.system_settings(enable_product_grade=True) @@ -805,7 +805,7 @@ def test_deduplication_performance_pghistory_no_async(self): self.system_settings(enable_deduplication=True) testuser = User.objects.get(username="admin") - testuser.usercontactinfo.import_execution_mode = "sync" + testuser.usercontactinfo.deduplication_execution_mode = "sync" testuser.usercontactinfo.save() self._deduplication_performance( diff --git a/unittests/test_jira_config_engagement.py b/unittests/test_jira_config_engagement.py index 7b8beb6e8ea..ca730fde0de 100644 --- a/unittests/test_jira_config_engagement.py +++ b/unittests/test_jira_config_engagement.py @@ -252,7 +252,7 @@ def setUp(self): self.system_settings(enable_jira=True) self.user = self.get_test_admin() self.client.force_login(self.user) - self.user.usercontactinfo.import_execution_mode = "sync" + self.user.usercontactinfo.deduplication_execution_mode = "sync" self.user.usercontactinfo.save() # product 3 has no jira project config, double check to make sure someone didn't molest the fixture # running this in __init__ throws database access denied error diff --git a/unittests/test_jira_config_engagement_epic.py b/unittests/test_jira_config_engagement_epic.py index 6f314b9748c..a77d005839a 100644 --- a/unittests/test_jira_config_engagement_epic.py +++ b/unittests/test_jira_config_engagement_epic.py @@ -39,7 +39,7 @@ def setUp(self): self.system_settings(enable_jira=True) self.user = self.get_test_admin() self.client.force_login(self.user) - self.user.usercontactinfo.import_execution_mode = "sync" + self.user.usercontactinfo.deduplication_execution_mode = "sync" self.user.usercontactinfo.save() # product 3 has no jira project config, double check to make sure someone didn't molest the fixture # running this in __init__ throws database access denied error diff --git a/unittests/test_jira_import_and_pushing_api.py b/unittests/test_jira_import_and_pushing_api.py index 7d472633363..89135bfada4 100644 --- a/unittests/test_jira_import_and_pushing_api.py +++ b/unittests/test_jira_import_and_pushing_api.py @@ -73,7 +73,7 @@ def setUp(self): self.system_settings(enable_jira=True) self.system_settings(enable_webhooks_notifications=True) self.testuser = User.objects.get(username="admin") - self.testuser.usercontactinfo.import_execution_mode = "sync" + self.testuser.usercontactinfo.deduplication_execution_mode = "sync" self.testuser.usercontactinfo.save() token = Token.objects.get(user=self.testuser) self.client = APIClient() diff --git a/unittests/test_notifications.py b/unittests/test_notifications.py index 06627273ce8..600098ccba8 100644 --- a/unittests/test_notifications.py +++ b/unittests/test_notifications.py @@ -641,9 +641,9 @@ class TestAsyncNotificationTaskBody(DojoTestCase): def run(self, result=None): # Same sync pattern used by TestNotificationWebhooks: run under an impersonated user - # with import_execution_mode="sync" so downstream dojo_dispatch_task calls execute inline. + # with deduplication_execution_mode="sync" so downstream dojo_dispatch_task calls execute inline. testuser = User.objects.get(username="admin") - testuser.usercontactinfo.import_execution_mode = "sync" + testuser.usercontactinfo.deduplication_execution_mode = "sync" testuser.save() with impersonate(testuser): super().run(result) @@ -677,8 +677,8 @@ def test_async_task_returns_silently_on_missing_instance(self, mock_process): @patch("dojo.notifications.helper.create_notification") def test_dispatch_respects_block_execution(self, mock_create): - """With import_execution_mode="sync" on the impersonated user, the post_save signal runs the task body inline.""" - # The run() wrapper impersonates admin with import_execution_mode="sync", so + """With deduplication_execution_mode="sync" on the impersonated user, the post_save signal runs the task body inline.""" + # The run() wrapper impersonates admin with deduplication_execution_mode="sync", so # dojo_dispatch_task takes the sync branch and the task body calls create_notification # (module-level helper inside async_create_notification) synchronously. prod = Product.objects.first() @@ -705,7 +705,7 @@ def run(self, result=None): if getattr(self, "__unittest_skip__", False): return super().run(result) testuser = User.objects.get(username="admin") - testuser.usercontactinfo.import_execution_mode = "sync" + testuser.usercontactinfo.deduplication_execution_mode = "sync" testuser.save() # unit tests are running without any user, which will result in actions like dedupe happening in the celery process diff --git a/unittests/test_prepare_duplicates_for_delete.py b/unittests/test_prepare_duplicates_for_delete.py index f879446f874..3bdecd44101 100644 --- a/unittests/test_prepare_duplicates_for_delete.py +++ b/unittests/test_prepare_duplicates_for_delete.py @@ -45,7 +45,7 @@ def setUp(self): is_staff=True, is_superuser=True, ) - UserContactInfo.objects.create(user=self.testuser, import_execution_mode="sync") + UserContactInfo.objects.create(user=self.testuser, deduplication_execution_mode="sync") self.system_settings(enable_deduplication=False) self.system_settings(enable_product_grade=False) diff --git a/unittests/test_product_grading.py b/unittests/test_product_grading.py index 4f929233b81..7d3080dcee5 100644 --- a/unittests/test_product_grading.py +++ b/unittests/test_product_grading.py @@ -12,7 +12,7 @@ class ProductGradeTest(DojoTestCase): def run(self, result=None): testuser = User.objects.get(username="admin") - testuser.usercontactinfo.import_execution_mode = "sync" + testuser.usercontactinfo.deduplication_execution_mode = "sync" testuser.save() # unit tests are running without any user, which will result in actions like dedupe happening in the celery process diff --git a/unittests/test_reimport_prefetch.py b/unittests/test_reimport_prefetch.py index fb6bff558c2..207ed54394d 100644 --- a/unittests/test_reimport_prefetch.py +++ b/unittests/test_reimport_prefetch.py @@ -72,7 +72,7 @@ class ReimportDuplicateFindingsTestBase(DojoTestCase): def setUp(self): super().setUp() testuser, _ = User.objects.get_or_create(username="admin") - UserContactInfo.objects.get_or_create(user=testuser, defaults={"import_execution_mode": "sync"}) + UserContactInfo.objects.get_or_create(user=testuser, defaults={"deduplication_execution_mode": "sync"}) self.system_settings(enable_deduplication=True) self.system_settings(enable_product_grade=False) diff --git a/unittests/test_tag_inheritance.py b/unittests/test_tag_inheritance.py index f28745b1921..25693313b51 100644 --- a/unittests/test_tag_inheritance.py +++ b/unittests/test_tag_inheritance.py @@ -674,7 +674,7 @@ class InheritedTagsImportTestAPI(DojoAPITestCase, InheritedTagsImportMixin): def setUp(self): super().setUp() testuser = User.objects.get(username="admin") - testuser.usercontactinfo.import_execution_mode = "sync" + testuser.usercontactinfo.deduplication_execution_mode = "sync" testuser.usercontactinfo.save() self.login_as_admin() settings.SECURE_SSL_REDIRECT = False @@ -692,7 +692,7 @@ class InheritedTagsImportTestUI(DojoAPITestCase, InheritedTagsImportMixin): def setUp(self): super().setUp() testuser = User.objects.get(username="admin") - testuser.usercontactinfo.import_execution_mode = "sync" + testuser.usercontactinfo.deduplication_execution_mode = "sync" testuser.usercontactinfo.save() self.login_as_admin() settings.SECURE_SSL_REDIRECT = False diff --git a/unittests/test_tags.py b/unittests/test_tags.py index 533d9d356b1..6d35b2a87e9 100644 --- a/unittests/test_tags.py +++ b/unittests/test_tags.py @@ -387,7 +387,7 @@ def setUp(self): super().setUp() settings.SECURE_SSL_REDIRECT = False testuser = User.objects.get(username="admin") - testuser.usercontactinfo.import_execution_mode = "sync" + testuser.usercontactinfo.deduplication_execution_mode = "sync" testuser.usercontactinfo.save() self.login_as_admin() TagImportMixin.setUp(self) @@ -404,7 +404,7 @@ def setUp(self): super().setUp() settings.SECURE_SSL_REDIRECT = False testuser = User.objects.get(username="admin") - testuser.usercontactinfo.import_execution_mode = "sync" + testuser.usercontactinfo.deduplication_execution_mode = "sync" testuser.usercontactinfo.save() self.login_as_admin() self.client_ui = Client() diff --git a/unittests/test_watson_async_search_index.py b/unittests/test_watson_async_search_index.py index fe54d61c0f8..916607b92a7 100644 --- a/unittests/test_watson_async_search_index.py +++ b/unittests/test_watson_async_search_index.py @@ -20,7 +20,7 @@ def setUp(self): super().setUp() self.testuser = User.objects.create(username="admin", is_staff=True, is_superuser=True) - UserContactInfo.objects.create(user=self.testuser, import_execution_mode="sync") + UserContactInfo.objects.create(user=self.testuser, deduplication_execution_mode="sync") self.system_settings(enable_product_grade=False) self.system_settings(enable_github=False) From 5933370e1aa992e6a25ca4872e4bb4c18b6fa047 Mon Sep 17 00:00:00 2001 From: Valentijn Scholten Date: Sun, 14 Jun 2026 15:07:52 +0200 Subject: [PATCH 05/12] refactor: rename DD_IMPORT_ASYNC_WAIT_TIMEOUT to DD_DEDUPLICATION_ASYNC_WAIT_TIMEOUT, default 60s --- dojo/importers/base_importer.py | 4 ++-- dojo/settings/settings.dist.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/dojo/importers/base_importer.py b/dojo/importers/base_importer.py index ab2f205b9f9..281e9f9a125 100644 --- a/dojo/importers/base_importer.py +++ b/dojo/importers/base_importer.py @@ -123,7 +123,7 @@ def wait_for_post_processing(self): returned statistics reflect the deduplicated state. Only relevant in the 'async_wait' execution mode; bounded by - settings.IMPORT_ASYNC_WAIT_TIMEOUT so a stuck/missing worker degrades + settings.DEDUPLICATION_ASYNC_WAIT_TIMEOUT so a stuck/missing worker degrades to the historical (respond-anyway) behavior instead of hanging. """ if self.deduplication_execution_mode == DEDUPLICATION_EXECUTION_MODE_SYNC: @@ -139,7 +139,7 @@ def wait_for_post_processing(self): # Nothing was dispatched (e.g. empty import) — dedup is trivially done. self.deduplication_complete = True return - timeout = getattr(settings, "IMPORT_ASYNC_WAIT_TIMEOUT", 120) + timeout = getattr(settings, "DEDUPLICATION_ASYNC_WAIT_TIMEOUT", 60) logger.debug("async_wait: waiting for %d post-processing task(s) (timeout=%ss)", len(results), timeout) success = True for result in results: diff --git a/dojo/settings/settings.dist.py b/dojo/settings/settings.dist.py index 2c680693477..c554c7d3d76 100644 --- a/dojo/settings/settings.dist.py +++ b/dojo/settings/settings.dist.py @@ -96,9 +96,9 @@ DD_CELERY_BROKER_PARAMS=(str, ""), DD_CELERY_BROKER_TRANSPORT_OPTIONS=(str, ""), DD_CELERY_TASK_IGNORE_RESULT=(bool, True), - # Max seconds the 'async_wait' import execution mode will wait for background - # deduplication/post-processing to finish before responding anyway. - DD_IMPORT_ASYNC_WAIT_TIMEOUT=(int, 120), + # Max seconds the 'async_wait' deduplication execution mode will wait for + # background deduplication/post-processing to finish before responding anyway. + DD_DEDUPLICATION_ASYNC_WAIT_TIMEOUT=(int, 60), DD_CELERY_RESULT_BACKEND=(str, "django-db"), DD_CELERY_RESULT_EXPIRES=(int, 86400), DD_CELERY_BEAT_SCHEDULE_FILENAME=(str, root("dojo.celery.beat.db")), @@ -858,7 +858,7 @@ def generate_url(scheme, double_slashes, user, password, host, port, path, param params=env("DD_CELERY_BROKER_PARAMS"), ) CELERY_TASK_IGNORE_RESULT = env("DD_CELERY_TASK_IGNORE_RESULT") -IMPORT_ASYNC_WAIT_TIMEOUT = env("DD_IMPORT_ASYNC_WAIT_TIMEOUT") +DEDUPLICATION_ASYNC_WAIT_TIMEOUT = env("DD_DEDUPLICATION_ASYNC_WAIT_TIMEOUT") CELERY_RESULT_BACKEND = env("DD_CELERY_RESULT_BACKEND") CELERY_TIMEZONE = TIME_ZONE CELERY_RESULT_EXPIRES = env("DD_CELERY_RESULT_EXPIRES") From 495db2353822daa6a5982fc5d4edb46d1b433fa3 Mon Sep 17 00:00:00 2001 From: Valentijn Scholten Date: Sun, 14 Jun 2026 15:34:36 +0200 Subject: [PATCH 06/12] refactor: keep block_execution as global switch, deduplication_execution_mode for import dedup block_execution gates every async task (notifications, jira, grading, deletes, reindex, ...) via we_want_async, so it must remain. Restore it as the global "run all async tasks in the foreground" flag and make deduplication_execution_mode an independent field that only controls how import/reimport deduplication post-processing is dispatched and awaited. - Restore the block_execution field; wants_block_execution() reads it again. - resolve_deduplication_execution_mode() falls back to block_execution -> sync. - Replace the destructive remove migration (0270) with a seed-only data migration that maps block_execution=True -> deduplication_execution_mode 'sync'; keep both. - Restore fixtures, the profile form field, and the user detail view (both fields). - Revert the test sweep: tests that need global foreground use block_execution=True again; the dedicated deduplication_execution_mode tests are updated for the split. --- dojo/celery_dispatch.py | 6 +-- ...9_usercontactinfo_import_execution_mode.py | 2 +- ..._remove_usercontactinfo_block_execution.py | 28 -------------- .../0270_seed_deduplication_execution_mode.py | 30 +++++++++++++++ ...ame_import_execution_mode_deduplication.py | 2 +- dojo/decorators.py | 6 +-- dojo/fixtures/defect_dojo_sample_data.json | 3 ++ .../defect_dojo_sample_data_locations.json | 3 ++ dojo/fixtures/dojo_testdata.json | 3 ++ dojo/fixtures/dojo_testdata_locations.json | 3 ++ dojo/forms.py | 2 +- dojo/middleware.py | 2 +- dojo/models.py | 37 +++++++++++-------- dojo/templates/dojo/view_user.html | 12 +++++- dojo/templates_classic/dojo/view_user.html | 12 +++++- unittests/test_async_delete.py | 8 ++-- unittests/test_cascade_delete.py | 2 +- unittests/test_celery_dispatch_force_async.py | 2 +- unittests/test_deduplication_logic.py | 2 +- unittests/test_duplication_loops.py | 2 +- .../test_false_positive_history_logic.py | 2 +- unittests/test_finding_helper.py | 2 +- unittests/test_finding_model.py | 2 +- unittests/test_import_execution_mode.py | 29 ++++++++++++--- unittests/test_import_reimport.py | 4 +- unittests/test_importers_deduplication.py | 2 +- unittests/test_importers_performance.py | 14 +++---- unittests/test_jira_config_engagement.py | 2 +- unittests/test_jira_config_engagement_epic.py | 2 +- unittests/test_jira_import_and_pushing_api.py | 2 +- unittests/test_notifications.py | 10 ++--- .../test_prepare_duplicates_for_delete.py | 2 +- unittests/test_product_grading.py | 2 +- unittests/test_reimport_prefetch.py | 2 +- unittests/test_tag_inheritance.py | 4 +- unittests/test_tags.py | 4 +- unittests/test_watson_async_search_index.py | 2 +- 37 files changed, 157 insertions(+), 97 deletions(-) delete mode 100644 dojo/db_migrations/0270_remove_usercontactinfo_block_execution.py create mode 100644 dojo/db_migrations/0270_seed_deduplication_execution_mode.py diff --git a/dojo/celery_dispatch.py b/dojo/celery_dispatch.py index 3171291d5ee..c4eee2aef08 100644 --- a/dojo/celery_dispatch.py +++ b/dojo/celery_dispatch.py @@ -62,10 +62,10 @@ def dojo_dispatch_task(task_or_sig: _SupportsSi | _SupportsApplyAsync | Signatur - Inject `async_user_id` if missing. - Capture and inject pghistory context if available. - Respect `force_sync=True` (foreground execution) and the user's - synchronous deduplication_execution_mode. + block_execution flag. - Respect `force_async=True` (background execution even when the caller - would otherwise run synchronously, e.g. user on the synchronous mode). - `force_async` wins over `force_sync` and the user's mode. + would otherwise run synchronously, e.g. user has block_execution). + `force_async` wins over `force_sync` and block_execution. - Support `countdown=` for async dispatch. Returns: diff --git a/dojo/db_migrations/0269_usercontactinfo_import_execution_mode.py b/dojo/db_migrations/0269_usercontactinfo_import_execution_mode.py index 220f0240b1a..d8219e5596c 100644 --- a/dojo/db_migrations/0269_usercontactinfo_import_execution_mode.py +++ b/dojo/db_migrations/0269_usercontactinfo_import_execution_mode.py @@ -11,6 +11,6 @@ class Migration(migrations.Migration): migrations.AddField( model_name='usercontactinfo', name='import_execution_mode', - field=models.CharField(blank=True, choices=[('async', 'Async (do not wait)'), ('async_wait', 'Async, wait for deduplication'), ('sync', 'Synchronous (block)')], help_text="Controls how import/reimport post-processing is executed. 'Async' returns immediately (default). 'Async, wait for deduplication' runs post-processing in the background but waits for deduplication to finish before responding, so notifications and statistics are accurate. 'Synchronous' runs everything inline (and blocks all async tasks in the foreground for this user, like the old 'block execution' flag). Can be overridden per request.", max_length=20, null=True), + field=models.CharField(blank=True, choices=[('async', 'Async (do not wait)'), ('async_wait', 'Async, wait for deduplication'), ('sync', 'Synchronous (block)')], help_text="Controls how import/reimport deduplication post-processing is executed. 'Async' dispatches it to the background and returns immediately (default). 'Async, wait for deduplication' dispatches to the background but waits for deduplication to finish before responding, so notifications and statistics reflect the deduplicated state. 'Synchronous' runs the import deduplication inline. Can be overridden per request. Independent of block_execution, which forces all async tasks (notifications, jira, ...) to the foreground.", max_length=20, null=True), ), ] diff --git a/dojo/db_migrations/0270_remove_usercontactinfo_block_execution.py b/dojo/db_migrations/0270_remove_usercontactinfo_block_execution.py deleted file mode 100644 index 56bba7e2886..00000000000 --- a/dojo/db_migrations/0270_remove_usercontactinfo_block_execution.py +++ /dev/null @@ -1,28 +0,0 @@ -from django.db import migrations - - -def block_execution_to_sync_mode(apps, schema_editor): - """Map the legacy block_execution=True flag to the synchronous import execution mode.""" - UserContactInfo = apps.get_model("dojo", "UserContactInfo") - UserContactInfo.objects.filter(block_execution=True).update(import_execution_mode="sync") - - -def sync_mode_to_block_execution(apps, schema_editor): - """Reverse: restore block_execution=True for users on the synchronous mode.""" - UserContactInfo = apps.get_model("dojo", "UserContactInfo") - UserContactInfo.objects.filter(import_execution_mode="sync").update(block_execution=True) - - -class Migration(migrations.Migration): - - dependencies = [ - ('dojo', '0269_usercontactinfo_import_execution_mode'), - ] - - operations = [ - migrations.RunPython(block_execution_to_sync_mode, sync_mode_to_block_execution), - migrations.RemoveField( - model_name='usercontactinfo', - name='block_execution', - ), - ] diff --git a/dojo/db_migrations/0270_seed_deduplication_execution_mode.py b/dojo/db_migrations/0270_seed_deduplication_execution_mode.py new file mode 100644 index 00000000000..862a2b0d929 --- /dev/null +++ b/dojo/db_migrations/0270_seed_deduplication_execution_mode.py @@ -0,0 +1,30 @@ +from django.db import migrations + + +def seed_deduplication_execution_mode(apps, schema_editor): + """ + Seed the new import deduplication execution mode from the legacy block_execution flag. + + block_execution remains the global "run all async tasks in the foreground" switch; + users who had it enabled get the synchronous deduplication mode so import behavior is + unchanged for them. + """ + UserContactInfo = apps.get_model("dojo", "UserContactInfo") + UserContactInfo.objects.filter(block_execution=True).update(import_execution_mode="sync") + + +def unseed_deduplication_execution_mode(apps, schema_editor): + """Reverse: clear the seeded synchronous mode.""" + UserContactInfo = apps.get_model("dojo", "UserContactInfo") + UserContactInfo.objects.filter(import_execution_mode="sync").update(import_execution_mode=None) + + +class Migration(migrations.Migration): + + dependencies = [ + ('dojo', '0269_usercontactinfo_import_execution_mode'), + ] + + operations = [ + migrations.RunPython(seed_deduplication_execution_mode, unseed_deduplication_execution_mode), + ] diff --git a/dojo/db_migrations/0271_rename_import_execution_mode_deduplication.py b/dojo/db_migrations/0271_rename_import_execution_mode_deduplication.py index 51812be16d3..1b136c9050b 100644 --- a/dojo/db_migrations/0271_rename_import_execution_mode_deduplication.py +++ b/dojo/db_migrations/0271_rename_import_execution_mode_deduplication.py @@ -4,7 +4,7 @@ class Migration(migrations.Migration): dependencies = [ - ('dojo', '0270_remove_usercontactinfo_block_execution'), + ('dojo', '0270_seed_deduplication_execution_mode'), ] operations = [ diff --git a/dojo/decorators.py b/dojo/decorators.py index d938c6e2661..cbe33de732d 100644 --- a/dojo/decorators.py +++ b/dojo/decorators.py @@ -76,15 +76,15 @@ def we_want_async(*args, func=None, **kwargs): return True if Dojo_User.wants_block_execution(user): - logger.debug("dojo_async_task %s: running task in the foreground as deduplication_execution_mode is 'sync' for %s", func, user) + logger.debug("dojo_async_task %s: running task in the foreground as block_execution is set to True for %s", func, user) return False - logger.debug("dojo_async_task %s: running task in the background as deduplication_execution_mode is not 'sync' for %s", func, user) + logger.debug("dojo_async_task %s: running task in the background as user has not set block_execution to True for %s", func, user) return True # Defect Dojo performs all tasks asynchrnonously using celery -# *unless* the user initiating the task has set deduplication_execution_mode to 'sync' in their usercontactinfo profile +# *unless* the user initiating the task has set block_execution to True in their usercontactinfo profile def dojo_async_task(func=None, *, signature=False): def decorator(func): @wraps(func) diff --git a/dojo/fixtures/defect_dojo_sample_data.json b/dojo/fixtures/defect_dojo_sample_data.json index bd28299fb27..7923a91dd38 100644 --- a/dojo/fixtures/defect_dojo_sample_data.json +++ b/dojo/fixtures/defect_dojo_sample_data.json @@ -687,6 +687,7 @@ }, { "fields": { + "block_execution": false, "cell_number": "", "force_password_reset": false, "github_username": null, @@ -704,6 +705,7 @@ }, { "fields": { + "block_execution": false, "cell_number": "", "force_password_reset": false, "github_username": null, @@ -721,6 +723,7 @@ }, { "fields": { + "block_execution": false, "cell_number": "", "force_password_reset": false, "github_username": null, diff --git a/dojo/fixtures/defect_dojo_sample_data_locations.json b/dojo/fixtures/defect_dojo_sample_data_locations.json index 3f7f04208f9..0b08e888e03 100644 --- a/dojo/fixtures/defect_dojo_sample_data_locations.json +++ b/dojo/fixtures/defect_dojo_sample_data_locations.json @@ -687,6 +687,7 @@ }, { "fields": { + "block_execution": false, "cell_number": "", "force_password_reset": false, "github_username": null, @@ -706,6 +707,7 @@ }, { "fields": { + "block_execution": false, "cell_number": "", "force_password_reset": false, "github_username": null, @@ -725,6 +727,7 @@ }, { "fields": { + "block_execution": false, "cell_number": "", "force_password_reset": false, "github_username": null, diff --git a/dojo/fixtures/dojo_testdata.json b/dojo/fixtures/dojo_testdata.json index b1471612a4c..50707a2d2bf 100644 --- a/dojo/fixtures/dojo_testdata.json +++ b/dojo/fixtures/dojo_testdata.json @@ -277,6 +277,7 @@ "title": null, "twitter_username": "#admin", "user": 1, + "block_execution": false, "github_username": null } }, @@ -290,6 +291,7 @@ "title": null, "twitter_username": null, "user": 2, + "block_execution": false, "github_username": null } }, @@ -303,6 +305,7 @@ "title": null, "twitter_username": null, "user": 3, + "block_execution": false, "github_username": null } }, diff --git a/dojo/fixtures/dojo_testdata_locations.json b/dojo/fixtures/dojo_testdata_locations.json index 99e9e875953..3d4eb06ff9b 100644 --- a/dojo/fixtures/dojo_testdata_locations.json +++ b/dojo/fixtures/dojo_testdata_locations.json @@ -277,6 +277,7 @@ "title": null, "twitter_username": "#admin", "user": 1, + "block_execution": false, "github_username": null } }, @@ -290,6 +291,7 @@ "title": null, "twitter_username": null, "user": 2, + "block_execution": false, "github_username": null } }, @@ -303,6 +305,7 @@ "title": null, "twitter_username": null, "user": 3, + "block_execution": false, "github_username": null } }, diff --git a/dojo/forms.py b/dojo/forms.py index 07ef3c16c68..599746763df 100644 --- a/dojo/forms.py +++ b/dojo/forms.py @@ -2393,7 +2393,7 @@ class Meta: # Swap order: password_last_reset before token_last_reset field_order = [ "title", "phone_number", "cell_number", "twitter_username", "github_username", - "slack_username", "ui_use_tailwind", "deduplication_execution_mode", "force_password_reset", "reset_api_token", + "slack_username", "ui_use_tailwind", "block_execution", "deduplication_execution_mode", "force_password_reset", "reset_api_token", "password_last_reset", "token_last_reset", ] diff --git a/dojo/middleware.py b/dojo/middleware.py index d9cdd626b1e..127ab462a09 100644 --- a/dojo/middleware.py +++ b/dojo/middleware.py @@ -281,7 +281,7 @@ def _drain_search_context_to_async(objects, source): for model_name, pk_list in model_groups.items(): batches = [pk_list[i:i + batch_size] for i in range(0, len(pk_list), batch_size)] # force_async=True keeps indexing off the request path even for users - # on the synchronous deduplication_execution_mode — index updates are slow and + # with block_execution=True — index updates are slow and # never need to be synchronous from the user's perspective. for i, batch in enumerate(batches, 1): logger.debug(f"{source}: Triggering batch {i}/{len(batches)} for {model_name}: {len(batch)} instances") diff --git a/dojo/models.py b/dojo/models.py index f3c9e4f7f9b..f7881e6d358 100644 --- a/dojo/models.py +++ b/dojo/models.py @@ -235,24 +235,30 @@ def __str__(self): @staticmethod def wants_block_execution(user): - # this returns False if there is no user, i.e. in celery processes, unittests, etc. - # The synchronous import execution mode is the successor of the old block_execution - # flag and governs whether async tasks run in the foreground for this user. - return hasattr(user, "usercontactinfo") and user.usercontactinfo.deduplication_execution_mode == DEDUPLICATION_EXECUTION_MODE_SYNC + # this return False if there is no user, i.e. in celery processes, unittests, etc. + # block_execution is the global "run all async tasks in the foreground" switch and + # governs every dojo_dispatch_task/dojo_async_task call (notifications, jira, grading, + # deduplication, ...). It is distinct from deduplication_execution_mode, which only + # controls how import/reimport deduplication post-processing is dispatched/awaited. + return hasattr(user, "usercontactinfo") and user.usercontactinfo.block_execution @staticmethod def resolve_deduplication_execution_mode(user, override=None): """ - Resolve the effective import post-processing execution mode. + Resolve the effective import/reimport deduplication execution mode. - Priority: explicit request override > user profile setting > default async. + Priority: explicit request override > user profile deduplication_execution_mode > + legacy block_execution (which forces everything sync) > default async. Returns one of DEDUPLICATION_EXECUTION_MODE_ASYNC / _ASYNC_WAIT / _SYNC. """ if override in DEDUPLICATION_EXECUTION_MODES: return override info = getattr(user, "usercontactinfo", None) - if info is not None and info.deduplication_execution_mode in DEDUPLICATION_EXECUTION_MODES: - return info.deduplication_execution_mode + if info is not None: + if info.deduplication_execution_mode in DEDUPLICATION_EXECUTION_MODES: + return info.deduplication_execution_mode + if info.block_execution: + return DEDUPLICATION_EXECUTION_MODE_SYNC return DEDUPLICATION_EXECUTION_MODE_ASYNC @staticmethod @@ -294,19 +300,20 @@ class UserContactInfo(models.Model): github_username = models.CharField(blank=True, null=True, max_length=150) slack_username = models.CharField(blank=True, null=True, max_length=150, help_text=_("Email address associated with your slack account"), verbose_name=_("Slack Email Address")) slack_user_id = models.CharField(blank=True, null=True, max_length=25) + block_execution = models.BooleanField(default=False, help_text=_("Instead of async deduping a finding the findings will be deduped synchronously and will 'block' the user until completion.")) deduplication_execution_mode = models.CharField( max_length=20, choices=DEDUPLICATION_EXECUTION_MODE_CHOICES, null=True, blank=True, help_text=_( - "Controls how import/reimport post-processing is executed. " - "'Async' returns immediately (default). 'Async, wait for deduplication' " - "runs post-processing in the background but waits for deduplication to " - "finish before responding, so notifications and statistics are accurate. " - "'Synchronous' runs everything inline (and blocks all async tasks in the " - "foreground for this user, like the old 'block execution' flag). Can be " - "overridden per request.", + "Controls how import/reimport deduplication post-processing is executed. " + "'Async' dispatches it to the background and returns immediately (default). " + "'Async, wait for deduplication' dispatches to the background but waits for " + "deduplication to finish before responding, so notifications and statistics " + "reflect the deduplicated state. 'Synchronous' runs the import deduplication " + "inline. Can be overridden per request. Independent of block_execution, which " + "forces all async tasks (notifications, jira, ...) to the foreground.", ), ) force_password_reset = models.BooleanField(default=False, help_text=_("Forces this user to reset their password on next login.")) diff --git a/dojo/templates/dojo/view_user.html b/dojo/templates/dojo/view_user.html index acc31927a48..75c30fd6edf 100644 --- a/dojo/templates/dojo/view_user.html +++ b/dojo/templates/dojo/view_user.html @@ -279,7 +279,17 @@

- {% trans "Import execution mode" %} + {% trans "Block execution" %} + + {% if user.usercontactinfo.block_execution %} + + {% else %} + + {% endif %} + + + + {% trans "Deduplication execution mode" %} {{ user.usercontactinfo.get_deduplication_execution_mode_display|default:_("Async (do not wait)") }} diff --git a/dojo/templates_classic/dojo/view_user.html b/dojo/templates_classic/dojo/view_user.html index 238694e1c5f..ed9e90ee8cc 100644 --- a/dojo/templates_classic/dojo/view_user.html +++ b/dojo/templates_classic/dojo/view_user.html @@ -279,7 +279,17 @@

- {% trans "Import execution mode" %} + {% trans "Block execution" %} + + {% if user.usercontactinfo.block_execution %} + + {% else %} + + {% endif %} + + + + {% trans "Deduplication execution mode" %} {{ user.usercontactinfo.get_deduplication_execution_mode_display|default:_("Async (do not wait)") }} diff --git a/unittests/test_async_delete.py b/unittests/test_async_delete.py index b7c2b5413a4..7aa8f0769a1 100644 --- a/unittests/test_async_delete.py +++ b/unittests/test_async_delete.py @@ -27,21 +27,21 @@ class TestAsyncDelete(DojoTestCase): """ Test async_delete functionality with dojo_dispatch_task kwargs injection. - These tests use deduplication_execution_mode="sync" and crum.impersonate to run tasks synchronously, + These tests use block_execution=True and crum.impersonate to run tasks synchronously, which allows errors to surface immediately rather than being lost in background workers. """ def setUp(self): - """Set up test user with deduplication_execution_mode="sync" and disable unneeded features.""" + """Set up test user with block_execution=True and disable unneeded features.""" super().setUp() - # Create test user with deduplication_execution_mode="sync" to run tasks synchronously + # Create test user with block_execution=True to run tasks synchronously self.testuser = User.objects.create( username="test_async_delete_user", is_staff=True, is_superuser=True, ) - UserContactInfo.objects.create(user=self.testuser, deduplication_execution_mode="sync") + UserContactInfo.objects.create(user=self.testuser, block_execution=True) # Log in as the test user (for API client) self.client.force_login(self.testuser) diff --git a/unittests/test_cascade_delete.py b/unittests/test_cascade_delete.py index ab86c942b43..f24667e44f8 100644 --- a/unittests/test_cascade_delete.py +++ b/unittests/test_cascade_delete.py @@ -37,7 +37,7 @@ def setUp(self): is_staff=True, is_superuser=True, ) - UserContactInfo.objects.create(user=self.testuser, deduplication_execution_mode="sync") + UserContactInfo.objects.create(user=self.testuser, block_execution=True) self.system_settings(enable_deduplication=False) self.system_settings(enable_product_grade=False) diff --git a/unittests/test_celery_dispatch_force_async.py b/unittests/test_celery_dispatch_force_async.py index d0479a02d7b..71170e1bb09 100644 --- a/unittests/test_celery_dispatch_force_async.py +++ b/unittests/test_celery_dispatch_force_async.py @@ -3,7 +3,7 @@ `force_async=True` is for callers (e.g. the watson async indexer middleware) that should always run their celery task in the background even when the -current user has `deduplication_execution_mode="sync"` or the caller passes `force_sync=True`. +current user has `block_execution=True` or the caller passes `force_sync=True`. """ from unittest.mock import patch diff --git a/unittests/test_deduplication_logic.py b/unittests/test_deduplication_logic.py index 1c8ed8773ab..fd6b5d2847c 100644 --- a/unittests/test_deduplication_logic.py +++ b/unittests/test_deduplication_logic.py @@ -162,7 +162,7 @@ class TestDuplicationLogic(DojoTestCase): def run(self, result=None): testuser = User.objects.get(username="admin") - testuser.usercontactinfo.deduplication_execution_mode = "sync" + testuser.usercontactinfo.block_execution = True testuser.save() # unit tests are running without any user, which will result in actions like dedupe happening in the celery process diff --git a/unittests/test_duplication_loops.py b/unittests/test_duplication_loops.py index 4c4616dafc4..a3a70cf5c2b 100644 --- a/unittests/test_duplication_loops.py +++ b/unittests/test_duplication_loops.py @@ -19,7 +19,7 @@ class TestDuplicationLoops(DojoTestCase): def run(self, result=None): testuser = User.objects.get(username="admin") - testuser.usercontactinfo.deduplication_execution_mode = "sync" + testuser.usercontactinfo.block_execution = True testuser.save() # unit tests are running without any user, which will result in actions like dedupe happening in the celery process diff --git a/unittests/test_false_positive_history_logic.py b/unittests/test_false_positive_history_logic.py index c81563f9eb0..8748239bedd 100644 --- a/unittests/test_false_positive_history_logic.py +++ b/unittests/test_false_positive_history_logic.py @@ -131,7 +131,7 @@ class TestFalsePositiveHistoryLogic(DojoTestCase): def run(self, result=None): testuser = User.objects.get(username="admin") - testuser.usercontactinfo.deduplication_execution_mode = "sync" + testuser.usercontactinfo.block_execution = True testuser.save() # Unit tests are running without any user, which will result in actions like dedupe happening in the celery process diff --git a/unittests/test_finding_helper.py b/unittests/test_finding_helper.py index 0f78154cb9a..fa6fd2d9ea5 100644 --- a/unittests/test_finding_helper.py +++ b/unittests/test_finding_helper.py @@ -253,7 +253,7 @@ def setUp(self): super().setUp() self.system_settings(enable_jira=True) self.testuser = User.objects.get(username="admin") - self.testuser.usercontactinfo.deduplication_execution_mode = "sync" + self.testuser.usercontactinfo.block_execution = True self.testuser.usercontactinfo.save() token = Token.objects.get(user=self.testuser) self.client = APIClient() diff --git a/unittests/test_finding_model.py b/unittests/test_finding_model.py index 2b3a626d9e6..78ee40693f2 100644 --- a/unittests/test_finding_model.py +++ b/unittests/test_finding_model.py @@ -507,7 +507,7 @@ class TestFindingSLAExpiration(DojoTestCase): def run(self, result=None): testuser = User.objects.get(username="admin") - testuser.usercontactinfo.deduplication_execution_mode = "sync" + testuser.usercontactinfo.block_execution = True testuser.save() # unit tests are running without any user, which will result in actions like dedupe happening in the celery process diff --git a/unittests/test_import_execution_mode.py b/unittests/test_import_execution_mode.py index 8c2bd4080ad..f7e8b9660c9 100644 --- a/unittests/test_import_execution_mode.py +++ b/unittests/test_import_execution_mode.py @@ -59,12 +59,31 @@ def test_invalid_override_ignored(self): def test_no_user(self): self.assertEqual(DEDUPLICATION_EXECUTION_MODE_ASYNC, Dojo_User.resolve_deduplication_execution_mode(None)) - def test_wants_block_execution_only_for_sync_mode(self): - self._set_profile(mode=DEDUPLICATION_EXECUTION_MODE_SYNC) + def test_block_execution_falls_back_to_sync(self): + # legacy global block_execution flag implies synchronous deduplication + UserContactInfo.objects.update_or_create(user=self.user, defaults={"block_execution": True}) + self.user.refresh_from_db() + self.assertEqual(DEDUPLICATION_EXECUTION_MODE_SYNC, Dojo_User.resolve_deduplication_execution_mode(self.user)) + + def test_mode_takes_precedence_over_block_execution(self): + UserContactInfo.objects.update_or_create( + user=self.user, + defaults={"block_execution": True, "deduplication_execution_mode": DEDUPLICATION_EXECUTION_MODE_ASYNC_WAIT}, + ) + self.user.refresh_from_db() + self.assertEqual(DEDUPLICATION_EXECUTION_MODE_ASYNC_WAIT, Dojo_User.resolve_deduplication_execution_mode(self.user)) + + def test_wants_block_execution_reads_block_execution_not_mode(self): + # wants_block_execution is the global switch and is independent of the dedup mode + UserContactInfo.objects.update_or_create(user=self.user, defaults={"block_execution": True}) + self.user.refresh_from_db() self.assertTrue(Dojo_User.wants_block_execution(self.user)) - self._set_profile(mode=DEDUPLICATION_EXECUTION_MODE_ASYNC_WAIT) - self.assertFalse(Dojo_User.wants_block_execution(self.user)) - self._set_profile(mode=None) + UserContactInfo.objects.update_or_create( + user=self.user, + defaults={"block_execution": False, "deduplication_execution_mode": DEDUPLICATION_EXECUTION_MODE_SYNC}, + ) + self.user.refresh_from_db() + # a 'sync' dedup mode alone does NOT force global foreground execution self.assertFalse(Dojo_User.wants_block_execution(self.user)) diff --git a/unittests/test_import_reimport.py b/unittests/test_import_reimport.py index 486d4e4eef2..417fe0ea9ea 100644 --- a/unittests/test_import_reimport.py +++ b/unittests/test_import_reimport.py @@ -2323,7 +2323,7 @@ def __init__(self, *args, **kwargs): def setUp(self): testuser = User.objects.get(username="admin") - testuser.usercontactinfo.deduplication_execution_mode = "sync" + testuser.usercontactinfo.block_execution = True testuser.usercontactinfo.save() self.login_as_admin() # self.url = reverse(self.viewname + '-list') @@ -3122,7 +3122,7 @@ def __init__(self, *args, **kwargs): def setUp(self): # still using the API to verify results testuser = User.objects.get(username="admin") - testuser.usercontactinfo.deduplication_execution_mode = "sync" + testuser.usercontactinfo.block_execution = True testuser.usercontactinfo.save() self.login_as_admin() # self.url = reverse(self.viewname + '-list') diff --git a/unittests/test_importers_deduplication.py b/unittests/test_importers_deduplication.py index f4c692c7101..7c1359dcc36 100644 --- a/unittests/test_importers_deduplication.py +++ b/unittests/test_importers_deduplication.py @@ -37,7 +37,7 @@ def setUp(self): testuser.is_superuser = True testuser.is_staff = True testuser.save() - UserContactInfo.objects.create(user=testuser, deduplication_execution_mode="sync") + UserContactInfo.objects.create(user=testuser, block_execution=True) # Authenticate API client as admin for import endpoints self.login_as_admin() diff --git a/unittests/test_importers_performance.py b/unittests/test_importers_performance.py index 8b10b2f8a02..ce9133e4a6b 100644 --- a/unittests/test_importers_performance.py +++ b/unittests/test_importers_performance.py @@ -67,7 +67,7 @@ def setUp(self): super().setUp() testuser, _ = User.objects.get_or_create(username="admin") - UserContactInfo.objects.update_or_create(user=testuser, defaults={"deduplication_execution_mode": None}) + UserContactInfo.objects.update_or_create(user=testuser, defaults={"block_execution": False}) self.system_settings(enable_product_grade=False) self.system_settings(enable_github=False) @@ -363,7 +363,7 @@ def test_import_reimport_reimport_performance_pghistory_no_async(self): configure_pghistory_triggers() testuser = User.objects.get(username="admin") - testuser.usercontactinfo.deduplication_execution_mode = "sync" + testuser.usercontactinfo.block_execution = True testuser.usercontactinfo.save() self._import_reimport_performance( @@ -387,7 +387,7 @@ def test_import_reimport_reimport_performance_pghistory_no_async_with_product_gr configure_pghistory_triggers() testuser = User.objects.get(username="admin") - testuser.usercontactinfo.deduplication_execution_mode = "sync" + testuser.usercontactinfo.block_execution = True testuser.usercontactinfo.save() self.system_settings(enable_product_grade=True) @@ -541,7 +541,7 @@ def test_deduplication_performance_pghistory_no_async(self): self.system_settings(enable_deduplication=True) testuser = User.objects.get(username="admin") - testuser.usercontactinfo.deduplication_execution_mode = "sync" + testuser.usercontactinfo.block_execution = True testuser.usercontactinfo.save() self._deduplication_performance( @@ -653,7 +653,7 @@ def test_import_reimport_reimport_performance_pghistory_no_async(self): configure_pghistory_triggers() testuser = User.objects.get(username="admin") - testuser.usercontactinfo.deduplication_execution_mode = "sync" + testuser.usercontactinfo.block_execution = True testuser.usercontactinfo.save() self._import_reimport_performance( @@ -677,7 +677,7 @@ def test_import_reimport_reimport_performance_pghistory_no_async_with_product_gr configure_pghistory_triggers() testuser = User.objects.get(username="admin") - testuser.usercontactinfo.deduplication_execution_mode = "sync" + testuser.usercontactinfo.block_execution = True testuser.usercontactinfo.save() self.system_settings(enable_product_grade=True) @@ -805,7 +805,7 @@ def test_deduplication_performance_pghistory_no_async(self): self.system_settings(enable_deduplication=True) testuser = User.objects.get(username="admin") - testuser.usercontactinfo.deduplication_execution_mode = "sync" + testuser.usercontactinfo.block_execution = True testuser.usercontactinfo.save() self._deduplication_performance( diff --git a/unittests/test_jira_config_engagement.py b/unittests/test_jira_config_engagement.py index ca730fde0de..aaabfdddc5e 100644 --- a/unittests/test_jira_config_engagement.py +++ b/unittests/test_jira_config_engagement.py @@ -252,7 +252,7 @@ def setUp(self): self.system_settings(enable_jira=True) self.user = self.get_test_admin() self.client.force_login(self.user) - self.user.usercontactinfo.deduplication_execution_mode = "sync" + self.user.usercontactinfo.block_execution = True self.user.usercontactinfo.save() # product 3 has no jira project config, double check to make sure someone didn't molest the fixture # running this in __init__ throws database access denied error diff --git a/unittests/test_jira_config_engagement_epic.py b/unittests/test_jira_config_engagement_epic.py index a77d005839a..614a10fbe57 100644 --- a/unittests/test_jira_config_engagement_epic.py +++ b/unittests/test_jira_config_engagement_epic.py @@ -39,7 +39,7 @@ def setUp(self): self.system_settings(enable_jira=True) self.user = self.get_test_admin() self.client.force_login(self.user) - self.user.usercontactinfo.deduplication_execution_mode = "sync" + self.user.usercontactinfo.block_execution = True self.user.usercontactinfo.save() # product 3 has no jira project config, double check to make sure someone didn't molest the fixture # running this in __init__ throws database access denied error diff --git a/unittests/test_jira_import_and_pushing_api.py b/unittests/test_jira_import_and_pushing_api.py index 89135bfada4..eb3f0692dbc 100644 --- a/unittests/test_jira_import_and_pushing_api.py +++ b/unittests/test_jira_import_and_pushing_api.py @@ -73,7 +73,7 @@ def setUp(self): self.system_settings(enable_jira=True) self.system_settings(enable_webhooks_notifications=True) self.testuser = User.objects.get(username="admin") - self.testuser.usercontactinfo.deduplication_execution_mode = "sync" + self.testuser.usercontactinfo.block_execution = True self.testuser.usercontactinfo.save() token = Token.objects.get(user=self.testuser) self.client = APIClient() diff --git a/unittests/test_notifications.py b/unittests/test_notifications.py index 600098ccba8..28921fdf961 100644 --- a/unittests/test_notifications.py +++ b/unittests/test_notifications.py @@ -641,9 +641,9 @@ class TestAsyncNotificationTaskBody(DojoTestCase): def run(self, result=None): # Same sync pattern used by TestNotificationWebhooks: run under an impersonated user - # with deduplication_execution_mode="sync" so downstream dojo_dispatch_task calls execute inline. + # with block_execution=True so downstream dojo_dispatch_task calls execute inline. testuser = User.objects.get(username="admin") - testuser.usercontactinfo.deduplication_execution_mode = "sync" + testuser.usercontactinfo.block_execution = True testuser.save() with impersonate(testuser): super().run(result) @@ -677,8 +677,8 @@ def test_async_task_returns_silently_on_missing_instance(self, mock_process): @patch("dojo.notifications.helper.create_notification") def test_dispatch_respects_block_execution(self, mock_create): - """With deduplication_execution_mode="sync" on the impersonated user, the post_save signal runs the task body inline.""" - # The run() wrapper impersonates admin with deduplication_execution_mode="sync", so + """With block_execution=True on the impersonated user, the post_save signal runs the task body inline.""" + # The run() wrapper impersonates admin with block_execution=True, so # dojo_dispatch_task takes the sync branch and the task body calls create_notification # (module-level helper inside async_create_notification) synchronously. prod = Product.objects.first() @@ -705,7 +705,7 @@ def run(self, result=None): if getattr(self, "__unittest_skip__", False): return super().run(result) testuser = User.objects.get(username="admin") - testuser.usercontactinfo.deduplication_execution_mode = "sync" + testuser.usercontactinfo.block_execution = True testuser.save() # unit tests are running without any user, which will result in actions like dedupe happening in the celery process diff --git a/unittests/test_prepare_duplicates_for_delete.py b/unittests/test_prepare_duplicates_for_delete.py index 3bdecd44101..78d612dfdec 100644 --- a/unittests/test_prepare_duplicates_for_delete.py +++ b/unittests/test_prepare_duplicates_for_delete.py @@ -45,7 +45,7 @@ def setUp(self): is_staff=True, is_superuser=True, ) - UserContactInfo.objects.create(user=self.testuser, deduplication_execution_mode="sync") + UserContactInfo.objects.create(user=self.testuser, block_execution=True) self.system_settings(enable_deduplication=False) self.system_settings(enable_product_grade=False) diff --git a/unittests/test_product_grading.py b/unittests/test_product_grading.py index 7d3080dcee5..5f4680c3315 100644 --- a/unittests/test_product_grading.py +++ b/unittests/test_product_grading.py @@ -12,7 +12,7 @@ class ProductGradeTest(DojoTestCase): def run(self, result=None): testuser = User.objects.get(username="admin") - testuser.usercontactinfo.deduplication_execution_mode = "sync" + testuser.usercontactinfo.block_execution = True testuser.save() # unit tests are running without any user, which will result in actions like dedupe happening in the celery process diff --git a/unittests/test_reimport_prefetch.py b/unittests/test_reimport_prefetch.py index 207ed54394d..cc5b2f40e1e 100644 --- a/unittests/test_reimport_prefetch.py +++ b/unittests/test_reimport_prefetch.py @@ -72,7 +72,7 @@ class ReimportDuplicateFindingsTestBase(DojoTestCase): def setUp(self): super().setUp() testuser, _ = User.objects.get_or_create(username="admin") - UserContactInfo.objects.get_or_create(user=testuser, defaults={"deduplication_execution_mode": "sync"}) + UserContactInfo.objects.get_or_create(user=testuser, defaults={"block_execution": True}) self.system_settings(enable_deduplication=True) self.system_settings(enable_product_grade=False) diff --git a/unittests/test_tag_inheritance.py b/unittests/test_tag_inheritance.py index 25693313b51..964094fa8ed 100644 --- a/unittests/test_tag_inheritance.py +++ b/unittests/test_tag_inheritance.py @@ -674,7 +674,7 @@ class InheritedTagsImportTestAPI(DojoAPITestCase, InheritedTagsImportMixin): def setUp(self): super().setUp() testuser = User.objects.get(username="admin") - testuser.usercontactinfo.deduplication_execution_mode = "sync" + testuser.usercontactinfo.block_execution = True testuser.usercontactinfo.save() self.login_as_admin() settings.SECURE_SSL_REDIRECT = False @@ -692,7 +692,7 @@ class InheritedTagsImportTestUI(DojoAPITestCase, InheritedTagsImportMixin): def setUp(self): super().setUp() testuser = User.objects.get(username="admin") - testuser.usercontactinfo.deduplication_execution_mode = "sync" + testuser.usercontactinfo.block_execution = True testuser.usercontactinfo.save() self.login_as_admin() settings.SECURE_SSL_REDIRECT = False diff --git a/unittests/test_tags.py b/unittests/test_tags.py index 6d35b2a87e9..ea460f247bf 100644 --- a/unittests/test_tags.py +++ b/unittests/test_tags.py @@ -387,7 +387,7 @@ def setUp(self): super().setUp() settings.SECURE_SSL_REDIRECT = False testuser = User.objects.get(username="admin") - testuser.usercontactinfo.deduplication_execution_mode = "sync" + testuser.usercontactinfo.block_execution = True testuser.usercontactinfo.save() self.login_as_admin() TagImportMixin.setUp(self) @@ -404,7 +404,7 @@ def setUp(self): super().setUp() settings.SECURE_SSL_REDIRECT = False testuser = User.objects.get(username="admin") - testuser.usercontactinfo.deduplication_execution_mode = "sync" + testuser.usercontactinfo.block_execution = True testuser.usercontactinfo.save() self.login_as_admin() self.client_ui = Client() diff --git a/unittests/test_watson_async_search_index.py b/unittests/test_watson_async_search_index.py index 916607b92a7..8edba606ac7 100644 --- a/unittests/test_watson_async_search_index.py +++ b/unittests/test_watson_async_search_index.py @@ -20,7 +20,7 @@ def setUp(self): super().setUp() self.testuser = User.objects.create(username="admin", is_staff=True, is_superuser=True) - UserContactInfo.objects.create(user=self.testuser, deduplication_execution_mode="sync") + UserContactInfo.objects.create(user=self.testuser, block_execution=True) self.system_settings(enable_product_grade=False) self.system_settings(enable_github=False) From ed0307912aae2875c8a7458d10720f706964e59a Mon Sep 17 00:00:00 2001 From: Valentijn Scholten Date: Sun, 14 Jun 2026 15:41:41 +0200 Subject: [PATCH 07/12] docs: 2.60 upgrade notes for deduplication_execution_mode --- docs/content/en/open_source/upgrading/2.60.md | 43 ++++++++++++++++++- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/docs/content/en/open_source/upgrading/2.60.md b/docs/content/en/open_source/upgrading/2.60.md index e7811aa0b99..529dd8ce54c 100644 --- a/docs/content/en/open_source/upgrading/2.60.md +++ b/docs/content/en/open_source/upgrading/2.60.md @@ -2,6 +2,45 @@ title: 'Upgrading to DefectDojo Version 2.60.x' toc_hide: true weight: -20260601 -description: No special instructions. +description: New deduplication execution mode for import/reimport. --- -There are no special instructions for upgrading to 2.60.x. Check the [Release Notes](https://github.com/DefectDojo/django-DefectDojo/releases/tag/2.60.0) for the contents of the release. + +## Deduplication execution mode for import/reimport + +This release adds a new `deduplication_execution_mode` setting that controls how +import/reimport deduplication post-processing is dispatched and whether the API +response waits for it. It can be set per user (profile) and overridden per request +on the import and reimport endpoints. + +Modes: + +- `async` (default): deduplication and the rest of post-processing are dispatched + to the background and the response returns immediately. This is the historical + behavior; nothing changes for existing users. +- `async_wait`: post-processing is still dispatched to the background, but the + request waits for deduplication to finish before responding. As a result the + `scan_added` notification and the statistics in the import/reimport response + reflect the deduplicated state (findings that turned out to be duplicates are + no longer counted/listed as new). JIRA push, product grading and other + non-deduplication tasks remain asynchronous and are not awaited. +- `sync`: import deduplication runs inline in the web request. + +The wait in `async_wait` is bounded by the new `DD_DEDUPLICATION_ASYNC_WAIT_TIMEOUT` +environment variable (default `60` seconds). If no worker picks up the work within +the timeout, the request responds anyway (degrading to the `async` outcome) rather +than hanging. + +The import/reimport response now also includes a `deduplication_complete` boolean +indicating whether deduplication had finished by the time the response was produced. + +### Relationship to `block_execution` + +The existing `block_execution` profile flag is unchanged. It remains the global +switch that forces **all** of a user's asynchronous tasks (notifications, JIRA +push, product grading, deduplication, ...) to run in the foreground. +`deduplication_execution_mode` is independent and narrower — it only affects +import/reimport deduplication post-processing. A user who has `block_execution` +enabled continues to get fully synchronous imports; the upgrade migration seeds +their `deduplication_execution_mode` to `sync` so behavior is unchanged. + +No action is required to upgrade. Check the [Release Notes](https://github.com/DefectDojo/django-DefectDojo/releases/tag/2.60.0) for the contents of the release. From eb47b01861cb1881e01e7d1aec2604d6d575b294 Mon Sep 17 00:00:00 2001 From: Valentijn Scholten Date: Sun, 14 Jun 2026 16:00:52 +0200 Subject: [PATCH 08/12] fix(importers): honor profile deduplication_execution_mode for UI import/reimport The API resolves deduplication_execution_mode in the serializer, but the UI import/reimport views built their context straight from the form and never resolved it, so UI imports silently defaulted to async regardless of the user's profile. Resolve it from the profile in both UI process_form paths. --- dojo/engagement/views.py | 4 ++++ dojo/test/views.py | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/dojo/engagement/views.py b/dojo/engagement/views.py index 0154aa5d336..bba6323b260 100644 --- a/dojo/engagement/views.py +++ b/dojo/engagement/views.py @@ -954,6 +954,10 @@ def process_form( "create_finding_groups_for_all_findings": form.cleaned_data.get("create_finding_groups_for_all_findings", None), "environment": self.get_development_environment(environment_name=form.cleaned_data.get("environment")), }) + # Honor the user's profile deduplication_execution_mode for UI imports. The API resolves + # this in the serializer; the UI has no per-import selector, so fall back to the profile + # (or block_execution) instead of silently defaulting to async. + context["deduplication_execution_mode"] = Dojo_User.resolve_deduplication_execution_mode(request.user) # Create the engagement if necessary self.create_engagement(context) # close_old_findings_product_scope is a modifier of close_old_findings. diff --git a/dojo/test/views.py b/dojo/test/views.py index 4e7f9c54dba..77666524ffd 100644 --- a/dojo/test/views.py +++ b/dojo/test/views.py @@ -47,6 +47,7 @@ from dojo.location.models import Location from dojo.models import ( BurpRawRequestResponse, + Dojo_User, Endpoint, Finding, Finding_Group, @@ -960,6 +961,10 @@ def process_form( "close_old_findings": form.cleaned_data.get("close_old_findings", None), "create_finding_groups_for_all_findings": form.cleaned_data.get("create_finding_groups_for_all_findings", None), }) + # Honor the user's profile deduplication_execution_mode for UI reimports. The API resolves + # this in the serializer; the UI has no per-import selector, so fall back to the profile + # (or block_execution) instead of silently defaulting to async. + context["deduplication_execution_mode"] = Dojo_User.resolve_deduplication_execution_mode(request.user) # Override the form values of active and verified if activeChoice := form.cleaned_data.get("active", None): if activeChoice == "force_to_true": From 71a7517635f147048b8b05c57c0d132f4896e192 Mon Sep 17 00:00:00 2001 From: Valentijn Scholten Date: Sun, 14 Jun 2026 16:16:59 +0200 Subject: [PATCH 09/12] test: revert set_block_execution to the block_execution checkbox Option B keeps block_execution as a checkbox in the profile form, so the integration helper toggles id_block_execution again instead of the deduplication_execution_mode select (which no longer existed under that id). --- tests/base_test_class.py | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/tests/base_test_class.py b/tests/base_test_class.py index 2798d2f2a58..d9043a43fd5 100644 --- a/tests/base_test_class.py +++ b/tests/base_test_class.py @@ -10,7 +10,7 @@ from selenium.webdriver.chrome.options import Options from selenium.webdriver.common.by import By from selenium.webdriver.support import expected_conditions -from selenium.webdriver.support.ui import Select, WebDriverWait +from selenium.webdriver.support.ui import WebDriverWait # import time logging.basicConfig( @@ -329,24 +329,22 @@ def enable_github(self): return self.enable_system_setting("id_enable_github") def set_block_execution(self, *, block_execution=True): - # we set the admin user (ourselves) to the synchronous import execution mode - # (the successor of the old block_execution flag) when block_execution=True. - # This forces dedupe to happen synchronously, among other things like - # notifications, rules, ... Otherwise we select the default async mode. - target_mode = "sync" if block_execution else "async" - logger.info("setting import execution mode to: %s", target_mode) + # we set the admin user (ourselves) to have block_execution checked + # this will force dedupe to happen synchronously, among other things like notifications, rules, ... + logger.info("setting block execution to: %s", block_execution) driver = self.driver driver.get(self.base_url + "profile") - select = Select(driver.find_element(By.ID, "id_import_execution_mode")) - if select.first_selected_option.get_attribute("value") != target_mode: - select.select_by_value(target_mode) + if ( + driver.find_element(By.ID, "id_block_execution").is_selected() + != block_execution + ): + driver.find_element(By.XPATH, '//*[@id="id_block_execution"]').click() # save settings driver.find_element(By.CSS_SELECTOR, "input.btn.btn-primary").click() - # check if it's applied after reload - select = Select(driver.find_element(By.ID, "id_import_execution_mode")) + # check if it's enabled after reload self.assertEqual( - select.first_selected_option.get_attribute("value"), - target_mode, + driver.find_element(By.ID, "id_block_execution").is_selected(), + block_execution, ) return driver From 68fc3cdd0ac11b9c7af506d1a1c5168ad3e53c24 Mon Sep 17 00:00:00 2001 From: Valentijn Scholten Date: Sun, 14 Jun 2026 16:40:08 +0200 Subject: [PATCH 10/12] refactor(migrations): split into schema add + data seed (2 migrations) Collapse the add/rename pair into a single AddField that creates deduplication_execution_mode with its final name, and keep the seed as a separate data migration, per the convention of keeping schema and data migrations apart. Drops the interim rename migration. --- ...rcontactinfo_deduplication_execution_mode.py} | 2 +- .../0270_seed_deduplication_execution_mode.py | 6 +++--- ...rename_import_execution_mode_deduplication.py | 16 ---------------- 3 files changed, 4 insertions(+), 20 deletions(-) rename dojo/db_migrations/{0269_usercontactinfo_import_execution_mode.py => 0269_usercontactinfo_deduplication_execution_mode.py} (95%) delete mode 100644 dojo/db_migrations/0271_rename_import_execution_mode_deduplication.py diff --git a/dojo/db_migrations/0269_usercontactinfo_import_execution_mode.py b/dojo/db_migrations/0269_usercontactinfo_deduplication_execution_mode.py similarity index 95% rename from dojo/db_migrations/0269_usercontactinfo_import_execution_mode.py rename to dojo/db_migrations/0269_usercontactinfo_deduplication_execution_mode.py index d8219e5596c..94f1e29f255 100644 --- a/dojo/db_migrations/0269_usercontactinfo_import_execution_mode.py +++ b/dojo/db_migrations/0269_usercontactinfo_deduplication_execution_mode.py @@ -10,7 +10,7 @@ class Migration(migrations.Migration): operations = [ migrations.AddField( model_name='usercontactinfo', - name='import_execution_mode', + name='deduplication_execution_mode', field=models.CharField(blank=True, choices=[('async', 'Async (do not wait)'), ('async_wait', 'Async, wait for deduplication'), ('sync', 'Synchronous (block)')], help_text="Controls how import/reimport deduplication post-processing is executed. 'Async' dispatches it to the background and returns immediately (default). 'Async, wait for deduplication' dispatches to the background but waits for deduplication to finish before responding, so notifications and statistics reflect the deduplicated state. 'Synchronous' runs the import deduplication inline. Can be overridden per request. Independent of block_execution, which forces all async tasks (notifications, jira, ...) to the foreground.", max_length=20, null=True), ), ] diff --git a/dojo/db_migrations/0270_seed_deduplication_execution_mode.py b/dojo/db_migrations/0270_seed_deduplication_execution_mode.py index 862a2b0d929..7e56f1674f4 100644 --- a/dojo/db_migrations/0270_seed_deduplication_execution_mode.py +++ b/dojo/db_migrations/0270_seed_deduplication_execution_mode.py @@ -10,19 +10,19 @@ def seed_deduplication_execution_mode(apps, schema_editor): unchanged for them. """ UserContactInfo = apps.get_model("dojo", "UserContactInfo") - UserContactInfo.objects.filter(block_execution=True).update(import_execution_mode="sync") + UserContactInfo.objects.filter(block_execution=True).update(deduplication_execution_mode="sync") def unseed_deduplication_execution_mode(apps, schema_editor): """Reverse: clear the seeded synchronous mode.""" UserContactInfo = apps.get_model("dojo", "UserContactInfo") - UserContactInfo.objects.filter(import_execution_mode="sync").update(import_execution_mode=None) + UserContactInfo.objects.filter(deduplication_execution_mode="sync").update(deduplication_execution_mode=None) class Migration(migrations.Migration): dependencies = [ - ('dojo', '0269_usercontactinfo_import_execution_mode'), + ('dojo', '0269_usercontactinfo_deduplication_execution_mode'), ] operations = [ diff --git a/dojo/db_migrations/0271_rename_import_execution_mode_deduplication.py b/dojo/db_migrations/0271_rename_import_execution_mode_deduplication.py deleted file mode 100644 index 1b136c9050b..00000000000 --- a/dojo/db_migrations/0271_rename_import_execution_mode_deduplication.py +++ /dev/null @@ -1,16 +0,0 @@ -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('dojo', '0270_seed_deduplication_execution_mode'), - ] - - operations = [ - migrations.RenameField( - model_name='usercontactinfo', - old_name='import_execution_mode', - new_name='deduplication_execution_mode', - ), - ] From 0fb37f4fda3cf7b7cc1b5b71f90772c430812850 Mon Sep 17 00:00:00 2001 From: Valentijn Scholten Date: Sun, 14 Jun 2026 16:46:44 +0200 Subject: [PATCH 11/12] test: use versioned_fixtures so dedup mode tests pass under V3_FEATURE_LOCATIONS The CI 'true' matrix variant enables V3_FEATURE_LOCATIONS, where loading dojo_testdata.json (containing Endpoints) raises NotImplementedError. Decorate the test classes with @versioned_fixtures so the locations fixture is used in that mode, matching the other import/reimport test suites. --- unittests/test_import_execution_mode.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/unittests/test_import_execution_mode.py b/unittests/test_import_execution_mode.py index f7e8b9660c9..e575b987c01 100644 --- a/unittests/test_import_execution_mode.py +++ b/unittests/test_import_execution_mode.py @@ -15,9 +15,10 @@ UserContactInfo, ) -from .dojo_test_case import DojoAPITestCase, DojoTestCase, get_unit_tests_path +from .dojo_test_case import DojoAPITestCase, DojoTestCase, get_unit_tests_path, versioned_fixtures +@versioned_fixtures class ImportExecutionModeResolverTest(DojoTestCase): """resolve_deduplication_execution_mode: request override > profile > default.""" @@ -87,6 +88,7 @@ def test_wants_block_execution_reads_block_execution_not_mode(self): self.assertFalse(Dojo_User.wants_block_execution(self.user)) +@versioned_fixtures class ImporterDispatchKwargsTest(DojoTestCase): """deduplication_execution_mode -> dojo_dispatch_task force flags.""" @@ -121,6 +123,7 @@ def test_external_force_sync_promotes_to_sync_mode(self): self.assertEqual(DEDUPLICATION_EXECUTION_MODE_SYNC, importer.deduplication_execution_mode) +@versioned_fixtures @override_settings(CELERY_TASK_ALWAYS_EAGER=True) class ImportExecutionModeAPITest(DojoAPITestCase): @@ -170,6 +173,7 @@ def test_import_rejects_invalid_mode(self): self.import_scan(payload, 400) +@versioned_fixtures class NotificationDeduplicationRefreshTest(DojoTestCase): """notify_scan_added refreshes duplicate status from the DB once dedup is complete.""" From 69aa97bb5e86dda4bbf76a301b8f5e182881cd92 Mon Sep 17 00:00:00 2001 From: Valentijn Scholten Date: Sun, 14 Jun 2026 22:04:14 +0200 Subject: [PATCH 12/12] test(perf): add async_wait deduplication performance test Covers the 'async_wait' execution mode: post-processing is dispatched to a background worker and the request joins on it before responding. The dedup queries run in the worker (separate connection), not the web request, so the only web-side delta over the plain async path is the post-dedup notification refresh SELECT (+1): 109->110 first import, 89->90 second. Does not use CELERY_TASK_ALWAYS_EAGER (that would run the task inline on the request connection and wrongly count worker queries). The dedup batch is dispatched async and the join (AsyncResult.get) is mocked to return instantly, simulating a finished worker. Adds an optional dedup_mode param to _deduplication_performance. --- unittests/test_importers_performance.py | 38 ++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/unittests/test_importers_performance.py b/unittests/test_importers_performance.py index ce9133e4a6b..3eac03f4ffb 100644 --- a/unittests/test_importers_performance.py +++ b/unittests/test_importers_performance.py @@ -403,7 +403,7 @@ def test_import_reimport_reimport_performance_pghistory_no_async_with_product_gr ) # Deduplication is enabled in the tests above, but to properly test it we must run the same import twice and capture the results. - def _deduplication_performance(self, expected_num_queries1, expected_num_async_tasks1, expected_num_queries2, expected_num_async_tasks2, *, check_duplicates=True): + def _deduplication_performance(self, expected_num_queries1, expected_num_async_tasks1, expected_num_queries2, expected_num_async_tasks2, *, check_duplicates=True, dedup_mode=None): """ Test method to measure deduplication performance by importing the same scan twice. The second import should result in all findings being marked as duplicates. @@ -444,6 +444,7 @@ def _deduplication_performance(self, expected_num_queries1, expected_num_async_t "verified": True, "scan_type": STACK_HAWK_SCAN_TYPE, "engagement": engagement, + **({"deduplication_execution_mode": dedup_mode} if dedup_mode else {}), } importer = DefaultImporter(**import_options) _, _, len_new_findings1, len_closed_findings1, _, _, _ = importer.process_scan(scan) @@ -471,6 +472,7 @@ def _deduplication_performance(self, expected_num_queries1, expected_num_async_t "verified": True, "scan_type": STACK_HAWK_SCAN_TYPE, "engagement": engagement, + **({"deduplication_execution_mode": dedup_mode} if dedup_mode else {}), } importer = DefaultImporter(**import_options) _, _, len_new_findings2, len_closed_findings2, _, _, _ = importer.process_scan(scan) @@ -551,6 +553,40 @@ def test_deduplication_performance_pghistory_no_async(self): expected_num_async_tasks2=2, ) + @override_settings(ENABLE_AUDITLOG=True) + def test_deduplication_performance_pghistory_async_wait(self): + """ + Deduplication performance in the 'async_wait' execution mode: post-processing is + dispatched to a background worker, then the request joins on the result before + responding. The dedup queries run in the worker (a separate connection), NOT in + the web request, so the only web-side cost over the plain async path is the single + post-dedup notification refresh SELECT (+1). + + We do not use CELERY_TASK_ALWAYS_EAGER here — that would run the dispatched task + inline on the request's connection and wrongly count the worker's dedup queries. + Instead the dedup batch is dispatched async (not executed in-process) and the join + (AsyncResult.get) is mocked to return immediately, simulating a worker that has + finished. deduplication_complete is therefore True (so the refresh runs), but the + findings are not actually deduplicated in-test, so check_duplicates is False. + """ + configure_audit_system() + configure_pghistory_triggers() + + # Enable deduplication + self.system_settings(enable_deduplication=True) + + # Simulate the background worker's post-processing having completed so the join + # returns instantly without executing dedup on the request's DB connection. + with patch("celery.result.AsyncResult.get", return_value=None): + self._deduplication_performance( + expected_num_queries1=110, + expected_num_async_tasks1=2, + expected_num_queries2=90, + expected_num_async_tasks2=2, + dedup_mode="async_wait", + check_duplicates=False, + ) + @tag("performance") @override_settings(V3_FEATURE_LOCATIONS=True)