Ved 982 updated dpsfull batch scenario (#1371)

* added DPSFULL batch scenario

* updated fix

Author: FimranNHS

tests/e2e_automation/features/APITests/search.feature

@@ -235,7 +235,6 @@ Feature: Search the immunization of a patient
         When I send a search request with Post method using identifier parameter for Immunization event created
         Then The request will be successful with the status code '200'
         And correct immunization event is returned in the response
-        And MNS event will not be triggered for the update event
 
     @Delete_cleanUp @vaccine_type_FLU @patient_id_Random  @supplier_name_Postman_Auth
     Scenario: Flu event is created and search post request fetch the only one record matched with identifier and _elements

tests/e2e_automation/features/APITests/steps/common_steps.py

@@ -1,3 +1,4 @@
+import copy
 import json
 import random
 import uuid
@@ -329,7 +330,8 @@ def validate_etag_in_header(context):
 @when("I subsequently update the vaccination details of the original immunization event")
 def send_update_for_vaccination_detail(context):
     get_update_url_header(context, str(context.expected_version))
-    context.update_object = convert_to_update(context.immunization_object, context.ImmsID)
+    context.update_object = copy.deepcopy(context.immunization_object)
+    context.update_object = convert_to_update(context.update_object, context.ImmsID)
     context.update_object.extension = [build_vaccine_procedure_extension(context.vaccine_type.upper())]
     vaccine_details = get_vaccine_details(context.vaccine_type.upper())
     context.update_object.vaccineCode = vaccine_details["vaccine_code"]
@@ -341,7 +343,8 @@ def send_update_for_vaccination_detail(context):
 @when("I update the address of the original immunization event")
 def send_update_for_immunization_event(context):
     get_update_url_header(context, str(context.expected_version))
-    context.update_object = convert_to_update(context.immunization_object, context.ImmsID)
+    context.update_object = copy.deepcopy(context.immunization_object)
+    context.update_object = convert_to_update(context.update_object, context.ImmsID)
     context.update_object.contained[1].address[0].city = "Updated City"
     context.update_object.contained[1].address[0].state = "Updated State"
     trigger_the_updated_request(context)
@@ -395,14 +398,7 @@ def mns_event_will_not_be_triggered_for_the_event(context):
 
 @then("MNS event will not be triggered for the update event")
 def validate_mns_event_not_triggered_for_updated_event(context):
-    message_body = read_message(
-        context,
-        queue_type="notification",
-        wait_time_seconds=5,
-        max_total_wait_seconds=20,
-    )
-    print("no MNS update event is created")
-    assert message_body is None, "Not expected a message but queue returned a message"
+    mns_event_will_not_be_triggered_for_the_event(context)
 
 
 def trigger_the_updated_request(context):
@@ -482,10 +478,11 @@ def validate_sqs_message(context, message_body, action):
             f"msn event for {action} Filtering is missing in the message body",
         )
 
-        check.is_true(
-            normalize(message_body.filtering.generalpractitioner) == normalize(context.gp_code),
-            f"msn event for {action} GP code mismatch: expected {context.gp_code}, got {message_body.filtering.generalpractitioner}",
-        )
+        if context.gp_code:
+            check.is_true(
+                normalize(message_body.filtering.generalpractitioner) == normalize(context.gp_code),
+                f"msn event for {action} GP code mismatch: expected {context.gp_code}, got {message_body.filtering.generalpractitioner}",
+            )
 
         expected_org = context.immunization_object.performer[1].actor.identifier.value
         check.is_true(
@@ -498,10 +495,11 @@ def validate_sqs_message(context, message_body, action):
             f"msn event for {action} Source application mismatch: expected {context.supplier_name}, got {message_body.filtering.sourceapplication}",
         )
 
-        check.is_true(
-            message_body.filtering.subjectage == context.patient_age,
-            f"msn event for {action} Age mismatch: expected {context.patient_age}, got {message_body.filtering.subjectage}",
-        )
+        if context.patient_age:
+            check.is_true(
+                message_body.filtering.subjectage == context.patient_age,
+                f"msn event for {action} Age mismatch: expected {context.patient_age}, got {message_body.filtering.subjectage}",
+            )
 
         check.is_true(
             message_body.filtering.immunisationtype == context.vaccine_type.upper(),
@@ -544,7 +542,12 @@ def mns_event_will_be_triggered_with_correct_data(context, action):
         print(f"Read {action}d message from SQS: {message_body}")
         assert message_body is not None, f"Expected a {action} message but queue returned empty"
         context.gp_code = get_gp_code_by_nhs_number(context.patient.identifier[0].value)
-        context.patient_age = calculate_age(context.patient.birthDate, context.immunization_object.occurrenceDateTime)
+        patient_DOB = (
+            context.immunization_object.contained[1].birthDate
+            if action.upper() == "CREATE"
+            else context.update_object.contained[1].birthDate
+        )
+        context.patient_age = calculate_age(patient_DOB, context.immunization_object.occurrenceDateTime)
         validate_sqs_message(context, message_body, action)
     else:
         print(

tests/e2e_automation/features/APITests/steps/test_update_steps.py

@@ -1,3 +1,4 @@
+import copy
 import uuid
 
 from pytest_bdd import parsers, scenarios, then, when
@@ -58,15 +59,17 @@ def validate_delta_table_for_updated_event(context):
 )
 def send_update_for_immunization_event_with_occurrenceDateTime(context, DateText):
     get_update_url_header(context, str(context.expected_version))
-    context.update_object = convert_to_update(context.immunization_object, context.ImmsID)
+    context.update_object = copy.deepcopy(context.immunization_object)
+    context.update_object = convert_to_update(context.update_object, context.ImmsID)
     context.update_object.occurrenceDateTime = generate_date(DateText)
     trigger_the_updated_request(context)
 
 
 @when(parsers.parse("Send a update for Immunization event created with recorded being updated to '{DateText}'"))
 def send_update_for_immunization_event_with_recorded_date_update(context, DateText):
     get_update_url_header(context, str(context.expected_version))
-    context.update_object = convert_to_update(context.immunization_object, context.ImmsID)
+    context.update_object = copy.deepcopy(context.immunization_object)
+    context.update_object = convert_to_update(context.update_object, context.ImmsID)
     context.update_object.recorded = generate_date(DateText)
     trigger_the_updated_request(context)
 
@@ -76,15 +79,17 @@ def send_update_for_immunization_event_with_recorded_date_update(context, DateTe
 )
 def send_update_for_immunization_event_with_dob_update(context, DateText):
     get_update_url_header(context, str(context.expected_version))
-    context.update_object = convert_to_update(context.immunization_object, context.ImmsID)
+    context.update_object = copy.deepcopy(context.immunization_object)
+    context.update_object = convert_to_update(context.update_object, context.ImmsID)
     context.update_object.contained[1].birthDate = generate_date(DateText)
     trigger_the_updated_request(context)
 
 
 @when(parsers.parse("Send a update for Immunization event created with expiration date being updated to '{DateText}'"))
 def send_update_for_immunization_event_with_expiration_date_update(context, DateText):
     get_update_url_header(context, str(context.expected_version))
-    context.update_object = convert_to_update(context.immunization_object, context.ImmsID)
+    context.update_object = copy.deepcopy(context.immunization_object)
+    context.update_object = convert_to_update(context.update_object, context.ImmsID)
     context.update_object.expirationDate = generate_date(DateText)
     trigger_the_updated_request(context)
 
@@ -94,7 +99,8 @@ def send_update_request_for_invalid_immunization_id(context):
     valid_json_payload_is_created(context)
     context.ImmsID = str(uuid.uuid4())
     get_update_url_header(context, str(context.expected_version))
-    context.update_object = convert_to_update(context.immunization_object, context.ImmsID)
+    context.update_object = copy.deepcopy(context.immunization_object)
+    context.update_object = convert_to_update(context.update_object, context.ImmsID)
     trigger_the_updated_request(context)
 
 
@@ -104,7 +110,8 @@ def send_update_request_for_invalid_etag(context, Etag):
     context.ImmsID = str(uuid.uuid4())
     context.version = Etag
     get_update_url_header(context, Etag)
-    context.update_object = convert_to_update(context.immunization_object, context.ImmsID)
+    context.update_object = copy.deepcopy(context.immunization_object)
+    context.update_object = convert_to_update(context.update_object, context.ImmsID)
     trigger_the_updated_request(context)
 
 

tests/e2e_automation/features/APITests/update.feature

@@ -22,7 +22,7 @@ Feature: Update the immunization of a patient
         And The Response JSONs should contain correct error message for 'forbidden' access
 
 
-    @delete_cleanup @vaccine_type_RSV @patient_id_NullNHS @supplier_name_RAVS
+    @delete_cleanup @vaccine_type_RSV @patient_id_Random @supplier_name_RAVS
     Scenario: verify that vaccination record can be updated with valid vaccination detail
         Given I have created a valid vaccination record
         When I update the address of the original immunization event
@@ -31,7 +31,7 @@ Feature: Update the immunization of a patient
         And The X-Request-ID and X-Correlation-ID keys in header will populate correctly
         And The imms event table will be populated with the correct data for 'updated' event
         And The delta table will be populated with the correct data for updated event
-        And MNS event will not be triggered for the update event
+        And MNS event will be triggered with correct data for Updated event
 
 
     @smoke
@@ -80,6 +80,7 @@ Feature: Update the immunization of a patient
         When Send a update for Immunization event created with occurrenceDateTime being updated to '<Date>'
         Then The request will be unsuccessful with the status code '400'
         And The Response JSONs should contain correct error message for 'invalid_OccurrenceDateTime'
+        And MNS event will not be triggered for the update event
         Examples:
             | Date              |
             | future_occurrence |

tests/e2e_automation/features/batchTests/Steps/batch_common_steps.py

@@ -100,6 +100,21 @@ def valid_batch_file_is_created_with_details(datatable, context):
     create_batch_file(context)
 
 
+@given("batch file is received for below data form DPS with all three action flag for each record")
+@ignore_if_local_run
+def valid_batch_file_is_created_for_DPSFULL(datatable, context):
+    build_dataFrame_using_datatable(datatable, context)
+    df_new = context.vaccine_df.copy()
+    df_update = df_new.copy()
+    df_update[["ACTION_FLAG", "EXPIRY_DATE"]] = ["UPDATE", "20281231"]
+    df_delete = df_new.copy()
+    df_delete[["ACTION_FLAG", "EXPIRY_DATE"]] = ["DELETE", "20281231"]
+    context.vaccine_df = pd.concat([df_new, df_update, df_delete], ignore_index=True)
+    file_name = f"{context.vaccine_type}_Vaccinations_v5_DPSFULL"
+    create_batch_file(context, fileName=file_name)
+    context.expected_version = 2
+
+
 @given("batch file is created for below data as full dataset with file extension dat")
 @ignore_if_local_run
 def valid_batch_file_is_created_with_details_and_dat_extension(datatable, context):
@@ -227,6 +242,34 @@ def validate_imms_delta_table_for_created_records_in_batch_file(context):
     validate_imms_delta_table_for_newly_created_records_in_batch_file(context)
 
 
+@then("The delta table will not be populated with DPSFULL records in batch file")
+def validate_imms_delta_table_for_dpsfull_records(context):
+    df = context.vaccine_df
+
+    check.is_true("IMMS_ID" in df.columns, "Column 'IMMS_ID' not found in vaccine_df")
+
+    valid_rows = df[df["IMMS_ID"].notnull()]
+    check.is_true(not valid_rows.empty, "No rows with non-null IMMS_ID found in vaccine_df")
+
+    grouped = valid_rows.groupby("IMMS_ID")
+
+    for imms_id in grouped:
+        clean_id = imms_id[0].replace("Immunization#", "")
+
+        delta_items = fetch_immunization_int_delta_detail_by_immsID(
+            context.aws_profile_name,
+            clean_id,
+            context.S3_env,
+            max_attempts=2,
+            delay=5,
+        )
+
+        check.is_true(
+            not delta_items,
+            f"Delta records were unexpectedly found for IMMS_ID: {clean_id}",
+        )
+
+
 @then("The delta table will be populated with the correct data for all updated records in batch file")
 def validate_imms_delta_table_for_updated_records(context):
     if context.delta_cache is None:
@@ -483,16 +526,18 @@ def validate_imms_delta_table_for_deleted_records_in_batch_file(context):
 
 
 def _is_null_nhs_row(row) -> bool:
-    return str(row.UNIQUE_ID).startswith("NullNHS") or str(row.NHS_NUMBER).strip() in ("", "None", "nan")
+    return str(row.UNIQUE_ID).startswith("NullNHS") or str(row.NHS_NUMBER).strip() in (
+        "",
+        "None",
+        "nan",
+    )
 
 
 def _assert_no_mns_events_for_null_nhs_rows(context, null_nhs_rows, wait_seconds=20):
     if not null_nhs_rows:
         print("No NullNHS rows — skipping negative MNS check.")
         return
 
-    # Use an unreachable expected_count so the poller runs for the full wait_seconds,
-    # then check that nothing arrived for these datarefs.
     unexpected = read_messages_for_batch(
         context,
         queue_type="notification",

tests/e2e_automation/features/batchTests/create_batch.feature

@@ -300,3 +300,25 @@ Feature: Create the immunization event for a patient through batch file
         And The imms event table will be populated with the correct data for 'created' event for records in batch file
         And The delta table will be populated with the correct data for all created records in batch file
         And MNS event will be triggered with correct data for all 'CREATE' events where NHS is not null
+
+
+    @vaccine_type_HPV  @supplier_name_DPSFULL
+    Scenario: Verify that vaccination record will be created through batch file received from DPS
+        Given batch file is received for below data form DPS with all three action flag for each record
+            | patient_id   | unique_id              |
+            | Random       | Valid_NhsNumber        |
+            | InvalidInPDS | InvalidInPDS_NhsNumber |
+            | SFlag        | SFlag_NhsNumber        |
+            | Mod11_NHS    | Mod11_NhSNumber        |
+            | NullNHS      | NullNHS_NhsNumber      |
+            | OldNHSNo     | OldNHS_NhsNumber       |
+            | NO_GP_NHS    | NO_GP_NhsNumber        |
+        When batch file is uploaded in s3 bucket
+        Then file will be moved to destination bucket and inf ack file will be created
+        And inf ack file has success status for processed batch file
+        And bus ack files will be created
+        And CSV bus ack will not have any entry of successfully processed records
+        And Json bus ack will only contain file metadata and no failure record entry
+        And Audit table will have correct status, queue name and record count for the processed batch file
+        And The imms event table will be populated with the correct data for 'deleted' event for records in batch file
+        And The delta table will not be populated with DPSFULL records in batch file

tests/e2e_automation/features/conftest.py

@@ -135,7 +135,8 @@ def context(request, global_context, temp_apigee_apps: list[ApigeeApp] | None) -
             ctx.patient_id = tag.split("patient_id_")[1]
         if tag.startswith("supplier_name_"):
             ctx.supplier_name = tag.split("supplier_name_")[1]
-            get_tokens(ctx, ctx.supplier_name)
+            if ctx.supplier_name != "DPSFULL":
+                get_tokens(ctx, ctx.supplier_name)
             ctx.supplier_name = tag.split("supplier_name_")[1]
             ctx.supplier_ods_code = SupplierNameWithODSCode[ctx.supplier_name].value
 
@@ -163,7 +164,8 @@ def pytest_bdd_after_scenario(request, feature, scenario):
 
     if "delete_cleanup_batch" in tags:
         if "IMMS_ID" in context.vaccine_df.columns and context.vaccine_df["IMMS_ID"].notna().any():
-            get_tokens(context, context.supplier_name)
+            supplier_name = context.supplier_name if context.supplier_name != "DPSFULL" else "Postman_Auth"
+            get_tokens(context, supplier_name)
 
             df = context.vaccine_df.dropna(subset=["IMMS_ID"]).copy()
             df["IMMS_ID_CLEAN"] = df["IMMS_ID"].astype(str).str.replace("Immunization#", "", regex=False)

tests/e2e_automation/pytest.ini

@@ -19,10 +19,17 @@ markers =
     Delete_Batch_Feature: tag for delete batch feature tests
     Delete_cleanUp: tag for scenarios that require cleanup after execution
     delete_cleanup_batch: tag for batch scenarios that require cleanup after execution
+    smoke: test tag for smoke suite
+    functional: test tag for regression suite
     patient_id_OldNHS: tag for old NHS patient ID scenarios
     patient_id_ValidNHS: tag for valid NHS patient ID scenarios
     patient_id_SFlag: tag for patient id SFlag scenario
     patient_id_SupersedeNhsNo: tag for patient id Supersede NHS Number scenario    
+    patient_id_NullNHS: tag for null NHS patient ID scenarios
+    patient_id_InvalidInPDS: tag for invalid NHS patient ID scenarios
+    patient_id_InvalidMOD11Check: tag for invalid MOD11 check NHS patient ID scenarios
+    patient_id_Mod11_NHS: tag for Mod11 NHS patient ID scenarios
+    patient_id_NO_GP_NHS: tag for NHS patient ID with no GP scenarios
     vaccine_type_RSV: tag for RSV vaccine type scenarios
     vaccine_type_PNEUMOCOCCAL: tag for PNEUMOCOCCAL vaccine type scenarios
     vaccine_type_FLU: tag for FLU vaccine type scenarios
@@ -52,4 +59,6 @@ markers =
     supplier_name_PINNACLE: tag for Pinnacle supplier name scenarios
     supplier_name_TPP: tag for TPP supplier name scenarios
     supplier_name_MEDICUS: tag for MEDICUS supplier name scenarios
-    supplier_name_CEGEDIM: tag for CEGEDIM supplier name scenarios 
+    supplier_name_CEGEDIM: tag for CEGEDIM supplier name scenarios
+    supplier_name_DPSFULL: tag for DPSFULL supplier name scenarios
+

tests/e2e_automation/src/dynamoDB/dynamo_db_helper.py

@@ -89,13 +89,17 @@ def fetch_immunization_events_detail_by_IdentifierPK(
     return response
 
 
-def fetch_immunization_int_delta_detail_by_immsID(aws_profile_name: str, ImmsID: str, env: str, expected_item: int = 1):
+def fetch_immunization_int_delta_detail_by_immsID(
+    aws_profile_name: str,
+    ImmsID: str,
+    env: str,
+    expected_item: int = 1,
+    max_attempts=5,
+    delay=2,
+):
     db = DynamoDBHelper(aws_profile_name, env)
     tableImmsDelta = db.get_delta_table()
 
-    max_attempts = 5
-    delay = 2  # seconds
-
     for attempt in range(1, max_attempts + 1):
         response = tableImmsDelta.query(
             IndexName="ImmunisationIdIndex",

tests/e2e_automation/utilities/enums.py

@@ -23,6 +23,7 @@ class SupplierNameWithODSCode(Enum):
     MEDICUS = "YGMYW"
     CEGEDIM = "YGM04"
     Postman_Auth = "Postman_Auth"
+    DPSFULL = "DPSFULL"
 
 
 class GenderCode(Enum):