CCM-15589: Write automation tests for urgent letter priority (#500)

* Refactor schemas and add test for letter priority

* Fix rename

* Bump internal events version

* Fix rename again

* Fix helper name

* Add integration-tests to pw config

* Change default variant map and refactor poll helper

* try

* add todo

* check letter queue; fix lock file

* fix lint

* remove commented code

* lint

* peer review

* phrasing

* fix imports

Author: francisco-videira-nhs

infrastructure/terraform/components/api/README.md

@@ -37,7 +37,7 @@ No requirements.
 | <a name="input_kms_deletion_window"></a> [kms\_deletion\_window](#input\_kms\_deletion\_window) | When a kms key is deleted, how long should it wait in the pending deletion state? | `string` | `"30"` | no |
 | <a name="input_letter_event_source"></a> [letter\_event\_source](#input\_letter\_event\_source) | Source value to use for the letter status event updates | `string` | `"/data-plane/supplier-api/nhs-supplier-api-prod/main/update-status"` | no |
 | <a name="input_letter_table_ttl_hours"></a> [letter\_table\_ttl\_hours](#input\_letter\_table\_ttl\_hours) | Number of hours to set as TTL on letters table | `number` | `24` | no |
-| <a name="input_letter_variant_map"></a> [letter\_variant\_map](#input\_letter\_variant\_map) | n/a | `map(object({ supplierId = string, specId = string, priority = number, billingId = string }))` | <pre>{<br/>  "lv1": {<br/>    "billingId": "billing1",<br/>    "priority": 10,<br/>    "specId": "spec1",<br/>    "supplierId": "supplier1"<br/>  },<br/>  "lv2": {<br/>    "billingId": "billing1",<br/>    "priority": 10,<br/>    "specId": "spec2",<br/>    "supplierId": "supplier1"<br/>  },<br/>  "lv3": {<br/>    "billingId": "billing1",<br/>    "priority": 10,<br/>    "specId": "spec3",<br/>    "supplierId": "supplier2"<br/>  }<br/>}</pre> | no |
+| <a name="input_letter_variant_map"></a> [letter\_variant\_map](#input\_letter\_variant\_map) | n/a | `map(object({ supplierId = string, specId = string, priority = number, billingId = string }))` | <pre>{<br/>  "digitrials-aspiring": {<br/>    "billingId": "digitrials-aspiring-billing",<br/>    "priority": "0",<br/>    "specId": "digitrials-aspiring",<br/>    "supplierId": "supplier1"<br/>  },<br/>  "digitrials-dmapp": {<br/>    "billingId": "notify-admail-billing",<br/>    "priority": "1",<br/>    "specId": "notify-admail",<br/>    "supplierId": "supplier1"<br/>  },<br/>  "digitrials-globalminds": {<br/>    "billingId": "digitrials-globalminds-billing",<br/>    "priority": "2",<br/>    "specId": "digitrials-globalminds",<br/>    "supplierId": "supplier1"<br/>  },<br/>  "digitrials-mymelanoma": {<br/>    "billingId": "digitrials-mymelanoma-billing",<br/>    "priority": "3",<br/>    "specId": "digitrials-mymelanoma",<br/>    "supplierId": "supplier1"<br/>  },<br/>  "digitrials-ofh": {<br/>    "billingId": "digitrials-ofh-billing",<br/>    "priority": "4",<br/>    "specId": "digitrials-ofh",<br/>    "supplierId": "supplier1"<br/>  },<br/>  "digitrials-prostateprogress": {<br/>    "billingId": "digitrials-prostateprogress-billing",<br/>    "priority": "5",<br/>    "specId": "digitrials-prostateprogress",<br/>    "supplierId": "supplier1"<br/>  },<br/>  "digitrials-protectc": {<br/>    "billingId": "notify-c5-colour-billing",<br/>    "priority": "6",<br/>    "specId": "notify-c5-colour",<br/>    "supplierId": "supplier1"<br/>  },<br/>  "digitrials-restore": {<br/>    "billingId": "digitrials-restore-billing",<br/>    "priority": "7",<br/>    "specId": "digitrials-restore",<br/>    "supplierId": "supplier1"<br/>  },<br/>  "gpreg-admail": {<br/>    "billingId": "notify-admail-billing",<br/>    "priority": "8",<br/>    "specId": "notify-admail",<br/>    "supplierId": "supplier1"<br/>  },<br/>  "nces-abnormal-results": {<br/>    "billingId": "nces-abnormal-results-billing",<br/>    "priority": "9",<br/>    "specId": "nces-abnormal-results",<br/>    "supplierId": "supplier1"<br/>  },<br/>  "nces-abnormal-results-braille": {<br/>    "billingId": "nces-abnormal-results-braille-billing",<br/>    "priority": "10",<br/>    "specId": "nces-abnormal-results-braille",<br/>    "supplierId": "supplier1"<br/>  },<br/>  "nces-invites": {<br/>    "billingId": "nces-invites-billing",<br/>    "priority": "10",<br/>    "specId": "nces-invites",<br/>    "supplierId": "supplier1"<br/>  },<br/>  "nces-invites-braille": {<br/>    "billingId": "nces-invites-braille-billing",<br/>    "priority": "10",<br/>    "specId": "nces-invites-braille",<br/>    "supplierId": "supplier1"<br/>  },<br/>  "nces-standard": {<br/>    "billingId": "notify-c5-whitemail-billing",<br/>    "priority": "11",<br/>    "specId": "notify-c5-whitemail",<br/>    "supplierId": "supplier1"<br/>  },<br/>  "nces-standard-braille": {<br/>    "billingId": "notify-braille-whitemail-billing",<br/>    "priority": "12",<br/>    "specId": "notify-braille-whitemail",<br/>    "supplierId": "supplier1"<br/>  },<br/>  "notify-braille": {<br/>    "billingId": "notify-braille-billing",<br/>    "priority": "13",<br/>    "specId": "notify-braille",<br/>    "supplierId": "supplier1"<br/>  },<br/>  "notify-digital-letters-standard": {<br/>    "billingId": "notify-c5-billing",<br/>    "priority": "97",<br/>    "specId": "notify-c5",<br/>    "supplierId": "supplier1"<br/>  },<br/>  "notify-standard": {<br/>    "billingId": "notify-c5-billing",<br/>    "priority": "98",<br/>    "specId": "notify-c5",<br/>    "supplierId": "supplier1"<br/>  },<br/>  "notify-standard-colour": {<br/>    "billingId": "notify-c5-colour-billing",<br/>    "priority": "99",<br/>    "specId": "notify-c5-colour",<br/>    "supplierId": "supplier1"<br/>  }<br/>}</pre> | no |
 | <a name="input_log_level"></a> [log\_level](#input\_log\_level) | The log level to be used in lambda functions within the component. Any log with a lower severity than the configured value will not be logged: https://docs.python.org/3/library/logging.html#levels | `string` | `"INFO"` | no |
 | <a name="input_log_retention_in_days"></a> [log\_retention\_in\_days](#input\_log\_retention\_in\_days) | The retention period in days for the Cloudwatch Logs events to be retained, default of 0 is indefinite | `number` | `0` | no |
 | <a name="input_manually_configure_mtls_truststore"></a> [manually\_configure\_mtls\_truststore](#input\_manually\_configure\_mtls\_truststore) | Manually manage the truststore used for API Gateway mTLS (e.g. for prod environment) | `bool` | `false` | no |

infrastructure/terraform/components/api/variables.tf

@@ -138,9 +138,25 @@ variable "eventpub_control_plane_bus_arn" {
 variable "letter_variant_map" {
   type = map(object({ supplierId = string, specId = string, priority = number, billingId = string }))
   default = {
-    "lv1" = { supplierId = "supplier1", specId = "spec1", priority = 10, billingId = "billing1" },
-    "lv2" = { supplierId = "supplier1", specId = "spec2", priority = 10, billingId = "billing1" },
-    "lv3" = { supplierId = "supplier2", specId = "spec3", priority = 10, billingId = "billing1" }
+    "digitrials-aspiring"             = { supplierId = "supplier1", specId = "digitrials-aspiring", priority = "0", billingId = "digitrials-aspiring-billing" },
+    "digitrials-dmapp"                = { supplierId = "supplier1", specId = "notify-admail", priority = "1", billingId = "notify-admail-billing" },
+    "digitrials-globalminds"          = { supplierId = "supplier1", specId = "digitrials-globalminds", priority = "2", billingId = "digitrials-globalminds-billing" },
+    "digitrials-mymelanoma"           = { supplierId = "supplier1", specId = "digitrials-mymelanoma", priority = "3", billingId = "digitrials-mymelanoma-billing" },
+    "digitrials-ofh"                  = { supplierId = "supplier1", specId = "digitrials-ofh", priority = "4", billingId = "digitrials-ofh-billing" },
+    "digitrials-prostateprogress"     = { supplierId = "supplier1", specId = "digitrials-prostateprogress", priority = "5", billingId = "digitrials-prostateprogress-billing" },
+    "digitrials-protectc"             = { supplierId = "supplier1", specId = "notify-c5-colour", priority = "6", billingId = "notify-c5-colour-billing" },
+    "digitrials-restore"              = { supplierId = "supplier1", specId = "digitrials-restore", priority = "7", billingId = "digitrials-restore-billing" },
+    "gpreg-admail"                    = { supplierId = "supplier1", specId = "notify-admail", priority = "8", billingId = "notify-admail-billing" },
+    "nces-abnormal-results"           = { supplierId = "supplier1", specId = "nces-abnormal-results", priority = "9", billingId = "nces-abnormal-results-billing" },
+    "nces-abnormal-results-braille"   = { supplierId = "supplier1", specId = "nces-abnormal-results-braille", priority = "10", billingId = "nces-abnormal-results-braille-billing" },
+    "nces-invites"                    = { supplierId = "supplier1", specId = "nces-invites", priority = "10", billingId = "nces-invites-billing" },
+    "nces-invites-braille"            = { supplierId = "supplier1", specId = "nces-invites-braille", priority = "10", billingId = "nces-invites-braille-billing" },
+    "nces-standard"                   = { supplierId = "supplier1", specId = "notify-c5-whitemail", priority = "11", billingId = "notify-c5-whitemail-billing" },
+    "nces-standard-braille"           = { supplierId = "supplier1", specId = "notify-braille-whitemail", priority = "12", billingId = "notify-braille-whitemail-billing" },
+    "notify-braille"                  = { supplierId = "supplier1", specId = "notify-braille", priority = "13", billingId = "notify-braille-billing" },
+    "notify-digital-letters-standard" = { supplierId = "supplier1", specId = "notify-c5", priority = "97", billingId = "notify-c5-billing" },
+    "notify-standard"                 = { supplierId = "supplier1", specId = "notify-c5", priority = "98", billingId = "notify-c5-billing" },
+    "notify-standard-colour"          = { supplierId = "supplier1", specId = "notify-c5-colour", priority = "99", billingId = "notify-c5-colour-billing" }
   }
 }
 

internal/event-builders/src/__tests__/letter-mapper.test.ts

@@ -1,4 +1,4 @@
-import { $LetterEvent } from "@nhsdigital/nhs-notify-event-schemas-supplier-api/src";
+import { $LetterStatusChangeEvent } from "@nhsdigital/nhs-notify-event-schemas-supplier-api/src";
 import { Letter } from "@internal/datastore";
 import { mapLetterToCloudEvent } from "../letter-mapper";
 
@@ -22,7 +22,7 @@ describe("letter-mapper", () => {
     const event = mapLetterToCloudEvent(letter, source);
 
     // Check it conforms to the letter event schema - parse will throw an error if not
-    $LetterEvent.parse(event);
+    $LetterStatusChangeEvent.parse(event);
     expect(event.type).toBe("uk.nhs.notify.supplier-api.letter.PRINTED.v1");
     expect(event.dataschema).toBe(
       `https://notify.nhs.uk/cloudevents/schemas/supplier-api/letter.PRINTED.${event.dataschemaversion}.schema.json`,

internal/event-builders/src/letter-mapper.ts

@@ -1,13 +1,13 @@
 import { randomBytes, randomUUID } from "node:crypto";
 import eventSchemaPackage from "@nhsdigital/nhs-notify-event-schemas-supplier-api/package.json";
 import { Letter } from "@internal/datastore";
-import { LetterEvent } from "@nhsdigital/nhs-notify-event-schemas-supplier-api/src/events/letter-events";
+import { LetterStatusChangeEvent } from "@nhsdigital/nhs-notify-event-schemas-supplier-api/src/events/letter-events";
 
 // eslint-disable-next-line import-x/prefer-default-export
 export function mapLetterToCloudEvent(
   letter: Letter,
   source: string,
-): LetterEvent {
+): LetterStatusChangeEvent {
   const eventId = randomUUID();
   const dataschemaversion = eventSchemaPackage.version;
   return {
@@ -21,7 +21,7 @@ export function mapLetterToCloudEvent(
     subject: `letter-origin/letter-rendering/letter/${letter.id}`,
 
     data: {
-      domainId: letter.id as LetterEvent["data"]["domainId"],
+      domainId: letter.id as LetterStatusChangeEvent["data"]["domainId"],
       status: letter.status,
       specificationId: letter.specificationId,
       billingRef: letter.billingRef,

internal/events/package.json

@@ -36,5 +36,5 @@
     "typecheck": "tsc --noEmit"
   },
   "types": "dist/index.d.ts",
-  "version": "1.0.17"
+  "version": "1.0.18"
 }

internal/events/src/cli/generate-json.ts

@@ -2,7 +2,7 @@ import { z } from "zod";
 import * as fs from "node:fs";
 import { $Letter } from "@nhsdigital/nhs-notify-event-schemas-supplier-api/src/domain/letter";
 import {
-  $LetterEvent,
+  $LetterStatusChangeEvent,
   letterEventMap,
 } from "@nhsdigital/nhs-notify-event-schemas-supplier-api/src/events/letter-events";
 import { $MISubmittedEvent } from "@nhsdigital/nhs-notify-event-schemas-supplier-api/src/events/mi-events";
@@ -36,7 +36,7 @@ for (const [key, schema] of Object.entries(letterEventMap)) {
 }
 
 // Generic letter status change event schema
-const json = z.toJSONSchema($LetterEvent, {
+const json = z.toJSONSchema($LetterStatusChangeEvent, {
   io: "input",
   target: "openapi-3.0",
   reused: "ref",

internal/events/src/events/letter-events.ts

@@ -9,7 +9,7 @@ import { EventEnvelope } from "@nhsdigital/nhs-notify-event-schemas-supplier-api
 /**
  * A generic schema for parsing any letter status change event
  */
-export const $LetterEvent = EventEnvelope(
+export const $LetterStatusChangeEvent = EventEnvelope(
   "letter",
   "letter",
   $Letter,
@@ -19,7 +19,7 @@ export const $LetterEvent = EventEnvelope(
   title: `letter.* Event`,
   description: `Event schema for generic letter status change`,
 });
-export type LetterEvent = z.infer<typeof $LetterEvent>;
+export type LetterStatusChangeEvent = z.infer<typeof $LetterStatusChangeEvent>;
 
 /**
  * Specialise the generic event schema for a single status

lambdas/api-handler/src/handlers/amendment-event-transformer.ts

@@ -1,6 +1,6 @@
 import { SQSBatchItemFailure, SQSEvent, SQSHandler } from "aws-lambda";
 import { PublishCommand } from "@aws-sdk/client-sns";
-import { LetterEvent } from "@nhsdigital/nhs-notify-event-schemas-supplier-api/src/events/letter-events";
+import { LetterStatusChangeEvent } from "@nhsdigital/nhs-notify-event-schemas-supplier-api/src/events/letter-events";
 import { MetricEntry, MetricStatus, buildEMFObject } from "@internal/helpers";
 import { mapLetterToCloudEvent } from "@internal/event-builders/src";
 import { Unit } from "aws-embedded-metrics";
@@ -66,7 +66,7 @@ export default function createTransformAmendmentEventHandler(
 }
 
 function buildSnsCommand(
-  letterEvent: LetterEvent,
+  letterEvent: LetterStatusChangeEvent,
   topicArn: string,
 ): PublishCommand {
   return new PublishCommand({

lambdas/letter-updates-transformer/src/letter-updates-transformer.ts

@@ -9,7 +9,7 @@ import {
   PublishBatchCommand,
   PublishBatchRequestEntry,
 } from "@aws-sdk/client-sns";
-import { LetterEvent } from "@nhsdigital/nhs-notify-event-schemas-supplier-api/src";
+import { LetterStatusChangeEvent } from "@nhsdigital/nhs-notify-event-schemas-supplier-api/src";
 import { MetricEntry, buildEMFObject } from "@internal/helpers";
 import { Letter, LetterSchema } from "@internal/datastore";
 import { mapLetterToCloudEvent } from "@internal/event-builders/src";
@@ -33,7 +33,7 @@ export default function createHandler(deps: Deps): Handler<KinesisStreamEvent> {
       extractPayload(record, deps),
     );
 
-    const cloudEvents: LetterEvent[] = ddbRecords
+    const cloudEvents: LetterStatusChangeEvent[] = ddbRecords
       .filter((record) => filterRecord(record, deps))
       .map((element) => extractNewLetter(element))
       .map((element) => mapLetterToCloudEvent(element, deps.env.EVENT_SOURCE));
@@ -59,7 +59,7 @@ export default function createHandler(deps: Deps): Handler<KinesisStreamEvent> {
   };
 }
 
-function populateEventTypeMap(cloudEvents: LetterEvent[]) {
+function populateEventTypeMap(cloudEvents: LetterStatusChangeEvent[]) {
   const evtMap = new Map<string, number>();
   for (const event of cloudEvents) {
     evtMap.set(event.type, (evtMap.get(event.type) || 0) + 1);
@@ -142,14 +142,14 @@ function extractNewLetter(record: DynamoDBRecord): Letter {
   return LetterSchema.parse(unmarshall(newImage as any));
 }
 
-function* generateBatches(events: LetterEvent[]) {
+function* generateBatches(events: LetterStatusChangeEvent[]) {
   for (let i = 0; i < events.length; i += BATCH_SIZE) {
     yield events.slice(i, i + BATCH_SIZE);
   }
 }
 
 function buildMessage(
-  event: LetterEvent,
+  event: LetterStatusChangeEvent,
   index: number,
 ): PublishBatchRequestEntry {
   return {

lambdas/supplier-allocator/src/handler/__tests__/allocate-handler.test.ts

@@ -4,8 +4,8 @@ import { SQSClient, SendMessageCommand } from "@aws-sdk/client-sqs";
 import { LetterRequestPreparedEventV2 } from "@nhsdigital/nhs-notify-event-schemas-letter-rendering";
 import { LetterRequestPreparedEvent } from "@nhsdigital/nhs-notify-event-schemas-letter-rendering-v1";
 import {
-  $LetterEvent,
-  LetterEvent,
+  $LetterStatusChangeEvent,
+  LetterStatusChangeEvent,
 } from "@nhsdigital/nhs-notify-event-schemas-supplier-api/src/events/letter-events";
 import { SupplierConfigRepository } from "@internal/datastore";
 import createSupplierAllocatorHandler from "../allocate-handler";
@@ -99,10 +99,10 @@ function createPreparedV2Event(
 
 function createSupplierStatusChangeEvent(
   overrides: Partial<any> = {},
-): LetterEvent {
+): LetterStatusChangeEvent {
   const now = new Date().toISOString();
 
-  return $LetterEvent.parse({
+  return $LetterStatusChangeEvent.parse({
     data: {
       domainId: overrides.domainId ?? "f47ac10b-58cc-4372-a567-0e02b2c3d479",
       groupId: "client_template",