CCM-12997 CCM-13001 CCM-13697: Add upsert lambda (#280)

* Add upsert lambda

* Fix names

* Fix unit test

* Fix unit test dates

* Fix lambda handler name export

* Add event-schemas-letter-rendering

* Add GitHub NPM registry configuration to .npmrc

* Add version check and Node.js setup to CI workflow

* TO REVERT AFTER TEST: conditional check for event schema version update

* Refactor event schema version checks in CI workflow

* Add GitHub NPM registry URL to Node.js setup in workflows

* Log npm response

* Add NODE_AUTH_TOKEN to GitHub Actions environment for NPM access

* Revert "Add GitHub NPM registry URL to Node.js setup in workflows"

This reverts commit f1444fd.

* Move NODE_AUTH_TOKEN to npm view step in CI workflow

* Reapply "Add GitHub NPM registry URL to Node.js setup in workflows"

This reverts commit 5741a86.

* Add NODE_AUTH_TOKEN to NPM install steps in workflows

* Re-enable conditional check on published package check

* Add NODE_AUTH_TOKEN input for GitHub package registry access in workflows

* Add packages:read to permissions

* delete old test

* fix trivy scan step

* Fix gh pkg registry test stage

* fix local npmrc

* Fix unit test timestamps, add source

* Add notification parsing logic

* fix tf var

* Fix ddb permissions

* Add latest version event schema letter rendering

* Add trying v1 parse if v2 fails

* Add subject

* Split upsert operations

* Letter variant map to vars

* Revert supplierStatusSk

* Fix dependencies

* CCM-13697: Letters Key and Source Data Item (#323)

* swap partition and sort for letters

* store changes and source, subject, billingref

* unsafe marker

* rename correctly

* missing close brace

* minor refactor upsert unit tests

* remove dup line

* gitleaksignore

* bump package version

* Fix get API for component tests

---------

Co-authored-by: Mike Houston <michael.houston4@nhs.net>
Co-authored-by: Mike Houston <60653100+m-houston@users.noreply.github.com>
Co-authored-by: Mark Slowey <113013138+masl2@users.noreply.github.com>
Co-authored-by: Mark Slowey <mark.slowey1@nhs.net>
Co-authored-by: Steve Buxton <steve.buxton@nhs.net>

Author: 4 people

.devcontainer/devcontainer.json

@@ -83,7 +83,8 @@
   },
   "mounts": [
     "source=${localEnv:HOME}/.ssh,target=/home/vscode/.ssh,type=bind,consistency=cached",
-    "source=${localEnv:HOME}/.aws,target=/home/vscode/.aws,type=bind,consistency=cached"
+    "source=${localEnv:HOME}/.aws,target=/home/vscode/.aws,type=bind,consistency=cached",
+    "source=${localEnv:HOME}/.npmrc,target=/home/vscode/.npmrc,type=bind,consistency=cached"
   ],
   "name": "Devcontainer",
   "postCreateCommand": "scripts/devcontainer/postcreatecommand.sh"

.github/workflows/stage-1-commit.yaml

@@ -76,7 +76,7 @@ jobs:
     needs: detect-terraform-changes
     if: needs.detect-terraform-changes.outputs.terraform_changed == 'true'
     permissions:
-        contents: write
+      contents: write
     steps:
       - name: "Checkout code"
         uses: actions/checkout@v5
@@ -317,7 +317,7 @@ jobs:
         uses: actions/setup-node@v4
         with:
           node-version: ${{ inputs.nodejs_version }}
-          registry-url: 'https://npm.pkg.github.com'
+          registry-url: "https://npm.pkg.github.com"
 
       - name: check if local version differs from latest published version
         id: check-version

.github/workflows/stage-2-test.yaml

@@ -38,7 +38,7 @@ env:
 
 permissions:
   id-token: write # This is required for requesting the JWT
-  contents: read  # This is required for actions/checkout
+  contents: read # This is required for actions/checkout
   packages: read # This is required for downloading from GitHub Package Registry
 
 jobs:
@@ -49,6 +49,11 @@ jobs:
     steps:
       - name: "Checkout code"
         uses: actions/checkout@v5
+      - name: Setup NodeJS
+        uses: actions/setup-node@v4
+        with:
+          node-version: ${{ inputs.nodejs_version }}
+          registry-url: "https://npm.pkg.github.com"
       - name: "Cache node_modules"
         uses: actions/cache@v4
         with:
@@ -73,6 +78,11 @@ jobs:
     steps:
       - name: "Checkout code"
         uses: actions/checkout@v5
+      - name: Setup NodeJS
+        uses: actions/setup-node@v4
+        with:
+          node-version: ${{ inputs.nodejs_version }}
+          registry-url: "https://npm.pkg.github.com"
       - name: "Cache node_modules"
         uses: actions/cache@v4
         with:
@@ -142,6 +152,11 @@ jobs:
     steps:
       - name: "Checkout code"
         uses: actions/checkout@v5
+      - name: Setup NodeJS
+        uses: actions/setup-node@v4
+        with:
+          node-version: ${{ inputs.nodejs_version }}
+          registry-url: "https://npm.pkg.github.com"
       - name: "Cache node_modules"
         uses: actions/cache@v4
         with:
@@ -168,6 +183,11 @@ jobs:
     steps:
       - name: "Checkout code"
         uses: actions/checkout@v5
+      - name: Setup NodeJS
+        uses: actions/setup-node@v4
+        with:
+          node-version: ${{ inputs.nodejs_version }}
+          registry-url: "https://npm.pkg.github.com"
       - name: "Cache node_modules"
         uses: actions/cache@v4
         with:

.gitleaksignore

@@ -21,3 +21,4 @@ e12407e09151898bfd8d049d57eee9db9977d56b:.github/copilot-instructions.md:generic
 82cf3b2e89ea24b97c4ffc09e618700fb1b0aff3:pact-contracts/pacts/letter-rendering/supplier-api-letter-request-prepared.json:generic-api-key:10
 82f6be3e657b46d8447e77cdc1894fba0b855c26:tests/component-tests/testCases/create-letter-request.spec.ts:generic-api-key:10
 debc75a97cfe551a69fd1e8694be483213322a9d:pact-contracts/pacts/letter-rendering/supplier-api-letter-request-prepared.json:generic-api-key:10
+777eb4047ad06b9e939a292ee18664a0ffee4f29:tests/resources/prepared-letter.json:generic-api-key:4

infrastructure/terraform/components/api/README.md

@@ -26,6 +26,7 @@ No requirements.
 | <a name="input_group"></a> [group](#input\_group) | The group variables are being inherited from (often synonmous with account short-name) | `string` | n/a | yes |
 | <a name="input_kms_deletion_window"></a> [kms\_deletion\_window](#input\_kms\_deletion\_window) | When a kms key is deleted, how long should it wait in the pending deletion state? | `string` | `"30"` | no |
 | <a name="input_letter_table_ttl_hours"></a> [letter\_table\_ttl\_hours](#input\_letter\_table\_ttl\_hours) | Number of hours to set as TTL on letters table | `number` | `24` | no |
+| <a name="input_letter_variant_map"></a> [letter\_variant\_map](#input\_letter\_variant\_map) | n/a | `map(object({ supplierId = string, specId = string }))` | <pre>{<br/>  "lv1": {<br/>    "specId": "spec1",<br/>    "supplierId": "supplier1"<br/>  },<br/>  "lv2": {<br/>    "specId": "spec2",<br/>    "supplierId": "supplier1"<br/>  },<br/>  "lv3": {<br/>    "specId": "spec3",<br/>    "supplierId": "supplier2"<br/>  }<br/>}</pre> | no |
 | <a name="input_log_level"></a> [log\_level](#input\_log\_level) | The log level to be used in lambda functions within the component. Any log with a lower severity than the configured value will not be logged: https://docs.python.org/3/library/logging.html#levels | `string` | `"INFO"` | no |
 | <a name="input_log_retention_in_days"></a> [log\_retention\_in\_days](#input\_log\_retention\_in\_days) | The retention period in days for the Cloudwatch Logs events to be retained, default of 0 is indefinite | `number` | `0` | no |
 | <a name="input_manually_configure_mtls_truststore"></a> [manually\_configure\_mtls\_truststore](#input\_manually\_configure\_mtls\_truststore) | Manually manage the truststore used for API Gateway mTLS (e.g. for prod environment) | `bool` | `false` | no |

infrastructure/terraform/components/api/ddb_table_letters.tf

@@ -2,8 +2,8 @@ resource "aws_dynamodb_table" "letters" {
   name         = "${local.csi}-letters"
   billing_mode = "PAY_PER_REQUEST"
 
-  hash_key  = "supplierId"
-  range_key = "id"
+  hash_key  = "id"
+  range_key = "supplierId"
 
   ttl {
     attribute_name = "ttl"

infrastructure/terraform/components/api/module_lambda_upsert_letter.tf

@@ -1,7 +1,7 @@
 module "upsert_letter" {
   source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip"
 
-  function_name = "upsert-letter"
+  function_name = "upsert_letter"
   description   = "Update or Insert the letter data in the letters table"
 
   aws_account_id = var.aws_account_id
@@ -22,7 +22,7 @@ module "upsert_letter" {
   function_code_base_path = local.aws_lambda_functions_dir_path
   function_code_dir       = "upsert-letter/dist"
   function_include_common = true
-  handler_function_name   = "handler"
+  handler_function_name   = "upsertLetterHandler"
   runtime                 = "nodejs22.x"
   memory                  = 128
   timeout                 = 29
@@ -34,7 +34,9 @@ module "upsert_letter" {
   log_destination_arn       = local.destination_arn
   log_subscription_role_arn = local.acct.log_subscription_role_arn
 
-  lambda_env_vars = merge(local.common_lambda_env_vars, {})
+  lambda_env_vars = merge(local.common_lambda_env_vars, {
+    VARIANT_MAP = jsonencode(var.letter_variant_map)
+  })
 }
 
 data "aws_iam_policy_document" "upsert_letter_lambda" {
@@ -57,7 +59,10 @@ data "aws_iam_policy_document" "upsert_letter_lambda" {
     effect = "Allow"
 
     actions = [
-      "dynamodb:PutItem"
+      "dynamodb:PutItem",
+      "dynamodb:GetItem",
+      "dynamodb:Query",
+      "dynamodb:UpdateItem"
     ]
 
     resources = [

infrastructure/terraform/components/api/variables.tf

@@ -141,6 +141,15 @@ variable "eventpub_control_plane_bus_arn" {
   default     = ""
 }
 
+variable "letter_variant_map" {
+  type = map(object({ supplierId = string, specId = string }))
+  default = {
+    "lv1" = { supplierId = "supplier1", specId = "spec1" },
+    "lv2" = { supplierId = "supplier1", specId = "spec2" },
+    "lv3" = { supplierId = "supplier2", specId = "spec3" }
+  }
+}
+
 variable "disable_gateway_execute_endpoint" {
   type        = bool
   description = "Disable the execution endpoint for the API Gateway"
@@ -157,4 +166,5 @@ variable "core_environment" {
   type        = string
   description = "Environment of Core"
   default     = "prod"
+
 }

internal/datastore/src/__test__/db.ts

@@ -51,8 +51,8 @@ const createLetterTableCommand = new CreateTableCommand({
   TableName: "letters",
   BillingMode: "PAY_PER_REQUEST",
   KeySchema: [
-    { AttributeName: "supplierId", KeyType: "HASH" }, // Partition key
-    { AttributeName: "id", KeyType: "RANGE" }, // Sort key
+    { AttributeName: "id", KeyType: "HASH" }, // Partition key (letter ID)
+    { AttributeName: "supplierId", KeyType: "RANGE" }, // Sort key
   ],
   GlobalSecondaryIndexes: [
     {

internal/datastore/src/__test__/heathcheck.test.ts

@@ -24,6 +24,10 @@ describe("DBHealthcheck", () => {
     await deleteTables(db);
   });
 
+  afterAll(async () => {
+    await db.container.stop();
+  });
+
   it("passes when the database is available", async () => {
     const dbHealthCheck = new DBHealthcheck(db.docClient, db.config);
     await expect(dbHealthCheck.check()).resolves.not.toThrow();