Skip to content

Commit c277f10

Browse files
nhsd-david-wassmasl2
nhsd-david-wass
and
masl2
authored
Feature/ccm 12952 publish mi events (#275)
* CM-12952 - MI Updates Transformer * Added terraform and stream forwarder * Unique stream name * copy fixes from base commit * new envelope fields * logging * lint fixes * update lackage lock * fix imports * back out export changes * Remove stream forwarder and pass direct from db to kinesis * rebase updates * logging and refactor * logging for investigation * fix extract payload * fix mapping * Fix default export in index.ts * rebase fix * review changes * trivy job setup of node for nhs packages * tsconfig to use project base * Add guard to non insert events * add NODE_AUTH_TOKEN to workflow script * possible unit test fix and update letter-rendering schema to 2.0.1 --------- Co-authored-by: Mark Slowey <mark.slowey1@nhs.net>
1 parent 5c7e01d commit c277f10

19 files changed

Lines changed: 7422 additions & 10343 deletions

File tree

‎.github/workflows/stage-2-test.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,8 @@ jobs:
7575
name: "Unit tests"
7676
runs-on: ubuntu-latest
7777
timeout-minutes: 5
78+
env:
79+
NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
7880
steps:
7981
- name: "Checkout code"
8082
uses: actions/checkout@v5
@@ -149,6 +151,8 @@ jobs:
149151
name: "Linting"
150152
runs-on: ubuntu-latest
151153
timeout-minutes: 5
154+
env:
155+
NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
152156
steps:
153157
- name: "Checkout code"
154158
uses: actions/checkout@v5
@@ -180,6 +184,8 @@ jobs:
180184
name: "Typecheck"
181185
runs-on: ubuntu-latest
182186
timeout-minutes: 5
187+
env:
188+
NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
183189
steps:
184190
- name: "Checkout code"
185191
uses: actions/checkout@v5

‎infrastructure/terraform/components/api/README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ No requirements.
5252
| <a name="module_letter_status_updates_queue"></a> [letter\_status\_updates\_queue](#module\_letter\_status\_updates\_queue) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.24/terraform-sqs.zip | n/a |
5353
| <a name="module_letter_updates_transformer"></a> [letter\_updates\_transformer](#module\_letter\_updates\_transformer) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a |
5454
| <a name="module_logging_bucket"></a> [logging\_bucket](#module\_logging\_bucket) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.26/terraform-s3bucket.zip | n/a |
55+
| <a name="module_mi_updates_transformer"></a> [mi\_updates\_transformer](#module\_mi\_updates\_transformer) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.26/terraform-lambda.zip | n/a |
5556
| <a name="module_patch_letter"></a> [patch\_letter](#module\_patch\_letter) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a |
5657
| <a name="module_post_letters"></a> [post\_letters](#module\_post\_letters) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a |
5758
| <a name="module_post_mi"></a> [post\_mi](#module\_post\_mi) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a |

‎infrastructure/terraform/components/api/event_source_mapping_mi_updates.tf‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
resource "aws_lambda_event_source_mapping" "mi_updates_transformer_kinesis" {
2+
event_source_arn = aws_kinesis_stream.mi_change_stream.arn
3+
function_name = module.mi_updates_transformer.function_arn
4+
starting_position = "LATEST"
5+
batch_size = 10
6+
maximum_batching_window_in_seconds = 1
7+
8+
depends_on = [
9+
module.mi_updates_transformer # ensures updates transformer exists
10+
]
11+
}

‎infrastructure/terraform/components/api/kinesis_mi_change_stream.tf‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
resource "aws_kinesis_stream" "mi_change_stream" {
2+
name = "${local.csi}-mi-change-stream"
3+
shard_count = 1
4+
retention_period = 24
5+
}
6+
7+
resource "aws_dynamodb_kinesis_streaming_destination" "mi_streaming_destination" {
8+
stream_arn = aws_kinesis_stream.mi_change_stream.arn
9+
table_name = aws_dynamodb_table.mi.name
10+
approximate_creation_date_time_precision = "MILLISECOND"
11+
}

‎infrastructure/terraform/components/api/module_lambda_mi_updates_transformer.tf‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
module "mi_updates_transformer" {
2+
source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.26/terraform-lambda.zip"
3+
4+
function_name = "mi-updates-transformer"
5+
description = "MI Update Filter/Producer"
6+
7+
aws_account_id = var.aws_account_id
8+
component = var.component
9+
environment = var.environment
10+
project = var.project
11+
region = var.region
12+
group = var.group
13+
14+
log_retention_in_days = var.log_retention_in_days
15+
kms_key_arn = module.kms.key_arn
16+
17+
iam_policy_document = {
18+
body = data.aws_iam_policy_document.mi_updates_transformer_lambda.json
19+
}
20+
21+
function_s3_bucket = local.acct.s3_buckets["lambda_function_artefacts"]["id"]
22+
function_code_base_path = local.aws_lambda_functions_dir_path
23+
function_code_dir = "mi-updates-transformer/dist"
24+
function_include_common = true
25+
handler_function_name = "handler"
26+
runtime = "nodejs22.x"
27+
memory = 128
28+
timeout = 29
29+
log_level = var.log_level
30+
31+
force_lambda_code_deploy = var.force_lambda_code_deploy
32+
enable_lambda_insights = false
33+
34+
send_to_firehose = true
35+
log_destination_arn = local.destination_arn
36+
log_subscription_role_arn = local.acct.log_subscription_role_arn
37+
38+
lambda_env_vars = merge(local.common_lambda_env_vars, {
39+
EVENTPUB_SNS_TOPIC_ARN = "${module.eventpub.sns_topic.arn}"
40+
})
41+
}
42+
43+
data "aws_iam_policy_document" "mi_updates_transformer_lambda" {
44+
statement {
45+
sid = "AllowSNSPublish"
46+
effect = "Allow"
47+
48+
actions = [
49+
"sns:Publish"
50+
]
51+
52+
resources = [
53+
module.eventpub.sns_topic.arn
54+
]
55+
}
56+
57+
statement {
58+
sid = "AllowKinesisGet"
59+
effect = "Allow"
60+
61+
actions = [
62+
"kinesis:GetRecords",
63+
"kinesis:GetShardIterator",
64+
"kinesis:DescribeStream",
65+
"kinesis:DescribeStreamSummary",
66+
"kinesis:ListShards",
67+
"kinesis:ListStreams",
68+
]
69+
70+
resources = [
71+
aws_kinesis_stream.mi_change_stream.arn
72+
]
73+
}
74+
}

‎lambdas/api-handler/package.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,8 @@
1919
"jest": "^30.2.0",
2020
"jest-mock-extended": "^4.0.0",
2121
"ts-jest": "^29.4.0",
22-
"typescript": "^5.9.3"
22+
"typescript": "^5.9.3",
23+
"zod": "^4.1.11"
2324
},
2425
"name": "nhs-notify-supplier-api-handler",
2526
"private": true,

‎lambdas/mi-updates-transformer/jest.config.ts‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
import type { Config } from "jest";
2+
3+
export const baseJestConfig: Config = {
4+
preset: "ts-jest",
5+
6+
// Automatically clear mock calls, instances, contexts and results before every test
7+
clearMocks: true,
8+
9+
// Indicates whether the coverage information should be collected while executing the test
10+
collectCoverage: true,
11+
12+
// The directory where Jest should output its coverage files
13+
coverageDirectory: "./.reports/unit/coverage",
14+
15+
// Indicates which provider should be used to instrument code for coverage
16+
coverageProvider: "babel",
17+
18+
coverageThreshold: {
19+
global: {
20+
branches: 100,
21+
functions: 100,
22+
lines: 100,
23+
statements: -10,
24+
},
25+
},
26+
27+
coveragePathIgnorePatterns: ["/__tests__/"],
28+
transform: { "^.+\\.ts$": "ts-jest" },
29+
testPathIgnorePatterns: [".build"],
30+
testMatch: ["**/?(*.)+(spec|test).[jt]s?(x)"],
31+
32+
// Use this configuration option to add custom reporters to Jest
33+
reporters: [
34+
"default",
35+
[
36+
"jest-html-reporter",
37+
{
38+
pageTitle: "Test Report",
39+
outputPath: "./.reports/unit/test-report.html",
40+
includeFailureMsg: true,
41+
},
42+
],
43+
],
44+
45+
// The test environment that will be used for testing
46+
testEnvironment: "jsdom",
47+
};
48+
49+
const utilsJestConfig = {
50+
...baseJestConfig,
51+
52+
testEnvironment: "node",
53+
54+
coveragePathIgnorePatterns: [
55+
...(baseJestConfig.coveragePathIgnorePatterns ?? []),
56+
"zod-validators.ts",
57+
],
58+
};
59+
60+
export default utilsJestConfig;

‎lambdas/mi-updates-transformer/package.json‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
{
2+
"dependencies": {
3+
"@aws-sdk/client-sns": "^3.954.0",
4+
"@aws-sdk/util-dynamodb": "^3.943.0",
5+
"@internal/datastore": "^0.1.0",
6+
"@nhsdigital/nhs-notify-event-schemas-supplier-api": "*",
7+
"aws-lambda": "^1.0.7",
8+
"esbuild": "^0.24.0",
9+
"pino": "^10.1.0",
10+
"zod": "^4.1.13"
11+
},
12+
"devDependencies": {
13+
"@tsconfig/node22": "^22.0.2",
14+
"@types/aws-lambda": "^8.10.148",
15+
"@types/jest": "^30.0.0",
16+
"jest": "^30.2.0",
17+
"jest-mock-extended": "^4.0.0",
18+
"typescript": "^5.8.3"
19+
},
20+
"name": "nhs-notify-supplier-api-mi-updates-transformer",
21+
"private": true,
22+
"scripts": {
23+
"lambda-build": "rm -rf dist && npx esbuild --bundle --minify --sourcemap --target=es2020 --platform=node --loader:.node=file --entry-names=[name] --outdir=dist src/index.ts",
24+
"lint": "eslint .",
25+
"lint:fix": "eslint . --fix",
26+
"test:unit": "jest",
27+
"typecheck": "tsc --noEmit"
28+
},
29+
"version": "0.0.1"
30+
}

0 commit comments

Comments
 (0)