Skip to content

Commit 2778623

Browse files
authored
Feature/ccm 12934 create letter queue (#410)
* CCM-12934 Add LetterQueue table * Fix metrics * Switch table indexes around * Fix resource name for lambda * Fix resource name * More terraform fixes * Fix return value in case of error * Fix LSI name
1 parent c7a8a06 commit 2778623

23 files changed

Lines changed: 9061 additions & 13200 deletions

infrastructure/terraform/components/api/README.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@ No requirements.
6666
| <a name="module_sqs_supplier_allocator"></a> [sqs\_supplier\_allocator](#module\_sqs\_supplier\_allocator) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.26/terraform-sqs.zip | n/a |
6767
| <a name="module_supplier_allocator"></a> [supplier\_allocator](#module\_supplier\_allocator) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a |
6868
| <a name="module_supplier_ssl"></a> [supplier\_ssl](#module\_supplier\_ssl) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.26/terraform-ssl.zip | n/a |
69+
| <a name="module_update_letter_queue"></a> [update\_letter\_queue](#module\_update\_letter\_queue) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a |
6970
| <a name="module_upsert_letter"></a> [upsert\_letter](#module\_upsert\_letter) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a |
7071
## Outputs
7172

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
resource "aws_dynamodb_table" "letter_queue" {
2+
name = "${local.csi}-letter-queue"
3+
billing_mode = "PAY_PER_REQUEST"
4+
5+
hash_key = "supplierId"
6+
range_key = "queueTimestamp"
7+
8+
ttl {
9+
attribute_name = "ttl"
10+
enabled = true
11+
}
12+
13+
local_secondary_index {
14+
name = "letterId-index"
15+
range_key = "letterId"
16+
projection_type = "ALL"
17+
}
18+
19+
attribute {
20+
name = "supplierId"
21+
type = "S"
22+
}
23+
24+
attribute {
25+
name = "letterId"
26+
type = "S"
27+
}
28+
29+
attribute {
30+
name = "queueTimestamp"
31+
type = "S"
32+
}
33+
34+
point_in_time_recovery {
35+
enabled = true
36+
}
37+
38+
tags = merge(
39+
local.default_tags,
40+
{
41+
NHSE-Enable-Dynamo-Backup-Acct = "True"
42+
}
43+
)
44+
}
Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
resource "aws_lambda_event_source_mapping" "update_letter_queue_kinesis" {
2+
event_source_arn = aws_kinesis_stream.letter_change_stream.arn
3+
function_name = module.update_letter_queue.function_arn
4+
starting_position = "LATEST"
5+
batch_size = 10
6+
maximum_batching_window_in_seconds = 1
7+
8+
depends_on = [
9+
module.update_letter_queue # ensures update letter queue lambda exists
10+
]
11+
}
Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
module "update_letter_queue" {
2+
source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip"
3+
4+
function_name = "update_letter_queue"
5+
description = "Populates the letter queue table with new pending letters from the letter change stream"
6+
7+
aws_account_id = var.aws_account_id
8+
component = var.component
9+
environment = var.environment
10+
project = var.project
11+
region = var.region
12+
group = var.group
13+
14+
log_retention_in_days = var.log_retention_in_days
15+
kms_key_arn = module.kms.key_arn
16+
17+
iam_policy_document = {
18+
body = data.aws_iam_policy_document.update_letter_queue_lambda.json
19+
}
20+
21+
function_s3_bucket = local.acct.s3_buckets["lambda_function_artefacts"]["id"]
22+
function_code_base_path = local.aws_lambda_functions_dir_path
23+
function_code_dir = "update-letter-queue/dist"
24+
function_include_common = true
25+
handler_function_name = "handler"
26+
runtime = "nodejs22.x"
27+
memory = 512
28+
timeout = 29
29+
log_level = var.log_level
30+
31+
force_lambda_code_deploy = var.force_lambda_code_deploy
32+
enable_lambda_insights = false
33+
34+
log_destination_arn = local.destination_arn
35+
log_subscription_role_arn = local.acct.log_subscription_role_arn
36+
37+
lambda_env_vars = merge(local.common_lambda_env_vars, {
38+
LETTER_QUEUE_TABLE_NAME = aws_dynamodb_table.letter_queue.name,
39+
LETTER_QUEUE_TTL_HOURS = 168 # 7 days
40+
})
41+
}
42+
43+
data "aws_iam_policy_document" "update_letter_queue_lambda" {
44+
statement {
45+
sid = "AllowDynamoDBWrite"
46+
effect = "Allow"
47+
48+
actions = [
49+
"dynamodb:PutItem",
50+
]
51+
52+
resources = [
53+
aws_dynamodb_table.letter_queue.arn,
54+
"${aws_dynamodb_table.letter_queue.arn}/index/*"
55+
]
56+
}
57+
58+
statement {
59+
sid = "AllowKinesisGet"
60+
effect = "Allow"
61+
62+
actions = [
63+
"kinesis:GetRecords",
64+
"kinesis:GetShardIterator",
65+
"kinesis:DescribeStream",
66+
"kinesis:DescribeStreamSummary",
67+
"kinesis:ListShards",
68+
"kinesis:ListStreams",
69+
]
70+
71+
resources = [
72+
aws_kinesis_stream.letter_change_stream.arn
73+
]
74+
}
75+
}

internal/datastore/src/__test__/db.ts

Lines changed: 41 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -30,9 +30,11 @@ export async function setupDynamoDBContainer() {
3030
region: "us-west-2",
3131
endpoint,
3232
lettersTableName: "letters",
33+
letterQueueTableName: "letter-queue",
3334
miTableName: "management-info",
3435
suppliersTableName: "suppliers",
3536
lettersTtlHours: 1,
37+
letterQueueTtlHours: 1,
3638
miTtlHours: 1,
3739
};
3840

@@ -118,6 +120,32 @@ const createSupplierTableCommand = new CreateTableCommand({
118120
],
119121
});
120122

123+
const createLetterQueueTableCommand = new CreateTableCommand({
124+
TableName: "letter-queue",
125+
BillingMode: "PAY_PER_REQUEST",
126+
KeySchema: [
127+
{ AttributeName: "supplierId", KeyType: "HASH" }, // Partition key
128+
{ AttributeName: "letterId", KeyType: "RANGE" }, // Sort key
129+
],
130+
LocalSecondaryIndexes: [
131+
{
132+
IndexName: "timestamp-index",
133+
KeySchema: [
134+
{ AttributeName: "supplierId", KeyType: "HASH" }, // Partition key for LSI
135+
{ AttributeName: "queueTimestamp", KeyType: "RANGE" }, // Sort key for LSI
136+
],
137+
Projection: {
138+
ProjectionType: "ALL",
139+
},
140+
},
141+
],
142+
AttributeDefinitions: [
143+
{ AttributeName: "supplierId", AttributeType: "S" },
144+
{ AttributeName: "letterId", AttributeType: "S" },
145+
{ AttributeName: "queueTimestamp", AttributeType: "S" },
146+
],
147+
});
148+
121149
export async function createTables(context: DBContext) {
122150
const { ddbClient } = context;
123151

@@ -126,26 +154,22 @@ export async function createTables(context: DBContext) {
126154

127155
await ddbClient.send(createMITableCommand);
128156
await ddbClient.send(createSupplierTableCommand);
157+
await ddbClient.send(createLetterQueueTableCommand);
129158
}
130159

131160
export async function deleteTables(context: DBContext) {
132161
const { ddbClient } = context;
133162

134-
await ddbClient.send(
135-
new DeleteTableCommand({
136-
TableName: "letters",
137-
}),
138-
);
139-
140-
await ddbClient.send(
141-
new DeleteTableCommand({
142-
TableName: "management-info",
143-
}),
144-
);
145-
146-
await ddbClient.send(
147-
new DeleteTableCommand({
148-
TableName: "suppliers",
149-
}),
150-
);
163+
for (const tableName of [
164+
"letters",
165+
"management-info",
166+
"suppliers",
167+
"letter-queue",
168+
]) {
169+
await ddbClient.send(
170+
new DeleteTableCommand({
171+
TableName: tableName,
172+
}),
173+
);
174+
}
151175
}
Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
import { Logger } from "pino";
2+
import {
3+
DBContext,
4+
createTables,
5+
deleteTables,
6+
setupDynamoDBContainer,
7+
} from "./db";
8+
import LetterQueueRepository from "../letter-queue-repository";
9+
import { InsertPendingLetter } from "../types";
10+
import { LetterAlreadyExistsError } from "../errors";
11+
import { createTestLogger } from "./logs";
12+
13+
function createLetter(letterId = "letter1"): InsertPendingLetter {
14+
return {
15+
letterId,
16+
supplierId: "supplier1",
17+
specificationId: "specification1",
18+
groupId: "group1",
19+
};
20+
}
21+
22+
// Database tests can take longer, especially with setup and teardown
23+
jest.setTimeout(30_000);
24+
25+
describe("LetterQueueRepository", () => {
26+
let db: DBContext;
27+
let letterQueueRepository: LetterQueueRepository;
28+
let logger: Logger;
29+
30+
beforeAll(async () => {
31+
db = await setupDynamoDBContainer();
32+
});
33+
34+
beforeEach(async () => {
35+
await createTables(db);
36+
({ logger } = createTestLogger());
37+
38+
letterQueueRepository = new LetterQueueRepository(
39+
db.docClient,
40+
logger,
41+
db.config,
42+
);
43+
});
44+
45+
afterEach(async () => {
46+
await deleteTables(db);
47+
jest.useRealTimers();
48+
});
49+
50+
afterAll(async () => {
51+
await db.container.stop();
52+
});
53+
54+
function assertTtl(ttl: number, before: number, after: number) {
55+
const expectedLower = Math.floor(
56+
before / 1000 + 60 * 60 * db.config.letterQueueTtlHours,
57+
);
58+
const expectedUpper = Math.floor(
59+
after / 1000 + 60 * 60 * db.config.lettersTtlHours,
60+
);
61+
expect(ttl).toBeGreaterThanOrEqual(expectedLower);
62+
expect(ttl).toBeLessThanOrEqual(expectedUpper);
63+
}
64+
65+
describe("putLetter", () => {
66+
it("adds a letter to the database", async () => {
67+
const before = Date.now();
68+
69+
const pendingLetter =
70+
await letterQueueRepository.putLetter(createLetter());
71+
72+
const after = Date.now();
73+
74+
const timestampInMillis = new Date(
75+
pendingLetter.queueTimestamp,
76+
).valueOf();
77+
expect(timestampInMillis).toBeGreaterThanOrEqual(before);
78+
expect(timestampInMillis).toBeLessThanOrEqual(after);
79+
assertTtl(pendingLetter.ttl, before, after);
80+
});
81+
82+
it("throws LetterAlreadyExistsError when creating a letter which already exists", async () => {
83+
await letterQueueRepository.putLetter(createLetter());
84+
85+
await expect(
86+
letterQueueRepository.putLetter(createLetter()),
87+
).rejects.toThrow(LetterAlreadyExistsError);
88+
});
89+
90+
it("rethrows errors from DynamoDB when creating a letter", async () => {
91+
const misconfiguredRepository = new LetterQueueRepository(
92+
db.docClient,
93+
logger,
94+
{
95+
...db.config,
96+
letterQueueTableName: "nonexistent-table",
97+
},
98+
);
99+
await expect(
100+
misconfiguredRepository.putLetter(createLetter()),
101+
).rejects.toThrow("Cannot do operations on a non-existent table");
102+
});
103+
});
104+
});

internal/datastore/src/config.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,10 @@ export type DatastoreConfig = {
22
region: string;
33
endpoint?: string;
44
lettersTableName: string;
5+
letterQueueTableName: string;
56
miTableName: string;
67
suppliersTableName: string;
78
lettersTtlHours: number;
9+
letterQueueTtlHours: number;
810
miTtlHours: number;
911
};

internal/datastore/src/errors.ts

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
/**
2+
* Error thrown when attempting to create a letter that already exists in the database.
3+
*/
4+
// eslint-disable-next-line import-x/prefer-default-export
5+
export class LetterAlreadyExistsError extends Error {
6+
constructor(
7+
public readonly supplierId: string,
8+
public readonly letterId: string,
9+
) {
10+
super(
11+
`Letter already exists: supplierId=${supplierId}, letterId=${letterId}`,
12+
);
13+
this.name = "LetterAlreadyExistsError";
14+
}
15+
}

internal/datastore/src/index.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
export * from "./types";
2+
export * from "./errors";
23
export * from "./mi-repository";
34
export * from "./letter-repository";
45
export * from "./supplier-repository";
6+
export { default as LetterQueueRepository } from "./letter-queue-repository";
57
export { default as DBHealthcheck } from "./healthcheck";

0 commit comments

Comments
 (0)