Skip to content

Commit b47ebbd

Browse files
nhsd-david-wassnhsd-david-wass
authored
CCM-12939 event audit pipeline (#368)
* workflow * Enable firehose * Status updates for sub * override fast-xml-parser * add bucket to eventsub outputs * add schedule * review changes * moved s3 policy documents to module * terraform format
1 parent 7c88a00 commit b47ebbd

15 files changed

Lines changed: 288 additions & 16 deletions

‎.github/workflows/manual-proxy-environment-deploy.yaml‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -36,11 +36,10 @@ jobs:
3636
node-version: 22
3737

3838
- name: Npm install
39-
working-directory: .
40-
env:
41-
NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
42-
run: npm ci
43-
shell: bash
39+
uses: ./.github/actions/node-install
40+
with:
41+
node-version: 22
42+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
4443

4544
- name: "Check if pull request exists for this branch and set ENVIRONMENT/APIM_ENV"
4645
id: pr_exists

‎infrastructure/terraform/components/api/README.md‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,9 @@ No requirements.
1818
| <a name="input_default_tags"></a> [default\_tags](#input\_default\_tags) | A map of default tags to apply to all taggable resources within the component | `map(string)` | `{}` | no |
1919
| <a name="input_disable_gateway_execute_endpoint"></a> [disable\_gateway\_execute\_endpoint](#input\_disable\_gateway\_execute\_endpoint) | Disable the execution endpoint for the API Gateway | `bool` | `true` | no |
2020
| <a name="input_enable_api_data_trace"></a> [enable\_api\_data\_trace](#input\_enable\_api\_data\_trace) | Enable API Gateway data trace logging | `bool` | `false` | no |
21-
| <a name="input_enable_event_cache"></a> [enable\_event\_cache](#input\_enable\_event\_cache) | Enable caching of events to an S3 bucket | `bool` | `false` | no |
22-
| <a name="input_enable_sns_delivery_logging"></a> [enable\_sns\_delivery\_logging](#input\_enable\_sns\_delivery\_logging) | Enable SNS Delivery Failure Notifications | `bool` | `false` | no |
21+
| <a name="input_enable_backups"></a> [enable\_backups](#input\_enable\_backups) | Enable backups | `bool` | `false` | no |
22+
| <a name="input_enable_event_cache"></a> [enable\_event\_cache](#input\_enable\_event\_cache) | Enable caching of events to an S3 bucket | `bool` | `true` | no |
23+
| <a name="input_enable_sns_delivery_logging"></a> [enable\_sns\_delivery\_logging](#input\_enable\_sns\_delivery\_logging) | Enable SNS Delivery Failure Notifications | `bool` | `true` | no |
2324
| <a name="input_environment"></a> [environment](#input\_environment) | The name of the tfscaffold environment | `string` | n/a | yes |
2425
| <a name="input_eventpub_control_plane_bus_arn"></a> [eventpub\_control\_plane\_bus\_arn](#input\_eventpub\_control\_plane\_bus\_arn) | ARN of the EventBridge control plane bus for eventpub | `string` | `""` | no |
2526
| <a name="input_eventpub_data_plane_bus_arn"></a> [eventpub\_data\_plane\_bus\_arn](#input\_eventpub\_data\_plane\_bus\_arn) | ARN of the EventBridge data plane bus for eventpub | `string` | `""` | no |
@@ -44,7 +45,7 @@ No requirements.
4445
|------|--------|---------|
4546
| <a name="module_authorizer_lambda"></a> [authorizer\_lambda](#module\_authorizer\_lambda) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a |
4647
| <a name="module_domain_truststore"></a> [domain\_truststore](#module\_domain\_truststore) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.26/terraform-s3bucket.zip | n/a |
47-
| <a name="module_eventpub"></a> [eventpub](#module\_eventpub) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.26/terraform-eventpub.zip | n/a |
48+
| <a name="module_eventpub"></a> [eventpub](#module\_eventpub) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.31/terraform-eventpub.zip | n/a |
4849
| <a name="module_eventsub"></a> [eventsub](#module\_eventsub) | ../../modules/eventsub | n/a |
4950
| <a name="module_get_letter"></a> [get\_letter](#module\_get\_letter) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a |
5051
| <a name="module_get_letter_data"></a> [get\_letter\_data](#module\_get\_letter\_data) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a |

‎infrastructure/terraform/components/api/glue_catalog_database_supplier.tf‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
resource "aws_glue_catalog_database" "supplier" {
2+
name = "${local.csi}-supplier"
3+
description = "Glue catalog database for Suppliers API"
4+
}

‎infrastructure/terraform/components/api/glue_crawler_event_crawler.tf‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
resource "aws_glue_crawler" "event_crawler" {
2+
count = local.event_cache_bucket_name != null ? 1 : 0
3+
name = "${local.csi}-audit-event-crawler"
4+
database_name = aws_glue_catalog_database.supplier.name
5+
role = aws_iam_role.glue_role.arn
6+
7+
table_prefix = ""
8+
s3_target {
9+
path = "s3://${local.event_cache_bucket_name}/"
10+
}
11+
12+
s3_target {
13+
path = "s3://${local.eventsub_event_cache_bucket_name}/"
14+
}
15+
16+
schedule = "cron(0 * * * ? *)"
17+
recrawl_policy {
18+
recrawl_behavior = "CRAWL_NEW_FOLDERS_ONLY"
19+
}
20+
21+
schema_change_policy {
22+
delete_behavior = "LOG"
23+
update_behavior = "LOG"
24+
}
25+
26+
}

‎infrastructure/terraform/components/api/iam_role_glue.tf‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
resource "aws_iam_role" "glue_role" {
2+
name = "${local.csi}-glue-role"
3+
assume_role_policy = data.aws_iam_policy_document.glue_assume_role.json
4+
}
5+
6+
data "aws_iam_policy_document" "glue_assume_role" {
7+
statement {
8+
sid = "AllowGlueServiceAssumeRole"
9+
effect = "Allow"
10+
11+
principals {
12+
type = "Service"
13+
identifiers = ["glue.amazonaws.com"]
14+
}
15+
16+
actions = [
17+
"sts:AssumeRole",
18+
]
19+
}
20+
}
21+
22+
resource "aws_iam_policy" "glue_service_policy" {
23+
name = "${local.csi}-glue-service-policy"
24+
description = "Policy for ${local.csi} Glue Service Role"
25+
policy = data.aws_iam_policy_document.glue_service_policy.json
26+
}
27+
28+
data "aws_iam_policy_document" "glue_service_policy" {
29+
statement {
30+
sid = "AllowGlueLogging"
31+
effect = "Allow"
32+
33+
actions = [
34+
"logs:CreateLogGroup",
35+
"logs:CreateLogStream",
36+
"logs:PutLogEvents"
37+
]
38+
resources = ["arn:aws:logs:*:*:*"]
39+
}
40+
41+
statement {
42+
sid = "AllowListBucketAndGetLocation"
43+
effect = "Allow"
44+
45+
actions = [
46+
"s3:ListBucket",
47+
"s3:GetBucketLocation"
48+
]
49+
50+
resources = [
51+
"arn:aws:s3:::${local.event_cache_bucket_name}",
52+
"arn:aws:s3:::${local.eventsub_event_cache_bucket_name}"
53+
]
54+
}
55+
statement {
56+
sid = "AllowS3Access"
57+
effect = "Allow"
58+
59+
actions = [
60+
"s3:GetObject",
61+
"s3:GetObjectVersion",
62+
"s3:PutObject",
63+
"s3:DeleteObject"
64+
]
65+
resources = [
66+
"arn:aws:s3:::${local.event_cache_bucket_name}/*",
67+
"arn:aws:s3:::${local.eventsub_event_cache_bucket_name}/*"
68+
]
69+
}
70+
statement {
71+
sid = "GlueCatalogAccess"
72+
effect = "Allow"
73+
actions = [
74+
"glue:GetDatabase",
75+
"glue:GetDatabases",
76+
"glue:GetTable",
77+
"glue:GetTables",
78+
"glue:CreateTable",
79+
"glue:UpdateTable",
80+
"glue:CreatePartition",
81+
"glue:BatchCreatePartition",
82+
"glue:GetPartition",
83+
"glue:BatchGetPartition",
84+
"glue:UpdatePartition"
85+
]
86+
resources = ["*"]
87+
}
88+
statement {
89+
sid = "S3TempAndGlueETL"
90+
effect = "Allow"
91+
actions = [
92+
"s3:PutObject",
93+
"s3:GetObject"
94+
]
95+
resources = [
96+
"arn:aws:s3:::aws-glue-*",
97+
"arn:aws:s3:::aws-glue-*/*"
98+
]
99+
}
100+
}
101+
102+
resource "aws_iam_role_policy_attachment" "gllue_attach_policy" {
103+
role = aws_iam_role.glue_role.name
104+
policy_arn = aws_iam_policy.glue_service_policy.arn
105+
}

‎infrastructure/terraform/components/api/locals.tf‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,4 +31,7 @@ locals {
3131

3232
core_pdf_bucket_arn = "arn:aws:s3:::comms-${var.core_account_id}-eu-west-2-${var.core_environment}-api-stg-pdf-pipeline"
3333
core_s3_kms_key_alias_name = "alias/comms-${var.core_environment}-api-s3"
34+
35+
event_cache_bucket_name = lookup(module.eventpub.s3_bucket_event_cache, "bucket", null)
36+
eventsub_event_cache_bucket_name = lookup(module.eventsub.s3_bucket_event_cache, "bucket", null)
3437
}

‎infrastructure/terraform/components/api/modules_eventpub.tf‎

Lines changed: 48 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
module "eventpub" {
2-
source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.26/terraform-eventpub.zip"
2+
source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.31/terraform-eventpub.zip"
33

44
name = "eventpub"
55

@@ -27,4 +27,51 @@ module "eventpub" {
2727

2828
data_plane_bus_arn = var.eventpub_data_plane_bus_arn
2929
control_plane_bus_arn = var.eventpub_control_plane_bus_arn
30+
31+
additional_policies_for_event_cache_bucket = [
32+
data.aws_iam_policy_document.eventcache[0].json
33+
]
34+
}
35+
data "aws_iam_policy_document" "eventcache" {
36+
count = local.event_cache_bucket_name != null ? 1 : 0
37+
statement {
38+
sid = "AllowGlueListBucketAndGetLocation"
39+
effect = "Allow"
40+
41+
principals {
42+
type = "AWS"
43+
identifiers = [aws_iam_role.glue_role.arn]
44+
}
45+
46+
actions = [
47+
"s3:ListBucket",
48+
"s3:GetBucketLocation"
49+
]
50+
51+
resources = [
52+
"arn:aws:s3:::${local.csi_global}-eventcache"
53+
]
54+
}
55+
56+
# Object-level permissions: Get/Put/Delete objects
57+
statement {
58+
sid = "AllowGlueObjectAccess"
59+
effect = "Allow"
60+
61+
principals {
62+
type = "AWS"
63+
identifiers = [aws_iam_role.glue_role.arn]
64+
}
65+
66+
actions = [
67+
"s3:GetObject",
68+
"s3:GetObjectVersion",
69+
"s3:PutObject",
70+
"s3:DeleteObject"
71+
]
72+
73+
resources = [
74+
"arn:aws:s3:::${local.csi_global}-eventcache/*"
75+
]
76+
}
3077
}

‎infrastructure/terraform/components/api/modules_eventsub.tf‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@ module "eventsub" {
1212

1313
default_tags = local.default_tags
1414

15+
glue_role_arn = aws_iam_role.glue_role.arn
16+
1517
kms_key_arn = module.kms.key_arn
1618
log_retention_in_days = var.log_retention_in_days
1719
log_level = "INFO"

‎infrastructure/terraform/components/api/s3_event_reporting.tf‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
resource "aws_s3_bucket" "event_reporting" {
2+
bucket = "${local.csi_global}-event-reporting"
3+
4+
tags = merge(local.default_tags, { "Enable-Backup" = var.enable_backups }, { "Enable-S3-Continuous-Backup" = var.enable_backups })
5+
}
6+
resource "aws_s3_bucket_ownership_controls" "event_reporting" {
7+
bucket = aws_s3_bucket.event_reporting.id
8+
9+
rule {
10+
object_ownership = "BucketOwnerPreferred"
11+
}
12+
}
13+
resource "aws_s3_bucket_versioning" "event_reporting" {
14+
bucket = aws_s3_bucket.event_reporting.id
15+
16+
versioning_configuration {
17+
status = "Enabled"
18+
}
19+
}

‎infrastructure/terraform/components/api/variables.tf‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -163,17 +163,23 @@ variable "core_environment" {
163163

164164
}
165165

166+
variable "enable_backups" {
167+
type = bool
168+
description = "Enable backups"
169+
default = false
170+
}
171+
166172
# Event Pub/Sub cache settings
167173
variable "enable_event_cache" {
168174
type = bool
169175
description = "Enable caching of events to an S3 bucket"
170-
default = false
176+
default = true
171177
}
172178

173179
variable "enable_sns_delivery_logging" {
174180
type = bool
175181
description = "Enable SNS Delivery Failure Notifications"
176-
default = false
182+
default = true
177183
}
178184

179185
variable "sns_success_logging_sample_percent" {

0 commit comments

Comments
 (0)