Skip to content

Commit 4b557a4

Browse files
nhsd-david-wassnhsd-david-wass
committed
glue crawler
1 parent 8d4f327 commit 4b557a4

6 files changed

Lines changed: 161 additions & 2 deletions

File tree

‎infrastructure/terraform/components/api/README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,8 @@ No requirements.
1717
| <a name="input_core_environment"></a> [core\_environment](#input\_core\_environment) | Environment of Core | `string` | `"prod"` | no |
1818
| <a name="input_default_tags"></a> [default\_tags](#input\_default\_tags) | A map of default tags to apply to all taggable resources within the component | `map(string)` | `{}` | no |
1919
| <a name="input_disable_gateway_execute_endpoint"></a> [disable\_gateway\_execute\_endpoint](#input\_disable\_gateway\_execute\_endpoint) | Disable the execution endpoint for the API Gateway | `bool` | `true` | no |
20-
| <a name="input_enable_backups"></a> [enable\_backups](#input\_enable\_backups) | Enable backups | `bool` | `false` | no |
2120
| <a name="input_enable_api_data_trace"></a> [enable\_api\_data\_trace](#input\_enable\_api\_data\_trace) | Enable API Gateway data trace logging | `bool` | `false` | no |
21+
| <a name="input_enable_backups"></a> [enable\_backups](#input\_enable\_backups) | Enable backups | `bool` | `false` | no |
2222
| <a name="input_enable_event_cache"></a> [enable\_event\_cache](#input\_enable\_event\_cache) | Enable caching of events to an S3 bucket | `bool` | `false` | no |
2323
| <a name="input_enable_sns_delivery_logging"></a> [enable\_sns\_delivery\_logging](#input\_enable\_sns\_delivery\_logging) | Enable SNS Delivery Failure Notifications | `bool` | `false` | no |
2424
| <a name="input_environment"></a> [environment](#input\_environment) | The name of the tfscaffold environment | `string` | n/a | yes |

‎infrastructure/terraform/components/api/glue_catalog_table_events.tf‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
resource "aws_glue_catalog_table" "events" {
2-
name = "events_history"
2+
name = "${local.csi}-events_history"
33
database_name = aws_glue_catalog_database.supplier.name
44

55
table_type = "EXTERNAL_TABLE"

‎infrastructure/terraform/components/api/glue_crawler_event_crawler.tf‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
resource "aws_glue_crawler" "event_crawler" {
2+
name = "event-crawler-${aws_glue_catalog_table.events.name}"
3+
database_name = aws_glue_catalog_database.supplier.name
4+
role = aws_iam_role.glue_role.arn
5+
6+
table_prefix = ""
7+
s3_target {
8+
path = "s3://${local.csi_global}-eventcache/"
9+
}
10+
recrawl_policy {
11+
recrawl_behavior = "CRAWL_EVERYTHING"
12+
}
13+
14+
schema_change_policy {
15+
delete_behavior = "LOG"
16+
update_behavior = "UPDATE_IN_DATABASE"
17+
}
18+
19+
}

‎infrastructure/terraform/components/api/iam_role_glue.tf‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
resource "aws_iam_role" "glue_role" {
2+
name = "${local.csi}-glue-role"
3+
assume_role_policy = data.aws_iam_policy_document.glue_assume_role.json
4+
}
5+
6+
data "aws_iam_policy_document" "glue_assume_role" {
7+
statement {
8+
sid = "AllowGlueServiceAssumeRole"
9+
effect = "Allow"
10+
11+
principals {
12+
type = "Service"
13+
identifiers = ["glue.amazonaws.com"]
14+
}
15+
16+
actions = [
17+
"sts:AssumeRole",
18+
]
19+
}
20+
}
21+
22+
resource "aws_iam_policy" "glue_service_policy" {
23+
name = "${local.csi}-glue-service-policy"
24+
description = "Policy for ${local.csi} Glue Service Role"
25+
policy = data.aws_iam_policy_document.glue_service_policy.json
26+
}
27+
28+
data "aws_iam_policy_document" "glue_service_policy" {
29+
statement {
30+
sid = "AllowGlueLogging"
31+
effect = "Allow"
32+
33+
actions = [
34+
"logs:CreateLogGroup",
35+
"logs:CreateLogStream",
36+
"logs:PutLogEvents"
37+
]
38+
resources = ["arn:aws:logs:*:*:*"]
39+
}
40+
41+
statement {
42+
sid = "AllowS3Access"
43+
effect = "Allow"
44+
45+
actions = [
46+
"s3:GetObject",
47+
"s3:PutObject",
48+
"s3:ListBucket",
49+
"s3:GetBucketLocation",
50+
"s3:DeleteObject"
51+
]
52+
resources = ["arn:aws:s3:::${local.csi}-glue-bucket/*",
53+
"arn:aws:s3:::${local.csi_global}-event-reporting/*"]
54+
}
55+
statement {
56+
sid = "GlueCatalogAccess"
57+
effect = "Allow"
58+
actions = [
59+
"glue:GetDatabase",
60+
"glue:GetDatabases",
61+
"glue:GetTable",
62+
"glue:GetTables",
63+
"glue:CreateTable",
64+
"glue:UpdateTable",
65+
"glue:CreatePartition",
66+
"glue:BatchCreatePartition",
67+
"glue:GetPartition",
68+
"glue:BatchGetPartition"
69+
]
70+
resources = ["*"]
71+
}
72+
statement {
73+
sid = "S3TempAndGlueETL"
74+
effect = "Allow"
75+
actions = [
76+
"s3:PutObject",
77+
"s3:GetObject"
78+
]
79+
resources = [
80+
"arn:aws:s3:::aws-glue-*",
81+
"arn:aws:s3:::aws-glue-*/*"
82+
]
83+
}
84+
}
85+
86+
resource "aws_iam_role_policy_attachment" "gllue_attach_policy" {
87+
role = aws_iam_role.glue_role.name
88+
policy_arn = aws_iam_policy.glue_service_policy.arn
89+
}

‎infrastructure/terraform/components/api/locals.tf‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,4 +31,6 @@ locals {
3131

3232
core_pdf_bucket_arn = "arn:aws:s3:::comms-${var.core_account_id}-eu-west-2-${var.core_environment}-api-stg-pdf-pipeline"
3333
core_s3_kms_key_alias_name = "alias/comms-${var.core_environment}-api-s3"
34+
35+
event_cache_bucket_name = lookup(module.eventpub.s3_bucket_event_cache, "bucket", null)
3436
}

‎infrastructure/terraform/components/api/s3_bucket_policy_eventcache.tf‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
resource "aws_s3_bucket_policy" "eventcache" {
2+
count = local.event_cache_bucket_name != null ? 1 : 0
3+
bucket = local.event_cache_bucket_name
4+
policy = data.aws_iam_policy_document.eventcache[0].json
5+
}
6+
7+
data "aws_iam_policy_document" "eventcache" {
8+
count = local.event_cache_bucket_name != null ? 1 : 0
9+
statement {
10+
sid = "AllowGlueListBucketAndGetLocation"
11+
effect = "Allow"
12+
13+
principals {
14+
type = "AWS"
15+
identifiers = [aws_iam_role.glue_role.arn]
16+
}
17+
18+
actions = [
19+
"s3:ListBucket",
20+
"s3:GetBucketLocation"
21+
]
22+
23+
resources = [
24+
"arn:aws:s3:::${local.csi_global}-eventcache"
25+
]
26+
}
27+
28+
# Object-level permissions: Get/Put/Delete objects
29+
statement {
30+
sid = "AllowGlueObjectAccess"
31+
effect = "Allow"
32+
33+
principals {
34+
type = "AWS"
35+
identifiers = [aws_iam_role.glue_role.arn]
36+
}
37+
38+
actions = [
39+
"s3:GetObject",
40+
"s3:GetObjectVersion",
41+
"s3:PutObject",
42+
"s3:DeleteObject"
43+
]
44+
45+
resources = [
46+
"arn:aws:s3:::${local.csi_global}-eventcache/*"
47+
]
48+
}
49+
}

0 commit comments

Comments
 (0)