glue crawler

Author: nhsd-david-wass

infrastructure/terraform/components/api/README.md

@@ -17,8 +17,8 @@ No requirements.
 | <a name="input_core_environment"></a> [core\_environment](#input\_core\_environment) | Environment of Core | `string` | `"prod"` | no |
 | <a name="input_default_tags"></a> [default\_tags](#input\_default\_tags) | A map of default tags to apply to all taggable resources within the component | `map(string)` | `{}` | no |
 | <a name="input_disable_gateway_execute_endpoint"></a> [disable\_gateway\_execute\_endpoint](#input\_disable\_gateway\_execute\_endpoint) | Disable the execution endpoint for the API Gateway | `bool` | `true` | no |
-| <a name="input_enable_backups"></a> [enable\_backups](#input\_enable\_backups) | Enable backups | `bool` | `false` | no |
 | <a name="input_enable_api_data_trace"></a> [enable\_api\_data\_trace](#input\_enable\_api\_data\_trace) | Enable API Gateway data trace logging | `bool` | `false` | no |
+| <a name="input_enable_backups"></a> [enable\_backups](#input\_enable\_backups) | Enable backups | `bool` | `false` | no |
 | <a name="input_enable_event_cache"></a> [enable\_event\_cache](#input\_enable\_event\_cache) | Enable caching of events to an S3 bucket | `bool` | `false` | no |
 | <a name="input_enable_sns_delivery_logging"></a> [enable\_sns\_delivery\_logging](#input\_enable\_sns\_delivery\_logging) | Enable SNS Delivery Failure Notifications | `bool` | `false` | no |
 | <a name="input_environment"></a> [environment](#input\_environment) | The name of the tfscaffold environment | `string` | n/a | yes |

infrastructure/terraform/components/api/glue_catalog_table_events.tf

@@ -1,5 +1,5 @@
 resource "aws_glue_catalog_table" "events" {
-  name          = "events_history"
+  name          = "${local.csi}-events_history"
   database_name = aws_glue_catalog_database.supplier.name
 
   table_type = "EXTERNAL_TABLE"

infrastructure/terraform/components/api/glue_crawler_event_crawler.tf

@@ -0,0 +1,19 @@
+resource "aws_glue_crawler" "event_crawler" {
+  name          = "event-crawler-${aws_glue_catalog_table.events.name}"
+  database_name = aws_glue_catalog_database.supplier.name
+  role          = aws_iam_role.glue_role.arn
+
+  table_prefix = ""
+  s3_target {
+    path = "s3://${local.csi_global}-eventcache/"
+  }
+  recrawl_policy {
+    recrawl_behavior = "CRAWL_EVERYTHING"
+  }
+
+  schema_change_policy {
+    delete_behavior = "LOG"
+    update_behavior = "UPDATE_IN_DATABASE"
+  }
+
+}

infrastructure/terraform/components/api/iam_role_glue.tf

@@ -0,0 +1,89 @@
+resource "aws_iam_role" "glue_role" {
+  name               = "${local.csi}-glue-role"
+  assume_role_policy = data.aws_iam_policy_document.glue_assume_role.json
+}
+
+data "aws_iam_policy_document" "glue_assume_role" {
+  statement {
+    sid    = "AllowGlueServiceAssumeRole"
+    effect = "Allow"
+
+    principals {
+      type        = "Service"
+      identifiers = ["glue.amazonaws.com"]
+    }
+
+    actions = [
+      "sts:AssumeRole",
+    ]
+  }
+}
+
+resource "aws_iam_policy" "glue_service_policy" {
+  name        = "${local.csi}-glue-service-policy"
+  description = "Policy for ${local.csi} Glue Service Role"
+  policy      = data.aws_iam_policy_document.glue_service_policy.json
+}
+
+data "aws_iam_policy_document" "glue_service_policy" {
+  statement {
+    sid    = "AllowGlueLogging"
+    effect = "Allow"
+
+    actions = [
+      "logs:CreateLogGroup",
+      "logs:CreateLogStream",
+      "logs:PutLogEvents"
+    ]
+    resources = ["arn:aws:logs:*:*:*"]
+  }
+
+  statement {
+    sid    = "AllowS3Access"
+    effect = "Allow"
+
+    actions = [
+      "s3:GetObject",
+      "s3:PutObject",
+      "s3:ListBucket",
+      "s3:GetBucketLocation",
+      "s3:DeleteObject"
+    ]
+    resources = ["arn:aws:s3:::${local.csi}-glue-bucket/*",
+    "arn:aws:s3:::${local.csi_global}-event-reporting/*"]
+  }
+  statement {
+    sid    = "GlueCatalogAccess"
+    effect = "Allow"
+    actions = [
+      "glue:GetDatabase",
+      "glue:GetDatabases",
+      "glue:GetTable",
+      "glue:GetTables",
+      "glue:CreateTable",
+      "glue:UpdateTable",
+      "glue:CreatePartition",
+      "glue:BatchCreatePartition",
+      "glue:GetPartition",
+      "glue:BatchGetPartition"
+    ]
+    resources = ["*"]
+  }
+  statement {
+    sid    = "S3TempAndGlueETL"
+    effect = "Allow"
+    actions = [
+      "s3:PutObject",
+      "s3:GetObject"
+    ]
+    resources = [
+      "arn:aws:s3:::aws-glue-*",
+      "arn:aws:s3:::aws-glue-*/*"
+    ]
+  }
+}
+
+resource "aws_iam_role_policy_attachment" "gllue_attach_policy" {
+  role       = aws_iam_role.glue_role.name
+  policy_arn = aws_iam_policy.glue_service_policy.arn
+}

infrastructure/terraform/components/api/s3_bucket_policy_eventcache.tf

@@ -0,0 +1,47 @@
+resource "aws_s3_bucket_policy" "eventcache" {
+  bucket = module.eventpub.s3_bucket_event_cache.bucket
+  policy = data.aws_iam_policy_document.eventcache.json
+}
+
+data "aws_iam_policy_document" "eventcache" {
+  statement {
+    sid    = "AllowGlueListBucketAndGetLocation"
+    effect = "Allow"
+
+    principals {
+      type        = "AWS"
+      identifiers = [aws_iam_role.glue_role.arn]
+    }
+
+    actions = [
+      "s3:ListBucket",
+      "s3:GetBucketLocation"
+    ]
+
+    resources = [
+      "arn:aws:s3:::${local.csi_global}-eventcache"
+    ]
+  }
+
+  # Object-level permissions: Get/Put/Delete objects
+  statement {
+    sid    = "AllowGlueObjectAccess"
+    effect = "Allow"
+
+    principals {
+      type        = "AWS"
+      identifiers = [aws_iam_role.glue_role.arn]
+    }
+
+    actions = [
+      "s3:GetObject",
+      "s3:GetObjectVersion",
+      "s3:PutObject",
+      "s3:DeleteObject"
+    ]
+
+    resources = [
+      "arn:aws:s3:::${local.csi_global}-eventcache/*"
+    ]
+  }
+}