glue crawler

Author: nhsd-david-wass

infrastructure/terraform/components/api/glue_catalog_table_events.tf

@@ -1,5 +1,5 @@
 resource "aws_glue_catalog_table" "events" {
-  name          = "events_history"
+  name          = "${local.csi}-events_history"
   database_name = aws_glue_catalog_database.supplier.name
 
   table_type = "EXTERNAL_TABLE"

infrastructure/terraform/components/api/glue_crawler_event_crawler.tf

@@ -0,0 +1,22 @@
+resource "aws_glue_crawler" "event_crawler" {
+  name        = "event-crawler-${aws_glue_catalog_table.events.name}"
+  database_name = aws_glue_catalog_database.supplier.name
+  role        = aws_iam_role.glue_role.arn
+
+  table_prefix = ""
+  s3_target {
+    path = "s3://${aws_s3_bucket.event_reporting.bucket}/events/"
+  }
+  recrawl_policy {
+    recrawl_behavior = "CRAWL_EVERYTHING"
+  }
+
+  configuration = jsonencode({
+    Version = 1.0
+    CrawlerOutput = {
+      Partitions = {
+        AddOrUpdateBehavior = "InheritFromTable"
+      }
+    }
+  })
+}

infrastructure/terraform/components/api/iam_role_glue.tf

@@ -0,0 +1,89 @@
+resource "aws_iam_role" "glue_role" {
+  name               = "${local.csi}-glue-role"
+  assume_role_policy = data.aws_iam_policy_document.glue_assume_role.json
+}
+
+data "aws_iam_policy_document" "glue_assume_role" {
+  statement {
+    sid    = "AllowGlueServiceAssumeRole"
+    effect = "Allow"
+
+    principals {
+      type        = "Service"
+      identifiers = ["glue.amazonaws.com"]
+    }
+
+    actions = [
+      "sts:AssumeRole",
+    ]
+  }
+}
+
+resource "aws_iam_policy" "glue_service_policy" {
+  name        = "${local.csi}-glue-service-policy"
+  description = "Policy for ${local.csi} Glue Service Role"
+  policy      = data.aws_iam_policy_document.glue_service_policy.json
+}
+
+data "aws_iam_policy_document" "glue_service_policy" {
+  statement {
+    sid    = "AllowGlueLogging"
+    effect = "Allow"
+
+    actions = [
+      "logs:CreateLogGroup",
+      "logs:CreateLogStream",
+      "logs:PutLogEvents"
+    ]
+    resources = ["arn:aws:logs:*:*:*"]
+  }
+
+  statement {
+    sid    = "AllowS3Access"
+    effect = "Allow"
+
+    actions = [
+      "s3:GetObject",
+      "s3:PutObject",
+      "s3:ListBucket",
+      "s3:GetBucketLocation",
+      "s3:DeleteObject"
+    ]
+    resources = ["arn:aws:s3:::${local.csi}-glue-bucket/*",
+    "arn:aws:s3:::${local.csi_global}-event-reporting/*"]
+  }
+  statement {
+    sid    = "GlueCatalogAccess"
+    effect = "Allow"
+    actions = [
+      "glue:GetDatabase",
+      "glue:GetDatabases",
+      "glue:GetTable",
+      "glue:GetTables",
+      "glue:CreateTable",
+      "glue:UpdateTable",
+      "glue:CreatePartition",
+      "glue:BatchCreatePartition",
+      "glue:GetPartition",
+      "glue:BatchGetPartition"
+    ]
+    resources = ["*"]
+  }
+  statement {
+    sid    = "S3TempAndGlueETL"
+    effect = "Allow"
+    actions = [
+      "s3:PutObject",
+      "s3:GetObject"
+    ]
+    resources = [
+      "arn:aws:s3:::aws-glue-*",
+      "arn:aws:s3:::aws-glue-*/*"
+    ]
+  }
+}
+
+resource "aws_iam_role_policy_attachment" "gllue_attach_policy" {
+  role       = aws_iam_role.glue_role.name
+  policy_arn = aws_iam_policy.glue_service_policy.arn
+}