VED-1238: Refactor all remaining lambdas deployment to a dedicated pipeline stage (#1414)

* Refactor Lambda infrastructure by removing ECR repository and lifecycle policy configurations

- Deleted ECR repository configurations for record processor, batch processor filter, delta, filename processor, id sync, forwarder, redis sync, and mesh processor Lambdas.
- Removed associated ECR lifecycle policies for the above Lambdas.
- Updated Lambda functions to use image URIs directly from variables instead of modules.
- Added new variables for backend, batch processor filter, delta, filename processor, id sync, mesh processor, MNS publisher, record forwarder, and redis sync Lambda image URIs to ensure CI/CD integration.
- Cleaned up shared Lambda configurations and removed unused local variables.

* Update ECR repository names in CI/CD workflows and Terraform configuration

- Changed ECR repository names for various Lambda functions in the deploy-backend.yml and pr-teardown.yml workflows to reflect updated naming conventions.
- Updated the lambda_ecr_repos.tf file to align with the new ECR repository names for backend, delta, filename processor, and record forwarder Lambdas, ensuring consistency across the infrastructure configuration.

* Refactor Lambda image handling in workflows to use a unified repository format

* Refactor Lambda deployment workflows to streamline ECR repository configurations

- Updated the deploy-backend.yml workflow to use lambda_dir as the tf_var_suffix for improved clarity and consistency.
- Simplified the lambda_ecr_repos.tf file by removing redundant local variables and directly defining the lifecycle policy for the recordprocessor repository.
- Enhanced the ECR repository policy to maintain image retrieval actions in a more structured manner.

* Add lambda ECR repository resource to Terraform configuration

* Refactor ECR repository references in Terraform configuration by removing array indexing for lambda repositories, simplifying the code structure.

---------

Co-authored-by: Akinola Olutola <akinola.olutola1@nhs.net>

Author: Thomas-Boyle

.github/workflows/deploy-backend.yml

@@ -9,7 +9,7 @@ on:
       lambda_build_flags:
         description: >
           JSON map of lambda_name -> force-build flag.
-          e.g. {"recordprocessor":true,"ack-backend":false}
+          e.g. {"backend":true,"recordprocessor":true,"ack-backend":false}
         required: false
         type: string
         default: "{}"
@@ -73,14 +73,14 @@ on:
       lambda_build_flags:
         description: >
           JSON map of lambda_name -> force-build flag.
-          e.g. {"recordprocessor":true,"ack-backend":false}
+          e.g. {"backend":true,"recordprocessor":true,"ack-backend":false}
         required: false
         type: string
         default: "{}"
       lambda_image_overrides:
         description: >
           JSON map of lambda_name -> immutable image selector for reuse mode.
-          e.g. {"recordprocessor":"internal-dev-git-abc123","ack-backend":"123456789012.dkr.ecr.eu-west-2.amazonaws.com/imms-ackbackend-repo@sha256:..."}
+          e.g. {"backend":"internal-dev-git-abc123","ack-backend":"123456789012.dkr.ecr.eu-west-2.amazonaws.com/imms-ackbackend-repo@sha256:..."}
         required: false
         type: string
         default: "{}"
@@ -118,34 +118,54 @@ jobs:
       fail-fast: false
       matrix:
         include:
+          - lambda_name: backend
+            ecr_repository: imms-backend-repo
+            lambda_dir: backend
+          - lambda_name: batch_processor_filter
+            ecr_repository: imms-batch-processor-filter-repo
+            lambda_dir: batch_processor_filter
+          - lambda_name: delta_backend
+            ecr_repository: imms-delta-backend-repo
+            lambda_dir: delta_backend
+          - lambda_name: filenameprocessor
+            ecr_repository: imms-filenameprocessor-repo
+            lambda_dir: filenameprocessor
+          - lambda_name: id_sync
+            ecr_repository: imms-id-sync-repo
+            lambda_dir: id_sync
+          - lambda_name: mesh_processor
+            ecr_repository: imms-mesh-processor-repo
+            lambda_dir: mesh_processor
+          - lambda_name: mns_publisher
+            ecr_repository: imms-mns-publisher-repo
+            lambda_dir: mns_publisher
+          - lambda_name: recordforwarder
+            ecr_repository: imms-recordforwarder-repo
+            lambda_dir: recordforwarder
+          - lambda_name: redis_sync
+            ecr_repository: imms-redis-sync-repo
+            lambda_dir: redis_sync
           - lambda_name: recordprocessor
-            tf_var_suffix: recordprocessor
             ecr_repository: imms-recordprocessor-repo
-            dockerfile_path: lambdas/recordprocessor/Dockerfile
-            lambda_paths: |
-              lambdas/recordprocessor/
+            lambda_dir: recordprocessor
           - lambda_name: ack-backend
-            tf_var_suffix: ack_backend
             ecr_repository: imms-ackbackend-repo
-            dockerfile_path: lambdas/ack_backend/Dockerfile
-            lambda_paths: |
-              lambdas/ack_backend/
+            lambda_dir: ack_backend
     uses: ./.github/workflows/deploy-lambda-artifact.yml
     with:
       lambda_name: ${{ matrix.lambda_name }}
-      tf_var_suffix: ${{ matrix.tf_var_suffix }}
+      tf_var_suffix: ${{ matrix.lambda_dir }}
       environment: ${{ inputs.environment }}
       sub_environment: ${{ inputs.sub_environment }}
       build_image: ${{ fromJson(inputs.lambda_build_flags)[matrix.lambda_name] || false }}
       image_version: ${{ fromJson(inputs.lambda_image_overrides)[matrix.lambda_name] || '' }}
       run_diff_check: ${{ inputs.run_diff_check }}
       diff_base_sha: ${{ inputs.diff_base_sha }}
       diff_head_sha: ${{ inputs.diff_head_sha }}
-      lambda_paths: ${{ matrix.lambda_paths }}
-      shared_paths: |
-        lambdas/shared/src/common/
+      lambda_paths: lambdas/${{ matrix.lambda_dir }}/
+      shared_paths: lambdas/shared/src/common/
       docker_context_path: lambdas
-      dockerfile_path: ${{ matrix.dockerfile_path }}
+      dockerfile_path: lambdas/${{ matrix.lambda_dir }}/Dockerfile
       ecr_repository: ${{ matrix.ecr_repository }}
       image_tag_prefix: ${{ inputs.sub_environment }}-
       allow_implicit_tag_prefix_reuse: ${{ inputs.sub_environment == 'internal-dev' || startsWith(inputs.sub_environment, 'pr-') }}
@@ -159,7 +179,6 @@ jobs:
     if: ${{ !cancelled() && needs.deploy-lambda-images.result == 'success' }}
     outputs:
       image_uris_json: ${{ steps.lambda-images.outputs.image_uris_json }}
-      terraform_image_uris_json: ${{ steps.lambda-images.outputs.terraform_image_uris_json }}
     runs-on: ubuntu-latest
     environment:
       name: ${{ inputs.environment }}
@@ -218,17 +237,11 @@ jobs:
               "${manifest_files[@]}"
           )"
 
-          terraform_image_uris_json="$(
-            jq -cs 'map(select(.tf_var_suffix != null and .tf_var_suffix != "" and .image_uri != null) | {(.tf_var_suffix): .image_uri}) | add' \
-              "${manifest_files[@]}"
-          )"
-
           echo "image_uris_json=${image_uris_json}" >> "$GITHUB_OUTPUT"
-          echo "terraform_image_uris_json=${terraform_image_uris_json}" >> "$GITHUB_OUTPUT"
           jq -er '
-            to_entries[]
-            | "TF_VAR_\(.key)_image_uri=\(.value)"
-          ' <<< "${terraform_image_uris_json}" >> "$GITHUB_ENV"
+            select(.tf_var_suffix != null and .tf_var_suffix != "" and .image_uri != null)
+            | "TF_VAR_\(.tf_var_suffix)_image_uri=\(.image_uri)"
+          ' "${manifest_files[@]}" >> "$GITHUB_ENV"
 
       - name: Terraform Init
         working-directory: infrastructure/instance
@@ -270,22 +283,6 @@ jobs:
         with:
           terraform_version: "1.12.2"
 
-      - name: Restore lambda image Terraform vars
-        env:
-          TERRAFORM_IMAGE_URIS_JSON: ${{ needs.terraform-plan.outputs.terraform_image_uris_json }}
-        run: |
-          set -euo pipefail
-
-          if [ -z "${TERRAFORM_IMAGE_URIS_JSON}" ] || [ "${TERRAFORM_IMAGE_URIS_JSON}" = "null" ]; then
-            echo "terraform-plan did not emit terraform_image_uris_json."
-            exit 1
-          fi
-
-          jq -er '
-            to_entries[]
-            | "TF_VAR_\(.key)_image_uri=\(.value)"
-          ' <<< "${TERRAFORM_IMAGE_URIS_JSON}" >> "$GITHUB_ENV"
-
       - name: Retrieve Terraform Plan
         uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c
         with:

.github/workflows/pr-deploy-and-test.yml

@@ -20,10 +20,6 @@ jobs:
     uses: ./.github/workflows/deploy-backend.yml
     with:
       apigee_environment: internal-dev
-      lambda_build_flags: >-
-        ${{ (github.event.action == 'opened' || github.event.action == 'reopened')
-            && '{"recordprocessor":true,"ack-backend":true}'
-            || '{}' }}
       diff_base_sha: ${{ github.event.action == 'synchronize' && github.event.before || github.event.pull_request.base.sha }}
       diff_head_sha: ${{ github.event.pull_request.head.sha }}
       run_diff_check: ${{ github.event.action == 'synchronize' }}

.github/workflows/pr-teardown.yml

@@ -20,6 +20,18 @@ jobs:
       APIGEE_ENVIRONMENT: internal-dev
       BACKEND_ENVIRONMENT: dev
       BACKEND_SUB_ENVIRONMENT: pr-${{ github.event_name == 'pull_request' && github.event.pull_request.number || inputs.pr_number }}
+      LAMBDA_IMAGE_REPOSITORIES: |
+        backend:imms-backend-repo
+        batch_processor_filter:imms-batch-processor-filter-repo
+        delta_backend:imms-delta-backend-repo
+        filenameprocessor:imms-filenameprocessor-repo
+        id_sync:imms-id-sync-repo
+        mesh_processor:imms-mesh-processor-repo
+        mns_publisher:imms-mns-publisher-repo
+        recordprocessor:imms-recordprocessor-repo
+        recordforwarder:imms-recordforwarder-repo
+        ack_backend:imms-ackbackend-repo
+        redis_sync:imms-redis-sync-repo
     permissions:
       id-token: write
       contents: read
@@ -53,8 +65,10 @@ jobs:
             local uri="$(make -s output "name=$1" 2>/dev/null || true)"
             echo "${uri:-placeholder.dkr.ecr.eu-west-2.amazonaws.com/$2@sha256:0000000000000000000000000000000000000000000000000000000000000000}"
           }
-          echo "TF_VAR_recordprocessor_image_uri=$(resolve_or_placeholder recordprocessor_image_uri imms-recordprocessor-repo)" >> $GITHUB_ENV
-          echo "TF_VAR_ack_backend_image_uri=$(resolve_or_placeholder ack_backend_image_uri imms-ackbackend-repo)" >> $GITHUB_ENV
+          while IFS=: read -r lambda_name repository_name; do
+            [ -n "${lambda_name}" ] || continue
+            echo "TF_VAR_${lambda_name}_image_uri=$(resolve_or_placeholder "${lambda_name}_image_uri" "${repository_name}")" >> $GITHUB_ENV
+          done <<< "${LAMBDA_IMAGE_REPOSITORIES}"
 
       - name: Install poetry
         run: pip install poetry==2.1.4
@@ -129,6 +143,8 @@ jobs:
               --output json
           }
 
-          for repository_name in imms-recordprocessor-repo imms-ackbackend-repo; do
+          while IFS=: read -r lambda_name repository_name; do
+            [ -n "${lambda_name}" ] || continue
+            [ -n "${repository_name}" ] || continue
             cleanup_repo_by_prefix "${repository_name}"
-          done
+          done <<< "${LAMBDA_IMAGE_REPOSITORIES}"

infrastructure/account/ackbackend_ecr_repo.tf

@@ -0,0 +1,143 @@
+locals {
+  lambda_source_arn_prefix = "arn:aws:lambda:${var.aws_region}:${var.imms_account_id}:function:imms-"
+
+  lambda_ecr_repositories = {
+    operation = {
+      name = "imms-backend-repo"
+      lambda_source_names = [
+        "*_get_status",
+        "*_not_found",
+        "*_search_imms",
+        "*_get_imms",
+        "*_delete_imms",
+        "*_create_imms",
+        "*_update_imms"
+      ]
+    }
+    batch_processor_filter = {
+      name                = "imms-batch-processor-filter-repo"
+      lambda_source_names = ["*-batch-processor-filter-lambda"]
+    }
+    delta = {
+      name                = "imms-delta-backend-repo"
+      lambda_source_names = ["*-delta-lambda"]
+    }
+    filenameprocessor = {
+      name                = "imms-filenameprocessor-repo"
+      lambda_source_names = ["*-filenameproc-lambda"]
+    }
+    id_sync = {
+      name                = "imms-id-sync-repo"
+      lambda_source_names = ["*-id-sync-lambda"]
+    }
+    mesh_processor = {
+      name                = "imms-mesh-processor-repo"
+      lambda_source_names = ["*-mesh-processor-lambda"]
+    }
+    mns_publisher = {
+      name                = "imms-mns-publisher-repo"
+      lambda_source_names = ["*-mns-publisher-lambda"]
+    }
+    ack_backend = {
+      name                = "imms-ackbackend-repo"
+      lambda_source_names = ["*-ack-lambda"]
+    }
+    recordforwarder = {
+      name                = "imms-recordforwarder-repo"
+      lambda_source_names = ["*-forwarding-lambda"]
+    }
+    recordprocessor = {
+      name = "imms-recordprocessor-repo"
+      lifecycle_policy = jsonencode({
+        rules = [
+          {
+            rulePriority = 1
+            description  = "Keep last 10 images."
+            selection = {
+              tagStatus   = "any"
+              countType   = "imageCountMoreThan"
+              countNumber = 10
+            }
+            action = {
+              type = "expire"
+            }
+          }
+        ]
+      })
+    }
+    redis_sync = {
+      name                = "imms-redis-sync-repo"
+      lambda_source_names = ["*-redis-sync-lambda"]
+    }
+  }
+}
+#lambda repo
+resource "aws_ecr_repository" "lambda_repository" {
+  for_each = local.lambda_ecr_repositories
+
+  image_scanning_configuration {
+    scan_on_push = true
+  }
+
+  image_tag_mutability = "IMMUTABLE"
+  name                 = each.value.name
+}
+
+resource "aws_ecr_repository_policy" "lambda_repository_image_retrieval_policy" {
+  for_each = {
+    for key, repo in local.lambda_ecr_repositories : key => repo if try(repo.lambda_source_names, null) != null
+  }
+
+  repository = aws_ecr_repository.lambda_repository[each.key].name
+
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Sid    = "LambdaECRImageRetrievalPolicy"
+        Effect = "Allow"
+        Principal = {
+          Service = "lambda.amazonaws.com"
+        }
+        Action = [
+          "ecr:BatchGetImage",
+          "ecr:GetDownloadUrlForLayer"
+        ]
+        Condition = {
+          StringLike = {
+            "aws:sourceArn" = formatlist("${local.lambda_source_arn_prefix}%s", each.value.lambda_source_names)
+          }
+        }
+      }
+    ]
+  })
+}
+
+resource "aws_ecr_lifecycle_policy" "lambda_repository_lifecycle_policy" {
+  for_each = {
+    for key, repo in local.lambda_ecr_repositories : key => repo if try(repo.lifecycle_policy, null) != null
+  }
+
+  repository = aws_ecr_repository.lambda_repository[each.key].name
+  policy     = each.value.lifecycle_policy
+}
+
+moved {
+  from = aws_ecr_repository.ackbackend_repository
+  to   = aws_ecr_repository.lambda_repository["ack_backend"]
+}
+
+moved {
+  from = aws_ecr_repository_policy.ackbackend_repository_lambda_image_retrieval_policy
+  to   = aws_ecr_repository_policy.lambda_repository_image_retrieval_policy["ack_backend"]
+}
+
+moved {
+  from = aws_ecr_repository.recordprocessor_repository
+  to   = aws_ecr_repository.lambda_repository["recordprocessor"]
+}
+
+moved {
+  from = aws_ecr_lifecycle_policy.recordprocessor_repository_lifecycle_policy
+  to   = aws_ecr_lifecycle_policy.lambda_repository_lifecycle_policy["recordprocessor"]
+}