VED-1157: Ack backend Lambda deployment to a dedicated pipeline stage (#1382)

Thomas-Boyle · web-flow · commit f3dfc6adfa0a · 2026-04-14T15:22:14.000+01:00
* Implement ack backend image deployment in CI/CD workflows

- Added support for building and deploying the ack backend image in the continuous deployment, backend deployment, and PR workflows.
- Introduced new variables for ack backend image version and build options in the relevant YAML files.
- Created a new ECR repository for the ack backend and updated the Terraform configuration accordingly.
- Enhanced the teardown process to clean up the ack backend image references.

This update ensures that the ack backend is properly integrated into the CI/CD pipeline, improving deployment consistency and management.

* Refactor image URI resolution in CI/CD teardown workflow and update ECR lifecycle policy format

- Simplified the process of resolving image URIs in the PR teardown workflow by introducing a reusable function that provides a placeholder if the output is unavailable.
- Updated the ECR lifecycle policy in Terraform to use the `jsonencode` function for better readability and maintainability.

* Remove deprecated ECR repository references and associated lifecycle configurations in temp.tf. This change is part of the cleanup process following the ack backend ECR ownership migration.

* Refactor CI/CD workflows to streamline lambda image deployment

- Replaced deprecated image version variables with a JSON map for lambda image overrides in the continuous deployment workflow.
- Consolidated the deployment of lambda images into a single job with a matrix strategy for better maintainability.
- Updated output variables to reflect the new image URI structure and removed redundant steps for deploying individual lambda images.
- Enhanced documentation for new input parameters related to lambda image deployment.

* Refactor deployment workflows to simplify image handling and cleanup process

* Refactor backend deployment workflows to use lambda build flags and simplify image build conditions

* Enhance deployment workflows by adding tf_var_suffix input for lambda artifacts and updating related processing logic

* Fix diff_base_sha assignment for synchronize action in PR deploy workflow

* Remove unused ECR lifecycle policy from ackbackend ECR repository configuration

* chore: empty commit
diff --git a/.github/workflows/continuous-deployment.yml b/.github/workflows/continuous-deployment.yml
@@ -20,7 +20,6 @@ jobs:
     uses: ./.github/workflows/deploy-backend.yml
     with:
       apigee_environment: internal-dev
-      build_recordprocessor_image: false
       diff_base_sha: ${{ github.event.before }}
       diff_head_sha: ${{ github.sha }}
       run_diff_check: true
@@ -86,9 +85,7 @@ jobs:
     uses: ./.github/workflows/deploy-backend.yml
     with:
       apigee_environment: ${{ matrix.sub_environment_name }}
-      recordprocessor_image_version: ${{ needs.deploy-internal-dev-backend.outputs.recordprocessor_image_version }}
-      diff_base_sha: ${{ github.event.before }}
-      diff_head_sha: ${{ github.sha }}
+      lambda_image_overrides: ${{ needs.deploy-internal-dev-backend.outputs.image_uris_json }}
       run_diff_check: false
       create_mns_subscription: true
       environment: dev
diff --git a/.github/workflows/deploy-backend.yml b/.github/workflows/deploy-backend.yml
@@ -6,14 +6,21 @@ on:
       apigee_environment:
         required: true
         type: string
-      build_recordprocessor_image:
+      lambda_build_flags:
+        description: >
+          JSON map of lambda_name -> force-build flag.
+          e.g. {"recordprocessor":true,"ack-backend":false}
         required: false
-        type: boolean
-        default: false
-      recordprocessor_image_version:
+        type: string
+        default: "{}"
+      lambda_image_overrides:
+        description: >
+          JSON map of lambda_name -> immutable image selector for reuse mode.
+          Supports tags, sha256 digests, or full image URIs. Empty object means
+          no explicit override.
         required: false
         type: string
-        default: ""
+        default: "{}"
       diff_base_sha:
         required: false
         type: string
@@ -37,9 +44,9 @@ on:
         required: true
         type: string
     outputs:
-      recordprocessor_image_version:
-        description: Selected immutable image selector used for recordprocessor deployment
-        value: ${{ jobs.deploy-recordprocessor-image.outputs.image_uri }}
+      image_uris_json:
+        description: JSON map of lambda_name -> immutable image URI used for deployment
+        value: ${{ jobs.terraform-plan.outputs.image_uris_json }}
   workflow_dispatch:
     inputs:
       apigee_environment:
@@ -63,16 +70,20 @@ on:
           - dev
           - preprod
           - prod
-      build_recordprocessor_image:
-        description: Build and publish a new recordprocessor image
-        required: true
-        type: boolean
-        default: true
-      recordprocessor_image_version:
-        description: Existing immutable recordprocessor image selector (tag, digest, or image URI) for reuse mode
+      lambda_build_flags:
+        description: >
+          JSON map of lambda_name -> force-build flag.
+          e.g. {"recordprocessor":true,"ack-backend":false}
         required: false
         type: string
-        default: ""
+        default: "{}"
+      lambda_image_overrides:
+        description: >
+          JSON map of lambda_name -> immutable image selector for reuse mode.
+          e.g. {"recordprocessor":"internal-dev-git-abc123","ack-backend":"123456789012.dkr.ecr.eu-west-2.amazonaws.com/imms-ackbackend-repo@sha256:..."}
+        required: false
+        type: string
+        default: "{}"
       diff_base_sha:
         description: Base commit SHA for diff checks
         required: false
@@ -100,25 +111,42 @@ env: # Sonarcloud - do not allow direct usage of untrusted data
 run-name: Deploy Backend - ${{ inputs.environment }} ${{ inputs.sub_environment }}
 
 jobs:
-  deploy-recordprocessor-image:
-    name: Deploy recordprocessor image
+  deploy-lambda-images:
+    name: Deploy ${{ matrix.lambda_name }} image
+    strategy:
+      # Surface every lambda image failure from the matrix in a single run.
+      fail-fast: false
+      matrix:
+        include:
+          - lambda_name: recordprocessor
+            tf_var_suffix: recordprocessor
+            ecr_repository: imms-recordprocessor-repo
+            dockerfile_path: lambdas/recordprocessor/Dockerfile
+            lambda_paths: |
+              lambdas/recordprocessor/
+          - lambda_name: ack-backend
+            tf_var_suffix: ack_backend
+            ecr_repository: imms-ackbackend-repo
+            dockerfile_path: lambdas/ack_backend/Dockerfile
+            lambda_paths: |
+              lambdas/ack_backend/
     uses: ./.github/workflows/deploy-lambda-artifact.yml
     with:
-      lambda_name: recordprocessor
+      lambda_name: ${{ matrix.lambda_name }}
+      tf_var_suffix: ${{ matrix.tf_var_suffix }}
       environment: ${{ inputs.environment }}
       sub_environment: ${{ inputs.sub_environment }}
-      build_image: ${{ inputs.build_recordprocessor_image }}
-      image_version: ${{ inputs.recordprocessor_image_version }}
+      build_image: ${{ fromJson(inputs.lambda_build_flags)[matrix.lambda_name] || false }}
+      image_version: ${{ fromJson(inputs.lambda_image_overrides)[matrix.lambda_name] || '' }}
       run_diff_check: ${{ inputs.run_diff_check }}
       diff_base_sha: ${{ inputs.diff_base_sha }}
       diff_head_sha: ${{ inputs.diff_head_sha }}
-      lambda_paths: |
-        lambdas/recordprocessor/
+      lambda_paths: ${{ matrix.lambda_paths }}
       shared_paths: |
         lambdas/shared/src/common/
       docker_context_path: lambdas
-      dockerfile_path: lambdas/recordprocessor/Dockerfile
-      ecr_repository: imms-recordprocessor-repo
+      dockerfile_path: ${{ matrix.dockerfile_path }}
+      ecr_repository: ${{ matrix.ecr_repository }}
       image_tag_prefix: ${{ inputs.sub_environment }}-
       allow_implicit_tag_prefix_reuse: ${{ inputs.sub_environment == 'internal-dev' || startsWith(inputs.sub_environment, 'pr-') }}
 
@@ -127,13 +155,12 @@ jobs:
       id-token: write
       contents: read
     needs:
-      - deploy-recordprocessor-image
-    if: ${{ !cancelled() && needs.deploy-recordprocessor-image.result == 'success' }}
+      - deploy-lambda-images
+    if: ${{ !cancelled() && needs.deploy-lambda-images.result == 'success' }}
     outputs:
-      recordprocessor_image_uri: ${{ needs.deploy-recordprocessor-image.outputs.image_uri }}
+      image_uris_json: ${{ steps.lambda-images.outputs.image_uris_json }}
+      terraform_image_uris_json: ${{ steps.lambda-images.outputs.terraform_image_uris_json }}
     runs-on: ubuntu-latest
-    env:
-      TF_VAR_recordprocessor_image_uri: ${{ needs.deploy-recordprocessor-image.outputs.image_uri }}
     environment:
       name: ${{ inputs.environment }}
     steps:
@@ -151,6 +178,58 @@ jobs:
         with:
           terraform_version: "1.12.2"
 
+      - name: Download lambda deployment manifests
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c
+        with:
+          pattern: deploy-manifest-*-${{ inputs.environment }}-${{ inputs.sub_environment }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/lambda-manifests
+          merge-multiple: true
+
+      - name: Assemble lambda image map and export Terraform vars
+        id: lambda-images
+        run: |
+          set -euo pipefail
+
+          manifest_dir="${RUNNER_TEMP}/lambda-manifests"
+          shopt -s nullglob
+          manifest_files=("${manifest_dir}"/*.json)
+
+          if [ "${#manifest_files[@]}" -eq 0 ]; then
+            echo "No lambda deployment manifests found."
+            exit 1
+          fi
+
+          jq -e -s '
+            all(
+              .[];
+              .lambda_name != null
+              and .lambda_name != ""
+              and .tf_var_suffix != null
+              and .tf_var_suffix != ""
+              and .image_uri != null
+              and .image_uri != ""
+            )
+            and ((map(.lambda_name) | unique | length) == length)
+            and ((map(.tf_var_suffix) | unique | length) == length)
+          ' "${manifest_files[@]}" > /dev/null
+
+          image_uris_json="$(
+            jq -cs 'map(select(.lambda_name != null and .image_uri != null) | {(.lambda_name): .image_uri}) | add' \
+              "${manifest_files[@]}"
+          )"
+
+          terraform_image_uris_json="$(
+            jq -cs 'map(select(.tf_var_suffix != null and .tf_var_suffix != "" and .image_uri != null) | {(.tf_var_suffix): .image_uri}) | add' \
+              "${manifest_files[@]}"
+          )"
+
+          echo "image_uris_json=${image_uris_json}" >> "$GITHUB_OUTPUT"
+          echo "terraform_image_uris_json=${terraform_image_uris_json}" >> "$GITHUB_OUTPUT"
+          jq -er '
+            to_entries[]
+            | "TF_VAR_\(.key)_image_uri=\(.value)"
+          ' <<< "${terraform_image_uris_json}" >> "$GITHUB_ENV"
+
       - name: Terraform Init
         working-directory: infrastructure/instance
         run: make init
@@ -175,8 +254,6 @@ jobs:
     needs: terraform-plan
     if: ${{ !cancelled() && needs.terraform-plan.result == 'success' }}
     runs-on: ubuntu-latest
-    env:
-      TF_VAR_recordprocessor_image_uri: ${{ needs.terraform-plan.outputs.recordprocessor_image_uri }}
     environment:
       name: ${{ inputs.environment }}
     steps:
@@ -193,6 +270,22 @@ jobs:
         with:
           terraform_version: "1.12.2"
 
+      - name: Restore lambda image Terraform vars
+        env:
+          TERRAFORM_IMAGE_URIS_JSON: ${{ needs.terraform-plan.outputs.terraform_image_uris_json }}
+        run: |
+          set -euo pipefail
+
+          if [ -z "${TERRAFORM_IMAGE_URIS_JSON}" ] || [ "${TERRAFORM_IMAGE_URIS_JSON}" = "null" ]; then
+            echo "terraform-plan did not emit terraform_image_uris_json."
+            exit 1
+          fi
+
+          jq -er '
+            to_entries[]
+            | "TF_VAR_\(.key)_image_uri=\(.value)"
+          ' <<< "${TERRAFORM_IMAGE_URIS_JSON}" >> "$GITHUB_ENV"
+
       - name: Retrieve Terraform Plan
         uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c
         with:
diff --git a/.github/workflows/deploy-lambda-artifact.yml b/.github/workflows/deploy-lambda-artifact.yml
@@ -7,6 +7,10 @@ on:
         description: Logical lambda identifier for logging and artifacts
         required: true
         type: string
+      tf_var_suffix:
+        description: Terraform variable suffix used to export TF_VAR_<suffix>_image_uri
+        required: true
+        type: string
       environment:
         description: GitHub environment containing AWS account variables
         required: true
@@ -106,6 +110,7 @@ jobs:
       DOCKERFILE_PATH: ${{ inputs.dockerfile_path }}
       TAG_PREFIX: ${{ inputs.image_tag_prefix }}
       LAMBDA_NAME: ${{ inputs.lambda_name }}
+      TF_VAR_SUFFIX: ${{ inputs.tf_var_suffix }}
       ENVIRONMENT: ${{ inputs.environment }}
       SUB_ENVIRONMENT: ${{ inputs.sub_environment }}
       BUILD_IMAGE: ${{ inputs.build_image }}
@@ -391,6 +396,7 @@ jobs:
 
           jq -n \
             --arg lambda_name "${LAMBDA_NAME}" \
+            --arg tf_var_suffix "${TF_VAR_SUFFIX}" \
             --arg environment "${ENVIRONMENT}" \
             --arg sub_environment "${SUB_ENVIRONMENT}" \
             --arg deployment_mode "${DEPLOYMENT_MODE}" \
@@ -408,6 +414,7 @@ jobs:
             --arg build_timestamp "${build_timestamp}" \
             '{
               lambda_name: $lambda_name,
+              tf_var_suffix: $tf_var_suffix,
               environment: $environment,
               sub_environment: $sub_environment,
               deployment_mode: $deployment_mode,
diff --git a/.github/workflows/pr-deploy-and-test.yml b/.github/workflows/pr-deploy-and-test.yml
@@ -20,7 +20,10 @@ jobs:
     uses: ./.github/workflows/deploy-backend.yml
     with:
       apigee_environment: internal-dev
-      build_recordprocessor_image: ${{ github.event.action == 'opened' || github.event.action == 'reopened' }}
+      lambda_build_flags: >-
+        ${{ (github.event.action == 'opened' || github.event.action == 'reopened')
+            && '{"recordprocessor":true,"ack-backend":true}'
+            || '{}' }}
       diff_base_sha: ${{ github.event.action == 'synchronize' && github.event.before || github.event.pull_request.base.sha }}
       diff_head_sha: ${{ github.event.pull_request.head.sha }}
       run_diff_check: ${{ github.event.action == 'synchronize' }}
diff --git a/.github/workflows/pr-teardown.yml b/.github/workflows/pr-teardown.yml
@@ -48,12 +48,13 @@ jobs:
           make init apigee_environment=$APIGEE_ENVIRONMENT environment=$BACKEND_ENVIRONMENT sub_environment=$BACKEND_SUB_ENVIRONMENT
           make workspace apigee_environment=$APIGEE_ENVIRONMENT environment=$BACKEND_ENVIRONMENT sub_environment=$BACKEND_SUB_ENVIRONMENT
           echo "ID_SYNC_QUEUE_ARN=$(make -s output name=id_sync_queue_arn)" >> $GITHUB_ENV
-          recordprocessor_image_uri="$(make -s output name=recordprocessor_image_uri 2>/dev/null || true)"
-          if [ -z "${recordprocessor_image_uri}" ]; then
-            # Destroy still evaluates variable validation, so provide a non-empty fallback when output is unavailable.
-            recordprocessor_image_uri="placeholder.dkr.ecr.eu-west-2.amazonaws.com/imms-recordprocessor-repo@sha256:0000000000000000000000000000000000000000000000000000000000000000"
-          fi
-          echo "TF_VAR_recordprocessor_image_uri=${recordprocessor_image_uri}" >> $GITHUB_ENV
+          # Destroy still evaluates variable validation, so provide a non-empty fallback when output is unavailable.
+          resolve_or_placeholder() {
+            local uri="$(make -s output "name=$1" 2>/dev/null || true)"
+            echo "${uri:-placeholder.dkr.ecr.eu-west-2.amazonaws.com/$2@sha256:0000000000000000000000000000000000000000000000000000000000000000}"
+          }
+          echo "TF_VAR_recordprocessor_image_uri=$(resolve_or_placeholder recordprocessor_image_uri imms-recordprocessor-repo)" >> $GITHUB_ENV
+          echo "TF_VAR_ack_backend_image_uri=$(resolve_or_placeholder ack_backend_image_uri imms-ackbackend-repo)" >> $GITHUB_ENV
 
       - name: Install poetry
         run: pip install poetry==2.1.4
@@ -128,4 +129,6 @@ jobs:
               --output json
           }
 
-          cleanup_repo_by_prefix "imms-recordprocessor-repo"
+          for repository_name in imms-recordprocessor-repo imms-ackbackend-repo; do
+            cleanup_repo_by_prefix "${repository_name}"
+          done
diff --git a/infrastructure/account/ackbackend_ecr_repo.tf b/infrastructure/account/ackbackend_ecr_repo.tf
@@ -0,0 +1,33 @@
+resource "aws_ecr_repository" "ackbackend_repository" {
+  image_scanning_configuration {
+    scan_on_push = true
+  }
+  image_tag_mutability = "IMMUTABLE"
+  name                 = "imms-ackbackend-repo"
+}
+
+resource "aws_ecr_repository_policy" "ackbackend_repository_lambda_image_retrieval_policy" {
+  repository = aws_ecr_repository.ackbackend_repository.name
+
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Sid    = "LambdaECRImageRetrievalPolicy"
+        Effect = "Allow"
+        Principal = {
+          Service = "lambda.amazonaws.com"
+        }
+        Action = [
+          "ecr:BatchGetImage",
+          "ecr:GetDownloadUrlForLayer"
+        ]
+        Condition = {
+          StringLike = {
+            "aws:sourceArn" = "arn:aws:lambda:${var.aws_region}:${var.imms_account_id}:function:imms-*-ack-lambda"
+          }
+        }
+      }
+    ]
+  })
+}
diff --git a/infrastructure/instance/ack_lambda.tf b/infrastructure/instance/ack_lambda.tf
diff --git a/infrastructure/instance/temp.tf b/infrastructure/instance/temp.tf
diff --git a/infrastructure/instance/variables.tf b/infrastructure/instance/variables.tf
