Merge remote-tracking branch 'origin/main' into feature/CCM-13116-Letter-Queue-Operations

Author: stevebux

.github/actions/e2e-tests/action.yml

@@ -0,0 +1,22 @@
+name: E2E tests
+description: "Run end-to-end tests for this repo"
+
+runs:
+  using: "composite"
+
+  steps:
+    - name: Install poetry and e2e test dependencies
+      shell: bash
+      run: |
+        pipx install poetry
+        cd tests/e2e-tests && poetry install
+
+    - name: Run e2e tests
+      shell: bash
+      run: |
+        echo "$INTERNAL_DEV_TEST_PEM" > "${GITHUB_WORKSPACE}/internal-dev-test-1.pem"
+        chmod 600 "${GITHUB_WORKSPACE}/internal-dev-test-1.pem"
+        export PROXY_NAME=nhs-notify-supplier--internal-dev--nhs-notify-supplier
+        export API_ENVIRONMENT=internal-dev
+        export NON_PROD_PRIVATE_KEY="${GITHUB_WORKSPACE}/internal-dev-test-1.pem"
+        make .internal-dev-test

.github/actions/test-types.json

@@ -1,3 +1,4 @@
 [
-  "component"
+  "component",
+  "sandbox"
 ]

.github/workflows/cicd-1-pull-request.yaml

@@ -29,6 +29,7 @@ jobs:
       does_pull_request_exist: ${{ steps.pr_exists.outputs.does_pull_request_exist }}
       pr_number: ${{ steps.pr_exists.outputs.pr_number }}
       skip_trivy_package: ${{ steps.skip_trivy.outputs.skip_trivy_package }}
+      deploy_proxy: ${{ steps.deploy_proxy.outputs.deploy_proxy }}
     steps:
       - name: "Checkout code"
         uses: actions/checkout@v5
@@ -87,6 +88,26 @@ jobs:
           else
             echo "skip_trivy_package=false" >> $GITHUB_OUTPUT
           fi
+      - name: "Determine if proxy should be deployed"
+        id: deploy_proxy
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          PR_NUMBER: ${{ steps.pr_exists.outputs.pr_number }}
+        run: |
+          if [[ -z "$PR_NUMBER" ]]; then
+            echo "No pull request detected; proxy deployment will run."
+            echo "deploy_proxy=true" >> $GITHUB_OUTPUT
+            exit 0
+          fi
+
+          labels=$(gh pr view "$PR_NUMBER" --json labels --jq '.labels[].name')
+          echo "Labels on PR #$PR_NUMBER: $labels"
+
+          if echo "$labels" | grep -Fxq 'deploy-proxy'; then
+            echo "deploy_proxy=true" >> $GITHUB_OUTPUT
+          else
+            echo "deploy_proxy=false" >> $GITHUB_OUTPUT
+          fi
       - name: "List variables"
         run: |
           export BUILD_DATETIME_LONDON="${{ steps.variables.outputs.build_datetime_london }}"
@@ -141,6 +162,7 @@ jobs:
       terraform_version: "${{ needs.metadata.outputs.terraform_version }}"
       version: "${{ needs.metadata.outputs.version }}"
       pr_number: "${{ needs.metadata.outputs.pr_number }}"
+      deploy_proxy: "${{ needs.metadata.outputs.deploy_proxy }}"
     secrets: inherit
   acceptance-stage: # Recommended maximum execution time is 10 minutes
     name: "Acceptance stage"
@@ -156,6 +178,7 @@ jobs:
       terraform_version: "${{ needs.metadata.outputs.terraform_version }}"
       version: "${{ needs.metadata.outputs.version }}"
       pr_number: ${{ needs.metadata.outputs.pr_number }}
+      proxy_deployed: "${{ needs.metadata.outputs.deploy_proxy }}"
     secrets: inherit
   publish-stage: # Recommended maximum execution time is 10 minutes
     name: "Publish stage"

.github/workflows/release_created.yaml

@@ -41,6 +41,7 @@ jobs:
             --targetComponent "api" \
             --terraformAction "apply"
   deploy-proxy:
+    needs: deploy-main #wait for backend deploy to complete
     name: "Deploy proxy"
     runs-on: ubuntu-latest
     timeout-minutes: 10

.github/workflows/stage-3-build.yaml

@@ -35,6 +35,10 @@ on:
         description: "PR Number if it exists"
         required: false
         type: string
+      deploy_proxy:
+        description: "True if the APIM proxy should be deployed"
+        required: true
+        type: string
 
 permissions:
   id-token: write # This is required for requesting the JWT
@@ -133,9 +137,13 @@ jobs:
   pr-create-dynamic-environment:
     name: Create Dynamic Environment
     runs-on: ubuntu-latest
-    if: inputs.pr_number != ''
+    outputs:
+      environment_name: ${{ steps.set-environment.outputs.environment_name }}
     steps:
       - uses: actions/checkout@v5
+      - name: Set environment name
+        id: set-environment
+        run: echo "environment_name=${{ inputs.pr_number != '' && format('pr{0}', inputs.pr_number) || 'main' }}" >> $GITHUB_OUTPUT
       - name: Trigger dynamic environment creation
         env:
           APP_CLIENT_ID: ${{ secrets.APP_CLIENT_ID }}
@@ -148,7 +156,7 @@ jobs:
             --infraRepoName "$(echo ${{ github.repository }} | cut -d'/' -f2)" \
             --releaseVersion ${{ github.head_ref || github.ref_name }} \
             --targetWorkflow "dispatch-deploy-dynamic-env.yaml" \
-            --targetEnvironment "pr${PR_NUMBER}" \
+            --targetEnvironment "${{ steps.set-environment.outputs.environment_name }}" \
             --targetComponent "api" \
             --targetAccountGroup "nhs-notify-supplier-api-dev" \
             --terraformAction "apply" \
@@ -157,12 +165,11 @@ jobs:
   artefact-proxies:
     name: "Build proxies"
     runs-on: ubuntu-latest
-    if: inputs.pr_number != ''
+    if: inputs.deploy_proxy == 'true'
     needs: [artefact-oas-spec-sandbox, pr-create-dynamic-environment]
     timeout-minutes: 10
     env:
       PROXYGEN_API_NAME: nhs-notify-supplier
-      PR_NUMBER: ${{ inputs.pr_number }}
       APP_CLIENT_ID: ${{ secrets.APP_CLIENT_ID }}
       APP_PEM_FILE: ${{ secrets.APP_PEM_FILE }}
     steps:
@@ -172,7 +179,7 @@ jobs:
         uses: ./.github/actions/build-proxies
         with:
           version: "${{ inputs.version }}"
-          environment: ${{ inputs.pr_number != '' && format('pr{0}', inputs.pr_number) || 'main' }}
+          environment: ${{ needs.pr-create-dynamic-environment.outputs.environment_name }}
           apimEnv: "internal-dev-sandbox"
           runId: "${{ github.run_id }}"
           buildSandbox: true

.github/workflows/stage-4-acceptance.yaml

@@ -34,6 +34,10 @@ on:
       pr_number:
         required: true
         type: string
+      proxy_deployed:
+        description: "True if the APIM proxy was deployed"
+        required: true
+        type: string
 
 permissions:
   id-token: write
@@ -77,3 +81,20 @@ jobs:
             --targetEnvironment "$ENVIRONMENT" \
             --targetAccountGroup "nhs-notify-supplier-api-dev" \
             --targetComponent "api"
+
+  run-e2e-tests:
+    name: Run End-to-End Tests
+    runs-on: ubuntu-latest
+    if: inputs.proxy_deployed == 'true'
+    steps:
+      - uses: actions/checkout@v5.0.0
+
+      - name: "Run e2e tests"
+        #uses: ./.github/actions/e2e-tests
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          NON_PROD_API_KEY: ${{ secrets.NON_PROD_API_KEY }}
+          INTERNAL_DEV_TEST_PEM: ${{ secrets.INTERNAL_DEV_TEST_PEM }}
+        shell: bash
+        run: |
+          echo "E2E tests are currently disabled. See CCM-14778"

Makefile

@@ -104,6 +104,9 @@ config:: _install-dependencies version # Configure development environment (main
 test-component:
 	(cd tests && npm install && npm run test:component)
 
+test-sandbox:
+	(cd tests && npm install && npm run test:sandbox)
+
 test-performance:
 	(cd tests && npm install && npm run test:performance)
 

infrastructure/terraform/components/api/README.md

@@ -20,15 +20,20 @@ No requirements.
 | <a name="input_enable_alarms"></a> [enable\_alarms](#input\_enable\_alarms) | Enable CloudWatch alarms for this deployed environment | `bool` | `true` | no |
 | <a name="input_enable_api_data_trace"></a> [enable\_api\_data\_trace](#input\_enable\_api\_data\_trace) | Enable API Gateway data trace logging | `bool` | `false` | no |
 | <a name="input_enable_backups"></a> [enable\_backups](#input\_enable\_backups) | Enable backups | `bool` | `false` | no |
+| <a name="input_enable_event_anomaly_detection"></a> [enable\_event\_anomaly\_detection](#input\_enable\_event\_anomaly\_detection) | Enable CloudWatch anomaly detection alarm for SNS message  Detects abnormal drops or spikes in event publishing volume. | `bool` | `true` | no |
 | <a name="input_enable_event_cache"></a> [enable\_event\_cache](#input\_enable\_event\_cache) | Enable caching of events to an S3 bucket | `bool` | `true` | no |
 | <a name="input_enable_sns_delivery_logging"></a> [enable\_sns\_delivery\_logging](#input\_enable\_sns\_delivery\_logging) | Enable SNS Delivery Failure Notifications | `bool` | `true` | no |
 | <a name="input_environment"></a> [environment](#input\_environment) | The name of the tfscaffold environment | `string` | n/a | yes |
+| <a name="input_event_anomaly_band_width"></a> [event\_anomaly\_band\_width](#input\_event\_anomaly\_band\_width) | The width of the anomaly detection band. Higher values (e.g. 4-6) reduce sensitivity and noise, lower values (e.g. 2-3) increase sensitivity. Recommended: 2-4. | `number` | `4` | no |
+| <a name="input_event_anomaly_evaluation_periods"></a> [event\_anomaly\_evaluation\_periods](#input\_event\_anomaly\_evaluation\_periods) | Number of evaluation periods for the anomaly alarm. Each period is defined by event\_anomaly\_period. | `number` | `3` | no |
+| <a name="input_event_anomaly_period"></a> [event\_anomaly\_period](#input\_event\_anomaly\_period) | The period in seconds over which the specified statistic is applied for anomaly detection. Minimum 300 seconds (5 minutes). Recommended: 300-600. | `number` | `300` | no |
 | <a name="input_eventpub_control_plane_bus_arn"></a> [eventpub\_control\_plane\_bus\_arn](#input\_eventpub\_control\_plane\_bus\_arn) | ARN of the EventBridge control plane bus for eventpub | `string` | `""` | no |
 | <a name="input_eventpub_data_plane_bus_arn"></a> [eventpub\_data\_plane\_bus\_arn](#input\_eventpub\_data\_plane\_bus\_arn) | ARN of the EventBridge data plane bus for eventpub | `string` | `""` | no |
 | <a name="input_force_destroy"></a> [force\_destroy](#input\_force\_destroy) | Flag to force deletion of S3 buckets | `bool` | `false` | no |
 | <a name="input_force_lambda_code_deploy"></a> [force\_lambda\_code\_deploy](#input\_force\_lambda\_code\_deploy) | If the lambda package in s3 has the same commit id tag as the terraform build branch, the lambda will not update automatically. Set to True if making changes to Lambda code from on the same commit for example during development | `bool` | `false` | no |
 | <a name="input_group"></a> [group](#input\_group) | The group variables are being inherited from (often synonmous with account short-name) | `string` | n/a | yes |
 | <a name="input_kms_deletion_window"></a> [kms\_deletion\_window](#input\_kms\_deletion\_window) | When a kms key is deleted, how long should it wait in the pending deletion state? | `string` | `"30"` | no |
+| <a name="input_letter_event_source"></a> [letter\_event\_source](#input\_letter\_event\_source) | Source value to use for the letter status event updates | `string` | `"/data-plane/supplier-api/nhs-supplier-api-prod/main/update-status"` | no |
 | <a name="input_letter_table_ttl_hours"></a> [letter\_table\_ttl\_hours](#input\_letter\_table\_ttl\_hours) | Number of hours to set as TTL on letters table | `number` | `24` | no |
 | <a name="input_letter_variant_map"></a> [letter\_variant\_map](#input\_letter\_variant\_map) | n/a | `map(object({ supplierId = string, specId = string }))` | <pre>{<br/>  "lv1": {<br/>    "specId": "spec1",<br/>    "supplierId": "supplier1"<br/>  },<br/>  "lv2": {<br/>    "specId": "spec2",<br/>    "supplierId": "supplier1"<br/>  },<br/>  "lv3": {<br/>    "specId": "spec3",<br/>    "supplierId": "supplier2"<br/>  }<br/>}</pre> | no |
 | <a name="input_log_level"></a> [log\_level](#input\_log\_level) | The log level to be used in lambda functions within the component. Any log with a lower severity than the configured value will not be logged: https://docs.python.org/3/library/logging.html#levels | `string` | `"INFO"` | no |
@@ -45,31 +50,31 @@ No requirements.
 | Name | Source | Version |
 |------|--------|---------|
 | <a name="module_amendment_event_transformer"></a> [amendment\_event\_transformer](#module\_amendment\_event\_transformer) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a |
-| <a name="module_amendments_queue"></a> [amendments\_queue](#module\_amendments\_queue) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.5/terraform-sqs.zip | n/a |
+| <a name="module_amendments_queue"></a> [amendments\_queue](#module\_amendments\_queue) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.6/terraform-sqs.zip | n/a |
 | <a name="module_authorizer_lambda"></a> [authorizer\_lambda](#module\_authorizer\_lambda) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a |
 | <a name="module_ddb_alarms_letter_queue"></a> [ddb\_alarms\_letter\_queue](#module\_ddb\_alarms\_letter\_queue) | ../../modules/alarms-ddb | n/a |
 | <a name="module_ddb_alarms_letters"></a> [ddb\_alarms\_letters](#module\_ddb\_alarms\_letters) | ../../modules/alarms-ddb | n/a |
 | <a name="module_ddb_alarms_mi"></a> [ddb\_alarms\_mi](#module\_ddb\_alarms\_mi) | ../../modules/alarms-ddb | n/a |
 | <a name="module_ddb_alarms_suppliers"></a> [ddb\_alarms\_suppliers](#module\_ddb\_alarms\_suppliers) | ../../modules/alarms-ddb | n/a |
-| <a name="module_domain_truststore"></a> [domain\_truststore](#module\_domain\_truststore) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.4/terraform-s3bucket.zip | n/a |
-| <a name="module_eventpub"></a> [eventpub](#module\_eventpub) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.4/terraform-eventpub.zip | n/a |
+| <a name="module_domain_truststore"></a> [domain\_truststore](#module\_domain\_truststore) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.6/terraform-s3bucket.zip | n/a |
+| <a name="module_eventpub"></a> [eventpub](#module\_eventpub) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.6/terraform-eventpub.zip | n/a |
 | <a name="module_eventsub"></a> [eventsub](#module\_eventsub) | ../../modules/eventsub | n/a |
 | <a name="module_get_letter"></a> [get\_letter](#module\_get\_letter) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a |
 | <a name="module_get_letter_data"></a> [get\_letter\_data](#module\_get\_letter\_data) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a |
 | <a name="module_get_letters"></a> [get\_letters](#module\_get\_letters) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a |
 | <a name="module_get_status"></a> [get\_status](#module\_get\_status) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a |
 | <a name="module_kms"></a> [kms](#module\_kms) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.26/terraform-kms.zip | n/a |
 | <a name="module_lambda_alarms"></a> [lambda\_alarms](#module\_lambda\_alarms) | ../../modules/alarms-lambda | n/a |
-| <a name="module_letter_status_updates_queue"></a> [letter\_status\_updates\_queue](#module\_letter\_status\_updates\_queue) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.5/terraform-sqs.zip | n/a |
+| <a name="module_letter_status_updates_queue"></a> [letter\_status\_updates\_queue](#module\_letter\_status\_updates\_queue) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.6/terraform-sqs.zip | n/a |
 | <a name="module_letter_updates_transformer"></a> [letter\_updates\_transformer](#module\_letter\_updates\_transformer) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a |
 | <a name="module_mi_updates_transformer"></a> [mi\_updates\_transformer](#module\_mi\_updates\_transformer) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.26/terraform-lambda.zip | n/a |
 | <a name="module_patch_letter"></a> [patch\_letter](#module\_patch\_letter) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a |
 | <a name="module_post_letters"></a> [post\_letters](#module\_post\_letters) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a |
 | <a name="module_post_mi"></a> [post\_mi](#module\_post\_mi) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a |
 | <a name="module_s3bucket_test_letters"></a> [s3bucket\_test\_letters](#module\_s3bucket\_test\_letters) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.26/terraform-s3bucket.zip | n/a |
 | <a name="module_sqs_alarms"></a> [sqs\_alarms](#module\_sqs\_alarms) | ../../modules/alarms-sqs | n/a |
-| <a name="module_sqs_letter_updates"></a> [sqs\_letter\_updates](#module\_sqs\_letter\_updates) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.5/terraform-sqs.zip | n/a |
-| <a name="module_sqs_supplier_allocator"></a> [sqs\_supplier\_allocator](#module\_sqs\_supplier\_allocator) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.5/terraform-sqs.zip | n/a |
+| <a name="module_sqs_letter_updates"></a> [sqs\_letter\_updates](#module\_sqs\_letter\_updates) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.6/terraform-sqs.zip | n/a |
+| <a name="module_sqs_supplier_allocator"></a> [sqs\_supplier\_allocator](#module\_sqs\_supplier\_allocator) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.6/terraform-sqs.zip | n/a |
 | <a name="module_supplier_allocator"></a> [supplier\_allocator](#module\_supplier\_allocator) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a |
 | <a name="module_supplier_ssl"></a> [supplier\_ssl](#module\_supplier\_ssl) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.26/terraform-ssl.zip | n/a |
 | <a name="module_update_letter_queue"></a> [update\_letter\_queue](#module\_update\_letter\_queue) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a |

infrastructure/terraform/components/api/module_domain_truststore.tf

@@ -1,5 +1,5 @@
 module "domain_truststore" {
-  source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.4/terraform-s3bucket.zip"
+  source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.6/terraform-s3bucket.zip"
 
   name           = "truststore"
   aws_account_id = var.aws_account_id

infrastructure/terraform/components/api/module_lambda_letter_updates_transformer.tf

@@ -36,7 +36,7 @@ module "letter_updates_transformer" {
 
   lambda_env_vars = merge(local.common_lambda_env_vars, {
     EVENTPUB_SNS_TOPIC_ARN = "${module.eventpub.sns_topic.arn}",
-    EVENT_SOURCE           = "/data-plane/supplier-api/${var.group}/${var.environment}/letters"
+    EVENT_SOURCE           = var.letter_event_source
   })
 }