Update dependencies and refactor tests to use unified mock_aws decorator (#1398)

- Bump `moto` version to 5.1.22 in both `poetry.lock` and `pyproject.toml` for improved AWS mocking capabilities.
- Update Python version requirements for `moto` and other dependencies in `poetry.lock`.
- Refactor test files to replace individual service mocks (e.g., `mock_s3`, `mock_dynamodb`) with a unified `mock_aws` decorator for cleaner test setup.
- Introduce helper functions for creating Boto3 clients and resetting common clients to streamline test initialization.
- Update environment variable mocks to include AWS credentials for testing purposes.

Author: Thomas-Boyle

lambdas/filenameprocessor/poetry.lock

@@ -15,7 +15,7 @@ python = "~3.11"
 boto3 = "~1.42.74"
 boto3-stubs-lite = { extras = ["dynamodb"], version = "~1.42.74" }
 aws-lambda-typing = "~2.20.0"
-moto = "^4"
+moto = "^5.1.22"
 requests = "~2.32.5"
 responses = "~0.26.0"
 pydantic = "~1.10.13"

lambdas/filenameprocessor/pyproject.toml

@@ -4,36 +4,37 @@
 from unittest import TestCase
 from unittest.mock import Mock, patch
 
-from boto3 import client as boto3_client
-from moto import mock_s3
+from moto import mock_aws
 
 from utils_for_tests.mock_environment_variables import MOCK_ENVIRONMENT_DICT
 from utils_for_tests.utils_for_filenameprocessor_tests import (
     GenericSetUp,
     GenericTearDown,
+    create_boto3_clients,
     create_mock_hget,
 )
 
 # Ensure environment variables are mocked before importing from src files
 with patch.dict("os.environ", MOCK_ENVIRONMENT_DICT):
-    from common.clients import REGION_NAME
     from elasticache import (
         get_supplier_permissions_from_cache,
         get_supplier_system_from_cache,
         get_valid_vaccine_types_from_cache,
     )
 
-s3_client = boto3_client("s3", region_name=REGION_NAME)
+s3_client = None
 
 
-@mock_s3
+@mock_aws
 @patch.dict("os.environ", MOCK_ENVIRONMENT_DICT)
 @patch("elasticache.get_redis_client")
 class TestElasticache(TestCase):
     """Tests for elasticache functions"""
 
     def setUp(self):
         """Set up the S3 buckets"""
+        global s3_client
+        (s3_client,) = create_boto3_clients("s3")
         GenericSetUp(s3_client)
 
     def tearDown(self):

lambdas/filenameprocessor/tests/test_elasticache.py

@@ -9,8 +9,7 @@
 from unittest.mock import ANY, Mock, patch
 
 import fakeredis
-from boto3 import client as boto3_client
-from moto import mock_dynamodb, mock_firehose, mock_s3, mock_sqs
+from moto import mock_aws
 
 from utils_for_tests.mock_environment_variables import (
     MOCK_ENVIRONMENT_DICT,
@@ -22,6 +21,7 @@
     GenericSetUp,
     GenericTearDown,
     assert_audit_table_entry,
+    create_boto3_clients,
     create_mock_hget,
 )
 from utils_for_tests.values_for_tests import (
@@ -34,15 +34,14 @@
 
 # Ensure environment variables are mocked before importing from src files
 with patch.dict("os.environ", MOCK_ENVIRONMENT_DICT):
-    from common.clients import REGION_NAME
     from common.models.batch_constants import AUDIT_TABLE_NAME, AuditTableKeys, FileStatus
     from constants import EXTENDED_ATTRIBUTES_VACC_TYPE
     from file_name_processor import handle_record, lambda_handler
 
-s3_client = boto3_client("s3", region_name=REGION_NAME)
-sqs_client = boto3_client("sqs", region_name=REGION_NAME)
-firehose_client = boto3_client("firehose", region_name=REGION_NAME)
-dynamodb_client = boto3_client("dynamodb", region_name=REGION_NAME)
+s3_client = None
+sqs_client = None
+firehose_client = None
+dynamodb_client = None
 
 # NOTE: The default throughout these tests is to use permissions config which allows all suppliers full permissions
 # for all vaccine types. This default is overridden for some specific tests.
@@ -52,10 +51,7 @@
 
 
 @patch.dict("os.environ", MOCK_ENVIRONMENT_DICT)
-@mock_s3
-@mock_sqs
-@mock_firehose
-@mock_dynamodb
+@mock_aws
 class TestLambdaHandlerDataSource(TestCase):
     """Tests for lambda_handler when a data sources (vaccine data) file is received."""
 
@@ -87,6 +83,10 @@ def run(self, result=None):
             super().run(result)
 
     def setUp(self):
+        global s3_client, sqs_client, firehose_client, dynamodb_client
+        s3_client, sqs_client, firehose_client, dynamodb_client = create_boto3_clients(
+            "s3", "sqs", "firehose", "dynamodb"
+        )
         GenericSetUp(s3_client, firehose_client, sqs_client, dynamodb_client)
         self.logger_patcher = patch("file_name_processor.logger")
         self.mock_logger = self.logger_patcher.start()
@@ -483,7 +483,9 @@ def test_lambda_handler_extended_attributes_extension_checks(self, mock_get_redi
         s3_client.put_object(Bucket=BucketNames.SOURCE, Key=bad_ext_key, Body=MOCK_EXTENDED_ATTRIBUTES_FILE_CONTENT)
         with patch("file_name_processor.uuid4", return_value="EA_bad_ext_id"):
             lambda_handler(self.make_event([self.make_record(bad_ext_key)]), None)
-        item = self.get_audit_table_items()[-1]
+        item = next(
+            item for item in self.get_audit_table_items() if item[AuditTableKeys.MESSAGE_ID]["S"] == "EA_bad_ext_id"
+        )
         self.assertEqual(item[AuditTableKeys.STATUS]["S"], "Failed")
         s3_client.get_object(Bucket=BucketNames.SOURCE, Key=f"extended-attributes-archive/{bad_ext_key}")
         """
@@ -673,14 +675,15 @@ def test_lambda_adds_event_to_audit_table_as_failed_when_unexpected_exception_is
 
 
 @patch.dict("os.environ", MOCK_ENVIRONMENT_DICT)
-@mock_s3
-@mock_dynamodb
-@mock_sqs
-@mock_firehose
+@mock_aws
 class TestUnexpectedBucket(TestCase):
     """Tests for lambda_handler when an unexpected bucket name is used"""
 
     def setUp(self):
+        global s3_client, sqs_client, firehose_client, dynamodb_client
+        s3_client, sqs_client, firehose_client, dynamodb_client = create_boto3_clients(
+            "s3", "sqs", "firehose", "dynamodb"
+        )
         GenericSetUp(s3_client, firehose_client, sqs_client, dynamodb_client)
 
     def tearDown(self):

lambdas/filenameprocessor/tests/test_lambda_handler.py

@@ -5,34 +5,38 @@
 from unittest import TestCase
 from unittest.mock import patch
 
-from boto3 import client as boto3_client
-from moto import mock_sqs
+from moto import mock_aws
 
 from utils_for_tests.mock_environment_variables import MOCK_ENVIRONMENT_DICT, Sqs
+from utils_for_tests.utils_for_filenameprocessor_tests import create_boto3_clients, reset_common_clients
 from utils_for_tests.values_for_tests import MockFileDetails
 
 # Ensure environment variables are mocked before importing from src files
 with patch.dict("os.environ", MOCK_ENVIRONMENT_DICT):
-    from common.clients import REGION_NAME
     from models.errors import UnhandledSqsError
     from send_sqs_message import make_and_send_sqs_message, send_to_supplier_queue
 
-sqs_client = boto3_client("sqs", region_name=REGION_NAME)
+sqs_client = None
 
 FLU_EMIS_FILE_DETAILS = MockFileDetails.emis_flu
 RSV_RAVS_FILE_DETAILS = MockFileDetails.ravs_rsv_1
 
 NON_EXISTENT_QUEUE_ERROR_MESSAGE = (
-    "An unexpected error occurred whilst sending to SQS: An error occurred (AWS.SimpleQueueService.NonExistent"
-    + "Queue) when calling the SendMessage operation: The specified queue does not exist for this wsdl version."
+    "An unexpected error occurred whilst sending to SQS: An error occurred "
+    "(AWS.SimpleQueueService.NonExistentQueue) when calling the SendMessage operation"
 )
 
 
-@mock_sqs
+@mock_aws
 @patch.dict("os.environ", MOCK_ENVIRONMENT_DICT)
 class TestSendSQSMessage(TestCase):
     """Tests for send_sqs_message functions"""
 
+    def setUp(self):
+        global sqs_client
+        reset_common_clients()
+        (sqs_client,) = create_boto3_clients("sqs")
+
     def test_send_to_supplier_queue_success(self):
         """Test send_to_supplier_queue function for a successful message send"""
         # Set up the sqs_queue
@@ -75,7 +79,7 @@ def test_send_to_supplier_queue_failure_due_to_queue_does_not_exist(self):
                 vaccine_type=FLU_EMIS_FILE_DETAILS.vaccine_type,
                 supplier=FLU_EMIS_FILE_DETAILS.supplier,
             )
-        self.assertEqual(NON_EXISTENT_QUEUE_ERROR_MESSAGE, str(context.exception))
+        self.assertIn(NON_EXISTENT_QUEUE_ERROR_MESSAGE, str(context.exception))
 
     def test_make_and_send_sqs_message_success(self):
         """Test make_and_send_sqs_message function for a successful message send"""

lambdas/filenameprocessor/tests/test_send_sqs_message.py

@@ -4,32 +4,33 @@
 from unittest import TestCase
 from unittest.mock import patch
 
-from boto3 import client as boto3_client
-from moto import mock_s3
+from moto import mock_aws
 
 from utils_for_tests.mock_environment_variables import (
     MOCK_ENVIRONMENT_DICT,
 )
 from utils_for_tests.utils_for_filenameprocessor_tests import (
     GenericSetUp,
     GenericTearDown,
+    create_boto3_clients,
 )
 
 # Ensure environment variables are mocked before importing from src files
 with patch.dict("os.environ", MOCK_ENVIRONMENT_DICT):
-    from common.clients import REGION_NAME
     from constants import AUDIT_TABLE_TTL_DAYS
     from utils_for_filenameprocessor import get_creation_and_expiry_times
 
-s3_client = boto3_client("s3", region_name=REGION_NAME)
+s3_client = None
 
 
-@mock_s3
+@mock_aws
 class TestUtilsForFilenameprocessor(TestCase):
     """Tests for utils_for_filenameprocessor functions"""
 
     def setUp(self):
         """Set up the s3 buckets"""
+        global s3_client
+        (s3_client,) = create_boto3_clients("s3")
         GenericSetUp(s3_client)
 
     def tearDown(self):

lambdas/filenameprocessor/tests/test_utils_for_filenameprocessor.py

@@ -35,6 +35,11 @@ class Sqs:
 # NOTE: FILE_NAME_GSI and CONFIG_BUCKET_NAME environment variables are set in the terraform,
 # but not used in the src code and so are not set here.
 MOCK_ENVIRONMENT_DICT = {
+    "AWS_ACCESS_KEY_ID": "testing",
+    "AWS_SECRET_ACCESS_KEY": "testing",
+    "AWS_SESSION_TOKEN": "testing",
+    "AWS_REGION": REGION_NAME,
+    "AWS_DEFAULT_REGION": REGION_NAME,
     "SOURCE_BUCKET_NAME": BucketNames.SOURCE,
     "ACK_BUCKET_NAME": BucketNames.DESTINATION,
     "DPS_BUCKET_NAME": BucketNames.DPS_DESTINATION,