diff --git a/tests/blackbox/s3_assertions.py b/tests/blackbox/s3_assertions.py index f848e8fd7abb..98ae75b9bcfc 100644 --- a/tests/blackbox/s3_assertions.py +++ b/tests/blackbox/s3_assertions.py @@ -778,6 +778,9 @@ def assert_get_access_point(request): assert "/v20180820/accesspoint/" in request.path, ( f"GetAccessPoint: expected /v20180820/accesspoint/ in {request.path}" ) + assert ( + request.headers.get("x-amz-account-id") is not None + ), "GetAccessPoint: required header x-amz-account-id is missing" def assert_get_caller_identity(request): @@ -785,3 +788,7 @@ def assert_get_caller_identity(request): assert request.method == "POST", ( f"GetCallerIdentity: expected POST, got {request.method}" ) + assert "Action=GetCallerIdentity" in request.body, ( + f"GetCallerIdentity: expected Action=GetCallerIdentity in body, " + f"got {request.body!r}" + ) diff --git a/tests/blackbox/test_cp_command.py b/tests/blackbox/test_cp_command.py index 3dae7d9262f1..4be7dab02cff 100644 --- a/tests/blackbox/test_cp_command.py +++ b/tests/blackbox/test_cp_command.py @@ -39,10 +39,10 @@ get_object_tagging_response, head_object_response, list_objects_xml, + mock_server, put_object_response, put_object_tagging_response, run_cli, - mock_server, setup_responses, upload_part_copy_response, upload_part_response, @@ -452,12 +452,8 @@ async def test_operations_used_in_download_file(self, aws_cli, tmp_path): assert rc == 0, stderr.decode() assert len(server.requests) == 2, format_requests(server) - assert_head_object( - server.requests[0], Bucket="bucket", Key="key.txt" - ) - assert_get_object( - server.requests[1], Bucket="bucket", Key="key.txt" - ) + assert_head_object(server.requests[0], Bucket="bucket", Key="key.txt") + assert_get_object(server.requests[1], Bucket="bucket", Key="key.txt") async def test_operations_used_in_recursive_download( self, aws_cli, tmp_path @@ -482,9 +478,7 @@ async def test_operations_used_in_recursive_download( assert rc == 0, stderr.decode() assert len(server.requests) == 1, format_requests(server) - assert_list_objects_v2( - server.requests[0], Bucket="bucket" - ) + assert_list_objects_v2(server.requests[0], Bucket="bucket") async def test_no_overwrite_flag_when_object_not_exists_on_target( self, aws_cli, tmp_path @@ -512,9 +506,7 @@ async def test_no_overwrite_flag_when_object_not_exists_on_target( assert rc == 0, stderr.decode() assert len(server.requests) == 1, format_requests(server) req = server.requests[0] - assert_put_object( - req, Bucket="bucket", Key="key.txt", IfNoneMatch="*" - ) + assert_put_object(req, Bucket="bucket", Key="key.txt", IfNoneMatch="*") async def test_no_overwrite_flag_when_object_exists_on_target( self, aws_cli, tmp_path @@ -551,9 +543,7 @@ async def test_no_overwrite_flag_when_object_exists_on_target( assert rc == 0, stderr.decode() assert len(server.requests) == 1, format_requests(server) req = server.requests[0] - assert_put_object( - req, Bucket="bucket", Key="key.txt", IfNoneMatch="*" - ) + assert_put_object(req, Bucket="bucket", Key="key.txt", IfNoneMatch="*") async def test_no_overwrite_flag_multipart_upload_when_object_not_exists_on_target( self, aws_cli, tmp_path @@ -686,9 +676,7 @@ async def test_no_overwrite_flag_on_copy_when_small_object_does_not_exist_on_tar assert rc == 0, stderr.decode() assert len(server.requests) == 2, format_requests(server) - assert_head_object( - server.requests[0], Bucket="bucket1", Key="key.txt" - ) + assert_head_object(server.requests[0], Bucket="bucket1", Key="key.txt") assert_copy_object( server.requests[1], Bucket="bucket", @@ -730,9 +718,7 @@ async def test_no_overwrite_flag_on_copy_when_small_object_exists_on_target( assert rc == 0, stderr.decode() assert len(server.requests) == 2, format_requests(server) - assert_head_object( - server.requests[0], Bucket="bucket1", Key="key.txt" - ) + assert_head_object(server.requests[0], Bucket="bucket1", Key="key.txt") assert_copy_object( server.requests[1], Bucket="bucket", @@ -861,9 +847,7 @@ async def test_metadata_copy(self, aws_cli, tmp_path): assert rc == 0, stderr.decode() assert len(server.requests) == 2, format_requests(server) - assert_head_object( - server.requests[0], Bucket="bucket", Key="key.txt" - ) + assert_head_object(server.requests[0], Bucket="bucket", Key="key.txt") assert_copy_object( server.requests[1], Bucket="bucket", @@ -974,9 +958,7 @@ async def test_metadata_directive_copy(self, aws_cli, tmp_path): assert rc == 0, stderr.decode() assert len(server.requests) == 2, format_requests(server) - assert_head_object( - server.requests[0], Bucket="bucket", Key="key.txt" - ) + assert_head_object(server.requests[0], Bucket="bucket", Key="key.txt") assert_copy_object( server.requests[1], Bucket="bucket", @@ -1056,9 +1038,7 @@ async def test_recursive_glacier_download_with_force_glacier( assert rc == 0, stderr.decode() assert len(server.requests) == 2, format_requests(server) - assert_list_objects_v2( - server.requests[0], Bucket="bucket" - ) + assert_list_objects_v2(server.requests[0], Bucket="bucket") assert_get_object( server.requests[1], Bucket="bucket", Key="foo/bar.txt" ) @@ -1098,9 +1078,7 @@ async def test_recursive_glacier_download_without_force_glacier( assert rc == 2 assert len(server.requests) == 1, format_requests(server) - assert_list_objects_v2( - server.requests[0], Bucket="bucket" - ) + assert_list_objects_v2(server.requests[0], Bucket="bucket") assert "GLACIER" in stderr.decode() async def test_warns_on_glacier_incompatible_operation( @@ -1134,9 +1112,7 @@ async def test_warns_on_glacier_incompatible_operation( assert rc == 2 assert len(server.requests) == 1, format_requests(server) - assert_head_object( - server.requests[0], Bucket="bucket", Key="key.txt" - ) + assert_head_object(server.requests[0], Bucket="bucket", Key="key.txt") assert "GLACIER" in stderr.decode() async def test_warns_on_deep_archive_incompatible_operation( @@ -1170,9 +1146,7 @@ async def test_warns_on_deep_archive_incompatible_operation( assert rc == 2 assert len(server.requests) == 1, format_requests(server) - assert_head_object( - server.requests[0], Bucket="bucket", Key="key.txt" - ) + assert_head_object(server.requests[0], Bucket="bucket", Key="key.txt") assert "GLACIER" in stderr.decode() async def test_turn_off_glacier_warnings(self, aws_cli, tmp_path): @@ -1210,9 +1184,7 @@ async def test_turn_off_glacier_warnings(self, aws_cli, tmp_path): assert rc == 0, stderr.decode() assert len(server.requests) == 1, format_requests(server) - assert_head_object( - server.requests[0], Bucket="bucket", Key="key.txt" - ) + assert_head_object(server.requests[0], Bucket="bucket", Key="key.txt") assert stderr.decode() == "" async def test_turn_off_glacier_warnings_for_deep_archive( @@ -1252,9 +1224,7 @@ async def test_turn_off_glacier_warnings_for_deep_archive( assert rc == 0, stderr.decode() assert len(server.requests) == 1, format_requests(server) - assert_head_object( - server.requests[0], Bucket="bucket", Key="key.txt" - ) + assert_head_object(server.requests[0], Bucket="bucket", Key="key.txt") assert stderr.decode() == "" async def test_cp_with_sse_flag(self, aws_cli, tmp_path): @@ -2189,9 +2159,7 @@ async def test_no_overwrite_flag_on_copy_when_large_object_does_not_exist_on_tar Bucket="bucket", Key="key1.txt", UploadId="foo", - MultipartUpload={ - "Parts": [{"PartNumber": "1"}, {"PartNumber": "2"}] - }, + MultipartUpload=[{"PartNumber": "1"}, {"PartNumber": "2"}], IfNoneMatch="*", ) @@ -2239,9 +2207,7 @@ async def test_no_overwrite_flag_on_copy_when_large_object_exists_on_target( Bucket="bucket", Key="key.txt", UploadId="foo", - MultipartUpload={ - "Parts": [{"PartNumber": "1"}, {"PartNumber": "2"}] - }, + MultipartUpload=[{"PartNumber": "1"}, {"PartNumber": "2"}], IfNoneMatch="*", ) # AbortMultipartUpload should have been called @@ -2277,9 +2243,7 @@ async def test_no_overwrite_flag_on_download_when_single_object_already_exists_a assert rc == 0, stderr.decode() assert len(server.requests) == 1, format_requests(server) - assert_head_object( - server.requests[0], Bucket="bucket", Key="foo.txt" - ) + assert_head_object(server.requests[0], Bucket="bucket", Key="foo.txt") # File should not have been overwritten assert target.read_text() == "existing content" @@ -2313,12 +2277,8 @@ async def test_no_overwrite_flag_on_download_when_single_object_does_not_exist_a assert rc == 0, stderr.decode() assert len(server.requests) == 2, format_requests(server) - assert_head_object( - server.requests[0], Bucket="bucket", Key="foo.txt" - ) - assert_get_object( - server.requests[1], Bucket="bucket", Key="foo.txt" - ) + assert_head_object(server.requests[0], Bucket="bucket", Key="foo.txt") + assert_get_object(server.requests[1], Bucket="bucket", Key="foo.txt") assert target.read_text() == "foo" async def test_warns_on_deep_arhive_incompatible_operation( @@ -2352,9 +2312,7 @@ async def test_warns_on_deep_arhive_incompatible_operation( assert rc == 2 assert len(server.requests) == 1, format_requests(server) - assert_head_object( - server.requests[0], Bucket="bucket", Key="key.txt" - ) + assert_head_object(server.requests[0], Bucket="bucket", Key="key.txt") assert "GLACIER" in stderr.decode() async def test_warns_on_glacier_incompatible_operation_for_multipart_file( @@ -2388,9 +2346,7 @@ async def test_warns_on_glacier_incompatible_operation_for_multipart_file( assert rc == 2 assert len(server.requests) == 1, format_requests(server) - assert_head_object( - server.requests[0], Bucket="bucket", Key="key.txt" - ) + assert_head_object(server.requests[0], Bucket="bucket", Key="key.txt") assert "GLACIER" in stderr.decode() async def test_warns_on_deep_archive_incompatible_op_for_multipart_file( @@ -2424,9 +2380,7 @@ async def test_warns_on_deep_archive_incompatible_op_for_multipart_file( assert rc == 2 assert len(server.requests) == 1, format_requests(server) - assert_head_object( - server.requests[0], Bucket="bucket", Key="key.txt" - ) + assert_head_object(server.requests[0], Bucket="bucket", Key="key.txt") assert "GLACIER" in stderr.decode() async def test_s3s3_cp_with_destination_sse_c_multipart( @@ -3104,9 +3058,7 @@ async def test_multipart_copy(self, aws_cli, tmp_path): Bucket="mybucket", Key="mykey", UploadId="id", - MultipartUpload={ - "Parts": [{"PartNumber": "1"}, {"PartNumber": "2"}] - }, + MultipartUpload=[{"PartNumber": "1"}, {"PartNumber": "2"}], RequestPayer="requester", ) @@ -3281,9 +3233,7 @@ async def test_upload(self, aws_cli, tmp_path): assert rc == 0, stderr.decode() assert len(server.requests) == 1, format_requests(server) - assert_put_object( - server.requests[0], Bucket=self.ARN, Key="mykey" - ) + assert_put_object(server.requests[0], Bucket=self.ARN, Key="mykey") async def test_recursive_upload(self, aws_cli, tmp_path): """cp local s3:/// --recursive uploads to accesspoint.""" @@ -3307,9 +3257,7 @@ async def test_recursive_upload(self, aws_cli, tmp_path): assert rc == 0, stderr.decode() assert len(server.requests) == 1, format_requests(server) - assert_put_object( - server.requests[0], Bucket=self.ARN, Key="myfile" - ) + assert_put_object(server.requests[0], Bucket=self.ARN, Key="myfile") async def test_download(self, aws_cli, tmp_path): """cp s3:///key local downloads from accesspoint.""" @@ -3338,9 +3286,7 @@ async def test_download(self, aws_cli, tmp_path): Key="mykey", ChecksumMode="ENABLED", ) - assert_get_object( - server.requests[1], Bucket=self.ARN, Key="mykey" - ) + assert_get_object(server.requests[1], Bucket=self.ARN, Key="mykey") async def test_recursive_download(self, aws_cli, tmp_path): """cp s3:// local --recursive downloads from accesspoint.""" @@ -3377,9 +3323,7 @@ async def test_recursive_download(self, aws_cli, tmp_path): server.requests[0], Bucket=self.ARN, ) - assert_get_object( - server.requests[1], Bucket=self.ARN, Key="mykey" - ) + assert_get_object(server.requests[1], Bucket=self.ARN, Key="mykey") async def test_copy(self, aws_cli, tmp_path): """cp s3:///key s3:///key copies between accesspoints.""" @@ -3414,9 +3358,7 @@ async def test_copy(self, aws_cli, tmp_path): Key="mykey", ChecksumMode="ENABLED", ) - assert_copy_object( - server.requests[1], Bucket=dest_arn, Key="mykey" - ) + assert_copy_object(server.requests[1], Bucket=dest_arn, Key="mykey") async def test_recursive_copy(self, aws_cli, tmp_path): """cp s3:// s3:// --recursive copies between accesspoints.""" @@ -3460,9 +3402,7 @@ async def test_recursive_copy(self, aws_cli, tmp_path): server.requests[0], Bucket=self.ARN, ) - assert_copy_object( - server.requests[1], Bucket=dest_arn, Key="mykey" - ) + assert_copy_object(server.requests[1], Bucket=dest_arn, Key="mykey") async def test_accepts_mrap_arns(self, aws_cli, tmp_path): """cp to MRAP ARN (colon separator) works.""" @@ -3482,9 +3422,7 @@ async def test_accepts_mrap_arns(self, aws_cli, tmp_path): assert rc == 0, stderr.decode() assert len(server.requests) == 1, format_requests(server) - assert_put_object( - server.requests[0], Bucket=mrap_arn, Key="mykey" - ) + assert_put_object(server.requests[0], Bucket=mrap_arn, Key="mykey") async def test_accepts_mrap_arns_with_slash(self, aws_cli, tmp_path): """cp to MRAP ARN (slash separator) works.""" @@ -3504,9 +3442,7 @@ async def test_accepts_mrap_arns_with_slash(self, aws_cli, tmp_path): assert rc == 0, stderr.decode() assert len(server.requests) == 1, format_requests(server) - assert_put_object( - server.requests[0], Bucket=mrap_arn, Key="mykey" - ) + assert_put_object(server.requests[0], Bucket=mrap_arn, Key="mykey") @pytest.mark.asyncio @@ -5290,9 +5226,7 @@ async def test_respects_source_region_for_recursive_mp_copy( assert rc == 0, stderr.decode() assert len(server.requests) == 7, format_requests(server) # Source region: ListObjectsV2 - assert_list_objects_v2( - server.requests[0], Bucket=self.SOURCE_BUCKET - ) + assert_list_objects_v2(server.requests[0], Bucket=self.SOURCE_BUCKET) assert server.requests[0].headers.get("host") == self.SOURCE_HOST # Source region: HeadObject (for metadata + size/etag) assert_head_object( @@ -5481,3 +5415,123 @@ async def test_s3_express_warn_emits_warning(self, aws_cli, tmp_path): assert rc == 0, stderr.decode() assert "warning: Recursive copies/moves" in stderr.decode() + + +@pytest.mark.asyncio +async def test_upload_key_with_spaces(aws_cli, tmp_path): + """cp uploads a file whose S3 key contains spaces.""" + src = tmp_path / "my file.txt" + src.write_text("content") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [put_object_response()]) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "cp", str(src), "s3://bucket/my file.txt"], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + # Space must be percent-encoded as %20, not + or literal space + assert server.requests[0].path == "/my%20file.txt" + + +@pytest.mark.asyncio +async def test_download_unicode_key_from_s3(aws_cli, tmp_path): + """cp --recursive downloads an object with a Unicode key.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + xml_response( + list_objects_xml( + contents=[ + { + "Key": "données.txt", + "Size": 5, + "LastModified": "2023-01-01T00:00:00Z", + } + ] + ) + ), + get_object_response(b"hello"), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "cp", "s3://bucket/", str(tmp_path), "--recursive"], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert (tmp_path / "données.txt").exists() + + +@pytest.mark.asyncio +async def test_upload_file_with_unicode_local_name(aws_cli, tmp_path): + """cp uploads a local file with a Unicode filename.""" + src = tmp_path / "données.txt" + src.write_text("content") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [put_object_response()]) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "cp", str(src), "s3://bucket/"], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + # Unicode filename is percent-encoded as UTF-8 on the wire + assert server.requests[0].path == "/donn%C3%A9es.txt" + + +@pytest.mark.asyncio +async def test_user_agent_contains_cli_version(aws_cli, tmp_path): + """Requests include a User-Agent with aws-cli version info.""" + src = tmp_path / "foo.txt" + src.write_text("content") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [put_object_response()]) + _, stderr, rc = await run_cli( + aws_cli, + ["s3", "cp", str(src), "s3://bucket/foo.txt"], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + ua = server.requests[0].headers.get("user-agent") + assert ua is not None, "User-Agent header missing" + assert "aws-cli/" in ua, f"Expected 'aws-cli/' in User-Agent: {ua}" + + +@pytest.mark.asyncio +async def test_user_agent_contains_command(aws_cli, tmp_path): + """User-Agent includes the command being run (e.g. s3.cp).""" + src = tmp_path / "foo.txt" + src.write_text("content") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [put_object_response()]) + _, stderr, rc = await run_cli( + aws_cli, + ["s3", "cp", str(src), "s3://bucket/foo.txt"], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + ua = server.requests[0].headers.get("user-agent") + assert "s3.cp" in ua, f"Expected 's3.cp' in User-Agent: {ua}" diff --git a/tests/blackbox/test_ls_command.py b/tests/blackbox/test_ls_command.py index 31239fcd4dad..848ab62c63bc 100644 --- a/tests/blackbox/test_ls_command.py +++ b/tests/blackbox/test_ls_command.py @@ -12,14 +12,13 @@ assert_list_buckets, assert_list_objects_v2, ) - from tests.blackbox.utils import ( cli_env, format_requests, get_query_params, list_objects_xml, - run_cli, mock_server, + run_cli, setup_responses, xml_response, ) @@ -572,3 +571,77 @@ async def test_list_objects_ignores_bucket_region(aws_cli: str) -> None: req = server.requests[0] params = get_query_params(req) assert "bucket-region" not in params + + +@pytest.mark.asyncio +async def test_list_objects_with_unicode_keys(aws_cli): + """ls handles objects with Unicode keys (CJK, emoji).""" + async with mock_server() as (server, proxy): + setup_responses( + server, + [ + xml_response( + list_objects_xml( + contents=[ + { + "Key": "文件.txt", + "Size": 100, + "LastModified": "2023-01-01T00:00:00Z", + }, + { + "Key": "📄data.csv", + "Size": 200, + "LastModified": "2023-01-01T00:00:00Z", + }, + ] + ) + ), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "ls", "s3://bucket/"], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + output = stdout.decode() + assert "文件.txt" in output + assert "📄data.csv" in output + + +@pytest.mark.asyncio +async def test_list_objects_with_url_encoded_keys(aws_cli): + """ls handles keys that S3 returns with URL encoding.""" + async with mock_server() as (server, proxy): + setup_responses( + server, + [ + xml_response( + list_objects_xml( + contents=[ + { + "Key": "my+file.txt", + "Size": 100, + "LastModified": "2023-01-01T00:00:00Z", + }, + { + "Key": "path/to/my%20doc.pdf", + "Size": 200, + "LastModified": "2023-01-01T00:00:00Z", + }, + ] + ) + ), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "ls", "s3://bucket/"], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + output = stdout.decode() + assert "my+file.txt" in output + assert "my%20doc.pdf" in output diff --git a/tests/blackbox/test_mb_command.py b/tests/blackbox/test_mb_command.py index bc86865e14f7..a9c33f7e2a51 100644 --- a/tests/blackbox/test_mb_command.py +++ b/tests/blackbox/test_mb_command.py @@ -9,13 +9,12 @@ from tests.blackbox.s3_assertions import ( assert_create_bucket, ) - from tests.blackbox.utils import ( cli_env, create_bucket_response, format_requests, - run_cli, mock_server, + run_cli, setup_responses, ) @@ -39,9 +38,7 @@ async def test_make_bucket(self, aws_cli): ) assert rc == 0, stderr.decode() assert len(server.requests) == 1, format_requests(server) - assert_create_bucket( - server.requests[0], Bucket="bucket" - ) + assert_create_bucket(server.requests[0], Bucket="bucket") async def test_adds_location_constraint(self, aws_cli): async with mock_server() as (server, proxy): @@ -298,3 +295,30 @@ async def test_tags_with_three_arguments_fails(self, aws_cli): assert rc == 252 assert len(server.requests) == 0, format_requests(server) assert "ParamValidation" in stderr.decode() + + +@pytest.mark.asyncio +async def test_create_bucket_with_non_ascii_tag_value(aws_cli): + """mb --tags with non-ASCII tag value sends correct XML body.""" + async with mock_server() as (server, proxy): + setup_responses(server, [create_bucket_response()]) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "mb", + "s3://bucket", + "--tags", + "Author", + "José García", + "--region", + "us-west-2", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + req = server.requests[0] + assert ( + "José" in req.body or "Jos" in req.body + ), f"Expected non-ASCII tag value in body, got: {req.body[:200]}" diff --git a/tests/blackbox/test_mv_command.py b/tests/blackbox/test_mv_command.py index 428c678f2c72..a00bb0ebe0a1 100644 --- a/tests/blackbox/test_mv_command.py +++ b/tests/blackbox/test_mv_command.py @@ -1348,7 +1348,11 @@ async def test_mv_works_if_access_point_arn_resolves_to_different_bucket( assert_head_object( server.requests[1], Bucket="bucket", Key="key" ) - assert_copy_object(server.requests[2]) + assert_copy_object( + server.requests[2], + Bucket="arn:aws:s3:us-west-2:123456789012:accesspoint/myaccesspoint", + Key="key", + ) assert_delete_object( server.requests[3], Bucket="bucket", Key="key" ) @@ -1390,7 +1394,11 @@ async def test_mv_works_if_access_point_alias_resolves_to_different_bucket( assert_head_object( server.requests[2], Bucket="bucket", Key="key" ) - assert_copy_object(server.requests[3]) + assert_copy_object( + server.requests[3], + Bucket="myaccesspoint-foobar-s3alias", + Key="key", + ) assert_delete_object( server.requests[4], Bucket="bucket", Key="key" ) @@ -1430,7 +1438,11 @@ async def test_mv_works_if_outpost_access_point_arn_resolves_to_different_bucket assert_head_object( server.requests[1], Bucket="bucket", Key="key" ) - assert_copy_object(server.requests[2]) + assert_copy_object( + server.requests[2], + Bucket="arn:aws:s3-outposts:us-east-1:123456789012:outpost/op-foobar/accesspoint/myaccesspoint", + Key="key", + ) assert_delete_object( server.requests[3], Bucket="bucket", Key="key" ) diff --git a/tests/blackbox/test_s3_config.py b/tests/blackbox/test_s3_config.py new file mode 100644 index 000000000000..5cbe52411bc8 --- /dev/null +++ b/tests/blackbox/test_s3_config.py @@ -0,0 +1,361 @@ +"""Blackbox tests for S3 configuration options. + +These config options affect how the CLI constructs requests on the wire: +endpoint selection, multipart behavior, payload signing, etc. +""" + +from __future__ import annotations + +import pytest +from localstub.handlers import handle_expect_header +from localstub.server import HTTPResponse + +from tests.blackbox.s3_assertions import ( + assert_complete_multipart_upload, + assert_create_multipart_upload, + assert_put_object, + assert_upload_part, +) +from tests.blackbox.utils import ( + cli_env, + complete_mpu_response, + create_mpu_response, + format_requests, + head_object_response, + mock_server, + put_object_response, + run_cli, + setup_responses, + upload_part_response, +) + + +def _cli_env_with_config(proxy, config_path): + env = cli_env(proxy) + env["AWS_CONFIG_FILE"] = config_path + return env + + +@pytest.mark.asyncio +async def test_multipart_threshold_single_put_below( + aws_cli, aws_config, tmp_path +): + """File below multipart_threshold uses single PutObject.""" + src = tmp_path / "small.bin" + src.write_bytes(b"x" * (4 * 1024 * 1024)) + config_path = aws_config( + {"default": {"s3": "\n multipart_threshold = 5MB"}} + ) + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [put_object_response()]) + _, stderr, rc = await run_cli( + aws_cli, + ["s3", "cp", str(src), "s3://bucket/small.bin"], + _cli_env_with_config(proxy, config_path), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + assert_put_object(server.requests[0], Bucket="bucket", Key="small.bin") + + +@pytest.mark.asyncio +async def test_multipart_threshold_multipart_above( + aws_cli, aws_config, tmp_path +): + """File above multipart_threshold uses multipart upload.""" + src = tmp_path / "big.bin" + src.write_bytes(b"x" * (6 * 1024 * 1024)) + config_path = aws_config( + {"default": {"s3": "\n multipart_threshold = 5MB"}} + ) + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + create_mpu_response("upload-id"), + upload_part_response("etag1"), + complete_mpu_response(), + ], + ) + _, stderr, rc = await run_cli( + aws_cli, + ["s3", "cp", str(src), "s3://bucket/big.bin"], + _cli_env_with_config(proxy, config_path), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 3, format_requests(server) + assert_create_multipart_upload( + server.requests[0], Bucket="bucket", Key="big.bin" + ) + assert_upload_part(server.requests[1], Bucket="bucket", Key="big.bin") + assert_complete_multipart_upload( + server.requests[2], + Bucket="bucket", + Key="big.bin", + UploadId="upload-id", + ) + + +@pytest.mark.asyncio +async def test_multipart_chunksize_controls_part_count( + aws_cli, aws_config, tmp_path +): + """multipart_chunksize controls the size of each part.""" + src = tmp_path / "big.bin" + src.write_bytes(b"x" * (12 * 1024 * 1024)) + config_path = aws_config( + { + "default": { + "s3": "\n multipart_threshold = 5MB\n multipart_chunksize = 5MB" + } + } + ) + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + create_mpu_response("upload-id"), + upload_part_response("etag1"), + upload_part_response("etag2"), + upload_part_response("etag3"), + complete_mpu_response(), + ], + ) + _, stderr, rc = await run_cli( + aws_cli, + ["s3", "cp", str(src), "s3://bucket/big.bin"], + _cli_env_with_config(proxy, config_path), + ) + + assert rc == 0, stderr.decode() + assert_create_multipart_upload( + server.requests[0], Bucket="bucket", Key="big.bin" + ) + part_reqs = [ + r + for r in server.requests + if r.method == "PUT" and "partNumber" in r.path + ] + assert len(part_reqs) == 3, format_requests(server) + assert_complete_multipart_upload( + server.requests[4], + Bucket="bucket", + Key="big.bin", + UploadId="upload-id", + ) + + +@pytest.mark.asyncio +async def test_addressing_style_path(aws_cli, aws_config, tmp_path): + """addressing_style=path puts bucket in the path, not the host.""" + src = tmp_path / "foo.txt" + src.write_text("content") + config_path = aws_config( + {"default": {"s3": "\n addressing_style = path"}} + ) + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [put_object_response()]) + _, stderr, rc = await run_cli( + aws_cli, + ["s3", "cp", str(src), "s3://bucket/foo.txt"], + _cli_env_with_config(proxy, config_path), + ) + + assert rc == 0, stderr.decode() + req = server.requests[0] + assert req.headers.get("host") == "s3.us-east-1.amazonaws.com" + assert req.path.startswith("/bucket/foo.txt") + + +@pytest.mark.asyncio +async def test_addressing_style_virtual(aws_cli, aws_config, tmp_path): + """addressing_style=virtual puts bucket in the host.""" + src = tmp_path / "foo.txt" + src.write_text("content") + config_path = aws_config( + {"default": {"s3": "\n addressing_style = virtual"}} + ) + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [put_object_response()]) + _, stderr, rc = await run_cli( + aws_cli, + ["s3", "cp", str(src), "s3://bucket/foo.txt"], + _cli_env_with_config(proxy, config_path), + ) + + assert rc == 0, stderr.decode() + req = server.requests[0] + assert "bucket" in req.headers.get("host", "") + assert req.path == "/foo.txt" or req.path.startswith("/foo.txt") + + +@pytest.mark.asyncio +async def test_use_accelerate_endpoint(aws_cli, aws_config, tmp_path): + """use_accelerate_endpoint=true routes to s3-accelerate.amazonaws.com.""" + src = tmp_path / "foo.txt" + src.write_text("content") + config_path = aws_config( + {"default": {"s3": "\n use_accelerate_endpoint = true"}} + ) + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [put_object_response()]) + _, stderr, rc = await run_cli( + aws_cli, + ["s3", "cp", str(src), "s3://bucket/foo.txt"], + _cli_env_with_config(proxy, config_path), + ) + + assert rc == 0, stderr.decode() + host = server.requests[0].headers.get("host", "") + assert ( + "s3-accelerate" in host + ), f"Expected s3-accelerate in host, got {host}" + + +@pytest.mark.asyncio +async def test_payload_signing_disabled(aws_cli, aws_config, tmp_path): + """payload_signing_enabled=false sends UNSIGNED-PAYLOAD.""" + config_path = aws_config( + {"default": {"s3": "\n payload_signing_enabled = false"}} + ) + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [head_object_response()]) + _, stderr, rc = await run_cli( + aws_cli, + ["s3api", "head-object", "--bucket", "bucket", "--key", "foo.txt"], + _cli_env_with_config(proxy, config_path), + ) + + assert rc == 0, stderr.decode() + sha256 = server.requests[0].headers.get("x-amz-content-sha256") + assert ( + sha256 == "UNSIGNED-PAYLOAD" + ), f"Expected UNSIGNED-PAYLOAD, got {sha256}" + + +@pytest.mark.asyncio +async def test_payload_signing_enabled(aws_cli, aws_config, tmp_path): + """payload_signing_enabled=true sends the actual SHA256 hash.""" + config_path = aws_config( + {"default": {"s3": "\n payload_signing_enabled = true"}} + ) + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [head_object_response()]) + _, stderr, rc = await run_cli( + aws_cli, + ["s3api", "head-object", "--bucket", "bucket", "--key", "foo.txt"], + _cli_env_with_config(proxy, config_path), + ) + + assert rc == 0, stderr.decode() + sha256 = server.requests[0].headers.get("x-amz-content-sha256") + # SHA256 of empty body + assert ( + sha256 + == "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + ), f"Expected SHA256 of empty body, got {sha256}" + + +@pytest.mark.asyncio +async def test_use_dualstack_endpoint(aws_cli, aws_config, tmp_path): + """use_dualstack_endpoint=true routes to dualstack endpoint.""" + src = tmp_path / "foo.txt" + src.write_text("content") + config_path = aws_config( + {"default": {"s3": "\n use_dualstack_endpoint = true"}} + ) + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [put_object_response()]) + _, stderr, rc = await run_cli( + aws_cli, + ["s3", "cp", str(src), "s3://bucket/foo.txt"], + _cli_env_with_config(proxy, config_path), + ) + + assert rc == 0, stderr.decode() + host = server.requests[0].headers.get("host", "") + assert "dualstack" in host, f"Expected dualstack in host, got {host}" + + +@pytest.mark.asyncio +async def test_use_dualstack_endpoint_false(aws_cli, aws_config, tmp_path): + """use_dualstack_endpoint=false uses the standard endpoint.""" + src = tmp_path / "foo.txt" + src.write_text("content") + config_path = aws_config( + {"default": {"s3": "\n use_dualstack_endpoint = false"}} + ) + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [put_object_response()]) + _, stderr, rc = await run_cli( + aws_cli, + ["s3", "cp", str(src), "s3://bucket/foo.txt"], + _cli_env_with_config(proxy, config_path), + ) + + assert rc == 0, stderr.decode() + host = server.requests[0].headers.get("host", "") + assert ( + "dualstack" not in host + ), f"Expected no dualstack in host, got {host}" + + + +@pytest.mark.asyncio +async def test_multipart_threshold_independent_of_chunksize( + aws_cli, aws_config, tmp_path +): + """File below multipart_threshold uses single PUT even when above chunksize. + + multipart_threshold and multipart_chunksize are independent settings. + A 15MB file with threshold=16MB should use single PUT regardless of + chunksize=8MB. + """ + src = tmp_path / "data.bin" + src.write_bytes(b"x" * (15 * 1024 * 1024)) + config_path = aws_config( + {"default": {"s3": "\n multipart_threshold = 16MB\n multipart_chunksize = 8MB"}} + ) + async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + setup_responses(server, [put_object_response()]) + _, stderr, rc = await run_cli( + aws_cli, + ["s3", "cp", str(src), "s3://bucket/data.bin"], + _cli_env_with_config(proxy, config_path), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + assert_put_object(server.requests[0], Bucket="bucket", Key="data.bin") diff --git a/tests/blackbox/test_sync_command.py b/tests/blackbox/test_sync_command.py new file mode 100644 index 000000000000..22a413572e99 --- /dev/null +++ b/tests/blackbox/test_sync_command.py @@ -0,0 +1,1476 @@ +"""Blackbox tests for `aws s3 sync` command.""" + +from __future__ import annotations + +import base64 +import os + +import pytest +from localstub.handlers import handle_expect_header +from localstub.server import HTTPResponse + +from tests.blackbox.s3_assertions import ( + assert_complete_multipart_upload, + assert_copy_object, + assert_create_multipart_upload, + assert_delete_object, + assert_get_object, + assert_get_object_tagging, + assert_head_object, + assert_list_objects_v2, + assert_put_object, + assert_put_object_tagging, + assert_upload_part_copy, +) +from tests.blackbox.utils import ( + cli_env, + complete_mpu_response, + copy_object_response, + create_mpu_response, + delete_response, + error_response, + format_requests, + get_object_response, + get_object_tagging_response, + head_object_response, + list_objects_xml, + mock_server, + put_object_response, + put_object_tagging_response, + run_cli, + setup_responses, + upload_part_copy_response, + xml_response, +) + + +def _b64(value: str) -> str: + """Base64-encode a string, as the CLI does for SSE-C keys on the wire.""" + return base64.b64encode(value.encode()).decode() + + +def _list_response(keys, **overrides): + """Build a ListObjectsV2 XML response with the given keys.""" + contents = [] + for k in keys: + entry = { + "Key": k, + "Size": overrides.get("Size", 100), + "LastModified": "2014-01-09T20:45:49.000Z", + "ETag": overrides.get( + "ETag", '"c8afdb36c52cf4727836669019e69222"' + ), + } + if "StorageClass" in overrides: + entry["StorageClass"] = overrides["StorageClass"] + contents.append(entry) + return xml_response(list_objects_xml(contents=contents if keys else None)) + + +def _empty_list_response(): + return xml_response(list_objects_xml()) + + +@pytest.mark.asyncio +class TestSyncCommand: + async def test_website_redirect_ignore_paramfile(self, aws_cli, tmp_path): + """sync local s3:// --website-redirect uses the URL value.""" + src = tmp_path / "foo.txt" + src.write_text("mycontent") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + _empty_list_response(), + put_object_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "sync", + str(tmp_path), + "s3://bucket/key.txt", + "--website-redirect", + "http://someserver", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 2, format_requests(server) + assert_list_objects_v2(server.requests[0], Bucket="bucket") + assert_put_object( + server.requests[1], + Bucket="bucket", + Key="key.txt/foo.txt", + WebsiteRedirectLocation="http://someserver", + ) + + async def test_no_recursive_option(self, aws_cli, tmp_path): + """sync does not accept --recursive.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, []) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "sync", ".", "s3://mybucket", "--recursive"], + cli_env(proxy), + ) + + assert rc == 252 + + async def test_sync_from_non_existent_directory(self, aws_cli, tmp_path): + """sync from non-existent local dir fails.""" + fakedir = str(tmp_path / "fakedir") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [_empty_list_response()]) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "sync", fakedir, "s3://bucket/"], + cli_env(proxy), + ) + + assert rc == 255 + assert b"does not exist" in stderr + + async def test_sync_to_non_existent_directory(self, aws_cli, tmp_path): + """sync s3->local creates the target directory.""" + target = tmp_path / "fakedir" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + _list_response(["foo.txt"]), + get_object_response(b"foo"), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "sync", "s3://bucket/", str(target)], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert (target / "foo.txt").exists() + + async def test_dryrun_sync(self, aws_cli, tmp_path): + """sync --dryrun only lists, does not transfer.""" + src = tmp_path / "file.txt" + src.write_text("mycontent") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [_empty_list_response()]) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "sync", str(tmp_path), "s3://bucket/", "--dryrun"], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + assert_list_objects_v2(server.requests[0], Bucket="bucket") + assert b"(dryrun) upload:" in stdout + + async def test_glacier_sync_with_force_glacier(self, aws_cli, tmp_path): + """sync s3->local --force-glacier-transfer downloads glacier objects.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + _list_response(["foo/bar.txt"], StorageClass="GLACIER"), + get_object_response(b"foo"), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "sync", + "s3://bucket/foo", + str(tmp_path), + "--force-glacier-transfer", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 2, format_requests(server) + assert_list_objects_v2(server.requests[0], Bucket="bucket") + assert_get_object( + server.requests[1], Bucket="bucket", Key="foo/bar.txt" + ) + + async def test_handles_glacier_incompatible_operations( + self, aws_cli, tmp_path + ): + """sync s3->local skips glacier/deep archive objects with warning.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + xml_response( + list_objects_xml( + contents=[ + { + "Key": "foo", + "Size": 100, + "LastModified": "2014-01-09T20:45:49.000Z", + "StorageClass": "GLACIER", + }, + { + "Key": "bar", + "Size": 100, + "LastModified": "2014-01-09T20:45:49.000Z", + "StorageClass": "DEEP_ARCHIVE", + }, + ] + ) + ), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "sync", "s3://bucket/", str(tmp_path)], + cli_env(proxy), + ) + + assert rc == 2 + assert len(server.requests) == 1, format_requests(server) + assert b"GLACIER" in stderr + assert b"s3://bucket/foo" in stderr + assert b"s3://bucket/bar" in stderr + + async def test_turn_off_glacier_warnings(self, aws_cli, tmp_path): + """sync s3->local --ignore-glacier-warnings suppresses warnings.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + xml_response( + list_objects_xml( + contents=[ + { + "Key": "foo", + "Size": 100, + "LastModified": "2014-01-09T20:45:49.000Z", + "StorageClass": "GLACIER", + }, + { + "Key": "bar", + "Size": 100, + "LastModified": "2014-01-09T20:45:49.000Z", + "StorageClass": "DEEP_ARCHIVE", + }, + ] + ) + ), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "sync", + "s3://bucket/", + str(tmp_path), + "--ignore-glacier-warnings", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert stderr.decode().strip() == "" + + async def test_sync_with_delete_on_downloads(self, aws_cli, tmp_path): + """sync s3->local --delete removes local files not in S3.""" + local_file = tmp_path / "foo.txt" + local_file.write_text("mycontent") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [_empty_list_response()]) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "sync", "s3://bucket", str(tmp_path), "--delete"], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + assert_list_objects_v2(server.requests[0], Bucket="bucket") + assert not local_file.exists() + + async def test_request_payer(self, aws_cli, tmp_path): + """sync s3->s3 --request-payer sends RequestPayer on all ops.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + _list_response(["mykey"]), + _empty_list_response(), + copy_object_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "sync", + "s3://sourcebucket/", + "s3://mybucket", + "--request-payer", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 3, format_requests(server) + assert_list_objects_v2( + server.requests[0], Bucket="sourcebucket", RequestPayer="requester" + ) + assert_list_objects_v2( + server.requests[1], Bucket="mybucket", RequestPayer="requester" + ) + assert_copy_object( + server.requests[2], + Bucket="mybucket", + Key="mykey", + RequestPayer="requester", + ) + + async def test_request_payer_with_deletes(self, aws_cli, tmp_path): + """sync s3->s3 --request-payer --delete sends RequestPayer on delete.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + _empty_list_response(), + _list_response(["key-to-delete"]), + delete_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "sync", + "s3://sourcebucket/", + "s3://mybucket", + "--request-payer", + "--delete", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 3, format_requests(server) + assert_list_objects_v2( + server.requests[0], Bucket="sourcebucket", RequestPayer="requester" + ) + assert_list_objects_v2( + server.requests[1], Bucket="mybucket", RequestPayer="requester" + ) + assert_delete_object( + server.requests[2], + Bucket="mybucket", + Key="key-to-delete", + RequestPayer="requester", + ) + + async def test_s3s3_sync_with_destination_sse_c(self, aws_cli, tmp_path): + """sync s3->s3 --sse-c sends SSE-C headers on CopyObject.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + _list_response(["mykey"]), + _empty_list_response(), + copy_object_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "sync", + "s3://sourcebucket/", + "s3://mybucket", + "--sse-c", + "AES256", + "--sse-c-key", + "destination-key", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 3, format_requests(server) + # SSE-C key is base64-encoded on the wire + assert_copy_object( + server.requests[2], + Bucket="mybucket", + Key="mykey", + SSECustomerAlgorithm="AES256", + SSECustomerKey=_b64("destination-key"), + ) + + async def test_s3s3_sync_with_different_sse_c_keys( + self, aws_cli, tmp_path + ): + """sync s3->s3 with both source and destination SSE-C keys.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + _list_response(["mykey"]), + _empty_list_response(), + copy_object_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "sync", + "s3://sourcebucket/", + "s3://mybucket", + "--sse-c-copy-source", + "AES256", + "--sse-c-copy-source-key", + "source-key", + "--sse-c", + "AES256", + "--sse-c-key", + "destination-key", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 3, format_requests(server) + # SSE-C keys are base64-encoded on the wire + assert_copy_object( + server.requests[2], + Bucket="mybucket", + Key="mykey", + SSECustomerAlgorithm="AES256", + SSECustomerKey=_b64("destination-key"), + CopySourceSSECustomerAlgorithm="AES256", + CopySourceSSECustomerKey=_b64("source-key"), + ) + + async def test_upload_with_checksum_algorithm_crc32( + self, aws_cli, tmp_path + ): + """sync local->s3 --checksum-algorithm CRC32 sends the algorithm.""" + (tmp_path / "foo.txt").write_text("contents") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, [_empty_list_response(), put_object_response()] + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "sync", + str(tmp_path), + "s3://bucket/", + "--checksum-algorithm", + "CRC32", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert_put_object( + server.requests[1], + Bucket="bucket", + Key="foo.txt", + ChecksumAlgorithm="CRC32", + ) + + async def test_upload_with_checksum_algorithm_sha256( + self, aws_cli, tmp_path + ): + """sync local->s3 --checksum-algorithm SHA256.""" + (tmp_path / "foo.txt").write_text("contents") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, [_empty_list_response(), put_object_response()] + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "sync", + str(tmp_path), + "s3://bucket/", + "--checksum-algorithm", + "SHA256", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert_put_object( + server.requests[1], + Bucket="bucket", + Key="foo.txt", + ChecksumAlgorithm="SHA256", + ) + + async def test_upload_with_checksum_algorithm_sha1( + self, aws_cli, tmp_path + ): + """sync local->s3 --checksum-algorithm SHA1.""" + (tmp_path / "foo.txt").write_text("contents") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, [_empty_list_response(), put_object_response()] + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "sync", + str(tmp_path), + "s3://bucket/", + "--checksum-algorithm", + "SHA1", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert_put_object( + server.requests[1], + Bucket="bucket", + Key="foo.txt", + ChecksumAlgorithm="SHA1", + ) + + async def test_download_with_checksum_mode_enabled( + self, aws_cli, tmp_path + ): + """sync s3->local --checksum-mode ENABLED sends ChecksumMode on GetObject.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + _list_response(["foo.txt"]), + get_object_response( + b"foo", **{"x-amz-checksum-crc32": "jHNlIQ=="} + ), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "sync", + "s3://bucket/foo", + str(tmp_path), + "--checksum-mode", + "ENABLED", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert_get_object( + server.requests[1], + Bucket="bucket", + Key="foo.txt", + ChecksumMode="ENABLED", + ) + + async def test_sync_upload_no_overwrite_file_not_at_destination( + self, aws_cli, tmp_path + ): + """sync local->s3 --no-overwrite uploads files not at destination.""" + (tmp_path / "new_file.txt").write_text("mycontent") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + _list_response(["file.txt"]), + put_object_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "sync", str(tmp_path), "s3://bucket", "--no-overwrite"], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 2, format_requests(server) + assert_list_objects_v2(server.requests[0], Bucket="bucket") + assert_put_object( + server.requests[1], Bucket="bucket", Key="new_file.txt" + ) + + async def test_sync_upload_no_overwrite_file_exists_at_destination( + self, aws_cli, tmp_path + ): + """sync local->s3 --no-overwrite skips files already at destination.""" + (tmp_path / "new_file.txt").write_text("mycontent") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [_list_response(["new_file.txt"])]) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "sync", str(tmp_path), "s3://bucket", "--no-overwrite"], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + assert_list_objects_v2(server.requests[0], Bucket="bucket") + + async def test_sync_download_no_overwrite_file_not_at_destination( + self, aws_cli, tmp_path + ): + """sync s3->local --no-overwrite downloads files not present locally.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + _list_response(["new_file.txt"]), + get_object_response(b"foo"), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "sync", + "s3://bucket/", + str(tmp_path), + "--no-overwrite", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 2, format_requests(server) + assert (tmp_path / "new_file.txt").exists() + + async def test_sync_download_no_overwrite_file_exists_at_destination( + self, aws_cli, tmp_path + ): + """sync s3->local --no-overwrite skips files already present locally.""" + (tmp_path / "file.txt").write_text("My content") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [_list_response(["file.txt"])]) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "sync", + "s3://bucket/", + str(tmp_path), + "--no-overwrite", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + + async def test_sync_copy_no_overwrite_file_not_at_destination( + self, aws_cli, tmp_path + ): + """sync s3->s3 --no-overwrite copies files not at destination.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + _list_response(["new_file.txt"]), + _list_response(["file1.txt"]), + copy_object_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "sync", + "s3://bucket/", + "s3://bucket2/", + "--no-overwrite", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 3, format_requests(server) + assert_copy_object( + server.requests[2], Bucket="bucket2", Key="new_file.txt" + ) + + async def test_sync_copy_no_overwrite_file_exists_at_destination( + self, aws_cli, tmp_path + ): + """sync s3->s3 --no-overwrite skips files already at destination.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + _list_response(["new_file.txt"]), + _list_response(["new_file.txt", "file1.txt"]), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "sync", + "s3://bucket/", + "s3://bucket2/", + "--no-overwrite", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 2, format_requests(server) + + async def test_with_accesspoint_arn(self, aws_cli, tmp_path): + """sync s3:/// local downloads from access point.""" + arn = "arn:aws:s3:us-west-2:123456789012:accesspoint/endpoint" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + _list_response(["mykey"]), + get_object_response(b"foo"), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "sync", f"s3://{arn}", str(tmp_path)], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 2, format_requests(server) + assert_list_objects_v2(server.requests[0], Bucket=arn) + assert_get_object(server.requests[1], Bucket=arn, Key="mykey") + + async def test_upload_sync(self, aws_cli, tmp_path): + """sync local->s3 uploads new files.""" + (tmp_path / "myfile").write_text("mycontent") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, [_empty_list_response(), put_object_response()] + ) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "sync", str(tmp_path), "s3://bucket/"], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 2, format_requests(server) + assert_list_objects_v2(server.requests[0], Bucket="bucket") + assert_put_object(server.requests[1], Bucket="bucket", Key="myfile") + + async def test_download_sync(self, aws_cli, tmp_path): + """sync s3->local downloads new files.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + _list_response(["key"]), + get_object_response(b"content"), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "sync", "s3://bucket/", str(tmp_path)], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 2, format_requests(server) + assert_list_objects_v2(server.requests[0], Bucket="bucket") + assert_get_object(server.requests[1], Bucket="bucket", Key="key") + assert (tmp_path / "key").exists() + + async def test_upload_sync_with_delete(self, aws_cli, tmp_path): + """sync local->s3 --delete uploads new files and deletes remote extras.""" + (tmp_path / "a-file").write_text("mycontent") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + _list_response(["delete-this"]), + put_object_response(), + delete_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "sync", str(tmp_path), "s3://bucket/", "--delete"], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 3, format_requests(server) + assert_list_objects_v2(server.requests[0], Bucket="bucket") + # Upload and delete are submitted concurrently; order is not guaranteed + non_list = server.requests[1:] + put_reqs = [r for r in non_list if r.method == "PUT"] + del_reqs = [r for r in non_list if r.method == "DELETE"] + assert len(put_reqs) == 1, format_requests(server) + assert len(del_reqs) == 1, format_requests(server) + assert_put_object(put_reqs[0], Bucket="bucket", Key="a-file") + assert_delete_object(del_reqs[0], Bucket="bucket", Key="delete-this") + + async def test_download_sync_with_delete(self, aws_cli, tmp_path): + """sync s3->local --delete downloads files and deletes local extras.""" + (tmp_path / "delete-this").write_text("content") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + _list_response(["key"]), + get_object_response(b"content"), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "sync", "s3://bucket/", str(tmp_path), "--delete"], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert_get_object(server.requests[1], Bucket="bucket", Key="key") + assert not (tmp_path / "delete-this").exists() + + async def test_copy_sync(self, aws_cli, tmp_path): + """sync s3->s3 copies objects.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + _list_response(["key"]), + _empty_list_response(), + copy_object_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "sync", "s3://bucket/", "s3://otherbucket/"], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 3, format_requests(server) + assert_copy_object(server.requests[2], Bucket="otherbucket", Key="key") + + async def test_respects_source_region(self, aws_cli, tmp_path): + """sync s3->s3 --source-region routes list to source region.""" + source_host = "sourcebucket.s3.af-south-1.amazonaws.com" + target_host = "bucket.s3.us-east-1.amazonaws.com" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + _list_response(["key"]), + _empty_list_response(), + copy_object_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "sync", + "s3://sourcebucket/", + "s3://bucket/", + "--region", + "us-east-1", + "--source-region", + "af-south-1", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 3, format_requests(server) + assert server.requests[0].headers.get("host") == source_host + assert server.requests[1].headers.get("host") == target_host + assert server.requests[2].headers.get("host") == target_host + + +@pytest.mark.asyncio +class TestSyncCommandWithS3Express: + async def test_incompatible_with_sync_upload(self, aws_cli, tmp_path): + """sync local->s3 with directory bucket is rejected.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, []) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "sync", + str(tmp_path), + "s3://testdirectorybucket--usw2-az1--x-s3/", + ], + cli_env(proxy), + ) + + assert rc == 252 + assert b"Cannot use sync command with a directory bucket" in stderr + + async def test_incompatible_with_sync_download(self, aws_cli, tmp_path): + """sync s3->local with directory bucket is rejected.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, []) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "sync", + "s3://testdirectorybucket--usw2-az1--x-s3/", + str(tmp_path), + ], + cli_env(proxy), + ) + + assert rc == 252 + assert b"Cannot use sync command with a directory bucket" in stderr + + async def test_incompatible_with_sync_copy(self, aws_cli, tmp_path): + """sync s3->s3 with directory bucket as destination is rejected.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, []) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "sync", + "s3://bucket/", + "s3://testdirectorybucket--usw2-az1--x-s3/", + ], + cli_env(proxy), + ) + + assert rc == 252 + assert b"Cannot use sync command with a directory bucket" in stderr + + async def test_incompatible_with_sync_with_delete(self, aws_cli, tmp_path): + """sync s3->s3 --delete with directory bucket is rejected.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, []) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "sync", + "s3://bucket/", + "s3://testdirectorybucket--usw2-az1--x-s3/", + "--delete", + ], + cli_env(proxy), + ) + + assert rc == 252 + assert b"Cannot use sync command with a directory bucket" in stderr + + +def _is_case_insensitive() -> bool: + """Check if the filesystem is case-insensitive.""" + import tempfile + + with tempfile.TemporaryDirectory() as d: + upper = os.path.join(d, "A") + open(upper, "w").close() + return os.path.exists(os.path.join(d, "a")) + + +@pytest.mark.asyncio +class TestSyncCaseConflict: + @pytest.mark.skipif( + not _is_case_insensitive(), + reason="Requires case-insensitive filesystem", + ) + async def test_error_with_existing_file(self, aws_cli, tmp_path): + """sync s3->local --case-conflict error fails on case conflict with existing file.""" + (tmp_path / "a.txt").write_text("mycontent") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + xml_response( + list_objects_xml( + contents=[ + { + "Key": "A.txt", + "Size": 100, + "LastModified": "2023-01-01T00:00:00Z", + } + ], + ) + ), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "sync", + "s3://bucket", + str(tmp_path), + "--case-conflict", + "error", + ], + cli_env(proxy), + ) + + assert rc == 1 + assert b"Failed to download bucket/A.txt" in stderr + + async def test_error_with_case_conflicts_in_s3(self, aws_cli, tmp_path): + """sync s3->local --case-conflict error fails on conflicting keys in S3.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + xml_response( + list_objects_xml( + contents=[ + { + "Key": "A.txt", + "Size": 100, + "LastModified": "2023-01-01T00:00:00Z", + }, + { + "Key": "a.txt", + "Size": 100, + "LastModified": "2023-01-01T00:00:00Z", + }, + ], + ) + ), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "sync", + "s3://bucket", + str(tmp_path), + "--case-conflict", + "error", + ], + cli_env(proxy), + ) + + assert rc == 1 + assert b"Failed to download bucket/a.txt" in stderr + + @pytest.mark.skipif( + not _is_case_insensitive(), + reason="Requires case-insensitive filesystem", + ) + async def test_warn_with_existing_file(self, aws_cli, tmp_path): + """sync s3->local --case-conflict warn warns on conflict with existing file.""" + (tmp_path / "a.txt").write_text("mycontent") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + xml_response( + list_objects_xml( + contents=[ + { + "Key": "A.txt", + "Size": 100, + "LastModified": "2023-01-01T00:00:00Z", + } + ], + ) + ), + get_object_response(b"foo"), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "sync", + "s3://bucket", + str(tmp_path), + "--case-conflict", + "warn", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert b"warning: Downloading bucket/A.txt" in stderr + + async def test_warn_with_case_conflicts_in_s3(self, aws_cli, tmp_path): + """sync s3->local --case-conflict warn warns on conflicting keys.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + xml_response( + list_objects_xml( + contents=[ + { + "Key": "A.txt", + "Size": 100, + "LastModified": "2023-01-01T00:00:00Z", + }, + { + "Key": "a.txt", + "Size": 100, + "LastModified": "2023-01-01T00:00:00Z", + }, + ], + ) + ), + get_object_response(b"foo"), + get_object_response(b"bar"), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "sync", + "s3://bucket", + str(tmp_path), + "--case-conflict", + "warn", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert b"warning: Downloading bucket/a.txt" in stderr + + @pytest.mark.skipif( + not _is_case_insensitive(), + reason="Requires case-insensitive filesystem", + ) + async def test_skip_with_existing_file(self, aws_cli, tmp_path): + """sync s3->local --case-conflict skip skips conflicting file.""" + (tmp_path / "a.txt").write_text("mycontent") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + xml_response( + list_objects_xml( + contents=[ + { + "Key": "A.txt", + "Size": 100, + "LastModified": "2023-01-01T00:00:00Z", + } + ], + ) + ), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "sync", + "s3://bucket", + str(tmp_path), + "--case-conflict", + "skip", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert b"warning: Skipping bucket/A.txt" in stderr + + async def test_skip_with_case_conflicts_in_s3(self, aws_cli, tmp_path): + """sync s3->local --case-conflict skip skips conflicting keys.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + xml_response( + list_objects_xml( + contents=[ + { + "Key": "A.txt", + "Size": 100, + "LastModified": "2023-01-01T00:00:00Z", + }, + { + "Key": "a.txt", + "Size": 100, + "LastModified": "2023-01-01T00:00:00Z", + }, + ], + ) + ), + get_object_response(b"foo"), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "sync", + "s3://bucket", + str(tmp_path), + "--case-conflict", + "skip", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert b"warning: Skipping bucket/a.txt" in stderr + + @pytest.mark.skipif( + not _is_case_insensitive(), + reason="Requires case-insensitive filesystem", + ) + async def test_ignore_with_existing_file(self, aws_cli, tmp_path): + """sync s3->local --case-conflict ignore proceeds without warning.""" + (tmp_path / "a.txt").write_text("mycontent") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + xml_response( + list_objects_xml( + contents=[ + { + "Key": "A.txt", + "Size": 100, + "LastModified": "2023-01-01T00:00:00Z", + } + ], + ) + ), + get_object_response(b"foo"), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "sync", + "s3://bucket", + str(tmp_path), + "--case-conflict", + "ignore", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + + async def test_ignore_with_case_conflicts_in_s3(self, aws_cli, tmp_path): + """sync s3->local --case-conflict ignore downloads all without warning.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + xml_response( + list_objects_xml( + contents=[ + { + "Key": "A.txt", + "Size": 100, + "LastModified": "2023-01-01T00:00:00Z", + }, + { + "Key": "a.txt", + "Size": 100, + "LastModified": "2023-01-01T00:00:00Z", + }, + ], + ) + ), + get_object_response(b"foo"), + get_object_response(b"bar"), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "sync", + "s3://bucket", + str(tmp_path), + "--case-conflict", + "ignore", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + + +@pytest.mark.asyncio +async def test_download_url_encoded_key_from_list(aws_cli, tmp_path): + """sync downloads objects whose keys contain spaces from ListObjectsV2.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + xml_response( + list_objects_xml( + contents=[ + { + "Key": "my file.txt", + "Size": 3, + "LastModified": "2023-01-01T00:00:00Z", + } + ] + ) + ), + get_object_response(b"foo"), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "sync", "s3://bucket/", str(tmp_path)], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert (tmp_path / "my file.txt").exists() diff --git a/tests/blackbox/utils.py b/tests/blackbox/utils.py index be8bb20fde73..e8a9b8299093 100644 --- a/tests/blackbox/utils.py +++ b/tests/blackbox/utils.py @@ -76,7 +76,11 @@ def get_query_params(request) -> dict[str, list[str]]: async def run_cli( - aws_cli: str, args: list[str], env: dict, stdin: bytes | None = None + aws_cli: str, + args: list[str], + env: dict, + stdin: bytes | None = None, + timeout: float = 60, ) -> tuple[bytes, bytes, int]: proc = await asyncio.create_subprocess_exec( aws_cli, @@ -86,7 +90,17 @@ async def run_cli( stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, ) - stdout, stderr = await proc.communicate(input=stdin) + try: + stdout, stderr = await asyncio.wait_for( + proc.communicate(input=stdin), timeout=timeout + ) + except asyncio.TimeoutError: + proc.kill() + await proc.wait() + raise AssertionError( + f"CLI process timed out after {timeout}s. " + f"Command: {aws_cli} {' '.join(args)}" + ) return stdout, stderr, proc.returncode