From 095613d85a3131d39fd756d8e3bbcc92970de56e Mon Sep 17 00:00:00 2001 From: aemous Date: Mon, 24 Aug 2026 09:25:13 -0400 Subject: [PATCH 1/7] Remove unnecessary comment. --- tests/blackbox/s3_assertions.py | 7 +++++++ tests/blackbox/test_cp_command.py | 12 +++--------- tests/blackbox/test_mv_command.py | 18 +++++++++++++++--- 3 files changed, 25 insertions(+), 12 deletions(-) diff --git a/tests/blackbox/s3_assertions.py b/tests/blackbox/s3_assertions.py index f848e8fd7abb..98ae75b9bcfc 100644 --- a/tests/blackbox/s3_assertions.py +++ b/tests/blackbox/s3_assertions.py @@ -778,6 +778,9 @@ def assert_get_access_point(request): assert "/v20180820/accesspoint/" in request.path, ( f"GetAccessPoint: expected /v20180820/accesspoint/ in {request.path}" ) + assert ( + request.headers.get("x-amz-account-id") is not None + ), "GetAccessPoint: required header x-amz-account-id is missing" def assert_get_caller_identity(request): @@ -785,3 +788,7 @@ def assert_get_caller_identity(request): assert request.method == "POST", ( f"GetCallerIdentity: expected POST, got {request.method}" ) + assert "Action=GetCallerIdentity" in request.body, ( + f"GetCallerIdentity: expected Action=GetCallerIdentity in body, " + f"got {request.body!r}" + ) diff --git a/tests/blackbox/test_cp_command.py b/tests/blackbox/test_cp_command.py index 3dae7d9262f1..1490ee5a08d7 100644 --- a/tests/blackbox/test_cp_command.py +++ b/tests/blackbox/test_cp_command.py @@ -2189,9 +2189,7 @@ async def test_no_overwrite_flag_on_copy_when_large_object_does_not_exist_on_tar Bucket="bucket", Key="key1.txt", UploadId="foo", - MultipartUpload={ - "Parts": [{"PartNumber": "1"}, {"PartNumber": "2"}] - }, + MultipartUpload=[{"PartNumber": "1"}, {"PartNumber": "2"}], IfNoneMatch="*", ) @@ -2239,9 +2237,7 @@ async def test_no_overwrite_flag_on_copy_when_large_object_exists_on_target( Bucket="bucket", Key="key.txt", UploadId="foo", - MultipartUpload={ - "Parts": [{"PartNumber": "1"}, {"PartNumber": "2"}] - }, + MultipartUpload=[{"PartNumber": "1"}, {"PartNumber": "2"}], IfNoneMatch="*", ) # AbortMultipartUpload should have been called @@ -3104,9 +3100,7 @@ async def test_multipart_copy(self, aws_cli, tmp_path): Bucket="mybucket", Key="mykey", UploadId="id", - MultipartUpload={ - "Parts": [{"PartNumber": "1"}, {"PartNumber": "2"}] - }, + MultipartUpload=[{"PartNumber": "1"}, {"PartNumber": "2"}], RequestPayer="requester", ) diff --git a/tests/blackbox/test_mv_command.py b/tests/blackbox/test_mv_command.py index 428c678f2c72..a00bb0ebe0a1 100644 --- a/tests/blackbox/test_mv_command.py +++ b/tests/blackbox/test_mv_command.py @@ -1348,7 +1348,11 @@ async def test_mv_works_if_access_point_arn_resolves_to_different_bucket( assert_head_object( server.requests[1], Bucket="bucket", Key="key" ) - assert_copy_object(server.requests[2]) + assert_copy_object( + server.requests[2], + Bucket="arn:aws:s3:us-west-2:123456789012:accesspoint/myaccesspoint", + Key="key", + ) assert_delete_object( server.requests[3], Bucket="bucket", Key="key" ) @@ -1390,7 +1394,11 @@ async def test_mv_works_if_access_point_alias_resolves_to_different_bucket( assert_head_object( server.requests[2], Bucket="bucket", Key="key" ) - assert_copy_object(server.requests[3]) + assert_copy_object( + server.requests[3], + Bucket="myaccesspoint-foobar-s3alias", + Key="key", + ) assert_delete_object( server.requests[4], Bucket="bucket", Key="key" ) @@ -1430,7 +1438,11 @@ async def test_mv_works_if_outpost_access_point_arn_resolves_to_different_bucket assert_head_object( server.requests[1], Bucket="bucket", Key="key" ) - assert_copy_object(server.requests[2]) + assert_copy_object( + server.requests[2], + Bucket="arn:aws:s3-outposts:us-east-1:123456789012:outpost/op-foobar/accesspoint/myaccesspoint", + Key="key", + ) assert_delete_object( server.requests[3], Bucket="bucket", Key="key" ) From fba5676a6916979408f5fc4c67bcb57ce1dd20f1 Mon Sep 17 00:00:00 2001 From: aemous Date: Thu, 27 Aug 2026 14:21:53 -0400 Subject: [PATCH 2/7] Progress on porting sync functional tests to blackbox tests. --- tests/blackbox/test_sync_command.py | 891 ++++++++++++++++++++++++++++ tests/blackbox/utils.py | 11 +- 2 files changed, 901 insertions(+), 1 deletion(-) create mode 100644 tests/blackbox/test_sync_command.py diff --git a/tests/blackbox/test_sync_command.py b/tests/blackbox/test_sync_command.py new file mode 100644 index 000000000000..682013243061 --- /dev/null +++ b/tests/blackbox/test_sync_command.py @@ -0,0 +1,891 @@ +"""Blackbox tests for `aws s3 sync` command.""" +from __future__ import annotations + +import base64 +import os + +import pytest +from localstub.handlers import handle_expect_header +from localstub.server import HTTPResponse + +from tests.blackbox.s3_assertions import ( + assert_copy_object, + assert_delete_object, + assert_get_object, + assert_get_object_tagging, + assert_head_object, + assert_list_objects_v2, + assert_put_object, + assert_put_object_tagging, + assert_upload_part_copy, + assert_complete_multipart_upload, + assert_create_multipart_upload, +) +from tests.blackbox.utils import ( + cli_env, + complete_mpu_response, + copy_object_response, + create_mpu_response, + delete_response, + error_response, + format_requests, + get_object_response, + get_object_tagging_response, + head_object_response, + list_objects_xml, + mock_server, + put_object_response, + put_object_tagging_response, + run_cli, + setup_responses, + upload_part_copy_response, + xml_response, +) + + +def _b64(value: str) -> str: + """Base64-encode a string, as the CLI does for SSE-C keys on the wire.""" + return base64.b64encode(value.encode()).decode() + + +def _list_response(keys, **overrides): + """Build a ListObjectsV2 XML response with the given keys.""" + contents = [] + for k in keys: + entry = { + "Key": k, + "Size": overrides.get("Size", 100), + "LastModified": "2014-01-09T20:45:49.000Z", + "ETag": overrides.get("ETag", '"c8afdb36c52cf4727836669019e69222"'), + } + if "StorageClass" in overrides: + entry["StorageClass"] = overrides["StorageClass"] + contents.append(entry) + return xml_response(list_objects_xml(contents=contents if keys else None)) + + +def _empty_list_response(): + return xml_response(list_objects_xml()) + + +@pytest.mark.asyncio +class TestSyncCommand: + async def test_website_redirect_ignore_paramfile(self, aws_cli, tmp_path): + """sync local s3:// --website-redirect uses the URL value.""" + src = tmp_path / "foo.txt" + src.write_text("mycontent") + async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + setup_responses(server, [ + _empty_list_response(), + put_object_response(), + ]) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "sync", str(tmp_path), "s3://bucket/key.txt", + "--website-redirect", "http://someserver"], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 2, format_requests(server) + assert_list_objects_v2(server.requests[0], Bucket="bucket") + assert_put_object(server.requests[1], Bucket="bucket", Key="key.txt/foo.txt", + WebsiteRedirectLocation="http://someserver") + + async def test_no_recursive_option(self, aws_cli, tmp_path): + """sync does not accept --recursive.""" + async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + setup_responses(server, []) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "sync", ".", "s3://mybucket", "--recursive"], + cli_env(proxy), + ) + + assert rc == 252 + + async def test_sync_from_non_existent_directory(self, aws_cli, tmp_path): + """sync from non-existent local dir fails.""" + fakedir = str(tmp_path / "fakedir") + async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + setup_responses(server, [_empty_list_response()]) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "sync", fakedir, "s3://bucket/"], + cli_env(proxy), + ) + + assert rc == 255 + assert b"does not exist" in stderr + + async def test_sync_to_non_existent_directory(self, aws_cli, tmp_path): + """sync s3->local creates the target directory.""" + target = tmp_path / "fakedir" + async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + setup_responses(server, [ + _list_response(["foo.txt"]), + get_object_response(b"foo"), + ]) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "sync", "s3://bucket/", str(target)], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert (target / "foo.txt").exists() + + async def test_dryrun_sync(self, aws_cli, tmp_path): + """sync --dryrun only lists, does not transfer.""" + src = tmp_path / "file.txt" + src.write_text("mycontent") + async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + setup_responses(server, [_empty_list_response()]) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "sync", str(tmp_path), "s3://bucket/", "--dryrun"], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + assert_list_objects_v2(server.requests[0], Bucket="bucket") + assert b"(dryrun) upload:" in stdout + + async def test_glacier_sync_with_force_glacier(self, aws_cli, tmp_path): + """sync s3->local --force-glacier-transfer downloads glacier objects.""" + async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + setup_responses(server, [ + _list_response(["foo/bar.txt"], StorageClass="GLACIER"), + get_object_response(b"foo"), + ]) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "sync", "s3://bucket/foo", str(tmp_path), + "--force-glacier-transfer"], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 2, format_requests(server) + assert_list_objects_v2(server.requests[0], Bucket="bucket") + assert_get_object(server.requests[1], Bucket="bucket", Key="foo/bar.txt") + + async def test_handles_glacier_incompatible_operations(self, aws_cli, tmp_path): + """sync s3->local skips glacier/deep archive objects with warning.""" + async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + setup_responses(server, [ + xml_response(list_objects_xml(contents=[ + {"Key": "foo", "Size": 100, "LastModified": "2014-01-09T20:45:49.000Z", + "StorageClass": "GLACIER"}, + {"Key": "bar", "Size": 100, "LastModified": "2014-01-09T20:45:49.000Z", + "StorageClass": "DEEP_ARCHIVE"}, + ])), + ]) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "sync", "s3://bucket/", str(tmp_path)], + cli_env(proxy), + ) + + assert rc == 2 + assert len(server.requests) == 1, format_requests(server) + assert b"GLACIER" in stderr + assert b"s3://bucket/foo" in stderr + assert b"s3://bucket/bar" in stderr + + async def test_turn_off_glacier_warnings(self, aws_cli, tmp_path): + """sync s3->local --ignore-glacier-warnings suppresses warnings.""" + async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + setup_responses(server, [ + xml_response(list_objects_xml(contents=[ + {"Key": "foo", "Size": 100, "LastModified": "2014-01-09T20:45:49.000Z", + "StorageClass": "GLACIER"}, + {"Key": "bar", "Size": 100, "LastModified": "2014-01-09T20:45:49.000Z", + "StorageClass": "DEEP_ARCHIVE"}, + ])), + ]) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "sync", "s3://bucket/", str(tmp_path), + "--ignore-glacier-warnings"], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert stderr.decode().strip() == "" + + async def test_sync_with_delete_on_downloads(self, aws_cli, tmp_path): + """sync s3->local --delete removes local files not in S3.""" + local_file = tmp_path / "foo.txt" + local_file.write_text("mycontent") + async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + setup_responses(server, [_empty_list_response()]) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "sync", "s3://bucket", str(tmp_path), "--delete"], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + assert_list_objects_v2(server.requests[0], Bucket="bucket") + assert not local_file.exists() + + async def test_request_payer(self, aws_cli, tmp_path): + """sync s3->s3 --request-payer sends RequestPayer on all ops.""" + async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + setup_responses(server, [ + _list_response(["mykey"]), + _empty_list_response(), + copy_object_response(), + ]) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "sync", "s3://sourcebucket/", "s3://mybucket", + "--request-payer"], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 3, format_requests(server) + assert_list_objects_v2(server.requests[0], Bucket="sourcebucket", + RequestPayer="requester") + assert_list_objects_v2(server.requests[1], Bucket="mybucket", + RequestPayer="requester") + assert_copy_object(server.requests[2], Bucket="mybucket", Key="mykey", + RequestPayer="requester") + + async def test_request_payer_with_deletes(self, aws_cli, tmp_path): + """sync s3->s3 --request-payer --delete sends RequestPayer on delete.""" + async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + setup_responses(server, [ + _empty_list_response(), + _list_response(["key-to-delete"]), + delete_response(), + ]) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "sync", "s3://sourcebucket/", "s3://mybucket", + "--request-payer", "--delete"], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 3, format_requests(server) + assert_list_objects_v2(server.requests[0], Bucket="sourcebucket", + RequestPayer="requester") + assert_list_objects_v2(server.requests[1], Bucket="mybucket", + RequestPayer="requester") + assert_delete_object(server.requests[2], Bucket="mybucket", Key="key-to-delete", + RequestPayer="requester") + + async def test_s3s3_sync_with_destination_sse_c(self, aws_cli, tmp_path): + """sync s3->s3 --sse-c sends SSE-C headers on CopyObject.""" + async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + setup_responses(server, [ + _list_response(["mykey"]), + _empty_list_response(), + copy_object_response(), + ]) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "sync", "s3://sourcebucket/", "s3://mybucket", + "--sse-c", "AES256", "--sse-c-key", "destination-key"], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 3, format_requests(server) + # SSE-C key is base64-encoded on the wire + assert_copy_object(server.requests[2], Bucket="mybucket", Key="mykey", + SSECustomerAlgorithm="AES256", + SSECustomerKey=_b64("destination-key")) + + async def test_s3s3_sync_with_different_sse_c_keys(self, aws_cli, tmp_path): + """sync s3->s3 with both source and destination SSE-C keys.""" + async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + setup_responses(server, [ + _list_response(["mykey"]), + _empty_list_response(), + copy_object_response(), + ]) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "sync", "s3://sourcebucket/", "s3://mybucket", + "--sse-c-copy-source", "AES256", + "--sse-c-copy-source-key", "source-key", + "--sse-c", "AES256", "--sse-c-key", "destination-key"], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 3, format_requests(server) + # SSE-C keys are base64-encoded on the wire + assert_copy_object(server.requests[2], Bucket="mybucket", Key="mykey", + SSECustomerAlgorithm="AES256", + SSECustomerKey=_b64("destination-key"), + CopySourceSSECustomerAlgorithm="AES256", + CopySourceSSECustomerKey=_b64("source-key")) + + async def test_upload_with_checksum_algorithm_crc32(self, aws_cli, tmp_path): + """sync local->s3 --checksum-algorithm CRC32 sends the algorithm.""" + (tmp_path / "foo.txt").write_text("contents") + async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + setup_responses(server, [_empty_list_response(), put_object_response()]) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "sync", str(tmp_path), "s3://bucket/", + "--checksum-algorithm", "CRC32"], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert_put_object(server.requests[1], Bucket="bucket", Key="foo.txt", + ChecksumAlgorithm="CRC32") + + async def test_upload_with_checksum_algorithm_sha256(self, aws_cli, tmp_path): + """sync local->s3 --checksum-algorithm SHA256.""" + (tmp_path / "foo.txt").write_text("contents") + async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + setup_responses(server, [_empty_list_response(), put_object_response()]) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "sync", str(tmp_path), "s3://bucket/", + "--checksum-algorithm", "SHA256"], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert_put_object(server.requests[1], Bucket="bucket", Key="foo.txt", + ChecksumAlgorithm="SHA256") + + async def test_upload_with_checksum_algorithm_sha1(self, aws_cli, tmp_path): + """sync local->s3 --checksum-algorithm SHA1.""" + (tmp_path / "foo.txt").write_text("contents") + async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + setup_responses(server, [_empty_list_response(), put_object_response()]) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "sync", str(tmp_path), "s3://bucket/", + "--checksum-algorithm", "SHA1"], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert_put_object(server.requests[1], Bucket="bucket", Key="foo.txt", + ChecksumAlgorithm="SHA1") + + async def test_download_with_checksum_mode_enabled(self, aws_cli, tmp_path): + """sync s3->local --checksum-mode ENABLED sends ChecksumMode on GetObject.""" + async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + setup_responses(server, [ + _list_response(["foo.txt"]), + get_object_response(b"foo", **{"x-amz-checksum-crc32": "jHNlIQ=="}), + ]) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "sync", "s3://bucket/foo", str(tmp_path), + "--checksum-mode", "ENABLED"], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert_get_object(server.requests[1], Bucket="bucket", Key="foo.txt", + ChecksumMode="ENABLED") + + async def test_sync_upload_no_overwrite_file_not_at_destination(self, aws_cli, tmp_path): + """sync local->s3 --no-overwrite uploads files not at destination.""" + (tmp_path / "new_file.txt").write_text("mycontent") + async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + setup_responses(server, [ + _list_response(["file.txt"]), + put_object_response(), + ]) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "sync", str(tmp_path), "s3://bucket", "--no-overwrite"], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 2, format_requests(server) + assert_list_objects_v2(server.requests[0], Bucket="bucket") + assert_put_object(server.requests[1], Bucket="bucket", Key="new_file.txt") + + async def test_sync_upload_no_overwrite_file_exists_at_destination(self, aws_cli, tmp_path): + """sync local->s3 --no-overwrite skips files already at destination.""" + (tmp_path / "new_file.txt").write_text("mycontent") + async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + setup_responses(server, [_list_response(["new_file.txt"])]) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "sync", str(tmp_path), "s3://bucket", "--no-overwrite"], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + assert_list_objects_v2(server.requests[0], Bucket="bucket") + + async def test_sync_download_no_overwrite_file_not_at_destination(self, aws_cli, tmp_path): + """sync s3->local --no-overwrite downloads files not present locally.""" + async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + setup_responses(server, [ + _list_response(["new_file.txt"]), + get_object_response(b"foo"), + ]) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "sync", "s3://bucket/", str(tmp_path), "--no-overwrite"], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 2, format_requests(server) + assert (tmp_path / "new_file.txt").exists() + + async def test_sync_download_no_overwrite_file_exists_at_destination(self, aws_cli, tmp_path): + """sync s3->local --no-overwrite skips files already present locally.""" + (tmp_path / "file.txt").write_text("My content") + async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + setup_responses(server, [_list_response(["file.txt"])]) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "sync", "s3://bucket/", str(tmp_path), "--no-overwrite"], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + + async def test_sync_copy_no_overwrite_file_not_at_destination(self, aws_cli, tmp_path): + """sync s3->s3 --no-overwrite copies files not at destination.""" + async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + setup_responses(server, [ + _list_response(["new_file.txt"]), + _list_response(["file1.txt"]), + copy_object_response(), + ]) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "sync", "s3://bucket/", "s3://bucket2/", "--no-overwrite"], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 3, format_requests(server) + assert_copy_object(server.requests[2], Bucket="bucket2", Key="new_file.txt") + + async def test_sync_copy_no_overwrite_file_exists_at_destination(self, aws_cli, tmp_path): + """sync s3->s3 --no-overwrite skips files already at destination.""" + async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + setup_responses(server, [ + _list_response(["new_file.txt"]), + _list_response(["new_file.txt", "file1.txt"]), + ]) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "sync", "s3://bucket/", "s3://bucket2/", "--no-overwrite"], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 2, format_requests(server) + + async def test_with_accesspoint_arn(self, aws_cli, tmp_path): + """sync s3:/// local downloads from access point.""" + arn = "arn:aws:s3:us-west-2:123456789012:accesspoint/endpoint" + async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + setup_responses(server, [ + _list_response(["mykey"]), + get_object_response(b"foo"), + ]) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "sync", f"s3://{arn}", str(tmp_path)], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 2, format_requests(server) + assert_list_objects_v2(server.requests[0], Bucket=arn) + assert_get_object(server.requests[1], Bucket=arn, Key="mykey") + + async def test_upload_sync(self, aws_cli, tmp_path): + """sync local->s3 uploads new files.""" + (tmp_path / "myfile").write_text("mycontent") + async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + setup_responses(server, [_empty_list_response(), put_object_response()]) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "sync", str(tmp_path), "s3://bucket/"], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 2, format_requests(server) + assert_list_objects_v2(server.requests[0], Bucket="bucket") + assert_put_object(server.requests[1], Bucket="bucket", Key="myfile") + + async def test_download_sync(self, aws_cli, tmp_path): + """sync s3->local downloads new files.""" + async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + setup_responses(server, [ + _list_response(["key"]), + get_object_response(b"content"), + ]) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "sync", "s3://bucket/", str(tmp_path)], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 2, format_requests(server) + assert_list_objects_v2(server.requests[0], Bucket="bucket") + assert_get_object(server.requests[1], Bucket="bucket", Key="key") + assert (tmp_path / "key").exists() + + async def test_upload_sync_with_delete(self, aws_cli, tmp_path): + """sync local->s3 --delete uploads new files and deletes remote extras.""" + (tmp_path / "a-file").write_text("mycontent") + async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + setup_responses(server, [ + _list_response(["delete-this"]), + put_object_response(), + delete_response(), + ]) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "sync", str(tmp_path), "s3://bucket/", "--delete"], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 3, format_requests(server) + assert_list_objects_v2(server.requests[0], Bucket="bucket") + # Upload and delete are submitted concurrently; order is not guaranteed + non_list = server.requests[1:] + put_reqs = [r for r in non_list if r.method == "PUT"] + del_reqs = [r for r in non_list if r.method == "DELETE"] + assert len(put_reqs) == 1, format_requests(server) + assert len(del_reqs) == 1, format_requests(server) + assert_put_object(put_reqs[0], Bucket="bucket", Key="a-file") + assert_delete_object(del_reqs[0], Bucket="bucket", Key="delete-this") + + async def test_download_sync_with_delete(self, aws_cli, tmp_path): + """sync s3->local --delete downloads files and deletes local extras.""" + (tmp_path / "delete-this").write_text("content") + async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + setup_responses(server, [ + _list_response(["key"]), + get_object_response(b"content"), + ]) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "sync", "s3://bucket/", str(tmp_path), "--delete"], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert_get_object(server.requests[1], Bucket="bucket", Key="key") + assert not (tmp_path / "delete-this").exists() + + async def test_copy_sync(self, aws_cli, tmp_path): + """sync s3->s3 copies objects.""" + async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + setup_responses(server, [ + _list_response(["key"]), + _empty_list_response(), + copy_object_response(), + ]) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "sync", "s3://bucket/", "s3://otherbucket/"], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 3, format_requests(server) + assert_copy_object(server.requests[2], Bucket="otherbucket", Key="key") + + async def test_respects_source_region(self, aws_cli, tmp_path): + """sync s3->s3 --source-region routes list to source region.""" + source_host = "sourcebucket.s3.af-south-1.amazonaws.com" + target_host = "bucket.s3.us-east-1.amazonaws.com" + async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + setup_responses(server, [ + _list_response(["key"]), + _empty_list_response(), + copy_object_response(), + ]) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "sync", "s3://sourcebucket/", "s3://bucket/", + "--region", "us-east-1", "--source-region", "af-south-1"], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 3, format_requests(server) + assert server.requests[0].headers.get("host") == source_host + assert server.requests[1].headers.get("host") == target_host + assert server.requests[2].headers.get("host") == target_host + + +@pytest.mark.asyncio +class TestSyncCommandWithS3Express: + async def test_incompatible_with_sync_upload(self, aws_cli, tmp_path): + """sync local->s3 with directory bucket is rejected.""" + async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + setup_responses(server, []) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "sync", str(tmp_path), + "s3://testdirectorybucket--usw2-az1--x-s3/"], + cli_env(proxy), + ) + + assert rc == 252 + assert b"Cannot use sync command with a directory bucket" in stderr + + async def test_incompatible_with_sync_download(self, aws_cli, tmp_path): + """sync s3->local with directory bucket is rejected.""" + async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + setup_responses(server, []) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "sync", + "s3://testdirectorybucket--usw2-az1--x-s3/", + str(tmp_path)], + cli_env(proxy), + ) + + assert rc == 252 + assert b"Cannot use sync command with a directory bucket" in stderr + + async def test_incompatible_with_sync_copy(self, aws_cli, tmp_path): + """sync s3->s3 with directory bucket as destination is rejected.""" + async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + setup_responses(server, []) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "sync", "s3://bucket/", + "s3://testdirectorybucket--usw2-az1--x-s3/"], + cli_env(proxy), + ) + + assert rc == 252 + assert b"Cannot use sync command with a directory bucket" in stderr + + async def test_incompatible_with_sync_with_delete(self, aws_cli, tmp_path): + """sync s3->s3 --delete with directory bucket is rejected.""" + async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + setup_responses(server, []) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "sync", "s3://bucket/", + "s3://testdirectorybucket--usw2-az1--x-s3/", "--delete"], + cli_env(proxy), + ) + + assert rc == 252 + assert b"Cannot use sync command with a directory bucket" in stderr + + +def _is_case_insensitive() -> bool: + """Check if the filesystem is case-insensitive.""" + import tempfile + with tempfile.TemporaryDirectory() as d: + upper = os.path.join(d, "A") + open(upper, "w").close() + return os.path.exists(os.path.join(d, "a")) + + +@pytest.mark.asyncio +class TestSyncCaseConflict: + @pytest.mark.skipif( + not _is_case_insensitive(), + reason="Requires case-insensitive filesystem", + ) + async def test_error_with_existing_file(self, aws_cli, tmp_path): + """sync s3->local --case-conflict error fails on case conflict with existing file.""" + (tmp_path / "a.txt").write_text("mycontent") + async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + setup_responses(server, [ + xml_response(list_objects_xml( + contents=[{"Key": "A.txt", "Size": 100, + "LastModified": "2023-01-01T00:00:00Z"}], + )), + ]) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "sync", "s3://bucket", str(tmp_path), + "--case-conflict", "error"], + cli_env(proxy), + ) + + assert rc == 1 + assert b"Failed to download bucket/A.txt" in stderr + + async def test_error_with_case_conflicts_in_s3(self, aws_cli, tmp_path): + """sync s3->local --case-conflict error fails on conflicting keys in S3.""" + async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + setup_responses(server, [ + xml_response(list_objects_xml( + contents=[ + {"Key": "A.txt", "Size": 100, "LastModified": "2023-01-01T00:00:00Z"}, + {"Key": "a.txt", "Size": 100, "LastModified": "2023-01-01T00:00:00Z"}, + ], + )), + ]) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "sync", "s3://bucket", str(tmp_path), + "--case-conflict", "error"], + cli_env(proxy), + ) + + assert rc == 1 + assert b"Failed to download bucket/a.txt" in stderr + + @pytest.mark.skipif( + not _is_case_insensitive(), + reason="Requires case-insensitive filesystem", + ) + async def test_warn_with_existing_file(self, aws_cli, tmp_path): + """sync s3->local --case-conflict warn warns on conflict with existing file.""" + (tmp_path / "a.txt").write_text("mycontent") + async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + setup_responses(server, [ + xml_response(list_objects_xml( + contents=[{"Key": "A.txt", "Size": 100, + "LastModified": "2023-01-01T00:00:00Z"}], + )), + get_object_response(b"foo"), + ]) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "sync", "s3://bucket", str(tmp_path), + "--case-conflict", "warn"], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert b"warning: Downloading bucket/A.txt" in stderr + + async def test_warn_with_case_conflicts_in_s3(self, aws_cli, tmp_path): + """sync s3->local --case-conflict warn warns on conflicting keys.""" + async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + setup_responses(server, [ + xml_response(list_objects_xml( + contents=[ + {"Key": "A.txt", "Size": 100, "LastModified": "2023-01-01T00:00:00Z"}, + {"Key": "a.txt", "Size": 100, "LastModified": "2023-01-01T00:00:00Z"}, + ], + )), + get_object_response(b"foo"), + get_object_response(b"bar"), + ]) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "sync", "s3://bucket", str(tmp_path), + "--case-conflict", "warn"], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert b"warning: Downloading bucket/a.txt" in stderr + + @pytest.mark.skipif( + not _is_case_insensitive(), + reason="Requires case-insensitive filesystem", + ) + async def test_skip_with_existing_file(self, aws_cli, tmp_path): + """sync s3->local --case-conflict skip skips conflicting file.""" + (tmp_path / "a.txt").write_text("mycontent") + async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + setup_responses(server, [ + xml_response(list_objects_xml( + contents=[{"Key": "A.txt", "Size": 100, + "LastModified": "2023-01-01T00:00:00Z"}], + )), + ]) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "sync", "s3://bucket", str(tmp_path), + "--case-conflict", "skip"], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert b"warning: Skipping bucket/A.txt" in stderr + + async def test_skip_with_case_conflicts_in_s3(self, aws_cli, tmp_path): + """sync s3->local --case-conflict skip skips conflicting keys.""" + async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + setup_responses(server, [ + xml_response(list_objects_xml( + contents=[ + {"Key": "A.txt", "Size": 100, "LastModified": "2023-01-01T00:00:00Z"}, + {"Key": "a.txt", "Size": 100, "LastModified": "2023-01-01T00:00:00Z"}, + ], + )), + get_object_response(b"foo"), + ]) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "sync", "s3://bucket", str(tmp_path), + "--case-conflict", "skip"], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert b"warning: Skipping bucket/a.txt" in stderr + + @pytest.mark.skipif( + not _is_case_insensitive(), + reason="Requires case-insensitive filesystem", + ) + async def test_ignore_with_existing_file(self, aws_cli, tmp_path): + """sync s3->local --case-conflict ignore proceeds without warning.""" + (tmp_path / "a.txt").write_text("mycontent") + async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + setup_responses(server, [ + xml_response(list_objects_xml( + contents=[{"Key": "A.txt", "Size": 100, + "LastModified": "2023-01-01T00:00:00Z"}], + )), + get_object_response(b"foo"), + ]) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "sync", "s3://bucket", str(tmp_path), + "--case-conflict", "ignore"], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + + async def test_ignore_with_case_conflicts_in_s3(self, aws_cli, tmp_path): + """sync s3->local --case-conflict ignore downloads all without warning.""" + async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + setup_responses(server, [ + xml_response(list_objects_xml( + contents=[ + {"Key": "A.txt", "Size": 100, "LastModified": "2023-01-01T00:00:00Z"}, + {"Key": "a.txt", "Size": 100, "LastModified": "2023-01-01T00:00:00Z"}, + ], + )), + get_object_response(b"foo"), + get_object_response(b"bar"), + ]) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "sync", "s3://bucket", str(tmp_path), + "--case-conflict", "ignore"], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() diff --git a/tests/blackbox/utils.py b/tests/blackbox/utils.py index be8bb20fde73..c851dcc0ae22 100644 --- a/tests/blackbox/utils.py +++ b/tests/blackbox/utils.py @@ -11,6 +11,15 @@ from localstub.tlsproxy import AsyncTLSInterceptProxy +import tempfile + +_CRT_CONFIG = tempfile.NamedTemporaryFile( + mode="w", suffix=".cfg", delete=False, prefix="blackbox_crt_" +) +_CRT_CONFIG.write("[default]\ns3 =\n preferred_transfer_client = crt\n") +_CRT_CONFIG.close() + + def cli_env(proxy: AsyncTLSInterceptProxy) -> dict[str, str]: return { "PATH": os.environ.get("PATH", ""), @@ -20,7 +29,7 @@ def cli_env(proxy: AsyncTLSInterceptProxy) -> dict[str, str]: "AWS_REGION": "us-east-1", "AWS_DEFAULT_REGION": "us-east-1", "AWS_MAX_ATTEMPTS": "1", - "AWS_CONFIG_FILE": "", + "AWS_CONFIG_FILE": _CRT_CONFIG.name, "AWS_SHARED_CREDENTIALS_FILE": "", "HTTPS_PROXY": proxy.endpoint_url, "HTTP_PROXY": proxy.endpoint_url, From fa19e963a42df82dfbe0576770e3cf1ececd3aff Mon Sep 17 00:00:00 2001 From: aemous Date: Wed, 2 Sep 2026 10:48:29 -0400 Subject: [PATCH 3/7] Port sync functional tests to blackbox tests and improve test coverage. --- tests/blackbox/test_cp_command.py | 78 +++++++++++++++++++++++++++++ tests/blackbox/test_ls_command.py | 75 +++++++++++++++++++++++++++ tests/blackbox/test_mb_command.py | 28 +++++++++++ tests/blackbox/test_sync_command.py | 35 +++++++++++++ tests/blackbox/utils.py | 11 +--- 5 files changed, 217 insertions(+), 10 deletions(-) diff --git a/tests/blackbox/test_cp_command.py b/tests/blackbox/test_cp_command.py index 1490ee5a08d7..36d606d20431 100644 --- a/tests/blackbox/test_cp_command.py +++ b/tests/blackbox/test_cp_command.py @@ -5475,3 +5475,81 @@ async def test_s3_express_warn_emits_warning(self, aws_cli, tmp_path): assert rc == 0, stderr.decode() assert "warning: Recursive copies/moves" in stderr.decode() + + + +@pytest.mark.asyncio +async def test_upload_key_with_spaces(aws_cli, tmp_path): + """cp uploads a file whose S3 key contains spaces.""" + src = tmp_path / "my file.txt" + src.write_text("content") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [put_object_response()]) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "cp", str(src), "s3://bucket/my file.txt"], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + # Space must be percent-encoded as %20, not + or literal space + assert server.requests[0].path == "/my%20file.txt" + + +@pytest.mark.asyncio +async def test_download_unicode_key_from_s3(aws_cli, tmp_path): + """cp --recursive downloads an object with a Unicode key.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + xml_response( + list_objects_xml( + contents=[ + { + "Key": "données.txt", + "Size": 5, + "LastModified": "2023-01-01T00:00:00Z", + } + ] + ) + ), + get_object_response(b"hello"), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "cp", "s3://bucket/", str(tmp_path), "--recursive"], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert (tmp_path / "données.txt").exists() + + +@pytest.mark.asyncio +async def test_upload_file_with_unicode_local_name(aws_cli, tmp_path): + """cp uploads a local file with a Unicode filename.""" + src = tmp_path / "données.txt" + src.write_text("content") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [put_object_response()]) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "cp", str(src), "s3://bucket/"], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + # Unicode filename is percent-encoded as UTF-8 on the wire + assert server.requests[0].path == "/donn%C3%A9es.txt" diff --git a/tests/blackbox/test_ls_command.py b/tests/blackbox/test_ls_command.py index 31239fcd4dad..21cf8df10243 100644 --- a/tests/blackbox/test_ls_command.py +++ b/tests/blackbox/test_ls_command.py @@ -572,3 +572,78 @@ async def test_list_objects_ignores_bucket_region(aws_cli: str) -> None: req = server.requests[0] params = get_query_params(req) assert "bucket-region" not in params + + + +@pytest.mark.asyncio +async def test_list_objects_with_unicode_keys(aws_cli): + """ls handles objects with Unicode keys (CJK, emoji).""" + async with mock_server() as (server, proxy): + setup_responses( + server, + [ + xml_response( + list_objects_xml( + contents=[ + { + "Key": "文件.txt", + "Size": 100, + "LastModified": "2023-01-01T00:00:00Z", + }, + { + "Key": "📄data.csv", + "Size": 200, + "LastModified": "2023-01-01T00:00:00Z", + }, + ] + ) + ), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "ls", "s3://bucket/"], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + output = stdout.decode() + assert "文件.txt" in output + assert "📄data.csv" in output + + +@pytest.mark.asyncio +async def test_list_objects_with_url_encoded_keys(aws_cli): + """ls handles keys that S3 returns with URL encoding.""" + async with mock_server() as (server, proxy): + setup_responses( + server, + [ + xml_response( + list_objects_xml( + contents=[ + { + "Key": "my+file.txt", + "Size": 100, + "LastModified": "2023-01-01T00:00:00Z", + }, + { + "Key": "path/to/my%20doc.pdf", + "Size": 200, + "LastModified": "2023-01-01T00:00:00Z", + }, + ] + ) + ), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "ls", "s3://bucket/"], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + output = stdout.decode() + assert "my+file.txt" in output + assert "my%20doc.pdf" in output diff --git a/tests/blackbox/test_mb_command.py b/tests/blackbox/test_mb_command.py index bc86865e14f7..374b486d891f 100644 --- a/tests/blackbox/test_mb_command.py +++ b/tests/blackbox/test_mb_command.py @@ -298,3 +298,31 @@ async def test_tags_with_three_arguments_fails(self, aws_cli): assert rc == 252 assert len(server.requests) == 0, format_requests(server) assert "ParamValidation" in stderr.decode() + + + +@pytest.mark.asyncio +async def test_create_bucket_with_non_ascii_tag_value(aws_cli): + """mb --tags with non-ASCII tag value sends correct XML body.""" + async with mock_server() as (server, proxy): + setup_responses(server, [create_bucket_response()]) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "mb", + "s3://bucket", + "--tags", + "Author", + "José García", + "--region", + "us-west-2", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + req = server.requests[0] + assert "José" in req.body or "Jos" in req.body, ( + f"Expected non-ASCII tag value in body, got: {req.body[:200]}" + ) diff --git a/tests/blackbox/test_sync_command.py b/tests/blackbox/test_sync_command.py index 682013243061..d45dcc9ce5e0 100644 --- a/tests/blackbox/test_sync_command.py +++ b/tests/blackbox/test_sync_command.py @@ -889,3 +889,38 @@ async def test_ignore_with_case_conflicts_in_s3(self, aws_cli, tmp_path): ) assert rc == 0, stderr.decode() + + + +@pytest.mark.asyncio +async def test_download_url_encoded_key_from_list(aws_cli, tmp_path): + """sync downloads objects whose keys contain spaces from ListObjectsV2.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + xml_response( + list_objects_xml( + contents=[ + { + "Key": "my file.txt", + "Size": 3, + "LastModified": "2023-01-01T00:00:00Z", + } + ] + ) + ), + get_object_response(b"foo"), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "sync", "s3://bucket/", str(tmp_path)], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert (tmp_path / "my file.txt").exists() diff --git a/tests/blackbox/utils.py b/tests/blackbox/utils.py index c851dcc0ae22..be8bb20fde73 100644 --- a/tests/blackbox/utils.py +++ b/tests/blackbox/utils.py @@ -11,15 +11,6 @@ from localstub.tlsproxy import AsyncTLSInterceptProxy -import tempfile - -_CRT_CONFIG = tempfile.NamedTemporaryFile( - mode="w", suffix=".cfg", delete=False, prefix="blackbox_crt_" -) -_CRT_CONFIG.write("[default]\ns3 =\n preferred_transfer_client = crt\n") -_CRT_CONFIG.close() - - def cli_env(proxy: AsyncTLSInterceptProxy) -> dict[str, str]: return { "PATH": os.environ.get("PATH", ""), @@ -29,7 +20,7 @@ def cli_env(proxy: AsyncTLSInterceptProxy) -> dict[str, str]: "AWS_REGION": "us-east-1", "AWS_DEFAULT_REGION": "us-east-1", "AWS_MAX_ATTEMPTS": "1", - "AWS_CONFIG_FILE": _CRT_CONFIG.name, + "AWS_CONFIG_FILE": "", "AWS_SHARED_CREDENTIALS_FILE": "", "HTTPS_PROXY": proxy.endpoint_url, "HTTP_PROXY": proxy.endpoint_url, From 6062f1ef3d1fbf391abf7b47c6e85c8ac96ed76c Mon Sep 17 00:00:00 2001 From: aemous Date: Wed, 2 Sep 2026 12:24:10 -0400 Subject: [PATCH 4/7] Add S3 blackbox tests using non-default config settings. --- tests/blackbox/test_s3_config.py | 242 +++++++++++++++++++++++++++++++ 1 file changed, 242 insertions(+) create mode 100644 tests/blackbox/test_s3_config.py diff --git a/tests/blackbox/test_s3_config.py b/tests/blackbox/test_s3_config.py new file mode 100644 index 000000000000..c4cc43fc9fc6 --- /dev/null +++ b/tests/blackbox/test_s3_config.py @@ -0,0 +1,242 @@ +"""Blackbox tests for S3 configuration options. + +These config options affect how the CLI constructs requests on the wire: +endpoint selection, multipart behavior, payload signing, etc. +""" +from __future__ import annotations + +import pytest +from localstub.handlers import handle_expect_header +from localstub.server import HTTPResponse + +from tests.blackbox.s3_assertions import ( + assert_complete_multipart_upload, + assert_create_multipart_upload, + assert_put_object, + assert_upload_part, +) +from tests.blackbox.utils import ( + cli_env, + complete_mpu_response, + create_mpu_response, + format_requests, + head_object_response, + mock_server, + put_object_response, + run_cli, + setup_responses, + upload_part_response, +) + + +def _cli_env_with_config(proxy, config_path): + env = cli_env(proxy) + env["AWS_CONFIG_FILE"] = config_path + return env + + +@pytest.mark.asyncio +async def test_multipart_threshold_single_put_below(aws_cli, aws_config, tmp_path): + """File below multipart_threshold uses single PutObject.""" + src = tmp_path / "small.bin" + src.write_bytes(b"x" * (4 * 1024 * 1024)) + config_path = aws_config({"default": {"s3": "\n multipart_threshold = 5MB"}}) + async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + setup_responses(server, [put_object_response()]) + _, stderr, rc = await run_cli( + aws_cli, + ["s3", "cp", str(src), "s3://bucket/small.bin"], + _cli_env_with_config(proxy, config_path), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + assert_put_object(server.requests[0], Bucket="bucket", Key="small.bin") + + +@pytest.mark.asyncio +async def test_multipart_threshold_multipart_above(aws_cli, aws_config, tmp_path): + """File above multipart_threshold uses multipart upload.""" + src = tmp_path / "big.bin" + src.write_bytes(b"x" * (6 * 1024 * 1024)) + config_path = aws_config({"default": {"s3": "\n multipart_threshold = 5MB"}}) + async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + setup_responses(server, [ + create_mpu_response("upload-id"), + upload_part_response("etag1"), + complete_mpu_response(), + ]) + _, stderr, rc = await run_cli( + aws_cli, + ["s3", "cp", str(src), "s3://bucket/big.bin"], + _cli_env_with_config(proxy, config_path), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 3, format_requests(server) + assert_create_multipart_upload(server.requests[0], Bucket="bucket", Key="big.bin") + assert_upload_part(server.requests[1], Bucket="bucket", Key="big.bin") + assert_complete_multipart_upload(server.requests[2], Bucket="bucket", Key="big.bin", UploadId="upload-id") + + +@pytest.mark.asyncio +async def test_multipart_chunksize_controls_part_count(aws_cli, aws_config, tmp_path): + """multipart_chunksize controls the size of each part.""" + src = tmp_path / "big.bin" + src.write_bytes(b"x" * (12 * 1024 * 1024)) + config_path = aws_config( + {"default": {"s3": "\n multipart_threshold = 5MB\n multipart_chunksize = 5MB"}} + ) + async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + setup_responses(server, [ + create_mpu_response("upload-id"), + upload_part_response("etag1"), + upload_part_response("etag2"), + upload_part_response("etag3"), + complete_mpu_response(), + ]) + _, stderr, rc = await run_cli( + aws_cli, + ["s3", "cp", str(src), "s3://bucket/big.bin"], + _cli_env_with_config(proxy, config_path), + ) + + assert rc == 0, stderr.decode() + assert_create_multipart_upload(server.requests[0], Bucket="bucket", Key="big.bin") + part_reqs = [r for r in server.requests if r.method == "PUT" and "partNumber" in r.path] + assert len(part_reqs) == 3, format_requests(server) + assert_complete_multipart_upload(server.requests[4], Bucket="bucket", Key="big.bin", UploadId="upload-id") + + +@pytest.mark.asyncio +async def test_addressing_style_path(aws_cli, aws_config, tmp_path): + """addressing_style=path puts bucket in the path, not the host.""" + src = tmp_path / "foo.txt" + src.write_text("content") + config_path = aws_config({"default": {"s3": "\n addressing_style = path"}}) + async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + setup_responses(server, [put_object_response()]) + _, stderr, rc = await run_cli( + aws_cli, + ["s3", "cp", str(src), "s3://bucket/foo.txt"], + _cli_env_with_config(proxy, config_path), + ) + + assert rc == 0, stderr.decode() + req = server.requests[0] + assert req.headers.get("host") == "s3.us-east-1.amazonaws.com" + assert req.path.startswith("/bucket/foo.txt") + + +@pytest.mark.asyncio +async def test_addressing_style_virtual(aws_cli, aws_config, tmp_path): + """addressing_style=virtual puts bucket in the host.""" + src = tmp_path / "foo.txt" + src.write_text("content") + config_path = aws_config({"default": {"s3": "\n addressing_style = virtual"}}) + async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + setup_responses(server, [put_object_response()]) + _, stderr, rc = await run_cli( + aws_cli, + ["s3", "cp", str(src), "s3://bucket/foo.txt"], + _cli_env_with_config(proxy, config_path), + ) + + assert rc == 0, stderr.decode() + req = server.requests[0] + assert "bucket" in req.headers.get("host", "") + assert req.path == "/foo.txt" or req.path.startswith("/foo.txt") + + +@pytest.mark.asyncio +async def test_use_accelerate_endpoint(aws_cli, aws_config, tmp_path): + """use_accelerate_endpoint=true routes to s3-accelerate.amazonaws.com.""" + src = tmp_path / "foo.txt" + src.write_text("content") + config_path = aws_config({"default": {"s3": "\n use_accelerate_endpoint = true"}}) + async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + setup_responses(server, [put_object_response()]) + _, stderr, rc = await run_cli( + aws_cli, + ["s3", "cp", str(src), "s3://bucket/foo.txt"], + _cli_env_with_config(proxy, config_path), + ) + + assert rc == 0, stderr.decode() + host = server.requests[0].headers.get("host", "") + assert "s3-accelerate" in host, f"Expected s3-accelerate in host, got {host}" + + +@pytest.mark.asyncio +async def test_payload_signing_disabled(aws_cli, aws_config, tmp_path): + """payload_signing_enabled=false sends UNSIGNED-PAYLOAD.""" + config_path = aws_config({"default": {"s3": "\n payload_signing_enabled = false"}}) + async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + setup_responses(server, [head_object_response()]) + _, stderr, rc = await run_cli( + aws_cli, + ["s3api", "head-object", "--bucket", "bucket", "--key", "foo.txt"], + _cli_env_with_config(proxy, config_path), + ) + + assert rc == 0, stderr.decode() + sha256 = server.requests[0].headers.get("x-amz-content-sha256") + assert sha256 == "UNSIGNED-PAYLOAD", f"Expected UNSIGNED-PAYLOAD, got {sha256}" + + +@pytest.mark.asyncio +async def test_payload_signing_enabled(aws_cli, aws_config, tmp_path): + """payload_signing_enabled=true sends the actual SHA256 hash.""" + config_path = aws_config({"default": {"s3": "\n payload_signing_enabled = true"}}) + async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + setup_responses(server, [head_object_response()]) + _, stderr, rc = await run_cli( + aws_cli, + ["s3api", "head-object", "--bucket", "bucket", "--key", "foo.txt"], + _cli_env_with_config(proxy, config_path), + ) + + assert rc == 0, stderr.decode() + sha256 = server.requests[0].headers.get("x-amz-content-sha256") + # SHA256 of empty body + assert sha256 == "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", ( + f"Expected SHA256 of empty body, got {sha256}" + ) + + +@pytest.mark.asyncio +async def test_use_dualstack_endpoint(aws_cli, aws_config, tmp_path): + """use_dualstack_endpoint=true routes to dualstack endpoint.""" + src = tmp_path / "foo.txt" + src.write_text("content") + config_path = aws_config({"default": {"s3": "\n use_dualstack_endpoint = true"}}) + async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + setup_responses(server, [put_object_response()]) + _, stderr, rc = await run_cli( + aws_cli, + ["s3", "cp", str(src), "s3://bucket/foo.txt"], + _cli_env_with_config(proxy, config_path), + ) + + assert rc == 0, stderr.decode() + host = server.requests[0].headers.get("host", "") + assert "dualstack" in host, f"Expected dualstack in host, got {host}" + + +@pytest.mark.asyncio +async def test_use_dualstack_endpoint_false(aws_cli, aws_config, tmp_path): + """use_dualstack_endpoint=false uses the standard endpoint.""" + src = tmp_path / "foo.txt" + src.write_text("content") + config_path = aws_config({"default": {"s3": "\n use_dualstack_endpoint = false"}}) + async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + setup_responses(server, [put_object_response()]) + _, stderr, rc = await run_cli( + aws_cli, + ["s3", "cp", str(src), "s3://bucket/foo.txt"], + _cli_env_with_config(proxy, config_path), + ) + + assert rc == 0, stderr.decode() + host = server.requests[0].headers.get("host", "") + assert "dualstack" not in host, f"Expected no dualstack in host, got {host}" From f42fb2032f7c6fecf79bb34929f5404ddda04ecc Mon Sep 17 00:00:00 2001 From: aemous Date: Wed, 2 Sep 2026 12:47:55 -0400 Subject: [PATCH 5/7] Add User-agent tests --- tests/blackbox/test_cp_command.py | 45 ++++++++++++++++++++++++++++++- tests/blackbox/test_mb_command.py | 1 - 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/tests/blackbox/test_cp_command.py b/tests/blackbox/test_cp_command.py index 36d606d20431..5de4c089c28a 100644 --- a/tests/blackbox/test_cp_command.py +++ b/tests/blackbox/test_cp_command.py @@ -5477,7 +5477,6 @@ async def test_s3_express_warn_emits_warning(self, aws_cli, tmp_path): assert "warning: Recursive copies/moves" in stderr.decode() - @pytest.mark.asyncio async def test_upload_key_with_spaces(aws_cli, tmp_path): """cp uploads a file whose S3 key contains spaces.""" @@ -5553,3 +5552,47 @@ async def test_upload_file_with_unicode_local_name(aws_cli, tmp_path): assert len(server.requests) == 1, format_requests(server) # Unicode filename is percent-encoded as UTF-8 on the wire assert server.requests[0].path == "/donn%C3%A9es.txt" + + + +@pytest.mark.asyncio +async def test_user_agent_contains_cli_version(aws_cli, tmp_path): + """Requests include a User-Agent with aws-cli version info.""" + src = tmp_path / "foo.txt" + src.write_text("content") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [put_object_response()]) + _, stderr, rc = await run_cli( + aws_cli, + ["s3", "cp", str(src), "s3://bucket/foo.txt"], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + ua = server.requests[0].headers.get("user-agent") + assert ua is not None, "User-Agent header missing" + assert "aws-cli/" in ua, f"Expected 'aws-cli/' in User-Agent: {ua}" + + +@pytest.mark.asyncio +async def test_user_agent_contains_command(aws_cli, tmp_path): + """User-Agent includes the command being run (e.g. s3.cp).""" + src = tmp_path / "foo.txt" + src.write_text("content") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [put_object_response()]) + _, stderr, rc = await run_cli( + aws_cli, + ["s3", "cp", str(src), "s3://bucket/foo.txt"], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + ua = server.requests[0].headers.get("user-agent") + assert "s3.cp" in ua, f"Expected 's3.cp' in User-Agent: {ua}" diff --git a/tests/blackbox/test_mb_command.py b/tests/blackbox/test_mb_command.py index 374b486d891f..2108085729f9 100644 --- a/tests/blackbox/test_mb_command.py +++ b/tests/blackbox/test_mb_command.py @@ -300,7 +300,6 @@ async def test_tags_with_three_arguments_fails(self, aws_cli): assert "ParamValidation" in stderr.decode() - @pytest.mark.asyncio async def test_create_bucket_with_non_ascii_tag_value(aws_cli): """mb --tags with non-ASCII tag value sends correct XML body.""" From 10133c22c3be4991b72a5dfe3c9deeb0eab4a6e7 Mon Sep 17 00:00:00 2001 From: aemous Date: Wed, 2 Sep 2026 12:49:02 -0400 Subject: [PATCH 6/7] Formatting --- tests/blackbox/test_cp_command.py | 123 +-- tests/blackbox/test_ls_command.py | 4 +- tests/blackbox/test_mb_command.py | 13 +- tests/blackbox/test_s3_config.py | 182 +++-- tests/blackbox/test_sync_command.py | 1168 ++++++++++++++++++++------- 5 files changed, 1032 insertions(+), 458 deletions(-) diff --git a/tests/blackbox/test_cp_command.py b/tests/blackbox/test_cp_command.py index 5de4c089c28a..4be7dab02cff 100644 --- a/tests/blackbox/test_cp_command.py +++ b/tests/blackbox/test_cp_command.py @@ -39,10 +39,10 @@ get_object_tagging_response, head_object_response, list_objects_xml, + mock_server, put_object_response, put_object_tagging_response, run_cli, - mock_server, setup_responses, upload_part_copy_response, upload_part_response, @@ -452,12 +452,8 @@ async def test_operations_used_in_download_file(self, aws_cli, tmp_path): assert rc == 0, stderr.decode() assert len(server.requests) == 2, format_requests(server) - assert_head_object( - server.requests[0], Bucket="bucket", Key="key.txt" - ) - assert_get_object( - server.requests[1], Bucket="bucket", Key="key.txt" - ) + assert_head_object(server.requests[0], Bucket="bucket", Key="key.txt") + assert_get_object(server.requests[1], Bucket="bucket", Key="key.txt") async def test_operations_used_in_recursive_download( self, aws_cli, tmp_path @@ -482,9 +478,7 @@ async def test_operations_used_in_recursive_download( assert rc == 0, stderr.decode() assert len(server.requests) == 1, format_requests(server) - assert_list_objects_v2( - server.requests[0], Bucket="bucket" - ) + assert_list_objects_v2(server.requests[0], Bucket="bucket") async def test_no_overwrite_flag_when_object_not_exists_on_target( self, aws_cli, tmp_path @@ -512,9 +506,7 @@ async def test_no_overwrite_flag_when_object_not_exists_on_target( assert rc == 0, stderr.decode() assert len(server.requests) == 1, format_requests(server) req = server.requests[0] - assert_put_object( - req, Bucket="bucket", Key="key.txt", IfNoneMatch="*" - ) + assert_put_object(req, Bucket="bucket", Key="key.txt", IfNoneMatch="*") async def test_no_overwrite_flag_when_object_exists_on_target( self, aws_cli, tmp_path @@ -551,9 +543,7 @@ async def test_no_overwrite_flag_when_object_exists_on_target( assert rc == 0, stderr.decode() assert len(server.requests) == 1, format_requests(server) req = server.requests[0] - assert_put_object( - req, Bucket="bucket", Key="key.txt", IfNoneMatch="*" - ) + assert_put_object(req, Bucket="bucket", Key="key.txt", IfNoneMatch="*") async def test_no_overwrite_flag_multipart_upload_when_object_not_exists_on_target( self, aws_cli, tmp_path @@ -686,9 +676,7 @@ async def test_no_overwrite_flag_on_copy_when_small_object_does_not_exist_on_tar assert rc == 0, stderr.decode() assert len(server.requests) == 2, format_requests(server) - assert_head_object( - server.requests[0], Bucket="bucket1", Key="key.txt" - ) + assert_head_object(server.requests[0], Bucket="bucket1", Key="key.txt") assert_copy_object( server.requests[1], Bucket="bucket", @@ -730,9 +718,7 @@ async def test_no_overwrite_flag_on_copy_when_small_object_exists_on_target( assert rc == 0, stderr.decode() assert len(server.requests) == 2, format_requests(server) - assert_head_object( - server.requests[0], Bucket="bucket1", Key="key.txt" - ) + assert_head_object(server.requests[0], Bucket="bucket1", Key="key.txt") assert_copy_object( server.requests[1], Bucket="bucket", @@ -861,9 +847,7 @@ async def test_metadata_copy(self, aws_cli, tmp_path): assert rc == 0, stderr.decode() assert len(server.requests) == 2, format_requests(server) - assert_head_object( - server.requests[0], Bucket="bucket", Key="key.txt" - ) + assert_head_object(server.requests[0], Bucket="bucket", Key="key.txt") assert_copy_object( server.requests[1], Bucket="bucket", @@ -974,9 +958,7 @@ async def test_metadata_directive_copy(self, aws_cli, tmp_path): assert rc == 0, stderr.decode() assert len(server.requests) == 2, format_requests(server) - assert_head_object( - server.requests[0], Bucket="bucket", Key="key.txt" - ) + assert_head_object(server.requests[0], Bucket="bucket", Key="key.txt") assert_copy_object( server.requests[1], Bucket="bucket", @@ -1056,9 +1038,7 @@ async def test_recursive_glacier_download_with_force_glacier( assert rc == 0, stderr.decode() assert len(server.requests) == 2, format_requests(server) - assert_list_objects_v2( - server.requests[0], Bucket="bucket" - ) + assert_list_objects_v2(server.requests[0], Bucket="bucket") assert_get_object( server.requests[1], Bucket="bucket", Key="foo/bar.txt" ) @@ -1098,9 +1078,7 @@ async def test_recursive_glacier_download_without_force_glacier( assert rc == 2 assert len(server.requests) == 1, format_requests(server) - assert_list_objects_v2( - server.requests[0], Bucket="bucket" - ) + assert_list_objects_v2(server.requests[0], Bucket="bucket") assert "GLACIER" in stderr.decode() async def test_warns_on_glacier_incompatible_operation( @@ -1134,9 +1112,7 @@ async def test_warns_on_glacier_incompatible_operation( assert rc == 2 assert len(server.requests) == 1, format_requests(server) - assert_head_object( - server.requests[0], Bucket="bucket", Key="key.txt" - ) + assert_head_object(server.requests[0], Bucket="bucket", Key="key.txt") assert "GLACIER" in stderr.decode() async def test_warns_on_deep_archive_incompatible_operation( @@ -1170,9 +1146,7 @@ async def test_warns_on_deep_archive_incompatible_operation( assert rc == 2 assert len(server.requests) == 1, format_requests(server) - assert_head_object( - server.requests[0], Bucket="bucket", Key="key.txt" - ) + assert_head_object(server.requests[0], Bucket="bucket", Key="key.txt") assert "GLACIER" in stderr.decode() async def test_turn_off_glacier_warnings(self, aws_cli, tmp_path): @@ -1210,9 +1184,7 @@ async def test_turn_off_glacier_warnings(self, aws_cli, tmp_path): assert rc == 0, stderr.decode() assert len(server.requests) == 1, format_requests(server) - assert_head_object( - server.requests[0], Bucket="bucket", Key="key.txt" - ) + assert_head_object(server.requests[0], Bucket="bucket", Key="key.txt") assert stderr.decode() == "" async def test_turn_off_glacier_warnings_for_deep_archive( @@ -1252,9 +1224,7 @@ async def test_turn_off_glacier_warnings_for_deep_archive( assert rc == 0, stderr.decode() assert len(server.requests) == 1, format_requests(server) - assert_head_object( - server.requests[0], Bucket="bucket", Key="key.txt" - ) + assert_head_object(server.requests[0], Bucket="bucket", Key="key.txt") assert stderr.decode() == "" async def test_cp_with_sse_flag(self, aws_cli, tmp_path): @@ -2273,9 +2243,7 @@ async def test_no_overwrite_flag_on_download_when_single_object_already_exists_a assert rc == 0, stderr.decode() assert len(server.requests) == 1, format_requests(server) - assert_head_object( - server.requests[0], Bucket="bucket", Key="foo.txt" - ) + assert_head_object(server.requests[0], Bucket="bucket", Key="foo.txt") # File should not have been overwritten assert target.read_text() == "existing content" @@ -2309,12 +2277,8 @@ async def test_no_overwrite_flag_on_download_when_single_object_does_not_exist_a assert rc == 0, stderr.decode() assert len(server.requests) == 2, format_requests(server) - assert_head_object( - server.requests[0], Bucket="bucket", Key="foo.txt" - ) - assert_get_object( - server.requests[1], Bucket="bucket", Key="foo.txt" - ) + assert_head_object(server.requests[0], Bucket="bucket", Key="foo.txt") + assert_get_object(server.requests[1], Bucket="bucket", Key="foo.txt") assert target.read_text() == "foo" async def test_warns_on_deep_arhive_incompatible_operation( @@ -2348,9 +2312,7 @@ async def test_warns_on_deep_arhive_incompatible_operation( assert rc == 2 assert len(server.requests) == 1, format_requests(server) - assert_head_object( - server.requests[0], Bucket="bucket", Key="key.txt" - ) + assert_head_object(server.requests[0], Bucket="bucket", Key="key.txt") assert "GLACIER" in stderr.decode() async def test_warns_on_glacier_incompatible_operation_for_multipart_file( @@ -2384,9 +2346,7 @@ async def test_warns_on_glacier_incompatible_operation_for_multipart_file( assert rc == 2 assert len(server.requests) == 1, format_requests(server) - assert_head_object( - server.requests[0], Bucket="bucket", Key="key.txt" - ) + assert_head_object(server.requests[0], Bucket="bucket", Key="key.txt") assert "GLACIER" in stderr.decode() async def test_warns_on_deep_archive_incompatible_op_for_multipart_file( @@ -2420,9 +2380,7 @@ async def test_warns_on_deep_archive_incompatible_op_for_multipart_file( assert rc == 2 assert len(server.requests) == 1, format_requests(server) - assert_head_object( - server.requests[0], Bucket="bucket", Key="key.txt" - ) + assert_head_object(server.requests[0], Bucket="bucket", Key="key.txt") assert "GLACIER" in stderr.decode() async def test_s3s3_cp_with_destination_sse_c_multipart( @@ -3275,9 +3233,7 @@ async def test_upload(self, aws_cli, tmp_path): assert rc == 0, stderr.decode() assert len(server.requests) == 1, format_requests(server) - assert_put_object( - server.requests[0], Bucket=self.ARN, Key="mykey" - ) + assert_put_object(server.requests[0], Bucket=self.ARN, Key="mykey") async def test_recursive_upload(self, aws_cli, tmp_path): """cp local s3:/// --recursive uploads to accesspoint.""" @@ -3301,9 +3257,7 @@ async def test_recursive_upload(self, aws_cli, tmp_path): assert rc == 0, stderr.decode() assert len(server.requests) == 1, format_requests(server) - assert_put_object( - server.requests[0], Bucket=self.ARN, Key="myfile" - ) + assert_put_object(server.requests[0], Bucket=self.ARN, Key="myfile") async def test_download(self, aws_cli, tmp_path): """cp s3:///key local downloads from accesspoint.""" @@ -3332,9 +3286,7 @@ async def test_download(self, aws_cli, tmp_path): Key="mykey", ChecksumMode="ENABLED", ) - assert_get_object( - server.requests[1], Bucket=self.ARN, Key="mykey" - ) + assert_get_object(server.requests[1], Bucket=self.ARN, Key="mykey") async def test_recursive_download(self, aws_cli, tmp_path): """cp s3:// local --recursive downloads from accesspoint.""" @@ -3371,9 +3323,7 @@ async def test_recursive_download(self, aws_cli, tmp_path): server.requests[0], Bucket=self.ARN, ) - assert_get_object( - server.requests[1], Bucket=self.ARN, Key="mykey" - ) + assert_get_object(server.requests[1], Bucket=self.ARN, Key="mykey") async def test_copy(self, aws_cli, tmp_path): """cp s3:///key s3:///key copies between accesspoints.""" @@ -3408,9 +3358,7 @@ async def test_copy(self, aws_cli, tmp_path): Key="mykey", ChecksumMode="ENABLED", ) - assert_copy_object( - server.requests[1], Bucket=dest_arn, Key="mykey" - ) + assert_copy_object(server.requests[1], Bucket=dest_arn, Key="mykey") async def test_recursive_copy(self, aws_cli, tmp_path): """cp s3:// s3:// --recursive copies between accesspoints.""" @@ -3454,9 +3402,7 @@ async def test_recursive_copy(self, aws_cli, tmp_path): server.requests[0], Bucket=self.ARN, ) - assert_copy_object( - server.requests[1], Bucket=dest_arn, Key="mykey" - ) + assert_copy_object(server.requests[1], Bucket=dest_arn, Key="mykey") async def test_accepts_mrap_arns(self, aws_cli, tmp_path): """cp to MRAP ARN (colon separator) works.""" @@ -3476,9 +3422,7 @@ async def test_accepts_mrap_arns(self, aws_cli, tmp_path): assert rc == 0, stderr.decode() assert len(server.requests) == 1, format_requests(server) - assert_put_object( - server.requests[0], Bucket=mrap_arn, Key="mykey" - ) + assert_put_object(server.requests[0], Bucket=mrap_arn, Key="mykey") async def test_accepts_mrap_arns_with_slash(self, aws_cli, tmp_path): """cp to MRAP ARN (slash separator) works.""" @@ -3498,9 +3442,7 @@ async def test_accepts_mrap_arns_with_slash(self, aws_cli, tmp_path): assert rc == 0, stderr.decode() assert len(server.requests) == 1, format_requests(server) - assert_put_object( - server.requests[0], Bucket=mrap_arn, Key="mykey" - ) + assert_put_object(server.requests[0], Bucket=mrap_arn, Key="mykey") @pytest.mark.asyncio @@ -5284,9 +5226,7 @@ async def test_respects_source_region_for_recursive_mp_copy( assert rc == 0, stderr.decode() assert len(server.requests) == 7, format_requests(server) # Source region: ListObjectsV2 - assert_list_objects_v2( - server.requests[0], Bucket=self.SOURCE_BUCKET - ) + assert_list_objects_v2(server.requests[0], Bucket=self.SOURCE_BUCKET) assert server.requests[0].headers.get("host") == self.SOURCE_HOST # Source region: HeadObject (for metadata + size/etag) assert_head_object( @@ -5554,7 +5494,6 @@ async def test_upload_file_with_unicode_local_name(aws_cli, tmp_path): assert server.requests[0].path == "/donn%C3%A9es.txt" - @pytest.mark.asyncio async def test_user_agent_contains_cli_version(aws_cli, tmp_path): """Requests include a User-Agent with aws-cli version info.""" diff --git a/tests/blackbox/test_ls_command.py b/tests/blackbox/test_ls_command.py index 21cf8df10243..848ab62c63bc 100644 --- a/tests/blackbox/test_ls_command.py +++ b/tests/blackbox/test_ls_command.py @@ -12,14 +12,13 @@ assert_list_buckets, assert_list_objects_v2, ) - from tests.blackbox.utils import ( cli_env, format_requests, get_query_params, list_objects_xml, - run_cli, mock_server, + run_cli, setup_responses, xml_response, ) @@ -574,7 +573,6 @@ async def test_list_objects_ignores_bucket_region(aws_cli: str) -> None: assert "bucket-region" not in params - @pytest.mark.asyncio async def test_list_objects_with_unicode_keys(aws_cli): """ls handles objects with Unicode keys (CJK, emoji).""" diff --git a/tests/blackbox/test_mb_command.py b/tests/blackbox/test_mb_command.py index 2108085729f9..a9c33f7e2a51 100644 --- a/tests/blackbox/test_mb_command.py +++ b/tests/blackbox/test_mb_command.py @@ -9,13 +9,12 @@ from tests.blackbox.s3_assertions import ( assert_create_bucket, ) - from tests.blackbox.utils import ( cli_env, create_bucket_response, format_requests, - run_cli, mock_server, + run_cli, setup_responses, ) @@ -39,9 +38,7 @@ async def test_make_bucket(self, aws_cli): ) assert rc == 0, stderr.decode() assert len(server.requests) == 1, format_requests(server) - assert_create_bucket( - server.requests[0], Bucket="bucket" - ) + assert_create_bucket(server.requests[0], Bucket="bucket") async def test_adds_location_constraint(self, aws_cli): async with mock_server() as (server, proxy): @@ -322,6 +319,6 @@ async def test_create_bucket_with_non_ascii_tag_value(aws_cli): assert rc == 0, stderr.decode() req = server.requests[0] - assert "José" in req.body or "Jos" in req.body, ( - f"Expected non-ASCII tag value in body, got: {req.body[:200]}" - ) + assert ( + "José" in req.body or "Jos" in req.body + ), f"Expected non-ASCII tag value in body, got: {req.body[:200]}" diff --git a/tests/blackbox/test_s3_config.py b/tests/blackbox/test_s3_config.py index c4cc43fc9fc6..b9a67684e1f1 100644 --- a/tests/blackbox/test_s3_config.py +++ b/tests/blackbox/test_s3_config.py @@ -3,6 +3,7 @@ These config options affect how the CLI constructs requests on the wire: endpoint selection, multipart behavior, payload signing, etc. """ + from __future__ import annotations import pytest @@ -36,12 +37,19 @@ def _cli_env_with_config(proxy, config_path): @pytest.mark.asyncio -async def test_multipart_threshold_single_put_below(aws_cli, aws_config, tmp_path): +async def test_multipart_threshold_single_put_below( + aws_cli, aws_config, tmp_path +): """File below multipart_threshold uses single PutObject.""" src = tmp_path / "small.bin" src.write_bytes(b"x" * (4 * 1024 * 1024)) - config_path = aws_config({"default": {"s3": "\n multipart_threshold = 5MB"}}) - async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + config_path = aws_config( + {"default": {"s3": "\n multipart_threshold = 5MB"}} + ) + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): setup_responses(server, [put_object_response()]) _, stderr, rc = await run_cli( aws_cli, @@ -55,17 +63,27 @@ async def test_multipart_threshold_single_put_below(aws_cli, aws_config, tmp_pat @pytest.mark.asyncio -async def test_multipart_threshold_multipart_above(aws_cli, aws_config, tmp_path): +async def test_multipart_threshold_multipart_above( + aws_cli, aws_config, tmp_path +): """File above multipart_threshold uses multipart upload.""" src = tmp_path / "big.bin" src.write_bytes(b"x" * (6 * 1024 * 1024)) - config_path = aws_config({"default": {"s3": "\n multipart_threshold = 5MB"}}) - async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): - setup_responses(server, [ - create_mpu_response("upload-id"), - upload_part_response("etag1"), - complete_mpu_response(), - ]) + config_path = aws_config( + {"default": {"s3": "\n multipart_threshold = 5MB"}} + ) + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + create_mpu_response("upload-id"), + upload_part_response("etag1"), + complete_mpu_response(), + ], + ) _, stderr, rc = await run_cli( aws_cli, ["s3", "cp", str(src), "s3://bucket/big.bin"], @@ -74,27 +92,46 @@ async def test_multipart_threshold_multipart_above(aws_cli, aws_config, tmp_path assert rc == 0, stderr.decode() assert len(server.requests) == 3, format_requests(server) - assert_create_multipart_upload(server.requests[0], Bucket="bucket", Key="big.bin") + assert_create_multipart_upload( + server.requests[0], Bucket="bucket", Key="big.bin" + ) assert_upload_part(server.requests[1], Bucket="bucket", Key="big.bin") - assert_complete_multipart_upload(server.requests[2], Bucket="bucket", Key="big.bin", UploadId="upload-id") + assert_complete_multipart_upload( + server.requests[2], + Bucket="bucket", + Key="big.bin", + UploadId="upload-id", + ) @pytest.mark.asyncio -async def test_multipart_chunksize_controls_part_count(aws_cli, aws_config, tmp_path): +async def test_multipart_chunksize_controls_part_count( + aws_cli, aws_config, tmp_path +): """multipart_chunksize controls the size of each part.""" src = tmp_path / "big.bin" src.write_bytes(b"x" * (12 * 1024 * 1024)) config_path = aws_config( - {"default": {"s3": "\n multipart_threshold = 5MB\n multipart_chunksize = 5MB"}} + { + "default": { + "s3": "\n multipart_threshold = 5MB\n multipart_chunksize = 5MB" + } + } ) - async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): - setup_responses(server, [ - create_mpu_response("upload-id"), - upload_part_response("etag1"), - upload_part_response("etag2"), - upload_part_response("etag3"), - complete_mpu_response(), - ]) + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + create_mpu_response("upload-id"), + upload_part_response("etag1"), + upload_part_response("etag2"), + upload_part_response("etag3"), + complete_mpu_response(), + ], + ) _, stderr, rc = await run_cli( aws_cli, ["s3", "cp", str(src), "s3://bucket/big.bin"], @@ -102,10 +139,21 @@ async def test_multipart_chunksize_controls_part_count(aws_cli, aws_config, tmp_ ) assert rc == 0, stderr.decode() - assert_create_multipart_upload(server.requests[0], Bucket="bucket", Key="big.bin") - part_reqs = [r for r in server.requests if r.method == "PUT" and "partNumber" in r.path] + assert_create_multipart_upload( + server.requests[0], Bucket="bucket", Key="big.bin" + ) + part_reqs = [ + r + for r in server.requests + if r.method == "PUT" and "partNumber" in r.path + ] assert len(part_reqs) == 3, format_requests(server) - assert_complete_multipart_upload(server.requests[4], Bucket="bucket", Key="big.bin", UploadId="upload-id") + assert_complete_multipart_upload( + server.requests[4], + Bucket="bucket", + Key="big.bin", + UploadId="upload-id", + ) @pytest.mark.asyncio @@ -113,8 +161,13 @@ async def test_addressing_style_path(aws_cli, aws_config, tmp_path): """addressing_style=path puts bucket in the path, not the host.""" src = tmp_path / "foo.txt" src.write_text("content") - config_path = aws_config({"default": {"s3": "\n addressing_style = path"}}) - async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + config_path = aws_config( + {"default": {"s3": "\n addressing_style = path"}} + ) + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): setup_responses(server, [put_object_response()]) _, stderr, rc = await run_cli( aws_cli, @@ -133,8 +186,13 @@ async def test_addressing_style_virtual(aws_cli, aws_config, tmp_path): """addressing_style=virtual puts bucket in the host.""" src = tmp_path / "foo.txt" src.write_text("content") - config_path = aws_config({"default": {"s3": "\n addressing_style = virtual"}}) - async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + config_path = aws_config( + {"default": {"s3": "\n addressing_style = virtual"}} + ) + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): setup_responses(server, [put_object_response()]) _, stderr, rc = await run_cli( aws_cli, @@ -153,8 +211,13 @@ async def test_use_accelerate_endpoint(aws_cli, aws_config, tmp_path): """use_accelerate_endpoint=true routes to s3-accelerate.amazonaws.com.""" src = tmp_path / "foo.txt" src.write_text("content") - config_path = aws_config({"default": {"s3": "\n use_accelerate_endpoint = true"}}) - async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + config_path = aws_config( + {"default": {"s3": "\n use_accelerate_endpoint = true"}} + ) + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): setup_responses(server, [put_object_response()]) _, stderr, rc = await run_cli( aws_cli, @@ -164,14 +227,21 @@ async def test_use_accelerate_endpoint(aws_cli, aws_config, tmp_path): assert rc == 0, stderr.decode() host = server.requests[0].headers.get("host", "") - assert "s3-accelerate" in host, f"Expected s3-accelerate in host, got {host}" + assert ( + "s3-accelerate" in host + ), f"Expected s3-accelerate in host, got {host}" @pytest.mark.asyncio async def test_payload_signing_disabled(aws_cli, aws_config, tmp_path): """payload_signing_enabled=false sends UNSIGNED-PAYLOAD.""" - config_path = aws_config({"default": {"s3": "\n payload_signing_enabled = false"}}) - async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + config_path = aws_config( + {"default": {"s3": "\n payload_signing_enabled = false"}} + ) + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): setup_responses(server, [head_object_response()]) _, stderr, rc = await run_cli( aws_cli, @@ -181,14 +251,21 @@ async def test_payload_signing_disabled(aws_cli, aws_config, tmp_path): assert rc == 0, stderr.decode() sha256 = server.requests[0].headers.get("x-amz-content-sha256") - assert sha256 == "UNSIGNED-PAYLOAD", f"Expected UNSIGNED-PAYLOAD, got {sha256}" + assert ( + sha256 == "UNSIGNED-PAYLOAD" + ), f"Expected UNSIGNED-PAYLOAD, got {sha256}" @pytest.mark.asyncio async def test_payload_signing_enabled(aws_cli, aws_config, tmp_path): """payload_signing_enabled=true sends the actual SHA256 hash.""" - config_path = aws_config({"default": {"s3": "\n payload_signing_enabled = true"}}) - async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + config_path = aws_config( + {"default": {"s3": "\n payload_signing_enabled = true"}} + ) + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): setup_responses(server, [head_object_response()]) _, stderr, rc = await run_cli( aws_cli, @@ -199,9 +276,10 @@ async def test_payload_signing_enabled(aws_cli, aws_config, tmp_path): assert rc == 0, stderr.decode() sha256 = server.requests[0].headers.get("x-amz-content-sha256") # SHA256 of empty body - assert sha256 == "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", ( - f"Expected SHA256 of empty body, got {sha256}" - ) + assert ( + sha256 + == "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + ), f"Expected SHA256 of empty body, got {sha256}" @pytest.mark.asyncio @@ -209,8 +287,13 @@ async def test_use_dualstack_endpoint(aws_cli, aws_config, tmp_path): """use_dualstack_endpoint=true routes to dualstack endpoint.""" src = tmp_path / "foo.txt" src.write_text("content") - config_path = aws_config({"default": {"s3": "\n use_dualstack_endpoint = true"}}) - async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + config_path = aws_config( + {"default": {"s3": "\n use_dualstack_endpoint = true"}} + ) + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): setup_responses(server, [put_object_response()]) _, stderr, rc = await run_cli( aws_cli, @@ -228,8 +311,13 @@ async def test_use_dualstack_endpoint_false(aws_cli, aws_config, tmp_path): """use_dualstack_endpoint=false uses the standard endpoint.""" src = tmp_path / "foo.txt" src.write_text("content") - config_path = aws_config({"default": {"s3": "\n use_dualstack_endpoint = false"}}) - async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + config_path = aws_config( + {"default": {"s3": "\n use_dualstack_endpoint = false"}} + ) + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): setup_responses(server, [put_object_response()]) _, stderr, rc = await run_cli( aws_cli, @@ -239,4 +327,6 @@ async def test_use_dualstack_endpoint_false(aws_cli, aws_config, tmp_path): assert rc == 0, stderr.decode() host = server.requests[0].headers.get("host", "") - assert "dualstack" not in host, f"Expected no dualstack in host, got {host}" + assert ( + "dualstack" not in host + ), f"Expected no dualstack in host, got {host}" diff --git a/tests/blackbox/test_sync_command.py b/tests/blackbox/test_sync_command.py index d45dcc9ce5e0..22a413572e99 100644 --- a/tests/blackbox/test_sync_command.py +++ b/tests/blackbox/test_sync_command.py @@ -1,4 +1,5 @@ """Blackbox tests for `aws s3 sync` command.""" + from __future__ import annotations import base64 @@ -9,7 +10,9 @@ from localstub.server import HTTPResponse from tests.blackbox.s3_assertions import ( + assert_complete_multipart_upload, assert_copy_object, + assert_create_multipart_upload, assert_delete_object, assert_get_object, assert_get_object_tagging, @@ -18,8 +21,6 @@ assert_put_object, assert_put_object_tagging, assert_upload_part_copy, - assert_complete_multipart_upload, - assert_create_multipart_upload, ) from tests.blackbox.utils import ( cli_env, @@ -56,7 +57,9 @@ def _list_response(keys, **overrides): "Key": k, "Size": overrides.get("Size", 100), "LastModified": "2014-01-09T20:45:49.000Z", - "ETag": overrides.get("ETag", '"c8afdb36c52cf4727836669019e69222"'), + "ETag": overrides.get( + "ETag", '"c8afdb36c52cf4727836669019e69222"' + ), } if "StorageClass" in overrides: entry["StorageClass"] = overrides["StorageClass"] @@ -74,27 +77,46 @@ async def test_website_redirect_ignore_paramfile(self, aws_cli, tmp_path): """sync local s3:// --website-redirect uses the URL value.""" src = tmp_path / "foo.txt" src.write_text("mycontent") - async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): - setup_responses(server, [ - _empty_list_response(), - put_object_response(), - ]) + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + _empty_list_response(), + put_object_response(), + ], + ) stdout, stderr, rc = await run_cli( aws_cli, - ["s3", "sync", str(tmp_path), "s3://bucket/key.txt", - "--website-redirect", "http://someserver"], + [ + "s3", + "sync", + str(tmp_path), + "s3://bucket/key.txt", + "--website-redirect", + "http://someserver", + ], cli_env(proxy), ) assert rc == 0, stderr.decode() assert len(server.requests) == 2, format_requests(server) assert_list_objects_v2(server.requests[0], Bucket="bucket") - assert_put_object(server.requests[1], Bucket="bucket", Key="key.txt/foo.txt", - WebsiteRedirectLocation="http://someserver") + assert_put_object( + server.requests[1], + Bucket="bucket", + Key="key.txt/foo.txt", + WebsiteRedirectLocation="http://someserver", + ) async def test_no_recursive_option(self, aws_cli, tmp_path): """sync does not accept --recursive.""" - async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): setup_responses(server, []) stdout, stderr, rc = await run_cli( aws_cli, @@ -107,7 +129,10 @@ async def test_no_recursive_option(self, aws_cli, tmp_path): async def test_sync_from_non_existent_directory(self, aws_cli, tmp_path): """sync from non-existent local dir fails.""" fakedir = str(tmp_path / "fakedir") - async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): setup_responses(server, [_empty_list_response()]) stdout, stderr, rc = await run_cli( aws_cli, @@ -121,11 +146,17 @@ async def test_sync_from_non_existent_directory(self, aws_cli, tmp_path): async def test_sync_to_non_existent_directory(self, aws_cli, tmp_path): """sync s3->local creates the target directory.""" target = tmp_path / "fakedir" - async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): - setup_responses(server, [ - _list_response(["foo.txt"]), - get_object_response(b"foo"), - ]) + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + _list_response(["foo.txt"]), + get_object_response(b"foo"), + ], + ) stdout, stderr, rc = await run_cli( aws_cli, ["s3", "sync", "s3://bucket/", str(target)], @@ -139,7 +170,10 @@ async def test_dryrun_sync(self, aws_cli, tmp_path): """sync --dryrun only lists, does not transfer.""" src = tmp_path / "file.txt" src.write_text("mycontent") - async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): setup_responses(server, [_empty_list_response()]) stdout, stderr, rc = await run_cli( aws_cli, @@ -154,34 +188,67 @@ async def test_dryrun_sync(self, aws_cli, tmp_path): async def test_glacier_sync_with_force_glacier(self, aws_cli, tmp_path): """sync s3->local --force-glacier-transfer downloads glacier objects.""" - async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): - setup_responses(server, [ - _list_response(["foo/bar.txt"], StorageClass="GLACIER"), - get_object_response(b"foo"), - ]) + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + _list_response(["foo/bar.txt"], StorageClass="GLACIER"), + get_object_response(b"foo"), + ], + ) stdout, stderr, rc = await run_cli( aws_cli, - ["s3", "sync", "s3://bucket/foo", str(tmp_path), - "--force-glacier-transfer"], + [ + "s3", + "sync", + "s3://bucket/foo", + str(tmp_path), + "--force-glacier-transfer", + ], cli_env(proxy), ) assert rc == 0, stderr.decode() assert len(server.requests) == 2, format_requests(server) assert_list_objects_v2(server.requests[0], Bucket="bucket") - assert_get_object(server.requests[1], Bucket="bucket", Key="foo/bar.txt") + assert_get_object( + server.requests[1], Bucket="bucket", Key="foo/bar.txt" + ) - async def test_handles_glacier_incompatible_operations(self, aws_cli, tmp_path): + async def test_handles_glacier_incompatible_operations( + self, aws_cli, tmp_path + ): """sync s3->local skips glacier/deep archive objects with warning.""" - async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): - setup_responses(server, [ - xml_response(list_objects_xml(contents=[ - {"Key": "foo", "Size": 100, "LastModified": "2014-01-09T20:45:49.000Z", - "StorageClass": "GLACIER"}, - {"Key": "bar", "Size": 100, "LastModified": "2014-01-09T20:45:49.000Z", - "StorageClass": "DEEP_ARCHIVE"}, - ])), - ]) + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + xml_response( + list_objects_xml( + contents=[ + { + "Key": "foo", + "Size": 100, + "LastModified": "2014-01-09T20:45:49.000Z", + "StorageClass": "GLACIER", + }, + { + "Key": "bar", + "Size": 100, + "LastModified": "2014-01-09T20:45:49.000Z", + "StorageClass": "DEEP_ARCHIVE", + }, + ] + ) + ), + ], + ) stdout, stderr, rc = await run_cli( aws_cli, ["s3", "sync", "s3://bucket/", str(tmp_path)], @@ -196,19 +263,42 @@ async def test_handles_glacier_incompatible_operations(self, aws_cli, tmp_path): async def test_turn_off_glacier_warnings(self, aws_cli, tmp_path): """sync s3->local --ignore-glacier-warnings suppresses warnings.""" - async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): - setup_responses(server, [ - xml_response(list_objects_xml(contents=[ - {"Key": "foo", "Size": 100, "LastModified": "2014-01-09T20:45:49.000Z", - "StorageClass": "GLACIER"}, - {"Key": "bar", "Size": 100, "LastModified": "2014-01-09T20:45:49.000Z", - "StorageClass": "DEEP_ARCHIVE"}, - ])), - ]) + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + xml_response( + list_objects_xml( + contents=[ + { + "Key": "foo", + "Size": 100, + "LastModified": "2014-01-09T20:45:49.000Z", + "StorageClass": "GLACIER", + }, + { + "Key": "bar", + "Size": 100, + "LastModified": "2014-01-09T20:45:49.000Z", + "StorageClass": "DEEP_ARCHIVE", + }, + ] + ) + ), + ], + ) stdout, stderr, rc = await run_cli( aws_cli, - ["s3", "sync", "s3://bucket/", str(tmp_path), - "--ignore-glacier-warnings"], + [ + "s3", + "sync", + "s3://bucket/", + str(tmp_path), + "--ignore-glacier-warnings", + ], cli_env(proxy), ) @@ -219,7 +309,10 @@ async def test_sync_with_delete_on_downloads(self, aws_cli, tmp_path): """sync s3->local --delete removes local files not in S3.""" local_file = tmp_path / "foo.txt" local_file.write_text("mycontent") - async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): setup_responses(server, [_empty_list_response()]) stdout, stderr, rc = await run_cli( aws_cli, @@ -234,174 +327,328 @@ async def test_sync_with_delete_on_downloads(self, aws_cli, tmp_path): async def test_request_payer(self, aws_cli, tmp_path): """sync s3->s3 --request-payer sends RequestPayer on all ops.""" - async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): - setup_responses(server, [ - _list_response(["mykey"]), - _empty_list_response(), - copy_object_response(), - ]) + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + _list_response(["mykey"]), + _empty_list_response(), + copy_object_response(), + ], + ) stdout, stderr, rc = await run_cli( aws_cli, - ["s3", "sync", "s3://sourcebucket/", "s3://mybucket", - "--request-payer"], + [ + "s3", + "sync", + "s3://sourcebucket/", + "s3://mybucket", + "--request-payer", + ], cli_env(proxy), ) assert rc == 0, stderr.decode() assert len(server.requests) == 3, format_requests(server) - assert_list_objects_v2(server.requests[0], Bucket="sourcebucket", - RequestPayer="requester") - assert_list_objects_v2(server.requests[1], Bucket="mybucket", - RequestPayer="requester") - assert_copy_object(server.requests[2], Bucket="mybucket", Key="mykey", - RequestPayer="requester") + assert_list_objects_v2( + server.requests[0], Bucket="sourcebucket", RequestPayer="requester" + ) + assert_list_objects_v2( + server.requests[1], Bucket="mybucket", RequestPayer="requester" + ) + assert_copy_object( + server.requests[2], + Bucket="mybucket", + Key="mykey", + RequestPayer="requester", + ) async def test_request_payer_with_deletes(self, aws_cli, tmp_path): """sync s3->s3 --request-payer --delete sends RequestPayer on delete.""" - async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): - setup_responses(server, [ - _empty_list_response(), - _list_response(["key-to-delete"]), - delete_response(), - ]) + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + _empty_list_response(), + _list_response(["key-to-delete"]), + delete_response(), + ], + ) stdout, stderr, rc = await run_cli( aws_cli, - ["s3", "sync", "s3://sourcebucket/", "s3://mybucket", - "--request-payer", "--delete"], + [ + "s3", + "sync", + "s3://sourcebucket/", + "s3://mybucket", + "--request-payer", + "--delete", + ], cli_env(proxy), ) assert rc == 0, stderr.decode() assert len(server.requests) == 3, format_requests(server) - assert_list_objects_v2(server.requests[0], Bucket="sourcebucket", - RequestPayer="requester") - assert_list_objects_v2(server.requests[1], Bucket="mybucket", - RequestPayer="requester") - assert_delete_object(server.requests[2], Bucket="mybucket", Key="key-to-delete", - RequestPayer="requester") + assert_list_objects_v2( + server.requests[0], Bucket="sourcebucket", RequestPayer="requester" + ) + assert_list_objects_v2( + server.requests[1], Bucket="mybucket", RequestPayer="requester" + ) + assert_delete_object( + server.requests[2], + Bucket="mybucket", + Key="key-to-delete", + RequestPayer="requester", + ) async def test_s3s3_sync_with_destination_sse_c(self, aws_cli, tmp_path): """sync s3->s3 --sse-c sends SSE-C headers on CopyObject.""" - async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): - setup_responses(server, [ - _list_response(["mykey"]), - _empty_list_response(), - copy_object_response(), - ]) + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + _list_response(["mykey"]), + _empty_list_response(), + copy_object_response(), + ], + ) stdout, stderr, rc = await run_cli( aws_cli, - ["s3", "sync", "s3://sourcebucket/", "s3://mybucket", - "--sse-c", "AES256", "--sse-c-key", "destination-key"], + [ + "s3", + "sync", + "s3://sourcebucket/", + "s3://mybucket", + "--sse-c", + "AES256", + "--sse-c-key", + "destination-key", + ], cli_env(proxy), ) assert rc == 0, stderr.decode() assert len(server.requests) == 3, format_requests(server) # SSE-C key is base64-encoded on the wire - assert_copy_object(server.requests[2], Bucket="mybucket", Key="mykey", - SSECustomerAlgorithm="AES256", - SSECustomerKey=_b64("destination-key")) + assert_copy_object( + server.requests[2], + Bucket="mybucket", + Key="mykey", + SSECustomerAlgorithm="AES256", + SSECustomerKey=_b64("destination-key"), + ) - async def test_s3s3_sync_with_different_sse_c_keys(self, aws_cli, tmp_path): + async def test_s3s3_sync_with_different_sse_c_keys( + self, aws_cli, tmp_path + ): """sync s3->s3 with both source and destination SSE-C keys.""" - async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): - setup_responses(server, [ - _list_response(["mykey"]), - _empty_list_response(), - copy_object_response(), - ]) + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + _list_response(["mykey"]), + _empty_list_response(), + copy_object_response(), + ], + ) stdout, stderr, rc = await run_cli( aws_cli, - ["s3", "sync", "s3://sourcebucket/", "s3://mybucket", - "--sse-c-copy-source", "AES256", - "--sse-c-copy-source-key", "source-key", - "--sse-c", "AES256", "--sse-c-key", "destination-key"], + [ + "s3", + "sync", + "s3://sourcebucket/", + "s3://mybucket", + "--sse-c-copy-source", + "AES256", + "--sse-c-copy-source-key", + "source-key", + "--sse-c", + "AES256", + "--sse-c-key", + "destination-key", + ], cli_env(proxy), ) assert rc == 0, stderr.decode() assert len(server.requests) == 3, format_requests(server) # SSE-C keys are base64-encoded on the wire - assert_copy_object(server.requests[2], Bucket="mybucket", Key="mykey", - SSECustomerAlgorithm="AES256", - SSECustomerKey=_b64("destination-key"), - CopySourceSSECustomerAlgorithm="AES256", - CopySourceSSECustomerKey=_b64("source-key")) + assert_copy_object( + server.requests[2], + Bucket="mybucket", + Key="mykey", + SSECustomerAlgorithm="AES256", + SSECustomerKey=_b64("destination-key"), + CopySourceSSECustomerAlgorithm="AES256", + CopySourceSSECustomerKey=_b64("source-key"), + ) - async def test_upload_with_checksum_algorithm_crc32(self, aws_cli, tmp_path): + async def test_upload_with_checksum_algorithm_crc32( + self, aws_cli, tmp_path + ): """sync local->s3 --checksum-algorithm CRC32 sends the algorithm.""" (tmp_path / "foo.txt").write_text("contents") - async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): - setup_responses(server, [_empty_list_response(), put_object_response()]) + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, [_empty_list_response(), put_object_response()] + ) stdout, stderr, rc = await run_cli( aws_cli, - ["s3", "sync", str(tmp_path), "s3://bucket/", - "--checksum-algorithm", "CRC32"], + [ + "s3", + "sync", + str(tmp_path), + "s3://bucket/", + "--checksum-algorithm", + "CRC32", + ], cli_env(proxy), ) assert rc == 0, stderr.decode() - assert_put_object(server.requests[1], Bucket="bucket", Key="foo.txt", - ChecksumAlgorithm="CRC32") + assert_put_object( + server.requests[1], + Bucket="bucket", + Key="foo.txt", + ChecksumAlgorithm="CRC32", + ) - async def test_upload_with_checksum_algorithm_sha256(self, aws_cli, tmp_path): + async def test_upload_with_checksum_algorithm_sha256( + self, aws_cli, tmp_path + ): """sync local->s3 --checksum-algorithm SHA256.""" (tmp_path / "foo.txt").write_text("contents") - async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): - setup_responses(server, [_empty_list_response(), put_object_response()]) + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, [_empty_list_response(), put_object_response()] + ) stdout, stderr, rc = await run_cli( aws_cli, - ["s3", "sync", str(tmp_path), "s3://bucket/", - "--checksum-algorithm", "SHA256"], + [ + "s3", + "sync", + str(tmp_path), + "s3://bucket/", + "--checksum-algorithm", + "SHA256", + ], cli_env(proxy), ) assert rc == 0, stderr.decode() - assert_put_object(server.requests[1], Bucket="bucket", Key="foo.txt", - ChecksumAlgorithm="SHA256") + assert_put_object( + server.requests[1], + Bucket="bucket", + Key="foo.txt", + ChecksumAlgorithm="SHA256", + ) - async def test_upload_with_checksum_algorithm_sha1(self, aws_cli, tmp_path): + async def test_upload_with_checksum_algorithm_sha1( + self, aws_cli, tmp_path + ): """sync local->s3 --checksum-algorithm SHA1.""" (tmp_path / "foo.txt").write_text("contents") - async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): - setup_responses(server, [_empty_list_response(), put_object_response()]) + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, [_empty_list_response(), put_object_response()] + ) stdout, stderr, rc = await run_cli( aws_cli, - ["s3", "sync", str(tmp_path), "s3://bucket/", - "--checksum-algorithm", "SHA1"], + [ + "s3", + "sync", + str(tmp_path), + "s3://bucket/", + "--checksum-algorithm", + "SHA1", + ], cli_env(proxy), ) assert rc == 0, stderr.decode() - assert_put_object(server.requests[1], Bucket="bucket", Key="foo.txt", - ChecksumAlgorithm="SHA1") + assert_put_object( + server.requests[1], + Bucket="bucket", + Key="foo.txt", + ChecksumAlgorithm="SHA1", + ) - async def test_download_with_checksum_mode_enabled(self, aws_cli, tmp_path): + async def test_download_with_checksum_mode_enabled( + self, aws_cli, tmp_path + ): """sync s3->local --checksum-mode ENABLED sends ChecksumMode on GetObject.""" - async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): - setup_responses(server, [ - _list_response(["foo.txt"]), - get_object_response(b"foo", **{"x-amz-checksum-crc32": "jHNlIQ=="}), - ]) + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + _list_response(["foo.txt"]), + get_object_response( + b"foo", **{"x-amz-checksum-crc32": "jHNlIQ=="} + ), + ], + ) stdout, stderr, rc = await run_cli( aws_cli, - ["s3", "sync", "s3://bucket/foo", str(tmp_path), - "--checksum-mode", "ENABLED"], + [ + "s3", + "sync", + "s3://bucket/foo", + str(tmp_path), + "--checksum-mode", + "ENABLED", + ], cli_env(proxy), ) assert rc == 0, stderr.decode() - assert_get_object(server.requests[1], Bucket="bucket", Key="foo.txt", - ChecksumMode="ENABLED") + assert_get_object( + server.requests[1], + Bucket="bucket", + Key="foo.txt", + ChecksumMode="ENABLED", + ) - async def test_sync_upload_no_overwrite_file_not_at_destination(self, aws_cli, tmp_path): + async def test_sync_upload_no_overwrite_file_not_at_destination( + self, aws_cli, tmp_path + ): """sync local->s3 --no-overwrite uploads files not at destination.""" (tmp_path / "new_file.txt").write_text("mycontent") - async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): - setup_responses(server, [ - _list_response(["file.txt"]), - put_object_response(), - ]) + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + _list_response(["file.txt"]), + put_object_response(), + ], + ) stdout, stderr, rc = await run_cli( aws_cli, ["s3", "sync", str(tmp_path), "s3://bucket", "--no-overwrite"], @@ -411,12 +658,19 @@ async def test_sync_upload_no_overwrite_file_not_at_destination(self, aws_cli, t assert rc == 0, stderr.decode() assert len(server.requests) == 2, format_requests(server) assert_list_objects_v2(server.requests[0], Bucket="bucket") - assert_put_object(server.requests[1], Bucket="bucket", Key="new_file.txt") + assert_put_object( + server.requests[1], Bucket="bucket", Key="new_file.txt" + ) - async def test_sync_upload_no_overwrite_file_exists_at_destination(self, aws_cli, tmp_path): + async def test_sync_upload_no_overwrite_file_exists_at_destination( + self, aws_cli, tmp_path + ): """sync local->s3 --no-overwrite skips files already at destination.""" (tmp_path / "new_file.txt").write_text("mycontent") - async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): setup_responses(server, [_list_response(["new_file.txt"])]) stdout, stderr, rc = await run_cli( aws_cli, @@ -428,16 +682,30 @@ async def test_sync_upload_no_overwrite_file_exists_at_destination(self, aws_cli assert len(server.requests) == 1, format_requests(server) assert_list_objects_v2(server.requests[0], Bucket="bucket") - async def test_sync_download_no_overwrite_file_not_at_destination(self, aws_cli, tmp_path): + async def test_sync_download_no_overwrite_file_not_at_destination( + self, aws_cli, tmp_path + ): """sync s3->local --no-overwrite downloads files not present locally.""" - async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): - setup_responses(server, [ - _list_response(["new_file.txt"]), - get_object_response(b"foo"), - ]) + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + _list_response(["new_file.txt"]), + get_object_response(b"foo"), + ], + ) stdout, stderr, rc = await run_cli( aws_cli, - ["s3", "sync", "s3://bucket/", str(tmp_path), "--no-overwrite"], + [ + "s3", + "sync", + "s3://bucket/", + str(tmp_path), + "--no-overwrite", + ], cli_env(proxy), ) @@ -445,48 +713,89 @@ async def test_sync_download_no_overwrite_file_not_at_destination(self, aws_cli, assert len(server.requests) == 2, format_requests(server) assert (tmp_path / "new_file.txt").exists() - async def test_sync_download_no_overwrite_file_exists_at_destination(self, aws_cli, tmp_path): + async def test_sync_download_no_overwrite_file_exists_at_destination( + self, aws_cli, tmp_path + ): """sync s3->local --no-overwrite skips files already present locally.""" (tmp_path / "file.txt").write_text("My content") - async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): setup_responses(server, [_list_response(["file.txt"])]) stdout, stderr, rc = await run_cli( aws_cli, - ["s3", "sync", "s3://bucket/", str(tmp_path), "--no-overwrite"], + [ + "s3", + "sync", + "s3://bucket/", + str(tmp_path), + "--no-overwrite", + ], cli_env(proxy), ) assert rc == 0, stderr.decode() assert len(server.requests) == 1, format_requests(server) - async def test_sync_copy_no_overwrite_file_not_at_destination(self, aws_cli, tmp_path): + async def test_sync_copy_no_overwrite_file_not_at_destination( + self, aws_cli, tmp_path + ): """sync s3->s3 --no-overwrite copies files not at destination.""" - async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): - setup_responses(server, [ - _list_response(["new_file.txt"]), - _list_response(["file1.txt"]), - copy_object_response(), - ]) + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + _list_response(["new_file.txt"]), + _list_response(["file1.txt"]), + copy_object_response(), + ], + ) stdout, stderr, rc = await run_cli( aws_cli, - ["s3", "sync", "s3://bucket/", "s3://bucket2/", "--no-overwrite"], + [ + "s3", + "sync", + "s3://bucket/", + "s3://bucket2/", + "--no-overwrite", + ], cli_env(proxy), ) assert rc == 0, stderr.decode() assert len(server.requests) == 3, format_requests(server) - assert_copy_object(server.requests[2], Bucket="bucket2", Key="new_file.txt") + assert_copy_object( + server.requests[2], Bucket="bucket2", Key="new_file.txt" + ) - async def test_sync_copy_no_overwrite_file_exists_at_destination(self, aws_cli, tmp_path): + async def test_sync_copy_no_overwrite_file_exists_at_destination( + self, aws_cli, tmp_path + ): """sync s3->s3 --no-overwrite skips files already at destination.""" - async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): - setup_responses(server, [ - _list_response(["new_file.txt"]), - _list_response(["new_file.txt", "file1.txt"]), - ]) + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + _list_response(["new_file.txt"]), + _list_response(["new_file.txt", "file1.txt"]), + ], + ) stdout, stderr, rc = await run_cli( aws_cli, - ["s3", "sync", "s3://bucket/", "s3://bucket2/", "--no-overwrite"], + [ + "s3", + "sync", + "s3://bucket/", + "s3://bucket2/", + "--no-overwrite", + ], cli_env(proxy), ) @@ -496,11 +805,17 @@ async def test_sync_copy_no_overwrite_file_exists_at_destination(self, aws_cli, async def test_with_accesspoint_arn(self, aws_cli, tmp_path): """sync s3:/// local downloads from access point.""" arn = "arn:aws:s3:us-west-2:123456789012:accesspoint/endpoint" - async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): - setup_responses(server, [ - _list_response(["mykey"]), - get_object_response(b"foo"), - ]) + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + _list_response(["mykey"]), + get_object_response(b"foo"), + ], + ) stdout, stderr, rc = await run_cli( aws_cli, ["s3", "sync", f"s3://{arn}", str(tmp_path)], @@ -515,8 +830,13 @@ async def test_with_accesspoint_arn(self, aws_cli, tmp_path): async def test_upload_sync(self, aws_cli, tmp_path): """sync local->s3 uploads new files.""" (tmp_path / "myfile").write_text("mycontent") - async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): - setup_responses(server, [_empty_list_response(), put_object_response()]) + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, [_empty_list_response(), put_object_response()] + ) stdout, stderr, rc = await run_cli( aws_cli, ["s3", "sync", str(tmp_path), "s3://bucket/"], @@ -530,11 +850,17 @@ async def test_upload_sync(self, aws_cli, tmp_path): async def test_download_sync(self, aws_cli, tmp_path): """sync s3->local downloads new files.""" - async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): - setup_responses(server, [ - _list_response(["key"]), - get_object_response(b"content"), - ]) + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + _list_response(["key"]), + get_object_response(b"content"), + ], + ) stdout, stderr, rc = await run_cli( aws_cli, ["s3", "sync", "s3://bucket/", str(tmp_path)], @@ -550,12 +876,18 @@ async def test_download_sync(self, aws_cli, tmp_path): async def test_upload_sync_with_delete(self, aws_cli, tmp_path): """sync local->s3 --delete uploads new files and deletes remote extras.""" (tmp_path / "a-file").write_text("mycontent") - async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): - setup_responses(server, [ - _list_response(["delete-this"]), - put_object_response(), - delete_response(), - ]) + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + _list_response(["delete-this"]), + put_object_response(), + delete_response(), + ], + ) stdout, stderr, rc = await run_cli( aws_cli, ["s3", "sync", str(tmp_path), "s3://bucket/", "--delete"], @@ -577,11 +909,17 @@ async def test_upload_sync_with_delete(self, aws_cli, tmp_path): async def test_download_sync_with_delete(self, aws_cli, tmp_path): """sync s3->local --delete downloads files and deletes local extras.""" (tmp_path / "delete-this").write_text("content") - async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): - setup_responses(server, [ - _list_response(["key"]), - get_object_response(b"content"), - ]) + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + _list_response(["key"]), + get_object_response(b"content"), + ], + ) stdout, stderr, rc = await run_cli( aws_cli, ["s3", "sync", "s3://bucket/", str(tmp_path), "--delete"], @@ -594,12 +932,18 @@ async def test_download_sync_with_delete(self, aws_cli, tmp_path): async def test_copy_sync(self, aws_cli, tmp_path): """sync s3->s3 copies objects.""" - async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): - setup_responses(server, [ - _list_response(["key"]), - _empty_list_response(), - copy_object_response(), - ]) + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + _list_response(["key"]), + _empty_list_response(), + copy_object_response(), + ], + ) stdout, stderr, rc = await run_cli( aws_cli, ["s3", "sync", "s3://bucket/", "s3://otherbucket/"], @@ -614,16 +958,30 @@ async def test_respects_source_region(self, aws_cli, tmp_path): """sync s3->s3 --source-region routes list to source region.""" source_host = "sourcebucket.s3.af-south-1.amazonaws.com" target_host = "bucket.s3.us-east-1.amazonaws.com" - async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): - setup_responses(server, [ - _list_response(["key"]), - _empty_list_response(), - copy_object_response(), - ]) + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + _list_response(["key"]), + _empty_list_response(), + copy_object_response(), + ], + ) stdout, stderr, rc = await run_cli( aws_cli, - ["s3", "sync", "s3://sourcebucket/", "s3://bucket/", - "--region", "us-east-1", "--source-region", "af-south-1"], + [ + "s3", + "sync", + "s3://sourcebucket/", + "s3://bucket/", + "--region", + "us-east-1", + "--source-region", + "af-south-1", + ], cli_env(proxy), ) @@ -638,12 +996,19 @@ async def test_respects_source_region(self, aws_cli, tmp_path): class TestSyncCommandWithS3Express: async def test_incompatible_with_sync_upload(self, aws_cli, tmp_path): """sync local->s3 with directory bucket is rejected.""" - async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): setup_responses(server, []) stdout, stderr, rc = await run_cli( aws_cli, - ["s3", "sync", str(tmp_path), - "s3://testdirectorybucket--usw2-az1--x-s3/"], + [ + "s3", + "sync", + str(tmp_path), + "s3://testdirectorybucket--usw2-az1--x-s3/", + ], cli_env(proxy), ) @@ -652,13 +1017,19 @@ async def test_incompatible_with_sync_upload(self, aws_cli, tmp_path): async def test_incompatible_with_sync_download(self, aws_cli, tmp_path): """sync s3->local with directory bucket is rejected.""" - async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): setup_responses(server, []) stdout, stderr, rc = await run_cli( aws_cli, - ["s3", "sync", - "s3://testdirectorybucket--usw2-az1--x-s3/", - str(tmp_path)], + [ + "s3", + "sync", + "s3://testdirectorybucket--usw2-az1--x-s3/", + str(tmp_path), + ], cli_env(proxy), ) @@ -667,12 +1038,19 @@ async def test_incompatible_with_sync_download(self, aws_cli, tmp_path): async def test_incompatible_with_sync_copy(self, aws_cli, tmp_path): """sync s3->s3 with directory bucket as destination is rejected.""" - async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): setup_responses(server, []) stdout, stderr, rc = await run_cli( aws_cli, - ["s3", "sync", "s3://bucket/", - "s3://testdirectorybucket--usw2-az1--x-s3/"], + [ + "s3", + "sync", + "s3://bucket/", + "s3://testdirectorybucket--usw2-az1--x-s3/", + ], cli_env(proxy), ) @@ -681,12 +1059,20 @@ async def test_incompatible_with_sync_copy(self, aws_cli, tmp_path): async def test_incompatible_with_sync_with_delete(self, aws_cli, tmp_path): """sync s3->s3 --delete with directory bucket is rejected.""" - async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): setup_responses(server, []) stdout, stderr, rc = await run_cli( aws_cli, - ["s3", "sync", "s3://bucket/", - "s3://testdirectorybucket--usw2-az1--x-s3/", "--delete"], + [ + "s3", + "sync", + "s3://bucket/", + "s3://testdirectorybucket--usw2-az1--x-s3/", + "--delete", + ], cli_env(proxy), ) @@ -697,6 +1083,7 @@ async def test_incompatible_with_sync_with_delete(self, aws_cli, tmp_path): def _is_case_insensitive() -> bool: """Check if the filesystem is case-insensitive.""" import tempfile + with tempfile.TemporaryDirectory() as d: upper = os.path.join(d, "A") open(upper, "w").close() @@ -712,17 +1099,36 @@ class TestSyncCaseConflict: async def test_error_with_existing_file(self, aws_cli, tmp_path): """sync s3->local --case-conflict error fails on case conflict with existing file.""" (tmp_path / "a.txt").write_text("mycontent") - async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): - setup_responses(server, [ - xml_response(list_objects_xml( - contents=[{"Key": "A.txt", "Size": 100, - "LastModified": "2023-01-01T00:00:00Z"}], - )), - ]) + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + xml_response( + list_objects_xml( + contents=[ + { + "Key": "A.txt", + "Size": 100, + "LastModified": "2023-01-01T00:00:00Z", + } + ], + ) + ), + ], + ) stdout, stderr, rc = await run_cli( aws_cli, - ["s3", "sync", "s3://bucket", str(tmp_path), - "--case-conflict", "error"], + [ + "s3", + "sync", + "s3://bucket", + str(tmp_path), + "--case-conflict", + "error", + ], cli_env(proxy), ) @@ -731,19 +1137,41 @@ async def test_error_with_existing_file(self, aws_cli, tmp_path): async def test_error_with_case_conflicts_in_s3(self, aws_cli, tmp_path): """sync s3->local --case-conflict error fails on conflicting keys in S3.""" - async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): - setup_responses(server, [ - xml_response(list_objects_xml( - contents=[ - {"Key": "A.txt", "Size": 100, "LastModified": "2023-01-01T00:00:00Z"}, - {"Key": "a.txt", "Size": 100, "LastModified": "2023-01-01T00:00:00Z"}, - ], - )), - ]) + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + xml_response( + list_objects_xml( + contents=[ + { + "Key": "A.txt", + "Size": 100, + "LastModified": "2023-01-01T00:00:00Z", + }, + { + "Key": "a.txt", + "Size": 100, + "LastModified": "2023-01-01T00:00:00Z", + }, + ], + ) + ), + ], + ) stdout, stderr, rc = await run_cli( aws_cli, - ["s3", "sync", "s3://bucket", str(tmp_path), - "--case-conflict", "error"], + [ + "s3", + "sync", + "s3://bucket", + str(tmp_path), + "--case-conflict", + "error", + ], cli_env(proxy), ) @@ -757,18 +1185,37 @@ async def test_error_with_case_conflicts_in_s3(self, aws_cli, tmp_path): async def test_warn_with_existing_file(self, aws_cli, tmp_path): """sync s3->local --case-conflict warn warns on conflict with existing file.""" (tmp_path / "a.txt").write_text("mycontent") - async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): - setup_responses(server, [ - xml_response(list_objects_xml( - contents=[{"Key": "A.txt", "Size": 100, - "LastModified": "2023-01-01T00:00:00Z"}], - )), - get_object_response(b"foo"), - ]) + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + xml_response( + list_objects_xml( + contents=[ + { + "Key": "A.txt", + "Size": 100, + "LastModified": "2023-01-01T00:00:00Z", + } + ], + ) + ), + get_object_response(b"foo"), + ], + ) stdout, stderr, rc = await run_cli( aws_cli, - ["s3", "sync", "s3://bucket", str(tmp_path), - "--case-conflict", "warn"], + [ + "s3", + "sync", + "s3://bucket", + str(tmp_path), + "--case-conflict", + "warn", + ], cli_env(proxy), ) @@ -777,21 +1224,43 @@ async def test_warn_with_existing_file(self, aws_cli, tmp_path): async def test_warn_with_case_conflicts_in_s3(self, aws_cli, tmp_path): """sync s3->local --case-conflict warn warns on conflicting keys.""" - async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): - setup_responses(server, [ - xml_response(list_objects_xml( - contents=[ - {"Key": "A.txt", "Size": 100, "LastModified": "2023-01-01T00:00:00Z"}, - {"Key": "a.txt", "Size": 100, "LastModified": "2023-01-01T00:00:00Z"}, - ], - )), - get_object_response(b"foo"), - get_object_response(b"bar"), - ]) + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + xml_response( + list_objects_xml( + contents=[ + { + "Key": "A.txt", + "Size": 100, + "LastModified": "2023-01-01T00:00:00Z", + }, + { + "Key": "a.txt", + "Size": 100, + "LastModified": "2023-01-01T00:00:00Z", + }, + ], + ) + ), + get_object_response(b"foo"), + get_object_response(b"bar"), + ], + ) stdout, stderr, rc = await run_cli( aws_cli, - ["s3", "sync", "s3://bucket", str(tmp_path), - "--case-conflict", "warn"], + [ + "s3", + "sync", + "s3://bucket", + str(tmp_path), + "--case-conflict", + "warn", + ], cli_env(proxy), ) @@ -805,17 +1274,36 @@ async def test_warn_with_case_conflicts_in_s3(self, aws_cli, tmp_path): async def test_skip_with_existing_file(self, aws_cli, tmp_path): """sync s3->local --case-conflict skip skips conflicting file.""" (tmp_path / "a.txt").write_text("mycontent") - async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): - setup_responses(server, [ - xml_response(list_objects_xml( - contents=[{"Key": "A.txt", "Size": 100, - "LastModified": "2023-01-01T00:00:00Z"}], - )), - ]) + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + xml_response( + list_objects_xml( + contents=[ + { + "Key": "A.txt", + "Size": 100, + "LastModified": "2023-01-01T00:00:00Z", + } + ], + ) + ), + ], + ) stdout, stderr, rc = await run_cli( aws_cli, - ["s3", "sync", "s3://bucket", str(tmp_path), - "--case-conflict", "skip"], + [ + "s3", + "sync", + "s3://bucket", + str(tmp_path), + "--case-conflict", + "skip", + ], cli_env(proxy), ) @@ -824,20 +1312,42 @@ async def test_skip_with_existing_file(self, aws_cli, tmp_path): async def test_skip_with_case_conflicts_in_s3(self, aws_cli, tmp_path): """sync s3->local --case-conflict skip skips conflicting keys.""" - async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): - setup_responses(server, [ - xml_response(list_objects_xml( - contents=[ - {"Key": "A.txt", "Size": 100, "LastModified": "2023-01-01T00:00:00Z"}, - {"Key": "a.txt", "Size": 100, "LastModified": "2023-01-01T00:00:00Z"}, - ], - )), - get_object_response(b"foo"), - ]) + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + xml_response( + list_objects_xml( + contents=[ + { + "Key": "A.txt", + "Size": 100, + "LastModified": "2023-01-01T00:00:00Z", + }, + { + "Key": "a.txt", + "Size": 100, + "LastModified": "2023-01-01T00:00:00Z", + }, + ], + ) + ), + get_object_response(b"foo"), + ], + ) stdout, stderr, rc = await run_cli( aws_cli, - ["s3", "sync", "s3://bucket", str(tmp_path), - "--case-conflict", "skip"], + [ + "s3", + "sync", + "s3://bucket", + str(tmp_path), + "--case-conflict", + "skip", + ], cli_env(proxy), ) @@ -851,18 +1361,37 @@ async def test_skip_with_case_conflicts_in_s3(self, aws_cli, tmp_path): async def test_ignore_with_existing_file(self, aws_cli, tmp_path): """sync s3->local --case-conflict ignore proceeds without warning.""" (tmp_path / "a.txt").write_text("mycontent") - async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): - setup_responses(server, [ - xml_response(list_objects_xml( - contents=[{"Key": "A.txt", "Size": 100, - "LastModified": "2023-01-01T00:00:00Z"}], - )), - get_object_response(b"foo"), - ]) + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + xml_response( + list_objects_xml( + contents=[ + { + "Key": "A.txt", + "Size": 100, + "LastModified": "2023-01-01T00:00:00Z", + } + ], + ) + ), + get_object_response(b"foo"), + ], + ) stdout, stderr, rc = await run_cli( aws_cli, - ["s3", "sync", "s3://bucket", str(tmp_path), - "--case-conflict", "ignore"], + [ + "s3", + "sync", + "s3://bucket", + str(tmp_path), + "--case-conflict", + "ignore", + ], cli_env(proxy), ) @@ -870,28 +1399,49 @@ async def test_ignore_with_existing_file(self, aws_cli, tmp_path): async def test_ignore_with_case_conflicts_in_s3(self, aws_cli, tmp_path): """sync s3->local --case-conflict ignore downloads all without warning.""" - async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): - setup_responses(server, [ - xml_response(list_objects_xml( - contents=[ - {"Key": "A.txt", "Size": 100, "LastModified": "2023-01-01T00:00:00Z"}, - {"Key": "a.txt", "Size": 100, "LastModified": "2023-01-01T00:00:00Z"}, - ], - )), - get_object_response(b"foo"), - get_object_response(b"bar"), - ]) + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + xml_response( + list_objects_xml( + contents=[ + { + "Key": "A.txt", + "Size": 100, + "LastModified": "2023-01-01T00:00:00Z", + }, + { + "Key": "a.txt", + "Size": 100, + "LastModified": "2023-01-01T00:00:00Z", + }, + ], + ) + ), + get_object_response(b"foo"), + get_object_response(b"bar"), + ], + ) stdout, stderr, rc = await run_cli( aws_cli, - ["s3", "sync", "s3://bucket", str(tmp_path), - "--case-conflict", "ignore"], + [ + "s3", + "sync", + "s3://bucket", + str(tmp_path), + "--case-conflict", + "ignore", + ], cli_env(proxy), ) assert rc == 0, stderr.decode() - @pytest.mark.asyncio async def test_download_url_encoded_key_from_list(aws_cli, tmp_path): """sync downloads objects whose keys contain spaces from ListObjectsV2.""" From bf2b27036940a241369d7ba5d4c3ba86b0ada488 Mon Sep 17 00:00:00 2001 From: aemous Date: Thu, 3 Sep 2026 10:14:39 -0400 Subject: [PATCH 7/7] Add 60s default test timeout. --- tests/blackbox/test_s3_config.py | 29 +++++++++++++++++++++++++++++ tests/blackbox/utils.py | 18 ++++++++++++++++-- 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/tests/blackbox/test_s3_config.py b/tests/blackbox/test_s3_config.py index b9a67684e1f1..5cbe52411bc8 100644 --- a/tests/blackbox/test_s3_config.py +++ b/tests/blackbox/test_s3_config.py @@ -330,3 +330,32 @@ async def test_use_dualstack_endpoint_false(aws_cli, aws_config, tmp_path): assert ( "dualstack" not in host ), f"Expected no dualstack in host, got {host}" + + + +@pytest.mark.asyncio +async def test_multipart_threshold_independent_of_chunksize( + aws_cli, aws_config, tmp_path +): + """File below multipart_threshold uses single PUT even when above chunksize. + + multipart_threshold and multipart_chunksize are independent settings. + A 15MB file with threshold=16MB should use single PUT regardless of + chunksize=8MB. + """ + src = tmp_path / "data.bin" + src.write_bytes(b"x" * (15 * 1024 * 1024)) + config_path = aws_config( + {"default": {"s3": "\n multipart_threshold = 16MB\n multipart_chunksize = 8MB"}} + ) + async with mock_server(on_headers_received=handle_expect_header) as (server, proxy): + setup_responses(server, [put_object_response()]) + _, stderr, rc = await run_cli( + aws_cli, + ["s3", "cp", str(src), "s3://bucket/data.bin"], + _cli_env_with_config(proxy, config_path), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + assert_put_object(server.requests[0], Bucket="bucket", Key="data.bin") diff --git a/tests/blackbox/utils.py b/tests/blackbox/utils.py index be8bb20fde73..e8a9b8299093 100644 --- a/tests/blackbox/utils.py +++ b/tests/blackbox/utils.py @@ -76,7 +76,11 @@ def get_query_params(request) -> dict[str, list[str]]: async def run_cli( - aws_cli: str, args: list[str], env: dict, stdin: bytes | None = None + aws_cli: str, + args: list[str], + env: dict, + stdin: bytes | None = None, + timeout: float = 60, ) -> tuple[bytes, bytes, int]: proc = await asyncio.create_subprocess_exec( aws_cli, @@ -86,7 +90,17 @@ async def run_cli( stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, ) - stdout, stderr = await proc.communicate(input=stdin) + try: + stdout, stderr = await asyncio.wait_for( + proc.communicate(input=stdin), timeout=timeout + ) + except asyncio.TimeoutError: + proc.kill() + await proc.wait() + raise AssertionError( + f"CLI process timed out after {timeout}s. " + f"Command: {aws_cli} {' '.join(args)}" + ) return stdout, stderr, proc.returncode