diff --git a/src/crawlee/_utils/requests.py b/src/crawlee/_utils/requests.py index fa31d4621d..3fe2ee7fd9 100644 --- a/src/crawlee/_utils/requests.py +++ b/src/crawlee/_utils/requests.py @@ -17,10 +17,11 @@ def normalize_url(url: str, *, keep_url_fragment: bool = False) -> str: """Normalize a URL. This function cleans and standardizes a URL by removing leading and trailing whitespaces, - converting the scheme and netloc to lower case, stripping unwanted tracking parameters + converting the scheme and host to lower case, stripping unwanted tracking parameters (specifically those beginning with 'utm_'), sorting the remaining query parameters alphabetically, - and optionally retaining the URL fragment. The goal is to ensure that URLs that are functionally - identical but differ in trivial ways (such as parameter order or casing) are treated as the same. + and optionally retaining the URL fragment. Per RFC 3986 only the scheme and host are + case-insensitive: the path, query, and fragment keep their original casing, so case-distinct + pages are not silently deduplicated. Args: url: The URL to be normalized. @@ -29,7 +30,8 @@ def normalize_url(url: str, *, keep_url_fragment: bool = False) -> str: Returns: A string containing the normalized URL. """ - # Parse the URL + # Parse the URL. yarl already lowercases the scheme and host per RFC 3986 while + # preserving the casing of the path, query, and fragment. parsed_url = URL(url.strip()) # Remove any 'utm_' parameters @@ -44,7 +46,7 @@ def normalize_url(url: str, *, keep_url_fragment: bool = False) -> str: yarl_new_url.path.removesuffix('/'), keep_query=True, keep_fragment=keep_url_fragment ) - return str(yarl_new_url).lower() + return str(yarl_new_url) def compute_unique_key( diff --git a/tests/unit/_utils/test_requests.py b/tests/unit/_utils/test_requests.py index 8198909592..033b0bab4f 100644 --- a/tests/unit/_utils/test_requests.py +++ b/tests/unit/_utils/test_requests.py @@ -15,22 +15,37 @@ 'http://example.com/?another_key=another_value&key=value', False, ), - ('HTTPS://EXAMPLE.COM/?KEY=VALUE', 'https://example.com/?key=value', False), + ('HTTPS://EXAMPLE.COM/?KEY=VALUE', 'https://example.com/?KEY=VALUE', False), ('', '', False), ('http://example.com/#fragment', 'http://example.com/#fragment', True), ('http://example.com/#fragment', 'http://example.com', False), (' https://example.com/ ', 'https://example.com', False), ('http://example.com/?b=2&a=1', 'http://example.com/?a=1&b=2', False), + # RFC 3986: only the scheme and host are case-insensitive. Path, query, + # and fragment casing is significant and must survive normalization, so + # case-distinct pages are not silently deduplicated (#2008). + ('https://example.com/Product/ABC', 'https://example.com/Product/ABC', False), + ('https://example.com/Product/ABC/', 'https://example.com/Product/ABC', False), + ('https://example.com/?token=SeCrEt', 'https://example.com/?token=SeCrEt', False), + ( + 'https://example.com/Path?Token=SeCrEt#Frag', + 'https://example.com/Path?Token=SeCrEt#Frag', + True, + ), ], ids=[ 'remove_utm_params', 'retain_sort_non_utm_params', - 'convert_scheme_netloc_to_lowercase', + 'lowercase_scheme_and_host_only', 'handle_empty_url', 'retain_fragment', 'remove_fragment', 'trim_whitespace', 'sort_query_params', + 'preserve_path_case', + 'preserve_path_case_with_trailing_slash', + 'preserve_query_value_case', + 'preserve_path_query_fragment_case', ], ) def test_normalize_url(url: str, expected_output: str, *, keep_url_fragment: bool) -> None: @@ -38,6 +53,16 @@ def test_normalize_url(url: str, expected_output: str, *, keep_url_fragment: boo assert output == expected_output +def test_normalize_url_does_not_collapse_case_distinct_paths() -> None: + # Two URLs differing only in path casing must normalize to different URLs, + # so compute_unique_key does not silently deduplicate case-distinct pages. + upper = normalize_url('https://example.com/Product/ABC') + lower = normalize_url('https://example.com/product/abc') + assert upper != lower + assert upper == 'https://example.com/Product/ABC' + assert lower == 'https://example.com/product/abc' + + def test_compute_unique_key_basic() -> None: url = 'https://crawlee.dev' uk_get = compute_unique_key(url, method='GET')