diff --git a/src/crawlee/request_loaders/_throttling_request_manager.py b/src/crawlee/request_loaders/_throttling_request_manager.py index d48c49ec92..4a62eb1b26 100644 --- a/src/crawlee/request_loaders/_throttling_request_manager.py +++ b/src/crawlee/request_loaders/_throttling_request_manager.py @@ -93,13 +93,17 @@ def __init__( locator, ensuring consistency with the crawler's storage backend. base_delay: Initial delay after the first 429 response from a domain. max_delay: Maximum delay between requests to a rate-limited domain. + + Raises: + ValueError: If an entry of `domains` is not a hostname the URL parser can read. """ self._inner: TRequestManager = inner self._service_locator = service_locator if service_locator is not None else global_service_locator self._base_delay = base_delay self._max_delay = max_delay self._request_manager_opener = request_manager_opener - self._domain_states: dict[str, _DomainState] = {d.lower(): _DomainState(domain=d.lower()) for d in domains if d} + domain_keys = [self._parse_configured_domain(d) for d in domains if d] + self._domain_states: dict[str, _DomainState] = {key: _DomainState(domain=key) for key in domain_keys} self._sub_managers: dict[str, TRequestManager] = {} self._new_work_event = asyncio.Event() """Set whenever a request is added or reclaimed. Lets `fetch_next_request` wake from a throttle @@ -354,9 +358,31 @@ def set_crawl_delay(self, url: str, delay_seconds: int) -> None: logger.debug(f'Set crawl-delay for domain "{state.domain}" to {delay_seconds}s') @staticmethod - def _extract_domain(url: str) -> str: - """Extract the domain (hostname) from a URL.""" - return URL(url).host or '' + def _normalize_domain(hostname: str) -> str: + """Bring a parsed hostname to the form domain keys are stored in, root dot and all casing gone.""" + return hostname.lower().removesuffix('.') + + @classmethod + def _parse_configured_domain(cls, domain: str) -> str: + """Turn one `domains` entry, a bare hostname or a URL, into the key its requests are looked up under.""" + try: + # A bare hostname reaches the parser, and with it IDNA and IPv6 handling, only through a synthetic URL. + host = (URL(domain) if '://' in domain else URL(f'https://{domain}')).host + except ValueError: + host = None + + if not host: + raise ValueError( + f'"{domain}" is not a valid hostname. The `domains` option takes bare hostnames such as ' + f'"example.com"; an IPv6 address has to be bracketed, as in "[::1]".' + ) + + return cls._normalize_domain(host) + + @classmethod + def _extract_domain(cls, url: str) -> str: + """Extract the domain key from a URL.""" + return cls._normalize_domain(URL(url).host or '') @staticmethod def _get_url_from_request(request: str | Request) -> str: diff --git a/tests/unit/test_throttling_request_manager.py b/tests/unit/test_throttling_request_manager.py index 0451297fff..ea39b1750a 100644 --- a/tests/unit/test_throttling_request_manager.py +++ b/tests/unit/test_throttling_request_manager.py @@ -120,6 +120,48 @@ async def test_domain_matching_is_case_insensitive( assert manager._is_domain_throttled('example.com') +@pytest.mark.parametrize( + ('configured', 'url'), + [ + pytest.param('xn--hky-ela4t.cz', 'https://háčky.cz/page', id='punycode_configured'), + pytest.param('háčky.cz', 'https://xn--hky-ela4t.cz/page', id='punycode_url'), + pytest.param('example.com', 'http://example.com./page', id='root_dot_url'), + pytest.param('example.com.', 'http://example.com/page', id='root_dot_configured'), + pytest.param('[::1]', 'http://[::1]:8080/page', id='ipv6_literal'), + pytest.param('https://example.com/products', 'https://example.com/page', id='full_url'), + ], +) +async def test_domain_matching_normalizes_spelling( + configured: str, + url: str, + inner_queue: RequestQueue, + service_locator: ServiceLocator, +) -> None: + """A configured domain and a crawled URL must land on the same key however each of them is spelled.""" + manager = ThrottlingRequestManager( + inner_queue, + domains=[configured], + request_manager_opener=RequestQueue.open, + service_locator=service_locator, + ) + + assert manager.record_domain_delay(url) is True + + +async def test_unreadable_domain_is_rejected( + inner_queue: RequestQueue, + service_locator: ServiceLocator, +) -> None: + """An entry the URL parser cannot read is rejected at construction instead of never matching anything.""" + with pytest.raises(ValueError, match='not a valid hostname'): + ThrottlingRequestManager( + inner_queue, + domains=['::1'], + request_manager_opener=RequestQueue.open, + service_locator=service_locator, + ) + + async def test_add_requests_routes_mixed_domains( manager: ThrottlingRequestManager[RequestQueue], inner_queue: RequestQueue,