From 1acd35f846738afd38efd42319a0f6d3bda971fc Mon Sep 17 00:00:00 2001 From: "Aryan Singh K." <70511529+aryansk@users.noreply.github.com> Date: Sat, 15 Aug 2026 10:55:28 +0530 Subject: [PATCH] refactor(sessions): move blocked_status_codes from Session to crawler Deciding whether a response status means "blocked" is the crawler's concern, not the session's - the crawler already owns additional_http_error_status_codes and ignore_http_error_status_codes. Move blocked status code handling out of Session/SessionPool and onto BasicCrawler, matching the JS implementation (apify/crawlee#3496). - BasicCrawler gains a `blocked_status_codes` option (default [401, 403, 429]); the blocked check in _raise_for_session_blocked_status_code now consults the crawler's set minus ignore_http_error_status_codes instead of Session - Session drops the blocked_status_codes parameter, the is_blocked_status_code method, and the model field; get_state no longer serializes it - docs: session examples configure blocked_status_codes on the crawler - tests: session/model tests updated; new crawler tests cover the default set, custom values, and ignore-code interaction Generated with Codebuff Co-Authored-By: Codebuff --- .../change_handle_error_status.py | 3 +- .../session_management/multi_sessions_http.py | 3 +- .../session_management/one_session_http.py | 4 +- .../session_management/sm_standalone.py | 2 +- src/crawlee/crawlers/_basic/_basic_crawler.py | 19 ++++++--- src/crawlee/sessions/_models.py | 1 - src/crawlee/sessions/_session.py | 27 +------------ .../crawlers/_basic/test_basic_crawler.py | 39 +++++++++++++++++++ tests/unit/sessions/test_models.py | 3 -- tests/unit/sessions/test_session.py | 15 ------- 10 files changed, 60 insertions(+), 56 deletions(-) diff --git a/docs/guides/code_examples/error_handling/change_handle_error_status.py b/docs/guides/code_examples/error_handling/change_handle_error_status.py index 55bf5a0e61..0902ea8fb8 100644 --- a/docs/guides/code_examples/error_handling/change_handle_error_status.py +++ b/docs/guides/code_examples/error_handling/change_handle_error_status.py @@ -4,7 +4,6 @@ from crawlee import HttpHeaders from crawlee.crawlers import HttpCrawler, HttpCrawlingContext from crawlee.errors import HttpStatusCodeError -from crawlee.sessions import SessionPool # Using a placeholder refresh token for this example REFRESH_TOKEN = 'PLACEHOLDER' @@ -15,7 +14,7 @@ async def main() -> None: crawler = HttpCrawler( max_request_retries=2, # Only treat 403 as a blocking status code, not 401 - session_pool=SessionPool(create_session_settings={'blocked_status_codes': [403]}), + blocked_status_codes=[403], # Don't treat 401 responses as errors ignore_http_error_status_codes=[UNAUTHORIZED_CODE], ) diff --git a/docs/guides/code_examples/session_management/multi_sessions_http.py b/docs/guides/code_examples/session_management/multi_sessions_http.py index 0bd4a88beb..3bc6fff569 100644 --- a/docs/guides/code_examples/session_management/multi_sessions_http.py +++ b/docs/guides/code_examples/session_management/multi_sessions_http.py @@ -21,7 +21,6 @@ def create_session() -> Session: max_usage_count=999_999, max_age=timedelta(hours=999_999), max_error_score=100, - blocked_status_codes=[403], ) return create_session @@ -33,6 +32,8 @@ async def main() -> None: concurrency_settings=ConcurrencySettings(max_tasks_per_minute=500), # Requests are bound to specific sessions, no rotation needed max_session_rotations=0, + # A 403 status usually indicates we're already blocked; retire the session + blocked_status_codes=[403], session_pool=SessionPool( max_pool_size=10, create_session_function=create_session_function() ), diff --git a/docs/guides/code_examples/session_management/one_session_http.py b/docs/guides/code_examples/session_management/one_session_http.py index 28cec44b63..0b48b4f6ab 100644 --- a/docs/guides/code_examples/session_management/one_session_http.py +++ b/docs/guides/code_examples/session_management/one_session_http.py @@ -13,6 +13,8 @@ async def main() -> None: concurrency_settings=ConcurrencySettings(max_tasks_per_minute=50), # Disable session rotation max_session_rotations=0, + # A 403 status usually indicates we're already blocked; retire the session + blocked_status_codes=[403], session_pool=SessionPool( # Only one session in the pool max_pool_size=1, @@ -25,8 +27,6 @@ async def main() -> None: # before crawlee decides the session is blocked # Make sure you know how to handle these errors 'max_error_score': 100, - # 403 status usually indicates you're already blocked - 'blocked_status_codes': [403], }, ), ) diff --git a/docs/guides/code_examples/session_management/sm_standalone.py b/docs/guides/code_examples/session_management/sm_standalone.py index 32989dc7e0..a15e1e4f59 100644 --- a/docs/guides/code_examples/session_management/sm_standalone.py +++ b/docs/guides/code_examples/session_management/sm_standalone.py @@ -7,7 +7,7 @@ async def main() -> None: # Override the default Session pool configuration. async with SessionPool( max_pool_size=100, - create_session_settings={'max_usage_count': 10, 'blocked_status_codes': [403]}, + create_session_settings={'max_usage_count': 10}, ) as session_pool: session = await session_pool.get_session() diff --git a/src/crawlee/crawlers/_basic/_basic_crawler.py b/src/crawlee/crawlers/_basic/_basic_crawler.py index 96ff205350..659a379b74 100644 --- a/src/crawlee/crawlers/_basic/_basic_crawler.py +++ b/src/crawlee/crawlers/_basic/_basic_crawler.py @@ -17,7 +17,7 @@ from http import HTTPStatus from io import StringIO from pathlib import Path -from typing import TYPE_CHECKING, Any, Generic, Literal, ParamSpec, cast +from typing import TYPE_CHECKING, Any, ClassVar, Generic, Literal, ParamSpec, cast from weakref import WeakKeyDictionary from cachetools import LRUCache @@ -168,6 +168,12 @@ class _BasicCrawlerOptions(TypedDict): retry_on_blocked: NotRequired[bool] """If True, the crawler attempts to bypass bot protections automatically.""" + blocked_status_codes: NotRequired[Iterable[int]] + """HTTP status codes that indicate the session should be retired/rotated. + + The default is ``[401, 403, 429]``. + """ + concurrency_settings: NotRequired[ConcurrencySettings] """Settings to fine-tune concurrency levels.""" @@ -273,6 +279,8 @@ class BasicCrawler(Generic[TCrawlingContext, TStatisticsState]): _CRAWLEE_STATE_KEY = 'CRAWLEE_STATE' _request_handler_timeout_text = 'Request handler timed out after' + _DEFAULT_BLOCKED_STATUS_CODES: ClassVar = [401, 403, 429] + """Default status codes that indicate a session is blocked.""" __next_id = 0 def __init__( @@ -292,6 +300,7 @@ def __init__( max_crawl_depth: int | None = None, use_session_pool: bool = True, retry_on_blocked: bool = True, + blocked_status_codes: Iterable[int] | None = None, additional_http_error_status_codes: Iterable[int] | None = None, ignore_http_error_status_codes: Iterable[int] | None = None, concurrency_settings: ConcurrencySettings | None = None, @@ -339,6 +348,8 @@ def __init__( from those requests. If not set, crawling continues without depth restrictions. use_session_pool: Enable the use of a session pool for managing sessions during crawling. retry_on_blocked: If True, the crawler attempts to bypass bot protections automatically. + blocked_status_codes: HTTP status codes that indicate the session should be retired. + Defaults to ``[401, 403, 429]``. additional_http_error_status_codes: Additional HTTP status codes to treat as errors, triggering automatic retries when encountered. ignore_http_error_status_codes: HTTP status codes that are typically considered errors but should be treated @@ -403,6 +414,7 @@ def __init__( self._ignore_http_error_status_codes = ( set(ignore_http_error_status_codes) if ignore_http_error_status_codes else set() ) + self._blocked_status_codes = set(blocked_status_codes or self._DEFAULT_BLOCKED_STATUS_CODES) self._http_client = http_client or ImpitHttpClient() @@ -1664,10 +1676,7 @@ def _raise_for_session_blocked_status_code( level=logging.WARNING, ) - if session is not None and session.is_blocked_status_code( - status_code=status_code, - ignore_http_error_status_codes=self._ignore_http_error_status_codes, - ): + if session is not None and status_code in (self._blocked_status_codes - self._ignore_http_error_status_codes): raise SessionError(f'Assuming the session is blocked based on HTTP status code {status_code}') def _check_request_collision(self, request: Request, session: Session | None) -> None: diff --git a/src/crawlee/sessions/_models.py b/src/crawlee/sessions/_models.py index fa0b64042c..aa54d5ed37 100644 --- a/src/crawlee/sessions/_models.py +++ b/src/crawlee/sessions/_models.py @@ -35,7 +35,6 @@ class SessionModel(BaseModel): max_usage_count: Annotated[int, Field(alias='maxUsageCount')] error_score: Annotated[float, Field(alias='errorScore')] cookies: Annotated[list[CookieParam], Field(alias='cookies')] - blocked_status_codes: Annotated[list[int], Field(alias='blockedStatusCodes')] class SessionPoolModel(BaseModel): diff --git a/src/crawlee/sessions/_session.py b/src/crawlee/sessions/_session.py index 4ff3ee7efa..c91f44111c 100644 --- a/src/crawlee/sessions/_session.py +++ b/src/crawlee/sessions/_session.py @@ -4,7 +4,7 @@ from datetime import datetime, timedelta, timezone from logging import getLogger -from typing import TYPE_CHECKING, ClassVar, Literal, overload +from typing import TYPE_CHECKING, Literal, overload from crawlee._utils.crypto import crypto_random_object_id from crawlee._utils.docs import docs_group @@ -30,9 +30,6 @@ class Session: usage count, and expiration. """ - _DEFAULT_BLOCKED_STATUS_CODES: ClassVar = [401, 403, 429] - """Default status codes that indicate a session is blocked.""" - def __init__( self, *, @@ -46,7 +43,6 @@ def __init__( max_usage_count: int = 50, error_score: float = 0.0, cookies: SessionCookies | CookieJar | dict[str, str] | list[CookieParam] | None = None, - blocked_status_codes: list | None = None, ) -> None: """Initialize a new instance. @@ -61,7 +57,6 @@ def __init__( max_usage_count: Maximum allowable uses of the session before it is considered expired. error_score: Current error score of the session. cookies: Cookies associated with the session. - blocked_status_codes: HTTP status codes that indicate a session should be blocked. """ self._id = id or crypto_random_object_id(length=10) self._max_age = max_age @@ -73,7 +68,6 @@ def __init__( self._max_usage_count = max_usage_count self._error_score = error_score self._cookies = SessionCookies(cookies) or SessionCookies() - self._blocked_status_codes = set(blocked_status_codes or self._DEFAULT_BLOCKED_STATUS_CODES) @classmethod def from_model(cls, model: SessionModel) -> Session: @@ -184,7 +178,6 @@ def get_state(self, *, as_dict: bool = False) -> SessionModel | dict: max_usage_count=self._max_usage_count, error_score=self._error_score, cookies=self._cookies.get_cookies_as_dicts(), - blocked_status_codes=list(self._blocked_status_codes), ) if as_dict: return model.model_dump() @@ -219,21 +212,3 @@ def retire(self) -> None: to use `mark_bad` method. """ self._error_score += self._max_error_score - - def is_blocked_status_code( - self, - *, - status_code: int, - ignore_http_error_status_codes: set[int] | None = None, - ) -> bool: - """Evaluate whether a session should be retired based on the received HTTP status code. - - Args: - status_code: The HTTP status code received from a server response. - ignore_http_error_status_codes: Optional status codes to allow suppression of - codes from `blocked_status_codes`. - - Returns: - True if the session should be retired, False otherwise. - """ - return status_code in (self._blocked_status_codes - (ignore_http_error_status_codes or set())) diff --git a/tests/unit/crawlers/_basic/test_basic_crawler.py b/tests/unit/crawlers/_basic/test_basic_crawler.py index 56ba257e86..bc2625d54f 100644 --- a/tests/unit/crawlers/_basic/test_basic_crawler.py +++ b/tests/unit/crawlers/_basic/test_basic_crawler.py @@ -2460,6 +2460,45 @@ async def handler(context: BasicCrawlingContext) -> None: assert global_event_manager.active is False +async def test_blocked_status_codes_default() -> None: + """The crawler defaults to the canonical [401, 403, 429] blocked status codes.""" + crawler = BasicCrawler() + assert crawler._blocked_status_codes == {401, 403, 429} + + custom = BasicCrawler(blocked_status_codes=[403, 451]) + assert custom._blocked_status_codes == {403, 451} + + +async def test_blocked_status_codes_retire_session_on_matching_response() -> None: + """A custom `blocked_status_codes` value raises SessionError on a matching response.""" + + crawler = BasicCrawler(blocked_status_codes=[418]) + session = Session(id='test_session') + + # 418 is not blocked by default, but is with the custom config. + with pytest.raises(SessionError): + crawler._raise_for_session_blocked_status_code(session, 418, request_url='https://crawlee.dev/') + + crawler_default = BasicCrawler() + with pytest.raises(SessionError): + crawler_default._raise_for_session_blocked_status_code(session, 403, request_url='https://crawlee.dev/') + + # No session -> no check performed. + crawler._raise_for_session_blocked_status_code(None, 418, request_url='https://crawlee.dev/') + + +async def test_blocked_status_codes_respect_ignore_http_error_codes() -> None: + """Codes in `ignore_http_error_status_codes` are excluded from the blocked set.""" + + crawler = BasicCrawler(blocked_status_codes=[401, 403], ignore_http_error_status_codes=[401]) + session = Session(id='test_session') + + # 401 is ignored -> no SessionError; 403 still blocks. + crawler._raise_for_session_blocked_status_code(session, 401, request_url='https://crawlee.dev/') + with pytest.raises(SessionError): + crawler._raise_for_session_blocked_status_code(session, 403, request_url='https://crawlee.dev/') + + async def test_warn_no_throttling_manager_once_on_429(caplog: pytest.LogCaptureFixture) -> None: """A 429 from a crawler without ThrottlingRequestManager logs a recommendation, only once per instance.""" crawler = BasicCrawler(configure_logging=False) diff --git a/tests/unit/sessions/test_models.py b/tests/unit/sessions/test_models.py index fee469c475..03bd7d262a 100644 --- a/tests/unit/sessions/test_models.py +++ b/tests/unit/sessions/test_models.py @@ -24,7 +24,6 @@ def session_direct() -> SessionModel: max_usage_count=10, error_score=0.0, cookies=[CookieParam({'name': 'cookie_key', 'value': 'cookie_value'})], - blocked_status_codes=[401, 403, 429], ) @@ -42,7 +41,6 @@ def session_args_camel() -> dict: 'maxUsageCount': 10, 'errorScore': 0.0, 'cookies': [CookieParam({'name': 'cookie_key', 'value': 'cookie_value'})], - 'blockedStatusCodes': [401, 403, 429], } @@ -60,7 +58,6 @@ def session_args_snake() -> dict: 'max_usage_count': 10, 'error_score': 0.0, 'cookies': [CookieParam({'name': 'cookie_key', 'value': 'cookie_value'})], - 'blocked_status_codes': [401, 403, 429], } diff --git a/tests/unit/sessions/test_session.py b/tests/unit/sessions/test_session.py index 5d1ca29383..d2a453d793 100644 --- a/tests/unit/sessions/test_session.py +++ b/tests/unit/sessions/test_session.py @@ -21,7 +21,6 @@ def session() -> Session: max_usage_count=10, error_score=0.0, cookies={'cookie_key': 'cookie_value'}, - blocked_status_codes=[401, 403, 429], ) @@ -105,20 +104,6 @@ def test_mark_bad_at_usage_limit_no_double_increment() -> None: assert not session.is_usable -def test_retire_on_blocked_status_code(session: Session) -> None: - """Test retiring the session based on specific HTTP status codes.""" - status_code = 403 - result = session.is_blocked_status_code(status_code=status_code) - assert result is True - - -def test_not_retire_on_not_block_status_code(session: Session) -> None: - """Test that the session is not retired on a non-blocked status code.""" - status_code = 200 - result = session.is_blocked_status_code(status_code=status_code) - assert result is False - - def test_session_expiration() -> None: """Test the expiration logic of the session.""" session = Session(created_at=datetime.now(timezone.utc) - timedelta(hours=1))