refactor(sessions): move blocked_status_codes from Session to crawler

docs/guides/code_examples/error_handling/change_handle_error_status.py

@@ -4,7 +4,6 @@
 from crawlee import HttpHeaders
 from crawlee.crawlers import HttpCrawler, HttpCrawlingContext
 from crawlee.errors import HttpStatusCodeError
-from crawlee.sessions import SessionPool
 
 # Using a placeholder refresh token for this example
 REFRESH_TOKEN = 'PLACEHOLDER'
@@ -15,7 +14,7 @@ async def main() -> None:
     crawler = HttpCrawler(
         max_request_retries=2,
         # Only treat 403 as a blocking status code, not 401
-        session_pool=SessionPool(create_session_settings={'blocked_status_codes': [403]}),
+        blocked_status_codes=[403],
         # Don't treat 401 responses as errors
         ignore_http_error_status_codes=[UNAUTHORIZED_CODE],
     )

docs/guides/code_examples/session_management/multi_sessions_http.py

@@ -21,7 +21,6 @@ def create_session() -> Session:
             max_usage_count=999_999,
             max_age=timedelta(hours=999_999),
             max_error_score=100,
-            blocked_status_codes=[403],
         )
 
     return create_session
@@ -33,6 +32,8 @@ async def main() -> None:
         concurrency_settings=ConcurrencySettings(max_tasks_per_minute=500),
         # Requests are bound to specific sessions, no rotation needed
         max_session_rotations=0,
+        # A 403 status usually indicates we're already blocked; retire the session
+        blocked_status_codes=[403],
         session_pool=SessionPool(
             max_pool_size=10, create_session_function=create_session_function()
         ),

docs/guides/code_examples/session_management/one_session_http.py

@@ -13,6 +13,8 @@ async def main() -> None:
         concurrency_settings=ConcurrencySettings(max_tasks_per_minute=50),
         # Disable session rotation
         max_session_rotations=0,
+        # A 403 status usually indicates we're already blocked; retire the session
+        blocked_status_codes=[403],
         session_pool=SessionPool(
             # Only one session in the pool
             max_pool_size=1,
@@ -25,8 +27,6 @@ async def main() -> None:
                 # before crawlee decides the session is blocked
                 # Make sure you know how to handle these errors
                 'max_error_score': 100,
-                # 403 status usually indicates you're already blocked
-                'blocked_status_codes': [403],
             },
         ),
     )

docs/guides/code_examples/session_management/sm_standalone.py

@@ -7,7 +7,7 @@ async def main() -> None:
     # Override the default Session pool configuration.
     async with SessionPool(
         max_pool_size=100,
-        create_session_settings={'max_usage_count': 10, 'blocked_status_codes': [403]},
+        create_session_settings={'max_usage_count': 10},
     ) as session_pool:
         session = await session_pool.get_session()
 

src/crawlee/crawlers/_basic/_basic_crawler.py

@@ -17,7 +17,7 @@
 from http import HTTPStatus
 from io import StringIO
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Generic, Literal, ParamSpec, cast
+from typing import TYPE_CHECKING, Any, ClassVar, Generic, Literal, ParamSpec, cast
 from weakref import WeakKeyDictionary
 
 from cachetools import LRUCache
@@ -168,6 +168,12 @@ class _BasicCrawlerOptions(TypedDict):
     retry_on_blocked: NotRequired[bool]
     """If True, the crawler attempts to bypass bot protections automatically."""
 
+    blocked_status_codes: NotRequired[Iterable[int]]
+    """HTTP status codes that indicate the session should be retired/rotated.
+
+    The default is ``[401, 403, 429]``.
+    """
+
     concurrency_settings: NotRequired[ConcurrencySettings]
     """Settings to fine-tune concurrency levels."""
 
@@ -273,6 +279,8 @@ class BasicCrawler(Generic[TCrawlingContext, TStatisticsState]):
 
     _CRAWLEE_STATE_KEY = 'CRAWLEE_STATE'
     _request_handler_timeout_text = 'Request handler timed out after'
+    _DEFAULT_BLOCKED_STATUS_CODES: ClassVar = [401, 403, 429]
+    """Default status codes that indicate a session is blocked."""
     __next_id = 0
 
     def __init__(
@@ -292,6 +300,7 @@ def __init__(
         max_crawl_depth: int | None = None,
         use_session_pool: bool = True,
         retry_on_blocked: bool = True,
+        blocked_status_codes: Iterable[int] | None = None,
         additional_http_error_status_codes: Iterable[int] | None = None,
         ignore_http_error_status_codes: Iterable[int] | None = None,
         concurrency_settings: ConcurrencySettings | None = None,
@@ -339,6 +348,8 @@ def __init__(
                 from those requests. If not set, crawling continues without depth restrictions.
             use_session_pool: Enable the use of a session pool for managing sessions during crawling.
             retry_on_blocked: If True, the crawler attempts to bypass bot protections automatically.
+            blocked_status_codes: HTTP status codes that indicate the session should be retired.
+                Defaults to ``[401, 403, 429]``.
             additional_http_error_status_codes: Additional HTTP status codes to treat as errors,
                 triggering automatic retries when encountered.
             ignore_http_error_status_codes: HTTP status codes that are typically considered errors but should be treated
@@ -403,6 +414,7 @@ def __init__(
         self._ignore_http_error_status_codes = (
             set(ignore_http_error_status_codes) if ignore_http_error_status_codes else set()
         )
+        self._blocked_status_codes = set(blocked_status_codes or self._DEFAULT_BLOCKED_STATUS_CODES)
 
         self._http_client = http_client or ImpitHttpClient()
 
@@ -1664,10 +1676,7 @@ def _raise_for_session_blocked_status_code(
                     level=logging.WARNING,
                 )
 
-        if session is not None and session.is_blocked_status_code(
-            status_code=status_code,
-            ignore_http_error_status_codes=self._ignore_http_error_status_codes,
-        ):
+        if session is not None and status_code in (self._blocked_status_codes - self._ignore_http_error_status_codes):
             raise SessionError(f'Assuming the session is blocked based on HTTP status code {status_code}')
 
     def _check_request_collision(self, request: Request, session: Session | None) -> None:

src/crawlee/sessions/_models.py

@@ -35,7 +35,6 @@ class SessionModel(BaseModel):
     max_usage_count: Annotated[int, Field(alias='maxUsageCount')]
     error_score: Annotated[float, Field(alias='errorScore')]
     cookies: Annotated[list[CookieParam], Field(alias='cookies')]
-    blocked_status_codes: Annotated[list[int], Field(alias='blockedStatusCodes')]
 
 
 class SessionPoolModel(BaseModel):

src/crawlee/sessions/_session.py

@@ -4,7 +4,7 @@
 
 from datetime import datetime, timedelta, timezone
 from logging import getLogger
-from typing import TYPE_CHECKING, ClassVar, Literal, overload
+from typing import TYPE_CHECKING, Literal, overload
 
 from crawlee._utils.crypto import crypto_random_object_id
 from crawlee._utils.docs import docs_group
@@ -30,9 +30,6 @@ class Session:
     usage count, and expiration.
     """
 
-    _DEFAULT_BLOCKED_STATUS_CODES: ClassVar = [401, 403, 429]
-    """Default status codes that indicate a session is blocked."""
-
     def __init__(
         self,
         *,
@@ -46,7 +43,6 @@ def __init__(
         max_usage_count: int = 50,
         error_score: float = 0.0,
         cookies: SessionCookies | CookieJar | dict[str, str] | list[CookieParam] | None = None,
-        blocked_status_codes: list | None = None,
     ) -> None:
         """Initialize a new instance.
 
@@ -61,7 +57,6 @@ def __init__(
             max_usage_count: Maximum allowable uses of the session before it is considered expired.
             error_score: Current error score of the session.
             cookies: Cookies associated with the session.
-            blocked_status_codes: HTTP status codes that indicate a session should be blocked.
         """
         self._id = id or crypto_random_object_id(length=10)
         self._max_age = max_age
@@ -73,7 +68,6 @@ def __init__(
         self._max_usage_count = max_usage_count
         self._error_score = error_score
         self._cookies = SessionCookies(cookies) or SessionCookies()
-        self._blocked_status_codes = set(blocked_status_codes or self._DEFAULT_BLOCKED_STATUS_CODES)
 
     @classmethod
     def from_model(cls, model: SessionModel) -> Session:
@@ -184,7 +178,6 @@ def get_state(self, *, as_dict: bool = False) -> SessionModel | dict:
             max_usage_count=self._max_usage_count,
             error_score=self._error_score,
             cookies=self._cookies.get_cookies_as_dicts(),
-            blocked_status_codes=list(self._blocked_status_codes),
         )
         if as_dict:
             return model.model_dump()
@@ -219,21 +212,3 @@ def retire(self) -> None:
         to use `mark_bad` method.
         """
         self._error_score += self._max_error_score
-
-    def is_blocked_status_code(
-        self,
-        *,
-        status_code: int,
-        ignore_http_error_status_codes: set[int] | None = None,
-    ) -> bool:
-        """Evaluate whether a session should be retired based on the received HTTP status code.
-
-        Args:
-            status_code: The HTTP status code received from a server response.
-            ignore_http_error_status_codes: Optional status codes to allow suppression of
-            codes from `blocked_status_codes`.
-
-        Returns:
-            True if the session should be retired, False otherwise.
-        """
-        return status_code in (self._blocked_status_codes - (ignore_http_error_status_codes or set()))

tests/unit/crawlers/_basic/test_basic_crawler.py

@@ -2460,6 +2460,45 @@ async def handler(context: BasicCrawlingContext) -> None:
     assert global_event_manager.active is False
 
 
+async def test_blocked_status_codes_default() -> None:
+    """The crawler defaults to the canonical [401, 403, 429] blocked status codes."""
+    crawler = BasicCrawler()
+    assert crawler._blocked_status_codes == {401, 403, 429}
+
+    custom = BasicCrawler(blocked_status_codes=[403, 451])
+    assert custom._blocked_status_codes == {403, 451}
+
+
+async def test_blocked_status_codes_retire_session_on_matching_response() -> None:
+    """A custom `blocked_status_codes` value raises SessionError on a matching response."""
+
+    crawler = BasicCrawler(blocked_status_codes=[418])
+    session = Session(id='test_session')
+
+    # 418 is not blocked by default, but is with the custom config.
+    with pytest.raises(SessionError):
+        crawler._raise_for_session_blocked_status_code(session, 418, request_url='https://crawlee.dev/')
+
+    crawler_default = BasicCrawler()
+    with pytest.raises(SessionError):
+        crawler_default._raise_for_session_blocked_status_code(session, 403, request_url='https://crawlee.dev/')
+
+    # No session -> no check performed.
+    crawler._raise_for_session_blocked_status_code(None, 418, request_url='https://crawlee.dev/')
+
+
+async def test_blocked_status_codes_respect_ignore_http_error_codes() -> None:
+    """Codes in `ignore_http_error_status_codes` are excluded from the blocked set."""
+
+    crawler = BasicCrawler(blocked_status_codes=[401, 403], ignore_http_error_status_codes=[401])
+    session = Session(id='test_session')
+
+    # 401 is ignored -> no SessionError; 403 still blocks.
+    crawler._raise_for_session_blocked_status_code(session, 401, request_url='https://crawlee.dev/')
+    with pytest.raises(SessionError):
+        crawler._raise_for_session_blocked_status_code(session, 403, request_url='https://crawlee.dev/')
+
+
 async def test_warn_no_throttling_manager_once_on_429(caplog: pytest.LogCaptureFixture) -> None:
     """A 429 from a crawler without ThrottlingRequestManager logs a recommendation, only once per instance."""
     crawler = BasicCrawler(configure_logging=False)

tests/unit/sessions/test_models.py

@@ -24,7 +24,6 @@ def session_direct() -> SessionModel:
         max_usage_count=10,
         error_score=0.0,
         cookies=[CookieParam({'name': 'cookie_key', 'value': 'cookie_value'})],
-        blocked_status_codes=[401, 403, 429],
     )
 
 
@@ -42,7 +41,6 @@ def session_args_camel() -> dict:
         'maxUsageCount': 10,
         'errorScore': 0.0,
         'cookies': [CookieParam({'name': 'cookie_key', 'value': 'cookie_value'})],
-        'blockedStatusCodes': [401, 403, 429],
     }
 
 
@@ -60,7 +58,6 @@ def session_args_snake() -> dict:
         'max_usage_count': 10,
         'error_score': 0.0,
         'cookies': [CookieParam({'name': 'cookie_key', 'value': 'cookie_value'})],
-        'blocked_status_codes': [401, 403, 429],
     }
 
 

tests/unit/sessions/test_session.py

@@ -21,7 +21,6 @@ def session() -> Session:
         max_usage_count=10,
         error_score=0.0,
         cookies={'cookie_key': 'cookie_value'},
-        blocked_status_codes=[401, 403, 429],
     )
 
 
@@ -105,20 +104,6 @@ def test_mark_bad_at_usage_limit_no_double_increment() -> None:
     assert not session.is_usable
 
 
-def test_retire_on_blocked_status_code(session: Session) -> None:
-    """Test retiring the session based on specific HTTP status codes."""
-    status_code = 403
-    result = session.is_blocked_status_code(status_code=status_code)
-    assert result is True
-
-
-def test_not_retire_on_not_block_status_code(session: Session) -> None:
-    """Test that the session is not retired on a non-blocked status code."""
-    status_code = 200
-    result = session.is_blocked_status_code(status_code=status_code)
-    assert result is False
-
-
 def test_session_expiration() -> None:
     """Test the expiration logic of the session."""
     session = Session(created_at=datetime.now(timezone.utc) - timedelta(hours=1))