diff --git a/src/crawlee/_utils/sitemap.py b/src/crawlee/_utils/sitemap.py index d110f0225c..d34cd04581 100644 --- a/src/crawlee/_utils/sitemap.py +++ b/src/crawlee/_utils/sitemap.py @@ -496,8 +496,17 @@ async def load( ) @classmethod - async def from_xml_string(cls, content: str) -> Sitemap: - return await cls.parse([SitemapSource(type='raw', content=content)]) + async def from_xml_string( + cls, + content: str, + *, + sitemap_url: str | None = None, + parse_sitemap_options: ParseSitemapOptions | None = None, + ) -> Sitemap: + source: SitemapSource = {'type': 'raw', 'content': content} + if sitemap_url is not None: + source['url'] = sitemap_url + return await cls.parse([source], parse_sitemap_options=parse_sitemap_options) @classmethod async def parse( diff --git a/tests/unit/_utils/test_sitemap.py b/tests/unit/_utils/test_sitemap.py index e1030844ab..5dcc0fa6ab 100644 --- a/tests/unit/_utils/test_sitemap.py +++ b/tests/unit/_utils/test_sitemap.py @@ -321,6 +321,37 @@ async def test_sitemap_from_string() -> None: assert set(sitemap.urls) == get_basic_results() +async def test_sitemap_from_string_keeps_same_host_with_sitemap_url() -> None: + """URLs on the sitemap's own host survive the default `same-hostname` filter.""" + sitemap = await Sitemap.from_xml_string( + get_basic_sitemap(), + sitemap_url=f'{DEFAULT_URL}sitemap.xml', + ) + + assert set(sitemap.urls) == get_basic_results() + + +async def test_sitemap_from_string_filters_cross_host_with_sitemap_url() -> None: + """`from_xml_string` opts into host filtering when `sitemap_url` is given.""" + sitemap = await Sitemap.from_xml_string( + get_basic_sitemap(url='https://other.com/'), + sitemap_url=f'{DEFAULT_URL}sitemap.xml', + ) + + assert sitemap.urls == [] + + +async def test_sitemap_from_string_allows_cross_host_with_strategy_all() -> None: + """`enqueue_strategy='all'` disables host filtering for raw string sitemaps too.""" + sitemap = await Sitemap.from_xml_string( + get_basic_sitemap(url='https://other.com/'), + sitemap_url=f'{DEFAULT_URL}sitemap.xml', + parse_sitemap_options={'enqueue_strategy': 'all'}, + ) + + assert set(sitemap.urls) == get_basic_results('https://other.com/') + + async def test_malformed_sitemap_keeps_urls() -> None: """A parse error must not discard the URLs collected before it.""" malformed = (