Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 11 additions & 2 deletions src/crawlee/_utils/sitemap.py
Original file line number Diff line number Diff line change
Expand Up @@ -496,8 +496,17 @@ async def load(
)

@classmethod
async def from_xml_string(cls, content: str) -> Sitemap:
return await cls.parse([SitemapSource(type='raw', content=content)])
async def from_xml_string(
cls,
content: str,
*,
sitemap_url: str | None = None,
parse_sitemap_options: ParseSitemapOptions | None = None,
) -> Sitemap:
source: SitemapSource = {'type': 'raw', 'content': content}
if sitemap_url is not None:
source['url'] = sitemap_url
return await cls.parse([source], parse_sitemap_options=parse_sitemap_options)

@classmethod
async def parse(
Expand Down
31 changes: 31 additions & 0 deletions tests/unit/_utils/test_sitemap.py
Original file line number Diff line number Diff line change
Expand Up @@ -321,6 +321,37 @@ async def test_sitemap_from_string() -> None:
assert set(sitemap.urls) == get_basic_results()


async def test_sitemap_from_string_keeps_same_host_with_sitemap_url() -> None:
"""URLs on the sitemap's own host survive the default `same-hostname` filter."""
sitemap = await Sitemap.from_xml_string(
get_basic_sitemap(),
sitemap_url=f'{DEFAULT_URL}sitemap.xml',
)

assert set(sitemap.urls) == get_basic_results()


async def test_sitemap_from_string_filters_cross_host_with_sitemap_url() -> None:
"""`from_xml_string` opts into host filtering when `sitemap_url` is given."""
sitemap = await Sitemap.from_xml_string(
get_basic_sitemap(url='https://other.com/'),
sitemap_url=f'{DEFAULT_URL}sitemap.xml',
)

assert sitemap.urls == []


async def test_sitemap_from_string_allows_cross_host_with_strategy_all() -> None:
"""`enqueue_strategy='all'` disables host filtering for raw string sitemaps too."""
sitemap = await Sitemap.from_xml_string(
get_basic_sitemap(url='https://other.com/'),
sitemap_url=f'{DEFAULT_URL}sitemap.xml',
parse_sitemap_options={'enqueue_strategy': 'all'},
)

assert set(sitemap.urls) == get_basic_results('https://other.com/')


async def test_malformed_sitemap_keeps_urls() -> None:
"""A parse error must not discard the URLs collected before it."""
malformed = (
Expand Down