Skip to content

Commit 91c3d7e

Browse files
committed
fix(search): read truncation from SMW's query-continue-offset
- replace the count-versus-limit check, blind to $smwgQMaxLimit - stop reporting a complete set of exactly 'limit' results as truncated - add SearchParam.return_meta yielding SemanticSearchResult per query - report query, titles, count, truncated and next_offset to callers - leave return_json untouched, its raw dict already holds the offset
1 parent 9351f6b commit 91c3d7e

3 files changed

Lines changed: 258 additions & 28 deletions

File tree

‎src/osw/wiki_tools.py‎

Lines changed: 58 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,26 @@ def create_site_object(
137137
return site
138138

139139

140+
class SemanticSearchResult(OswBaseModel):
141+
"""Result of a single semantic query, including whether the wiki truncated it"""
142+
143+
query: str
144+
"""the query as sent to the wiki, including any limit appended by
145+
semantic_search"""
146+
titles: List[str]
147+
"""the page-title fulltext strings of the results that exist"""
148+
count: int
149+
"""the number of results this response carried, before dropping
150+
non-existing pages, so it can be larger than len(titles). It is not the
151+
total number of matching pages on the wiki, which SMW does not report"""
152+
truncated: bool
153+
"""True if the wiki reported further results beyond those returned"""
154+
next_offset: Optional[int] = None
155+
"""the absolute offset at which the remaining results start, to be passed
156+
back as '|offset='. None for a complete result set. It says where to
157+
continue, not how many results remain"""
158+
159+
140160
class SearchParam(OswBaseModel):
141161
"""Search parameters for semantic and prefix search"""
142162

@@ -148,6 +168,11 @@ class SearchParam(OswBaseModel):
148168
Ignored by semantic_search for a query that sets 'limit=' itself, since SMW
149169
honours the last limit in the query string"""
150170
return_json: Optional[bool] = False
171+
return_meta: Optional[bool] = False
172+
"""If True, semantic_search returns one SemanticSearchResult per query instead
173+
of a flat list of titles, so that a caller can report a truncated result set.
174+
Ignored when return_json is True, since the raw wiki response already carries
175+
the truncation signal"""
151176

152177
def __init__(self, **data):
153178
super().__init__(**data)
@@ -286,7 +311,7 @@ def get_query_limit(query: str) -> Optional[int]:
286311

287312
def semantic_search(
288313
site: mwclient.client.Site, query: Union[str, List[str], SearchParam]
289-
) -> Union[List[str], List[dict]]:
314+
) -> Union[List[str], List[dict], List[SemanticSearchResult]]:
290315
"""Semantic query
291316
292317
Parameters
@@ -301,9 +326,12 @@ def semantic_search(
301326
Returns
302327
-------
303328
result:
304-
With ``return_json=False`` (default): a flat list of page-title fulltext
305-
strings. With ``return_json=True``: a list of raw SMW ``ask`` result dicts,
306-
one per query (always a list, even for a single query).
329+
With ``return_json=False`` and ``return_meta=False`` (default): a flat
330+
list of page-title fulltext strings. With ``return_json=True``: a list of
331+
raw SMW ``ask`` result dicts, one per query (always a list, even for a
332+
single query). With ``return_meta=True``: a list of SemanticSearchResult,
333+
one per query, which reports whether the wiki truncated the result set.
334+
``return_json`` takes precedence if both are set.
307335
"""
308336
if not isinstance(query, SearchParam):
309337
query = SearchParam(query=query)
@@ -326,15 +354,21 @@ def semantic_search_(single_query):
326354
_logger.debug(f"Query '{single_query}' returned no results")
327355
else:
328356
_logger.debug(f"Query '{single_query}' returned {n} results")
329-
# No limit in force, or 'limit=0' asking for no results at all as a
330-
# count format does, means the result count says nothing about
331-
# truncation
332-
if limit and n >= limit:
357+
# SMW reports an incomplete result set with a top-level
358+
# 'query-continue-offset' holding the offset the remainder starts at,
359+
# and omits the key for a complete one. That replaces the earlier
360+
# comparison of the result count against the limit, which was wrong in
361+
# both directions: it could not see the wiki's own '$smwgQMaxLimit'
362+
# cap, and it reported a complete set of exactly 'limit' results as
363+
# truncated
364+
next_offset = result.get("query-continue-offset")
365+
truncated = next_offset is not None
366+
if truncated:
333367
_logger.warning(
334-
f"Query '{single_query}' returned {n} results, which meets the "
335-
f"requested limit of {limit}. Results are truncated - raise "
336-
f"the limit or page through with '|offset=' to retrieve the "
337-
f"remainder."
368+
f"Query '{single_query}' returned {n} results and the wiki "
369+
f"reports further ones. Results are truncated - raise the "
370+
f"limit or page through with '|offset={next_offset}' to "
371+
f"retrieve the remainder."
338372
)
339373
if query.return_json:
340374
return result
@@ -355,6 +389,14 @@ def semantic_search_(single_query):
355389
f"Query '{single_query}': {dropped} of {n} results were dropped "
356390
f"because the wiki reported them as non-existing pages."
357391
)
392+
if query.return_meta:
393+
return SemanticSearchResult(
394+
query=single_query,
395+
titles=page_list,
396+
count=n,
397+
truncated=truncated,
398+
next_offset=next_offset,
399+
)
358400
return page_list
359401

360402
if query.parallel:
@@ -364,10 +406,10 @@ def semantic_search_(single_query):
364406
else:
365407
query_results = [semantic_search_(single_query=sq) for sq in query.query]
366408

367-
if query.return_json:
368-
# Each entry of query_results is the raw SMW result dict for one query.
369-
# Do not flatten dicts; always return the list of result dicts (one per
370-
# query), even when only a single query was passed.
409+
if query.return_json or query.return_meta:
410+
# Each entry of query_results is the raw SMW result dict, or the
411+
# SemanticSearchResult, for one query. Do not flatten those; always
412+
# return one entry per query, even when only a single query was passed.
371413
return query_results
372414

373415
return [item for sublist in query_results for item in sublist]

‎src/osw/wtsite.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -638,7 +638,9 @@ def semantic_search(self, query: Union[str, SearchParam]):
638638
639639
Returns
640640
-------
641-
A list of page titles
641+
A list of page titles, or, if the SearchParam sets return_json or
642+
return_meta, one raw result dict or SemanticSearchResult per query.
643+
See wiki_tools.semantic_search for details.
642644
"""
643645
return wt.semantic_search(self._site, query)
644646

0 commit comments

Comments
 (0)