@@ -137,6 +137,26 @@ def create_site_object(
137137 return site
138138
139139
140+ class SemanticSearchResult (OswBaseModel ):
141+ """Result of a single semantic query, including whether the wiki truncated it"""
142+
143+ query : str
144+ """the query as sent to the wiki, including any limit appended by
145+ semantic_search"""
146+ titles : List [str ]
147+ """the page-title fulltext strings of the results that exist"""
148+ count : int
149+ """the number of results this response carried, before dropping
150+ non-existing pages, so it can be larger than len(titles). It is not the
151+ total number of matching pages on the wiki, which SMW does not report"""
152+ truncated : bool
153+ """True if the wiki reported further results beyond those returned"""
154+ next_offset : Optional [int ] = None
155+ """the absolute offset at which the remaining results start, to be passed
156+ back as '|offset='. None for a complete result set. It says where to
157+ continue, not how many results remain"""
158+
159+
140160class SearchParam (OswBaseModel ):
141161 """Search parameters for semantic and prefix search"""
142162
@@ -148,6 +168,11 @@ class SearchParam(OswBaseModel):
148168 Ignored by semantic_search for a query that sets 'limit=' itself, since SMW
149169 honours the last limit in the query string"""
150170 return_json : Optional [bool ] = False
171+ return_meta : Optional [bool ] = False
172+ """If True, semantic_search returns one SemanticSearchResult per query instead
173+ of a flat list of titles, so that a caller can report a truncated result set.
174+ Ignored when return_json is True, since the raw wiki response already carries
175+ the truncation signal"""
151176
152177 def __init__ (self , ** data ):
153178 super ().__init__ (** data )
@@ -286,7 +311,7 @@ def get_query_limit(query: str) -> Optional[int]:
286311
287312def semantic_search (
288313 site : mwclient .client .Site , query : Union [str , List [str ], SearchParam ]
289- ) -> Union [List [str ], List [dict ]]:
314+ ) -> Union [List [str ], List [dict ], List [ SemanticSearchResult ] ]:
290315 """Semantic query
291316
292317 Parameters
@@ -301,9 +326,12 @@ def semantic_search(
301326 Returns
302327 -------
303328 result:
304- With ``return_json=False`` (default): a flat list of page-title fulltext
305- strings. With ``return_json=True``: a list of raw SMW ``ask`` result dicts,
306- one per query (always a list, even for a single query).
329+ With ``return_json=False`` and ``return_meta=False`` (default): a flat
330+ list of page-title fulltext strings. With ``return_json=True``: a list of
331+ raw SMW ``ask`` result dicts, one per query (always a list, even for a
332+ single query). With ``return_meta=True``: a list of SemanticSearchResult,
333+ one per query, which reports whether the wiki truncated the result set.
334+ ``return_json`` takes precedence if both are set.
307335 """
308336 if not isinstance (query , SearchParam ):
309337 query = SearchParam (query = query )
@@ -326,15 +354,21 @@ def semantic_search_(single_query):
326354 _logger .debug (f"Query '{ single_query } ' returned no results" )
327355 else :
328356 _logger .debug (f"Query '{ single_query } ' returned { n } results" )
329- # No limit in force, or 'limit=0' asking for no results at all as a
330- # count format does, means the result count says nothing about
331- # truncation
332- if limit and n >= limit :
357+ # SMW reports an incomplete result set with a top-level
358+ # 'query-continue-offset' holding the offset the remainder starts at,
359+ # and omits the key for a complete one. That replaces the earlier
360+ # comparison of the result count against the limit, which was wrong in
361+ # both directions: it could not see the wiki's own '$smwgQMaxLimit'
362+ # cap, and it reported a complete set of exactly 'limit' results as
363+ # truncated
364+ next_offset = result .get ("query-continue-offset" )
365+ truncated = next_offset is not None
366+ if truncated :
333367 _logger .warning (
334- f"Query '{ single_query } ' returned { n } results, which meets the "
335- f"requested limit of { limit } . Results are truncated - raise "
336- f"the limit or page through with '|offset=' to retrieve the "
337- f"remainder."
368+ f"Query '{ single_query } ' returned { n } results and the wiki "
369+ f"reports further ones . Results are truncated - raise the "
370+ f"limit or page through with '|offset={ next_offset } ' to "
371+ f"retrieve the remainder."
338372 )
339373 if query .return_json :
340374 return result
@@ -355,6 +389,14 @@ def semantic_search_(single_query):
355389 f"Query '{ single_query } ': { dropped } of { n } results were dropped "
356390 f"because the wiki reported them as non-existing pages."
357391 )
392+ if query .return_meta :
393+ return SemanticSearchResult (
394+ query = single_query ,
395+ titles = page_list ,
396+ count = n ,
397+ truncated = truncated ,
398+ next_offset = next_offset ,
399+ )
358400 return page_list
359401
360402 if query .parallel :
@@ -364,10 +406,10 @@ def semantic_search_(single_query):
364406 else :
365407 query_results = [semantic_search_ (single_query = sq ) for sq in query .query ]
366408
367- if query .return_json :
368- # Each entry of query_results is the raw SMW result dict for one query.
369- # Do not flatten dicts ; always return the list of result dicts (one per
370- # query) , even when only a single query was passed.
409+ if query .return_json or query . return_meta :
410+ # Each entry of query_results is the raw SMW result dict, or the
411+ # SemanticSearchResult, for one query. Do not flatten those ; always
412+ # return one entry per query, even when only a single query was passed.
371413 return query_results
372414
373415 return [item for sublist in query_results for item in sublist ]
0 commit comments