diff --git a/cornac/eval_methods/next_item_evaluation.py b/cornac/eval_methods/next_item_evaluation.py index aceb1a26c..4b83f57bd 100644 --- a/cornac/eval_methods/next_item_evaluation.py +++ b/cornac/eval_methods/next_item_evaluation.py @@ -22,7 +22,8 @@ from ..data import SequentialDataset from ..experiment.result import Result -from ..models import NextItemRecommender +from ..models import NextItemRecommender, SemanticIDMixin +from ..metrics import Recall, NDCG from ..utils import validate_format from . import BaseMethod @@ -31,6 +32,51 @@ "next", ]) + +def _semantic_metrics(model, metrics, enabled): + if not enabled or not isinstance(model, SemanticIDMixin): + return [] + return [mt for mt in metrics if type(mt) in (Recall, NDCG) + and isinstance(mt.k, (int, np.integer)) and mt.k > 0] + + +def _semantic_metric_scores(predictions, target_sid, group_sizes, metrics): + """SID matching and uniform expected item credit after compaction.""" + seen = set() + sid_rank = None + expanded_before = 0 + target_before = 0 + for row in predictions: + sid = tuple(row) + if sid not in group_sizes or sid in seen: + continue + if sid == target_sid: + sid_rank = len(seen) + target_before = expanded_before + seen.add(sid) + expanded_before += group_sizes[sid] + + values = [] + size = group_sizes[target_sid] + for view in ("SID", "CCE"): + for metric in metrics: + if sid_rank is None: + value = 0.0 + elif view == "SID": + value = float(sid_rank < metric.k) + if type(metric) is NDCG: + value /= np.log2(sid_rank + 2) + else: + included = min(size, max(0, metric.k - target_before)) + if type(metric) is Recall: + value = included / size + else: + positions = target_before + np.arange(included) + 2 + value = float(np.sum(1.0 / np.log2(positions)) / size) + values.append(value) + return values, len(seen) + + def ranking_eval( model, metrics, @@ -40,6 +86,7 @@ def ranking_eval( exclude_unknowns=True, mode="last", verbose=False, + semantic_id_metrics=False, ): """Evaluate model on provided ranking metrics. @@ -64,6 +111,12 @@ def ranking_eval( verbose: bool, optional, default: False Output evaluation progress. + semantic_id_metrics: bool, optional, default: False + Set True to add SID and CCE views for finite-K Recall/NDCG on + SemanticIDMixin models. + Returned lists contain ordinary metrics first, then supported SID metrics, + then supported CCE metrics, in the requested metric order within each view. + Returns ------- res: (List, List) @@ -76,9 +129,24 @@ def ranking_eval( if len(metrics) == 0: return [], [] + semantic_metrics = _semantic_metrics(model, metrics, semantic_id_metrics) + if isinstance(model, SemanticIDMixin) and not semantic_id_metrics: + warnings.warn("Semantic ID metrics are disabled; reporting ordinary item metrics only.") + result_count = len(metrics) + 2 * len(semantic_metrics) avg_results = [] - session_results = [defaultdict(list) for _ in enumerate(metrics)] - user_results = [defaultdict(list) for _ in enumerate(metrics)] + session_results = [defaultdict(list) for _ in range(result_count)] + user_results = [defaultdict(list) for _ in range(result_count)] + short_queries = 0 + if semantic_metrics: + catalog = np.asarray(model.semantic_id_catalog()) + if (catalog.ndim != 2 or catalog.shape[0] != model.total_items or + catalog.shape[1] == 0 or not np.issubdtype(catalog.dtype, np.integer)): + raise ValueError("semantic_id_catalog must be an integer array (total_items, SID_length)") + candidate_count = train_set.num_items if exclude_unknowns else test_set.num_items + group_sizes = defaultdict(int) + for row in catalog[:candidate_count]: + group_sizes[tuple(row)] += 1 + max_cutoff = max(mt.k for mt in semantic_metrics) user_sessions = defaultdict(list) session_ids = [] @@ -117,7 +185,8 @@ def ranking_eval( u_gt_neg_items = np.nonzero(u_gt_neg_mask)[0] item_indices = np.nonzero(u_gt_pos_mask + u_gt_neg_mask)[0] - item_rank, item_scores = model.rank( + rank = model.rank_with_semantic_ids if semantic_metrics else model.rank + predictions = rank( user_idx, item_indices, history_items=session_items[:test_pos], @@ -127,21 +196,43 @@ def ranking_eval( extra_data=test_set.extra_data, ) - for i, mt in enumerate(metrics): - mt_score = mt.compute( + item_rank, item_scores = predictions[:2] + scores = [] + for mt in metrics: + scores.append(mt.compute( gt_pos=u_gt_pos_items, gt_neg=u_gt_neg_items, pd_rank=item_rank, pd_scores=item_scores, item_indices=item_indices, - ) + )) + if semantic_metrics: + semantic_ids = np.asarray(predictions[2]) + if (semantic_ids.ndim != 2 or semantic_ids.shape[1] != catalog.shape[1] or + not np.issubdtype(semantic_ids.dtype, np.integer)): + raise ValueError("predicted Semantic IDs must be an integer array (n_predictions, SID_length)") + if len(u_gt_pos_items): + extra_scores, distinct_count = _semantic_metric_scores( + semantic_ids, tuple(catalog[test_pos_items]), group_sizes, semantic_metrics) + else: + extra_scores = [0.0] * (2 * len(semantic_metrics)) + distinct_count = len({tuple(row) for row in semantic_ids if tuple(row) in group_sizes}) + scores.extend(extra_scores) + short_queries += distinct_count < max_cutoff + for i, mt_score in enumerate(scores): if user_based: user_results[i][user_idx].append(mt_score) else: session_results[i][sid].append(mt_score) + if short_queries: + warnings.warn( + f"{short_queries} evaluation queries returned fewer than {max_cutoff} " + "valid distinct Semantic IDs; SID/CCE metrics use the available predictions " + "without increasing the decoding budget.") + # avg results of ranking metrics - for i, mt in enumerate(metrics): + for i in range(result_count): if user_based: user_ids = list(user_sessions.keys()) user_avg_results = [np.mean(user_results[i][user_idx]) for user_idx in user_ids] @@ -190,8 +281,26 @@ class NextItemEvaluation(BaseMethod): verbose: bool, optional, default: False Output running log. + semantic_id_metrics: bool, optional, default: False + Set True for models implementing :class:`cornac.models.SemanticIDMixin` + to report ``SID-Recall@K``, ``SID-NDCG@K``, ``CCE-Recall@K`` and + ``CCE-NDCG@K`` for requested finite-K Recall/NDCG metrics. Ordinary item + metrics retain their existing names and model resolution policy. Other + metrics and non-SID models still run their ordinary item evaluation. + False reports ordinary item metrics only and warns for SID models. + Notes ----- + **Semantic IDs.** Full IDs include disambiguating suffixes. Invalid and + repeated predictions are removed before computing compact SID ranks. CCE + expands each surviving ID to its admitted item bucket and averages item + Recall/NDCG under uniform order within each bucket. Thus CCE is expected + item performance, while ordinary metrics evaluate the model's actual item + ranking. Candidate filtering also restricts bucket sizes. Predictions are + obtained once per query for all views; configured decoding budgets stay + unchanged, with one warning per split when distinct valid IDs fall below + the largest supported cutoff. No ungenerated candidates are added. + **Item content.** Modalities (e.g., ``item_feature=FeatureModality(features=..., ids=...)``) can be passed as keyword arguments to the constructor or any of the class-method builders @@ -223,8 +332,10 @@ def __init__( mode="last", exclude_unknowns=True, verbose=False, + semantic_id_metrics=False, **kwargs, ): + self.semantic_id_metrics = semantic_id_metrics super().__init__( data=data, data_size=0 if data is None else len(data), @@ -314,6 +425,7 @@ def eval( user_based=False, verbose=False, mode="last", + semantic_id_metrics=False, **kwargs, ): metric_avg_results = OrderedDict() @@ -328,11 +440,15 @@ def eval( exclude_unknowns=exclude_unknowns, mode=mode, verbose=verbose, + semantic_id_metrics=semantic_id_metrics, ) - for i, mt in enumerate(ranking_metrics): - metric_avg_results[mt.name] = avg_results[i] - metric_user_results[mt.name] = user_results[i] + names = [mt.name for mt in ranking_metrics] + semantic_metrics = _semantic_metrics(model, ranking_metrics, semantic_id_metrics) + names.extend(f"{view}-{mt.name}" for view in ("SID", "CCE") for mt in semantic_metrics) + for i, name in enumerate(names): + metric_avg_results[name] = avg_results[i] + metric_user_results[name] = user_results[i] return Result(model.name, metric_avg_results, metric_user_results) @@ -403,6 +519,7 @@ def evaluate(self, model, metrics, user_based, show_validation=True): user_based=user_based, mode=self.mode, verbose=self.verbose, + semantic_id_metrics=self.semantic_id_metrics, ) test_time = time.time() - start test_result.metric_avg_results["Train (s)"] = train_time @@ -422,6 +539,7 @@ def evaluate(self, model, metrics, user_based, show_validation=True): user_based=user_based, mode=self.mode, verbose=self.verbose, + semantic_id_metrics=self.semantic_id_metrics, ) val_time = time.time() - start val_result.metric_avg_results["Time (s)"] = val_time diff --git a/cornac/experiment/result.py b/cornac/experiment/result.py index 0c7f4e827..c63d5adb9 100644 --- a/cornac/experiment/result.py +++ b/cornac/experiment/result.py @@ -187,10 +187,11 @@ class ExperimentResult(list): """ def __str__(self): - headers = list(self[0].metric_avg_results.keys()) + headers = list(dict.fromkeys(m for r in self for m in r.metric_avg_results)) data, index = [], [] for r in self: - data.append([NUM_FMT.format(r.metric_avg_results[m]) for m in headers]) + data.append([NUM_FMT.format(r.metric_avg_results[m]) if m in r.metric_avg_results + else "N/A" for m in headers]) index.append(r.model_name) return _table_format(data, headers, index, h_bars=[1]) diff --git a/cornac/models/__init__.py b/cornac/models/__init__.py index 5b95a2aae..994a344db 100644 --- a/cornac/models/__init__.py +++ b/cornac/models/__init__.py @@ -16,6 +16,7 @@ from .recommender import Recommender from .recommender import NextBasketRecommender from .recommender import NextItemRecommender +from .recommender import SemanticIDMixin from .amr import AMR from .ann import AnnoyANN diff --git a/cornac/models/diffgrm/README.md b/cornac/models/diffgrm/README.md index 9b8c31992..f5af4ebc8 100644 --- a/cornac/models/diffgrm/README.md +++ b/cornac/models/diffgrm/README.md @@ -31,6 +31,7 @@ from cornac.models.diffgrm import DiffGRM, DIFFGRM_SPORTS_CONFIG item_feature = FeatureModality(features=item_embeddings, ids=item_ids) eval_method = NextItemEvaluation.from_splits( + semantic_id_metrics=True, train_data=train_data, val_data=val_data, test_data=test_data, @@ -209,3 +210,16 @@ The exported paper configurations use four 256-way digits, one encoder layer, fo - Zhao Liu, Yichen Zhu, Yiqing Yang, Guoping Tang, Rui Huang, Qiang Luo, Xiao Lv, Ruiming Tang, Kun Gai, and Guorui Zhou. [DiffGRM: Diffusion-based Generative Recommendation Model](https://arxiv.org/abs/2510.21805). WWW 2026. - [Released DiffGRM repository](https://github.com/liuzhao09/DiffGRM), behavior audited at commit `ad7b971c7e525e9fea6fb8e362a5c49dccb2473c`. + + +### Semantic ID evaluation + +Set `NextItemEvaluation(..., semantic_id_metrics=True)` to report ordinary +item metrics plus `SID-Recall@K`, `SID-NDCG@K`, `CCE-Recall@K` and `CCE-NDCG@K` for +requested finite-K Recall/NDCG metrics. All views share one inference; CCE averages +item credit over uniform order within each admitted collision bucket. Invalid +and repeated predicted IDs are compacted. Other requested metrics keep ordinary +item evaluation. Configured beam sizes remain unchanged; short candidate lists +warn. Set `semantic_id_metrics=False` for ordinary item metrics only. + +See the [evaluation documentation](../../../docs/source/api_ref/eval_methods.rst). diff --git a/cornac/models/diffgrm/diffgrm.py b/cornac/models/diffgrm/diffgrm.py index 2734a33ef..9ade9b784 100644 --- a/cornac/models/diffgrm/diffgrm.py +++ b/cornac/models/diffgrm/diffgrm.py @@ -360,12 +360,15 @@ def cpd_decode_batch( digit_order=None, greedy_final=False, return_diagnostics=False, + return_raw=False, ): """Batched confidence-prioritized decoding over digit/code assignments. At every step, every still-masked digit competes globally. ``constrained`` additionally removes partial assignments that cannot lead to a catalog semantic ID. Complete duplicate IDs are collapsed by maximum path score. + When ``return_raw`` is true, the additional result retains every final + beam path, including invalid and duplicate addresses, before that collapse. """ device = memory.device batch_size = memory.size(0) @@ -478,13 +481,25 @@ def cpd_decode_batch( tuple(row) for row in np.asarray(catalog_codes).tolist() } batch_codes, batch_scores, batch_diagnostics = [], [], [] + if return_raw: + batch_raw = [] for row in range(batch_size): complete = {} + if return_raw: + raw_codes = [] + raw_scores = [] + raw_validity = [] for codes, score in zip( beams[row].cpu().tolist(), beam_scores[row].cpu().tolist() ): key = tuple(codes) - if -1 in key or not np.isfinite(score): + is_complete = -1 not in key and np.isfinite(score) + if return_raw: + raw_codes.append(key) + raw_scores.append(float(score)) + raw_validity.append(bool(is_complete and ( + catalog_set is None or key in catalog_set))) + if not is_complete: continue complete[key] = max(complete.get(key, -float("inf")), score) best = { @@ -504,9 +519,21 @@ def cpd_decode_batch( "duplicate_paths": len(beams[row]) - len(complete), } ) + if return_raw: + batch_raw.append( + { + "address_sids": raw_codes, + "scores": raw_scores, + "catalog_validity": raw_validity, + } + ) + if return_diagnostics and return_raw: + return batch_codes, batch_scores, batch_diagnostics, batch_raw if return_diagnostics: return batch_codes, batch_scores, batch_diagnostics + if return_raw: + return batch_codes, batch_scores, batch_raw return batch_codes, batch_scores @@ -522,6 +549,7 @@ def cpd_decode( digit_order=None, greedy_final=False, return_diagnostics=False, + return_raw=False, ): """Single-history wrapper around :func:`cpd_decode_batch`.""" if memory.size(0) != 1: @@ -537,9 +565,16 @@ def cpd_decode( digit_order=digit_order, greedy_final=greedy_final, return_diagnostics=return_diagnostics, + return_raw=return_raw, ) + if return_diagnostics and return_raw: + codes, scores, diagnostics, raw = result + return codes[0], scores[0], diagnostics[0], raw[0] if return_diagnostics: codes, scores, diagnostics = result return codes[0], scores[0], diagnostics[0] + if return_raw: + codes, scores, raw = result + return codes[0], scores[0], raw[0] codes, scores = result return codes[0], scores[0] diff --git a/cornac/models/diffgrm/recom_diffgrm.py b/cornac/models/diffgrm/recom_diffgrm.py index 1841d891e..e9b3db618 100644 --- a/cornac/models/diffgrm/recom_diffgrm.py +++ b/cornac/models/diffgrm/recom_diffgrm.py @@ -20,7 +20,7 @@ import numpy as np from tqdm.auto import trange -from cornac.models.recommender import NextItemRecommender +from ..recommender import NextItemRecommender, SemanticIDMixin from ...utils import get_rng @@ -82,7 +82,7 @@ def _diffgrm_session_iter( ) -class DiffGRM(NextItemRecommender): +class DiffGRM(NextItemRecommender, SemanticIDMixin): """DiffGRM: Diffusion-based Generative Recommendation Model. DiffGRM combines parallel OPQ semantic IDs (PSE), on-policy @@ -688,13 +688,16 @@ def _ensure_device(self, torch): if next(self.model.parameters()).device != requested: self.model.to(requested) - def _score_history(self, history_items): + def _score_history(self, history_items, return_semantic_ids=False): import torch from .diffgrm import cpd_decode - if not history_items: - return np.ones(self.total_items, dtype="float") + if len(history_items) == 0: + scores = np.ones(self.total_items, dtype="float") + if return_semantic_ids: + return scores, np.empty((0, self.n_digit), dtype="int64") + return scores self._ensure_device(torch) history = list(history_items)[-self.max_len :] input_iids = np.full((1, self.max_len), self.pad_idx, dtype="int64") @@ -709,7 +712,7 @@ def _score_history(self, history_items): decode_start = time.perf_counter() with torch.no_grad(): memory, padding_mask = self.model.encode_history(inputs, masks) - codes, path_scores, diagnostics = cpd_decode( + decoded = cpd_decode( self.model, memory, padding_mask, @@ -722,10 +725,16 @@ def _score_history(self, history_items): else None, greedy_final=self.scoring == "released", return_diagnostics=True, + return_raw=return_semantic_ids, ) + codes, path_scores, diagnostics = decoded[:3] self.last_decode_time_ = time.perf_counter() - decode_start self.last_decode_diagnostics_ = diagnostics - return self._decoded_item_scores(codes, path_scores) + scores = self._decoded_item_scores(codes, path_scores) + if return_semantic_ids: + semantic_ids = np.asarray(decoded[3]["address_sids"], dtype="int64").reshape(-1, self.n_digit) + return scores, semantic_ids + return scores def _decoded_item_scores(self, codes, path_scores): scores = np.full(self.total_items, -1e10, dtype="float") @@ -740,3 +749,12 @@ def _decoded_item_scores(self, codes, path_scores): def score(self, user_idx, history_items, **kwargs): return self._score_history(history_items) + + def semantic_id_catalog(self): + """Return complete IDs independently of native collision policy.""" + if not hasattr(self, "sid_table"): + raise ValueError("DiffGRM must be fitted before exporting its SID catalog") + return self.sid_table + + def _score_with_semantic_ids(self, user_idx, history_items, **kwargs): + return self._score_history(history_items, return_semantic_ids=True) diff --git a/cornac/models/letter/README.md b/cornac/models/letter/README.md index ea72b59da..c812e44af 100644 --- a/cornac/models/letter/README.md +++ b/cornac/models/letter/README.md @@ -27,6 +27,7 @@ from cornac.models import LETTER from cornac.models.letter import LETTER_BEAUTY_CONFIG eval_method = NextItemEvaluation.from_splits( + semantic_id_metrics=True, train_data=train, val_data=val, test_data=test, @@ -94,3 +95,16 @@ The authors' Beauty index contains 12,101 items but only 12,088 unique Semantic ### Reproduction limitation The released repository provides the final Beauty Semantic-ID table, but not the trained tokenizer checkpoint, content embeddings, or SASRec checkpoint used to produce it. Exact reproduction of the paper's learned Semantic IDs therefore requires those missing artifacts; the end-to-end result above is a runnable substitute rather than an exact reconstruction of the paper's tokenizer inputs. + + +### Semantic ID evaluation + +Set `NextItemEvaluation(..., semantic_id_metrics=True)` to report ordinary +item metrics plus `SID-Recall@K`, `SID-NDCG@K`, `CCE-Recall@K` and `CCE-NDCG@K` for +requested finite-K Recall/NDCG metrics. All views share one inference; CCE averages +item credit over uniform order within each admitted collision bucket. Invalid +and repeated predicted IDs are compacted. Other requested metrics keep ordinary +item evaluation. Configured beam sizes remain unchanged; short candidate lists +warn. Set `semantic_id_metrics=False` for ordinary item metrics only. + +See the [evaluation documentation](../../../docs/source/api_ref/eval_methods.rst). diff --git a/cornac/models/letter/recom_letter.py b/cornac/models/letter/recom_letter.py index b32cde156..d8dacfe10 100644 --- a/cornac/models/letter/recom_letter.py +++ b/cornac/models/letter/recom_letter.py @@ -521,11 +521,9 @@ def lr_lambda(step): if best_state is not None: self.model.load_state_dict(best_state) - def score(self, user_idx, history_items, **kwargs): + def _predict_history(self, history_items, return_semantic_ids=False): import torch - if len(history_items) == 0: - return np.ones(self.total_items, dtype="float") self._ensure_device(torch) enc_tokens, enc_mask = self._encoder_batch( torch, [list(history_items)[-self.max_len :]] @@ -538,7 +536,8 @@ def score(self, user_idx, history_items, **kwargs): ) scores = np.full(self.total_items, -1e10, dtype="float") for sid, log_prob in zip(beams, log_probs): - scores[self.sid_to_item[sid]] = log_prob + if sid in self.sid_to_item: + scores[self.sid_to_item[sid]] = log_prob else: sid_table = torch.as_tensor( self.sid_table, dtype=torch.long, device=self.device_ @@ -546,4 +545,14 @@ def score(self, user_idx, history_items, **kwargs): scores = self.model.score_all_items( enc_tokens, enc_mask, sid_table, self.scoring_batch_size ).astype("float") + if return_semantic_ids: + semantic_ids = (np.asarray(beams, dtype="int64").reshape(-1, self.sid_table.shape[1]) + if self.scoring == "beam" else self._semantic_ids_from_scores(scores)) + return scores, semantic_ids return scores + + def semantic_id_catalog(self): + """Return suffix-free IDs aligned to item indices.""" + if not hasattr(self, "sid_table"): + raise ValueError("LETTER must be fitted before exporting its SID catalog") + return self.sid_table diff --git a/cornac/models/recommender.py b/cornac/models/recommender.py index 5d8514e95..68f8e7d5a 100644 --- a/cornac/models/recommender.py +++ b/cornac/models/recommender.py @@ -502,6 +502,10 @@ def rank(self, user_idx, item_indices=None, k=-1, **kwargs): except ScoreException: known_item_scores = np.ones(self.total_items) * self.default_score() + return self._rank_from_scores(known_item_scores, item_indices, k) + + def _rank_from_scores(self, known_item_scores, item_indices=None, k=-1): + """Apply the standard item ranking to already computed scores.""" # check if the returned scores also cover unknown items # if not, all unknown items will be given the MIN score if len(known_item_scores) == self.total_items: @@ -764,3 +768,46 @@ def score(self, user_idx, history_items, **kwargs): """ raise NotImplementedError("The algorithm is not able to make score prediction!") + + +class SemanticIDMixin: + """Optional Semantic ID capability for next-item recommenders. + + Complete IDs include any item-disambiguating suffix. The combined ranking + exposes ordinary item predictions and generated IDs from the same inference; + it never reconstructs beam candidates from item-score filler values. + """ + + def semantic_id_catalog(self): + """Return an integer array ``(total_items, SID_length)`` aligned to items.""" + raise NotImplementedError + + def _score_with_semantic_ids(self, user_idx, history_items, **kwargs): + """Return full item scores and ranked complete IDs from one inference.""" + raise NotImplementedError + + def rank_with_semantic_ids(self, user_idx, item_indices=None, k=-1, **kwargs): + """Return item ranks, aligned item scores and raw ranked Semantic IDs. + + Item ranking has the same contract as :meth:`Recommender.rank`. + The third array has integer shape ``(n_predictions, SID_length)`` and + preserves the full generated list, including invalid/repeated rows, + independently of item-ranking ``k``. It is not retained on the model. + History and other context are passed in ``kwargs`` as with ``rank``. + """ + try: + scores, semantic_ids = self._score_with_semantic_ids(user_idx, **kwargs) + except ScoreException: + scores = np.ones(self.total_items) * self.default_score() + semantic_ids = np.empty((0, self.semantic_id_catalog().shape[1]), dtype="int64") + item_rank, item_scores = self._rank_from_scores(scores, item_indices, k) + return item_rank, item_scores, semantic_ids + + def _semantic_ids_from_scores(self, scores): + """Rank exhaustive SID scores by maximum member, then lexicographic ID.""" + catalog = self.semantic_id_catalog() + group_scores = {} + for sid, score in zip(map(tuple, catalog), scores): + group_scores[sid] = max(group_scores.get(sid, -np.inf), score) + ranked = sorted(group_scores, key=lambda sid: (-group_scores[sid], sid)) + return np.asarray(ranked, dtype="int64").reshape(-1, catalog.shape[1]) diff --git a/cornac/models/rpg/README.md b/cornac/models/rpg/README.md index 4cd84808b..11f4d60af 100644 --- a/cornac/models/rpg/README.md +++ b/cornac/models/rpg/README.md @@ -35,6 +35,7 @@ from cornac.models import RPG from cornac.models.rpg import RPG_BEAUTY_CONFIG eval_method = NextItemEvaluation.from_splits( + semantic_id_metrics=True, train_data=train, val_data=val, test_data=test, @@ -140,3 +141,16 @@ within 3.1% and 8.3% of the paper's R@5, respectively; Toys is 3.5% higher. - Graph decoding is approximate and stochastic because its initial beam is sampled. Set `seed` for repeatability, or use exact scoring when deterministic full-catalog ranks are required. + + +### Semantic ID evaluation + +Set `NextItemEvaluation(..., semantic_id_metrics=True)` to report ordinary +item metrics plus `SID-Recall@K`, `SID-NDCG@K`, `CCE-Recall@K` and `CCE-NDCG@K` for +requested finite-K Recall/NDCG metrics. All views share one inference; CCE averages +item credit over uniform order within each admitted collision bucket. Invalid +and repeated predicted IDs are compacted. Other requested metrics keep ordinary +item evaluation. Configured beam sizes remain unchanged; short candidate lists +warn. Set `semantic_id_metrics=False` for ordinary item metrics only. + +See the [evaluation documentation](../../../docs/source/api_ref/eval_methods.rst). diff --git a/cornac/models/rpg/recom_rpg.py b/cornac/models/rpg/recom_rpg.py index c66f0f12a..acb2edcff 100644 --- a/cornac/models/rpg/recom_rpg.py +++ b/cornac/models/rpg/recom_rpg.py @@ -18,7 +18,7 @@ import numpy as np from tqdm.auto import trange -from cornac.models.recommender import NextItemRecommender +from ..recommender import NextItemRecommender, SemanticIDMixin from ...utils import get_rng @@ -98,7 +98,7 @@ def _rpg_session_iter( ) -class RPG(NextItemRecommender): +class RPG(NextItemRecommender, SemanticIDMixin): """RPG: Generating Long Semantic IDs in Parallel for Recommendation. RPG is the architectural counterpoint to TIGER. Instead of short *ordered* @@ -646,13 +646,16 @@ def _ensure_device(self, torch): if next(self.model.parameters()).device.type != torch.device(self.device_).type: self.model.to(self.device_) - def score(self, user_idx, history_items, **kwargs): + def _predict_history(self, history_items, return_semantic_ids=False): import torch from .rpg import graph_decode if len(history_items) == 0: - return np.ones(self.total_items, dtype="float") + scores = np.ones(self.total_items, dtype="float") + if return_semantic_ids: + return scores, np.empty((0, self.sid_table.shape[1]), dtype="int64") + return scores self._ensure_device(torch) hist = list(history_items)[-self.max_len :] hist = [self.pad_idx] * (self.max_len - len(hist)) + hist @@ -663,9 +666,10 @@ def score(self, user_idx, history_items, **kwargs): logits = self.model.next_item_logits(input_ids, attn_mask)[0] # (M, K) if self.scoring == "exact": - return ( - self._exact_item_scores(logits)[0].cpu().numpy().astype("float") - ) + scores = self._exact_item_scores(logits)[0].cpu().numpy().astype("float") + if return_semantic_ids: + return scores, self._semantic_ids_from_scores(scores) + return scores # graph-guided beam decoding over the final propagated beam flat = logits.reshape(-1).cpu().numpy() @@ -679,4 +683,18 @@ def score(self, user_idx, history_items, **kwargs): ) scores = np.full(self.total_items, -1e10, dtype="float") scores[candidates] = candidate_scores + if return_semantic_ids: + return scores, self.sid_table[candidates] return scores + + def semantic_id_catalog(self): + """Return complete, un-offset OPQ codes aligned to item indices.""" + if not hasattr(self, "sid_table"): + raise ValueError("RPG must be fitted before exporting its SID catalog") + return self.sid_table + + def _score_with_semantic_ids(self, user_idx, history_items, **kwargs): + return self._predict_history(history_items, return_semantic_ids=True) + + def score(self, user_idx, history_items, **kwargs): + return self._predict_history(history_items) diff --git a/cornac/models/tiger/README.md b/cornac/models/tiger/README.md index d697cc987..370df8123 100644 --- a/cornac/models/tiger/README.md +++ b/cornac/models/tiger/README.md @@ -13,6 +13,7 @@ from cornac.models import TIGER from cornac.models.tiger import GRID_CONFIG, PAISCHER_CONFIG eval_method = NextItemEvaluation.from_splits( + semantic_id_metrics=True, train_data=train, test_data=test, val_data=val, mode="last", item_feature=FeatureModality(features=item_embeddings, ids=item_ids), ) @@ -79,3 +80,19 @@ SASRec tuned over loss {`ce`, `bce`, `bpr`, `bpr-max`} × emb {64, 128} × lr {5 - `TIGER` likely uses `Sentence-T5-XXL` embeddings, but improvements over ST5-base are minimal (sometimes even decrease). OpenAI's `text-embedding-3-large` (provided by [`RPG`](https://github.com/facebookresearch/RPG_KDD2025)) is also available, but it doesn't guarantee performance improvement. - `beauty` and `sports` are somewhat reproducible, but `toys` is ***not***, likely due to ambiguity in the item text used (possibly `descriptions` only). - Well-tuned `SASRec` and `BERT4Rec` (within cornac) still outperform `TIGER`: tuned SASRec on beauty wins every metric except a marginal R@20 edge for `TIGER` + openai embedding, at a fraction of the cost (pure ID-based recommendation). + + +### Semantic ID evaluation + +Set `NextItemEvaluation(..., semantic_id_metrics=True)` to report ordinary +item metrics plus `SID-Recall@K`, `SID-NDCG@K`, `CCE-Recall@K` and `CCE-NDCG@K` for +requested finite-K Recall/NDCG metrics. All views share one inference; CCE averages +item credit over uniform order within each admitted collision bucket. Invalid +and repeated predicted IDs are compacted. Other requested metrics keep ordinary +item evaluation. Configured beam sizes remain unchanged; short candidate lists +warn. Set `semantic_id_metrics=False` for ordinary item metrics only. + +See the [evaluation documentation](../../../docs/source/api_ref/eval_methods.rst). + +TIGER uses complete IDs including the disambiguating suffix, so its SID and CCE +metrics coincide even when tokenizer prefixes collide. diff --git a/cornac/models/tiger/recom_tiger.py b/cornac/models/tiger/recom_tiger.py index 1ec31bbcd..805676e9d 100644 --- a/cornac/models/tiger/recom_tiger.py +++ b/cornac/models/tiger/recom_tiger.py @@ -19,7 +19,7 @@ import numpy as np from tqdm.auto import trange -from cornac.models.recommender import NextItemRecommender +from ..recommender import NextItemRecommender, SemanticIDMixin from ...utils import get_rng from ..seq_utils import session_seq_iter @@ -29,7 +29,7 @@ SUPPORTED_LR_SCHEDULES = ("constant", "cosine") -class TIGER(NextItemRecommender): +class TIGER(NextItemRecommender, SemanticIDMixin): """TIGER: Recommender Systems with Generative Retrieval. Two-stage generative retrieval with semantic IDs: (1) an RQ-VAE quantizes @@ -564,11 +564,9 @@ def _ensure_device(self, torch): if next(self.model.parameters()).device.type != torch.device(self.device_).type: self.model.to(self.device_) - def score(self, user_idx, history_items, **kwargs): + def _predict_history(self, history_items, return_semantic_ids=False): import torch - if len(history_items) == 0: - return np.ones(self.total_items, dtype="float") self._ensure_device(torch) hist = list(history_items)[-self.max_len :] hist = [self.pad_idx] * (self.max_len - len(hist)) + hist @@ -584,10 +582,32 @@ def score(self, user_idx, history_items, **kwargs): beams, logps = self.model.generate_beam(enc_tokens_t, enc_mask_t, self.n_beams, self.prefix_children) scores = np.full(self.total_items, -1e10, dtype="float") for sid, lp in zip(beams, logps): - scores[self.sid_to_item[sid]] = lp + if sid in self.sid_to_item: + scores[self.sid_to_item[sid]] = lp else: # "exact" sid_table_t = torch.tensor(self.sid_table, dtype=torch.long, device=self.device_) scores = self.model.score_all_items( enc_tokens_t, enc_mask_t, sid_table_t, self.scoring_batch_size ).astype("float") + if return_semantic_ids: + semantic_ids = (np.asarray(beams, dtype="int64").reshape(-1, self.sid_table.shape[1]) + if self.scoring == "beam" else self._semantic_ids_from_scores(scores)) + return scores, semantic_ids return scores + + def semantic_id_catalog(self): + """Return complete, suffix-bearing IDs aligned to item indices.""" + if not hasattr(self, "sid_table"): + raise ValueError("TIGER must be fitted before exporting its SID catalog") + return self.sid_table + + def _score_with_semantic_ids(self, user_idx, history_items, **kwargs): + if len(history_items) == 0: + return np.ones(self.total_items, dtype="float"), np.empty( + (0, self.semantic_id_catalog().shape[1]), dtype="int64") + return self._predict_history(history_items, return_semantic_ids=True) + + def score(self, user_idx, history_items, **kwargs): + if len(history_items) == 0: + return np.ones(self.total_items, dtype="float") + return self._predict_history(history_items) diff --git a/docs/source/api_ref/eval_methods.rst b/docs/source/api_ref/eval_methods.rst index d482f66da..28cd5017d 100644 --- a/docs/source/api_ref/eval_methods.rst +++ b/docs/source/api_ref/eval_methods.rst @@ -32,3 +32,80 @@ Timestamp Split --------------- .. automodule:: cornac.eval_methods.timestamp_split :members: + +Next Item Evaluation +-------------------- +.. automodule:: cornac.eval_methods.next_item_evaluation + :members: + +Recommendation Evaluation with Semantic IDs +------------------------------------------ + +Use ``NextItemEvaluation`` for both ordinary next-item models (such as SASRec) +and Semantic ID models (TIGER, LETTER, RPG and DiffGRM), including a mixed +``Experiment``. Its optional ``semantic_id_metrics`` switch defaults to False. +Set it to True in Semantic ID experiment setup to enable the extra views. +Ordinary item metric names and model-specific item resolution are preserved. +With the switch enabled, for each requested finite-K ``Recall`` or ``NDCG``, +SID models additionally report ``SID-Recall@K``, ``SID-NDCG@K``, ``CCE-Recall@K`` and ``CCE-NDCG@K``. +Other metrics, including AUC, MAP, MRR and unbounded cutoffs, still run their +ordinary item evaluation without additional views. Non-SID models have no SID +or CCE values; mixed result tables show ``N/A`` for those entries. + +SID metrics match the target's complete generated ID, including any +item-disambiguating suffix. Invalid IDs, IDs absent from the admitted candidate +catalog, and repeated IDs after their first occurrence are removed before +ranking. Candidate filtering respects ``exclude_unknowns`` and also restricts +each collision bucket's size. No ungenerated fallback candidates are introduced. + +Collision-Corrected Evaluation (CCE) expands surviving IDs in their original +order into their full admitted item buckets. Earlier buckets consume their full +widths. Items within a bucket have uniformly random order. CCE Recall is the +fraction of target-bucket positions inside K, and CCE NDCG averages the log +discount over those positions, divided by the full target-bucket size. + +For example, suppose items a and b share the first predicted ID, c has the +second ID, and b is the target. A model that ranks a first has ordinary +Recall@1 of 0, SID-Recall@1 of 1 and CCE-Recall@1 of 0.5. CCE-Recall@2 is 1 +because both positions of the target bucket fit. This is expected item credit, +not the accuracy of a concrete item resolver. A catalog whose complete IDs are +all unique yields identical SID and CCE metrics. TIGER's suffix-bearing IDs +have this property even when their tokenizer prefixes collide. + +CCE follows Qian Zhang, Lech Szymanski, Haibo Zhang and Jeremiah D. Deng, +`Faithful Evaluation of Semantic-ID Tokenizers for Generative Recommendation +`_ (CIKM 2026). Ordinary item metrics remain +the direct measure of the model's concrete recommendations; CCE evaluates the +uniform within-bucket assumption. + +All three views share one model inference per query. Configured decoding budgets +are preserved. Short outputs are evaluated as returned, and one warning per +split reports the number of queries with fewer valid distinct IDs than the +largest requested SID/CCE cutoff. Increase the model's beam setting explicitly +if a larger candidate budget is desired. With ``semantic_id_metrics=False``, +only ordinary item metrics are reported; SID models issue a warning. +``mode='last'``/``mode='next'`` and per-query/per-user aggregation apply to every +view in the same way. + +For example:: + + evaluation = NextItemEvaluation.from_splits( + train, test, fmt="USIT", item_feature=features, + semantic_id_metrics=True) + Experiment(evaluation, [TIGER(device="cpu"), SASRec()], + [Recall(10), NDCG(10)]).run() + +Adapter authors implement ``SemanticIDMixin`` alongside ``NextItemRecommender``. +``semantic_id_catalog()`` returns a two-dimensional integer array with one row +per global item, storing complete generated IDs. ``rank_with_semantic_ids`` has +the same inputs as ``rank`` and returns ordinary item ranks, item scores aligned +to the candidate indices, and a two-dimensional integer array of raw ranked SID +rows. Its full SID output is independent of item-ranking ``k``. The model keeps +its decoding and concrete SID-to-item policy. The mixin provides shared ranking +from ``_score_with_semantic_ids(user_idx, history_items, **kwargs)``, which returns +full item scores and SID rows from the same inference. Empty SID output has +shape ``(0, SID_length)``. No persistent prediction cache is required. + +For exhaustive scoring, the built-in adapters order SID groups by their maximum +member score, with lexicographic ID ties. Beam adapters expose only generated +rows, preserving decoder order before evaluator filtering and compaction. diff --git a/docs/source/api_ref/models.rst b/docs/source/api_ref/models.rst index 030b88912..fddac1a1a 100644 --- a/docs/source/api_ref/models.rst +++ b/docs/source/api_ref/models.rst @@ -333,3 +333,8 @@ Weighted Matrix Factorization (WMF) ----------------------------------- .. automodule:: cornac.models.wmf.recom_wmf :members: + +Semantic ID Model Contracts +-------------------------- +.. autoclass:: cornac.models.recommender.SemanticIDMixin + :members: diff --git a/examples/diffgrm_example.py b/examples/diffgrm_example.py index e01f14a14..f66c3c377 100644 --- a/examples/diffgrm_example.py +++ b/examples/diffgrm_example.py @@ -62,6 +62,7 @@ next_item_eval = NextItemEvaluation.leave_last_out( data=data, exclude_unknowns=True, + semantic_id_metrics=True, verbose=True, item_feature=FeatureModality(features=features, ids=item_ids), ) diff --git a/examples/letter_example.py b/examples/letter_example.py index 9bdb28489..45527333d 100644 --- a/examples/letter_example.py +++ b/examples/letter_example.py @@ -69,6 +69,7 @@ val_data=val_data, test_data=test_data, exclude_unknowns=True, + semantic_id_metrics=True, verbose=True, fmt="USIT", item_feature=FeatureModality(features=content, ids=item_ids), diff --git a/examples/rpg_example.py b/examples/rpg_example.py index 0e494031b..a05699ec5 100644 --- a/examples/rpg_example.py +++ b/examples/rpg_example.py @@ -59,6 +59,7 @@ val_data=val_data, test_data=test_data, exclude_unknowns=True, + semantic_id_metrics=True, verbose=True, fmt="USIT", item_feature=FeatureModality(features=features, ids=item_ids), diff --git a/examples/tiger_example.py b/examples/tiger_example.py index b6c407913..6b46253ec 100644 --- a/examples/tiger_example.py +++ b/examples/tiger_example.py @@ -57,6 +57,7 @@ next_item_eval = NextItemEvaluation.leave_last_out( data=data, exclude_unknowns=True, + semantic_id_metrics=True, verbose=True, item_feature=FeatureModality(features=features, ids=item_ids), ) diff --git a/tests/cornac/eval_methods/test_semantic_id_metrics.py b/tests/cornac/eval_methods/test_semantic_id_metrics.py new file mode 100644 index 000000000..f8574b4f5 --- /dev/null +++ b/tests/cornac/eval_methods/test_semantic_id_metrics.py @@ -0,0 +1,378 @@ +# Copyright 2026 The Cornac Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Three-view next-item evaluation with backend-free fixed predictions.""" + +import itertools +import os +import subprocess +import sys +import warnings + +import numpy as np +import pytest + +from cornac.eval_methods import NextItemEvaluation +from cornac.eval_methods.next_item_evaluation import ( + _semantic_metric_scores, + ranking_eval, +) +from cornac.exception import ScoreException +from cornac.experiment.result import ExperimentResult +from cornac.metrics import Recall, NDCG, AUC, MAP, MRR +from cornac.models import NextItemRecommender, SemanticIDMixin, SPop + + +class FixedSID(NextItemRecommender, SemanticIDMixin): + def __init__(self, codes=(0, 0, 1), predicted=(0, 1), scores=(3.0, 2.0, 1.0)): + super().__init__("FixedSID") + self.codes = np.asarray(codes, dtype="int64").reshape(-1, 1) + self.predicted = np.asarray(predicted, dtype="int64").reshape(-1, 1) + self.scores = np.asarray(scores) + self.calls = [] + + def semantic_id_catalog(self): + return self.codes + + def _score_with_semantic_ids(self, user_idx, history_items, **kwargs): + self.calls.append((list(history_items), kwargs)) + return self.scores.copy(), self.predicted.copy() + + def score(self, user_idx, history_items, **kwargs): + return self._score_with_semantic_ids(user_idx, history_items, **kwargs)[0] + + +def evaluation(**kwargs): + train = [("u", "tr", "a", 0), ("u", "tr", "b", 1), ("u", "tr", "c", 2)] + test = [("u", "te", "a", 3), ("u", "te", "b", 4)] + kwargs.setdefault("semantic_id_metrics", True) + return NextItemEvaluation.from_splits(train, test, fmt="USIT", **kwargs) + + +@pytest.mark.parametrize("entrypoint", ["evaluate", "eval", "ranking_eval"]) +@pytest.mark.parametrize("flag", ["omitted", False, True]) +@pytest.mark.parametrize("model_class", [FixedSID, SPop]) +def test_semantic_views_require_explicit_opt_in(entrypoint, flag, model_class): + data = evaluation() + options = {} if flag == "omitted" else {"semantic_id_metrics": flag} + ev = NextItemEvaluation.from_splits( + [("u", "tr", "a", 0), ("u", "tr", "b", 1), ("u", "tr", "c", 2)], + [("u", "te", "a", 3), ("u", "te", "b", 4)], + fmt="USIT", + **options, + ) + model = model_class() + enabled = flag is True and model_class is FixedSID + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + if entrypoint == "evaluate": + result, _ = ev.evaluate(model, [Recall(1)], False) + values = list(result.metric_avg_results.values())[:-2] + names = result.metric_avg_results + else: + model.fit(data.train_set) + if entrypoint == "eval": + result = NextItemEvaluation.eval( + model, data.train_set, data.test_set, True, [Recall(1)], **options + ) + values = list(result.metric_avg_results.values()) + names = result.metric_avg_results + else: + values, _ = ranking_eval( + model, [Recall(1)], data.train_set, data.test_set, **options + ) + names = None + assert len(values) == (3 if enabled else 1) + if enabled: + assert values == [0.0, 1.0, 0.5] + if names is not None: + assert ("SID-Recall@1" in names) is enabled + assert ("CCE-Recall@1" in names) is enabled + assert len(caught) == int(model_class is FixedSID and not enabled) + if caught: + assert "disabled" in str(caught[0].message) + + +@pytest.mark.parametrize("predicted", [(0, 1, 2), (1, 0), (9, 0, 0, 2, 1), ()]) +def test_cce_matches_all_bucket_permutations(predicted): + groups = {0: (0, 1), 1: (2,), 2: (3, 4, 5)} + compact = list(dict.fromkeys(sid for sid in predicted if sid in groups)) + rankings = [ + sum((list(bucket) for bucket in permutation), []) + for permutation in itertools.product( + *(list(itertools.permutations(groups[sid])) for sid in compact) + ) + ] + for target in range(6): + target_sid = next(sid for sid, members in groups.items() if target in members) + for cutoff in range(1, 8): + values, distinct = _semantic_metric_scores( + np.asarray(predicted).reshape(-1, 1), + (target_sid,), + {(sid,): len(members) for sid, members in groups.items()}, + [Recall(cutoff), NDCG(cutoff)], + ) + oracle_hits, oracle_ndcg = [], [] + for ranking in rankings: + rank = ranking.index(target) if target in ranking else None + hit = float(rank is not None and rank < cutoff) + oracle_hits.append(hit) + oracle_ndcg.append(hit / np.log2(rank + 2) if rank is not None else 0.0) + assert values[2:] == pytest.approx( + [np.mean(oracle_hits), np.mean(oracle_ndcg)] + ) + assert distinct == len(compact) + + +def test_three_views_keep_ordinary_resolution_and_share_inference(): + model = FixedSID() + result, _ = evaluation().evaluate(model, [Recall(1), NDCG(1)], False) + values = result.metric_avg_results + assert values["Recall@1"] == 0.0 + assert values["SID-Recall@1"] == 1.0 + assert values["CCE-Recall@1"] == 0.5 + assert values["CCE-NDCG@1"] == 0.5 + assert len(model.calls) == 1 + assert model.calls[0][0] == [0] + assert { + "history_mapped_ids", + "sessions", + "session_indices", + "extra_data", + } <= model.calls[0][1].keys() + assert not hasattr(result, "semantic_id_trace") + + +def test_missing_candidates_cannot_get_credit_from_score_fillers(): + model = FixedSID(predicted=(1,), scores=(-1e10, -1e10, 3.0)) + with pytest.warns(UserWarning, match="1 evaluation queries returned fewer than 2"): + result, _ = evaluation().evaluate(model, [Recall(2)], False) + assert result.metric_avg_results["SID-Recall@2"] == 0.0 + assert result.metric_avg_results["CCE-Recall@2"] == 0.0 + + +def test_invalid_and_duplicate_predictions_compact_before_cutoff(): + model = FixedSID(predicted=(9, 1, 1, 0, 0)) + result, _ = evaluation().evaluate(model, [Recall(2)], False) + assert result.metric_avg_results["SID-Recall@2"] == 1.0 + assert result.metric_avg_results["CCE-Recall@2"] == 0.5 + + +@pytest.mark.parametrize("exclude", [False, True]) +def test_candidate_filtering_changes_bucket_sizes_and_removes_unknown_only_ids(exclude): + ev = NextItemEvaluation.from_splits( + [("u", "tr", "a", 0), ("u", "tr", "b", 1)], + [("u", "te", "x", 2), ("u", "te", "a", 3), ("u", "te", "b", 4)], + fmt="USIT", + exclude_unknowns=False, + semantic_id_metrics=True, + ) + # Keep the full catalog available, then restrict the evaluated candidates. + ev.exclude_unknowns = exclude + model = FixedSID(codes=(0, 0, 0), predicted=(0,), scores=(3.0, 2.0, 1.0)) + result, _ = ev.evaluate(model, [Recall(1)], False) + assert result.metric_avg_results["CCE-Recall@1"] == pytest.approx( + 1 / (2 if exclude else 3) + ) + # An unknown-only bucket consumes no slot when excluded. + model = FixedSID(codes=(0, 0, 1), predicted=(1, 0)) + result, _ = ev.evaluate(model, [Recall(1)], False) + assert result.metric_avg_results["SID-Recall@1"] == float(exclude) + + +@pytest.mark.parametrize("mode,count", [("last", 1), ("next", 2)]) +@pytest.mark.parametrize("user_based", [False, True]) +def test_modes_user_aggregation_and_validation_share_query_counts( + mode, count, user_based +): + train = [("u", "tr", "a", 0), ("u", "tr", "b", 1), ("u", "tr", "c", 2)] + test = [("u", "te", "a", 3), ("u", "te", "b", 4), ("u", "te", "c", 5)] + val = [("u", "va", "a", 6), ("u", "va", "b", 7)] + ev = NextItemEvaluation.from_splits( + train, test, val, fmt="USIT", mode=mode, semantic_id_metrics=True + ) + model = FixedSID() + test_result, val_result = ev.evaluate(model, [Recall(1)], user_based) + assert len(model.calls) == count + 1 + assert "CCE-Recall@1" in test_result.metric_avg_results + assert val_result.metric_avg_results["CCE-Recall@1"] == 0.5 + assert model.calls[-1][0] == [0] + + +def test_user_weighting_matches_ordinary_and_extra_views(): + ev = NextItemEvaluation.from_splits( + [("u", "tr1", "a", 0), ("u", "tr1", "b", 1), ("v", "tr2", "c", 2)], + [ + ("u", "te1", "a", 3), + ("u", "te1", "b", 4), + ("u", "te2", "a", 5), + ("u", "te2", "b", 6), + ("v", "te3", "a", 7), + ("v", "te3", "c", 8), + ], + fmt="USIT", + semantic_id_metrics=True, + ) + query, _ = ev.evaluate(FixedSID(), [Recall(1)], False) + user, _ = ev.evaluate(FixedSID(), [Recall(1)], True) + assert query.metric_avg_results["SID-Recall@1"] == pytest.approx(2 / 3) + assert user.metric_avg_results["SID-Recall@1"] == 0.5 + assert query.metric_avg_results["CCE-Recall@1"] == pytest.approx(1 / 3) + assert user.metric_avg_results["CCE-Recall@1"] == 0.25 + + +def test_only_finite_recall_ndcg_add_extra_views_without_rejecting_other_metrics(): + model = FixedSID() + metrics = [Recall(1), NDCG(1), Recall(), NDCG(), AUC(), MAP(), MRR()] + result, _ = evaluation().evaluate(model, metrics, False) + assert {mt.name for mt in metrics} <= result.metric_avg_results.keys() + assert { + name for name in result.metric_avg_results if name.startswith(("SID-", "CCE-")) + } == {"SID-Recall@1", "SID-NDCG@1", "CCE-Recall@1", "CCE-NDCG@1"} + assert len(model.calls) == 1 + with warnings.catch_warnings(record=True) as caught: + evaluation().evaluate(FixedSID(predicted=()), [MRR()], False) + assert not caught + + +def test_cutoff_lists_and_flag_off_preserve_legacy_item_metrics(): + ev = evaluation(semantic_id_metrics=False) + model = FixedSID() + with pytest.warns(UserWarning, match="disabled"): + result, _ = ev.evaluate(model, [Recall([1, 2])], False) + assert not any( + name.startswith(("SID-", "CCE-")) for name in result.metric_avg_results + ) + enabled, _ = evaluation().evaluate(FixedSID(), [Recall([1, 2])], False) + for name in ("Recall@1", "Recall@2"): + assert enabled.metric_avg_results[name] == result.metric_avg_results[name] + ordinary, _ = evaluation().evaluate(SPop(), [Recall(1)], False) + assert not any( + name.startswith(("SID-", "CCE-")) for name in ordinary.metric_avg_results + ) + + +@pytest.mark.parametrize("reverse", [False, True]) +def test_mixed_result_table_union_and_unavailable_values(reverse): + sid_result, _ = evaluation().evaluate(FixedSID(), [Recall(1)], False) + ordinary, _ = evaluation().evaluate(SPop(), [Recall(1)], False) + results = [sid_result, ordinary] + table = str(ExperimentResult(results[::-1] if reverse else results)) + assert "SID-Recall@1" in table and "CCE-Recall@1" in table + assert table.count("N/A") == 2 + assert "nan" not in table + assert "SID-Recall@1" not in ordinary.metric_avg_results + + +@pytest.mark.parametrize("k", [-1, 1, 2]) +def test_combined_ranking_preserves_full_sid_output_and_legacy_scores(k): + ev = evaluation() + model = FixedSID(predicted=(9, 0, 0, 1)) + model.fit(ev.train_set) + ordinary = model.rank(0, [2, 1, 0], k=k, history_items=[0]) + combined = model.rank_with_semantic_ids(0, [2, 1, 0], k=k, history_items=[0]) + for expected, actual in zip(ordinary, combined): + np.testing.assert_array_equal(expected, actual) + np.testing.assert_array_equal(combined[2], model.predicted) + + +def test_rank_fallback_and_unknown_score_padding(): + ev = evaluation() + model = FixedSID(scores=(2.0, 1.0)) + model.fit(ev.train_set) + model.num_items = 2 + np.testing.assert_array_equal( + model.rank_with_semantic_ids(0, [2, 0], history_items=[0])[1], [1.0, 2.0] + ) + + def fail(*args, **kwargs): + raise ScoreException("no prediction") + + model._score_with_semantic_ids = fail + ordinary = model.rank(0, history_items=[0]) + combined = model.rank_with_semantic_ids(0, history_items=[0]) + np.testing.assert_array_equal(combined[0], ordinary[0]) + np.testing.assert_array_equal(combined[1], ordinary[1]) + assert combined[2].shape == (0, 1) + + +def test_lazy_import_without_computing_backends(): + script = """ +import sys +class BlockBackends: + def find_spec(self, fullname, path=None, target=None): + if fullname.split('.')[0] in {'torch', 'transformers', 'faiss'}: + raise ImportError('backend unavailable') +sys.meta_path.insert(0, BlockBackends()) +from cornac.models import TIGER, LETTER, RPG, DiffGRM, SemanticIDMixin +from cornac.eval_methods import NextItemEvaluation +assert 'torch' not in sys.modules +""" + subprocess.run([sys.executable, "-c", script], check=True, env=os.environ.copy()) + + +def test_score_fallback_has_no_semantic_credit(): + class Unscorable(FixedSID): + def _score_with_semantic_ids(self, *args, **kwargs): + raise ScoreException("no prediction") + + with pytest.warns(UserWarning, match="1 evaluation queries"): + result, _ = evaluation().evaluate(Unscorable(), [Recall(3)], False) + assert result.metric_avg_results["Recall@3"] == 1.0 + assert result.metric_avg_results["SID-Recall@3"] == 0.0 + assert result.metric_avg_results["CCE-Recall@3"] == 0.0 + + +def test_unique_complete_ids_make_all_three_views_equal(): + model = FixedSID(codes=(0, 1, 2), predicted=(0, 1, 2)) + result, _ = evaluation().evaluate(model, [Recall(2), NDCG(2)], False) + for name in ("Recall@2", "NDCG@2"): + assert ( + result.metric_avg_results[name] == result.metric_avg_results["SID-" + name] + ) + assert ( + result.metric_avg_results[name] == result.metric_avg_results["CCE-" + name] + ) + + +def test_empty_output_warns_once_per_split_without_budget_changes(): + train = [("u", "tr", "a", 0), ("u", "tr", "b", 1), ("u", "tr", "c", 2)] + test = [ + ("u", "te1", "a", 3), + ("u", "te1", "b", 4), + ("u", "te2", "a", 5), + ("u", "te2", "c", 6), + ] + ev = NextItemEvaluation.from_splits( + train, test, fmt="USIT", semantic_id_metrics=True + ) + model = FixedSID(predicted=()) + with pytest.warns( + UserWarning, match="2 evaluation queries returned fewer than 2" + ) as caught: + result, _ = ev.evaluate(model, [Recall(2), NDCG(2), Recall()], False) + assert len(caught) == 1 + assert all( + result.metric_avg_results[name] == 0.0 + for name in ("SID-Recall@2", "CCE-Recall@2", "SID-NDCG@2", "CCE-NDCG@2") + ) + assert len(model.calls) == 2 + + +def test_exhaustive_sid_order_uses_max_member_and_lexicographic_ties(): + model = FixedSID(codes=(2, 2, 1), scores=(1.0, 3.0, 3.0)) + np.testing.assert_array_equal( + model._semantic_ids_from_scores(model.scores), [[1], [2]] + ) diff --git a/tests/cornac/models/test_semantic_id_adapters.py b/tests/cornac/models/test_semantic_id_adapters.py new file mode 100644 index 000000000..709a0f6f1 --- /dev/null +++ b/tests/cornac/models/test_semantic_id_adapters.py @@ -0,0 +1,441 @@ +# Copyright 2026 The Cornac Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Semantic ID adapters share decoding without changing native item scoring.""" + +import numpy as np +import pytest + +torch = pytest.importorskip("torch") +pytest.importorskip("transformers") +from cornac.models import TIGER + + +def tiny_tiger(model_class=TIGER): + torch.manual_seed(7) + model = model_class( + rqvae_num_levels=2, + rqvae_codebook_size=2, + n_epochs=0, + d_model=8, + d_ff=16, + num_heads=2, + d_kv=4, + num_enc_layers=1, + num_dec_layers=1, + max_len=2, + dropout=0, + n_beams=2, + device="cpu", + seed=7, + ) + model.num_items = 3 + model.device_ = "cpu" + model._build_semantic_ids(np.array([[0, 0], [0, 0], [1, 1]])) + model._fit_seq2seq(torch, None) + return model + + +def native_tiger_prediction(model, history, budget): + """Direct decoder invocation with master's padding and encoder layout.""" + hist = list(history)[-model.max_len :] + hist = [model.pad_idx] * (model.max_len - len(hist)) + hist + tokens = torch.tensor(model.enc_token_table[hist].reshape(1, -1)) + mask = (tokens != 0).float() + model.model.eval() + with torch.no_grad(): + if model.scoring == "beam": + return model.model.generate_beam( + tokens, mask, budget, model.prefix_children + ) + return model.model.score_all_items( + tokens, mask, torch.tensor(model.sid_table), model.scoring_batch_size + ) + + +def tiny_letter(): + from cornac.data import FeatureModality + from cornac.eval_methods import NextItemEvaluation + from cornac.models import LETTER + + evaluation = NextItemEvaluation.from_splits( + [("u", "tr", "a", 0), ("u", "tr", "b", 1), ("u", "tr", "c", 2)], + [("u", "te", "a", 3), ("u", "te", "b", 4)], + fmt="USIT", + item_feature=FeatureModality(features=np.eye(3), ids=["a", "b", "c"]), + ) + model = LETTER( + precomputed_semantic_ids=np.array([[0, 0], [0, 0], [1, 1]]), + rqvae_num_levels=2, + rqvae_codebook_size=2, + n_epochs=1, + d_model=8, + d_ff=16, + num_heads=2, + d_kv=4, + letter_base_vocab_size=8, + num_enc_layers=1, + num_dec_layers=1, + max_len=2, + dropout=0, + n_beams=2, + device="cpu", + seed=7, + ) + model.fit(evaluation.train_set) + return model, evaluation + + +def tiny_rpg(monkeypatch): + from cornac.data import FeatureModality + from cornac.eval_methods import NextItemEvaluation + from cornac.models import RPG + + evaluation = NextItemEvaluation.from_splits( + [("u", "tr", "a", 0), ("u", "tr", "b", 1), ("u", "tr", "c", 2)], + [("u", "te", "a", 3), ("u", "te", "b", 4)], + fmt="USIT", + item_feature=FeatureModality(features=np.eye(3), ids=["a", "b", "c"]), + ) + model = RPG( + n_codebook=2, + codebook_size=2, + n_epochs=1, + d_model=8, + n_layer=1, + n_head=2, + n_inner=16, + max_len=2, + resid_dropout=0, + embd_dropout=0, + attn_dropout=0, + n_beams=2, + graph_edges=2, + propagation_steps=1, + device="cpu", + seed=7, + ) + # faiss OPQ uses 8-bit PQ and needs >=256 training items; freeze only + # tokenization to keep this real training/graph-building CPU smoke tiny. + with monkeypatch.context() as patch: + patch.setattr( + RPG, + "_opq_tokenize", + lambda self, feats, train_mask: np.array([[0, 0], [0, 0], [1, 1]]), + ) + model.fit(evaluation.train_set) + return model, evaluation + + +class ConstantCPD: + n_digit = 2 + codebook_size = 2 + + def decode_logits(self, memory, padding, partial): + return torch.tensor([[[4.0, 0.0], [3.0, 0.0]]]).expand(len(partial), -1, -1) + + +@pytest.mark.parametrize("diagnostics", [False, True]) +@pytest.mark.parametrize("constrained", [False, True]) +def test_cpd_raw_single_batch_default_parity(diagnostics, constrained): + from cornac.models.diffgrm.diffgrm import cpd_decode, cpd_decode_batch + + memory, padding = torch.zeros(2, 1, 4), torch.zeros(2, 1, dtype=torch.bool) + kwargs = dict( + beam_size=4, + catalog_codes=np.array([[0, 0], [1, 1]]), + constrained=constrained, + return_diagnostics=diagnostics, + ) + default = cpd_decode_batch(ConstantCPD(), memory, padding, **kwargs) + extra = cpd_decode_batch(ConstantCPD(), memory, padding, return_raw=True, **kwargs) + assert extra[:-1] == default + raw = extra[-1] + assert len(raw) == 2 + for entry in raw: + assert ( + len(entry["address_sids"]) + == len(entry["scores"]) + == len(entry["catalog_validity"]) + == 4 + ) + # Parallel assignment orders produce repeated complete paths. + assert len(set(entry["address_sids"])) < len(entry["address_sids"]) + assert entry["catalog_validity"] == [ + sid in {(0, 0), (1, 1)} and np.isfinite(score) + for sid, score in zip(entry["address_sids"], entry["scores"]) + ] + single = cpd_decode( + ConstantCPD(), memory[:1], padding[:1], return_raw=True, **kwargs + ) + assert single == tuple(part[0] for part in extra) + legacy_single = cpd_decode(ConstantCPD(), memory[:1], padding[:1], **kwargs) + assert legacy_single == tuple(part[0] for part in default) + with pytest.raises(ValueError, match="one encoded history"): + cpd_decode(ConstantCPD(), memory, padding, **kwargs) + + +def test_cpd_raw_marks_nonfinite_and_invalid_slots(): + from cornac.models.diffgrm.diffgrm import cpd_decode + + memory, padding = torch.zeros(1, 1, 4), torch.zeros(1, 1, dtype=torch.bool) + codes, scores, raw = cpd_decode( + ConstantCPD(), + memory, + padding, + beam_size=16, + catalog_codes=np.array([[1, 1]]), + return_raw=True, + ) + assert len(raw["address_sids"]) > len(codes) + assert any(not valid for valid in raw["catalog_validity"]) + for sid, score, valid in zip( + raw["address_sids"], raw["scores"], raw["catalog_validity"] + ): + assert valid == (sid == (1, 1) and -1 not in sid and np.isfinite(score)) + with pytest.raises(ValueError, match="permutation"): + cpd_decode(ConstantCPD(), memory, padding, beam_size=2, digit_order=[0, 0]) + + +def tiny_diffgrm(): + from cornac.eval_methods import NextItemEvaluation + from cornac.models import DiffGRM + + evaluation = NextItemEvaluation.from_splits( + [("u", "tr", "a", 0), ("u", "tr", "b", 1), ("u", "tr", "c", 2)], + [("u", "te", "a", 3), ("u", "te", "b", 4)], + fmt="USIT", + ) + model = DiffGRM( + item_sids=np.array([[0, 0], [0, 0], [1, 1]]), + n_digit=2, + codebook_size=2, + d_model=8, + encoder_n_layer=1, + decoder_n_layer=1, + n_head=2, + n_inner=16, + dropout=0, + max_len=2, + min_history=1, + n_epochs=1, + n_views=1, + beam_size=4, + lr_schedule="constant", + device="cpu", + seed=7, + ) + model.fit(evaluation.train_set) + return model, evaluation + + +@pytest.mark.parametrize("scoring", ["beam", "exact"]) +def test_tiger_native_item_parity_and_full_ids(scoring, tmp_path): + model = tiny_tiger() + model.scoring = scoring + native = native_tiger_prediction(model, [0], model.n_beams) + if scoring == "beam": + sids, scores = native + expected = np.full(3, -1e10) + for sid, score in zip(sids, scores): + expected[model.sid_to_item[sid]] = score + else: + expected = native + sids = model._semantic_ids_from_scores(expected) + combined = model.rank_with_semantic_ids(0, np.arange(3), history_items=[0]) + np.testing.assert_allclose(combined[1], expected) + np.testing.assert_array_equal(combined[2], sids) + np.testing.assert_allclose(model.score(0, [0]), expected) + assert not np.array_equal( + model.semantic_id_catalog()[0], model.semantic_id_catalog()[1] + ) + restored = model.load(model.save(str(tmp_path))) + np.testing.assert_array_equal( + restored.rank_with_semantic_ids(0, history_items=[0])[2], combined[2] + ) + assert model.rank_with_semantic_ids(0, history_items=[])[2].shape == (0, 3) + + +@pytest.mark.parametrize("scoring", ["beam", "exact"]) +def test_letter_native_item_parity_and_collision_catalog(scoring): + model, evaluation = tiny_letter() + model.scoring = scoring + inputs, mask = model._encoder_batch(torch, [[0]]) + model.model.eval() + with torch.no_grad(): + if scoring == "beam": + sids, scores = model.model.generate_beam( + inputs, mask, model.n_beams, model.prefix_children + ) + expected = np.full(3, -1e10) + for sid, score in zip(sids, scores): + expected[model.sid_to_item[sid]] = score + else: + expected = model.model.score_all_items( + inputs, mask, torch.tensor(model.sid_table), model.scoring_batch_size + ) + sids = model._semantic_ids_from_scores(expected) + combined = model.rank_with_semantic_ids(0, np.arange(3), history_items=[0]) + np.testing.assert_allclose(combined[1], expected) + np.testing.assert_array_equal(combined[2], sids) + np.testing.assert_allclose(model.score(0, [0]), expected) + np.testing.assert_array_equal( + model.semantic_id_catalog()[0], model.semantic_id_catalog()[1] + ) + assert model.rank_with_semantic_ids(0, history_items=[])[2].shape == (0, 2) + + +@pytest.mark.parametrize("kind", ["tiger", "letter"]) +def test_beam_single_inference_keeps_raw_rows_and_configured_budget(kind, monkeypatch): + model = tiny_tiger() if kind == "tiger" else tiny_letter()[0] + sid = tuple(model.sid_table[0]) + invalid = (9,) * len(sid) + output = [sid, invalid, sid] + calls = [] + + def decode(tokens, mask, budget, children): + calls.append(budget) + return output, [-1.0, -2.0, -3.0] + + monkeypatch.setattr(model.model, "generate_beam", decode) + combined = model.rank_with_semantic_ids(0, [0, 1, 2], k=1, history_items=[0]) + assert calls == [model.n_beams] + np.testing.assert_array_equal(combined[2], output) + + +@pytest.mark.parametrize("scoring", ["graph", "exact"]) +def test_rpg_native_parity_and_single_rng_advance(monkeypatch, scoring, tmp_path): + from cornac.models.rpg.rpg import graph_decode + + model, evaluation = tiny_rpg(monkeypatch) + model.scoring = scoring + state = model.graph_rng.get_state() + model.model.eval() + with torch.no_grad(): + inputs = torch.tensor([[model.pad_idx, 0]]) + logits = model.model.next_item_logits( + inputs, (inputs != model.pad_idx).float() + )[0] + if scoring == "graph": + candidates, scores = graph_decode( + logits.reshape(-1).numpy(), + model.adjacency, + model.item_tokens, + model.n_beams, + model.propagation_steps, + model.graph_rng, + ) + after_native = model.graph_rng.get_state() + expected = np.full(3, -1e10) + expected[candidates] = scores + expected_sids = model.sid_table[candidates] + model.graph_rng.set_state(state) + else: + expected = model._exact_item_scores(logits)[0].numpy().astype(float) + expected_sids = model._semantic_ids_from_scores(expected) + combined = model.rank_with_semantic_ids(0, np.arange(3), history_items=[0]) + np.testing.assert_allclose(combined[1], expected) + np.testing.assert_array_equal(combined[2], expected_sids) + if scoring == "graph": + assert all( + np.array_equal(a, b) + for a, b in zip(after_native, model.graph_rng.get_state()) + ) + model.graph_rng.set_state(state) + ordinary = model.rank(0, np.arange(3), history_items=[0]) + np.testing.assert_array_equal(combined[0], ordinary[0]) + np.testing.assert_allclose(combined[1], ordinary[1]) + restored = model.load(model.save(str(tmp_path))) + np.testing.assert_array_equal( + restored.rank_with_semantic_ids(0, history_items=[0])[2], + model.rank_with_semantic_ids(0, history_items=[0])[2], + ) + assert model.rank_with_semantic_ids(0, history_items=[])[2].shape == (0, 2) + + +@pytest.mark.parametrize("scoring", ["released", "paper", "catalog", "fixed"]) +@pytest.mark.parametrize("policy", ["all", "first", "last"]) +def test_diffgrm_native_parity_and_raw_sids(scoring, policy): + from cornac.models.diffgrm.diffgrm import cpd_decode + + model, evaluation = tiny_diffgrm() + model.scoring = scoring + model.collision_policy = policy + model.model.eval() + with torch.no_grad(): + inputs = torch.tensor([[0, model.pad_idx]]) + memory, padding = model.model.encode_history(inputs, torch.tensor([[1.0, 0.0]])) + codes, scores, diagnostics, raw = cpd_decode( + model.model, + memory, + padding, + beam_size=model.beam_size, + catalog_codes=model.sid_table, + valid_code_set=model.sid_to_items_, + constrained=scoring == "catalog", + digit_order=model.fixed_decode_order_ if scoring == "fixed" else None, + greedy_final=scoring == "released", + return_diagnostics=True, + return_raw=True, + ) + expected = np.full(3, -1e10) + for sid, score in zip(codes, scores): + members = model.sid_to_items_[sid] + members = ( + members[:1] + if policy == "first" + else members[-1:] if policy == "last" else members + ) + expected[members] = score + combined = model.rank_with_semantic_ids(0, np.arange(3), history_items=[0]) + np.testing.assert_allclose(combined[1], expected) + np.testing.assert_array_equal(combined[2], raw["address_sids"]) + np.testing.assert_allclose(model.score(0, [0]), expected) + assert model.rank_with_semantic_ids(0, history_items=[])[2].shape == (0, 2) + + +def test_diffgrm_single_decode_resolution_independence_and_budget(monkeypatch): + import cornac.models.diffgrm.diffgrm as backend + from cornac.metrics import Recall + + model, evaluation = tiny_diffgrm() + model.trainable = False + original = backend.cpd_decode + calls = [] + + def decode(*args, **kwargs): + calls.append(kwargs["beam_size"]) + return original(*args, **kwargs) + + monkeypatch.setattr(backend, "cpd_decode", decode) + rows = [] + for policy in ("all", "first", "last"): + model.collision_policy = policy + result = evaluation.eval( + model, + evaluation.train_set, + evaluation.test_set, + True, + [Recall(1)], + semantic_id_metrics=True, + ) + rows.append( + [ + result.metric_avg_results[name] + for name in ("SID-Recall@1", "CCE-Recall@1") + ] + ) + np.testing.assert_allclose(rows, [rows[0]] * 3) + assert calls == [model.beam_size] * 3