diff --git a/plexe/CODE_INDEX.md b/plexe/CODE_INDEX.md index 51b1da24..3b4ec808 100644 --- a/plexe/CODE_INDEX.md +++ b/plexe/CODE_INDEX.md @@ -1,6 +1,6 @@ # Code Index: plexe -> Generated on 2026-03-05 21:32:55 +> Generated on 2026-10-10 04:54:35 Code structure and public interface documentation for the **plexe** package. @@ -225,8 +225,8 @@ Helper functions for workflow. - `metric_requires_probabilities(metric_name: str) -> bool` - Return True when a metric requires probability scores instead of hard labels. - `normalize_probability_predictions(y_true: np.ndarray, y_pred_proba: Any, metric_name: str) -> np.ndarray` - Normalize probability predictions for sklearn metric compatibility. - `evaluate_on_sample(spark: SparkSession, sample_uri: str, model_artifacts_path: Path, model_type: str, metric: str, target_columns: list[str], group_column: str | None, train_sample_uri: str | None) -> tuple[float, float | None]` - Evaluate model on validation sample, optionally also on training sample. -- `compute_metric_hardcoded(y_true, y_pred, metric_name: str) -> float` - Compute metric using hardcoded sklearn implementations. -- `compute_metric(y_true, y_pred, metric_name: str, group_ids) -> float` - Compute metric value. +- `compute_metric_hardcoded(y_true, y_pred, metric_name: str) -> float` - Compute a standard metric and reject undefined or non-finite results. +- `compute_metric(y_true, y_pred, metric_name: str, group_ids) -> float` - Compute a metric and reject undefined or non-finite results. --- ## `integrations/base.py` diff --git a/plexe/helpers.py b/plexe/helpers.py index be7398fd..1e7e0fd9 100644 --- a/plexe/helpers.py +++ b/plexe/helpers.py @@ -138,6 +138,13 @@ def normalize_probability_predictions(y_true: np.ndarray, y_pred_proba: Any, met return probabilities +def _validate_metric_result(value: float, metric_name: str) -> float: + """Reject metric results that cannot be ranked or reported reliably.""" + if not np.isfinite(value): + raise ValueError(f"Metric '{metric_name}' produced a non-finite result ({value}).") + return value + + def evaluate_on_sample( spark: SparkSession, sample_uri: str, @@ -237,7 +244,7 @@ def _evaluate_predictor( return compute_metric(y.values, predictions, metric, group_ids=group_ids) -def compute_metric_hardcoded(y_true, y_pred, metric_name: str) -> float: +def _compute_metric_hardcoded(y_true, y_pred, metric_name: str) -> float: """ Compute metric using hardcoded sklearn implementations. @@ -357,7 +364,24 @@ def compute_metric_hardcoded(y_true, y_pred, metric_name: str) -> float: ) -def compute_metric(y_true, y_pred, metric_name: str, group_ids=None) -> float: +def compute_metric_hardcoded(y_true, y_pred, metric_name: str) -> float: + """Compute a standard metric and reject undefined or non-finite results. + + Args: + y_true: True labels or values. + y_pred: Predicted labels or values. + metric_name: Metric name. + + Returns: + A finite metric value. A score of 0.0 is valid. + + Raises: + ValueError: If the metric is unsupported or produces a non-finite result. + """ + return _validate_metric_result(_compute_metric_hardcoded(y_true, y_pred, metric_name), metric_name) + + +def _compute_metric(y_true, y_pred, metric_name: str, group_ids=None) -> float: """ Compute metric value. @@ -471,3 +495,21 @@ def compute_metric(y_true, y_pred, metric_name: str, group_ids=None) -> float: # Non-ranking metrics use existing implementation return compute_metric_hardcoded(y_true, y_pred, metric_name) + + +def compute_metric(y_true, y_pred, metric_name: str, group_ids=None) -> float: + """Compute a metric and reject undefined or non-finite results. + + Args: + y_true: True labels or values. + y_pred: Predicted labels or values. + metric_name: Metric name. + group_ids: Optional group IDs for ranking metrics (query_id, session_id). + + Returns: + A finite metric value. A score of 0.0 is valid. + + Raises: + ValueError: If the metric produces a non-finite result. + """ + return _validate_metric_result(_compute_metric(y_true, y_pred, metric_name, group_ids), metric_name) diff --git a/tests/CODE_INDEX.md b/tests/CODE_INDEX.md index 1fbe1b68..bac68ca0 100644 --- a/tests/CODE_INDEX.md +++ b/tests/CODE_INDEX.md @@ -1,6 +1,6 @@ # Code Index: tests -> Generated on 2026-03-05 21:32:55 +> Generated on 2026-10-10 04:54:35 Test suite structure and test case documentation. @@ -220,6 +220,9 @@ Unit tests for workflow helper functions. **Functions:** - `test_compute_metric_accuracy()` - Test accuracy computation. - `test_compute_metric_rmse()` - Test RMSE computation. +- `test_metric_helpers_reject_non_finite_results(metric_function)` - An undefined one-row R2 result must not enter workflow state as a score. +- `test_metric_helpers_reject_infinite_results(metric_function, metric_name, y_true, y_pred)` - Overflowed metric calculations must not be returned as valid scores. +- `test_metric_helpers_preserve_zero_as_valid_score(metric_function)` - A zero-valued score is valid and must remain distinct from missing. - `test_compute_metric_f1_score()` - Test F1 score computation. - `test_compute_metric_unknown_raises()` - Test unknown metric raises ValueError. - `test_select_viable_model_types_defaults_image()` - Default model types intersect with IMAGE_PATH. diff --git a/tests/unit/test_helpers.py b/tests/unit/test_helpers.py index 1118653b..5358aab4 100644 --- a/tests/unit/test_helpers.py +++ b/tests/unit/test_helpers.py @@ -5,12 +5,14 @@ import numpy as np import pandas as pd import pytest +from sklearn.exceptions import UndefinedMetricWarning from sklearn.metrics import accuracy_score from plexe.config import DEFAULT_MODEL_TYPES, ModelType, detect_installed_frameworks from plexe.helpers import ( _evaluate_predictor, compute_metric, + compute_metric_hardcoded, metric_requires_probabilities, normalize_probability_predictions, select_viable_model_types, @@ -45,6 +47,37 @@ def test_compute_metric_rmse(): assert result == pytest.approx(expected, abs=0.01) +@pytest.mark.parametrize("metric_function", [compute_metric, compute_metric_hardcoded]) +def test_metric_helpers_reject_non_finite_results(metric_function): + """An undefined one-row R2 result must not enter workflow state as a score.""" + y_true = np.array([1.0]) + y_pred = np.array([1.0]) + + with pytest.warns(UndefinedMetricWarning): + with pytest.raises(ValueError, match="non-finite"): + metric_function(y_true, y_pred, "r2") + + +@pytest.mark.parametrize( + ("metric_function", "metric_name", "y_true", "y_pred"), + [ + (compute_metric, "mse", [0.0, 0.0], [1e308, -1e308]), + (compute_metric_hardcoded, "r2", [0.0, 1e154], [1e154, -1e154]), + ], +) +def test_metric_helpers_reject_infinite_results(metric_function, metric_name, y_true, y_pred): + """Overflowed metric calculations must not be returned as valid scores.""" + with pytest.warns(RuntimeWarning): + with pytest.raises(ValueError, match="non-finite"): + metric_function(np.array(y_true), np.array(y_pred), metric_name) + + +@pytest.mark.parametrize("metric_function", [compute_metric, compute_metric_hardcoded]) +def test_metric_helpers_preserve_zero_as_valid_score(metric_function): + """A zero-valued score is valid and must remain distinct from missing.""" + assert metric_function([0, 1], [1, 0], "accuracy") == 0.0 + + def test_compute_metric_f1_score(): """Test F1 score computation.""" y_true = [0, 1, 0, 1, 0, 1]