Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions plexe/CODE_INDEX.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# Code Index: plexe

> Generated on 2026-03-05 21:32:55
> Generated on 2026-10-10 04:54:35

Code structure and public interface documentation for the **plexe** package.

Expand Down Expand Up @@ -225,8 +225,8 @@ Helper functions for workflow.
- `metric_requires_probabilities(metric_name: str) -> bool` - Return True when a metric requires probability scores instead of hard labels.
- `normalize_probability_predictions(y_true: np.ndarray, y_pred_proba: Any, metric_name: str) -> np.ndarray` - Normalize probability predictions for sklearn metric compatibility.
- `evaluate_on_sample(spark: SparkSession, sample_uri: str, model_artifacts_path: Path, model_type: str, metric: str, target_columns: list[str], group_column: str | None, train_sample_uri: str | None) -> tuple[float, float | None]` - Evaluate model on validation sample, optionally also on training sample.
- `compute_metric_hardcoded(y_true, y_pred, metric_name: str) -> float` - Compute metric using hardcoded sklearn implementations.
- `compute_metric(y_true, y_pred, metric_name: str, group_ids) -> float` - Compute metric value.
- `compute_metric_hardcoded(y_true, y_pred, metric_name: str) -> float` - Compute a standard metric and reject undefined or non-finite results.
- `compute_metric(y_true, y_pred, metric_name: str, group_ids) -> float` - Compute a metric and reject undefined or non-finite results.

---
## `integrations/base.py`
Expand Down
46 changes: 44 additions & 2 deletions plexe/helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,13 @@ def normalize_probability_predictions(y_true: np.ndarray, y_pred_proba: Any, met
return probabilities


def _validate_metric_result(value: float, metric_name: str) -> float:
"""Reject metric results that cannot be ranked or reported reliably."""
if not np.isfinite(value):
raise ValueError(f"Metric '{metric_name}' produced a non-finite result ({value}).")
return value


def evaluate_on_sample(
spark: SparkSession,
sample_uri: str,
Expand Down Expand Up @@ -237,7 +244,7 @@ def _evaluate_predictor(
return compute_metric(y.values, predictions, metric, group_ids=group_ids)


def compute_metric_hardcoded(y_true, y_pred, metric_name: str) -> float:
def _compute_metric_hardcoded(y_true, y_pred, metric_name: str) -> float:
"""
Compute metric using hardcoded sklearn implementations.

Expand Down Expand Up @@ -357,7 +364,24 @@ def compute_metric_hardcoded(y_true, y_pred, metric_name: str) -> float:
)


def compute_metric(y_true, y_pred, metric_name: str, group_ids=None) -> float:
def compute_metric_hardcoded(y_true, y_pred, metric_name: str) -> float:
"""Compute a standard metric and reject undefined or non-finite results.

Args:
y_true: True labels or values.
y_pred: Predicted labels or values.
metric_name: Metric name.

Returns:
A finite metric value. A score of 0.0 is valid.

Raises:
ValueError: If the metric is unsupported or produces a non-finite result.
"""
return _validate_metric_result(_compute_metric_hardcoded(y_true, y_pred, metric_name), metric_name)


def _compute_metric(y_true, y_pred, metric_name: str, group_ids=None) -> float:
"""
Compute metric value.

Expand Down Expand Up @@ -471,3 +495,21 @@ def compute_metric(y_true, y_pred, metric_name: str, group_ids=None) -> float:

# Non-ranking metrics use existing implementation
return compute_metric_hardcoded(y_true, y_pred, metric_name)


def compute_metric(y_true, y_pred, metric_name: str, group_ids=None) -> float:
"""Compute a metric and reject undefined or non-finite results.

Args:
y_true: True labels or values.
y_pred: Predicted labels or values.
metric_name: Metric name.
group_ids: Optional group IDs for ranking metrics (query_id, session_id).

Returns:
A finite metric value. A score of 0.0 is valid.

Raises:
ValueError: If the metric produces a non-finite result.
"""
return _validate_metric_result(_compute_metric(y_true, y_pred, metric_name, group_ids), metric_name)
5 changes: 4 additions & 1 deletion tests/CODE_INDEX.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# Code Index: tests

> Generated on 2026-03-05 21:32:55
> Generated on 2026-10-10 04:54:35

Test suite structure and test case documentation.

Expand Down Expand Up @@ -220,6 +220,9 @@ Unit tests for workflow helper functions.
**Functions:**
- `test_compute_metric_accuracy()` - Test accuracy computation.
- `test_compute_metric_rmse()` - Test RMSE computation.
- `test_metric_helpers_reject_non_finite_results(metric_function)` - An undefined one-row R2 result must not enter workflow state as a score.
- `test_metric_helpers_reject_infinite_results(metric_function, metric_name, y_true, y_pred)` - Overflowed metric calculations must not be returned as valid scores.
- `test_metric_helpers_preserve_zero_as_valid_score(metric_function)` - A zero-valued score is valid and must remain distinct from missing.
- `test_compute_metric_f1_score()` - Test F1 score computation.
- `test_compute_metric_unknown_raises()` - Test unknown metric raises ValueError.
- `test_select_viable_model_types_defaults_image()` - Default model types intersect with IMAGE_PATH.
Expand Down
33 changes: 33 additions & 0 deletions tests/unit/test_helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,14 @@
import numpy as np
import pandas as pd
import pytest
from sklearn.exceptions import UndefinedMetricWarning
from sklearn.metrics import accuracy_score

from plexe.config import DEFAULT_MODEL_TYPES, ModelType, detect_installed_frameworks
from plexe.helpers import (
_evaluate_predictor,
compute_metric,
compute_metric_hardcoded,
metric_requires_probabilities,
normalize_probability_predictions,
select_viable_model_types,
Expand Down Expand Up @@ -45,6 +47,37 @@ def test_compute_metric_rmse():
assert result == pytest.approx(expected, abs=0.01)


@pytest.mark.parametrize("metric_function", [compute_metric, compute_metric_hardcoded])
def test_metric_helpers_reject_non_finite_results(metric_function):
"""An undefined one-row R2 result must not enter workflow state as a score."""
y_true = np.array([1.0])
y_pred = np.array([1.0])

with pytest.warns(UndefinedMetricWarning):
with pytest.raises(ValueError, match="non-finite"):
metric_function(y_true, y_pred, "r2")


@pytest.mark.parametrize(
("metric_function", "metric_name", "y_true", "y_pred"),
[
(compute_metric, "mse", [0.0, 0.0], [1e308, -1e308]),
(compute_metric_hardcoded, "r2", [0.0, 1e154], [1e154, -1e154]),
],
)
def test_metric_helpers_reject_infinite_results(metric_function, metric_name, y_true, y_pred):
"""Overflowed metric calculations must not be returned as valid scores."""
with pytest.warns(RuntimeWarning):
with pytest.raises(ValueError, match="non-finite"):
metric_function(np.array(y_true), np.array(y_pred), metric_name)


@pytest.mark.parametrize("metric_function", [compute_metric, compute_metric_hardcoded])
def test_metric_helpers_preserve_zero_as_valid_score(metric_function):
"""A zero-valued score is valid and must remain distinct from missing."""
assert metric_function([0, 1], [1, 0], "accuracy") == 0.0


def test_compute_metric_f1_score():
"""Test F1 score computation."""
y_true = [0, 1, 0, 1, 0, 1]
Expand Down