diff --git a/CLI-COMMANDS.md b/CLI-COMMANDS.md index 10ec349a..579b10d4 100644 --- a/CLI-COMMANDS.md +++ b/CLI-COMMANDS.md @@ -367,6 +367,12 @@ view. Items left in Trash are cleaned up automatically after 30 days. ### Inspect model evaluations +Compare models with `roboflow eval compare --project chess-pieces --version 131`. +Use `--frontier-metric mAP5095` to select a metric instead of the project default. +Use `--json` for all model metrics, median latency, exclusions, and server-computed frontier membership. +The command is read-only. It needs `model-eval:read` access. +See the [Model Comparison reference](https://docs.roboflow.com/models/evaluate/model-comparison) for the response contract and all command options. + ```bash # List evals in the workspace; filter by project, version, model, or status. roboflow eval list --status done --limit 10 diff --git a/roboflow/adapters/rfapi.py b/roboflow/adapters/rfapi.py index e29b2b2e..2169fb3d 100644 --- a/roboflow/adapters/rfapi.py +++ b/roboflow/adapters/rfapi.py @@ -2110,6 +2110,10 @@ class ModelEvalNotDoneError(RoboflowError): """Raised when reading panel data for an eval whose status is not ``done`` (HTTP 409).""" +class ModelEvalAccessError(RoboflowError): + """Raised when an evaluation read is not authorized (HTTP 401 or 403).""" + + class InvalidSplitError(RoboflowError): """Raised when ``split`` is not one of the accepted values (HTTP 400).""" @@ -2149,6 +2153,8 @@ def _model_eval_error_for(response): "invalid_confidence": InvalidConfidenceError, } cls = cls_by_code.get(code or "") + if response.status_code in (401, 403): + return ModelEvalAccessError(message) if cls is not None: return cls(message) if response.status_code == 404: @@ -2172,6 +2178,22 @@ def _eval_get(api_key, workspace_url, path, params=None): return response.json() +def compare_model_evals( + api_key: str, + workspace_url: str, + *, + project: str, + version: Union[str, int], + frontier_metric: Optional[str] = None, +) -> dict: + return _eval_get( + api_key, + workspace_url, + "/compare", + params={"project": project, "version": version, "frontierMetric": frontier_metric}, + ) + + def list_model_evals( api_key: str, workspace_url: str, diff --git a/roboflow/cli/handlers/eval.py b/roboflow/cli/handlers/eval.py index 0e41cc90..d00fe8ba 100644 --- a/roboflow/cli/handlers/eval.py +++ b/roboflow/cli/handlers/eval.py @@ -24,6 +24,67 @@ # --------------------------------------------------------------------------- +@eval_app.command("compare") +def compare_evals_cmd( + ctx: typer.Context, + project: Annotated[str, typer.Option("-p", "--project", help="Project slug")], + version: Annotated[int, typer.Option("-v", "--version", min=1, help="Dataset version number")], + frontier_metric: Annotated[ + Optional[str], + typer.Option("--frontier-metric", help="Metric for frontier membership (uses the project default if omitted)"), + ] = None, +) -> None: + """Compare test-set accuracy and median latency without starting evaluations.""" + from roboflow.adapters import rfapi + from roboflow.cli._output import output, output_error + from roboflow.cli._table import format_table + + args = ctx_to_args(ctx, project=project, version=version, frontier_metric=frontier_metric) + resolved = _resolve(args) + if not resolved: + return + workspace_url, api_key = resolved + try: + comparison = rfapi.compare_model_evals( + api_key, + workspace_url, + project=project, + version=version, + frontier_metric=frontier_metric, + ) + except Exception as exc: + output_error( + args, + str(exc), + hint="Check the project, version, frontier metric, and workspace access.", + exit_code=_eval_error_exit_code(exc), + ) + return + if args.json: + output(args, comparison) + return + frontier_metric = comparison.get("frontierMetric") + rows = [] + for model in comparison.get("models", []): + accuracy = model.get("metrics", {}).get(frontier_metric) if frontier_metric else None + latency = model.get("medianLatencyMs") + rows.append( + { + "model": model["modelId"], + "accuracy": f"{accuracy * 100:.1f}%" if accuracy is not None else "", + "latency": f"{latency:.2f}" if latency is not None else "", + "frontier": "Yes" if model.get("onFrontier") else "", + "exclusion": model.get("exclusionReason") or "", + } + ) + table = format_table( + rows, + columns=["model", "accuracy", "latency", "frontier", "exclusion"], + headers=["MODEL", frontier_metric or "ACCURACY", "MEDIAN LATENCY (ms)", "FRONTIER", "EXCLUSION"], + ) + output(args, comparison, text=table) + + @eval_app.command("list") def list_evals_cmd( ctx: typer.Context, @@ -171,6 +232,8 @@ def _eval_error_exit_code(exc: Exception) -> int: """ from roboflow.adapters import rfapi + if isinstance(exc, rfapi.ModelEvalAccessError): + return 2 if isinstance(exc, rfapi.ModelEvalNotFoundError): return 3 if isinstance(exc, rfapi.ModelEvalNotDoneError): diff --git a/roboflow/core/workspace.py b/roboflow/core/workspace.py index 69fd055f..d4e3a634 100644 --- a/roboflow/core/workspace.py +++ b/roboflow/core/workspace.py @@ -8,7 +8,7 @@ import tempfile import time import zipfile -from typing import TYPE_CHECKING, Any, Dict, Generator, List, Optional +from typing import TYPE_CHECKING, Any, Dict, Generator, List, Optional, Union import requests from requests.exceptions import HTTPError @@ -1572,6 +1572,32 @@ def upload_vision_event_image( # Model evaluations # ----------------------------------------------------------------- + def compare_model_evaluations( + self, + project: str, + version: Union[str, int], + *, + frontier_metric: Optional[str] = None, + ) -> dict: + """Compare model accuracy and median latency for a dataset version. + + Args: + project: Project URL slug. + version: Dataset version number. + frontier_metric: Metric for frontier membership. The server uses + the project default when this value is not specified. + + Returns: + The public model comparison response. + """ + return rfapi.compare_model_evals( + self.__api_key, + self.url, + project=project, + version=version, + frontier_metric=frontier_metric, + ) + def evals( self, *, diff --git a/tests/adapters/test_rfapi_model_evals.py b/tests/adapters/test_rfapi_model_evals.py index 41cc6942..b758d906 100644 --- a/tests/adapters/test_rfapi_model_evals.py +++ b/tests/adapters/test_rfapi_model_evals.py @@ -220,5 +220,31 @@ def test_non_json_body_falls_back_to_text(self, mock_get): self.assertIn("Bad Gateway", str(ctx.exception)) +class TestCompareModelEvals(unittest.TestCase): + @patch("roboflow.adapters.rfapi.requests.get") + def test_compare_returns_public_result_and_sends_frontier_metric(self, mock_get): + comparison = { + "project": "chess", + "version": "131", + "frontierMetric": "mAP5095", + "availableMetrics": ["mAP", "mAP5095"], + "models": [], + } + mock_get.return_value = _resp(200, comparison) + + result = rfapi.compare_model_evals("k", "ws", project="chess", version="131", frontier_metric="mAP5095") + + self.assertEqual(result, comparison) + mock_get.assert_called_once_with( + f"{API_URL}/ws/model-evals/compare", + params={ + "api_key": "k", + "project": "chess", + "version": "131", + "frontierMetric": "mAP5095", + }, + ) + + if __name__ == "__main__": unittest.main() diff --git a/tests/cli/test_eval_handler.py b/tests/cli/test_eval_handler.py index 0d4a4a30..45fda2bc 100644 --- a/tests/cli/test_eval_handler.py +++ b/tests/cli/test_eval_handler.py @@ -409,5 +409,116 @@ def test_exit_codes(self) -> None: self.assertEqual(_eval_error_exit_code(exc), expected) +class TestEvalCompareCommand(unittest.TestCase): + @patch("roboflow.adapters.rfapi.requests.get") + def test_json_preserves_the_full_comparison(self, mock_get): + from pathlib import Path + from unittest.mock import MagicMock + + comparison = json.loads((Path(__file__).parents[1] / "fixtures/model_eval_comparison.json").read_text()) + mock_get.return_value = MagicMock(status_code=200) + mock_get.return_value.json.return_value = comparison + + result = runner.invoke( + app, + [ + "--api-key", + "k", + "--workspace", + "ws", + "--json", + "eval", + "compare", + "--project", + "chess", + "--version", + "131", + "--frontier-metric", + "mAP5095", + ], + ) + + self.assertEqual(result.exit_code, 0, result.output) + self.assertEqual(json.loads(result.stdout), comparison) + self.assertEqual(mock_get.call_count, 1) + self.assertEqual(mock_get.call_args.kwargs["params"]["frontierMetric"], "mAP5095") + + @patch("roboflow.adapters.rfapi.requests.get") + def test_text_shows_server_frontier_and_exclusion_with_zero_values(self, mock_get): + from unittest.mock import MagicMock + + mock_get.return_value = MagicMock(status_code=200) + mock_get.return_value.json.return_value = { + "project": "chess", + "version": "131", + "frontierMetric": "mAP", + "availableMetrics": ["mAP"], + "models": [ + { + "modelId": "ws/chess-fast", + "evaluationId": "eval-fast", + "metrics": {"mAP": 0}, + "medianLatencyMs": 0, + "onFrontier": True, + "exclusionReason": None, + }, + { + "modelId": "ws/chess-old", + "evaluationId": "eval-old", + "metrics": {"mAP": 0.9}, + "medianLatencyMs": None, + "onFrontier": False, + "exclusionReason": "latency_unavailable", + }, + ], + } + result = runner.invoke( + app, + ["--api-key", "k", "--workspace", "ws", "eval", "compare", "--project", "chess", "--version", "131"], + ) + + self.assertEqual(result.exit_code, 0, result.output) + for text in [ + "MODEL", + "mAP", + "MEDIAN LATENCY (ms)", + "FRONTIER", + "EXCLUSION", + "ws/chess-fast", + "0.0%", + "0.00", + "Yes", + "latency_unavailable", + ]: + self.assertIn(text, result.stdout) + + @patch("roboflow.adapters.rfapi.requests.get") + def test_permission_failure_is_a_structured_auth_error(self, mock_get): + from unittest.mock import MagicMock + + mock_get.return_value = MagicMock(status_code=403, text="Comparison access denied") + mock_get.return_value.json.return_value = {"error": "forbidden", "message": "Comparison access denied"} + result = runner.invoke( + app, + [ + "--api-key", + "k", + "--workspace", + "ws", + "--json", + "eval", + "compare", + "--project", + "chess", + "--version", + "131", + ], + ) + + self.assertEqual(result.exit_code, 2) + self.assertEqual(json.loads(result.stderr)["error"]["message"], "Comparison access denied") + self.assertEqual(result.stdout, "") + + if __name__ == "__main__": unittest.main() diff --git a/tests/fixtures/model_eval_comparison.json b/tests/fixtures/model_eval_comparison.json new file mode 100644 index 00000000..909ea58e --- /dev/null +++ b/tests/fixtures/model_eval_comparison.json @@ -0,0 +1,63 @@ +{ + "project": "chess", + "version": "131", + "frontierMetric": "mAP", + "availableMetrics": [ + "mAP", + "mAP5095", + "mAP75", + "precision", + "recall", + "f1" + ], + "models": [ + { + "modelId": "acme/chess-fast", + "evaluationId": "eval-fast", + "metrics": { + "mAP": 0.8, + "mIoU": null, + "f1": 0.8, + "precision": 0.8, + "recall": 0.8, + "mAP5095": 0.8, + "mAP75": 0.8 + }, + "medianLatencyMs": 3, + "onFrontier": true, + "exclusionReason": null + }, + { + "modelId": "acme/chess-dominated", + "evaluationId": "eval-dominated", + "metrics": { + "mAP": 0.7, + "mIoU": null, + "f1": 0.7, + "precision": 0.7, + "recall": 0.7, + "mAP5095": 0.7, + "mAP75": 0.7 + }, + "medianLatencyMs": 5, + "onFrontier": false, + "exclusionReason": null + }, + { + "modelId": "acme/chess-missing-latency", + "evaluationId": "eval-missing-latency", + "metrics": { + "mAP": 0.9, + "mIoU": null, + "f1": 0.9, + "precision": 0.9, + "recall": 0.9, + "mAP5095": 0.9, + "mAP75": 0.9 + }, + "medianLatencyMs": null, + "onFrontier": false, + "exclusionReason": "latency_unavailable" + } + ] +} diff --git a/tests/test_model_eval.py b/tests/test_model_eval.py index 40072bab..31c5fec6 100644 --- a/tests/test_model_eval.py +++ b/tests/test_model_eval.py @@ -246,6 +246,28 @@ def test_refresh_404_propagates(self, mock_fn): class TestWorkspaceEvalAccessors(unittest.TestCase): + @patch("roboflow.adapters.rfapi.compare_model_evals") + def test_compare_model_evaluations_returns_public_comparison(self, mock_compare): + comparison = { + "project": "chess", + "version": "131", + "frontierMetric": "mAP5095", + "availableMetrics": ["mAP", "mAP5095"], + "models": [], + } + mock_compare.return_value = comparison + + result = _make_workspace().compare_model_evaluations("chess", 131, frontier_metric="mAP5095") + + self.assertEqual(result, comparison) + mock_compare.assert_called_once_with( + "k", + "lee-sandbox", + project="chess", + version=131, + frontier_metric="mAP5095", + ) + @patch("roboflow.adapters.rfapi.list_model_evals") def test_evals_returns_modeleval_instances(self, mock_list): from roboflow.core.model_eval import ModelEval