feat: compare version model evaluations in the CLI

CLI-COMMANDS.md

@@ -367,6 +367,12 @@ view. Items left in Trash are cleaned up automatically after 30 days.
 
 ### Inspect model evaluations
 
+Compare models with `roboflow eval compare --project chess-pieces --version 131`.
+Use `--frontier-metric mAP5095` to select a metric instead of the project default.
+Use `--json` for all model metrics, median latency, exclusions, and server-computed frontier membership.
+The command is read-only. It needs `model-eval:read` access.
+See the [Model Comparison reference](https://docs.roboflow.com/models/evaluate/model-comparison) for the response contract and all command options.
+
 ```bash
 # List evals in the workspace; filter by project, version, model, or status.
 roboflow eval list --status done --limit 10

roboflow/adapters/rfapi.py

@@ -2110,6 +2110,10 @@ class ModelEvalNotDoneError(RoboflowError):
     """Raised when reading panel data for an eval whose status is not ``done`` (HTTP 409)."""
 
 
+class ModelEvalAccessError(RoboflowError):
+    """Raised when an evaluation read is not authorized (HTTP 401 or 403)."""
+
+
 class InvalidSplitError(RoboflowError):
     """Raised when ``split`` is not one of the accepted values (HTTP 400)."""
 
@@ -2149,6 +2153,8 @@ def _model_eval_error_for(response):
         "invalid_confidence": InvalidConfidenceError,
     }
     cls = cls_by_code.get(code or "")
+    if response.status_code in (401, 403):
+        return ModelEvalAccessError(message)
     if cls is not None:
         return cls(message)
     if response.status_code == 404:
@@ -2172,6 +2178,22 @@ def _eval_get(api_key, workspace_url, path, params=None):
     return response.json()
 
 
+def compare_model_evals(
+    api_key: str,
+    workspace_url: str,
+    *,
+    project: str,
+    version: Union[str, int],
+    frontier_metric: Optional[str] = None,
+) -> dict:
+    return _eval_get(
+        api_key,
+        workspace_url,
+        "/compare",
+        params={"project": project, "version": version, "frontierMetric": frontier_metric},
+    )
+
+
 def list_model_evals(
     api_key: str,
     workspace_url: str,

roboflow/cli/handlers/eval.py

@@ -24,6 +24,67 @@
 # ---------------------------------------------------------------------------
 
 
+@eval_app.command("compare")
+def compare_evals_cmd(
+    ctx: typer.Context,
+    project: Annotated[str, typer.Option("-p", "--project", help="Project slug")],
+    version: Annotated[int, typer.Option("-v", "--version", min=1, help="Dataset version number")],
+    frontier_metric: Annotated[
+        Optional[str],
+        typer.Option("--frontier-metric", help="Metric for frontier membership (uses the project default if omitted)"),
+    ] = None,
+) -> None:
+    """Compare test-set accuracy and median latency without starting evaluations."""
+    from roboflow.adapters import rfapi
+    from roboflow.cli._output import output, output_error
+    from roboflow.cli._table import format_table
+
+    args = ctx_to_args(ctx, project=project, version=version, frontier_metric=frontier_metric)
+    resolved = _resolve(args)
+    if not resolved:
+        return
+    workspace_url, api_key = resolved
+    try:
+        comparison = rfapi.compare_model_evals(
+            api_key,
+            workspace_url,
+            project=project,
+            version=version,
+            frontier_metric=frontier_metric,
+        )
+    except Exception as exc:
+        output_error(
+            args,
+            str(exc),
+            hint="Check the project, version, frontier metric, and workspace access.",
+            exit_code=_eval_error_exit_code(exc),
+        )
+        return
+    if args.json:
+        output(args, comparison)
+        return
+    frontier_metric = comparison.get("frontierMetric")
+    rows = []
+    for model in comparison.get("models", []):
+        accuracy = model.get("metrics", {}).get(frontier_metric) if frontier_metric else None
+        latency = model.get("medianLatencyMs")
+        rows.append(
+            {
+                "model": model["modelId"],
+                "accuracy": f"{accuracy * 100:.1f}%" if accuracy is not None else "",
+                "latency": f"{latency:.2f}" if latency is not None else "",
+                "frontier": "Yes" if model.get("onFrontier") else "",
+                "exclusion": model.get("exclusionReason") or "",
+            }
+        )
+    table = format_table(
+        rows,
+        columns=["model", "accuracy", "latency", "frontier", "exclusion"],
+        headers=["MODEL", frontier_metric or "ACCURACY", "MEDIAN LATENCY (ms)", "FRONTIER", "EXCLUSION"],
+    )
+    output(args, comparison, text=table)
+
+
 @eval_app.command("list")
 def list_evals_cmd(
     ctx: typer.Context,
@@ -171,6 +232,8 @@ def _eval_error_exit_code(exc: Exception) -> int:
     """
     from roboflow.adapters import rfapi
 
+    if isinstance(exc, rfapi.ModelEvalAccessError):
+        return 2
     if isinstance(exc, rfapi.ModelEvalNotFoundError):
         return 3
     if isinstance(exc, rfapi.ModelEvalNotDoneError):

roboflow/core/workspace.py

@@ -8,7 +8,7 @@
 import tempfile
 import time
 import zipfile
-from typing import TYPE_CHECKING, Any, Dict, Generator, List, Optional
+from typing import TYPE_CHECKING, Any, Dict, Generator, List, Optional, Union
 
 import requests
 from requests.exceptions import HTTPError
@@ -1572,6 +1572,32 @@ def upload_vision_event_image(
     # Model evaluations
     # -----------------------------------------------------------------
 
+    def compare_model_evaluations(
+        self,
+        project: str,
+        version: Union[str, int],
+        *,
+        frontier_metric: Optional[str] = None,
+    ) -> dict:
+        """Compare model accuracy and median latency for a dataset version.
+
+        Args:
+            project: Project URL slug.
+            version: Dataset version number.
+            frontier_metric: Metric for frontier membership. The server uses
+                the project default when this value is not specified.
+
+        Returns:
+            The public model comparison response.
+        """
+        return rfapi.compare_model_evals(
+            self.__api_key,
+            self.url,
+            project=project,
+            version=version,
+            frontier_metric=frontier_metric,
+        )
+
     def evals(
         self,
         *,

tests/adapters/test_rfapi_model_evals.py

@@ -220,5 +220,31 @@ def test_non_json_body_falls_back_to_text(self, mock_get):
         self.assertIn("Bad Gateway", str(ctx.exception))
 
 
+class TestCompareModelEvals(unittest.TestCase):
+    @patch("roboflow.adapters.rfapi.requests.get")
+    def test_compare_returns_public_result_and_sends_frontier_metric(self, mock_get):
+        comparison = {
+            "project": "chess",
+            "version": "131",
+            "frontierMetric": "mAP5095",
+            "availableMetrics": ["mAP", "mAP5095"],
+            "models": [],
+        }
+        mock_get.return_value = _resp(200, comparison)
+
+        result = rfapi.compare_model_evals("k", "ws", project="chess", version="131", frontier_metric="mAP5095")
+
+        self.assertEqual(result, comparison)
+        mock_get.assert_called_once_with(
+            f"{API_URL}/ws/model-evals/compare",
+            params={
+                "api_key": "k",
+                "project": "chess",
+                "version": "131",
+                "frontierMetric": "mAP5095",
+            },
+        )
+
+
 if __name__ == "__main__":
     unittest.main()

tests/cli/test_eval_handler.py

@@ -409,5 +409,116 @@ def test_exit_codes(self) -> None:
                 self.assertEqual(_eval_error_exit_code(exc), expected)
 
 
+class TestEvalCompareCommand(unittest.TestCase):
+    @patch("roboflow.adapters.rfapi.requests.get")
+    def test_json_preserves_the_full_comparison(self, mock_get):
+        from pathlib import Path
+        from unittest.mock import MagicMock
+
+        comparison = json.loads((Path(__file__).parents[1] / "fixtures/model_eval_comparison.json").read_text())
+        mock_get.return_value = MagicMock(status_code=200)
+        mock_get.return_value.json.return_value = comparison
+
+        result = runner.invoke(
+            app,
+            [
+                "--api-key",
+                "k",
+                "--workspace",
+                "ws",
+                "--json",
+                "eval",
+                "compare",
+                "--project",
+                "chess",
+                "--version",
+                "131",
+                "--frontier-metric",
+                "mAP5095",
+            ],
+        )
+
+        self.assertEqual(result.exit_code, 0, result.output)
+        self.assertEqual(json.loads(result.stdout), comparison)
+        self.assertEqual(mock_get.call_count, 1)
+        self.assertEqual(mock_get.call_args.kwargs["params"]["frontierMetric"], "mAP5095")
+
+    @patch("roboflow.adapters.rfapi.requests.get")
+    def test_text_shows_server_frontier_and_exclusion_with_zero_values(self, mock_get):
+        from unittest.mock import MagicMock
+
+        mock_get.return_value = MagicMock(status_code=200)
+        mock_get.return_value.json.return_value = {
+            "project": "chess",
+            "version": "131",
+            "frontierMetric": "mAP",
+            "availableMetrics": ["mAP"],
+            "models": [
+                {
+                    "modelId": "ws/chess-fast",
+                    "evaluationId": "eval-fast",
+                    "metrics": {"mAP": 0},
+                    "medianLatencyMs": 0,
+                    "onFrontier": True,
+                    "exclusionReason": None,
+                },
+                {
+                    "modelId": "ws/chess-old",
+                    "evaluationId": "eval-old",
+                    "metrics": {"mAP": 0.9},
+                    "medianLatencyMs": None,
+                    "onFrontier": False,
+                    "exclusionReason": "latency_unavailable",
+                },
+            ],
+        }
+        result = runner.invoke(
+            app,
+            ["--api-key", "k", "--workspace", "ws", "eval", "compare", "--project", "chess", "--version", "131"],
+        )
+
+        self.assertEqual(result.exit_code, 0, result.output)
+        for text in [
+            "MODEL",
+            "mAP",
+            "MEDIAN LATENCY (ms)",
+            "FRONTIER",
+            "EXCLUSION",
+            "ws/chess-fast",
+            "0.0%",
+            "0.00",
+            "Yes",
+            "latency_unavailable",
+        ]:
+            self.assertIn(text, result.stdout)
+
+    @patch("roboflow.adapters.rfapi.requests.get")
+    def test_permission_failure_is_a_structured_auth_error(self, mock_get):
+        from unittest.mock import MagicMock
+
+        mock_get.return_value = MagicMock(status_code=403, text="Comparison access denied")
+        mock_get.return_value.json.return_value = {"error": "forbidden", "message": "Comparison access denied"}
+        result = runner.invoke(
+            app,
+            [
+                "--api-key",
+                "k",
+                "--workspace",
+                "ws",
+                "--json",
+                "eval",
+                "compare",
+                "--project",
+                "chess",
+                "--version",
+                "131",
+            ],
+        )
+
+        self.assertEqual(result.exit_code, 2)
+        self.assertEqual(json.loads(result.stderr)["error"]["message"], "Comparison access denied")
+        self.assertEqual(result.stdout, "")
+
+
 if __name__ == "__main__":
     unittest.main()

tests/fixtures/model_eval_comparison.json

@@ -0,0 +1,63 @@
+{
+    "project": "chess",
+    "version": "131",
+    "frontierMetric": "mAP",
+    "availableMetrics": [
+        "mAP",
+        "mAP5095",
+        "mAP75",
+        "precision",
+        "recall",
+        "f1"
+    ],
+    "models": [
+        {
+            "modelId": "acme/chess-fast",
+            "evaluationId": "eval-fast",
+            "metrics": {
+                "mAP": 0.8,
+                "mIoU": null,
+                "f1": 0.8,
+                "precision": 0.8,
+                "recall": 0.8,
+                "mAP5095": 0.8,
+                "mAP75": 0.8
+            },
+            "medianLatencyMs": 3,
+            "onFrontier": true,
+            "exclusionReason": null
+        },
+        {
+            "modelId": "acme/chess-dominated",
+            "evaluationId": "eval-dominated",
+            "metrics": {
+                "mAP": 0.7,
+                "mIoU": null,
+                "f1": 0.7,
+                "precision": 0.7,
+                "recall": 0.7,
+                "mAP5095": 0.7,
+                "mAP75": 0.7
+            },
+            "medianLatencyMs": 5,
+            "onFrontier": false,
+            "exclusionReason": null
+        },
+        {
+            "modelId": "acme/chess-missing-latency",
+            "evaluationId": "eval-missing-latency",
+            "metrics": {
+                "mAP": 0.9,
+                "mIoU": null,
+                "f1": 0.9,
+                "precision": 0.9,
+                "recall": 0.9,
+                "mAP5095": 0.9,
+                "mAP75": 0.9
+            },
+            "medianLatencyMs": null,
+            "onFrontier": false,
+            "exclusionReason": "latency_unavailable"
+        }
+    ]
+}