From 17cb7950c19f1a90230afb33a4ca07d3f6c50f38 Mon Sep 17 00:00:00 2001 From: lucas-fochesatto Date: Mon, 7 Sep 2026 18:15:25 -0300 Subject: [PATCH 1/7] Add hosted auto-label support to the SDK and CLI Expose the four public auto-label endpoints already used by the Roboflow MCP: list the foundation-model catalog, preview one image for free, start a job over a batch, and poll job progress. - rfapi: list_autolabel_models, preview_autolabel, start_autolabel_job, get_autolabel_job (pass-through, no client-side model whitelist) - Workspace.autolabel_models / autolabel_job - Project.autolabel / autolabel_preview / autolabel_job; Roboflow-trained models are sent as custom_roboflow with modelId in modelOptions - roboflow autolabel models | preview | start | job - util.autolabel_utils shared by SDK and CLI (image payload from URL, local file or base64; model_type resolution) Co-Authored-By: Claude Fable 5.1 --- CHANGELOG.md | 17 ++ CLI-COMMANDS.md | 29 +++ roboflow/adapters/rfapi.py | 106 +++++++++++ roboflow/cli/__init__.py | 2 + roboflow/cli/handlers/autolabel.py | 268 ++++++++++++++++++++++++++++ roboflow/core/project.py | 111 ++++++++++++ roboflow/core/workspace.py | 27 +++ roboflow/util/autolabel_utils.py | 39 ++++ tests/adapters/test_autolabel.py | 107 +++++++++++ tests/cli/test_autolabel_handler.py | 173 ++++++++++++++++++ tests/test_project_autolabel.py | 79 ++++++++ tests/util/test_autolabel_utils.py | 51 ++++++ 12 files changed, 1009 insertions(+) create mode 100644 roboflow/cli/handlers/autolabel.py create mode 100644 roboflow/util/autolabel_utils.py create mode 100644 tests/adapters/test_autolabel.py create mode 100644 tests/cli/test_autolabel_handler.py create mode 100644 tests/test_project_autolabel.py create mode 100644 tests/util/test_autolabel_utils.py diff --git a/CHANGELOG.md b/CHANGELOG.md index bb35f49c..ee016f18 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,23 @@ All notable changes to this project will be documented in this file. +## Unreleased + +### Added + +- Hosted auto-label support, matching the Roboflow MCP tools: + - `Workspace.autolabel_models()` — list the foundation-model catalog + (`gpt-6-astra-boxes`, `sam3-rle`, `gemini-boxes`, ...) with availability, + guidance and credits per image. + - `Project.autolabel_preview(model, image, ontology=...)` — free single-image + preview to compare models before starting a job. `image` accepts an HTTPS + URL, a local file path or a base64 string. + - `Project.autolabel(batch_id, model, model_type="foundational" | "roboflow", ...)` + — start a job over a batch; returns `{jobId, annotationJobId}`. + - `Project.autolabel_job(job_id)` / `Workspace.autolabel_job(job_id)` — poll + per-subjob progress. + - `roboflow autolabel models | preview | start | job` CLI commands. + ## 1.4.1 ### Added diff --git a/CLI-COMMANDS.md b/CLI-COMMANDS.md index 960a4489..70492c5c 100644 --- a/CLI-COMMANDS.md +++ b/CLI-COMMANDS.md @@ -239,6 +239,34 @@ project.accept_annotation_job_images( ) ``` +### Auto-label a batch with a foundation model + +```bash +roboflow autolabel models +roboflow autolabel preview -p my-project -m sam3-rle --image https://example.com/sample.jpg \ + --class cat --class dog +roboflow autolabel start -p my-project --batch-id -m gpt-6-astra-boxes \ + --ontology '{"cat": "a cat", "dog": "a dog"}' --confidence 0.5 --reviewer b@co.com +roboflow autolabel start -p my-project --batch-id -m my-project/3 --model-type roboflow +roboflow autolabel job +``` + +`models` lists the catalog for the workspace (id, availability, credits per +image, default). `preview` runs one image through a model for free so you can +compare candidates before spending credits. `start` creates the job and prints +`jobId` and `annotationJobId`; poll it with `job`. Pass the ontology either as +repeated `--class` flags or as `--ontology` JSON (class name to text prompt). +`--image` accepts an HTTPS URL or a local file. + +The same operations are available in Python: + +```python +models = workspace.autolabel_models()["models"] +preview = project.autolabel_preview("sam3-rle", "sample.jpg", ontology={"cat": "cat"}) +job = project.autolabel("batch-id", model="gpt-6-astra-boxes", ontology={"cat": "a cat"}) +project.autolabel_job(job["jobId"])["status"] +``` + ### RFDM devices (v2 deployments) Workspace-scoped device management — backed by the external Deployments API @@ -457,6 +485,7 @@ Version numbers are always numeric — that's how `x/y` is disambiguated between | `workflow` | Manage workflows | | `folder` | Manage workspace folders | | `annotation` | Annotation batches and jobs | +| `autolabel` | Auto-label batches with hosted foundation or Roboflow models | | `asynctasks` | Inspect async background tasks (e.g. project forks) | | `trash` | List items in Trash | | `universe` | Search Roboflow Universe | diff --git a/roboflow/adapters/rfapi.py b/roboflow/adapters/rfapi.py index 9e9b4fc2..45e1e2bb 100644 --- a/roboflow/adapters/rfapi.py +++ b/roboflow/adapters/rfapi.py @@ -1323,6 +1323,112 @@ def _annotation_administration_response(response): return response.json() +# --------------------------------------------------------------------------- +# Hosted auto-label endpoints +# --------------------------------------------------------------------------- + + +def list_autolabel_models(api_key, workspace_url): + """Fetch the foundation-model catalog for hosted auto-labeling. + + Calls ``GET /:workspace/autolabel/models``. Returns ``{models: [...]}`` + where each entry carries ``id``, ``name``, ``guidance``, ``ontologyFormat``, + ``creditsPerImage``, ``isDefault`` and per-workspace ``available`` (with an + ``unavailableReason`` when the plan blocks a model). + """ + response = requests.get( + f"{API_URL}/{workspace_url}/autolabel/models", + params={"api_key": api_key}, + ) + return _annotation_administration_response(response) + + +def preview_autolabel( + api_key, + workspace_url, + project_url, + *, + model_type, + image, + ontology=None, + confidence_threshold=None, +): + """Preview one image with a foundation model before starting a job. + + Calls ``POST /:workspace/:project/autolabel/preview``. Free: no job is + created and no credits are spent. ``image`` is + ``{"type": "url" | "base64", "value": ...}``. Returns + ``{model, predictions, summary, blockErrors?}``. + """ + payload = {"modelType": model_type, "image": image} + if ontology is not None: + payload["ontology"] = ontology + if confidence_threshold is not None: + payload["confidenceThreshold"] = confidence_threshold + response = requests.post( + f"{API_URL}/{workspace_url}/{project_url}/autolabel/preview", + params={"api_key": api_key}, + json=payload, + ) + return _annotation_administration_response(response) + + +def start_autolabel_job( + api_key, + workspace_url, + project_url, + *, + batch_id, + model_type, + ontology=None, + num_images_to_label=None, + default_confidence=None, + confidence_thresholds=None, + run_nms=None, + reviewer_email=None, + model_options=None, +): + """Start a hosted auto-label job over a batch. + + Calls ``POST /:workspace/:project/autolabel``. ``model_type`` is sent + as-is: a catalog id from ``list_autolabel_models`` (for example + ``gpt-6-astra-boxes`` or ``sam3-rle``) or ``custom_roboflow`` with the + Roboflow model id in ``model_options["modelId"]``. The backend fans + ``default_confidence`` out across the ontology when + ``confidence_thresholds`` is omitted and defaults ``num_images_to_label`` + to the whole batch. Returns ``{jobId, annotationJobId, message}``. + """ + payload = {"batchId": batch_id, "modelType": model_type} + optional = { + "ontology": ontology, + "numImagesToLabel": num_images_to_label, + "defaultConfidence": default_confidence, + "confidenceThresholds": confidence_thresholds, + "runNMS": run_nms, + "reviewerEmail": reviewer_email, + "modelOptions": model_options, + } + payload.update({key: value for key, value in optional.items() if value is not None}) + response = requests.post( + f"{API_URL}/{workspace_url}/{project_url}/autolabel", + params={"api_key": api_key}, + json=payload, + ) + return _annotation_administration_response(response) + + +def get_autolabel_job(api_key, workspace_url, job_id): + """Fetch per-subjob status and progress for a hosted auto-label job. + + Calls ``GET /:workspace/autolabel/jobs/:jobId``. + """ + response = requests.get( + f"{API_URL}/{workspace_url}/autolabel/jobs/{job_id}", + params={"api_key": api_key}, + ) + return _annotation_administration_response(response) + + # --------------------------------------------------------------------------- # Phase 2: Folder (project group) endpoints # --------------------------------------------------------------------------- diff --git a/roboflow/cli/__init__.py b/roboflow/cli/__init__.py index 3befa965..2c2610be 100644 --- a/roboflow/cli/__init__.py +++ b/roboflow/cli/__init__.py @@ -188,6 +188,7 @@ def _walk(group: Any, prefix: str = "") -> None: from roboflow.cli.handlers.api_key import api_key_app # noqa: E402 from roboflow.cli.handlers.asynctasks import asynctasks_app # noqa: E402 from roboflow.cli.handlers.auth import auth_app # noqa: E402 +from roboflow.cli.handlers.autolabel import autolabel_app # noqa: E402 from roboflow.cli.handlers.batch import batch_app # noqa: E402 from roboflow.cli.handlers.completion import completion_app # noqa: E402 from roboflow.cli.handlers.deployment import deployment_app # noqa: E402 @@ -213,6 +214,7 @@ def _walk(group: Any, prefix: str = "") -> None: app.add_typer(api_key_app, name="api-key") app.add_typer(asynctasks_app, name="asynctasks") app.add_typer(auth_app, name="auth") +app.add_typer(autolabel_app, name="autolabel") app.add_typer(batch_app, name="batch") app.add_typer(completion_app, name="completion") app.add_typer(deployment_app, name="deployment") diff --git a/roboflow/cli/handlers/autolabel.py b/roboflow/cli/handlers/autolabel.py new file mode 100644 index 00000000..15d91d54 --- /dev/null +++ b/roboflow/cli/handlers/autolabel.py @@ -0,0 +1,268 @@ +"""Hosted auto-label commands: list models, preview, start and track jobs.""" + +from __future__ import annotations + +import json +from typing import Annotated, Any, Callable, Optional + +import typer + +from roboflow.cli._compat import SortedGroup, ctx_to_args + +autolabel_app = typer.Typer(cls=SortedGroup, help="Hosted auto-label jobs", no_args_is_help=True) + + +@autolabel_app.command("models") +def models(ctx: typer.Context) -> None: + """List the foundation models available for auto-labeling in this workspace.""" + _models(ctx_to_args(ctx)) + + +@autolabel_app.command("preview") +def preview( + ctx: typer.Context, + project: Annotated[str, typer.Option("-p", "--project", help="Project ID")], + model: Annotated[str, typer.Option("-m", "--model", help="Foundation model ID from 'autolabel models'")], + image: Annotated[str, typer.Option("--image", help="Sample image: HTTPS URL or local file path")], + classes: Annotated[ + Optional[list[str]], + typer.Option("--class", help="Class name to detect; repeat for multiple classes"), + ] = None, + ontology: Annotated[ + Optional[str], + typer.Option("--ontology", help='JSON mapping of class name to prompt, e.g. \'{"cat": "a cat"}\''), + ] = None, + confidence: Annotated[ + Optional[float], typer.Option("--confidence", help="Detection threshold from 0.0 to 1.0 (sam3 only)") + ] = None, +) -> None: + """Preview one image with a foundation model. Free: no job is created.""" + args = ctx_to_args(ctx, project=project) + resolved_ontology = _parse_ontology(args, ontology, classes) + if resolved_ontology is _INVALID: + return + from roboflow.util.autolabel_utils import image_payload + + _project_command( + args, + lambda key, workspace, proj: _rfapi().preview_autolabel( + key, + workspace, + proj, + model_type=model, + image=image_payload(image), + ontology=resolved_ontology, + confidence_threshold=confidence, + ), + ) + + +@autolabel_app.command("start") +def start( + ctx: typer.Context, + project: Annotated[str, typer.Option("-p", "--project", help="Project ID")], + batch_id: Annotated[str, typer.Option("--batch-id", help="Source batch ID containing the images to label")], + model: Annotated[ + str, + typer.Option( + "-m", + "--model", + help="Foundation model ID from 'autolabel models', or a Roboflow model ID with --model-type roboflow", + ), + ], + model_type: Annotated[ + str, + typer.Option("--model-type", help="'foundational' (hosted foundation model) or 'roboflow' (trained model)"), + ] = "foundational", + classes: Annotated[ + Optional[list[str]], + typer.Option("--class", help="Class name to label; repeat for multiple classes"), + ] = None, + ontology: Annotated[ + Optional[str], + typer.Option("--ontology", help='JSON mapping of class name to prompt, e.g. \'{"cat": "a cat"}\''), + ] = None, + num_images: Annotated[ + Optional[int], typer.Option("--num-images", help="Number of images to label (default: whole batch)") + ] = None, + confidence: Annotated[ + Optional[float], typer.Option("--confidence", help="Confidence threshold applied to every class") + ] = None, + confidence_thresholds: Annotated[ + Optional[str], + typer.Option("--confidence-thresholds", help="JSON per-class thresholds, e.g. '{\"cat\": 0.5}'"), + ] = None, + no_nms: Annotated[bool, typer.Option("--no-nms", help="Disable non-max suppression")] = False, + reviewer: Annotated[ + Optional[str], typer.Option("--reviewer", help="Reviewer email for the resulting annotation job") + ] = None, + model_options: Annotated[ + Optional[str], + typer.Option("--model-options", help='JSON model options, e.g. \'{"outputFormat": "polygon"}\''), + ] = None, +) -> None: + """Start a hosted auto-label job over a batch of images.""" + args = ctx_to_args(ctx, project=project) + resolved_ontology = _parse_ontology(args, ontology, classes) + if resolved_ontology is _INVALID: + return + resolved_thresholds = _parse_json_option(args, "--confidence-thresholds", confidence_thresholds) + resolved_options = _parse_json_option(args, "--model-options", model_options) + if resolved_thresholds is _INVALID or resolved_options is _INVALID: + return + from roboflow.util.autolabel_utils import resolve_model + + def start_job(key: str, workspace: str, proj: str) -> Any: + wire_model_type, wire_options = resolve_model(model, model_type, resolved_options) + return _rfapi().start_autolabel_job( + key, + workspace, + proj, + batch_id=batch_id, + model_type=wire_model_type, + ontology=resolved_ontology, + num_images_to_label=num_images, + default_confidence=confidence, + confidence_thresholds=resolved_thresholds, + run_nms=False if no_nms else None, + reviewer_email=reviewer, + model_options=wire_options, + ) + + _project_command(args, start_job) + + +@autolabel_app.command("job") +def job( + ctx: typer.Context, + job_id: Annotated[str, typer.Argument(help="Auto-label job ID returned by 'autolabel start'")], +) -> None: + """Get status and per-subjob progress for an auto-label job.""" + args = ctx_to_args(ctx) + _workspace_command(args, lambda key, workspace: _rfapi().get_autolabel_job(key, workspace, job_id)) + + +# --------------------------------------------------------------------------- +# Business logic +# --------------------------------------------------------------------------- + +_INVALID = object() + + +def _rfapi(): + from roboflow.adapters import rfapi + + return rfapi + + +def _parse_json_option(args: Any, flag: str, raw: Optional[str]) -> Any: + from roboflow.cli._output import output_error + + if raw is None: + return None + try: + value = json.loads(raw) + except ValueError: + output_error(args, f"{flag} must be valid JSON.") + return _INVALID + if not isinstance(value, dict): + output_error(args, f"{flag} must be a JSON object.") + return _INVALID + return value + + +def _parse_ontology(args: Any, ontology: Optional[str], classes: Optional[list[str]]) -> Any: + """Build the ontology from --ontology JSON (takes precedence) or repeated --class.""" + if ontology is not None: + return _parse_json_option(args, "--ontology", ontology) + if classes: + return {name: name for name in classes} + return None + + +def _resolve_workspace(args: Any) -> tuple[Optional[str], Optional[str]]: + from roboflow.cli._output import output_error + from roboflow.cli._resolver import resolve_default_workspace + from roboflow.config import load_roboflow_api_key + + workspace_url = args.workspace or resolve_default_workspace(api_key=args.api_key) + if not workspace_url: + output_error(args, "No workspace specified.", hint="Use --workspace or run 'roboflow auth login'.") + return None, None + api_key = args.api_key or load_roboflow_api_key(workspace_url) + if not api_key: + output_error(args, "No API key found.", hint="Set ROBOFLOW_API_KEY or run 'roboflow auth login'.", exit_code=2) + return None, None + return api_key, workspace_url + + +def _resolve_project(args: Any) -> tuple[Optional[str], Optional[str], Optional[str]]: + from roboflow.cli._output import output_error + from roboflow.cli._resolver import resolve_resource + from roboflow.config import load_roboflow_api_key + + try: + workspace, project, _version = resolve_resource(args.project, workspace_override=args.workspace) + except ValueError as exc: + output_error(args, str(exc)) + return None, None, None + api_key = args.api_key or load_roboflow_api_key(workspace) + if not api_key: + output_error(args, "No API key found.", hint="Set ROBOFLOW_API_KEY or run 'roboflow auth login'.", exit_code=2) + return None, None, None + return api_key, workspace, project + + +def _run(args: Any, operation: Callable[[], Any], text: Optional[Callable[[Any], str]] = None) -> None: + from roboflow.cli._output import output, output_api_error, output_error + + try: + data = operation() + except _rfapi().RoboflowError as exc: + output_api_error(args, exc) + return + except ValueError as exc: + output_error(args, str(exc)) + return + output(args, data, text=text(data) if text else None) + + +def _workspace_command(args: Any, operation: Callable[[str, str], Any]) -> None: + api_key, workspace_url = _resolve_workspace(args) + if not workspace_url: + return + _run(args, lambda: operation(api_key, workspace_url)) + + +def _project_command(args: Any, operation: Callable[[str, str, str], Any]) -> None: + api_key, workspace, project = _resolve_project(args) + if not project: + return + _run(args, lambda: operation(api_key, workspace, project)) + + +def _models(args: Any) -> None: + from roboflow.cli._table import format_table + + def table(data: Any) -> str: + rows = [ + { + "id": model.get("id", ""), + "name": model.get("name", ""), + "available": "yes" if model.get("available", True) else "no", + "default": "yes" if model.get("isDefault") else "", + "credits": model.get("creditsPerImage", ""), + "ontology": model.get("ontologyFormat", ""), + } + for model in data.get("models", []) + ] + return format_table( + rows, + columns=["id", "name", "available", "default", "credits", "ontology"], + headers=["ID", "NAME", "AVAILABLE", "DEFAULT", "CREDITS/IMAGE", "ONTOLOGY"], + ) + + api_key, workspace_url = _resolve_workspace(args) + if not workspace_url: + return + _run(args, lambda: _rfapi().list_autolabel_models(api_key, workspace_url), text=table) diff --git a/roboflow/core/project.py b/roboflow/core/project.py index dae10928..5d5cc448 100644 --- a/roboflow/core/project.py +++ b/roboflow/core/project.py @@ -14,6 +14,8 @@ from roboflow.adapters.rfapi import AnnotationSaveError, ImageUploadError from roboflow.config import API_URL, DEMO_KEYS from roboflow.core.version import Version +from roboflow.util.autolabel_utils import image_payload as _autolabel_image_payload +from roboflow.util.autolabel_utils import resolve_model as _resolve_autolabel_model from roboflow.util.general import Retry from roboflow.util.image_utils import load_labelmap @@ -1155,6 +1157,115 @@ def delete_annotation_job_annotations(self, job_id: str) -> Dict: """Delete project annotations from every image assigned to a job.""" return rfapi.delete_annotation_job_annotations(self.__api_key, self.__workspace, self.__project_name, job_id) + def autolabel_preview( + self, + model: str, + image: str, + ontology: Optional[Union[Dict[str, str], List[str]]] = None, + confidence_threshold: Optional[float] = None, + ) -> Dict: + """Preview one image with a foundation model before starting an auto-label job. + + Previews are free: no job is created and no credits are spent. Use it to + compare the candidates from ``Workspace.autolabel_models()`` on a sample + image and start the real job with the winner. + + Args: + model: Foundation model id from ``Workspace.autolabel_models()`` + (e.g. ``"gpt-6-astra-boxes"``, ``"sam3-rle"``, ``"gemini-boxes"``). + image: HTTPS URL, local file path, or base64-encoded image. + ontology: ``{"class name": "text prompt"}`` or a plain list of class + names. Defaults to the dataset's own classes. + confidence_threshold: Detection threshold between 0.0 and 1.0 + (sam3 only; other models report fixed confidence). + + Returns: + Dict: ``{model, predictions, summary, blockErrors?}``. ``summary.byClass`` + carries per-class counts and max confidence, and + ``summary.classesWithNoDetections`` lists requested classes the model + did not find. + """ + return rfapi.preview_autolabel( + self.__api_key, + self.__workspace, + self.__project_name, + model_type=model, + image=_autolabel_image_payload(image), + ontology=ontology, + confidence_threshold=confidence_threshold, + ) + + def autolabel( + self, + batch_id: str, + model: str, + model_type: str = "foundational", + ontology: Optional[Dict[str, str]] = None, + num_images: Optional[int] = None, + confidence: Optional[float] = None, + confidence_thresholds: Optional[Dict[str, float]] = None, + run_nms: Optional[bool] = None, + reviewer_email: Optional[str] = None, + model_options: Optional[Dict] = None, + ) -> Dict: + """Start a hosted auto-label job over a batch of images. + + Args: + batch_id: Source batch containing the images to auto-label. + model: For ``model_type="foundational"``, a model id from + ``Workspace.autolabel_models()`` (e.g. ``"gpt-6-astra-boxes"``, + ``"sam3-rle"``, ``"sam3-polygon"``, ``"gemini-boxes"``). For + ``model_type="roboflow"``, a Roboflow model id such as + ``"project-slug/3"`` or ``"workspace-slug/model-id"``. + model_type: ``"foundational"`` (hosted foundation model, sent as-is; + the backend resolves catalog ids) or ``"roboflow"`` (a + Roboflow-trained model). + ontology: ``{"class name": "text prompt"}``. For models with + ``ontologyFormat="promptMap"`` (sam3) the prompts are sent to + the model; for ``ontologyFormat="classes"`` only the class + names are used. Defaults to the dataset's classes (or the + trained model's classes for ``model_type="roboflow"``). + num_images: Number of images from the batch to label. Defaults to + the whole batch. + confidence: Confidence threshold applied to every class (mirrors + the UI slider). Ignored when ``confidence_thresholds`` is set. + confidence_thresholds: Per-class threshold override, e.g. + ``{"cat": 0.5, "dog": 0.6}``. + run_nms: Whether to run non-max suppression (server default: True). + reviewer_email: Reviewer for the resulting annotation job. Must be a + workspace member; defaults to the workspace owner. + model_options: Model-specific options, e.g. + ``{"outputFormat": "polygon"}`` for segmentation output. + + Returns: + Dict: ``{jobId, annotationJobId, message}``. Poll progress with + ``autolabel_job(jobId)``. + + Example: + >>> job = project.autolabel("batch-id", model="gpt-6-astra-boxes", + ... ontology={"cat": "a cat", "dog": "a dog"}) + >>> project.autolabel_job(job["jobId"])["status"] + """ + wire_model_type, model_options = _resolve_autolabel_model(model, model_type, model_options) + return rfapi.start_autolabel_job( + self.__api_key, + self.__workspace, + self.__project_name, + batch_id=batch_id, + model_type=wire_model_type, + ontology=ontology, + num_images_to_label=num_images, + default_confidence=confidence, + confidence_thresholds=confidence_thresholds, + run_nms=run_nms, + reviewer_email=reviewer_email, + model_options=model_options, + ) + + def autolabel_job(self, job_id: str) -> Dict: + """Get status and per-subjob progress for an auto-label job started with ``autolabel``.""" + return rfapi.get_autolabel_job(self.__api_key, self.__workspace, job_id) + def get_batches(self) -> Dict: """ Get a list of all batches in the project. diff --git a/roboflow/core/workspace.py b/roboflow/core/workspace.py index 7a084102..69fd055f 100644 --- a/roboflow/core/workspace.py +++ b/roboflow/core/workspace.py @@ -1669,6 +1669,33 @@ def restore_from_trash(self, item_type: str, item_id: str, parent_id: Optional[s """ return rfapi.restore_trash_item(self.__api_key, self.url, item_type, item_id, parent_id) + def autolabel_models(self) -> dict: + """ + List the foundation models available for hosted auto-labeling in this workspace. + + Returns ``{models: [...]}``. Each entry's ``id`` (e.g. + ``gpt-6-astra-boxes``, ``sam3-rle``, ``gemini-boxes``) is a valid + ``model`` for ``Project.autolabel`` and ``Project.autolabel_preview``. + Entries carry ``guidance`` on when to prefer each model, the + ``projectTypes`` they support, ``ontologyFormat``, ``creditsPerImage``, + ``isDefault``, and ``available`` with an ``unavailableReason`` when the + workspace plan blocks a model. + + Example: + >>> for m in ws.autolabel_models()["models"]: + ... print(m["id"], m["available"]) + """ + return rfapi.list_autolabel_models(self.__api_key, self.url) + + def autolabel_job(self, job_id: str) -> dict: + """ + Get status and per-subjob progress for a hosted auto-label job. + + Args: + job_id: the ``jobId`` returned by ``Project.autolabel``. + """ + return rfapi.get_autolabel_job(self.__api_key, self.url, job_id) + # Permanent-delete actions (empty trash / delete a single trash item # immediately) are intentionally not exposed in the SDK — they destroy # data irrecoverably and are only available through the web UI's Trash diff --git a/roboflow/util/autolabel_utils.py b/roboflow/util/autolabel_utils.py new file mode 100644 index 00000000..b82d82b1 --- /dev/null +++ b/roboflow/util/autolabel_utils.py @@ -0,0 +1,39 @@ +"""Helpers shared by the SDK and CLI for hosted auto-label requests.""" + +from __future__ import annotations + +import base64 +import os +from typing import Any, Dict, Optional, Tuple + +MODEL_TYPES = ("foundational", "roboflow") + + +def image_payload(image: str) -> Dict[str, str]: + """Build the ``{type, value}`` image payload for the auto-label preview endpoint. + + Accepts an HTTP(S) URL, a local file path (read and base64-encoded), or an + already base64-encoded string. + """ + if image.startswith(("http://", "https://")): + return {"type": "url", "value": image} + if os.path.isfile(image): + with open(image, "rb") as handle: + return {"type": "base64", "value": base64.b64encode(handle.read()).decode("ascii")} + return {"type": "base64", "value": image} + + +def resolve_model( + model: str, model_type: str, model_options: Optional[Dict[str, Any]] = None +) -> Tuple[str, Optional[Dict[str, Any]]]: + """Translate the public ``model``/``model_type`` pair into wire values. + + Foundation models are sent as-is (the backend resolves catalog ids such as + ``gpt-6-astra-boxes``). Roboflow-trained models are sent as + ``custom_roboflow`` with the model id in ``modelOptions.modelId``. + """ + if model_type == "roboflow": + return "custom_roboflow", {**(model_options or {}), "modelId": model} + if model_type == "foundational": + return model, model_options + raise ValueError("model_type must be 'foundational' or 'roboflow'") diff --git a/tests/adapters/test_autolabel.py b/tests/adapters/test_autolabel.py new file mode 100644 index 00000000..9540531d --- /dev/null +++ b/tests/adapters/test_autolabel.py @@ -0,0 +1,107 @@ +"""HTTP contract tests for hosted auto-label adapters.""" + +import unittest +from unittest.mock import MagicMock, patch + +from roboflow.adapters import rfapi + + +def _response(payload=None, status_code=200, text="error"): + return MagicMock(status_code=status_code, text=text, json=lambda: payload or {"success": True}) + + +class TestAutolabelAdapters(unittest.TestCase): + @patch("roboflow.adapters.rfapi.requests.get") + def test_list_models_contract(self, mock_get): + mock_get.return_value = _response({"models": []}) + + self.assertEqual(rfapi.list_autolabel_models("key", "ws"), {"models": []}) + self.assertTrue(mock_get.call_args.args[0].endswith("/ws/autolabel/models")) + self.assertEqual(mock_get.call_args.kwargs["params"], {"api_key": "key"}) + + @patch("roboflow.adapters.rfapi.requests.get") + def test_get_job_contract(self, mock_get): + mock_get.return_value = _response({"status": "running"}) + + self.assertEqual(rfapi.get_autolabel_job("key", "ws", "job-1"), {"status": "running"}) + self.assertTrue(mock_get.call_args.args[0].endswith("/ws/autolabel/jobs/job-1")) + self.assertEqual(mock_get.call_args.kwargs["params"], {"api_key": "key"}) + + @patch("roboflow.adapters.rfapi.requests.post") + def test_preview_contract(self, mock_post): + mock_post.return_value = _response({"predictions": []}) + image = {"type": "url", "value": "https://example.com/cat.jpg"} + + rfapi.preview_autolabel("key", "ws", "proj", model_type="sam3-rle", image=image) + self.assertTrue(mock_post.call_args.args[0].endswith("/ws/proj/autolabel/preview")) + self.assertEqual(mock_post.call_args.kwargs["params"], {"api_key": "key"}) + self.assertEqual(mock_post.call_args.kwargs["json"], {"modelType": "sam3-rle", "image": image}) + + rfapi.preview_autolabel( + "key", + "ws", + "proj", + model_type="sam3-rle", + image=image, + ontology={"cat": "cat"}, + confidence_threshold=0.4, + ) + self.assertEqual( + mock_post.call_args.kwargs["json"], + {"modelType": "sam3-rle", "image": image, "ontology": {"cat": "cat"}, "confidenceThreshold": 0.4}, + ) + + @patch("roboflow.adapters.rfapi.requests.post") + def test_start_job_contract_omits_unset_fields(self, mock_post): + mock_post.return_value = _response({"jobId": "job-1"}) + + result = rfapi.start_autolabel_job("key", "ws", "proj", batch_id="batch-1", model_type="gpt-6-astra-boxes") + self.assertEqual(result, {"jobId": "job-1"}) + self.assertTrue(mock_post.call_args.args[0].endswith("/ws/proj/autolabel")) + self.assertEqual(mock_post.call_args.kwargs["params"], {"api_key": "key"}) + self.assertEqual( + mock_post.call_args.kwargs["json"], + {"batchId": "batch-1", "modelType": "gpt-6-astra-boxes"}, + ) + + @patch("roboflow.adapters.rfapi.requests.post") + def test_start_job_contract_full_payload(self, mock_post): + mock_post.return_value = _response({"jobId": "job-1"}) + + rfapi.start_autolabel_job( + "key", + "ws", + "proj", + batch_id="batch-1", + model_type="custom_roboflow", + ontology={"cat": "a cat"}, + num_images_to_label=10, + default_confidence=0.5, + confidence_thresholds={"cat": 0.6}, + run_nms=False, + reviewer_email="reviewer@example.com", + model_options={"modelId": "proj/3"}, + ) + self.assertEqual( + mock_post.call_args.kwargs["json"], + { + "batchId": "batch-1", + "modelType": "custom_roboflow", + "ontology": {"cat": "a cat"}, + "numImagesToLabel": 10, + "defaultConfidence": 0.5, + "confidenceThresholds": {"cat": 0.6}, + "runNMS": False, + "reviewerEmail": "reviewer@example.com", + "modelOptions": {"modelId": "proj/3"}, + }, + ) + + @patch("roboflow.adapters.rfapi.requests.post") + def test_errors_raise_roboflow_error_with_status(self, mock_post): + mock_post.return_value = _response({"error": {"message": "batch not found"}}, status_code=404) + + with self.assertRaises(rfapi.RoboflowError) as ctx: + rfapi.start_autolabel_job("key", "ws", "proj", batch_id="missing", model_type="sam3-rle") + self.assertEqual(str(ctx.exception), "batch not found") + self.assertEqual(ctx.exception.status_code, 404) diff --git a/tests/cli/test_autolabel_handler.py b/tests/cli/test_autolabel_handler.py new file mode 100644 index 00000000..26ab9d72 --- /dev/null +++ b/tests/cli/test_autolabel_handler.py @@ -0,0 +1,173 @@ +"""Unit tests for roboflow.cli.handlers.autolabel.""" + +import json +import unittest +from unittest.mock import patch + +from typer.testing import CliRunner + +from roboflow.adapters.rfapi import RoboflowError +from roboflow.cli import app + +runner = CliRunner() + +_RESOLVE_PROJECT = "roboflow.cli.handlers.autolabel._resolve_project" +_RESOLVE_WORKSPACE = "roboflow.cli.handlers.autolabel._resolve_workspace" + + +class TestAutolabelRegistration(unittest.TestCase): + def test_subcommands_have_help(self): + for name in ["models", "preview", "start", "job"]: + with self.subTest(command=name): + result = runner.invoke(app, ["autolabel", name, "--help"]) + self.assertEqual(result.exit_code, 0, result.output) + + +class TestAutolabelModels(unittest.TestCase): + @patch("roboflow.adapters.rfapi.list_autolabel_models") + @patch(_RESOLVE_WORKSPACE, return_value=("key", "ws")) + def test_text_output_is_a_table(self, _resolve, mock_api): + mock_api.return_value = { + "models": [ + {"id": "gpt-6-astra-boxes", "name": "GPT-6 Astra", "available": True, "isDefault": True}, + {"id": "gemini-boxes", "name": "Gemini", "available": False, "unavailableReason": "plan"}, + ] + } + result = runner.invoke(app, ["autolabel", "models"]) + self.assertEqual(result.exit_code, 0, result.output) + self.assertIn("gpt-6-astra-boxes", result.output) + self.assertIn("gemini-boxes", result.output) + mock_api.assert_called_once_with("key", "ws") + + @patch("roboflow.adapters.rfapi.list_autolabel_models", return_value={"models": [{"id": "sam3-rle"}]}) + @patch(_RESOLVE_WORKSPACE, return_value=("key", "ws")) + def test_json_output(self, _resolve, _mock_api): + result = runner.invoke(app, ["--json", "autolabel", "models"]) + self.assertEqual(json.loads(result.output), {"models": [{"id": "sam3-rle"}]}) + + +class TestAutolabelPreview(unittest.TestCase): + @patch("roboflow.adapters.rfapi.preview_autolabel", return_value={"summary": {"totalDetections": 2}}) + @patch(_RESOLVE_PROJECT, return_value=("key", "ws", "proj")) + def test_classes_become_identity_ontology(self, _resolve, mock_api): + result = runner.invoke( + app, + [ + "--json", + "autolabel", + "preview", + "-p", + "ws/proj", + "-m", + "sam3-rle", + "--image", + "https://example.com/cat.jpg", + "--class", + "cat", + "--class", + "dog", + "--confidence", + "0.4", + ], + ) + self.assertEqual(result.exit_code, 0, result.output) + self.assertEqual(json.loads(result.output), {"summary": {"totalDetections": 2}}) + mock_api.assert_called_once_with( + "key", + "ws", + "proj", + model_type="sam3-rle", + image={"type": "url", "value": "https://example.com/cat.jpg"}, + ontology={"cat": "cat", "dog": "dog"}, + confidence_threshold=0.4, + ) + + @patch("roboflow.adapters.rfapi.preview_autolabel", return_value={}) + @patch(_RESOLVE_PROJECT, return_value=("key", "ws", "proj")) + def test_ontology_json_takes_precedence_over_classes(self, _resolve, mock_api): + runner.invoke( + app, + ["autolabel", "preview", "-p", "ws/proj", "-m", "sam3-rle", "--image", "https://x/y.jpg"] + + ["--class", "cat", "--ontology", '{"cat": "a tabby cat"}'], + ) + self.assertEqual(mock_api.call_args.kwargs["ontology"], {"cat": "a tabby cat"}) + + @patch("roboflow.adapters.rfapi.preview_autolabel") + @patch(_RESOLVE_PROJECT, return_value=("key", "ws", "proj")) + def test_invalid_ontology_json_fails_without_calling_api(self, _resolve, mock_api): + result = runner.invoke( + app, + ["autolabel", "preview", "-p", "ws/proj", "-m", "sam3-rle", "--image", "https://x/y.jpg"] + + ["--ontology", "not-json"], + ) + self.assertNotEqual(result.exit_code, 0) + mock_api.assert_not_called() + + +class TestAutolabelStart(unittest.TestCase): + @patch("roboflow.adapters.rfapi.start_autolabel_job", return_value={"jobId": "job-1"}) + @patch(_RESOLVE_PROJECT, return_value=("key", "ws", "proj")) + def test_foundational_start(self, _resolve, mock_api): + result = runner.invoke( + app, + ["--json", "autolabel", "start", "-p", "ws/proj", "--batch-id", "batch-1", "-m", "gpt-6-astra-boxes"] + + ["--class", "cat", "--num-images", "10", "--confidence", "0.5", "--no-nms"] + + ["--reviewer", "r@example.com", "--model-options", '{"outputFormat": "polygon"}'], + ) + self.assertEqual(result.exit_code, 0, result.output) + self.assertEqual(json.loads(result.output), {"jobId": "job-1"}) + mock_api.assert_called_once_with( + "key", + "ws", + "proj", + batch_id="batch-1", + model_type="gpt-6-astra-boxes", + ontology={"cat": "cat"}, + num_images_to_label=10, + default_confidence=0.5, + confidence_thresholds=None, + run_nms=False, + reviewer_email="r@example.com", + model_options={"outputFormat": "polygon"}, + ) + + @patch("roboflow.adapters.rfapi.start_autolabel_job", return_value={"jobId": "job-1"}) + @patch(_RESOLVE_PROJECT, return_value=("key", "ws", "proj")) + def test_roboflow_model_type(self, _resolve, mock_api): + result = runner.invoke( + app, + ["autolabel", "start", "-p", "ws/proj", "--batch-id", "batch-1", "-m", "proj/3"] + + ["--model-type", "roboflow", "--confidence-thresholds", '{"cat": 0.6}'], + ) + self.assertEqual(result.exit_code, 0, result.output) + kwargs = mock_api.call_args.kwargs + self.assertEqual(kwargs["model_type"], "custom_roboflow") + self.assertEqual(kwargs["model_options"], {"modelId": "proj/3"}) + self.assertEqual(kwargs["confidence_thresholds"], {"cat": 0.6}) + self.assertIsNone(kwargs["run_nms"]) + + @patch("roboflow.adapters.rfapi.start_autolabel_job") + @patch(_RESOLVE_PROJECT, return_value=("key", "ws", "proj")) + def test_unknown_model_type_is_an_error(self, _resolve, mock_api): + result = runner.invoke( + app, + ["autolabel", "start", "-p", "ws/proj", "--batch-id", "batch-1", "-m", "x", "--model-type", "hosted"], + ) + self.assertNotEqual(result.exit_code, 0) + mock_api.assert_not_called() + + @patch("roboflow.adapters.rfapi.start_autolabel_job", side_effect=RoboflowError("batch not found", status_code=404)) + @patch(_RESOLVE_PROJECT, return_value=("key", "ws", "proj")) + def test_api_error_maps_to_not_found_exit_code(self, _resolve, _mock_api): + result = runner.invoke(app, ["autolabel", "start", "-p", "ws/proj", "--batch-id", "missing", "-m", "sam3-rle"]) + self.assertEqual(result.exit_code, 3, result.output) + + +class TestAutolabelJob(unittest.TestCase): + @patch("roboflow.adapters.rfapi.get_autolabel_job", return_value={"status": "running", "progress": 0.5}) + @patch(_RESOLVE_WORKSPACE, return_value=("key", "ws")) + def test_json_output(self, _resolve, mock_api): + result = runner.invoke(app, ["--json", "autolabel", "job", "job-1"]) + self.assertEqual(result.exit_code, 0, result.output) + self.assertEqual(json.loads(result.output)["status"], "running") + mock_api.assert_called_once_with("key", "ws", "job-1") diff --git a/tests/test_project_autolabel.py b/tests/test_project_autolabel.py new file mode 100644 index 00000000..476f5d22 --- /dev/null +++ b/tests/test_project_autolabel.py @@ -0,0 +1,79 @@ +"""Public Project and Workspace wrapper coverage for hosted auto-label.""" + +from unittest.mock import patch + +from tests import PROJECT_NAME, WORKSPACE_NAME, RoboflowTest + + +class TestProjectAutolabel(RoboflowTest): + @patch("roboflow.adapters.rfapi.start_autolabel_job", return_value={"jobId": "job-1"}) + def test_autolabel_foundational_passes_model_as_is(self, mock_start): + result = self.project.autolabel( + "batch-1", + "gpt-6-astra-boxes", + ontology={"cat": "a cat"}, + num_images=5, + confidence=0.4, + reviewer_email="reviewer@example.com", + ) + + self.assertEqual(result, {"jobId": "job-1"}) + mock_start.assert_called_once_with( + self.rf.api_key, + WORKSPACE_NAME, + PROJECT_NAME, + batch_id="batch-1", + model_type="gpt-6-astra-boxes", + ontology={"cat": "a cat"}, + num_images_to_label=5, + default_confidence=0.4, + confidence_thresholds=None, + run_nms=None, + reviewer_email="reviewer@example.com", + model_options=None, + ) + + @patch("roboflow.adapters.rfapi.start_autolabel_job", return_value={"jobId": "job-1"}) + def test_autolabel_roboflow_model_is_sent_as_custom_roboflow(self, mock_start): + self.project.autolabel("batch-1", "my-project/3", model_type="roboflow", model_options={"outputFormat": "rle"}) + + kwargs = mock_start.call_args.kwargs + self.assertEqual(kwargs["model_type"], "custom_roboflow") + self.assertEqual(kwargs["model_options"], {"outputFormat": "rle", "modelId": "my-project/3"}) + + def test_autolabel_rejects_unknown_model_type(self): + with self.assertRaises(ValueError): + self.project.autolabel("batch-1", "sam3-rle", model_type="hosted") + + @patch("roboflow.adapters.rfapi.preview_autolabel", return_value={"predictions": []}) + def test_autolabel_preview_builds_image_payload(self, mock_preview): + result = self.project.autolabel_preview( + "sam3-rle", + "https://example.com/cat.jpg", + ontology=["cat"], + confidence_threshold=0.3, + ) + + self.assertEqual(result, {"predictions": []}) + mock_preview.assert_called_once_with( + self.rf.api_key, + WORKSPACE_NAME, + PROJECT_NAME, + model_type="sam3-rle", + image={"type": "url", "value": "https://example.com/cat.jpg"}, + ontology=["cat"], + confidence_threshold=0.3, + ) + + @patch("roboflow.adapters.rfapi.get_autolabel_job", return_value={"status": "done"}) + def test_autolabel_job_wrappers_delegate(self, mock_get): + self.assertEqual(self.project.autolabel_job("job-1"), {"status": "done"}) + mock_get.assert_called_with(self.rf.api_key, WORKSPACE_NAME, "job-1") + + self.assertEqual(self.workspace.autolabel_job("job-2"), {"status": "done"}) + mock_get.assert_called_with(self.rf.api_key, WORKSPACE_NAME, "job-2") + + @patch("roboflow.adapters.rfapi.list_autolabel_models", return_value={"models": []}) + def test_workspace_autolabel_models_delegates(self, mock_list): + self.assertEqual(self.workspace.autolabel_models(), {"models": []}) + mock_list.assert_called_once_with(self.rf.api_key, WORKSPACE_NAME) diff --git a/tests/util/test_autolabel_utils.py b/tests/util/test_autolabel_utils.py new file mode 100644 index 00000000..42955b7d --- /dev/null +++ b/tests/util/test_autolabel_utils.py @@ -0,0 +1,51 @@ +"""Unit tests for roboflow.util.autolabel_utils.""" + +import base64 +import os +import tempfile +import unittest + +from roboflow.util.autolabel_utils import image_payload, resolve_model + + +class TestImagePayload(unittest.TestCase): + def test_url(self): + self.assertEqual( + image_payload("https://example.com/cat.jpg"), + {"type": "url", "value": "https://example.com/cat.jpg"}, + ) + + def test_local_file_is_base64_encoded(self): + with tempfile.NamedTemporaryFile(suffix=".jpg", delete=False) as handle: + handle.write(b"fake-image-bytes") + path = handle.name + try: + payload = image_payload(path) + finally: + os.unlink(path) + self.assertEqual(payload["type"], "base64") + self.assertEqual(base64.b64decode(payload["value"]), b"fake-image-bytes") + + def test_other_strings_are_treated_as_base64(self): + encoded = base64.b64encode(b"bytes").decode("ascii") + self.assertEqual(image_payload(encoded), {"type": "base64", "value": encoded}) + + +class TestResolveModel(unittest.TestCase): + def test_foundational_is_pass_through(self): + self.assertEqual(resolve_model("gpt-6-astra-boxes", "foundational"), ("gpt-6-astra-boxes", None)) + self.assertEqual( + resolve_model("sam3-rle", "foundational", {"outputFormat": "rle"}), + ("sam3-rle", {"outputFormat": "rle"}), + ) + + def test_roboflow_model_rides_in_model_options(self): + self.assertEqual( + resolve_model("proj/3", "roboflow", {"outputFormat": "polygon"}), + ("custom_roboflow", {"outputFormat": "polygon", "modelId": "proj/3"}), + ) + self.assertEqual(resolve_model("proj/3", "roboflow"), ("custom_roboflow", {"modelId": "proj/3"})) + + def test_unknown_model_type_raises(self): + with self.assertRaises(ValueError): + resolve_model("x", "hosted") From 65295ddd0063ff0c566e5a5f4b52540d8fa32324 Mon Sep 17 00:00:00 2001 From: lucas-fochesatto Date: Mon, 7 Sep 2026 18:34:40 -0300 Subject: [PATCH 2/7] Narrow Optional credentials in autolabel CLI helpers for mypy Co-Authored-By: Claude Fable 5.1 --- roboflow/cli/handlers/autolabel.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roboflow/cli/handlers/autolabel.py b/roboflow/cli/handlers/autolabel.py index 15d91d54..defd8eba 100644 --- a/roboflow/cli/handlers/autolabel.py +++ b/roboflow/cli/handlers/autolabel.py @@ -229,14 +229,14 @@ def _run(args: Any, operation: Callable[[], Any], text: Optional[Callable[[Any], def _workspace_command(args: Any, operation: Callable[[str, str], Any]) -> None: api_key, workspace_url = _resolve_workspace(args) - if not workspace_url: + if api_key is None or workspace_url is None: return _run(args, lambda: operation(api_key, workspace_url)) def _project_command(args: Any, operation: Callable[[str, str, str], Any]) -> None: api_key, workspace, project = _resolve_project(args) - if not project: + if api_key is None or workspace is None or project is None: return _run(args, lambda: operation(api_key, workspace, project)) From e116738bee16635b1918a5c0a6adfc8ca2d9fa14 Mon Sep 17 00:00:00 2001 From: lucas-fochesatto Date: Mon, 7 Sep 2026 18:58:21 -0300 Subject: [PATCH 3/7] Align autolabel CLI and adapter with repo conventions - JSON options accept @file references via the shared train parser - Auto-label adapters use their own response helper over a generic JSON-or-raise implementation Co-Authored-By: Claude Fable 5.1 --- CLI-COMMANDS.md | 1 + roboflow/adapters/rfapi.py | 17 ++++++++--- roboflow/cli/handlers/autolabel.py | 46 +++++++++++------------------ tests/cli/test_autolabel_handler.py | 19 ++++++++++++ 4 files changed, 51 insertions(+), 32 deletions(-) diff --git a/CLI-COMMANDS.md b/CLI-COMMANDS.md index 70492c5c..5dc4869e 100644 --- a/CLI-COMMANDS.md +++ b/CLI-COMMANDS.md @@ -256,6 +256,7 @@ image, default). `preview` runs one image through a model for free so you can compare candidates before spending credits. `start` creates the job and prints `jobId` and `annotationJobId`; poll it with `job`. Pass the ontology either as repeated `--class` flags or as `--ontology` JSON (class name to text prompt). +JSON options also accept a curl-style file reference (`--ontology @ontology.json`). `--image` accepts an HTTPS URL or a local file. The same operations are available in Python: diff --git a/roboflow/adapters/rfapi.py b/roboflow/adapters/rfapi.py index 45e1e2bb..1bc8fe14 100644 --- a/roboflow/adapters/rfapi.py +++ b/roboflow/adapters/rfapi.py @@ -1309,6 +1309,11 @@ def _annotation_pagination_params(api_key, *, limit, after=None, show_empty=None def _annotation_administration_response(response): + return _json_response_or_raise(response) + + +def _json_response_or_raise(response): + """Return the JSON body of a 2xx response; raise ``RoboflowError`` with the HTTP status otherwise.""" if not 200 <= response.status_code < 300: message = response.text try: @@ -1328,6 +1333,10 @@ def _annotation_administration_response(response): # --------------------------------------------------------------------------- +def _autolabel_response(response): + return _json_response_or_raise(response) + + def list_autolabel_models(api_key, workspace_url): """Fetch the foundation-model catalog for hosted auto-labeling. @@ -1340,7 +1349,7 @@ def list_autolabel_models(api_key, workspace_url): f"{API_URL}/{workspace_url}/autolabel/models", params={"api_key": api_key}, ) - return _annotation_administration_response(response) + return _autolabel_response(response) def preview_autolabel( @@ -1370,7 +1379,7 @@ def preview_autolabel( params={"api_key": api_key}, json=payload, ) - return _annotation_administration_response(response) + return _autolabel_response(response) def start_autolabel_job( @@ -1414,7 +1423,7 @@ def start_autolabel_job( params={"api_key": api_key}, json=payload, ) - return _annotation_administration_response(response) + return _autolabel_response(response) def get_autolabel_job(api_key, workspace_url, job_id): @@ -1426,7 +1435,7 @@ def get_autolabel_job(api_key, workspace_url, job_id): f"{API_URL}/{workspace_url}/autolabel/jobs/{job_id}", params={"api_key": api_key}, ) - return _annotation_administration_response(response) + return _autolabel_response(response) # --------------------------------------------------------------------------- diff --git a/roboflow/cli/handlers/autolabel.py b/roboflow/cli/handlers/autolabel.py index defd8eba..039a905f 100644 --- a/roboflow/cli/handlers/autolabel.py +++ b/roboflow/cli/handlers/autolabel.py @@ -2,7 +2,6 @@ from __future__ import annotations -import json from typing import Annotated, Any, Callable, Optional import typer @@ -30,7 +29,9 @@ def preview( ] = None, ontology: Annotated[ Optional[str], - typer.Option("--ontology", help='JSON mapping of class name to prompt, e.g. \'{"cat": "a cat"}\''), + typer.Option( + "--ontology", help='JSON mapping of class name to prompt, e.g. \'{"cat": "a cat"}\', or @ontology.json' + ), ] = None, confidence: Annotated[ Optional[float], typer.Option("--confidence", help="Detection threshold from 0.0 to 1.0 (sam3 only)") @@ -39,8 +40,6 @@ def preview( """Preview one image with a foundation model. Free: no job is created.""" args = ctx_to_args(ctx, project=project) resolved_ontology = _parse_ontology(args, ontology, classes) - if resolved_ontology is _INVALID: - return from roboflow.util.autolabel_utils import image_payload _project_command( @@ -80,7 +79,9 @@ def start( ] = None, ontology: Annotated[ Optional[str], - typer.Option("--ontology", help='JSON mapping of class name to prompt, e.g. \'{"cat": "a cat"}\''), + typer.Option( + "--ontology", help='JSON mapping of class name to prompt, e.g. \'{"cat": "a cat"}\', or @ontology.json' + ), ] = None, num_images: Annotated[ Optional[int], typer.Option("--num-images", help="Number of images to label (default: whole batch)") @@ -98,18 +99,16 @@ def start( ] = None, model_options: Annotated[ Optional[str], - typer.Option("--model-options", help='JSON model options, e.g. \'{"outputFormat": "polygon"}\''), + typer.Option( + "--model-options", help='JSON model options, e.g. \'{"outputFormat": "polygon"}\', or @options.json' + ), ] = None, ) -> None: """Start a hosted auto-label job over a batch of images.""" args = ctx_to_args(ctx, project=project) resolved_ontology = _parse_ontology(args, ontology, classes) - if resolved_ontology is _INVALID: - return resolved_thresholds = _parse_json_option(args, "--confidence-thresholds", confidence_thresholds) resolved_options = _parse_json_option(args, "--model-options", model_options) - if resolved_thresholds is _INVALID or resolved_options is _INVALID: - return from roboflow.util.autolabel_utils import resolve_model def start_job(key: str, workspace: str, proj: str) -> Any: @@ -146,8 +145,6 @@ def job( # Business logic # --------------------------------------------------------------------------- -_INVALID = object() - def _rfapi(): from roboflow.adapters import rfapi @@ -155,24 +152,17 @@ def _rfapi(): return rfapi -def _parse_json_option(args: Any, flag: str, raw: Optional[str]) -> Any: - from roboflow.cli._output import output_error - +def _parse_json_option(args: Any, flag: str, raw: Optional[str]) -> Optional[dict]: + """Parse an optional JSON-object flag: inline JSON or ``@path`` to a file. Exits on invalid input.""" if raw is None: return None - try: - value = json.loads(raw) - except ValueError: - output_error(args, f"{flag} must be valid JSON.") - return _INVALID - if not isinstance(value, dict): - output_error(args, f"{flag} must be a JSON object.") - return _INVALID - return value - - -def _parse_ontology(args: Any, ontology: Optional[str], classes: Optional[list[str]]) -> Any: - """Build the ontology from --ontology JSON (takes precedence) or repeated --class.""" + from roboflow.cli.handlers.train import _parse_json_flag + + return _parse_json_flag(args, raw, flag) + + +def _parse_ontology(args: Any, ontology: Optional[str], classes: Optional[list[str]]) -> Optional[dict]: + """Build the ontology from --ontology (JSON or @file, takes precedence) or repeated --class.""" if ontology is not None: return _parse_json_option(args, "--ontology", ontology) if classes: diff --git a/tests/cli/test_autolabel_handler.py b/tests/cli/test_autolabel_handler.py index 26ab9d72..ea9a2d9d 100644 --- a/tests/cli/test_autolabel_handler.py +++ b/tests/cli/test_autolabel_handler.py @@ -1,6 +1,8 @@ """Unit tests for roboflow.cli.handlers.autolabel.""" import json +import os +import tempfile import unittest from unittest.mock import patch @@ -92,6 +94,23 @@ def test_ontology_json_takes_precedence_over_classes(self, _resolve, mock_api): ) self.assertEqual(mock_api.call_args.kwargs["ontology"], {"cat": "a tabby cat"}) + @patch("roboflow.adapters.rfapi.preview_autolabel", return_value={}) + @patch(_RESOLVE_PROJECT, return_value=("key", "ws", "proj")) + def test_ontology_can_be_read_from_a_file(self, _resolve, mock_api): + with tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) as handle: + json.dump({"cat": "a tabby cat"}, handle) + path = handle.name + try: + result = runner.invoke( + app, + ["autolabel", "preview", "-p", "ws/proj", "-m", "sam3-rle", "--image", "https://x/y.jpg"] + + ["--ontology", f"@{path}"], + ) + finally: + os.unlink(path) + self.assertEqual(result.exit_code, 0, result.output) + self.assertEqual(mock_api.call_args.kwargs["ontology"], {"cat": "a tabby cat"}) + @patch("roboflow.adapters.rfapi.preview_autolabel") @patch(_RESOLVE_PROJECT, return_value=("key", "ws", "proj")) def test_invalid_ontology_json_fails_without_calling_api(self, _resolve, mock_api): From 5b33a1dd9d5936dbca6deee73032afd7b3874bfa Mon Sep 17 00:00:00 2001 From: Iuri de Silvio Date: Tue, 8 Sep 2026 09:26:07 +0200 Subject: [PATCH 4/7] Send the auto-label ontology in the explicit [{class, prompt}] wire form MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The public signature is `{"class name": "text prompt"}`, but the API's object form means the opposite — `normalizeOntology` treats the key as the prompt and the value as the class, which is the `CaptionOntology` shape the labeling worker consumes. Passing the dict straight through therefore inverted every non-identity ontology: `ontology={"cat": "a cat"}` prompted the model with "cat" and wrote the annotations under the class name "a cat". The `--class` path built an identity map, which is symmetric, so the tests never caught it. Serialize through `ontology_payload` instead. The list form names both sides, so nothing has to be inferred from key order; the backend already normalizes it on both the preview and the start path, and it is what the web app sends. `Project.autolabel` also accepts a plain list of class names now, matching `autolabel_preview`. Co-Authored-By: Claude Opus 5 --- roboflow/adapters/rfapi.py | 8 ++++++-- roboflow/cli/handlers/autolabel.py | 10 ++++++---- roboflow/core/project.py | 17 +++++++++------- roboflow/util/autolabel_utils.py | 24 +++++++++++++++++++++- tests/adapters/test_autolabel.py | 13 ++++++++---- tests/cli/test_autolabel_handler.py | 8 ++++---- tests/test_project_autolabel.py | 4 ++-- tests/util/test_autolabel_utils.py | 31 ++++++++++++++++++++++++++++- 8 files changed, 90 insertions(+), 25 deletions(-) diff --git a/roboflow/adapters/rfapi.py b/roboflow/adapters/rfapi.py index 1bc8fe14..a97be491 100644 --- a/roboflow/adapters/rfapi.py +++ b/roboflow/adapters/rfapi.py @@ -1366,7 +1366,9 @@ def preview_autolabel( Calls ``POST /:workspace/:project/autolabel/preview``. Free: no job is created and no credits are spent. ``image`` is - ``{"type": "url" | "base64", "value": ...}``. Returns + ``{"type": "url" | "base64", "value": ...}`` and ``ontology`` is the + ``[{"class": ..., "prompt": ...}]`` wire form (see + ``util.autolabel_utils.ontology_payload``). Returns ``{model, predictions, summary, blockErrors?}``. """ payload = {"modelType": model_type, "image": image} @@ -1402,7 +1404,9 @@ def start_autolabel_job( Calls ``POST /:workspace/:project/autolabel``. ``model_type`` is sent as-is: a catalog id from ``list_autolabel_models`` (for example ``gpt-6-astra-boxes`` or ``sam3-rle``) or ``custom_roboflow`` with the - Roboflow model id in ``model_options["modelId"]``. The backend fans + Roboflow model id in ``model_options["modelId"]``. ``ontology`` is the + ``[{"class": ..., "prompt": ...}]`` wire form (see + ``util.autolabel_utils.ontology_payload``). The backend fans ``default_confidence`` out across the ontology when ``confidence_thresholds`` is omitted and defaults ``num_images_to_label`` to the whole batch. Returns ``{jobId, annotationJobId, message}``. diff --git a/roboflow/cli/handlers/autolabel.py b/roboflow/cli/handlers/autolabel.py index 039a905f..99835d03 100644 --- a/roboflow/cli/handlers/autolabel.py +++ b/roboflow/cli/handlers/autolabel.py @@ -161,12 +161,14 @@ def _parse_json_option(args: Any, flag: str, raw: Optional[str]) -> Optional[dic return _parse_json_flag(args, raw, flag) -def _parse_ontology(args: Any, ontology: Optional[str], classes: Optional[list[str]]) -> Optional[dict]: - """Build the ontology from --ontology (JSON or @file, takes precedence) or repeated --class.""" +def _parse_ontology(args: Any, ontology: Optional[str], classes: Optional[list[str]]) -> Optional[list[dict]]: + """Build the wire ontology from --ontology (JSON or @file, takes precedence) or repeated --class.""" + from roboflow.util.autolabel_utils import ontology_payload + if ontology is not None: - return _parse_json_option(args, "--ontology", ontology) + return ontology_payload(_parse_json_option(args, "--ontology", ontology)) if classes: - return {name: name for name in classes} + return ontology_payload(classes) return None diff --git a/roboflow/core/project.py b/roboflow/core/project.py index 5d5cc448..70854f94 100644 --- a/roboflow/core/project.py +++ b/roboflow/core/project.py @@ -15,6 +15,7 @@ from roboflow.config import API_URL, DEMO_KEYS from roboflow.core.version import Version from roboflow.util.autolabel_utils import image_payload as _autolabel_image_payload +from roboflow.util.autolabel_utils import ontology_payload as _autolabel_ontology_payload from roboflow.util.autolabel_utils import resolve_model as _resolve_autolabel_model from roboflow.util.general import Retry from roboflow.util.image_utils import load_labelmap @@ -1175,7 +1176,8 @@ def autolabel_preview( (e.g. ``"gpt-6-astra-boxes"``, ``"sam3-rle"``, ``"gemini-boxes"``). image: HTTPS URL, local file path, or base64-encoded image. ontology: ``{"class name": "text prompt"}`` or a plain list of class - names. Defaults to the dataset's own classes. + names (each class is then its own prompt). Defaults to the + dataset's own classes. confidence_threshold: Detection threshold between 0.0 and 1.0 (sam3 only; other models report fixed confidence). @@ -1191,7 +1193,7 @@ def autolabel_preview( self.__project_name, model_type=model, image=_autolabel_image_payload(image), - ontology=ontology, + ontology=_autolabel_ontology_payload(ontology), confidence_threshold=confidence_threshold, ) @@ -1200,7 +1202,7 @@ def autolabel( batch_id: str, model: str, model_type: str = "foundational", - ontology: Optional[Dict[str, str]] = None, + ontology: Optional[Union[Dict[str, str], List[str]]] = None, num_images: Optional[int] = None, confidence: Optional[float] = None, confidence_thresholds: Optional[Dict[str, float]] = None, @@ -1220,9 +1222,10 @@ def autolabel( model_type: ``"foundational"`` (hosted foundation model, sent as-is; the backend resolves catalog ids) or ``"roboflow"`` (a Roboflow-trained model). - ontology: ``{"class name": "text prompt"}``. For models with - ``ontologyFormat="promptMap"`` (sam3) the prompts are sent to - the model; for ``ontologyFormat="classes"`` only the class + ontology: ``{"class name": "text prompt"}``, or a plain list of + class names (each class is then its own prompt). For models + with ``ontologyFormat="promptMap"`` (sam3) the prompts are sent + to the model; for ``ontologyFormat="classes"`` only the class names are used. Defaults to the dataset's classes (or the trained model's classes for ``model_type="roboflow"``). num_images: Number of images from the batch to label. Defaults to @@ -1253,7 +1256,7 @@ def autolabel( self.__project_name, batch_id=batch_id, model_type=wire_model_type, - ontology=ontology, + ontology=_autolabel_ontology_payload(ontology), num_images_to_label=num_images, default_confidence=confidence, confidence_thresholds=confidence_thresholds, diff --git a/roboflow/util/autolabel_utils.py b/roboflow/util/autolabel_utils.py index b82d82b1..429d6b30 100644 --- a/roboflow/util/autolabel_utils.py +++ b/roboflow/util/autolabel_utils.py @@ -4,7 +4,7 @@ import base64 import os -from typing import Any, Dict, Optional, Tuple +from typing import Any, Dict, Iterable, List, Optional, Tuple, Union MODEL_TYPES = ("foundational", "roboflow") @@ -23,6 +23,28 @@ def image_payload(image: str) -> Dict[str, str]: return {"type": "base64", "value": image} +def ontology_payload( + ontology: Optional[Union[Dict[str, str], Iterable[str]]], +) -> Optional[List[Dict[str, str]]]: + """Serialize an ontology into the ``[{"class", "prompt"}]`` wire form. + + The public SDK/CLI signature is the intuitive ``{"class name": "text + prompt"}``. The API's object form means the opposite (``{prompt: class}``, + the ``CaptionOntology`` shape the labeling worker consumes), so a bare dict + is ambiguous on the wire. The list form is explicit about which side is + which, is normalized by the backend for every endpoint, and is what the web + app already sends. + + A plain iterable of class names is treated as ``{"cat": "cat"}`` — each + class is its own prompt. + """ + if ontology is None: + return None + if isinstance(ontology, dict): + return [{"class": name, "prompt": prompt} for name, prompt in ontology.items()] + return [{"class": name, "prompt": name} for name in ontology] + + def resolve_model( model: str, model_type: str, model_options: Optional[Dict[str, Any]] = None ) -> Tuple[str, Optional[Dict[str, Any]]]: diff --git a/tests/adapters/test_autolabel.py b/tests/adapters/test_autolabel.py index 9540531d..7a3bb525 100644 --- a/tests/adapters/test_autolabel.py +++ b/tests/adapters/test_autolabel.py @@ -43,12 +43,17 @@ def test_preview_contract(self, mock_post): "proj", model_type="sam3-rle", image=image, - ontology={"cat": "cat"}, + ontology=[{"class": "cat", "prompt": "cat"}], confidence_threshold=0.4, ) self.assertEqual( mock_post.call_args.kwargs["json"], - {"modelType": "sam3-rle", "image": image, "ontology": {"cat": "cat"}, "confidenceThreshold": 0.4}, + { + "modelType": "sam3-rle", + "image": image, + "ontology": [{"class": "cat", "prompt": "cat"}], + "confidenceThreshold": 0.4, + }, ) @patch("roboflow.adapters.rfapi.requests.post") @@ -74,7 +79,7 @@ def test_start_job_contract_full_payload(self, mock_post): "proj", batch_id="batch-1", model_type="custom_roboflow", - ontology={"cat": "a cat"}, + ontology=[{"class": "cat", "prompt": "a cat"}], num_images_to_label=10, default_confidence=0.5, confidence_thresholds={"cat": 0.6}, @@ -87,7 +92,7 @@ def test_start_job_contract_full_payload(self, mock_post): { "batchId": "batch-1", "modelType": "custom_roboflow", - "ontology": {"cat": "a cat"}, + "ontology": [{"class": "cat", "prompt": "a cat"}], "numImagesToLabel": 10, "defaultConfidence": 0.5, "confidenceThresholds": {"cat": 0.6}, diff --git a/tests/cli/test_autolabel_handler.py b/tests/cli/test_autolabel_handler.py index ea9a2d9d..a3afb38e 100644 --- a/tests/cli/test_autolabel_handler.py +++ b/tests/cli/test_autolabel_handler.py @@ -80,7 +80,7 @@ def test_classes_become_identity_ontology(self, _resolve, mock_api): "proj", model_type="sam3-rle", image={"type": "url", "value": "https://example.com/cat.jpg"}, - ontology={"cat": "cat", "dog": "dog"}, + ontology=[{"class": "cat", "prompt": "cat"}, {"class": "dog", "prompt": "dog"}], confidence_threshold=0.4, ) @@ -92,7 +92,7 @@ def test_ontology_json_takes_precedence_over_classes(self, _resolve, mock_api): ["autolabel", "preview", "-p", "ws/proj", "-m", "sam3-rle", "--image", "https://x/y.jpg"] + ["--class", "cat", "--ontology", '{"cat": "a tabby cat"}'], ) - self.assertEqual(mock_api.call_args.kwargs["ontology"], {"cat": "a tabby cat"}) + self.assertEqual(mock_api.call_args.kwargs["ontology"], [{"class": "cat", "prompt": "a tabby cat"}]) @patch("roboflow.adapters.rfapi.preview_autolabel", return_value={}) @patch(_RESOLVE_PROJECT, return_value=("key", "ws", "proj")) @@ -109,7 +109,7 @@ def test_ontology_can_be_read_from_a_file(self, _resolve, mock_api): finally: os.unlink(path) self.assertEqual(result.exit_code, 0, result.output) - self.assertEqual(mock_api.call_args.kwargs["ontology"], {"cat": "a tabby cat"}) + self.assertEqual(mock_api.call_args.kwargs["ontology"], [{"class": "cat", "prompt": "a tabby cat"}]) @patch("roboflow.adapters.rfapi.preview_autolabel") @patch(_RESOLVE_PROJECT, return_value=("key", "ws", "proj")) @@ -141,7 +141,7 @@ def test_foundational_start(self, _resolve, mock_api): "proj", batch_id="batch-1", model_type="gpt-6-astra-boxes", - ontology={"cat": "cat"}, + ontology=[{"class": "cat", "prompt": "cat"}], num_images_to_label=10, default_confidence=0.5, confidence_thresholds=None, diff --git a/tests/test_project_autolabel.py b/tests/test_project_autolabel.py index 476f5d22..e5ad9b70 100644 --- a/tests/test_project_autolabel.py +++ b/tests/test_project_autolabel.py @@ -24,7 +24,7 @@ def test_autolabel_foundational_passes_model_as_is(self, mock_start): PROJECT_NAME, batch_id="batch-1", model_type="gpt-6-astra-boxes", - ontology={"cat": "a cat"}, + ontology=[{"class": "cat", "prompt": "a cat"}], num_images_to_label=5, default_confidence=0.4, confidence_thresholds=None, @@ -61,7 +61,7 @@ def test_autolabel_preview_builds_image_payload(self, mock_preview): PROJECT_NAME, model_type="sam3-rle", image={"type": "url", "value": "https://example.com/cat.jpg"}, - ontology=["cat"], + ontology=[{"class": "cat", "prompt": "cat"}], confidence_threshold=0.3, ) diff --git a/tests/util/test_autolabel_utils.py b/tests/util/test_autolabel_utils.py index 42955b7d..f78f8094 100644 --- a/tests/util/test_autolabel_utils.py +++ b/tests/util/test_autolabel_utils.py @@ -5,7 +5,7 @@ import tempfile import unittest -from roboflow.util.autolabel_utils import image_payload, resolve_model +from roboflow.util.autolabel_utils import image_payload, ontology_payload, resolve_model class TestImagePayload(unittest.TestCase): @@ -31,6 +31,35 @@ def test_other_strings_are_treated_as_base64(self): self.assertEqual(image_payload(encoded), {"type": "base64", "value": encoded}) +class TestOntologyPayload(unittest.TestCase): + def test_none_stays_none(self): + self.assertIsNone(ontology_payload(None)) + + def test_class_to_prompt_mapping_is_serialized_explicitly(self): + # The wire form names both sides, so the API never has to guess which + # of the two strings is the class and which is the prompt. + self.assertEqual( + ontology_payload({"cat": "a cat", "dog": "a dog"}), + [{"class": "cat", "prompt": "a cat"}, {"class": "dog", "prompt": "a dog"}], + ) + + def test_list_of_classes_becomes_identity_prompts(self): + self.assertEqual( + ontology_payload(["cat", "dog"]), + [{"class": "cat", "prompt": "cat"}, {"class": "dog", "prompt": "dog"}], + ) + + def test_several_classes_may_share_a_prompt(self): + # Impossible to express if the prompt were the key. + self.assertEqual( + ontology_payload({"cat": "animal", "dog": "animal"}), + [{"class": "cat", "prompt": "animal"}, {"class": "dog", "prompt": "animal"}], + ) + + def test_empty_is_preserved_as_empty(self): + self.assertEqual(ontology_payload({}), []) + + class TestResolveModel(unittest.TestCase): def test_foundational_is_pass_through(self): self.assertEqual(resolve_model("gpt-6-astra-boxes", "foundational"), ("gpt-6-astra-boxes", None)) From 6820f7939d22301a061efec97253b9c7791ade86 Mon Sep 17 00:00:00 2001 From: Iuri de Silvio Date: Tue, 8 Sep 2026 09:34:04 +0200 Subject: [PATCH 5/7] Accept a [{class, prompt}] ontology so one class can have several prompts The class-keyed dict the previous commit settled on cannot express the case the API's ontology exists for: several prompts collapsing to one output class, e.g. "kitten" and "tabby" both labeled `cat`. A dict has room for one prompt per class because its keys are unique. So accept the explicit list on the way in too, alongside the dict and the plain list of class names. It is the same shape already used on the wire, so this is one shape fewer to think about rather than one more. Reject the mirror case while we are here: two classes claiming one prompt. The API keys its ontology by prompt, so it keeps whichever class came last and drops the other, and the job then never labels that class with nothing in the response to say why. `ontology_payload` names the collision instead, and the CLI reports it before any network call rather than letting the ValueError escape as a traceback. `--ontology` accepts a JSON array as well as an object; `_parse_json_flag` grew an opt-in `allow_list` for that and stays object-only everywhere else. --- CLI-COMMANDS.md | 3 ++ roboflow/cli/handlers/autolabel.py | 35 +++++++++++----- roboflow/cli/handlers/train.py | 9 ++-- roboflow/core/project.py | 21 ++++++---- roboflow/util/autolabel_utils.py | 65 +++++++++++++++++++++++------ tests/cli/test_autolabel_handler.py | 25 +++++++++++ tests/util/test_autolabel_utils.py | 43 +++++++++++++++++-- 7 files changed, 162 insertions(+), 39 deletions(-) diff --git a/CLI-COMMANDS.md b/CLI-COMMANDS.md index 5dc4869e..c3be24e5 100644 --- a/CLI-COMMANDS.md +++ b/CLI-COMMANDS.md @@ -256,6 +256,9 @@ image, default). `preview` runs one image through a model for free so you can compare candidates before spending credits. `start` creates the job and prints `jobId` and `annotationJobId`; poll it with `job`. Pass the ontology either as repeated `--class` flags or as `--ontology` JSON (class name to text prompt). +To give one class several prompts, pass `--ontology` as an array instead: +`'[{"class": "cat", "prompt": "kitten"}, {"class": "cat", "prompt": "tabby"}]'` +(an object can only carry one prompt per class, since its keys are unique). JSON options also accept a curl-style file reference (`--ontology @ontology.json`). `--image` accepts an HTTPS URL or a local file. diff --git a/roboflow/cli/handlers/autolabel.py b/roboflow/cli/handlers/autolabel.py index 99835d03..587dce55 100644 --- a/roboflow/cli/handlers/autolabel.py +++ b/roboflow/cli/handlers/autolabel.py @@ -30,7 +30,9 @@ def preview( ontology: Annotated[ Optional[str], typer.Option( - "--ontology", help='JSON mapping of class name to prompt, e.g. \'{"cat": "a cat"}\', or @ontology.json' + "--ontology", + help='JSON class-to-prompt object, e.g. \'{"cat": "a cat"}\'; or a ' + '[{"class": ..., "prompt": ...}] array to give one class several prompts; or @ontology.json', ), ] = None, confidence: Annotated[ @@ -80,7 +82,9 @@ def start( ontology: Annotated[ Optional[str], typer.Option( - "--ontology", help='JSON mapping of class name to prompt, e.g. \'{"cat": "a cat"}\', or @ontology.json' + "--ontology", + help='JSON class-to-prompt object, e.g. \'{"cat": "a cat"}\'; or a ' + '[{"class": ..., "prompt": ...}] array to give one class several prompts; or @ontology.json', ), ] = None, num_images: Annotated[ @@ -152,24 +156,35 @@ def _rfapi(): return rfapi -def _parse_json_option(args: Any, flag: str, raw: Optional[str]) -> Optional[dict]: - """Parse an optional JSON-object flag: inline JSON or ``@path`` to a file. Exits on invalid input.""" +def _parse_json_option(args: Any, flag: str, raw: Optional[str], allow_list: bool = False) -> Optional[Any]: + """Parse an optional JSON flag: inline JSON or ``@path`` to a file. Exits on invalid input.""" if raw is None: return None from roboflow.cli.handlers.train import _parse_json_flag - return _parse_json_flag(args, raw, flag) + return _parse_json_flag(args, raw, flag, allow_list=allow_list) def _parse_ontology(args: Any, ontology: Optional[str], classes: Optional[list[str]]) -> Optional[list[dict]]: - """Build the wire ontology from --ontology (JSON or @file, takes precedence) or repeated --class.""" + """Build the wire ontology from --ontology (JSON or @file, takes precedence) or repeated --class. + + ``--ontology`` takes either the ``{"class": "prompt"}`` object or the + ``[{"class": ..., "prompt": ...}]`` array, which is the only one of the two + that can give a single class more than one prompt. + """ + from roboflow.cli._output import output_error from roboflow.util.autolabel_utils import ontology_payload + raw: Any = None if ontology is not None: - return ontology_payload(_parse_json_option(args, "--ontology", ontology)) - if classes: - return ontology_payload(classes) - return None + raw = _parse_json_option(args, "--ontology", ontology, allow_list=True) + elif classes: + raw = classes + try: + return ontology_payload(raw) + except ValueError as exc: + output_error(args, str(exc), hint="See 'roboflow autolabel start --help' for the accepted shapes.") + return None # unreachable: output_error sys.exits def _resolve_workspace(args: Any) -> tuple[Optional[str], Optional[str]]: diff --git a/roboflow/cli/handlers/train.py b/roboflow/cli/handlers/train.py index f2a69651..da7c9174 100644 --- a/roboflow/cli/handlers/train.py +++ b/roboflow/cli/handlers/train.py @@ -341,11 +341,12 @@ def _start(args): # noqa: ANN001 output(args, data, text=f"Training started for {project_slug} version {args.version_number}.") -def _parse_json_flag(args, raw, flag): +def _parse_json_flag(args, raw, flag, allow_list=False): """Parse a JSON-object CLI flag value; exits with a clean error on invalid input. Accepts inline JSON, or ``@path/to/file.json`` to read the JSON from a file (curl-style; unambiguous because ``@`` can never start valid JSON). + Set *allow_list* for flags whose value may also be a JSON array. """ import json import os @@ -372,10 +373,12 @@ def _parse_json_flag(args, raw, flag): except json.JSONDecodeError as exc: output_error(args, f"Invalid JSON in {flag} {source}: {exc}", hint="Pass a valid JSON string.") return None # unreachable: output_error sys.exits - if not isinstance(parsed, dict): + allowed = (dict, list) if allow_list else (dict,) + if not isinstance(parsed, allowed): + expected = "a JSON object or array" if allow_list else "a JSON object" output_error( args, - f"{flag} must be a JSON object, got {type(parsed).__name__}", + f"{flag} must be {expected}, got {type(parsed).__name__}", hint="Pass a JSON object string, e.g. '{\"lr\": 0.0002}'.", ) return None # unreachable: output_error sys.exits diff --git a/roboflow/core/project.py b/roboflow/core/project.py index 70854f94..c6028f41 100644 --- a/roboflow/core/project.py +++ b/roboflow/core/project.py @@ -14,6 +14,7 @@ from roboflow.adapters.rfapi import AnnotationSaveError, ImageUploadError from roboflow.config import API_URL, DEMO_KEYS from roboflow.core.version import Version +from roboflow.util.autolabel_utils import Ontology as _AutolabelOntology from roboflow.util.autolabel_utils import image_payload as _autolabel_image_payload from roboflow.util.autolabel_utils import ontology_payload as _autolabel_ontology_payload from roboflow.util.autolabel_utils import resolve_model as _resolve_autolabel_model @@ -1162,7 +1163,7 @@ def autolabel_preview( self, model: str, image: str, - ontology: Optional[Union[Dict[str, str], List[str]]] = None, + ontology: Optional[_AutolabelOntology] = None, confidence_threshold: Optional[float] = None, ) -> Dict: """Preview one image with a foundation model before starting an auto-label job. @@ -1175,9 +1176,9 @@ def autolabel_preview( model: Foundation model id from ``Workspace.autolabel_models()`` (e.g. ``"gpt-6-astra-boxes"``, ``"sam3-rle"``, ``"gemini-boxes"``). image: HTTPS URL, local file path, or base64-encoded image. - ontology: ``{"class name": "text prompt"}`` or a plain list of class - names (each class is then its own prompt). Defaults to the - dataset's own classes. + ontology: ``{"class name": "text prompt"}``, a plain list of class + names, or ``[{"class": ..., "prompt": ...}]`` when one class + needs several prompts. Defaults to the dataset's own classes. confidence_threshold: Detection threshold between 0.0 and 1.0 (sam3 only; other models report fixed confidence). @@ -1202,7 +1203,7 @@ def autolabel( batch_id: str, model: str, model_type: str = "foundational", - ontology: Optional[Union[Dict[str, str], List[str]]] = None, + ontology: Optional[_AutolabelOntology] = None, num_images: Optional[int] = None, confidence: Optional[float] = None, confidence_thresholds: Optional[Dict[str, float]] = None, @@ -1222,10 +1223,12 @@ def autolabel( model_type: ``"foundational"`` (hosted foundation model, sent as-is; the backend resolves catalog ids) or ``"roboflow"`` (a Roboflow-trained model). - ontology: ``{"class name": "text prompt"}``, or a plain list of - class names (each class is then its own prompt). For models - with ``ontologyFormat="promptMap"`` (sam3) the prompts are sent - to the model; for ``ontologyFormat="classes"`` only the class + ontology: ``{"class name": "text prompt"}``, a plain list of class + names, or ``[{"class": ..., "prompt": ...}]`` when one class + needs several prompts (``{"cat": ...}`` can only carry one, + since dict keys are unique). For models with + ``ontologyFormat="promptMap"`` (sam3) the prompts are sent to + the model; for ``ontologyFormat="classes"`` only the class names are used. Defaults to the dataset's classes (or the trained model's classes for ``model_type="roboflow"``). num_images: Number of images from the batch to label. Defaults to diff --git a/roboflow/util/autolabel_utils.py b/roboflow/util/autolabel_utils.py index 429d6b30..07b4c2a5 100644 --- a/roboflow/util/autolabel_utils.py +++ b/roboflow/util/autolabel_utils.py @@ -23,26 +23,65 @@ def image_payload(image: str) -> Dict[str, str]: return {"type": "base64", "value": image} -def ontology_payload( - ontology: Optional[Union[Dict[str, str], Iterable[str]]], -) -> Optional[List[Dict[str, str]]]: +OntologyEntry = Dict[str, str] +Ontology = Union[Dict[str, str], Iterable[Union[str, OntologyEntry]]] + + +def ontology_payload(ontology: Optional[Ontology]) -> Optional[List[OntologyEntry]]: """Serialize an ontology into the ``[{"class", "prompt"}]`` wire form. - The public SDK/CLI signature is the intuitive ``{"class name": "text - prompt"}``. The API's object form means the opposite (``{prompt: class}``, - the ``CaptionOntology`` shape the labeling worker consumes), so a bare dict - is ambiguous on the wire. The list form is explicit about which side is - which, is normalized by the backend for every endpoint, and is what the web - app already sends. + Accepts, in order of convenience: + + * ``{"cat": "a cat"}`` — one prompt per class, the common case. + * ``["cat", "dog"]`` — each class is its own prompt. + * ``[{"class": "cat", "prompt": "kitten"}, {"class": "cat", "prompt": "tabby"}]`` + — several prompts for one class, which a class-keyed dict cannot express + because its keys have to be unique. ``prompt`` defaults to ``class``. - A plain iterable of class names is treated as ``{"cat": "cat"}`` — each - class is its own prompt. + The API's own object form is ``{prompt: class}``, the ``CaptionOntology`` + shape the labeling worker consumes, so a bare dict is ambiguous on the + wire: the two sides are both strings and only key order says which is + which. The list form names them, and the backend normalizes it on both the + preview and the start path. """ if ontology is None: return None + if isinstance(ontology, str): + raise ValueError(f"ontology must be a mapping or a list of classes, not a bare string {ontology!r}") if isinstance(ontology, dict): - return [{"class": name, "prompt": prompt} for name, prompt in ontology.items()] - return [{"class": name, "prompt": name} for name in ontology] + entries = [{"class": name, "prompt": prompt} for name, prompt in ontology.items()] + else: + entries = [_ontology_entry(item) for item in ontology] + _reject_ambiguous_prompts(entries) + return entries + + +def _ontology_entry(item: Union[str, OntologyEntry]) -> OntologyEntry: + if isinstance(item, str): + return {"class": item, "prompt": item} + if isinstance(item, dict) and "class" in item: + return {"class": item["class"], "prompt": item.get("prompt", item["class"])} + raise ValueError( + f"ontology entries must be a class name or a {{'class': ..., 'prompt': ...}} mapping, got {item!r}" + ) + + +def _reject_ambiguous_prompts(entries: List[OntologyEntry]) -> None: + """Refuse a prompt claimed by two classes. + + The backend keys its ontology by prompt, so it would keep whichever class + came last and silently drop the other, leaving that class unlabeled for the + whole job with nothing in the response to explain why. + """ + by_prompt: Dict[str, str] = {} + for entry in entries: + claimed = by_prompt.setdefault(entry["prompt"], entry["class"]) + if claimed != entry["class"]: + raise ValueError( + f"ontology maps the prompt {entry['prompt']!r} to both {claimed!r} and " + f"{entry['class']!r}. The API keys its ontology by prompt, so one of the two " + "classes would be dropped. Give each class a distinct prompt." + ) def resolve_model( diff --git a/tests/cli/test_autolabel_handler.py b/tests/cli/test_autolabel_handler.py index a3afb38e..1cfb10d6 100644 --- a/tests/cli/test_autolabel_handler.py +++ b/tests/cli/test_autolabel_handler.py @@ -111,6 +111,31 @@ def test_ontology_can_be_read_from_a_file(self, _resolve, mock_api): self.assertEqual(result.exit_code, 0, result.output) self.assertEqual(mock_api.call_args.kwargs["ontology"], [{"class": "cat", "prompt": "a tabby cat"}]) + @patch("roboflow.adapters.rfapi.preview_autolabel", return_value={}) + @patch(_RESOLVE_PROJECT, return_value=("key", "ws", "proj")) + def test_ontology_accepts_a_json_array_for_multi_prompt_classes(self, _resolve, mock_api): + result = runner.invoke( + app, + ["autolabel", "preview", "-p", "ws/proj", "-m", "sam3-rle", "--image", "https://x/y.jpg"] + + ["--ontology", '[{"class": "cat", "prompt": "kitten"}, {"class": "cat", "prompt": "tabby"}]'], + ) + self.assertEqual(result.exit_code, 0, result.output) + self.assertEqual( + mock_api.call_args.kwargs["ontology"], + [{"class": "cat", "prompt": "kitten"}, {"class": "cat", "prompt": "tabby"}], + ) + + @patch("roboflow.adapters.rfapi.preview_autolabel") + @patch(_RESOLVE_PROJECT, return_value=("key", "ws", "proj")) + def test_one_prompt_for_two_classes_errors_without_calling_api(self, _resolve, mock_api): + result = runner.invoke( + app, + ["autolabel", "preview", "-p", "ws/proj", "-m", "sam3-rle", "--image", "https://x/y.jpg"] + + ["--ontology", '[{"class": "cat", "prompt": "animal"}, {"class": "dog", "prompt": "animal"}]'], + ) + self.assertNotEqual(result.exit_code, 0) + mock_api.assert_not_called() + @patch("roboflow.adapters.rfapi.preview_autolabel") @patch(_RESOLVE_PROJECT, return_value=("key", "ws", "proj")) def test_invalid_ontology_json_fails_without_calling_api(self, _resolve, mock_api): diff --git a/tests/util/test_autolabel_utils.py b/tests/util/test_autolabel_utils.py index f78f8094..3b384a44 100644 --- a/tests/util/test_autolabel_utils.py +++ b/tests/util/test_autolabel_utils.py @@ -49,13 +49,48 @@ def test_list_of_classes_becomes_identity_prompts(self): [{"class": "cat", "prompt": "cat"}, {"class": "dog", "prompt": "dog"}], ) - def test_several_classes_may_share_a_prompt(self): - # Impossible to express if the prompt were the key. + def test_one_class_may_carry_several_prompts(self): + # The reason the list form is accepted at all: {"cat": ...} has room for + # exactly one prompt, because dict keys are unique. self.assertEqual( - ontology_payload({"cat": "animal", "dog": "animal"}), - [{"class": "cat", "prompt": "animal"}, {"class": "dog", "prompt": "animal"}], + ontology_payload( + [ + {"class": "cat", "prompt": "kitten"}, + {"class": "cat", "prompt": "tabby"}, + ] + ), + [{"class": "cat", "prompt": "kitten"}, {"class": "cat", "prompt": "tabby"}], ) + def test_list_entries_default_the_prompt_to_the_class(self): + self.assertEqual( + ontology_payload([{"class": "cat"}, "dog"]), + [{"class": "cat", "prompt": "cat"}, {"class": "dog", "prompt": "dog"}], + ) + + def test_one_prompt_claimed_by_two_classes_is_rejected(self): + # The API keys its ontology by prompt, so it would keep "dog" and drop + # "cat" without saying so. Name the collision instead. + with self.assertRaises(ValueError) as ctx: + ontology_payload([{"class": "cat", "prompt": "animal"}, {"class": "dog", "prompt": "animal"}]) + self.assertIn("animal", str(ctx.exception)) + self.assertIn("cat", str(ctx.exception)) + self.assertIn("dog", str(ctx.exception)) + + def test_a_repeated_class_prompt_pair_is_not_a_collision(self): + self.assertEqual( + ontology_payload([{"class": "cat", "prompt": "cat"}, "cat"]), + [{"class": "cat", "prompt": "cat"}, {"class": "cat", "prompt": "cat"}], + ) + + def test_bare_string_is_rejected_rather_than_iterated_per_character(self): + with self.assertRaises(ValueError): + ontology_payload("cat") + + def test_malformed_entry_is_rejected(self): + with self.assertRaises(ValueError): + ontology_payload([{"prompt": "a cat"}]) + def test_empty_is_preserved_as_empty(self): self.assertEqual(ontology_payload({}), []) From 1dd4b4b8360b5156a22fc2cf9ab56f31fc72aa3b Mon Sep 17 00:00:00 2001 From: Iuri de Silvio Date: Tue, 8 Sep 2026 11:49:42 +0200 Subject: [PATCH 6/7] Key the ontology by prompt, matching the API Settles the direction question the last two commits worked around. The API's ontology is `{prompt: class name}`, and so is the CaptionOntology the labeling worker consumes, so the SDK and CLI now take that shape directly instead of translating a class-keyed one into it. That direction is the useful one, not an accident of the API: prompts are the unique side, so several of them can collapse onto one output class, `{"kitten": "cat", "tabby": "cat"}`. A class-keyed object has room for one prompt per class. The previous commit reached for a [{class, prompt}] list to get that expressiveness back, which the prompt-keyed object gives for free. Dropping the translation drops what surrounded it: the wire form, the entry-list shape, the guard against two classes claiming one prompt (a duplicate prompt is now impossible, it is a dict key), and the `allow_list` opt-in `_parse_json_flag` grew for the array. Net 87 lines lighter. A plain list of class names still works and still means "prompt each class with its own name". Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 4 +- CLI-COMMANDS.md | 14 +++--- roboflow/adapters/rfapi.py | 12 +++--- roboflow/cli/handlers/autolabel.py | 37 +++++++--------- roboflow/cli/handlers/train.py | 9 ++-- roboflow/core/project.py | 26 ++++++------ roboflow/util/autolabel_utils.py | 66 ++++++++--------------------- tests/adapters/test_autolabel.py | 13 ++---- tests/cli/test_autolabel_handler.py | 33 +++++---------- tests/test_project_autolabel.py | 6 +-- tests/util/test_autolabel_utils.py | 60 ++++++-------------------- 11 files changed, 95 insertions(+), 185 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ee016f18..e28f4c81 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,9 @@ All notable changes to this project will be documented in this file. preview to compare models before starting a job. `image` accepts an HTTPS URL, a local file path or a base64 string. - `Project.autolabel(batch_id, model, model_type="foundational" | "roboflow", ...)` - — start a job over a batch; returns `{jobId, annotationJobId}`. + — start a job over a batch; returns `{jobId, annotationJobId}`. The + `ontology` is keyed by prompt (`{"kitten": "cat", "tabby": "cat"}`), so + several prompts can share one output class. - `Project.autolabel_job(job_id)` / `Workspace.autolabel_job(job_id)` — poll per-subjob progress. - `roboflow autolabel models | preview | start | job` CLI commands. diff --git a/CLI-COMMANDS.md b/CLI-COMMANDS.md index c3be24e5..a7999ff4 100644 --- a/CLI-COMMANDS.md +++ b/CLI-COMMANDS.md @@ -246,7 +246,7 @@ roboflow autolabel models roboflow autolabel preview -p my-project -m sam3-rle --image https://example.com/sample.jpg \ --class cat --class dog roboflow autolabel start -p my-project --batch-id -m gpt-6-astra-boxes \ - --ontology '{"cat": "a cat", "dog": "a dog"}' --confidence 0.5 --reviewer b@co.com + --ontology '{"a cat": "cat", "a dog": "dog"}' --confidence 0.5 --reviewer b@co.com roboflow autolabel start -p my-project --batch-id -m my-project/3 --model-type roboflow roboflow autolabel job ``` @@ -255,11 +255,11 @@ roboflow autolabel job image, default). `preview` runs one image through a model for free so you can compare candidates before spending credits. `start` creates the job and prints `jobId` and `annotationJobId`; poll it with `job`. Pass the ontology either as -repeated `--class` flags or as `--ontology` JSON (class name to text prompt). -To give one class several prompts, pass `--ontology` as an array instead: -`'[{"class": "cat", "prompt": "kitten"}, {"class": "cat", "prompt": "tabby"}]'` -(an object can only carry one prompt per class, since its keys are unique). -JSON options also accept a curl-style file reference (`--ontology @ontology.json`). +repeated `--class` flags or as `--ontology` JSON. The ontology is keyed by +**prompt**, not by class: `'{"kitten": "cat", "tabby": "cat"}'` labels whatever +matches either prompt as class `cat`. That direction is what lets several +prompts share one output class. JSON options also accept a curl-style file +reference (`--ontology @ontology.json`). `--image` accepts an HTTPS URL or a local file. The same operations are available in Python: @@ -267,7 +267,7 @@ The same operations are available in Python: ```python models = workspace.autolabel_models()["models"] preview = project.autolabel_preview("sam3-rle", "sample.jpg", ontology={"cat": "cat"}) -job = project.autolabel("batch-id", model="gpt-6-astra-boxes", ontology={"cat": "a cat"}) +job = project.autolabel("batch-id", model="gpt-6-astra-boxes", ontology={"a cat": "cat"}) project.autolabel_job(job["jobId"])["status"] ``` diff --git a/roboflow/adapters/rfapi.py b/roboflow/adapters/rfapi.py index a97be491..535f6456 100644 --- a/roboflow/adapters/rfapi.py +++ b/roboflow/adapters/rfapi.py @@ -1366,9 +1366,9 @@ def preview_autolabel( Calls ``POST /:workspace/:project/autolabel/preview``. Free: no job is created and no credits are spent. ``image`` is - ``{"type": "url" | "base64", "value": ...}`` and ``ontology`` is the - ``[{"class": ..., "prompt": ...}]`` wire form (see - ``util.autolabel_utils.ontology_payload``). Returns + ``{"type": "url" | "base64", "value": ...}`` and ``ontology`` is keyed by + prompt: ``{"kitten": "cat"}`` labels prompt matches as class ``cat``. + Returns ``{model, predictions, summary, blockErrors?}``. """ payload = {"modelType": model_type, "image": image} @@ -1404,9 +1404,9 @@ def start_autolabel_job( Calls ``POST /:workspace/:project/autolabel``. ``model_type`` is sent as-is: a catalog id from ``list_autolabel_models`` (for example ``gpt-6-astra-boxes`` or ``sam3-rle``) or ``custom_roboflow`` with the - Roboflow model id in ``model_options["modelId"]``. ``ontology`` is the - ``[{"class": ..., "prompt": ...}]`` wire form (see - ``util.autolabel_utils.ontology_payload``). The backend fans + Roboflow model id in ``model_options["modelId"]``. ``ontology`` is keyed + by prompt: ``{"kitten": "cat"}`` labels prompt matches as class ``cat``. + The backend fans ``default_confidence`` out across the ontology when ``confidence_thresholds`` is omitted and defaults ``num_images_to_label`` to the whole batch. Returns ``{jobId, annotationJobId, message}``. diff --git a/roboflow/cli/handlers/autolabel.py b/roboflow/cli/handlers/autolabel.py index 587dce55..5061ba46 100644 --- a/roboflow/cli/handlers/autolabel.py +++ b/roboflow/cli/handlers/autolabel.py @@ -31,8 +31,8 @@ def preview( Optional[str], typer.Option( "--ontology", - help='JSON class-to-prompt object, e.g. \'{"cat": "a cat"}\'; or a ' - '[{"class": ..., "prompt": ...}] array to give one class several prompts; or @ontology.json', + help="JSON object mapping each text prompt to the class name it labels, e.g. " + '\'{"kitten": "cat", "tabby": "cat"}\'. Note the direction: prompt first. Also accepts @ontology.json', ), ] = None, confidence: Annotated[ @@ -83,8 +83,8 @@ def start( Optional[str], typer.Option( "--ontology", - help='JSON class-to-prompt object, e.g. \'{"cat": "a cat"}\'; or a ' - '[{"class": ..., "prompt": ...}] array to give one class several prompts; or @ontology.json', + help="JSON object mapping each text prompt to the class name it labels, e.g. " + '\'{"kitten": "cat", "tabby": "cat"}\'. Note the direction: prompt first. Also accepts @ontology.json', ), ] = None, num_images: Annotated[ @@ -156,35 +156,28 @@ def _rfapi(): return rfapi -def _parse_json_option(args: Any, flag: str, raw: Optional[str], allow_list: bool = False) -> Optional[Any]: - """Parse an optional JSON flag: inline JSON or ``@path`` to a file. Exits on invalid input.""" +def _parse_json_option(args: Any, flag: str, raw: Optional[str]) -> Optional[dict]: + """Parse an optional JSON-object flag: inline JSON or ``@path`` to a file. Exits on invalid input.""" if raw is None: return None from roboflow.cli.handlers.train import _parse_json_flag - return _parse_json_flag(args, raw, flag, allow_list=allow_list) + return _parse_json_flag(args, raw, flag) -def _parse_ontology(args: Any, ontology: Optional[str], classes: Optional[list[str]]) -> Optional[list[dict]]: - """Build the wire ontology from --ontology (JSON or @file, takes precedence) or repeated --class. +def _parse_ontology(args: Any, ontology: Optional[str], classes: Optional[list[str]]) -> Optional[dict]: + """Build the ontology from --ontology (JSON or @file, takes precedence) or repeated --class. - ``--ontology`` takes either the ``{"class": "prompt"}`` object or the - ``[{"class": ..., "prompt": ...}]`` array, which is the only one of the two - that can give a single class more than one prompt. + ``--ontology`` is keyed by prompt, not by class: ``{"kitten": "cat"}`` labels + whatever matches the prompt "kitten" as class ``cat``. """ - from roboflow.cli._output import output_error from roboflow.util.autolabel_utils import ontology_payload - raw: Any = None if ontology is not None: - raw = _parse_json_option(args, "--ontology", ontology, allow_list=True) - elif classes: - raw = classes - try: - return ontology_payload(raw) - except ValueError as exc: - output_error(args, str(exc), hint="See 'roboflow autolabel start --help' for the accepted shapes.") - return None # unreachable: output_error sys.exits + return ontology_payload(_parse_json_option(args, "--ontology", ontology)) + if classes: + return ontology_payload(classes) + return None def _resolve_workspace(args: Any) -> tuple[Optional[str], Optional[str]]: diff --git a/roboflow/cli/handlers/train.py b/roboflow/cli/handlers/train.py index da7c9174..f2a69651 100644 --- a/roboflow/cli/handlers/train.py +++ b/roboflow/cli/handlers/train.py @@ -341,12 +341,11 @@ def _start(args): # noqa: ANN001 output(args, data, text=f"Training started for {project_slug} version {args.version_number}.") -def _parse_json_flag(args, raw, flag, allow_list=False): +def _parse_json_flag(args, raw, flag): """Parse a JSON-object CLI flag value; exits with a clean error on invalid input. Accepts inline JSON, or ``@path/to/file.json`` to read the JSON from a file (curl-style; unambiguous because ``@`` can never start valid JSON). - Set *allow_list* for flags whose value may also be a JSON array. """ import json import os @@ -373,12 +372,10 @@ def _parse_json_flag(args, raw, flag, allow_list=False): except json.JSONDecodeError as exc: output_error(args, f"Invalid JSON in {flag} {source}: {exc}", hint="Pass a valid JSON string.") return None # unreachable: output_error sys.exits - allowed = (dict, list) if allow_list else (dict,) - if not isinstance(parsed, allowed): - expected = "a JSON object or array" if allow_list else "a JSON object" + if not isinstance(parsed, dict): output_error( args, - f"{flag} must be {expected}, got {type(parsed).__name__}", + f"{flag} must be a JSON object, got {type(parsed).__name__}", hint="Pass a JSON object string, e.g. '{\"lr\": 0.0002}'.", ) return None # unreachable: output_error sys.exits diff --git a/roboflow/core/project.py b/roboflow/core/project.py index c6028f41..e43edd22 100644 --- a/roboflow/core/project.py +++ b/roboflow/core/project.py @@ -1176,9 +1176,11 @@ def autolabel_preview( model: Foundation model id from ``Workspace.autolabel_models()`` (e.g. ``"gpt-6-astra-boxes"``, ``"sam3-rle"``, ``"gemini-boxes"``). image: HTTPS URL, local file path, or base64-encoded image. - ontology: ``{"class name": "text prompt"}``, a plain list of class - names, or ``[{"class": ..., "prompt": ...}]`` when one class - needs several prompts. Defaults to the dataset's own classes. + ontology: ``{"text prompt": "class name"}`` -- keyed by prompt, so + several prompts can share one class + (``{"kitten": "cat", "tabby": "cat"}``). A plain list of class + names prompts each class with its own name. Defaults to the + dataset's own classes. confidence_threshold: Detection threshold between 0.0 and 1.0 (sam3 only; other models report fixed confidence). @@ -1223,14 +1225,14 @@ def autolabel( model_type: ``"foundational"`` (hosted foundation model, sent as-is; the backend resolves catalog ids) or ``"roboflow"`` (a Roboflow-trained model). - ontology: ``{"class name": "text prompt"}``, a plain list of class - names, or ``[{"class": ..., "prompt": ...}]`` when one class - needs several prompts (``{"cat": ...}`` can only carry one, - since dict keys are unique). For models with - ``ontologyFormat="promptMap"`` (sam3) the prompts are sent to - the model; for ``ontologyFormat="classes"`` only the class - names are used. Defaults to the dataset's classes (or the - trained model's classes for ``model_type="roboflow"``). + ontology: ``{"text prompt": "class name"}`` -- keyed by prompt, not + by class, so several prompts can collapse onto one output class + (``{"kitten": "cat", "tabby": "cat"}`` labels both as ``cat``). + A plain list of class names prompts each class with its own + name. For models with ``ontologyFormat="promptMap"`` (sam3) the + prompts are sent to the model; for ``ontologyFormat="classes"`` + only the class names are used. Defaults to the dataset's classes + (or the trained model's classes for ``model_type="roboflow"``). num_images: Number of images from the batch to label. Defaults to the whole batch. confidence: Confidence threshold applied to every class (mirrors @@ -1249,7 +1251,7 @@ def autolabel( Example: >>> job = project.autolabel("batch-id", model="gpt-6-astra-boxes", - ... ontology={"cat": "a cat", "dog": "a dog"}) + ... ontology={"a cat": "cat", "a dog": "dog"}) >>> project.autolabel_job(job["jobId"])["status"] """ wire_model_type, model_options = _resolve_autolabel_model(model, model_type, model_options) diff --git a/roboflow/util/autolabel_utils.py b/roboflow/util/autolabel_utils.py index 07b4c2a5..91920405 100644 --- a/roboflow/util/autolabel_utils.py +++ b/roboflow/util/autolabel_utils.py @@ -4,7 +4,7 @@ import base64 import os -from typing import Any, Dict, Iterable, List, Optional, Tuple, Union +from typing import Any, Dict, Iterable, Optional, Tuple, Union MODEL_TYPES = ("foundational", "roboflow") @@ -23,65 +23,33 @@ def image_payload(image: str) -> Dict[str, str]: return {"type": "base64", "value": image} -OntologyEntry = Dict[str, str] -Ontology = Union[Dict[str, str], Iterable[Union[str, OntologyEntry]]] +Ontology = Union[Dict[str, str], Iterable[str]] -def ontology_payload(ontology: Optional[Ontology]) -> Optional[List[OntologyEntry]]: - """Serialize an ontology into the ``[{"class", "prompt"}]`` wire form. +def ontology_payload(ontology: Optional[Ontology]) -> Optional[Dict[str, str]]: + """Normalize an ontology into the API's ``{prompt: class name}`` object. - Accepts, in order of convenience: + Note the direction: the **key is the text prompt** sent to the model and the + **value is the class name** written onto the annotations. It reads backwards + at first, but it is the shape that lets several prompts collapse onto one + output class, which is what the ontology is for:: - * ``{"cat": "a cat"}`` — one prompt per class, the common case. - * ``["cat", "dog"]`` — each class is its own prompt. - * ``[{"class": "cat", "prompt": "kitten"}, {"class": "cat", "prompt": "tabby"}]`` - — several prompts for one class, which a class-keyed dict cannot express - because its keys have to be unique. ``prompt`` defaults to ``class``. + {"kitten": "cat", "tabby": "cat", "puppy": "dog"} - The API's own object form is ``{prompt: class}``, the ``CaptionOntology`` - shape the labeling worker consumes, so a bare dict is ambiguous on the - wire: the two sides are both strings and only key order says which is - which. The list form names them, and the backend normalizes it on both the - preview and the start path. + A class-keyed object could not express that, since its keys would have to be + unique. This is also the ``CaptionOntology`` shape the labeling worker + consumes, so nothing is translated on the way out. + + A plain iterable of class names is expanded to ``{"cat": "cat"}``, each class + prompted with its own name. """ if ontology is None: return None if isinstance(ontology, str): raise ValueError(f"ontology must be a mapping or a list of classes, not a bare string {ontology!r}") if isinstance(ontology, dict): - entries = [{"class": name, "prompt": prompt} for name, prompt in ontology.items()] - else: - entries = [_ontology_entry(item) for item in ontology] - _reject_ambiguous_prompts(entries) - return entries - - -def _ontology_entry(item: Union[str, OntologyEntry]) -> OntologyEntry: - if isinstance(item, str): - return {"class": item, "prompt": item} - if isinstance(item, dict) and "class" in item: - return {"class": item["class"], "prompt": item.get("prompt", item["class"])} - raise ValueError( - f"ontology entries must be a class name or a {{'class': ..., 'prompt': ...}} mapping, got {item!r}" - ) - - -def _reject_ambiguous_prompts(entries: List[OntologyEntry]) -> None: - """Refuse a prompt claimed by two classes. - - The backend keys its ontology by prompt, so it would keep whichever class - came last and silently drop the other, leaving that class unlabeled for the - whole job with nothing in the response to explain why. - """ - by_prompt: Dict[str, str] = {} - for entry in entries: - claimed = by_prompt.setdefault(entry["prompt"], entry["class"]) - if claimed != entry["class"]: - raise ValueError( - f"ontology maps the prompt {entry['prompt']!r} to both {claimed!r} and " - f"{entry['class']!r}. The API keys its ontology by prompt, so one of the two " - "classes would be dropped. Give each class a distinct prompt." - ) + return dict(ontology) + return {name: name for name in ontology} def resolve_model( diff --git a/tests/adapters/test_autolabel.py b/tests/adapters/test_autolabel.py index 7a3bb525..45e8023a 100644 --- a/tests/adapters/test_autolabel.py +++ b/tests/adapters/test_autolabel.py @@ -43,17 +43,12 @@ def test_preview_contract(self, mock_post): "proj", model_type="sam3-rle", image=image, - ontology=[{"class": "cat", "prompt": "cat"}], + ontology={"cat": "cat"}, confidence_threshold=0.4, ) self.assertEqual( mock_post.call_args.kwargs["json"], - { - "modelType": "sam3-rle", - "image": image, - "ontology": [{"class": "cat", "prompt": "cat"}], - "confidenceThreshold": 0.4, - }, + {"modelType": "sam3-rle", "image": image, "ontology": {"cat": "cat"}, "confidenceThreshold": 0.4}, ) @patch("roboflow.adapters.rfapi.requests.post") @@ -79,7 +74,7 @@ def test_start_job_contract_full_payload(self, mock_post): "proj", batch_id="batch-1", model_type="custom_roboflow", - ontology=[{"class": "cat", "prompt": "a cat"}], + ontology={"a cat": "cat"}, num_images_to_label=10, default_confidence=0.5, confidence_thresholds={"cat": 0.6}, @@ -92,7 +87,7 @@ def test_start_job_contract_full_payload(self, mock_post): { "batchId": "batch-1", "modelType": "custom_roboflow", - "ontology": [{"class": "cat", "prompt": "a cat"}], + "ontology": {"a cat": "cat"}, "numImagesToLabel": 10, "defaultConfidence": 0.5, "confidenceThresholds": {"cat": 0.6}, diff --git a/tests/cli/test_autolabel_handler.py b/tests/cli/test_autolabel_handler.py index 1cfb10d6..a7104475 100644 --- a/tests/cli/test_autolabel_handler.py +++ b/tests/cli/test_autolabel_handler.py @@ -80,7 +80,7 @@ def test_classes_become_identity_ontology(self, _resolve, mock_api): "proj", model_type="sam3-rle", image={"type": "url", "value": "https://example.com/cat.jpg"}, - ontology=[{"class": "cat", "prompt": "cat"}, {"class": "dog", "prompt": "dog"}], + ontology={"cat": "cat", "dog": "dog"}, confidence_threshold=0.4, ) @@ -90,15 +90,15 @@ def test_ontology_json_takes_precedence_over_classes(self, _resolve, mock_api): runner.invoke( app, ["autolabel", "preview", "-p", "ws/proj", "-m", "sam3-rle", "--image", "https://x/y.jpg"] - + ["--class", "cat", "--ontology", '{"cat": "a tabby cat"}'], + + ["--class", "cat", "--ontology", '{"a tabby cat": "cat"}'], ) - self.assertEqual(mock_api.call_args.kwargs["ontology"], [{"class": "cat", "prompt": "a tabby cat"}]) + self.assertEqual(mock_api.call_args.kwargs["ontology"], {"a tabby cat": "cat"}) @patch("roboflow.adapters.rfapi.preview_autolabel", return_value={}) @patch(_RESOLVE_PROJECT, return_value=("key", "ws", "proj")) def test_ontology_can_be_read_from_a_file(self, _resolve, mock_api): with tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) as handle: - json.dump({"cat": "a tabby cat"}, handle) + json.dump({"a tabby cat": "cat"}, handle) path = handle.name try: result = runner.invoke( @@ -109,32 +109,19 @@ def test_ontology_can_be_read_from_a_file(self, _resolve, mock_api): finally: os.unlink(path) self.assertEqual(result.exit_code, 0, result.output) - self.assertEqual(mock_api.call_args.kwargs["ontology"], [{"class": "cat", "prompt": "a tabby cat"}]) + self.assertEqual(mock_api.call_args.kwargs["ontology"], {"a tabby cat": "cat"}) @patch("roboflow.adapters.rfapi.preview_autolabel", return_value={}) @patch(_RESOLVE_PROJECT, return_value=("key", "ws", "proj")) - def test_ontology_accepts_a_json_array_for_multi_prompt_classes(self, _resolve, mock_api): + def test_several_prompts_may_share_one_class(self, _resolve, mock_api): + # Keying by prompt is what makes this expressible at all. result = runner.invoke( app, ["autolabel", "preview", "-p", "ws/proj", "-m", "sam3-rle", "--image", "https://x/y.jpg"] - + ["--ontology", '[{"class": "cat", "prompt": "kitten"}, {"class": "cat", "prompt": "tabby"}]'], + + ["--ontology", '{"kitten": "cat", "tabby": "cat"}'], ) self.assertEqual(result.exit_code, 0, result.output) - self.assertEqual( - mock_api.call_args.kwargs["ontology"], - [{"class": "cat", "prompt": "kitten"}, {"class": "cat", "prompt": "tabby"}], - ) - - @patch("roboflow.adapters.rfapi.preview_autolabel") - @patch(_RESOLVE_PROJECT, return_value=("key", "ws", "proj")) - def test_one_prompt_for_two_classes_errors_without_calling_api(self, _resolve, mock_api): - result = runner.invoke( - app, - ["autolabel", "preview", "-p", "ws/proj", "-m", "sam3-rle", "--image", "https://x/y.jpg"] - + ["--ontology", '[{"class": "cat", "prompt": "animal"}, {"class": "dog", "prompt": "animal"}]'], - ) - self.assertNotEqual(result.exit_code, 0) - mock_api.assert_not_called() + self.assertEqual(mock_api.call_args.kwargs["ontology"], {"kitten": "cat", "tabby": "cat"}) @patch("roboflow.adapters.rfapi.preview_autolabel") @patch(_RESOLVE_PROJECT, return_value=("key", "ws", "proj")) @@ -166,7 +153,7 @@ def test_foundational_start(self, _resolve, mock_api): "proj", batch_id="batch-1", model_type="gpt-6-astra-boxes", - ontology=[{"class": "cat", "prompt": "cat"}], + ontology={"cat": "cat"}, num_images_to_label=10, default_confidence=0.5, confidence_thresholds=None, diff --git a/tests/test_project_autolabel.py b/tests/test_project_autolabel.py index e5ad9b70..9b0deb0c 100644 --- a/tests/test_project_autolabel.py +++ b/tests/test_project_autolabel.py @@ -11,7 +11,7 @@ def test_autolabel_foundational_passes_model_as_is(self, mock_start): result = self.project.autolabel( "batch-1", "gpt-6-astra-boxes", - ontology={"cat": "a cat"}, + ontology={"a cat": "cat"}, num_images=5, confidence=0.4, reviewer_email="reviewer@example.com", @@ -24,7 +24,7 @@ def test_autolabel_foundational_passes_model_as_is(self, mock_start): PROJECT_NAME, batch_id="batch-1", model_type="gpt-6-astra-boxes", - ontology=[{"class": "cat", "prompt": "a cat"}], + ontology={"a cat": "cat"}, num_images_to_label=5, default_confidence=0.4, confidence_thresholds=None, @@ -61,7 +61,7 @@ def test_autolabel_preview_builds_image_payload(self, mock_preview): PROJECT_NAME, model_type="sam3-rle", image={"type": "url", "value": "https://example.com/cat.jpg"}, - ontology=[{"class": "cat", "prompt": "cat"}], + ontology={"cat": "cat"}, confidence_threshold=0.3, ) diff --git a/tests/util/test_autolabel_utils.py b/tests/util/test_autolabel_utils.py index 3b384a44..c5cd3aa6 100644 --- a/tests/util/test_autolabel_utils.py +++ b/tests/util/test_autolabel_utils.py @@ -35,64 +35,30 @@ class TestOntologyPayload(unittest.TestCase): def test_none_stays_none(self): self.assertIsNone(ontology_payload(None)) - def test_class_to_prompt_mapping_is_serialized_explicitly(self): - # The wire form names both sides, so the API never has to guess which - # of the two strings is the class and which is the prompt. - self.assertEqual( - ontology_payload({"cat": "a cat", "dog": "a dog"}), - [{"class": "cat", "prompt": "a cat"}, {"class": "dog", "prompt": "a dog"}], - ) - - def test_list_of_classes_becomes_identity_prompts(self): - self.assertEqual( - ontology_payload(["cat", "dog"]), - [{"class": "cat", "prompt": "cat"}, {"class": "dog", "prompt": "dog"}], - ) - - def test_one_class_may_carry_several_prompts(self): - # The reason the list form is accepted at all: {"cat": ...} has room for - # exactly one prompt, because dict keys are unique. - self.assertEqual( - ontology_payload( - [ - {"class": "cat", "prompt": "kitten"}, - {"class": "cat", "prompt": "tabby"}, - ] - ), - [{"class": "cat", "prompt": "kitten"}, {"class": "cat", "prompt": "tabby"}], - ) + def test_prompt_keyed_mapping_is_passed_through(self): + self.assertEqual(ontology_payload({"a cat": "cat"}), {"a cat": "cat"}) - def test_list_entries_default_the_prompt_to_the_class(self): + def test_several_prompts_may_share_one_class(self): + # The reason the object is keyed by prompt rather than by class: a + # class-keyed object has room for exactly one prompt per class. self.assertEqual( - ontology_payload([{"class": "cat"}, "dog"]), - [{"class": "cat", "prompt": "cat"}, {"class": "dog", "prompt": "dog"}], + ontology_payload({"kitten": "cat", "tabby": "cat", "puppy": "dog"}), + {"kitten": "cat", "tabby": "cat", "puppy": "dog"}, ) - def test_one_prompt_claimed_by_two_classes_is_rejected(self): - # The API keys its ontology by prompt, so it would keep "dog" and drop - # "cat" without saying so. Name the collision instead. - with self.assertRaises(ValueError) as ctx: - ontology_payload([{"class": "cat", "prompt": "animal"}, {"class": "dog", "prompt": "animal"}]) - self.assertIn("animal", str(ctx.exception)) - self.assertIn("cat", str(ctx.exception)) - self.assertIn("dog", str(ctx.exception)) + def test_list_of_classes_becomes_identity_prompts(self): + self.assertEqual(ontology_payload(["cat", "dog"]), {"cat": "cat", "dog": "dog"}) - def test_a_repeated_class_prompt_pair_is_not_a_collision(self): - self.assertEqual( - ontology_payload([{"class": "cat", "prompt": "cat"}, "cat"]), - [{"class": "cat", "prompt": "cat"}, {"class": "cat", "prompt": "cat"}], - ) + def test_the_result_is_a_copy(self): + source = {"a cat": "cat"} + self.assertIsNot(ontology_payload(source), source) def test_bare_string_is_rejected_rather_than_iterated_per_character(self): with self.assertRaises(ValueError): ontology_payload("cat") - def test_malformed_entry_is_rejected(self): - with self.assertRaises(ValueError): - ontology_payload([{"prompt": "a cat"}]) - def test_empty_is_preserved_as_empty(self): - self.assertEqual(ontology_payload({}), []) + self.assertEqual(ontology_payload({}), {}) class TestResolveModel(unittest.TestCase): From 3d4a4f3310914f4d89e52643bb0924d10acfa743 Mon Sep 17 00:00:00 2001 From: lucas-fochesatto Date: Tue, 8 Sep 2026 08:30:56 -0300 Subject: [PATCH 7/7] Address the merge-blocking review items on the auto-label CLI Five fixes from Iuri's review, in the order he listed them. A mistyped --image path was silently sent as base64. image_payload now expands ~, and anything that is neither a URL, an existing file nor valid base64 is rejected with "Image file not found" instead of reaching the API and failing there with a generic inference error. An unreadable image file escaped as a raw traceback, since the payload was built inside the operation _run wraps and _run only catches RoboflowError and ValueError. preview now builds the payload before resolving the project, so both the not-found and the OSError case print a structured error, and they fail before any network call. `autolabel job` could not find a job `autolabel start -p other-ws/proj` had just created, because start derived the workspace from the shorthand and job only read --workspace or the default. job now takes the same -p and resolves the workspace the same way. preserveExistingAnnotations was not exposed, and the server default (false) replaces annotations already on the batch images. Added `preserve_existing_annotations` to rfapi.start_autolabel_job and Project.autolabel, and `--preserve-existing` to the CLI. The handler re-implemented the credential resolvers and diverged: a missing default workspace exited 1 where the CLI contract says 2. It now uses resolve_ws_and_key, and the project variant is lifted from annotation.py into _resolver.py as resolve_project_context so there is one copy instead of three. Folding _models into _workspace_command fell out of the same change. Co-Authored-By: Claude Fable 5.1 --- CHANGELOG.md | 5 +- CLI-COMMANDS.md | 8 ++- roboflow/adapters/rfapi.py | 6 +- roboflow/cli/_resolver.py | 28 +++++++++ roboflow/cli/handlers/annotation.py | 15 +---- roboflow/cli/handlers/autolabel.py | 94 ++++++++++++++++------------- roboflow/core/project.py | 6 ++ roboflow/util/autolabel_utils.py | 32 ++++++++-- tests/adapters/test_autolabel.py | 2 + tests/cli/test_autolabel_handler.py | 79 ++++++++++++++++++++++-- tests/test_project_autolabel.py | 7 +++ tests/util/test_autolabel_utils.py | 34 +++++++++++ 12 files changed, 249 insertions(+), 67 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e28f4c81..0fc224cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,10 +16,13 @@ All notable changes to this project will be documented in this file. - `Project.autolabel(batch_id, model, model_type="foundational" | "roboflow", ...)` — start a job over a batch; returns `{jobId, annotationJobId}`. The `ontology` is keyed by prompt (`{"kitten": "cat", "tabby": "cat"}`), so - several prompts can share one output class. + several prompts can share one output class. `preserve_existing_annotations=True` + keeps annotations already on the images (the server default replaces them). - `Project.autolabel_job(job_id)` / `Workspace.autolabel_job(job_id)` — poll per-subjob progress. - `roboflow autolabel models | preview | start | job` CLI commands. + `start --preserve-existing` mirrors the SDK flag; `job -p ws/project` + resolves the workspace the same way `start` does. ## 1.4.1 diff --git a/CLI-COMMANDS.md b/CLI-COMMANDS.md index a7999ff4..10ec349a 100644 --- a/CLI-COMMANDS.md +++ b/CLI-COMMANDS.md @@ -248,7 +248,9 @@ roboflow autolabel preview -p my-project -m sam3-rle --image https://example.com roboflow autolabel start -p my-project --batch-id -m gpt-6-astra-boxes \ --ontology '{"a cat": "cat", "a dog": "dog"}' --confidence 0.5 --reviewer b@co.com roboflow autolabel start -p my-project --batch-id -m my-project/3 --model-type roboflow +roboflow autolabel start -p my-project --batch-id -m sam3-rle --preserve-existing roboflow autolabel job +roboflow autolabel job -p other-workspace/my-project ``` `models` lists the catalog for the workspace (id, availability, credits per @@ -260,7 +262,11 @@ repeated `--class` flags or as `--ontology` JSON. The ontology is keyed by matches either prompt as class `cat`. That direction is what lets several prompts share one output class. JSON options also accept a curl-style file reference (`--ontology @ontology.json`). -`--image` accepts an HTTPS URL or a local file. +`--image` accepts an HTTPS URL or a local file. By default a job replaces the +annotations already on the batch images; `--preserve-existing` keeps them and +only adds new ones. `job` looks the id up in your default workspace, so when +the job was started with a `workspace/project` shorthand pass the same `-p` to +`job`. The same operations are available in Python: diff --git a/roboflow/adapters/rfapi.py b/roboflow/adapters/rfapi.py index 535f6456..e29b2b2e 100644 --- a/roboflow/adapters/rfapi.py +++ b/roboflow/adapters/rfapi.py @@ -1398,6 +1398,7 @@ def start_autolabel_job( run_nms=None, reviewer_email=None, model_options=None, + preserve_existing_annotations=None, ): """Start a hosted auto-label job over a batch. @@ -1409,7 +1410,9 @@ def start_autolabel_job( The backend fans ``default_confidence`` out across the ontology when ``confidence_thresholds`` is omitted and defaults ``num_images_to_label`` - to the whole batch. Returns ``{jobId, annotationJobId, message}``. + to the whole batch. ``preserve_existing_annotations`` keeps annotations + already on the images and only adds new ones; the server default (False) + replaces them. Returns ``{jobId, annotationJobId, message}``. """ payload = {"batchId": batch_id, "modelType": model_type} optional = { @@ -1420,6 +1423,7 @@ def start_autolabel_job( "runNMS": run_nms, "reviewerEmail": reviewer_email, "modelOptions": model_options, + "preserveExistingAnnotations": preserve_existing_annotations, } payload.update({key: value for key, value in optional.items() if value is not None}) response = requests.post( diff --git a/roboflow/cli/_resolver.py b/roboflow/cli/_resolver.py index 11f5e3c5..25b45d1e 100644 --- a/roboflow/cli/_resolver.py +++ b/roboflow/cli/_resolver.py @@ -135,3 +135,31 @@ def resolve_ws_and_key(args) -> Optional[Tuple[str, str]]: return None return ws, api_key + + +def resolve_project_context(args) -> Optional[Tuple[str, str, str]]: + """Resolve API key, workspace and project from CLI args. + + Parses ``args.project`` (any ``resolve_resource`` shorthand, honouring + ``args.workspace`` as an override) and loads the API key for that + workspace. Returns ``(api_key, workspace_url, project_slug)`` or ``None`` + after calling ``output_error`` on failure; a missing key exits with the + auth code (2), matching ``resolve_ws_and_key``. + """ + from roboflow.cli._output import output_error + from roboflow.config import load_roboflow_api_key + + try: + workspace, project, _version = resolve_resource( + args.project, workspace_override=getattr(args, "workspace", None) + ) + except ValueError as exc: + output_error(args, str(exc)) + return None + + api_key = getattr(args, "api_key", None) or load_roboflow_api_key(workspace) + if not api_key: + output_error(args, "No API key found.", hint="Set ROBOFLOW_API_KEY or run 'roboflow auth login'.", exit_code=2) + return None + + return api_key, workspace, project diff --git a/roboflow/cli/handlers/annotation.py b/roboflow/cli/handlers/annotation.py index 2ceca056..f4da8b92 100644 --- a/roboflow/cli/handlers/annotation.py +++ b/roboflow/cli/handlers/annotation.py @@ -465,20 +465,9 @@ def job_delete_annotations( def _resolve_project_context(args: Any) -> Optional[tuple[str, str, str]]: - from roboflow.cli._output import output_error - from roboflow.cli._resolver import resolve_resource - from roboflow.config import load_roboflow_api_key + from roboflow.cli._resolver import resolve_project_context - try: - workspace, project, _version = resolve_resource(args.project, workspace_override=args.workspace) - except ValueError as exc: - output_error(args, str(exc)) - return None - api_key = args.api_key or load_roboflow_api_key(workspace) - if not api_key: - output_error(args, "No API key found.", hint="Set ROBOFLOW_API_KEY or run 'roboflow auth login'.", exit_code=2) - return None - return api_key, workspace, project + return resolve_project_context(args) def _call(args: Any, operation: Callable[[str, str, str], Any]) -> Any: diff --git a/roboflow/cli/handlers/autolabel.py b/roboflow/cli/handlers/autolabel.py index 5061ba46..0fe3eab2 100644 --- a/roboflow/cli/handlers/autolabel.py +++ b/roboflow/cli/handlers/autolabel.py @@ -42,7 +42,7 @@ def preview( """Preview one image with a foundation model. Free: no job is created.""" args = ctx_to_args(ctx, project=project) resolved_ontology = _parse_ontology(args, ontology, classes) - from roboflow.util.autolabel_utils import image_payload + resolved_image = _parse_image(args, image) _project_command( args, @@ -51,7 +51,7 @@ def preview( workspace, proj, model_type=model, - image=image_payload(image), + image=resolved_image, ontology=resolved_ontology, confidence_threshold=confidence, ), @@ -107,6 +107,14 @@ def start( "--model-options", help='JSON model options, e.g. \'{"outputFormat": "polygon"}\', or @options.json' ), ] = None, + preserve_existing: Annotated[ + bool, + typer.Option( + "--preserve-existing", + help="Keep annotations already on the batch images and only add new ones " + "(by default the job replaces them)", + ), + ] = False, ) -> None: """Start a hosted auto-label job over a batch of images.""" args = ctx_to_args(ctx, project=project) @@ -130,6 +138,7 @@ def start_job(key: str, workspace: str, proj: str) -> Any: run_nms=False if no_nms else None, reviewer_email=reviewer, model_options=wire_options, + preserve_existing_annotations=True if preserve_existing else None, ) _project_command(args, start_job) @@ -139,10 +148,26 @@ def start_job(key: str, workspace: str, proj: str) -> Any: def job( ctx: typer.Context, job_id: Annotated[str, typer.Argument(help="Auto-label job ID returned by 'autolabel start'")], + project: Annotated[ + Optional[str], + typer.Option( + "-p", + "--project", + help="Project the job was started on (accepts 'workspace/project'); resolves the workspace " + "the same way 'autolabel start' does. Defaults to --workspace or the default workspace.", + ), + ] = None, ) -> None: """Get status and per-subjob progress for an auto-label job.""" - args = ctx_to_args(ctx) - _workspace_command(args, lambda key, workspace: _rfapi().get_autolabel_job(key, workspace, job_id)) + args = ctx_to_args(ctx, project=project) + + def get_job(key: str, workspace: str, *_project: str) -> Any: + return _rfapi().get_autolabel_job(key, workspace, job_id) + + if project: + _project_command(args, get_job) + else: + _workspace_command(args, get_job) # --------------------------------------------------------------------------- @@ -180,37 +205,19 @@ def _parse_ontology(args: Any, ontology: Optional[str], classes: Optional[list[s return None -def _resolve_workspace(args: Any) -> tuple[Optional[str], Optional[str]]: +def _parse_image(args: Any, image: str) -> dict: + """Build the --image payload before any network call, so a bad path fails fast and cleanly.""" from roboflow.cli._output import output_error - from roboflow.cli._resolver import resolve_default_workspace - from roboflow.config import load_roboflow_api_key - - workspace_url = args.workspace or resolve_default_workspace(api_key=args.api_key) - if not workspace_url: - output_error(args, "No workspace specified.", hint="Use --workspace or run 'roboflow auth login'.") - return None, None - api_key = args.api_key or load_roboflow_api_key(workspace_url) - if not api_key: - output_error(args, "No API key found.", hint="Set ROBOFLOW_API_KEY or run 'roboflow auth login'.", exit_code=2) - return None, None - return api_key, workspace_url - - -def _resolve_project(args: Any) -> tuple[Optional[str], Optional[str], Optional[str]]: - from roboflow.cli._output import output_error - from roboflow.cli._resolver import resolve_resource - from roboflow.config import load_roboflow_api_key + from roboflow.util.autolabel_utils import image_payload + hint = "Pass --image as an HTTPS URL or the path of a readable local image file." try: - workspace, project, _version = resolve_resource(args.project, workspace_override=args.workspace) + return image_payload(image) + except OSError as exc: + output_error(args, f"Cannot read image {image}: {exc.strerror or exc}", hint=hint) except ValueError as exc: - output_error(args, str(exc)) - return None, None, None - api_key = args.api_key or load_roboflow_api_key(workspace) - if not api_key: - output_error(args, "No API key found.", hint="Set ROBOFLOW_API_KEY or run 'roboflow auth login'.", exit_code=2) - return None, None, None - return api_key, workspace, project + output_error(args, str(exc), hint=hint) + return {} # unreachable: output_error exits def _run(args: Any, operation: Callable[[], Any], text: Optional[Callable[[Any], str]] = None) -> None: @@ -227,17 +234,25 @@ def _run(args: Any, operation: Callable[[], Any], text: Optional[Callable[[Any], output(args, data, text=text(data) if text else None) -def _workspace_command(args: Any, operation: Callable[[str, str], Any]) -> None: - api_key, workspace_url = _resolve_workspace(args) - if api_key is None or workspace_url is None: +def _workspace_command( + args: Any, operation: Callable[[str, str], Any], text: Optional[Callable[[Any], str]] = None +) -> None: + from roboflow.cli._resolver import resolve_ws_and_key + + resolved = resolve_ws_and_key(args) + if resolved is None: return - _run(args, lambda: operation(api_key, workspace_url)) + workspace_url, api_key = resolved + _run(args, lambda: operation(api_key, workspace_url), text=text) def _project_command(args: Any, operation: Callable[[str, str, str], Any]) -> None: - api_key, workspace, project = _resolve_project(args) - if api_key is None or workspace is None or project is None: + from roboflow.cli._resolver import resolve_project_context + + resolved = resolve_project_context(args) + if resolved is None: return + api_key, workspace, project = resolved _run(args, lambda: operation(api_key, workspace, project)) @@ -262,7 +277,4 @@ def table(data: Any) -> str: headers=["ID", "NAME", "AVAILABLE", "DEFAULT", "CREDITS/IMAGE", "ONTOLOGY"], ) - api_key, workspace_url = _resolve_workspace(args) - if not workspace_url: - return - _run(args, lambda: _rfapi().list_autolabel_models(api_key, workspace_url), text=table) + _workspace_command(args, lambda key, workspace: _rfapi().list_autolabel_models(key, workspace), text=table) diff --git a/roboflow/core/project.py b/roboflow/core/project.py index e43edd22..f9fe8fb5 100644 --- a/roboflow/core/project.py +++ b/roboflow/core/project.py @@ -1212,6 +1212,7 @@ def autolabel( run_nms: Optional[bool] = None, reviewer_email: Optional[str] = None, model_options: Optional[Dict] = None, + preserve_existing_annotations: Optional[bool] = None, ) -> Dict: """Start a hosted auto-label job over a batch of images. @@ -1244,6 +1245,10 @@ def autolabel( workspace member; defaults to the workspace owner. model_options: Model-specific options, e.g. ``{"outputFormat": "polygon"}`` for segmentation output. + preserve_existing_annotations: ``True`` keeps annotations already + on the batch images and only adds new ones. The server default + (``False``) replaces them, so set this when the batch contains + images that were already labeled or reviewed. Returns: Dict: ``{jobId, annotationJobId, message}``. Poll progress with @@ -1268,6 +1273,7 @@ def autolabel( run_nms=run_nms, reviewer_email=reviewer_email, model_options=model_options, + preserve_existing_annotations=preserve_existing_annotations, ) def autolabel_job(self, job_id: str) -> Dict: diff --git a/roboflow/util/autolabel_utils.py b/roboflow/util/autolabel_utils.py index 91920405..097e733d 100644 --- a/roboflow/util/autolabel_utils.py +++ b/roboflow/util/autolabel_utils.py @@ -3,6 +3,7 @@ from __future__ import annotations import base64 +import binascii import os from typing import Any, Dict, Iterable, Optional, Tuple, Union @@ -12,15 +13,36 @@ def image_payload(image: str) -> Dict[str, str]: """Build the ``{type, value}`` image payload for the auto-label preview endpoint. - Accepts an HTTP(S) URL, a local file path (read and base64-encoded), or an - already base64-encoded string. + Accepts an HTTP(S) URL, a local file path (``~`` is expanded; the file is + read and base64-encoded) or an already base64-encoded string. Anything + else is treated as a mistyped path and rejected here, rather than being + sent to the API as "base64" and failing there with a generic inference + error. + + Raises: + ValueError: ``image`` is neither a URL, an existing file nor base64. + OSError: the file exists but cannot be read. """ if image.startswith(("http://", "https://")): return {"type": "url", "value": image} - if os.path.isfile(image): - with open(image, "rb") as handle: + path = os.path.expanduser(image) + if os.path.isfile(path): + with open(path, "rb") as handle: return {"type": "base64", "value": base64.b64encode(handle.read()).decode("ascii")} - return {"type": "base64", "value": image} + compact = "".join(image.split()) + if _is_base64(compact): + return {"type": "base64", "value": compact} + raise ValueError(f"Image file not found: {image} (expected an HTTPS URL, an existing file path or base64 data)") + + +def _is_base64(value: str) -> bool: + if not value: + return False + try: + base64.b64decode(value, validate=True) + except (binascii.Error, ValueError): + return False + return True Ontology = Union[Dict[str, str], Iterable[str]] diff --git a/tests/adapters/test_autolabel.py b/tests/adapters/test_autolabel.py index 45e8023a..27947d79 100644 --- a/tests/adapters/test_autolabel.py +++ b/tests/adapters/test_autolabel.py @@ -81,6 +81,7 @@ def test_start_job_contract_full_payload(self, mock_post): run_nms=False, reviewer_email="reviewer@example.com", model_options={"modelId": "proj/3"}, + preserve_existing_annotations=True, ) self.assertEqual( mock_post.call_args.kwargs["json"], @@ -94,6 +95,7 @@ def test_start_job_contract_full_payload(self, mock_post): "runNMS": False, "reviewerEmail": "reviewer@example.com", "modelOptions": {"modelId": "proj/3"}, + "preserveExistingAnnotations": True, }, ) diff --git a/tests/cli/test_autolabel_handler.py b/tests/cli/test_autolabel_handler.py index a7104475..580032f4 100644 --- a/tests/cli/test_autolabel_handler.py +++ b/tests/cli/test_autolabel_handler.py @@ -13,8 +13,12 @@ runner = CliRunner() -_RESOLVE_PROJECT = "roboflow.cli.handlers.autolabel._resolve_project" -_RESOLVE_WORKSPACE = "roboflow.cli.handlers.autolabel._resolve_workspace" +# The handler resolves credentials through the shared CLI resolvers (imported lazily), +# so patching them at their definition site is what the handler sees. +_RESOLVE_PROJECT = "roboflow.cli._resolver.resolve_project_context" +_RESOLVE_WORKSPACE = "roboflow.cli._resolver.resolve_ws_and_key" +_DEFAULT_WORKSPACE = "roboflow.cli._resolver.resolve_default_workspace" +_IMAGE_PAYLOAD = "roboflow.util.autolabel_utils.image_payload" class TestAutolabelRegistration(unittest.TestCase): @@ -27,7 +31,7 @@ def test_subcommands_have_help(self): class TestAutolabelModels(unittest.TestCase): @patch("roboflow.adapters.rfapi.list_autolabel_models") - @patch(_RESOLVE_WORKSPACE, return_value=("key", "ws")) + @patch(_RESOLVE_WORKSPACE, return_value=("ws", "key")) def test_text_output_is_a_table(self, _resolve, mock_api): mock_api.return_value = { "models": [ @@ -42,11 +46,20 @@ def test_text_output_is_a_table(self, _resolve, mock_api): mock_api.assert_called_once_with("key", "ws") @patch("roboflow.adapters.rfapi.list_autolabel_models", return_value={"models": [{"id": "sam3-rle"}]}) - @patch(_RESOLVE_WORKSPACE, return_value=("key", "ws")) + @patch(_RESOLVE_WORKSPACE, return_value=("ws", "key")) def test_json_output(self, _resolve, _mock_api): result = runner.invoke(app, ["--json", "autolabel", "models"]) self.assertEqual(json.loads(result.output), {"models": [{"id": "sam3-rle"}]}) + @patch("roboflow.adapters.rfapi.list_autolabel_models") + @patch(_DEFAULT_WORKSPACE, return_value=None) + def test_missing_workspace_exits_with_auth_code(self, _default, mock_api): + # CLAUDE.md pins exit code 2 for auth errors; 'workflow list' exits 2 for the same condition. + with patch.dict(os.environ, {"ROBOFLOW_API_KEY": ""}): + result = runner.invoke(app, ["autolabel", "models"]) + self.assertEqual(result.exit_code, 2, result.output) + mock_api.assert_not_called() + class TestAutolabelPreview(unittest.TestCase): @patch("roboflow.adapters.rfapi.preview_autolabel", return_value={"summary": {"totalDetections": 2}}) @@ -134,6 +147,31 @@ def test_invalid_ontology_json_fails_without_calling_api(self, _resolve, mock_ap self.assertNotEqual(result.exit_code, 0) mock_api.assert_not_called() + @patch("roboflow.adapters.rfapi.preview_autolabel") + @patch(_RESOLVE_PROJECT, return_value=("key", "ws", "proj")) + def test_mistyped_image_path_fails_before_resolving_credentials(self, mock_resolve, mock_api): + result = runner.invoke( + app, + ["autolabel", "preview", "-p", "ws/proj", "-m", "sam3-rle", "--image", "smaple.jpg", "--class", "cat"], + ) + self.assertEqual(result.exit_code, 1, result.output) + self.assertIn("smaple.jpg", result.output) + mock_resolve.assert_not_called() + mock_api.assert_not_called() + + @patch("roboflow.adapters.rfapi.preview_autolabel") + @patch(_RESOLVE_PROJECT, return_value=("key", "ws", "proj")) + @patch(_IMAGE_PAYLOAD, side_effect=PermissionError(13, "Permission denied")) + def test_unreadable_image_is_a_structured_error_not_a_traceback(self, _payload, _resolve, mock_api): + result = runner.invoke( + app, + ["autolabel", "preview", "-p", "ws/proj", "-m", "sam3-rle", "--image", "locked.jpg", "--class", "cat"], + ) + self.assertEqual(result.exit_code, 1, result.output) + self.assertNotIsInstance(result.exception, OSError) + self.assertIn("Permission denied", result.output) + mock_api.assert_not_called() + class TestAutolabelStart(unittest.TestCase): @patch("roboflow.adapters.rfapi.start_autolabel_job", return_value={"jobId": "job-1"}) @@ -160,7 +198,19 @@ def test_foundational_start(self, _resolve, mock_api): run_nms=False, reviewer_email="r@example.com", model_options={"outputFormat": "polygon"}, + preserve_existing_annotations=None, + ) + + @patch("roboflow.adapters.rfapi.start_autolabel_job", return_value={"jobId": "job-1"}) + @patch(_RESOLVE_PROJECT, return_value=("key", "ws", "proj")) + def test_preserve_existing_flag(self, _resolve, mock_api): + # The server default replaces annotations already on the batch images. + result = runner.invoke( + app, + ["autolabel", "start", "-p", "ws/proj", "--batch-id", "batch-1", "-m", "sam3-rle", "--preserve-existing"], ) + self.assertEqual(result.exit_code, 0, result.output) + self.assertIs(mock_api.call_args.kwargs["preserve_existing_annotations"], True) @patch("roboflow.adapters.rfapi.start_autolabel_job", return_value={"jobId": "job-1"}) @patch(_RESOLVE_PROJECT, return_value=("key", "ws", "proj")) @@ -196,9 +246,28 @@ def test_api_error_maps_to_not_found_exit_code(self, _resolve, _mock_api): class TestAutolabelJob(unittest.TestCase): @patch("roboflow.adapters.rfapi.get_autolabel_job", return_value={"status": "running", "progress": 0.5}) - @patch(_RESOLVE_WORKSPACE, return_value=("key", "ws")) + @patch(_RESOLVE_WORKSPACE, return_value=("ws", "key")) def test_json_output(self, _resolve, mock_api): result = runner.invoke(app, ["--json", "autolabel", "job", "job-1"]) self.assertEqual(result.exit_code, 0, result.output) self.assertEqual(json.loads(result.output)["status"], "running") mock_api.assert_called_once_with("key", "ws", "job-1") + + @patch("roboflow.adapters.rfapi.get_autolabel_job", return_value={"status": "running"}) + @patch(_RESOLVE_WORKSPACE) + @patch(_RESOLVE_PROJECT, return_value=("key", "other-ws", "proj")) + def test_project_shorthand_resolves_the_same_workspace_as_start(self, _resolve, mock_ws_resolve, mock_api): + # 'start -p other-ws/proj' creates the job in other-ws; 'job -p other-ws/proj' must look there too, + # since the API 404s on a job from another workspace. + result = runner.invoke(app, ["autolabel", "job", "job-1", "-p", "other-ws/proj"]) + self.assertEqual(result.exit_code, 0, result.output) + mock_api.assert_called_once_with("key", "other-ws", "job-1") + mock_ws_resolve.assert_not_called() + + @patch("roboflow.adapters.rfapi.get_autolabel_job") + @patch(_DEFAULT_WORKSPACE, return_value=None) + def test_missing_workspace_exits_with_auth_code(self, _default, mock_api): + with patch.dict(os.environ, {"ROBOFLOW_API_KEY": ""}): + result = runner.invoke(app, ["autolabel", "job", "job-1"]) + self.assertEqual(result.exit_code, 2, result.output) + mock_api.assert_not_called() diff --git a/tests/test_project_autolabel.py b/tests/test_project_autolabel.py index 9b0deb0c..d6f080cb 100644 --- a/tests/test_project_autolabel.py +++ b/tests/test_project_autolabel.py @@ -31,8 +31,15 @@ def test_autolabel_foundational_passes_model_as_is(self, mock_start): run_nms=None, reviewer_email="reviewer@example.com", model_options=None, + preserve_existing_annotations=None, ) + @patch("roboflow.adapters.rfapi.start_autolabel_job", return_value={"jobId": "job-1"}) + def test_autolabel_forwards_preserve_existing_annotations(self, mock_start): + self.project.autolabel("batch-1", "sam3-rle", preserve_existing_annotations=True) + + self.assertIs(mock_start.call_args.kwargs["preserve_existing_annotations"], True) + @patch("roboflow.adapters.rfapi.start_autolabel_job", return_value={"jobId": "job-1"}) def test_autolabel_roboflow_model_is_sent_as_custom_roboflow(self, mock_start): self.project.autolabel("batch-1", "my-project/3", model_type="roboflow", model_options={"outputFormat": "rle"}) diff --git a/tests/util/test_autolabel_utils.py b/tests/util/test_autolabel_utils.py index c5cd3aa6..59d0b44d 100644 --- a/tests/util/test_autolabel_utils.py +++ b/tests/util/test_autolabel_utils.py @@ -4,6 +4,7 @@ import os import tempfile import unittest +from unittest.mock import patch from roboflow.util.autolabel_utils import image_payload, ontology_payload, resolve_model @@ -30,6 +31,39 @@ def test_other_strings_are_treated_as_base64(self): encoded = base64.b64encode(b"bytes").decode("ascii") self.assertEqual(image_payload(encoded), {"type": "base64", "value": encoded}) + def test_line_wrapped_base64_is_compacted(self): + encoded = base64.b64encode(b"some longer image bytes").decode("ascii") + wrapped = encoded[:8] + "\n" + encoded[8:] + "\n" + self.assertEqual(image_payload(wrapped), {"type": "base64", "value": encoded}) + + def test_home_directory_is_expanded(self): + with tempfile.TemporaryDirectory() as home: + with open(os.path.join(home, "cat.jpg"), "wb") as handle: + handle.write(b"fake-image-bytes") + # expanduser reads HOME on POSIX and USERPROFILE on Windows. + with patch.dict(os.environ, {"HOME": home, "USERPROFILE": home}): + payload = image_payload("~/cat.jpg") + self.assertEqual(base64.b64decode(payload["value"]), b"fake-image-bytes") + + def test_mistyped_path_is_rejected_instead_of_sent_as_base64(self): + with self.assertRaises(ValueError) as ctx: + image_payload("smaple.jpg") + self.assertIn("smaple.jpg", str(ctx.exception)) + + def test_missing_home_path_is_rejected(self): + with self.assertRaises(ValueError): + image_payload("~/definitely-missing-image.png") + + def test_unreadable_file_raises_oserror(self): + with tempfile.NamedTemporaryFile(suffix=".jpg", delete=False) as handle: + path = handle.name + try: + with patch("roboflow.util.autolabel_utils.open", side_effect=PermissionError(13, "denied"), create=True): + with self.assertRaises(OSError): + image_payload(path) + finally: + os.unlink(path) + class TestOntologyPayload(unittest.TestCase): def test_none_stays_none(self):