diff --git a/README.md b/README.md index 1be0577f..98d79be1 100644 --- a/README.md +++ b/README.md @@ -223,6 +223,8 @@ Within the project object, you can retrieve metadata about the project, list ver Within the version object, you can download the dataset version in any model format, train the version on Roboflow, and deploy your own external model to Roboflow. +For custom OWLv2 checkpoints and Auto Label, see [Upload a custom OWLv2 model](docs/owlv2-upload.md). + ## 🏆 Contributing We would love your input on how we can improve the Roboflow Python package! Please see our [contributing guide](https://github.com/roboflow/roboflow-python/blob/main/CONTRIBUTING.md) to get started. Thank you 🙏 to all our contributors! diff --git a/docs/owlv2-upload.md b/docs/owlv2-upload.md new file mode 100644 index 00000000..2f0195a7 --- /dev/null +++ b/docs/owlv2-upload.md @@ -0,0 +1,81 @@ +# Upload a custom OWLv2 model + +Use `model_type="owlv2"` to upload fine-tuned OWLv2 weights for Auto Label and Workflows. +For a standard OWLv2 model, the only required file is the checkpoint: + +```text +weights-upload/ + owlv2.pth +``` + +`owlv2.pth` must contain the complete `Owlv2ForObjectDetection.state_dict()`, including the text encoder and detection heads. +A training checkpoint that wraps the weights under another key is not accepted. +To use image-only requests in Auto Label or the Workflows Object Detection Model block, add `class_names.txt` beside the checkpoint. +It contains unique, nonblank output labels, one per line. Class IDs follow this order. By default, these labels are also the text queries. +If you upload without `class_names.txt`, use `workspace.deploy_model`. An upload to a dataset version requires the labels file. + +The exporter assumes the standard `google/owlv2-base-patch16-ensemble` model configuration, image preprocessing, and tokenizer. +It bundles these defaults at revision `cfd3195ba4ea9592eec887ded089f4c08eff231d` and uses them offline. +Fine-tuning the weights does not require additional files if those components stay unchanged. +The exporter saves the resolved configuration and processor in the inference package. + +## Upload + +Use an object-detection project that the workspace can access. +Replace the workspace, project, folder, and model name with your values. +Set `ROBOFLOW_API_KEY` in your environment before you run this example. + +```python +from roboflow import Roboflow + +workspace = Roboflow().workspace("your-workspace") +workspace.deploy_model( + model_type="owlv2", + model_path="/absolute/path/to/weights-upload", + filename="owlv2.pth", + project_ids=["your-project"], + model_name="custom-owlv2", +) +``` + +## Run local inference + +On a local server that supports `/v2/models/infer`, send your model ID, image, and text queries. +Replace the model ID and image path below. The example uses the same API key as the upload. + +```bash +curl 'http://localhost:9001/v2/models/infer?model_id=your-workspace/custom-owlv2' \ + -H "Authorization: Bearer $ROBOFLOW_API_KEY" \ + -F 'image=@field.jpg' \ + -F 'inputs={"classes":["maize tassel","weed"]}' +``` + +Each class ID indexes the query list for that request. Explicit queries override any saved prompts for that call. +If you included `class_names.txt`, omit the `inputs` field to use the saved labels, or select this model in Auto Label. +If you uploaded only the weights, supply queries on every request. + +## Custom model or preprocessing + +If training changed a component, place its override files beside the weights. +Each supplied component takes precedence over its default. + +| Optional files | When to supply them | +| --- | --- | +| `config.json` | A different model architecture, saved with `model.config.save_pretrained(folder)` | +| `processor_config.json` or `preprocessor_config.json` | Different image preprocessing, such as resizing or normalization | +| Tokenizer files | Different tokenization, saved with the original tokenizer or processor | +| `owlv2_prompts.json` | Text prompts that differ from the output labels | + +A tokenizer override needs `tokenizer_config.json` and either `tokenizer.json` or both `vocab.json` and `merges.txt`. +Include additional tokenizer files that the original processor saved, such as `added_tokens.json` and `special_tokens_map.json`. +Partial tokenizer overrides produce an error. They are never mixed with the default tokenizer. +You can save a complete processor override with `processor.save_pretrained(folder)`. + +`owlv2_prompts.json` is a JSON array with one nonblank prompt per label, in label order. +For example, labels `dent` and `scratch` can use `["a dent in a metal panel", "a scratch in a metal panel"]`. + +The exporter strictly matches the weights to the selected model configuration and rejects partial or adapter-only checkpoints. +If the weights do not match the default architecture, supply the original `config.json`. +Weight shapes cannot identify custom resizing or normalization. Omitted image preprocessing files mean standard preprocessing. +The SDK packages files without loading the checkpoint. Before the upload, it checks only that the set of files is consistent. +The exporter validates the contents of the files. diff --git a/roboflow/core/version.py b/roboflow/core/version.py index e78c5b2f..c82827ef 100644 --- a/roboflow/core/version.py +++ b/roboflow/core/version.py @@ -6,6 +6,7 @@ import sys import time import warnings +from pathlib import Path from typing import TYPE_CHECKING, Optional, Union from dotenv import load_dotenv @@ -36,7 +37,11 @@ from roboflow.util import http from roboflow.util.annotations import amend_data_yaml from roboflow.util.general import extract_zip, write_line -from roboflow.util.model_processor import package_custom_weights_interactive, validate_model_type_for_project +from roboflow.util.model_processor import ( + MissingFileError, + package_custom_weights_interactive, + validate_model_type_for_project, +) from roboflow.util.train_recipe import fold_epochs_into_recipe from roboflow.util.versions import get_model_format, get_wrong_dependencies_versions @@ -663,6 +668,11 @@ def deploy(self, model_type: str, model_path: str, filename: str = "weights/best model_path (str): File path to the model weights to be uploaded. filename (str, optional): The name of the weights file. Defaults to "weights/best.pt". """ + if model_type == "owlv2" and not (Path(model_path) / "class_names.txt").is_file(): + raise MissingFileError( + "An OWLv2 upload to a dataset version needs class_names.txt. " + "Use workspace.deploy_model to upload weights without labels." + ) bundle = package_custom_weights_interactive(model_type, model_path, filename, build_dir=model_path) self._validate_against_project_type(bundle.model_type) diff --git a/roboflow/util/model_processor.py b/roboflow/util/model_processor.py index 48fc85be..04be15d2 100644 --- a/roboflow/util/model_processor.py +++ b/roboflow/util/model_processor.py @@ -68,6 +68,7 @@ "paligemma2", "florence-2", "rfdetr", + "owlv2", ) SUPPORTED_HUGGINGFACE_TYPES = ( @@ -462,7 +463,11 @@ def _process_model( f"It must start with a supported family: {', '.join(SUPPORTED_MODELS)}." ) - if model_type.startswith(("paligemma", "paligemma2", "florence-2")): + if model_type == "owlv2": + return _process_owlv2(model_path, filename, build_dir) + elif model_type.startswith("owlv2"): + raise UnsupportedModelError("Use model_type='owlv2' for OWLv2 checkpoints.") + elif model_type.startswith(("paligemma", "paligemma2", "florence-2")): return _process_huggingface(model_type, model_path, build_dir) if model_type.startswith("yolonas"): return _process_yolonas(model_type, model_path, filename, build_dir) @@ -1125,6 +1130,50 @@ def _process_rfdetr( return archive_path, model_type, warnings +def _process_owlv2(model_path: Path, filename: str, build_dir: Path) -> tuple[Path, str, list[str]]: + tokenizer_files = { + "added_tokens.json", + "merges.txt", + "special_tokens_map.json", + "tokenizer.json", + "tokenizer_config.json", + "vocab.json", + } + optional = tokenizer_files | { + "class_names.txt", + "config.json", + "owlv2_prompts.json", + "preprocessor_config.json", + "processor_config.json", + } + files = {name: _resolve_within_source(model_path, name) for name in optional} + present = {name for name, path in files.items() if path.is_file()} + # The export job is the authority on these files, because uploads can bypass the SDK. The SDK + # repeats only the file-set rules, so that a doomed upload fails here and not in a remote job + # after a multi-gigabyte transfer. File contents are validated by the export job alone. + if "owlv2_prompts.json" in present and "class_names.txt" not in present: + raise MissingFileError("owlv2_prompts.json requires class_names.txt to name the output labels.") + if present & tokenizer_files and ( + "tokenizer_config.json" not in present + or not ("tokenizer.json" in present or {"vocab.json", "merges.txt"} <= present) + ): + raise MissingFileError( + "Provide a complete tokenizer override: tokenizer_config.json and either tokenizer.json " + "or both vocab.json and merges.txt. Omit all tokenizer files to use the standard OWLv2 tokenizer." + ) + weights = _resolve_within_source(model_path, filename) + if not weights.is_file(): + raise MissingFileError( + f"OWLv2 weights file '{weights}' was not found. Set filename to the checkpoint's relative path." + ) + archive_path = build_dir / "roboflow_deploy.zip" + _write_zip( + archive_path, + [(weights, "weights.pt", True)] + [(path, name, False) for name, path in sorted(files.items())], + ) + return archive_path, "owlv2", [] + + def _process_huggingface( model_type: str, model_path: Path, diff --git a/tests/test_version.py b/tests/test_version.py index d00ae4cc..0afe46f6 100644 --- a/tests/test_version.py +++ b/tests/test_version.py @@ -1,6 +1,8 @@ import json import os +import tempfile import unittest +from pathlib import Path from types import SimpleNamespace from unittest.mock import MagicMock, patch @@ -20,6 +22,7 @@ from roboflow.core.training import Training from roboflow.core.version import Version, unwrap_version_id from roboflow.models.object_detection import ObjectDetectionModel +from roboflow.util.model_processor import MissingFileError from tests.helpers import get_version @@ -275,6 +278,15 @@ def test_classification_project_rejects_detection(self): self._version(TYPE_CLASSICATION)._validate_against_project_type("yolov11") +class TestDeployOwlV2(unittest.TestCase): + def test_weights_without_labels_are_rejected_before_packaging(self): + with tempfile.TemporaryDirectory() as model_path: + (Path(model_path) / "owlv2.pth").write_text("checkpoint") + with self.assertRaisesRegex(MissingFileError, "workspace.deploy_model"): + get_version().deploy("owlv2", model_path, "owlv2.pth") + self.assertEqual([path.name for path in Path(model_path).iterdir()], ["owlv2.pth"]) + + class TestConstructionDoesNotProbeNetwork(unittest.TestCase): @patch("roboflow.adapters.rfapi.get_version", side_effect=AssertionError("get_version should not be called")) def test_construction_makes_no_request_when_payload_has_no_model(self, _mock_get_version: MagicMock): diff --git a/tests/util/test_model_processor.py b/tests/util/test_model_processor.py index da39b96e..9f0539d3 100644 --- a/tests/util/test_model_processor.py +++ b/tests/util/test_model_processor.py @@ -728,6 +728,75 @@ def test_rfdetr_multiple_discovered_checkpoints_warns_which_used(self): bundle.cleanup() +class PackageOwlV2Test(unittest.TestCase): + def setUp(self): + self.directory = tempfile.TemporaryDirectory() + self.addCleanup(self.directory.cleanup) + self.source = Path(self.directory.name) + for name, contents in { + "config.json": '{"model_type": "owlv2"}', + "processor_config.json": "{}", + "tokenizer_config.json": "{}", + "tokenizer.json": "{}", + "class_names.txt": "dent\nscratch\n", + "owlv2.pth": "checkpoint", + }.items(): + (self.source / name).write_text(contents) + + def test_packages_raw_weights_without_loading_them(self): + with mock.patch.object(model_processor, "import_module", side_effect=AssertionError("No model load")): + bundle = package_custom_weights("owlv2", str(self.source), "owlv2.pth") + self.addCleanup(bundle.cleanup) + with zipfile.ZipFile(bundle.archive_path) as archive: + self.assertEqual(archive.read("weights.pt"), b"checkpoint") + self.assertEqual(archive.read("class_names.txt").decode().splitlines(), ["dent", "scratch"]) + self.assertIn("processor_config.json", archive.namelist()) + self.assertFalse((self.source / "weights.pt").exists()) + + def test_packages_only_weights_and_class_names(self): + for path in self.source.iterdir(): + if path.name not in {"owlv2.pth", "class_names.txt"}: + path.unlink() + bundle = package_custom_weights("owlv2", str(self.source), "owlv2.pth") + self.addCleanup(bundle.cleanup) + with zipfile.ZipFile(bundle.archive_path) as archive: + self.assertEqual(set(archive.namelist()), {"weights.pt", "class_names.txt"}) + + def test_packages_weights_alone_for_explicit_queries(self): + for path in self.source.iterdir(): + if path.name != "owlv2.pth": + path.unlink() + bundle = package_custom_weights("owlv2", str(self.source), "owlv2.pth") + self.addCleanup(bundle.cleanup) + with zipfile.ZipFile(bundle.archive_path) as archive: + self.assertEqual(archive.namelist(), ["weights.pt"]) + + def test_prompt_override_requires_output_labels(self): + (self.source / "class_names.txt").unlink() + (self.source / "owlv2_prompts.json").write_text('["a dent"]') + with self.assertRaisesRegex(MissingFileError, "class_names.txt"): + package_custom_weights("owlv2", str(self.source), "owlv2.pth") + + def test_rejects_partial_tokenizer_overrides(self): + (self.source / "tokenizer_config.json").unlink() + (self.source / "tokenizer.json").unlink() + for name in ("tokenizer_config.json", "tokenizer.json", "special_tokens_map.json", "vocab.json"): + with self.subTest(name=name): + (self.source / name).write_text("{}") + with self.assertRaisesRegex(MissingFileError, "complete tokenizer override"): + package_custom_weights("owlv2", str(self.source), "owlv2.pth") + (self.source / name).unlink() + + def test_packages_legacy_tokenizer_override(self): + (self.source / "tokenizer.json").unlink() + (self.source / "vocab.json").write_text("{}") + (self.source / "merges.txt").write_text("#version: 0.2\n") + bundle = package_custom_weights("owlv2", str(self.source), "owlv2.pth") + self.addCleanup(bundle.cleanup) + with zipfile.ZipFile(bundle.archive_path) as archive: + self.assertLessEqual({"tokenizer_config.json", "vocab.json", "merges.txt"}, set(archive.namelist())) + + class ProcessHuggingfaceTest(unittest.TestCase): """Packaging for HuggingFace-backed models (paligemma / florence-2)."""