diff --git a/.ai/skills/diffusers-cli/SKILL.md b/.ai/skills/diffusers-cli/SKILL.md index dd9ffd66dabd..204f683e1413 100644 --- a/.ai/skills/diffusers-cli/SKILL.md +++ b/.ai/skills/diffusers-cli/SKILL.md @@ -16,6 +16,7 @@ description: > | --------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `run` | Run any `DiffusionPipeline` or `ModularPipeline`. Forwards `--pipeline-kwargs` verbatim, saves output by detecting its runtime type, optionally runs on HF Jobs via `--remote`. | | `schema` | Print the input schema for a pipeline repo (kwarg names, types, defaults, descriptions). **No weights downloaded** — only the small index file. | +| `serve` | Serve a pipeline over HTTP with the Generative Media Spec API: model discovery, queued generations, polling or SSE progress. | | `custom_blocks` | Package a local `ModularPipelineBlocks` subclass for the Hub. | | `env` | Print versions of diffusers + torch + transformers + accelerate + safetensors + CUDA + GPU info. Use when investigating environment issues, dtype/precision support, or building bug reports. | @@ -33,6 +34,7 @@ The other commands are small enough that `diffusers-cli --help` is the ```bash diffusers-cli schema --help +diffusers-cli serve --help diffusers-cli custom_blocks --help diffusers-cli env --help ``` diff --git a/docs/source/en/_toctree.yml b/docs/source/en/_toctree.yml index bea3bdca0d91..628f2f6ff636 100644 --- a/docs/source/en/_toctree.yml +++ b/docs/source/en/_toctree.yml @@ -7,6 +7,8 @@ title: Quickstart - local: using-diffusers/cli title: Command line interface + - local: using-diffusers/serve + title: Serving title: Get started - isExpanded: true sections: diff --git a/docs/source/en/using-diffusers/cli.md b/docs/source/en/using-diffusers/cli.md index 8625001cef89..7dd6de84ca68 100644 --- a/docs/source/en/using-diffusers/cli.md +++ b/docs/source/en/using-diffusers/cli.md @@ -12,7 +12,8 @@ specific language governing permissions and limitations under the License. # Command line interface -`diffusers-cli` is a command line client for running, inspecting, and packaging Diffusers pipelines. +`diffusers-cli` is a command line client for running, inspecting, and packaging Diffusers pipelines. It is also +installed as `diffusers`, so `diffusers serve` and `diffusers-cli serve` are the same command. ## Available commands @@ -21,6 +22,7 @@ specific language governing permissions and limitations under the License. | [`env`](#env) | Print environment info for bug reports. | | [`schema`](#schema) | Inspect a pipeline's `__call__` signature without downloading weights. | | [`run`](#run) | Run a pipeline locally or in a Hugging Face Sandbox. | +| [`serve`](#serve) | Serve a pipeline over HTTP with the Generative Media Spec API. | | [`custom_blocks`](#customblocks) | Package a local `ModularPipelineBlocks` subclass for the Hub. | | [`fp16_safetensors`](#fp16safetensors) | Convert a checkpoint to fp16 `.safetensors`. | | [`skills`](#skills) | Install pre-authored skill bundles into your AI coding agent. | @@ -287,6 +289,15 @@ diffusers-cli run -m black-forest-labs/FLUX.1-dev --dtype bf16 \ hf sandbox kill ``` +## `serve` + +Serve a pipeline over HTTP with the Generative Media Spec API. See [Serving](./serve) for the full guide. + +```bash +pip install "diffusers[serve]" +diffusers-cli serve black-forest-labs/FLUX.1-dev +``` + ## `custom_blocks` Package a local `ModularPipelineBlocks` subclass for upload to the Hub. Reads a Python file, AST-scans it for diff --git a/docs/source/en/using-diffusers/serve.md b/docs/source/en/using-diffusers/serve.md new file mode 100644 index 000000000000..0d0ecad19305 --- /dev/null +++ b/docs/source/en/using-diffusers/serve.md @@ -0,0 +1,207 @@ + + +# Serving + +`diffusers serve` serves a pipeline over HTTP with the Generative Media Spec (GMS) API: clients discover what the +model accepts, create a generation, follow it while it runs, and download the results. + +```bash +pip install "diffusers[serve]" +diffusers serve black-forest-labs/FLUX.1-dev +``` + +`serve` takes the same loading and optimization flags as [`diffusers-cli run`](./cli#run). The server listens on +`127.0.0.1:8000` and has no authentication, so keep it on a trusted network. Run `diffusers serve --help` for every +option. + +## Generations + +Create a generation. `Prefer: wait` holds the response until it finishes, for up to 120 seconds here (the server caps it at 300): + +```bash +curl localhost:8000/v1/generations -H 'Prefer: wait=120' -d '{ + "model": "black-forest-labs/FLUX.1-dev", + "inputs": {"prompt": "an astronaut riding a horse"}, + "parameters": {"width": 1024, "height": 1024, "steps": 28, "seed": 42} +}' +``` + +```json +{ + "id": "gen_3f9a1c2b7d4e", + "model": "black-forest-labs/FLUX.1-dev", + "revision": "", + "status": "complete", + "created_at": "2026-11-05T10:00:00Z", + "progress": null, + "outputs": { + "image": [ + { + "media_type": "image/png", + "width": 1024, + "height": 1024, + "url": "http://localhost:8000/v1/generations/gen_3f9a1c2b7d4e/files/image-0.png" + } + ] + }, + "expires_at": "2026-11-05T11:00:12Z", + "seed": 42, + "usage": {"compute_seconds": 11.84} +} +``` + +Query the server from Python with [httpx](https://www.python-httpx.org/): + +```py +import httpx + +response = httpx.post( + "http://localhost:8000/v1/generations", + headers={"Prefer": "wait=120"}, + json={"model": "black-forest-labs/FLUX.1-dev", "inputs": {"prompt": "an astronaut riding a horse"}}, + timeout=130, +) +image_url = response.json()["outputs"]["image"][0]["url"] +with open("image.png", "wb") as f: + f.write(httpx.get(image_url).content) +``` + +Without `Prefer: wait`, the response comes back at once with `"status": "queued"`. Follow its progress as +server-sent events, or poll it: + +```bash +curl -N localhost:8000/v1/generations/gen_3f9a1c2b7d4e/events +``` + +```text +id: 1 +event: progress +data: {"step": 1, "total_steps": 28} + +... + +id: 29 +event: complete +data: {"id": "gen_3f9a1c2b7d4e", "status": "complete", "outputs": {...}, ...} +``` + +```bash +curl localhost:8000/v1/generations/gen_3f9a1c2b7d4e # poll +curl -X POST localhost:8000/v1/generations/gen_3f9a1c2b7d4e/cancel # cancel +curl -X DELETE localhost:8000/v1/generations/gen_3f9a1c2b7d4e # delete it and its files +curl -O http://localhost:8000/v1/generations/gen_3f9a1c2b7d4e/files/image-0.png +``` + +`inputs` holds the prompt and media (`{"type": "image", "url": "https://..."}`), and `parameters` holds settings. +Both use the names the model's manifest declares. A completed generation lists a download URL for each output, or +inline base64 with `"response_format": "b64_json"`. Outputs expire after one hour. + +Invalid requests are rejected with a pointer to the offending field: + +```json +{"error": {"code": "invalid_request", "message": "task 'text_to_image' declares no parameter 'cfg'", "pointer": "/parameters/cfg"}} +``` + +A request can also apply LoRAs for that generation only: + +```bash +curl localhost:8000/v1/generations -d '{ + "model": "black-forest-labs/FLUX.1-dev", + "inputs": {"prompt": "a tiny cat"}, + "adapters": [{"type": "lora", "path": "hf:alvdansen/littletinies", "scale": 0.8}] +}' +``` + +## Manifests + +`GET /v1/models/` returns the model's manifest: its tasks, inputs, parameters and capabilities. + +```bash +curl localhost:8000/v1/models/black-forest-labs/FLUX.1-dev +``` + +```json +{ + "spec": "gms/0.1", + "manifest": { + "id": "black-forest-labs/FLUX.1-dev", + "modalities": {"in": ["image"], "out": ["image"]}, + "tasks": { + "text_to_image": { + "inputs": {"prompt": {"type": "text", "required": true}, "prompt_2": {"type": "text"}}, + "outputs": {"image": {"type": "image"}}, + "parameters": { + "height": {"type": "integer"}, + "width": {"type": "integer"}, + "steps": {"type": "integer", "default": 28}, + "guidance_scale": {"type": "number", "default": 3.5}, + "seed": {"type": "integer"} + }, + "x-diffusers": {"parameters": {"steps": "num_inference_steps", "seed": "generator"}} + } + }, + "capabilities": ["sse", "wait", "adapters"], + "adapters": {"lora": {"targets": ["transformer", "text_encoder"]}} + } +} +``` + +`serve` uses the file passed with `--manifest`, else a `gms.json` in the model repo, else a manifest derived from the +pipeline's `__call__` signature. Publish a `gms.json` to add value ranges, presets, or a duration in seconds. The +server checks the manifest at startup and refuses to start if it declares something the pipeline can't do. + +`x-diffusers` maps a manifest's names to pipeline arguments, and `capabilities` lists what the server supports: +`sse` (event streams), `wait` (`Prefer: wait`) and `adapters` (per-request LoRAs). The manifest above is abridged. + +## Timeouts + +- `--generation-timeout 10m` fails a generation that runs longer than 10 minutes. +- `--idle-timeout 30m` shuts the server down after 30 minutes without requests. + +## Security + +The server has no authentication and listens on `127.0.0.1`. Before exposing it, put it behind a proxy that +authenticates requests. With `--remote`, requests already need your Hugging Face token. + +The server downloads media inputs given as URLs. By default it only fetches public addresses, up to 256 MB, and doesn't +follow redirects. To allow only specific hosts, list them: + +```bash +diffusers serve black-forest-labs/FLUX.1-dev --allowed-media-domains upload.wikimedia.org +``` + +Listed hosts are fetched even when they're private, such as `localhost` while testing. + +## Multiple GPUs + +`--data-parallel N` loads a copy of the pipeline on each of N GPUs and runs up to N generations at once. +`--context-parallel` is not supported by `serve` yet. + +```bash +diffusers serve black-forest-labs/FLUX.1-dev --data-parallel 4 +``` + +## Remote serving + +`--remote` runs the server in a [Hugging Face Job](https://huggingface.co/docs/huggingface_hub/guides/jobs) and +streams its logs. Once the model loads, it prints the server URL. Requests authenticate with your Hugging Face token. + +```bash +diffusers serve black-forest-labs/FLUX.1-dev --remote --flavor a10g-large + +curl /v1/generations -H 'Prefer: wait=120' -H "Authorization: Bearer $HF_TOKEN" \ + -d '{"model": "black-forest-labs/FLUX.1-dev", "inputs": {"prompt": "a cat on the moon"}}' +``` + +The Job stops when the server goes idle (`--idle-timeout` defaults to `30m`), and `--flavor`, `--image`, +`--dependencies` and `--namespace` work as in [`run --remote`](./cli#remote-execution---remote). diff --git a/setup.py b/setup.py index f490209e99f2..610177d3f135 100644 --- a/setup.py +++ b/setup.py @@ -146,6 +146,8 @@ "opencv-python", "timm", "flashpack", + "fastapi", + "uvicorn", ] # this is a lookup table with items like: @@ -253,6 +255,7 @@ def run(self): extras["nvidia_modelopt"] = deps_list("nvidia_modelopt[hf]") extras["sdnq"] = deps_list("sdnq") extras["flashpack"] = deps_list("flashpack") +extras["serve"] = deps_list("fastapi", "uvicorn") extras["dev"] = extras["quality"] + extras["test"] + extras["training"] + extras["docs"] + extras["torch"] @@ -287,7 +290,12 @@ def run(self): python_requires=">=3.11.0", install_requires=list(install_requires), extras_require=extras, - entry_points={"console_scripts": ["diffusers-cli=diffusers.commands.diffusers_cli:main"]}, + entry_points={ + "console_scripts": [ + "diffusers-cli=diffusers.commands.diffusers_cli:main", + "diffusers=diffusers.commands.diffusers_cli:main", + ] + }, classifiers=[ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", diff --git a/src/diffusers/commands/diffusers_cli.py b/src/diffusers/commands/diffusers_cli.py index 0e4f2c27fb64..32252a386d1e 100644 --- a/src/diffusers/commands/diffusers_cli.py +++ b/src/diffusers/commands/diffusers_cli.py @@ -22,6 +22,7 @@ from .fp16_safetensors import FP16SafetensorsCommand from .run import RunCommand from .schema import SchemaCommand +from .serve import ServeCommand from .skills import SkillsCommand @@ -49,6 +50,7 @@ def main(): CustomBlocksCommand.register_subcommand(commands_parser) RunCommand.register_subcommand(commands_parser) SchemaCommand.register_subcommand(commands_parser) + ServeCommand.register_subcommand(commands_parser) SkillsCommand.register_subcommand(commands_parser) # Let's go diff --git a/src/diffusers/commands/run.py b/src/diffusers/commands/run.py index ae9a465d5bad..c0a85ab550ec 100644 --- a/src/diffusers/commands/run.py +++ b/src/diffusers/commands/run.py @@ -148,8 +148,10 @@ # --------------------------------------------------------------------------- -def _add_loading_arguments(parser: ArgumentParser) -> None: - parser.add_argument("--model", "-m", required=True, help="Model id on the Hugging Face Hub or local path.") +def _add_loading_arguments(parser: ArgumentParser, require_model: bool = True) -> None: + parser.add_argument( + "--model", "-m", required=require_model, help="Model id on the Hugging Face Hub or local path." + ) parser.add_argument( "--device-map", default=None, diff --git a/src/diffusers/commands/serve/__init__.py b/src/diffusers/commands/serve/__init__.py new file mode 100644 index 000000000000..f340452f55f1 --- /dev/null +++ b/src/diffusers/commands/serve/__init__.py @@ -0,0 +1,345 @@ +# Copyright 2026 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""`diffusers-cli serve` — serve a pipeline over HTTP with the Generative Media Spec (GMS) API. + +Generations run on background workers, one per pipeline copy: a request returns immediately with a queued generation +that clients poll or follow over server-sent events. +""" + +from __future__ import annotations + +import copy +import dataclasses +import inspect +import random +import secrets +import threading +import time +import typing +from argparse import ArgumentParser, Namespace, RawDescriptionHelpFormatter, _SubParsersAction +from pathlib import Path +from typing import Any + +import torch +from huggingface_hub.cli._output import out + +from ...utils import BaseOutput, logging +from .. import BaseDiffusersCLICommand +from ..run import _add_loading_arguments, _add_optimization_arguments, _duration_to_seconds +from .remote import add_remote_arguments, serve_remote + + +logger = logging.get_logger("diffusers-cli/serve") + +DEFAULT_OUTPUT_DIR = str(Path.home() / ".diffusers" / "cli" / "serve" / "outputs") + + +def _resolve_pipeline(args: Namespace) -> tuple[type, inspect.Signature, set[str] | None, str | None]: + """Return the pipeline class, the signature of its call, the fields of its output, and the commit hash of a + standard Hub repo. The output fields are None when the pipeline doesn't annotate what it returns.""" + import diffusers + + from .manifest import blocks_signature + + # A modular repo ships `modular_model_index.json` instead of `model_index.json`, so `load_config` raises. + try: + config, commit_hash = diffusers.DiffusionPipeline.load_config( + args.model, token=args.token, revision=args.revision, return_commit_hash=True + ) + except OSError: + config, commit_hash = None, None + + if config is not None: + class_name = config.get("_class_name") + pipeline_cls = getattr(diffusers, str(class_name), None) + if not isinstance(pipeline_cls, type): + raise SystemExit( + f"Pipeline class {class_name!r} declared in {diffusers.DiffusionPipeline.config_name} is not " + "exported by the installed diffusers." + ) + if not issubclass(pipeline_cls, diffusers.ModularPipeline): + # Evaluated, so a module with `from __future__ import annotations` gives types rather than strings. A + # kwarg whose annotation doesn't evaluate is left out of a derived manifest. + try: + signature = inspect.signature(pipeline_cls.__call__, eval_str=True) + except (NameError, TypeError): + signature = inspect.signature(pipeline_cls.__call__) + returned = typing.get_args(signature.return_annotation) or (signature.return_annotation,) + outputs = [cls for cls in returned if isinstance(cls, type) and issubclass(cls, BaseOutput)] + output_fields = {field.name for field in dataclasses.fields(outputs[0])} if outputs else None + return pipeline_cls, signature, output_fields, commit_hash + + try: + pipeline = diffusers.ModularPipeline.from_pretrained( + args.model, trust_remote_code=args.trust_remote_code, token=args.token, revision=args.revision + ) + except OSError as e: + raise SystemExit(f"Could not read a pipeline config for {args.model!r}: {e}") from e + output_fields = {output.name for output in pipeline.blocks.intermediate_outputs} + return type(pipeline), blocks_signature(pipeline.blocks), output_fields, commit_hash + + +def _shut_down_when_idle(server: Any, app: Any, scheduler: Any, idle_timeout: str) -> None: + limit = _duration_to_seconds(idle_timeout) + while True: + time.sleep(min(limit, 30.0)) + if scheduler.busy(): + app.state.last_request = time.monotonic() + continue + if time.monotonic() - app.state.last_request >= limit: + logger.info(f"No requests for {idle_timeout}; shutting down.") + server.should_exit = True + return + + +class ServeCommand(BaseDiffusersCLICommand): + task = "serve" + + @staticmethod + def register_subcommand(subparsers: _SubParsersAction) -> None: + epilog = ( + "Examples\n" + " $ diffusers-cli serve black-forest-labs/FLUX.1-dev\n" + " $ diffusers-cli serve black-forest-labs/FLUX.1-dev --compile --port 9000\n" + "\n" + " $ curl localhost:8000/v1/generations -H 'Prefer: wait=120' \\\n" + ' -d \'{"model": "black-forest-labs/FLUX.1-dev", "inputs": {"prompt": "a cat on the moon"}}\'\n' + "\n" + "Learn more\n" + " Use `diffusers-cli --help` for more information about a command.\n" + " Read the documentation at https://huggingface.co/docs/diffusers\n" + ) + + parser: ArgumentParser = subparsers.add_parser( + "serve", + help="Serve a diffusers pipeline over HTTP with the Generative Media Spec API.", + usage="\n diffusers-cli serve [options]", + epilog=epilog, + formatter_class=RawDescriptionHelpFormatter, + ) + parser._optionals.title = "Options" + parser.add_argument( + "model_id", + nargs="?", + default=None, + metavar="MODEL", + help="Model id on the Hugging Face Hub or local path.", + ) + _add_loading_arguments(parser, require_model=False) + _add_optimization_arguments(parser) + parser.add_argument("--host", default="127.0.0.1", help="Interface to bind. Defaults to 127.0.0.1.") + parser.add_argument("--port", type=int, default=8000, help="Port to bind. Defaults to 8000.") + parser.add_argument( + "--public-url", + default=None, + metavar="URL", + help=( + "Base URL clients reach the server at, used to build output file links. Set it when the server " + "sits behind a reverse proxy; defaults to the address of each incoming request." + ), + ) + parser.add_argument( + "--manifest", + default=None, + metavar="PATH", + help=( + "Path to a GMS manifest (`gms.json`) describing the model. Defaults to the `gms.json` published " + "in the model repo; when there is none, a manifest is derived from the pipeline's call signature." + ), + ) + parser.add_argument( + "--output-dir", + default=DEFAULT_OUTPUT_DIR, + help=f"Directory generated files are stored in until they expire. Defaults to {DEFAULT_OUTPUT_DIR}.", + ) + parser.add_argument( + "--data-parallel", + type=int, + default=1, + metavar="N", + help=( + "Load a copy of the pipeline on each of the first N GPUs and run up to N generations at once. " + "Defaults to 1." + ), + ) + parser.add_argument( + "--seed", + type=int, + default=None, + help=( + "Seed for the generator that picks the seed of each request that does not set one. " + "Defaults to a random seed chosen at startup." + ), + ) + parser.add_argument( + "--allowed-media-domains", + action="append", + default=None, + metavar="HOST", + help=( + "Only fetch media input URLs from this host (e.g. upload.wikimedia.org). Repeat to allow several. " + "Defaults to any host with a public address." + ), + ) + parser.add_argument( + "--enable-cors", + action="store_true", + help="Allow cross-origin requests from any origin, so browser apps on other hosts can call the server.", + ) + parser.add_argument( + "--log-level", + choices=tuple(logging.get_log_levels_dict()), + default=None, + help="Logging level for diffusers and the HTTP server. Defaults to the current diffusers verbosity.", + ) + parser.add_argument( + "--generation-timeout", + default=None, + help=( + "Fail a generation that runs longer than this (e.g. 10m). It is interrupted at its next step; " + "/health reports a generation still running at twice the limit. Defaults to no limit." + ), + ) + parser.add_argument( + "--idle-timeout", + default=None, + help=( + "Shut the server down after this long without requests while no generation is queued or running " + "(e.g. 30m, 2h). Defaults to never, or to 30m with --remote." + ), + ) + add_remote_arguments(parser) + # `_load_pipeline` reads these `run` flags, which `serve` does not expose. + # Most served models are DiTs trained in bf16; `run`'s `auto` would load them in fp32. + parser.set_defaults(func=ServeCommand, workflow=None, offload_margin=None, dtype="bf16") + + def __init__(self, args: Namespace): + self.args = args + + def run(self) -> None: + args = self.args + if args.model_id is not None and args.model is not None and args.model_id != args.model: + raise SystemExit(f"Got two models: {args.model_id!r} and --model {args.model!r}. Pass only one.") + args.model = args.model or args.model_id + # Cleared so `serve --remote` does not forward it to the Job as `--model-id`. + args.model_id = None + if args.model is None: + raise SystemExit("Pass the model to serve, e.g. `diffusers-cli serve black-forest-labs/FLUX.1-dev`.") + for flag, value in (("--idle-timeout", args.idle_timeout), ("--generation-timeout", args.generation_timeout)): + if value is None: + continue + try: + seconds = _duration_to_seconds(value) + except ValueError: + raise SystemExit(f"{flag} {value!r} is not a duration such as 30m or 2h.") + if seconds <= 0: + raise SystemExit(f"{flag} must be a positive duration such as 30m or 2h, got {value!r}.") + if args.log_level is not None: + logging.set_verbosity(logging.get_log_levels_dict()[args.log_level]) + + # TODO(DN6): support --context-parallel once the encoders and the VAE decode are not repeated on every rank. + if args.context_parallel: + raise SystemExit( + "--context-parallel is not supported by `diffusers-cli serve` yet. Use --data-parallel N to serve " + "on multiple GPUs." + ) + if args.data_parallel < 1: + raise SystemExit(f"--data-parallel must be at least 1, got {args.data_parallel}.") + if args.data_parallel > 1 and args.compile is not None: + raise SystemExit( + "--compile is not supported with --data-parallel yet: the copies of the pipeline would compile " + "concurrently on separate threads, which torch.compile does not support." + ) + if args.data_parallel > 1 and args.device_map is not None: + raise SystemExit( + "--device-map does not apply with --data-parallel: each copy of the pipeline is loaded on its own GPU." + ) + if args.data_parallel > 1 and not args.remote and torch.cuda.device_count() < args.data_parallel: + raise SystemExit( + f"--data-parallel {args.data_parallel} needs {args.data_parallel} GPUs, but " + f"{torch.cuda.device_count()} are visible." + ) + + if args.remote: + serve_remote(args, self.task) + return + + try: + import fastapi # noqa: F401 + import uvicorn + except ImportError as e: + raise SystemExit( + '`diffusers-cli serve` requires FastAPI and uvicorn. Install them with: pip install "diffusers[serve]"' + ) from e + + from .app import create_app + from .backends import DiffusersBackend, backend_capabilities + from .manifest import load_manifest + from .scheduler import Scheduler + + pipeline_cls, signature, output_fields, commit_hash = _resolve_pipeline(args) + capabilities = backend_capabilities(args, pipeline_cls) + manifest = load_manifest( + args.model, + pipeline_cls, + signature, + capabilities, + manifest_path=args.manifest, + revision=args.revision, + commit_hash=commit_hash, + token=args.token, + output_fields=output_fields, + ) + + if args.data_parallel > 1: + backends = [] + for index in range(args.data_parallel): + replica_args = copy.copy(args) + replica_args.device_map = f"cuda:{index}" + backends.append(DiffusersBackend(replica_args, manifest)) + else: + backends = [DiffusersBackend(args, manifest)] + + seed = args.seed if args.seed is not None else secrets.randbelow(2**32) + # Seeds the generator `Manifest.resolve` draws request seeds from, like vLLM's `--seed`. + random.seed(seed) + timeout = None if args.generation_timeout is None else _duration_to_seconds(args.generation_timeout) + scheduler = Scheduler(backends, args.output_dir, timeout=timeout) + app = create_app( + manifest, + scheduler, + enable_cors=args.enable_cors, + public_url=args.public_url, + ) + out.result( + self.task, + url=f"http://{args.host}:{args.port}", + model=manifest.id, + pipeline_class=pipeline_cls.__name__, + tasks=list(manifest.tasks), + capabilities=manifest.capabilities, + seed=seed, + ) + server = uvicorn.Server( + uvicorn.Config(app, host=args.host, port=args.port, log_level=args.log_level or "info") + ) + if args.idle_timeout is not None: + threading.Thread( + target=_shut_down_when_idle, + args=(server, app, scheduler, args.idle_timeout), + name="diffusers-serve-idle", + daemon=True, + ).start() + server.run() diff --git a/src/diffusers/commands/serve/app.py b/src/diffusers/commands/serve/app.py new file mode 100644 index 000000000000..d816fe53a4c5 --- /dev/null +++ b/src/diffusers/commands/serve/app.py @@ -0,0 +1,225 @@ +# Copyright 2026 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The Generative Media Spec HTTP surface, as a FastAPI app over one model's manifest and its scheduler.""" + +from __future__ import annotations + +import base64 +import json +import re +import time +import uuid +from typing import Any +from urllib.parse import urlparse + +from fastapi import FastAPI, Request +from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import FileResponse, JSONResponse, Response, StreamingResponse +from starlette.exceptions import HTTPException + +from ...utils import logging +from .manifest import GMSError, Manifest +from .scheduler import GenerationJob, Scheduler + + +logger = logging.get_logger("diffusers-cli/serve") + +MAX_WAIT_SECONDS = 300.0 +REQUEST_ID_HEADER = "x-request-id" +_KEEP_ALIVE_SECONDS = 15.0 +_FILE_ROUTE = "/v1/generations/{generation_id}/files/{filename}" + + +def _timestamp(value: Any) -> str | None: + if value is None: + return None + return value.strftime("%Y-%m-%dT%H:%M:%SZ") + + +def create_app( + manifest: Manifest, + scheduler: Scheduler, + enable_cors: bool = False, + public_url: str | None = None, +) -> FastAPI: + # A proxy may or may not strip the path it publishes the server under before forwarding a request. + # With that path as the root path, routing accepts both forms. + root_path = urlparse(public_url).path.rstrip("/") if public_url is not None else "" + app = FastAPI(title="diffusers-cli serve", docs_url=None, redoc_url=None, openapi_url=None, root_path=root_path) + app.state.last_request = time.monotonic() + + def format_job_response(job: GenerationJob, request: Request) -> dict[str, Any]: + """The response body for a generation: outputs are linked under the server address, or inlined as base64. + + Raises the generation's error once it failed, which the error handler turns into the error envelope. + """ + if job.error is not None: + raise job.error + + # Each output file is listed with its metadata, plus a download link or its bytes inline. + outputs = None + if job.status == "complete": + # Output links must work for the client, so they are built on the address it reached the server at. + base_url = (public_url or str(request.base_url)).rstrip("/") + outputs = {} + for name, artifacts in job.outputs.items(): + outputs[name] = [] + for artifact in artifacts: + entry = {key: value for key, value in artifact.items() if key != "file"} + if job.request.response_format == "b64_json": + entry["base64"] = base64.b64encode((job.directory / artifact["file"]).read_bytes()).decode() + else: + entry["url"] = base_url + _FILE_ROUTE.format(generation_id=job.id, filename=artifact["file"]) + outputs[name].append(entry) + + return { + "id": job.id, + "model": manifest.id, + "revision": manifest.revision, + "status": job.status, + "created_at": _timestamp(job.created_at), + "progress": job.progress if job.status == "generating" else None, + "outputs": outputs, + "expires_at": _timestamp(job.expires_at), + "seed": job.request.seed if job.status == "complete" else None, + "usage": job.usage, + } + + @app.middleware("http") + async def request_id(request: Request, call_next) -> Response: + request.state.request_id = request.headers.get(REQUEST_ID_HEADER) or str(uuid.uuid4()) + response = await call_next(request) + response.headers[REQUEST_ID_HEADER] = request.state.request_id + return response + + @app.middleware("http") + async def track_activity(request: Request, call_next) -> Response: + # Health checks come from monitors rather than users, and must not keep an idle server alive. + if not request.url.path.rstrip("/").endswith("/health"): + app.state.last_request = time.monotonic() + return await call_next(request) + + if enable_cors: + app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_methods=["*"], + allow_headers=["*"], + expose_headers=[REQUEST_ID_HEADER], + ) + logger.warning("--enable-cors allows requests from any origin. Not recommended for production.") + + @app.exception_handler(GMSError) + async def gms_error(request: Request, error: GMSError) -> JSONResponse: + return JSONResponse(error.to_dict(), status_code=error.status) + + @app.exception_handler(HTTPException) + async def http_error(request: Request, error: HTTPException) -> JSONResponse: + if error.status_code == 404: + code = "not_found" + elif error.status_code < 500: + code = "invalid_request" + else: + code = "internal" + return JSONResponse({"error": {"code": code, "message": str(error.detail)}}, status_code=error.status_code) + + @app.exception_handler(Exception) + async def internal_error(request: Request, error: Exception) -> JSONResponse: + message = f"{type(error).__name__}: {error}" + # An unhandled error is answered outside the middleware stack, so the request id is attached here. + headers = {REQUEST_ID_HEADER: request.state.request_id} + return JSONResponse({"error": {"code": "internal", "message": message}}, status_code=500, headers=headers) + + @app.get("/health") + async def health() -> JSONResponse: + failure = scheduler.failure() + if failure is not None: + return JSONResponse({"status": "unhealthy", "reason": failure}, status_code=503) + return JSONResponse({"status": "ok"}) + + @app.get("/v1/models") + async def list_models() -> dict[str, Any]: + return {"data": [{"id": manifest.id, "modalities": manifest.modalities, "tasks": list(manifest.tasks)}]} + + @app.get("/v1/models/{model_id:path}") + async def get_model(model_id: str) -> Response: + if model_id != manifest.id: + raise GMSError("not_found", f"model {model_id!r} is not served here; this server hosts {manifest.id!r}") + return Response(manifest.discovery_document(), media_type="application/json") + + @app.post("/v1/generations", status_code=201) + async def create_generation(request: Request) -> dict[str, Any]: + try: + body = await request.json() + except json.JSONDecodeError as e: + raise GMSError("invalid_request", f"request body is not valid JSON: {e}") from e + generation = GenerationJob(manifest.resolve(body)) + scheduler.submit(generation) + + wait = re.fullmatch(r"\s*wait=(\d+)\s*", request.headers.get("prefer", "")) + # `Prefer` is a hint, so a server without the `wait` capability answers right away instead of failing. + if wait is not None and "wait" in manifest.capabilities: + await generation.wait(min(float(wait.group(1)), MAX_WAIT_SECONDS)) + return format_job_response(generation, request) + + @app.get("/v1/generations/{generation_id}") + async def get_generation(generation_id: str, request: Request) -> dict[str, Any]: + generation = scheduler.get(generation_id) + return format_job_response(generation, request) + + @app.delete("/v1/generations/{generation_id}", status_code=204) + async def delete_generation(generation_id: str) -> Response: + scheduler.delete(generation_id) + return Response(status_code=204) + + @app.post("/v1/generations/{generation_id}/cancel") + async def cancel_generation(generation_id: str, request: Request) -> dict[str, Any]: + generation = scheduler.cancel(generation_id) + return format_job_response(generation, request) + + @app.get("/v1/generations/{generation_id}/events") + async def get_generation_events(generation_id: str, request: Request) -> StreamingResponse: + if "sse" not in manifest.capabilities: + raise GMSError("not_implemented", "this server does not advertise the `sse` capability") + generation = scheduler.get(generation_id) + last_event_id = request.headers.get("last-event-id", "") + start = int(last_event_id) if last_event_id.isdigit() else 0 + + async def stream(): + index = start + while True: + events, done = await generation.wait_for_events(index, _KEEP_ALIVE_SECONDS) + for event, data in events: + index += 1 + payload = format_job_response(generation, request) if event == "complete" else data + yield f"id: {index}\nevent: {event}\ndata: {json.dumps(payload)}\n\n" + if done: + return + if not events: + yield ": keep-alive\n\n" + + return StreamingResponse(stream(), media_type="text/event-stream", headers={"Cache-Control": "no-cache"}) + + @app.get(_FILE_ROUTE) + async def get_generation_file(generation_id: str, filename: str) -> FileResponse: + generation = scheduler.get(generation_id) + # Only files the backend saved are served, so a filename can't reach anything else on disk. + for artifacts in (generation.outputs or {}).values(): + for artifact in artifacts: + if artifact["file"] == filename: + return FileResponse(generation.directory / filename, media_type=artifact["media_type"]) + raise GMSError("not_found", f"generation {generation_id!r} has no file {filename!r}") + + return app diff --git a/src/diffusers/commands/serve/backends.py b/src/diffusers/commands/serve/backends.py new file mode 100644 index 000000000000..778a5dc8ae67 --- /dev/null +++ b/src/diffusers/commands/serve/backends.py @@ -0,0 +1,329 @@ +# Copyright 2026 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The backend that executes a resolved generation in the server process.""" + +from __future__ import annotations + +import base64 +import contextlib +import inspect +import io +import ipaddress +import mimetypes +import socket +import tempfile +import wave +from argparse import Namespace +from pathlib import Path +from typing import Any, Callable +from urllib.parse import urlparse + +import numpy as np +import torch +from huggingface_hub.utils import httpx +from PIL import Image + +from ...modular_pipelines import InputParam, LoopSequentialPipelineBlocks, ModularPipeline, ModularPipelineBlocks +from ...utils import export_to_video, load_image, load_video, logging, numpy_to_pil +from ...utils.constants import DIFFUSERS_REQUEST_TIMEOUT +from ..run import _as_audio_arrays, _as_pil_list, _get_generator, _load_pipeline +from .manifest import OUTPUT_FIELDS, GMSError, Manifest, blocks_signature +from .scheduler import GenerationJob + + +logger = logging.get_logger("diffusers-cli/serve") + +# zlib level for PNG outputs. Pillow's default of 6 takes close to four times as long to encode a +# 1024x1024 image for a file about 7% smaller; the output is lossless either way. +PNG_COMPRESS_LEVEL = 1 +# Caps each media URL download, like vLLM's `VLLM_MAX_MEDIA_DOWNLOAD_SIZE_MB`, so a request can't make the server read +# an unbounded file into memory. +MAX_MEDIA_DOWNLOAD_BYTES = 256 * 2**20 +_SAMPLE_RATE_ATTRIBUTES = ( + "sample_rate", + "vocoder.config.output_sampling_rate", + "vocoder.config.sampling_rate", + "audio_sampling_rate", + "sound_sampling_rate", + "vae.config.sampling_rate", +) + + +def backend_capabilities(args: Namespace, pipeline_cls: type) -> list[str]: + capabilities = ["sse", "wait"] + # Swapping adapter weights retraces a compiled denoiser on every request, and LoRAs passed at + # startup are part of the deployment rather than something a request may replace. + if hasattr(pipeline_cls, "load_lora_weights") and args.compile is None and not args.lora: + capabilities.append("adapters") + return capabilities + + +def _load_media(name: str, value: dict[str, Any], pointer: str, allowed_domains: list[str] | None) -> Any: + try: + if "base64" in value: + content = base64.b64decode(value["base64"], validate=True) + else: + url = urlparse(value["url"]) + if allowed_domains is not None: + if url.hostname not in allowed_domains: + raise ValueError(f"host {url.hostname!r} is not in --allowed-media-domains {allowed_domains}") + else: + # Only public addresses are fetched, so a request can't reach the server's own network, e.g. a cloud + # metadata endpoint. The operator vouches for the hosts on an allowlist. + default_port = 443 if url.scheme == "https" else 80 + for info in socket.getaddrinfo(url.hostname, url.port or default_port): + if not ipaddress.ip_address(info[4][0].split("%")[0]).is_global: + raise ValueError( + f"host {url.hostname!r} resolves to the non-public address {info[4][0]}. Pass " + "--allowed-media-domains to fetch from it." + ) + # Redirects are not followed, so a public URL can't send the server to an address on its own network. + with httpx.stream( + "GET", value["url"], follow_redirects=False, timeout=DIFFUSERS_REQUEST_TIMEOUT + ) as response: + response.raise_for_status() + content = bytearray() + for chunk in response.iter_bytes(): + content += chunk + if len(content) > MAX_MEDIA_DOWNLOAD_BYTES: + raise ValueError(f"the download is larger than {MAX_MEDIA_DOWNLOAD_BYTES // 2**20} MB") + except (ValueError, OSError, httpx.HTTPError) as e: + raise GMSError("invalid_request", f"could not read input {name!r}: {e}", pointer) from e + + try: + if value["type"] in ("image", "mask"): + return load_image(Image.open(io.BytesIO(content))) + if value["type"] == "audio": + import torchaudio + + return torchaudio.load(io.BytesIO(content)) + if value.get("media_type"): + suffix = mimetypes.guess_extension(value["media_type"]) or ".mp4" + else: + suffix = Path(urlparse(value.get("url", "")).path).suffix or ".mp4" + with tempfile.NamedTemporaryFile(suffix=suffix) as file: + file.write(content) + file.flush() + return load_video(file.name) + except Exception as e: + raise GMSError("invalid_request", f"could not decode input {name!r}: {e}", pointer) from e + + +def _sample_rate(pipeline: Any) -> int: + for path in _SAMPLE_RATE_ATTRIBUTES: + value = pipeline + for attribute in path.split("."): + value = getattr(value, attribute, None) + if value: + return int(value) + raise SystemExit( + f"Cannot serve audio outputs: {type(pipeline).__name__} exposes no output sample rate. Looked for " + f"{list(_SAMPLE_RATE_ATTRIBUTES)} on the pipeline." + ) + + +class _Canceled(BaseException): + """Stops a modular pipeline from its step callback. Not an `Exception`, so the blocks don't log it as an error.""" + + +# TODO(DN6): remove once modular pipelines stream their steps with `pipe.stream()` (#14159). +class _StepCallback(ModularPipelineBlocks): + """Calls `callback_on_step_end` after each denoising step of a modular pipeline, as a `DiffusionPipeline` does. + + Modular pipelines have no `num_timesteps` attribute, so the step count is passed in `callback_kwargs`. + """ + + @property + def inputs(self): + return [InputParam("callback_on_step_end", type_hint=Callable), InputParam("timesteps")] + + def __call__(self, components, block_state, i, t): + if block_state.callback_on_step_end is not None: + block_state.callback_on_step_end(components, i, t, {"num_timesteps": len(block_state.timesteps)}) + return components, block_state + + +def _denoise_loops(block: ModularPipelineBlocks) -> list[LoopSequentialPipelineBlocks]: + loops = [] + if isinstance(block, LoopSequentialPipelineBlocks) and "denoiser" in block.sub_blocks: + loops.append(block) + for sub_block in getattr(block, "sub_blocks", {}).values(): + loops.extend(_denoise_loops(sub_block)) + return loops + + +class DiffusersBackend: + def __init__(self, args: Namespace, manifest: Manifest): + self.pipeline = _load_pipeline(args) + self._allowed_media_domains = args.allowed_media_domains + self._modular = isinstance(self.pipeline, ModularPipeline) + if self._modular: + # `pipeline.blocks` returns a copy, and the pipeline is already built, so the loops are edited in place. + for loop in _denoise_loops(self.pipeline._blocks): + loop.sub_blocks["step_callback"] = _StepCallback() + parameters = blocks_signature(self.pipeline.blocks).parameters + else: + parameters = inspect.signature(type(self.pipeline).__call__).parameters + self._reports_steps = "callback_on_step_end" in parameters + self._sample_rate = None + if any(spec["type"] == "audio" for task in manifest.tasks.values() for spec in task["outputs"].values()): + self._sample_rate = _sample_rate(self.pipeline) + + def generate(self, generation: GenerationJob) -> dict[str, list[dict[str, Any]]]: + """Call the pipeline with the request's kwargs and media, and save its outputs.""" + request = generation.request + kwargs = dict(request.kwargs) + for kwarg, (name, kind, value) in request.media.items(): + if isinstance(value, list): + loaded = [ + _load_media(name, item, f"/inputs/{name}/{index}", self._allowed_media_domains) + for index, item in enumerate(value) + ] + else: + loaded = [_load_media(name, value, f"/inputs/{name}", self._allowed_media_domains)] + if kind == "audio": + if kwarg == "initial_audio_waveforms": + kwargs.setdefault("initial_audio_sampling_rate", loaded[0][1]) + loaded = [waveform for waveform, _ in loaded] + kwargs[kwarg] = loaded if isinstance(value, list) else loaded[0] + + device = self.pipeline.device if hasattr(self.pipeline, "device") else torch.device("cpu") + if request.seed is not None: + # A CUDA generator must sit on the pipeline's own GPU, which is `cuda:1` and up with --data-parallel. + kwargs["generator"] = _get_generator(request.seed, str(device) if device.type == "cuda" else device.type) + + if self._reports_steps: + + def on_step_end(pipeline, step, timestep, callback_kwargs): + total_steps = callback_kwargs.get("num_timesteps", getattr(pipeline, "num_timesteps", None)) + if total_steps: + generation.report_progress(step + 1, total_steps) + if generation.cancel_requested: + # A block can't end the loop it runs in, so a modular pipeline is stopped by raising. + if self._modular: + raise _Canceled + pipeline._interrupt = True + return callback_kwargs + + kwargs["callback_on_step_end"] = on_step_end + + names = [f"request_{index}" for index in range(len(request.adapters))] + try: + for index, adapter in enumerate(request.adapters): + try: + self.pipeline.load_lora_weights(adapter["path"].removeprefix("hf:"), adapter_name=names[index]) + except Exception as e: + raise GMSError( + "invalid_request", + f"could not load adapter {adapter['path']!r}: {type(e).__name__}: {e}", + f"/adapters/{index}/path", + ) from e + if names: + self.pipeline.set_adapters(names, adapter_weights=[adapter["weight"] for adapter in request.adapters]) + # Worker threads start on GPU 0, and with --data-parallel this pipeline may sit on another one. + with torch.cuda.device(device) if device.type == "cuda" else contextlib.nullcontext(): + result = self.pipeline(**kwargs) + except _Canceled: + return {} + finally: + if request.adapters: + self.pipeline.unload_lora_weights() + if generation.cancel_requested: + return {} + + outputs = {} + for name, (kind, media_type) in request.outputs.items(): + value = None + for field in OUTPUT_FIELDS[kind]: + # A modular pipeline returns its `PipelineState`; a standard one returns an output object. + value = result.get(field) if self._modular else getattr(result, field, None) + if value is not None: + break + if value is None: + continue + + artifacts = [] + if kind == "image": + images = _as_pil_list(numpy_to_pil(value) if isinstance(value, np.ndarray) else value) + if images is None: + raise GMSError("internal", f"cannot encode output {name!r} as {media_type}: not a list of images") + for index, image in enumerate(images): + filename = f"{name}-{index}.{media_type.split('/')[1]}" + if media_type == "image/jpeg": + image = image.convert("RGB") + if media_type == "image/png": + image.save(generation.directory / filename, compress_level=PNG_COMPRESS_LEVEL) + else: + image.save(generation.directory / filename) + artifacts.append( + {"file": filename, "media_type": media_type, "width": image.width, "height": image.height} + ) + elif kind == "video": + for index, frames in enumerate(value): + frames = list(frames) + filename = f"{name}-{index}.mp4" + export_to_video(frames, str(generation.directory / filename), fps=request.fps) + first = frames[0] + width, height = first.size if isinstance(first, Image.Image) else (first.shape[1], first.shape[0]) + artifacts.append( + { + "file": filename, + "media_type": media_type, + "width": width, + "height": height, + "duration": round(len(frames) / request.fps, 2), + "fps": request.fps, + } + ) + else: + if isinstance(value, torch.Tensor): + value = value.detach().to(torch.float32).cpu().numpy() + arrays = _as_audio_arrays(value) + if arrays is None: + raise GMSError("internal", f"cannot encode output {name!r} as {media_type}: not an audio array") + # TODO(DN6): write the files with `export_to_wav` once it is added to `diffusers.utils`. + for index, audio in enumerate(arrays): + data = np.asarray(audio) + if data.dtype.kind == "f": + data = (np.clip(data, -1.0, 1.0) * 32767).astype(np.int16) + else: + data = data.astype(np.int16) + # `wave` takes interleaved samples x channels; the shorter axis is taken to be the channels. + if data.ndim == 1: + data = data[:, None] + elif data.shape[0] < data.shape[1]: + data = data.T + filename = f"{name}-{index}.wav" + with wave.open(str(generation.directory / filename), "wb") as file: + file.setnchannels(data.shape[1]) + file.setsampwidth(2) + file.setframerate(self._sample_rate) + file.writeframes(data.tobytes()) + artifacts.append( + { + "file": filename, + "media_type": media_type, + "duration": round(data.shape[0] / self._sample_rate, 2), + "sample_rate": self._sample_rate, + } + ) + outputs[name] = artifacts + if not outputs: + raise GMSError( + "internal", + f"{type(self.pipeline).__name__} returned none of the outputs {list(request.outputs)} declared for " + f"task {request.task!r}", + ) + return outputs diff --git a/src/diffusers/commands/serve/manifest.py b/src/diffusers/commands/serve/manifest.py new file mode 100644 index 000000000000..cf45eb635075 --- /dev/null +++ b/src/diffusers/commands/serve/manifest.py @@ -0,0 +1,895 @@ +# Copyright 2026 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Generative Media Spec (GMS) manifests. + +A manifest describes what this server offers: its tasks with their inputs and parameters, and its capabilities. It is +loaded from a published `gms.json` or derived from the pipeline signature, checked against the pipeline at startup, +served for discovery, and used to validate each request and map it to the pipeline call. +""" + +from __future__ import annotations + +import hashlib +import inspect +import json +import math +import operator +import os +import random +import re +import types +import typing +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from huggingface_hub import hf_hub_download +from huggingface_hub.errors import EntryNotFoundError +from huggingface_hub.utils import HFValidationError, validate_repo_id + +from ...pipelines.auto_pipeline import ( + AUTO_IMAGE2IMAGE_PIPELINES_MAPPING, + AUTO_IMAGE2VIDEO_PIPELINES_MAPPING, + AUTO_INPAINT_PIPELINES_MAPPING, + AUTO_TEXT2AUDIO_PIPELINES_MAPPING, + AUTO_TEXT2IMAGE_PIPELINES_MAPPING, + AUTO_TEXT2VIDEO_PIPELINES_MAPPING, + AUTO_VIDEO2VIDEO_PIPELINES_MAPPING, +) +from ...utils import logging +from ..run import _AUDIO_INPUT_KEYS, _IMAGE_INPUT_KEYS, _VIDEO_INPUT_KEYS + + +logger = logging.get_logger("diffusers-cli/serve") + + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +SPEC_VERSION = "gms/0.1" +MANIFEST_FILENAME = "gms.json" +EXTENSION_KEY = "x-diffusers" + +MAX_ADAPTERS = 4 +# The frame rate videos are encoded at when neither the pipeline nor the manifest gives one. Native rates range from 8 +# fps (CogVideoX) to 30 (Mochi), so this is a guess; a `duration` frame lattice in a published manifest sets the real one. +DEFAULT_FPS = 16 + +# First entry of each tuple is the default when the request names no `output_format`. +# The fields of a pipeline's output that hold each output type, tried in order. +OUTPUT_FIELDS = { + "image": ("images", "image"), + "video": ("frames", "videos", "video"), + "audio": ("audios", "audio", "sound"), +} +OUTPUT_FORMATS = { + "image": ("image/png", "image/jpeg", "image/webp"), + "video": ("video/mp4",), + "audio": ("audio/wav",), +} + +_ERROR_STATUS = { + "invalid_request": 400, + "not_found": 404, + "internal": 500, + "not_implemented": 501, +} +_REQUEST_FIELDS = ("model", "task", "preset", "inputs", "parameters", "output_format", "response_format", "adapters") +_ADAPTER_FIELDS = ("type", "path", "scale", "targets") + +# GMS name -> pipeline kwargs that carry the same meaning, tried in order before the name itself. +_PARAMETER_KWARGS = { + "seed": ("generator",), + "steps": ("num_inference_steps",), + "num_outputs": ("num_images_per_prompt", "num_videos_per_prompt", "num_waveforms_per_prompt"), + "fps": ("fps", "frame_rate"), + "duration": ("num_frames",), +} +_INPUT_KWARGS = {"mask": ("mask_image",)} + +# (task, classes registered for it, inputs the task requires, output modality) +_AUTO_TASKS = ( + ("text_to_image", AUTO_TEXT2IMAGE_PIPELINES_MAPPING, (), "image"), + ("image_to_image", AUTO_IMAGE2IMAGE_PIPELINES_MAPPING, ("image",), "image"), + ("inpaint", AUTO_INPAINT_PIPELINES_MAPPING, ("image", "mask"), "image"), + ("text_to_video", AUTO_TEXT2VIDEO_PIPELINES_MAPPING, (), "video"), + ("image_to_video", AUTO_IMAGE2VIDEO_PIPELINES_MAPPING, ("image",), "video"), + ("video_to_video", AUTO_VIDEO2VIDEO_PIPELINES_MAPPING, ("video",), "video"), + ("text_to_audio", AUTO_TEXT2AUDIO_PIPELINES_MAPPING, (), "audio"), +) + +_MEDIA_KWARGS = { + **dict.fromkeys(_IMAGE_INPUT_KEYS, "image"), + **dict.fromkeys(_VIDEO_INPUT_KEYS, "video"), + **dict.fromkeys(_AUDIO_INPUT_KEYS, "audio"), + "mask_image": "mask", +} +_SKIPPED_KWARGS = frozenset({"output_type", "return_dict", "callback_steps", "callback_on_step_end_tensor_inputs"}) +_AUDIO_OUTPUT_KWARGS = frozenset({"audio_length_in_s", "audio_end_in_s", "num_waveforms_per_prompt"}) +# In order of preference when an annotation allows several, so `int | float` is a number. +_SCALAR_TYPES = {float: "number", int: "integer", bool: "boolean", str: "string"} +_PYTHON_TYPES = {"number": (int, float), "integer": int, "boolean": bool, "string": str} +_COMPARISONS = { + "eq": operator.eq, + "gt": operator.gt, + "gte": operator.ge, + "lt": operator.lt, + "lte": operator.le, +} +_FRAME_LATTICE = re.compile(r"(\d+)k\+(\d+)") + + +# --------------------------------------------------------------------------- +# Types +# --------------------------------------------------------------------------- + + +class GMSError(Exception): + def __init__(self, code: str, message: str, pointer: str | None = None): + super().__init__(message) + self.code = code + self.message = message + self.pointer = pointer + self.status = _ERROR_STATUS[code] + + def to_dict(self) -> dict[str, Any]: + error = {"code": self.code, "message": self.message} + if self.pointer is not None: + error["pointer"] = self.pointer + return {"error": error} + + +@dataclass +class ResolvedRequest: + task: str + kwargs: dict[str, Any] + # pipeline kwarg -> (request input name, media type, media value or list of media values) + media: dict[str, tuple[str, str, Any]] + seed: int | None + # output name -> (output type, media type to encode it as) + outputs: dict[str, tuple[str, str]] + # The frame rate video outputs are encoded at, or None when the task produces no video. + fps: float | None + response_format: str + adapters: list[dict[str, Any]] + + +# --------------------------------------------------------------------------- +# Loading a manifest +# --------------------------------------------------------------------------- + + +def load_manifest( + model: str, + pipeline_cls: type, + signature: inspect.Signature, + capabilities: list[str], + manifest_path: str | None = None, + revision: str | None = None, + commit_hash: str | None = None, + token: str | None = None, + output_fields: set[str] | None = None, +) -> Manifest: + """Load the model's manifest and bind it to the pipeline: `--manifest`, else the repo's `gms.json`, else derived. + + `revision` is the one requested for the download; `commit_hash` is the one the pipeline was resolved at, which + generations report. + """ + source = None + raw = None + if manifest_path is not None: + raw, origin = Path(manifest_path).read_bytes(), manifest_path + elif Path(model).is_dir(): + published = Path(model) / MANIFEST_FILENAME + if published.is_file(): + raw, origin = published.read_bytes(), str(published) + else: + try: + published = hf_hub_download(model, MANIFEST_FILENAME, revision=revision, token=token) + except EntryNotFoundError: + published = None + if published is not None: + raw, origin = Path(published).read_bytes(), f"{model}/{MANIFEST_FILENAME}" + source = f"hf:{model}/{MANIFEST_FILENAME}" + + if raw is not None: + try: + data = json.loads(raw) + except json.JSONDecodeError as e: + raise SystemExit(f"GMS manifest {origin!r} is not valid JSON: {e}") from e + if not isinstance(data, dict) or not isinstance(data.get("id"), str): + raise SystemExit(f"GMS manifest {origin!r} must be a JSON object with a string `id`.") + if not isinstance(data.get("tasks"), dict) or not data["tasks"]: + raise SystemExit(f"GMS manifest {origin!r} declares no `tasks`, so there is nothing to serve.") + else: + model_id = Path(model).name if Path(model).is_dir() else model + data = derive_manifest(model_id, pipeline_cls, signature, capabilities, output_fields) + raw = json.dumps(data).encode() + return Manifest( + raw, + data, + pipeline_cls, + signature, + capabilities, + source=source, + revision=commit_hash, + output_fields=output_fields, + ) + + +# --------------------------------------------------------------------------- +# Deriving a manifest from the pipeline signature +# --------------------------------------------------------------------------- + + +def blocks_signature(blocks: Any) -> inspect.Signature: + """The call signature of a modular pipeline, which takes its keyword arguments from its blocks' inputs.""" + parameters = [] + for input_param in blocks.inputs: + if input_param.name is None: + continue + annotation = inspect.Parameter.empty if input_param.type_hint is None else input_param.type_hint + # A required input is given no default, as a required argument of a `__call__` has none. + default = inspect.Parameter.empty if input_param.required else input_param.default + parameters.append( + inspect.Parameter(input_param.name, inspect.Parameter.KEYWORD_ONLY, default=default, annotation=annotation) + ) + return inspect.Signature(parameters) + + +def derive_manifest( + model_id: str, + pipeline_cls: type, + signature: inspect.Signature, + capabilities: list[str], + output_fields: set[str] | None = None, +) -> dict[str, Any]: + """Build a server-authored manifest from the pipeline call `signature`, declaring the server's `capabilities`. + + `output_fields`, the fields of the pipeline's output when it annotates them, decides the output type. + """ + # Sort the call kwargs into GMS inputs and parameters, and record the ones whose GMS name differs. + inputs: dict[str, dict[str, Any]] = {} + parameters: dict[str, dict[str, Any]] = {} + input_kwargs: dict[str, str] = {} + parameter_kwargs: dict[str, str] = {} + # Pipeline kwarg -> its GMS parameter name. `num_frames` keeps its name: `duration` is in seconds, which only a + # written manifest's frame lattice converts. + kwarg_parameters = { + kwarg: name for name, kwargs in _PARAMETER_KWARGS.items() if name != "duration" for kwarg in kwargs + } + + for kwarg, parameter in signature.parameters.items(): + if kwarg == "self" or kwarg in _SKIPPED_KWARGS: + continue + if parameter.kind in (inspect.Parameter.VAR_POSITIONAL, inspect.Parameter.VAR_KEYWORD): + continue + has_default = parameter.default is not inspect.Parameter.empty + default = parameter.default if has_default else None + + if kwarg in _MEDIA_KWARGS: + name = "mask" if kwarg == "mask_image" else kwarg + spec: dict[str, Any] = {"type": _MEDIA_KWARGS[kwarg]} + if not has_default: + spec["required"] = True + if re.search(r"\b[Ll]ist\b", str(parameter.annotation)): + spec["multiple"] = {} + inputs[name] = spec + if name != kwarg: + input_kwargs[name] = kwarg + continue + + if kwarg == "generator": + parameters["seed"] = {"type": "integer"} + parameter_kwargs["seed"] = kwarg + continue + + annotation = parameter.annotation + if annotation is inspect.Parameter.empty and default is not None: + annotation = type(default) + schema = _parameter_schema(annotation) + # Modular blocks often declare `prompt` without a type hint or a default to infer one from. + if schema is None and kwarg == "prompt" and annotation is inspect.Parameter.empty: + schema = {"type": "string"} + if schema is None: + continue + if schema["type"] == "string" and re.fullmatch(r"(.*_)?prompt(_\d+)?", kwarg): + inputs[kwarg] = {"type": "text", "required": True} if kwarg == "prompt" else {"type": "text"} + continue + + name = kwarg_parameters.get(kwarg, kwarg) + if name in parameters: + continue + if default is not None and isinstance(default, (bool, int, float, str, list)): + schema["default"] = default + parameters[name] = schema + if name != kwarg: + parameter_kwargs[name] = kwarg + + extension: dict[str, Any] = {} + if input_kwargs: + extension["inputs"] = input_kwargs + if parameter_kwargs: + extension["parameters"] = parameter_kwargs + + registered = [entry for entry in _AUTO_TASKS if pipeline_cls in entry[1].values()] + returned = [kind for kind, fields in OUTPUT_FIELDS.items() if output_fields and set(fields) & output_fields] + if returned: + output = returned[0] + elif registered: + output = registered[0][3] + elif "num_frames" in signature.parameters or "Video" in pipeline_cls.__name__: + output = "video" + elif set(signature.parameters) & _AUDIO_OUTPUT_KWARGS: + output = "audio" + else: + output = "image" + + def task(required: tuple[str, ...], excluded: set[str]) -> dict[str, Any]: + task_inputs = { + name: {**spec, "required": True} if name in required else spec + for name, spec in inputs.items() + if name not in excluded + } + spec = {"inputs": task_inputs, "outputs": {output: {"type": output}}, "parameters": parameters} + if extension: + spec[EXTENSION_KEY] = extension + return spec + + primary = None + if "image" in inputs: + primary = "image" + elif "video" in inputs: + primary = "video" + tasks = {} + if registered: + # A class registered for several tasks takes each task's conditioning input optionally; an + # input another task requires is what tells the tasks apart, so it is left out of the rest. + for name, _, required, _ in registered: + if not set(required) <= set(inputs): + continue + others = set() + for other_name, _, other_required, _ in registered: + if other_name != name: + others.update(other_required) + tasks[name] = task(required, others - set(required)) + elif primary is None: + tasks[f"text_to_{output}"] = task((), set()) + elif inputs[primary].get("required"): + tasks[f"{primary}_to_{output}"] = task((), set()) + else: + tasks[f"text_to_{output}"] = task((), {primary}) + tasks[f"{primary}_to_{output}"] = task((primary,), set()) + + modalities_in = sorted({"image" if s["type"] == "mask" else s["type"] for s in inputs.values()} - {"text"}) + manifest: dict[str, Any] = { + "id": model_id, + "modalities": {"in": modalities_in or ["text"], "out": [output]}, + "tasks": tasks, + "capabilities": list(capabilities), + } + if "adapters" in capabilities: + manifest["adapters"] = {"lora": {"targets": list(getattr(pipeline_cls, "_lora_loadable_modules", []))}} + return manifest + + +def _parameter_schema(annotation: Any) -> dict[str, Any] | None: + """The GMS schema of a scalar annotation, a list of scalars, or a union of them; None for anything else.""" + if typing.get_origin(annotation) in (typing.Union, types.UnionType): + # Any member's values are valid, so members that aren't scalars or lists of them, like a tensor, are left out. + options = [schema for schema in map(_parameter_schema, typing.get_args(annotation)) if schema is not None] + # `str | list[str]` is a scalar that also accepts a batch; only a bare list is an array parameter. + options = [option for option in options if option["type"] != "array"] or options + order = list(_SCALAR_TYPES.values()) + return min(options, key=lambda option: order.index(option.get("items", option)["type"]), default=None) + if typing.get_origin(annotation) is list: + args = typing.get_args(annotation) + items = _parameter_schema(args[0]) if args else None + if items is None or items["type"] == "array": + return None + return {"type": "array", "items": items} + if annotation in _SCALAR_TYPES: + return {"type": _SCALAR_TYPES[annotation]} + return None + + +# --------------------------------------------------------------------------- +# The manifest bound to the pipeline: startup checks, discovery and request resolution +# --------------------------------------------------------------------------- + + +class Manifest: + """A model's manifest, bound to the pipeline that serves it: discovery and request resolution. + + The manifest describes this server, so everything it declares must be servable: each input and parameter maps + to a kwarg of the pipeline call, and each capability is one this server provides. Anything else is an error at + startup rather than a request that fails later. + """ + + def __init__( + self, + raw: bytes, + data: dict[str, Any], + pipeline_cls: type, + signature: inspect.Signature, + supported_capabilities: list[str], + source: str | None = None, + revision: str | None = None, + output_fields: set[str] | None = None, + ): + # The bytes are kept as published: discovery serves them unchanged, and the hash is computed over them. + self.raw = raw + self.source = source + self.id = data["id"] + self.revision = revision + self.tasks: dict[str, dict[str, Any]] = data["tasks"] + self.modalities = data.get("modalities", {}) + self.capabilities: list[str] = data.get("capabilities", []) + self.adapters = data.get("adapters", {}) + self._kwarg_names = set(signature.parameters) - {"self"} + self._pipeline_name = pipeline_cls.__name__ + fps_kwarg = self._pipeline_kwarg("fps", _PARAMETER_KWARGS, {}) + default_fps = signature.parameters[fps_kwarg].default if fps_kwarg is not None else None + self._default_fps = None if default_fps is inspect.Parameter.empty else default_fps + + for capability in self.capabilities: + if capability in supported_capabilities: + continue + message = f"The GMS manifest for {self.id!r} declares the {capability!r} capability, " + if capability == "adapters": + raise SystemExit( + message + f"but request adapters are off: {self._pipeline_name} cannot load LoRAs, or the server " + "was started with --compile or --lora." + ) + raise SystemExit( + message + f"which this server does not provide. It provides: {list(supported_capabilities)}." + ) + if "adapters" in self.capabilities and not self.adapters: + raise SystemExit( + f"The GMS manifest for {self.id!r} declares the `adapters` capability but no top-level `adapters` " + 'section naming the adapter types, e.g. {"lora": {"targets": ["transformer"]}}.' + ) + + # Per task: GMS input or parameter name -> the pipeline kwarg it sets. + self._input_kwargs: dict[str, dict[str, str]] = {} + self._parameter_kwargs: dict[str, dict[str, str]] = {} + # Per task: the pipeline kwarg that sets the frame rate, if the pipeline takes one. + self._fps_kwargs: dict[str, str | None] = {} + for name, task in self.tasks.items(): + declared = task.get(EXTENSION_KEY, {}) + self._input_kwargs[name] = { + key: self._require_kwarg(name, "input", key, _INPUT_KWARGS, declared.get("inputs", {})) + for key in task["inputs"] + } + self._parameter_kwargs[name] = { + key: self._require_kwarg(name, "parameter", key, _PARAMETER_KWARGS, declared.get("parameters", {})) + for key in task.get("parameters", {}) + } + self._fps_kwargs[name] = self._parameter_kwargs[name].get("fps", fps_kwarg) + unsupported = [spec["type"] for spec in task["outputs"].values() if spec["type"] not in OUTPUT_FORMATS] + if unsupported: + raise SystemExit( + f"The GMS manifest for {self.id!r} declares {unsupported} outputs on task {name!r}; this server " + f"produces {list(OUTPUT_FORMATS)} outputs." + ) + # `output_fields` is None when the pipeline doesn't annotate what it returns. + if output_fields is not None: + for spec in task["outputs"].values(): + if not set(OUTPUT_FIELDS[spec["type"]]) & output_fields: + raise SystemExit( + f"Task {name!r} outputs {spec['type']}, read from one of " + f"{list(OUTPUT_FIELDS[spec['type']])}, but {self._pipeline_name} returns " + f"{sorted(output_fields)}. `diffusers serve` only serves image, video and audio outputs." + ) + parameters = task.get("parameters", {}) + if "duration" in parameters and _frame_lattice(parameters["duration"]) is None: + raise SystemExit( + f"The GMS manifest for {self.id!r} declares `duration` on task {name!r} without a frame lattice, so " + 'seconds cannot be converted to frames. Add one, e.g. "lattice": {"frames": "8k+1", "fps": 24}.' + ) + produces_video = any(spec["type"] == "video" for spec in task["outputs"].values()) + if produces_video and self._default_fps is None and "duration" not in parameters: + logger.warning( + f"{self._pipeline_name} has no default `fps` or `frame_rate` argument, so task {name!r} encodes " + f"videos at {DEFAULT_FPS} fps, which may not be the model's native rate. To set it, declare " + '`duration` with a frame lattice in a manifest, e.g. "duration": {"type": "number", "lattice": ' + '{"frames": "4k+1", "fps": 24}}, and pass it with --manifest.' + ) + for preset, body in task.get("presets", {}).items(): + for key in body.get("parameters", {}): + if key not in parameters: + raise SystemExit( + f"The GMS manifest for {self.id!r} sets parameter {key!r} in preset {preset!r} of task " + f"{name!r}, which the task does not declare." + ) + + def _pipeline_kwarg(self, name: str, aliases: dict[str, tuple[str, ...]], declared: dict[str, str]) -> str | None: + candidates = [declared[name]] if name in declared else [*aliases.get(name, ()), name] + return next((kwarg for kwarg in candidates if kwarg in self._kwarg_names), None) + + def _require_kwarg( + self, task: str, kind: str, name: str, aliases: dict[str, tuple[str, ...]], declared: dict[str, str] + ) -> str: + kwarg = self._pipeline_kwarg(name, aliases, declared) + if kwarg is None: + raise SystemExit( + f"The GMS manifest for {self.id!r} declares {kind} {name!r} on task {task!r}, which " + f"{self._pipeline_name} does not take. Remove it from the manifest, or map it to a pipeline kwarg " + f'under `{EXTENSION_KEY}`, e.g. "{EXTENSION_KEY}": {{"{kind}s": {{"{name}": ""}}}}.' + ) + return kwarg + + def discovery_document(self) -> bytes: + # The manifest bytes are spliced in untouched: the spec defines verification as a hash of the served bytes. + document = '{"spec": ' + json.dumps(SPEC_VERSION) + if self.source is not None: + document += ', "source": ' + json.dumps(self.source) + document += f', "hash": "sha256:{hashlib.sha256(self.raw).hexdigest()}"' + document += ', "manifest": ' + return document.encode() + self.raw + b"}" + + def resolve(self, body: Any) -> ResolvedRequest: + """Validate a create-generation request body and map it to the pipeline call.""" + if not isinstance(body, dict): + raise GMSError("invalid_request", "request body must be a JSON object") + for key in body: + if key not in _REQUEST_FIELDS: + raise GMSError("invalid_request", f"unknown request field {key!r}", f"/{key}") + if not isinstance(body.get("model"), str): + raise GMSError("invalid_request", "model is required and must be a string", "/model") + if body["model"] != self.id: + raise GMSError("not_found", f"model {body['model']!r} is not served here; this server hosts {self.id!r}") + inputs = body.get("inputs") + if not isinstance(inputs, dict): + raise GMSError("invalid_request", "inputs is required and must be an object", "/inputs") + + # The task: the one requested, else the only one whose inputs match what was supplied. + task = body.get("task") + if task is not None: + if not isinstance(task, str) or task not in self.tasks: + raise GMSError( + "invalid_request", + f"task {task!r} is not hosted for {self.id!r}; hosted tasks: {list(self.tasks)}", + "/task", + ) + elif len(self.tasks) == 1: + task = next(iter(self.tasks)) + else: + candidates = [] + for name, spec in self.tasks.items(): + required = {key for key, input_spec in spec["inputs"].items() if input_spec.get("required")} + if required <= set(inputs) <= set(spec["inputs"]): + candidates.append(name) + if len(candidates) == 1: + task = candidates[0] + else: + raise GMSError( + "invalid_request", + f"could not infer the task from inputs {sorted(inputs)}; set `task` to one of {list(self.tasks)}", + "/task", + ) + declared_inputs = self.tasks[task]["inputs"] + declared_parameters = self.tasks[task].get("parameters", {}) + + # The parameter values: declared defaults, then the preset, then the request's own. Values the request set + # itself are held to stricter rules than defaults. + explicit = body.get("parameters", {}) + if not isinstance(explicit, dict): + raise GMSError("invalid_request", "parameters must be an object", "/parameters") + for key, value in explicit.items(): + if key not in declared_parameters: + raise GMSError("invalid_request", f"task {task!r} declares no parameter {key!r}", f"/parameters/{key}") + _check_value(key, value, declared_parameters[key], f"/parameters/{key}") + values = {key: spec["default"] for key, spec in declared_parameters.items() if "default" in spec} + preset = body.get("preset") + if preset is not None: + presets = self.tasks[task].get("presets", {}) + if preset not in presets: + raise GMSError( + "invalid_request", + f"preset {preset!r} is not declared for task {task!r}; declared presets: {list(presets)}", + "/preset", + ) + values.update(presets[preset].get("parameters", {})) + values.update(explicit) + + # Inputs: text is passed as is; media is checked here and loaded by the backend. + kwargs: dict[str, Any] = {} + media: dict[str, tuple[str, str, Any]] = {} + for key, value in inputs.items(): + pointer = f"/inputs/{key}" + spec = declared_inputs.get(key) + if spec is None: + raise GMSError("invalid_request", f"task {task!r} declares no input {key!r}", pointer) + kwarg = self._input_kwargs[task][key] + if "active_when" in spec and not _is_active(spec["active_when"], values): + raise GMSError( + "invalid_request", f"input {key!r} is inactive unless {json.dumps(spec['active_when'])}", pointer + ) + if spec["type"] == "text": + if not isinstance(value, str): + raise GMSError("invalid_request", f"{key} must be a string, got {value!r}", pointer) + kwargs[kwarg] = value + continue + if isinstance(value, list): + if "multiple" not in spec: + raise GMSError("invalid_request", f"{key} accepts a single media value, not a list", pointer) + limit = spec["multiple"].get("max") + if limit is not None and len(value) > limit: + raise GMSError( + "invalid_request", f"{key} accepts at most {limit} items, got {len(value)}", pointer + ) + for index, item in enumerate(value): + _check_media(key, item, spec, f"{pointer}/{index}") + else: + _check_media(key, value, spec, pointer) + media[kwarg] = (key, spec["type"], value) + for key, spec in declared_inputs.items(): + if spec.get("required") and key not in inputs: + raise GMSError("invalid_request", f"task {task!r} requires input {key!r}", f"/inputs/{key}") + + # Parameters. The seed is pulled out, since it seeds a generator instead of being passed as is. + seed = None + for key, value in values.items(): + spec = declared_parameters[key] + kwarg = self._parameter_kwargs[task][key] + pointer = f"/parameters/{key}" + if "active_when" in spec and not _is_active(spec["active_when"], values): + if key in explicit: + raise GMSError( + "invalid_request", + f"parameter {key!r} is inactive unless {json.dumps(spec['active_when'])}", + pointer, + ) + continue + if key == "seed": + if not 0 <= value < 2**63: + raise GMSError("invalid_request", f"seed must be between 0 and 2**63 - 1, got {value}", pointer) + seed = value + continue + if key == "duration": + lattice_fps = spec["lattice"]["fps"] + if values.get("fps", lattice_fps) != lattice_fps: + raise GMSError( + "invalid_request", + f"duration is declared on a {lattice_fps} fps frame lattice; fps must be {lattice_fps} " + f"when duration is set, got {values['fps']}", + "/parameters/fps", + ) + step, offset, _ = _frame_lattice(spec) + position = (value * lattice_fps - offset) / step + snap = spec["lattice"].get("snap", "nearest") + if snap == "reject" and not math.isclose(position, round(position), abs_tol=1e-9): + raise GMSError( + "invalid_request", + f"duration {value} is off the {spec['lattice']['frames']} frame lattice at {lattice_fps} fps", + pointer, + ) + index = math.ceil(position - 1e-9) if snap == "up" else round(position) + kwargs[kwarg] = step * max(index, 0) + offset + if self._fps_kwargs[task] is not None: + kwargs[self._fps_kwargs[task]] = lattice_fps + continue + kwargs[kwarg] = value + # The spec has the server pick the seed when the request omits it, so the result is reproducible. + if seed is None and "seed" in self._parameter_kwargs[task]: + seed = random.randrange(2**32) + + # Outputs, each with the media type it is encoded as. + output_formats = {kind: formats[0] for kind, formats in OUTPUT_FORMATS.items()} + output_format = body.get("output_format") + if output_format is not None: + kind = output_format.split("/")[0] if isinstance(output_format, str) else None + produced = {spec["type"] for spec in self.tasks[task]["outputs"].values()} + # The startup check limits output types to those in OUTPUT_FORMATS. + if kind not in produced or output_format not in OUTPUT_FORMATS[kind]: + supported = sorted( + media_type for produced_kind in produced for media_type in OUTPUT_FORMATS[produced_kind] + ) + raise GMSError( + "invalid_request", + f"output_format must be one of {supported} for task {task!r}, got {output_format!r}", + "/output_format", + ) + output_formats[kind] = output_format + outputs = { + name: (spec["type"], output_formats[spec["type"]]) for name, spec in self.tasks[task]["outputs"].items() + } + fps = None + if any(kind == "video" for kind, _ in outputs.values()): + duration = declared_parameters.get("duration") + fps = ( + kwargs.get(self._fps_kwargs[task]) + or self._default_fps + or (_frame_lattice(duration)[2] if duration is not None else DEFAULT_FPS) + ) + + response_format = body.get("response_format", "url") + if response_format not in ("url", "b64_json"): + raise GMSError( + "invalid_request", + f"response_format must be 'url' or 'b64_json', got {response_format!r}", + "/response_format", + ) + + # Adapters, each resolved to its path and the weight `set_adapters` takes. + adapters = [] + if body.get("adapters") is not None: + if "adapters" not in self.capabilities: + raise GMSError("not_implemented", "this server does not apply adapters", "/adapters") + if not isinstance(body["adapters"], list): + raise GMSError("invalid_request", "adapters must be an array", "/adapters") + if len(body["adapters"]) > MAX_ADAPTERS: + raise GMSError( + "invalid_request", + f"at most {MAX_ADAPTERS} adapters per request, got {len(body['adapters'])}", + "/adapters", + ) + for index, adapter in enumerate(body["adapters"]): + pointer = f"/adapters/{index}" + if not isinstance(adapter, dict): + raise GMSError("invalid_request", "an adapter must be an object", pointer) + for key in adapter: + if key not in _ADAPTER_FIELDS: + raise GMSError("invalid_request", f"unknown adapter field {key!r}", f"{pointer}/{key}") + if not isinstance(adapter.get("type"), str) or adapter["type"] not in self.adapters: + raise GMSError( + "invalid_request", + f"adapter type must be one of {list(self.adapters)}, got {adapter.get('type')!r}", + f"{pointer}/type", + ) + path = adapter.get("path") + repo_id = path.removeprefix("hf:") if isinstance(path, str) and path.startswith("hf:") else None + if repo_id is not None: + try: + validate_repo_id(repo_id) + except HFValidationError: + repo_id = None + # `load_lora_weights` prefers a local path over a Hub repo of the same name, so a repo id that also + # names a path on the server's disk is refused as well. + if repo_id is None or "/" not in repo_id or os.path.exists(repo_id): + raise GMSError( + "invalid_request", + f"adapter path must be a Hub repo id of the form 'hf:/', got {path!r}", + f"{pointer}/path", + ) + scale = adapter.get("scale", 1.0) + if isinstance(scale, bool) or not isinstance(scale, (int, float)): + raise GMSError( + "invalid_request", f"adapter scale must be a number, got {scale!r}", f"{pointer}/scale" + ) + valid_targets = self.adapters[adapter["type"]].get("targets", []) + targets = adapter.get("targets", {}) + if not isinstance(targets, dict): + raise GMSError("invalid_request", "adapter targets must be an object", f"{pointer}/targets") + for target, target_scale in targets.items(): + if target not in valid_targets: + raise GMSError( + "invalid_request", + f"adapter target must be one of {valid_targets}, got {target!r}", + f"{pointer}/targets/{target}", + ) + if isinstance(target_scale, bool) or not isinstance(target_scale, (int, float)): + raise GMSError( + "invalid_request", + f"adapter target scale must be a number, got {target_scale!r}", + f"{pointer}/targets/{target}", + ) + weight = {target: targets.get(target, scale) for target in valid_targets} if targets else scale + adapters.append({"path": path, "weight": weight}) + + return ResolvedRequest( + task=task, + kwargs=kwargs, + media=media, + seed=seed, + outputs=outputs, + fps=fps, + response_format=response_format, + adapters=adapters, + ) + + +# --------------------------------------------------------------------------- +# Value checks +# --------------------------------------------------------------------------- + + +def _check_value(label: str, value: Any, spec: dict[str, Any], pointer: str) -> None: + """Check a parameter value against its declared schema: type, enum, range and `multiple_of`.""" + kind = spec.get("type") + if kind == "array": + if not isinstance(value, list): + raise GMSError("invalid_request", f"{label} must be an array, got {value!r}", pointer) + if "maxItems" in spec and len(value) > spec["maxItems"]: + raise GMSError("invalid_request", f"{label} accepts at most {spec['maxItems']} items", pointer) + for index, item in enumerate(value): + _check_value(f"{label}[{index}]", item, spec["items"], f"{pointer}/{index}") + return + if kind == "object": + if not isinstance(value, dict): + raise GMSError("invalid_request", f"{label} must be an object, got {value!r}", pointer) + for key, item in value.items(): + if key not in spec["properties"]: + raise GMSError("invalid_request", f"{label} has no property {key!r}", f"{pointer}/{key}") + _check_value(f"{label}.{key}", item, spec["properties"][key], f"{pointer}/{key}") + return + + expected = _PYTHON_TYPES.get(kind) + is_bool = isinstance(value, bool) + if expected is not None and (not isinstance(value, expected) or (is_bool and kind != "boolean")): + raise GMSError("invalid_request", f"{label} must be of type {kind}, got {value!r}", pointer) + if isinstance(value, float) and not math.isfinite(value): + raise GMSError("invalid_request", f"{label} must be a finite number, got {value}", pointer) + if "enum" in spec and value not in spec["enum"]: + raise GMSError("invalid_request", f"{label} must be one of {spec['enum']}, got {value!r}", pointer) + if "min" in spec and value < spec["min"]: + raise GMSError("invalid_request", f"{label} must be at least {spec['min']}, got {value}", pointer) + if "max" in spec and value > spec["max"]: + raise GMSError("invalid_request", f"{label} must be at most {spec['max']}, got {value}", pointer) + if "multiple_of" in spec: + quotient = value / spec["multiple_of"] + if not math.isclose(quotient, round(quotient), abs_tol=1e-9): + raise GMSError( + "invalid_request", f"{label} must be a multiple of {spec['multiple_of']}, got {value}", pointer + ) + + +def _check_media(name: str, value: Any, spec: dict[str, Any], pointer: str) -> None: + if not isinstance(value, dict): + raise GMSError("invalid_request", f"{name} must be a media value object, got {value!r}", pointer) + if value.get("type") != spec["type"]: + raise GMSError( + "invalid_request", f"{name} must have type {spec['type']!r}, got {value.get('type')!r}", f"{pointer}/type" + ) + keys = set(value) + if keys == {"type", "base64", "media_type"}: + if not isinstance(value["base64"], str) or not isinstance(value["media_type"], str): + raise GMSError("invalid_request", f"{name} base64 and media_type must be strings", pointer) + return + if "url" not in keys or not keys <= {"type", "url", "media_type"}: + raise GMSError( + "invalid_request", + f"{name} must be either {{type, url, media_type?}} or {{type, base64, media_type}}, got keys {sorted(keys)}", + pointer, + ) + # A URL media value is fetched by the server, so anything but http(s) would read the server's own filesystem. + if not isinstance(value["url"], str) or not value["url"].startswith(("http://", "https://")): + raise GMSError("invalid_request", f"{name} url must be an http(s) URL", f"{pointer}/url") + + +def _is_active(condition: dict[str, Any], values: dict[str, Any]) -> bool: + """Evaluate an `active_when` condition against the parameter values.""" + if "all" in condition: + return all(_is_active(item, values) for item in condition["all"]) + actual = values.get(condition["parameter"]) + for name, compare in _COMPARISONS.items(): + if name not in condition: + continue + if actual is None or isinstance(actual, str) != isinstance(condition[name], str): + return False + if not compare(actual, condition[name]): + return False + return True + + +def _frame_lattice(spec: dict[str, Any]) -> tuple[int, int, float] | None: + """Parse a `duration` parameter's frame lattice, e.g. `8k+1` at 24 fps, into `(8, 1, 24)`.""" + lattice = spec.get("lattice") or {} + match = _FRAME_LATTICE.fullmatch(str(lattice.get("frames", "")).replace(" ", "")) + if match is None or not isinstance(lattice.get("fps"), (int, float)): + return None + return int(match.group(1)), int(match.group(2)), lattice["fps"] diff --git a/src/diffusers/commands/serve/remote.py b/src/diffusers/commands/serve/remote.py new file mode 100644 index 000000000000..e7b6a4f7ff61 --- /dev/null +++ b/src/diffusers/commands/serve/remote.py @@ -0,0 +1,162 @@ +# Copyright 2026 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""`diffusers-cli serve --remote` — run the server in a Hugging Face Job, reachable on the port the Job exposes.""" + +from __future__ import annotations + +import shlex +import sys +import threading +from argparse import ArgumentParser, Namespace +from pathlib import Path +from typing import Any + +from huggingface_hub import HfApi, get_token +from huggingface_hub.cli._output import out +from huggingface_hub.utils import httpx, send_telemetry + +from ... import __version__ +from ...utils import logging +from ...utils.constants import DIFFUSERS_TRUST_REMOTE_KERNELS +from ..run import _DEFAULT_REMOTE_DEPS, _DEFAULT_REMOTE_IMAGE, _build_task_kwargs, _kwargs_to_argv + + +logger = logging.get_logger("diffusers-cli/serve") + +# The port the server binds inside the Job. The Jobs proxy publishes it at `https://--.hf.jobs`. +JOB_PORT = 8000 +_SERVE_DEPS = ("fastapi", "uvicorn") +_DEFAULT_REMOTE_IDLE_TIMEOUT = "30m" +# The Job ends when the server shuts down after --idle-timeout. This cap only stops a server that never goes idle, +# so it can't run, and bill, without end. +_JOB_TIMEOUT = "24h" +_READY_POLL_SECONDS = 5.0 + + +def add_remote_arguments(parser: ArgumentParser) -> None: + parser.add_argument( + "--remote", + action="store_true", + help="Run the server in a Hugging Face Job instead of on the local machine.", + ) + parser.add_argument( + "--flavor", + default="a10g-small", + help="HF Jobs hardware flavor for --remote (e.g. a10g-small, a100-large).", + ) + parser.add_argument( + "--dependencies", + action="append", + default=None, + help="Extra pip dependencies to install in the Job. Repeat to add multiple.", + ) + parser.add_argument( + "--namespace", + default=None, + help="HF namespace to run the Job under (defaults to the current user).", + ) + parser.add_argument( + "--image", + default=None, + help=f"Job image for --remote. Defaults to {_DEFAULT_REMOTE_IMAGE!r}.", + ) + + +def _announce_when_ready(api: HfApi, job: Any, args: Namespace, stopped: threading.Event) -> None: + while not stopped.wait(_READY_POLL_SECONDS): + urls = api.inspect_job(job_id=job.id, namespace=job.owner.name).status.expose_urls + if not urls: + continue + try: + response = httpx.get(f"{urls[0]}/health", headers={"Authorization": f"Bearer {api.token}"}, timeout=10.0) + except httpx.HTTPError: + continue + if response.status_code != 200: + continue + out.result( + "remote-serve", + job_id=job.id, + url=urls[0], + model=args.model, + # The HF token is left as a variable so it is not printed to the terminal. + hint=f'curl {urls[0]}/v1/models -H "Authorization: Bearer $HF_TOKEN"', + ) + return + + +def serve_remote(args: Namespace, task: str) -> None: + if Path(args.model).exists(): + raise SystemExit( + f"--model {args.model!r} is a local path; the Job can't see it. " + "Pass a Hub repo id so the Job can download it." + ) + if args.manifest is not None: + raise SystemExit( + f"--manifest is not supported with --remote: the Job can't read {args.manifest!r}. Upload it as " + f"`gms.json` to the model repo, e.g. `hf upload {args.model} {args.manifest} gms.json`, and the " + "remote server will use it." + ) + + # The Job installs the dependencies, then starts the server in the foreground. A local --output-dir means nothing + # inside the Job, and --public-url is set from the Job id below. + task_kwargs = _build_task_kwargs(args) + task_kwargs.pop("output_dir", None) + task_kwargs.pop("public_url", None) + task_kwargs.update(host="0.0.0.0", port=JOB_PORT, idle_timeout=args.idle_timeout or _DEFAULT_REMOTE_IDLE_TIMEOUT) + dependencies = [*_DEFAULT_REMOTE_DEPS, *_SERVE_DEPS, *(args.dependencies or [])] + install = shlex.join(["uv", "pip", "install", "--system", "--break-system-packages", *dependencies]) + # An `--image` can put another environment first on PATH, with its own older `diffusers-cli`, so the server + # is started with the interpreter the install went into. The Job id is only known + # inside the Job, and output links must be built on the public address. + start = ( + 'exec "$(uv python find --system)" -m diffusers.commands.diffusers_cli --format quiet ' + f"{shlex.join(_kwargs_to_argv(task, task_kwargs))} " + f'--public-url "https://${{JOB_ID}}--{JOB_PORT}.hf.jobs"' + ) + hf_token = args.token or get_token() + env = {"HF_ENABLE_PARALLEL_LOADING": "1"} + if DIFFUSERS_TRUST_REMOTE_KERNELS: + env["DIFFUSERS_TRUST_REMOTE_KERNELS"] = "true" + + api = HfApi(token=hf_token) + logger.info(f"starting a Job on flavor={args.flavor!r}...") + job = api.run_job( + image=args.image or _DEFAULT_REMOTE_IMAGE, + command=["sh", "-c", f"{install} && {start}"], + env=env, + secrets={"HF_TOKEN": hf_token} if hf_token else None, + flavor=args.flavor, + timeout=_JOB_TIMEOUT, + expose=[JOB_PORT], + namespace=args.namespace, + ) + send_telemetry(topic="diffusers/cli/serve/remote", library_name="diffusers", library_version=__version__) + + stage = None + stopped = threading.Event() + try: + threading.Thread(target=_announce_when_ready, args=(api, job, args, stopped), daemon=True).start() + logger.info(f"streaming the logs of Job {job.id}; stop it with Ctrl-C.") + for line in api.fetch_job_logs(job_id=job.id, namespace=job.owner.name, follow=True): + sys.stderr.write(line + "\n") + sys.stderr.flush() + stage = api.inspect_job(job_id=job.id, namespace=job.owner.name).status.stage + finally: + stopped.set() + if stage is None: + api.cancel_job(job_id=job.id, namespace=job.owner.name) + + if stage != "COMPLETED": + raise SystemExit(f"remote server Job {job.id} ended with status {stage}") diff --git a/src/diffusers/commands/serve/scheduler.py b/src/diffusers/commands/serve/scheduler.py new file mode 100644 index 000000000000..6816fd676088 --- /dev/null +++ b/src/diffusers/commands/serve/scheduler.py @@ -0,0 +1,247 @@ +# Copyright 2026 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Generation jobs, and the scheduler that runs them in arrival order, one at a time on each backend.""" + +from __future__ import annotations + +import asyncio +import queue +import shutil +import threading +import time +import uuid +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any, Callable + +from ...utils import logging +from .manifest import GMSError, ResolvedRequest + + +logger = logging.get_logger("diffusers-cli/serve") + +EXPIRY_SECONDS = 3600 + + +class GenerationJob: + """One generation, from the request that created it to its outputs: the resource `/v1/generations/{id}` returns.""" + + def __init__(self, request: ResolvedRequest): + self.id = f"gen_{uuid.uuid4().hex[:12]}" + self.request = request + self.status = "queued" + self.created_at = datetime.now(timezone.utc) + self.expires_at: datetime | None = None + self.progress: dict[str, Any] | None = None + self.outputs: dict[str, list[dict[str, Any]]] | None = None + self.usage: dict[str, Any] | None = None + self.error: GMSError | None = None + self.canceled = False + # The `time.monotonic()` time a running generation must stop by, from `--generation-timeout`; None when it has + # no limit. + self.deadline: float | None = None + self.directory: Path | None = None + self.events: list[tuple[str, dict[str, Any] | None]] = [] + self.done = False + self._lock = threading.Lock() + # Requests waiting on the event loop for the job to change. The worker thread wakes them through their loop. + self._waiters: list[tuple[asyncio.AbstractEventLoop, asyncio.Event]] = [] + + @property + def cancel_requested(self) -> bool: + """Whether the pipeline should stop at its next step: a client canceled it, or it ran past its deadline.""" + return self.canceled or (self.deadline is not None and time.monotonic() > self.deadline) + + def report_progress(self, step: int, total_steps: int) -> None: + with self._lock: + self.progress = {"step": step, "total_steps": total_steps} + self.events.append(("progress", self.progress)) + self._notify_waiting_requests() + + async def wait_for_events( + self, index: int, timeout: float + ) -> tuple[list[tuple[str, dict[str, Any] | None]], bool]: + await self._wait_until(lambda: len(self.events) > index or self.done, timeout) + # Read together, so a stream never sees `done` without the final event that came with it. + with self._lock: + return self.events[index:], self.done + + async def wait(self, timeout: float) -> None: + await self._wait_until(lambda: self.done, timeout) + + def finish(self, status: str, error: GMSError | None = None) -> None: + with self._lock: + self.status = status + self.error = error + self.progress = None + self.expires_at = datetime.now(timezone.utc) + timedelta(seconds=EXPIRY_SECONDS) + if error is not None: + self.events.append(("error", {"code": error.code, "message": error.message})) + elif status == "complete": + self.events.append(("complete", None)) + self.done = True + self._notify_waiting_requests() + + async def _wait_until(self, condition: Callable[[], bool], timeout: float) -> None: + # Waiting on the event loop, not a pool thread, so any number of requests can wait at once. + loop = asyncio.get_running_loop() + give_up_at = loop.time() + timeout + while True: + event = asyncio.Event() + with self._lock: + if condition(): + return + self._waiters.append((loop, event)) + try: + await asyncio.wait_for(event.wait(), give_up_at - loop.time()) + except asyncio.TimeoutError: + return + finally: + with self._lock: + self._waiters.remove((loop, event)) + + def _notify_waiting_requests(self) -> None: + for loop, event in self._waiters: + # A loop closed at shutdown has no request left to wake, and raising here would end the worker. + if not loop.is_closed(): + loop.call_soon_threadsafe(event.set) + + +class Scheduler: + """Runs generation jobs first in, first out, on one worker thread per backend. + + It also keeps every job until it expires, so the API can look jobs up, cancel them and delete them. + """ + + def __init__(self, backends: list[Any], output_dir: str, timeout: float | None = None): + self._output_dir = Path(output_dir) + self._timeout = timeout + self._generations: dict[str, GenerationJob] = {} + self._lock = threading.Lock() + self._pending: queue.Queue[GenerationJob] = queue.Queue() + # The generations a worker is running now. + self._running: set[GenerationJob] = set() + # Every worker takes the next job from the same queue, so no job waits behind a long one on a busy backend. + for index, backend in enumerate(backends): + threading.Thread( + target=self._work, args=(backend,), name=f"diffusers-serve-worker-{index}", daemon=True + ).start() + + def failure(self) -> str | None: + if self._timeout is None: + return None + # A timed out generation stops at its next step. One still running long after that is stuck inside a step, + # or in a pipeline without step callbacks, and its worker takes no more jobs. + now = time.monotonic() + with self._lock: + for generation in self._running: + if now > generation.deadline + self._timeout: + return ( + f"generation {generation.id} has run for over twice the --generation-timeout of " + f"{self._timeout:g}s and did not stop when interrupted" + ) + return None + + def busy(self) -> bool: + return bool(self._running) or not self._pending.empty() + + def submit(self, generation: GenerationJob) -> None: + self._prune() + with self._lock: + self._generations[generation.id] = generation + self._pending.put(generation) + + def get(self, generation_id: str) -> GenerationJob: + self._prune() + with self._lock: + generation = self._generations.get(generation_id) + if generation is None: + raise GMSError("not_found", f"no generation with id {generation_id!r}") + return generation + + def cancel(self, generation_id: str) -> GenerationJob: + generation = self.get(generation_id) + with self._lock: + if generation.done: + return generation + generation.canceled = True + if generation.status == "queued": + generation.finish("canceled") + return generation + + def delete(self, generation_id: str) -> None: + generation = self.cancel(generation_id) + with self._lock: + del self._generations[generation.id] + # A generation still running keeps writing into its directory; the worker removes it once the backend returns. + if generation.done: + self._remove_files(generation) + + def _prune(self) -> None: + now = datetime.now(timezone.utc) + with self._lock: + expired = [g for g in self._generations.values() if g.expires_at is not None and g.expires_at <= now] + for generation in expired: + del self._generations[generation.id] + for generation in expired: + self._remove_files(generation) + + def _remove_files(self, generation: GenerationJob) -> None: + if generation.directory is not None: + shutil.rmtree(generation.directory, ignore_errors=True) + + def _work(self, backend: Any) -> None: + while True: + generation = self._pending.get() + with self._lock: + if generation.done: + continue + generation.status = "generating" + if self._timeout is not None: + generation.deadline = time.monotonic() + self._timeout + self._running.add(generation) + generation.directory = self._output_dir / generation.id + + started = time.perf_counter() + error = None + try: + generation.directory.mkdir(parents=True, exist_ok=True) + outputs = backend.generate(generation) + except GMSError as e: + error = e + # The helpers shared with `run` raise SystemExit, and anything escaping here would end this + # worker and leave its backend idle for good. + except BaseException as e: + logger.error(f"generation {generation.id} failed", exc_info=True) + error = GMSError("internal", f"{type(e).__name__}: {e}") + returned_at = time.monotonic() + + with self._lock: + deleted = generation.id not in self._generations + # Before `finish` wakes any waiter, so a finished job is never reported while the server looks busy. + self._running.discard(generation) + if error is not None: + generation.finish("failed", error) + elif generation.deadline is not None and returned_at > generation.deadline: + message = f"generation exceeded the --generation-timeout of {self._timeout:g}s" + generation.finish("failed", GMSError("internal", message)) + elif generation.canceled: + generation.finish("canceled") + else: + generation.outputs = outputs + generation.usage = {"compute_seconds": round(time.perf_counter() - started, 2)} + generation.finish("complete") + if deleted or generation.status != "complete": + self._remove_files(generation) diff --git a/src/diffusers/dependency_versions_table.py b/src/diffusers/dependency_versions_table.py index da572914dc57..9646698613ab 100644 --- a/src/diffusers/dependency_versions_table.py +++ b/src/diffusers/dependency_versions_table.py @@ -53,4 +53,6 @@ "opencv-python": "opencv-python", "timm": "timm", "flashpack": "flashpack", + "fastapi": "fastapi", + "uvicorn": "uvicorn", } diff --git a/tests/others/test_cli_commands.py b/tests/others/test_cli_commands.py index d05d2ff97f2e..ee98f5f4ffbd 100644 --- a/tests/others/test_cli_commands.py +++ b/tests/others/test_cli_commands.py @@ -54,7 +54,7 @@ ) -AVAILABLE_COMMANDS = ("env", "fp16_safetensors", "custom_blocks", "run", "schema", "skills") +AVAILABLE_COMMANDS = ("env", "fp16_safetensors", "custom_blocks", "run", "schema", "serve", "skills") class TestRunCommand: diff --git a/tests/others/test_cli_serve.py b/tests/others/test_cli_serve.py new file mode 100644 index 000000000000..d4757f385f88 --- /dev/null +++ b/tests/others/test_cli_serve.py @@ -0,0 +1,939 @@ +# Copyright 2026 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Unit tests for `diffusers-cli serve`. + +No weights are loaded: the backend runs a fake pipeline. +""" + +import asyncio +import base64 +import inspect +import io +import json +import re +import shlex +import threading +import time +from argparse import ArgumentParser, Namespace +from pathlib import Path +from types import SimpleNamespace + +import numpy as np +import pytest +import torch +from huggingface_hub.utils import httpx +from PIL import Image + +from diffusers.commands.run import _DEFAULT_REMOTE_IMAGE +from diffusers.commands.serve import ServeCommand, _shut_down_when_idle +from diffusers.commands.serve.backends import DiffusersBackend, _load_media, backend_capabilities +from diffusers.commands.serve.manifest import GMSError, Manifest, blocks_signature, derive_manifest +from diffusers.commands.serve.scheduler import GenerationJob, Scheduler +from diffusers.modular_pipelines import InputParam, LoopSequentialPipelineBlocks, ModularPipelineBlocks, OutputParam + + +MODEL_ID = "test-org/fake-model" +WAIT = {"Prefer": "wait=30"} + + +class FakePipeline: + _lora_loadable_modules = ["transformer", "text_encoder"] + device = torch.device("cpu") + num_timesteps = 0 + + def __init__(self): + self.calls = [] + self.lora_calls = [] + self.release = threading.Event() + self.release.set() + self.started = threading.Event() + self.error = None + self.result = None + + def __call__( + self, + prompt: str | list[str] = None, + negative_prompt: str | list[str] | None = None, + image: Image.Image | None = None, + height: int | None = None, + width: int | None = None, + num_inference_steps: int = 4, + sigmas: list[float] | None = None, + guidance_scale: float = 3.5, + num_images_per_prompt: int | None = 1, + generator: torch.Generator | None = None, + latents: torch.Tensor | None = None, + output_type: str | None = "pil", + return_dict: bool = True, + callback_on_step_end=None, + callback_on_step_end_tensor_inputs: list[str] = ["latents"], + ): + self.calls.append( + { + "prompt": prompt, + "negative_prompt": negative_prompt, + "image": image, + "height": height, + "width": width, + "num_inference_steps": num_inference_steps, + "guidance_scale": guidance_scale, + "generator": generator, + } + ) + self.started.set() + self.release.wait(timeout=30) + if self.error is not None: + raise self.error + self.num_timesteps = num_inference_steps + self._interrupt = False + for step in range(num_inference_steps): + if self._interrupt: + break + callback_on_step_end(self, step, 0, {}) + if self.result is not None: + return self.result + size = (width or 32, height or 32) + return SimpleNamespace(images=[Image.new("RGB", size, "red") for _ in range(num_images_per_prompt)]) + + def load_lora_weights(self, repo, adapter_name=None): + self.lora_calls.append(("load", repo, adapter_name)) + + def set_adapters(self, names, adapter_weights=None): + self.lora_calls.append(("set", names, adapter_weights)) + + def unload_lora_weights(self): + self.lora_calls.append(("unload",)) + + +FAKE_SIGNATURE = inspect.signature(FakePipeline.__call__) + + +class FakeImageBlock(ModularPipelineBlocks): + """A one-block modular pipeline that declares its inputs the way the built-in blocks do, mostly untyped.""" + + model_name = "fake" + + @property + def inputs(self): + return [ + InputParam("prompt"), + InputParam("image"), + InputParam("width"), + InputParam("num_inference_steps", default=4), + InputParam("generator"), + ] + + @property + def intermediate_outputs(self): + return [OutputParam("images")] + + def __call__(self, components, state): + block_state = self.get_block_state(state) + size = (block_state.width or 32, 32) + block_state.images = [Image.new("RGB", size, "red")] + self.set_block_state(state, block_state) + return components, state + + +class FakeDenoiser(ModularPipelineBlocks): + @property + def inputs(self): + return [] + + def __call__(self, components, block_state, i, t): + return components, block_state + + +class FakeDenoiseLoop(LoopSequentialPipelineBlocks): + """A modular pipeline whose denoising loop, like the built-in ones, has no step callback of its own.""" + + model_name = "fake" + block_classes = [FakeDenoiser] + block_names = ["denoiser"] + + @property + def description(self): + return "Denoises for `num_inference_steps` steps." + + @property + def loop_inputs(self): + return [InputParam("prompt"), InputParam("num_inference_steps", default=3)] + + @property + def loop_intermediate_outputs(self): + return [OutputParam("images")] + + def __call__(self, components, state): + block_state = self.get_block_state(state) + block_state.timesteps = list(range(block_state.num_inference_steps)) + for i, t in enumerate(block_state.timesteps): + components, block_state = self.loop_step(components, block_state, i=i, t=t) + block_state.images = [Image.new("RGB", (32, 32), "red")] + self.set_block_state(state, block_state) + return components, state + + +def _args(**overrides): + values = { + "compile": None, + "lora": None, + "allowed_media_domains": None, + } + return Namespace(**{**values, **overrides}) + + +def _manifest(manifest=None, pipeline_cls=FakePipeline, signature=None, args=None): + """A `Manifest` bound to `pipeline_cls`, from the given manifest dict or bytes, or derived from the pipeline.""" + signature = signature or inspect.signature(pipeline_cls.__call__) + capabilities = backend_capabilities(args or _args(), pipeline_cls) + if manifest is None: + manifest = derive_manifest(MODEL_ID, pipeline_cls, signature, capabilities) + raw = manifest if isinstance(manifest, bytes) else json.dumps(manifest).encode() + return Manifest(raw, json.loads(raw), pipeline_cls, signature, capabilities, revision="abc123") + + +@pytest.fixture +def serve(tmp_path, monkeypatch): + """Build a test client around a `DiffusersBackend` running `FakePipeline`, or around the given pipeline or + backend. Returns `(client, pipeline)`.""" + pytest.importorskip("fastapi", reason="`diffusers-cli serve` needs the `diffusers[serve]` extra") + from fastapi.testclient import TestClient + + from diffusers.commands.serve.app import create_app + + def _build(manifest=None, app_options=None, pipeline=None, signature=None, backend=None, **arg_overrides): + pipeline = pipeline or FakePipeline() + monkeypatch.setattr("diffusers.commands.serve.backends._load_pipeline", lambda args: pipeline) + args = _args(**arg_overrides) + bound = _manifest(manifest=manifest, pipeline_cls=type(pipeline), signature=signature, args=args) + backend = backend or DiffusersBackend(args, bound) + app = create_app(bound, Scheduler([backend], str(tmp_path)), **(app_options or {})) + return TestClient(app), pipeline + + return _build + + +LATTICE_MANIFEST = { + "id": MODEL_ID, + "modalities": {"in": ["image"], "out": ["video"]}, + "tasks": { + "text_to_video": { + "inputs": { + "prompt": {"type": "text", "required": True}, + "negative_prompt": {"type": "text", "active_when": {"parameter": "guidance_scale", "gt": 1}}, + }, + "outputs": {"video": {"type": "video"}}, + "parameters": { + "duration": {"type": "number", "min": 1, "max": 20, "lattice": {"frames": "8k+1", "fps": 24}}, + "width": {"type": "integer", "default": 768, "min": 512, "max": 1536, "multiple_of": 64}, + "steps": {"type": "integer", "default": 30, "min": 1, "max": 100}, + "fps": {"type": "number"}, + "guidance_scale": {"type": "number", "default": 3.0, "min": 1.0, "max": 10.0}, + "seed": {"type": "integer"}, + }, + "presets": { + "default": {}, + "distilled": {"parameters": {"steps": 8, "guidance_scale": 1.0}}, + }, + } + }, + "capabilities": ["sse", "wait"], +} + + +class FakeVideoPipeline: + def __call__( + self, + prompt: str = None, + negative_prompt: str | None = None, + width: int = 768, + num_frames: int = 121, + frame_rate: float = 24.0, + num_inference_steps: int = 40, + guidance_scale: float = 4.0, + generator: torch.Generator | None = None, + ): + pass + + +class FakeAudioVideoPipeline(FakePipeline): + vocoder = SimpleNamespace(config=SimpleNamespace(sampling_rate=24000)) + + def __call__(self, prompt: str = None, frame_rate: float = 24.0): + return self.result + + +class TestManifest: + def test_derive_manifest_from_signature(self): + manifest = derive_manifest(MODEL_ID, FakePipeline, FAKE_SIGNATURE, ["sse", "wait", "adapters"]) + + # An optional `image` kwarg yields one task without it and one that requires it. + assert list(manifest["tasks"]) == ["text_to_image", "image_to_image"], manifest["tasks"].keys() + text_to_image = manifest["tasks"]["text_to_image"] + assert text_to_image["inputs"] == { + "prompt": {"type": "text", "required": True}, + "negative_prompt": {"type": "text"}, + } + assert manifest["tasks"]["image_to_image"]["inputs"]["image"] == {"type": "image", "required": True} + assert text_to_image["outputs"] == {"image": {"type": "image"}} + # Core vocabulary names replace the pipeline's own; tensors, callbacks and output plumbing are dropped. + assert text_to_image["parameters"] == { + "height": {"type": "integer"}, + "width": {"type": "integer"}, + "steps": {"type": "integer", "default": 4}, + "sigmas": {"type": "array", "items": {"type": "number"}}, + "guidance_scale": {"type": "number", "default": 3.5}, + "num_outputs": {"type": "integer", "default": 1}, + "seed": {"type": "integer"}, + } + assert text_to_image["x-diffusers"] == { + "parameters": {"steps": "num_inference_steps", "num_outputs": "num_images_per_prompt", "seed": "generator"} + } + assert manifest["capabilities"] == ["sse", "wait", "adapters"] + assert manifest["adapters"] == {"lora": {"targets": ["transformer", "text_encoder"]}} + + def test_output_type_comes_from_the_pipeline_output_fields(self): + manifest = derive_manifest(MODEL_ID, FakePipeline, FAKE_SIGNATURE, [], output_fields={"frames"}) + assert list(manifest["modalities"]["out"]) == ["video"], "a pipeline returning `frames` should output video" + + manifest = derive_manifest(MODEL_ID, FakePipeline, FAKE_SIGNATURE, [], output_fields={"coords_nm"}) + with pytest.raises(SystemExit, match=re.escape("but FakePipeline returns ['coords_nm']")): + Manifest( + json.dumps(manifest).encode(), manifest, FakePipeline, FAKE_SIGNATURE, [], output_fields={"coords_nm"} + ) + + def test_derive_manifest_keeps_the_scalar_members_of_unions(self): + def __call__( + self, + prompt: str | list[str], + strength: float | torch.Tensor = 0.5, + control_mode: int | list[int] | list[list[int]] | None = None, + boxes: list[list[float]] | None = None, + latents: torch.Tensor | None = None, + ): + pass + + manifest = derive_manifest(MODEL_ID, FakePipeline, inspect.signature(__call__), []) + assert manifest["tasks"]["text_to_image"]["parameters"] == { + "strength": {"type": "number", "default": 0.5}, + "control_mode": {"type": "integer"}, + }, "a union should keep its scalar members, and nested lists and tensors should be left out" + + def test_derive_manifest_marks_required_modular_inputs(self): + class RequiredImageBlock(FakeImageBlock): + @property + def inputs(self): + return [InputParam("prompt", required=True), InputParam("image", required=True)] + + pipeline = RequiredImageBlock().init_pipeline() + manifest = derive_manifest(MODEL_ID, type(pipeline), blocks_signature(pipeline.blocks), []) + assert list(manifest["tasks"]) == ["image_to_image"], ( + "a required modular `image` should not yield text_to_image" + ) + + @pytest.mark.parametrize( + "body, code, pointer", + [ + ({"parameters": {"cfg": 3}}, "invalid_request", "/parameters/cfg"), + ({"parameters": {"steps": 500}}, "invalid_request", "/parameters/steps"), + ({"parameters": {"steps": "8"}}, "invalid_request", "/parameters/steps"), + ({"inputs": {}}, "invalid_request", "/inputs/prompt"), + ({"task": "image_to_video"}, "invalid_request", "/task"), + # The lattice pins duration to 24 fps, so another fps would deliver a different length. + ({"parameters": {"duration": 4, "fps": 12}}, "invalid_request", "/parameters/fps"), + # `negative_prompt` is only honored while guidance_scale > 1. + ( + {"inputs": {"prompt": "a cat", "negative_prompt": "blurry"}, "preset": "distilled"}, + "invalid_request", + "/inputs/negative_prompt", + ), + ], + ) + def test_resolve_rejects_invalid_requests(self, body, code, pointer): + bound = _manifest(manifest=LATTICE_MANIFEST, pipeline_cls=FakeVideoPipeline) + with pytest.raises(GMSError) as error: + bound.resolve({"model": MODEL_ID, "inputs": {"prompt": "a cat"}, **body}) + assert (error.value.code, error.value.pointer) == (code, pointer), error.value.message + + def test_resolve_rejects_non_http_media_urls(self): + bound = _manifest() + body = {"model": MODEL_ID, "inputs": {"prompt": "a cat", "image": {"type": "image", "url": "/etc/passwd"}}} + with pytest.raises(GMSError) as error: + bound.resolve(body) + assert error.value.pointer == "/inputs/image/url", error.value.message + + @pytest.mark.parametrize( + "path", + ["hf:/etc/passwd", "hf:../loras/x", "hf:../outputs", "hf:detail", "/etc/passwd", "https://x/y.safetensors"], + ) + def test_resolve_rejects_adapter_paths_that_are_not_repo_ids(self, path): + body = {"model": MODEL_ID, "inputs": {"prompt": "a cat"}, "adapters": [{"type": "lora", "path": path}]} + with pytest.raises(GMSError) as error: + _manifest().resolve(body) + assert (error.value.code, error.value.pointer) == ("invalid_request", "/adapters/0/path"), error.value.message + + def test_resolve_rejects_adapter_repo_ids_that_name_a_local_path(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + (tmp_path / "someone" / "detail").mkdir(parents=True) + body = { + "model": MODEL_ID, + "inputs": {"prompt": "a cat"}, + "adapters": [{"type": "lora", "path": "hf:someone/detail"}], + } + with pytest.raises(GMSError, match="adapter path"): + _manifest().resolve(body) + + def test_resolve_maps_request_to_pipeline_kwargs(self): + bound = _manifest(manifest=LATTICE_MANIFEST, pipeline_cls=FakeVideoPipeline) + resolved = bound.resolve( + { + "model": MODEL_ID, + "preset": "distilled", + "inputs": {"prompt": "a cat"}, + "parameters": {"duration": 8, "seed": 7}, + } + ) + # 8 s at 24 fps is 192 frames; the nearest point on the 8k+1 lattice is 193. The preset overrides the + # task defaults. + assert resolved.kwargs == { + "prompt": "a cat", + "num_frames": 193, + "frame_rate": 24, + "width": 768, + "num_inference_steps": 8, + "guidance_scale": 1.0, + }, resolved.kwargs + assert resolved.seed == 7 + assert resolved.task == "text_to_video" + + @pytest.mark.parametrize( + "change, args, message", + [ + # flux2-klein's published manifest declares `guidance_type`, which its pipeline does not take. + ( + {"parameters": {"guidance_type": {"type": "string", "enum": ["distilled"]}}}, + {}, + "declares parameter 'guidance_type' on task 'text_to_video', which FakeVideoPipeline does not take", + ), + ({"capabilities": ["adapters"]}, {}, "request adapters are off"), + ({"capabilities": ["stream"]}, {}, "declares the 'stream' capability, which this server does not provide"), + ({"parameters": {"duration": {"type": "number"}}}, {}, "without a frame lattice"), + ], + ) + def test_manifest_the_server_cannot_honor_fails_startup(self, change, args, message): + manifest = json.loads(json.dumps(LATTICE_MANIFEST)) + if "parameters" in change: + manifest["tasks"]["text_to_video"]["parameters"].update(change["parameters"]) + else: + manifest.update(change) + with pytest.raises(SystemExit, match=re.escape(message)): + _manifest(manifest=manifest, pipeline_cls=FakeVideoPipeline, args=_args(**args)) + + +class TestServeApp: + def test_generation_lifecycle(self, serve): + client, pipeline = serve() + body = { + "model": MODEL_ID, + "inputs": {"prompt": "a cat"}, + "parameters": {"width": 64, "height": 48, "steps": 3, "seed": 11}, + } + response = client.post("/v1/generations", json=body, headers=WAIT) + assert response.status_code == 201, response.text + generation = response.json() + assert generation["status"] == "complete", generation + assert generation["seed"] == 11 + assert generation["revision"] == "abc123" + assert generation["progress"] is None + assert generation["usage"]["compute_seconds"] >= 0 + + call = pipeline.calls[0] + assert (call["prompt"], call["width"], call["height"], call["num_inference_steps"]) == ("a cat", 64, 48, 3) + assert torch.equal(call["generator"].get_state(), torch.Generator().manual_seed(11).get_state()) + + (artifact,) = generation["outputs"]["image"] + assert {k: artifact[k] for k in ("media_type", "width", "height")} == { + "media_type": "image/png", + "width": 64, + "height": 48, + } + image = Image.open(io.BytesIO(client.get(artifact["url"]).content)) + assert (image.format, image.size) == ("PNG", (64, 48)) + + events = client.get(f"/v1/generations/{generation['id']}/events").text + assert events.count("event: progress") == 3, events + assert 'data: {"step": 3, "total_steps": 3}' in events + assert events.rstrip().split("\n\n")[-1].startswith("id: 4\nevent: complete\n") + # Resuming after the last progress event replays only what followed it. + resumed = client.get(f"/v1/generations/{generation['id']}/events", headers={"Last-Event-ID": "3"}).text + assert "event: progress" not in resumed and "event: complete" in resumed, resumed + + assert client.delete(f"/v1/generations/{generation['id']}").status_code == 204 + missing = client.get(f"/v1/generations/{generation['id']}") + assert (missing.status_code, missing.json()["error"]["code"]) == (404, "not_found") + assert client.get(artifact["url"]).status_code == 404 + + def test_inline_response_and_media_input(self, serve): + client, pipeline = serve() + buffer = io.BytesIO() + Image.new("RGB", (8, 8), "blue").save(buffer, format="PNG") + body = { + "model": MODEL_ID, + "inputs": { + "prompt": "a cat", + "image": { + "type": "image", + "base64": base64.b64encode(buffer.getvalue()).decode(), + "media_type": "image/png", + }, + }, + "output_format": "image/jpeg", + "response_format": "b64_json", + } + generation = client.post("/v1/generations", json=body, headers=WAIT).json() + assert generation["status"] == "complete", generation + # The task is inferred from the inputs: only `image_to_image` accepts an image. + assert pipeline.calls[0]["image"].size == (8, 8) + (artifact,) = generation["outputs"]["image"] + assert artifact["media_type"] == "image/jpeg" and "url" not in artifact + assert Image.open(io.BytesIO(base64.b64decode(artifact["base64"]))).format == "JPEG" + + def test_media_urls_are_capped_and_not_redirected(self, monkeypatch): + def respond(request): + if request.url.path == "/redirect": + return httpx.Response(302, headers={"location": "http://169.254.169.254/"}) + return httpx.Response(200, content=b"x" * 16) + + client = httpx.Client(transport=httpx.MockTransport(respond)) + monkeypatch.setattr("diffusers.commands.serve.backends.httpx.stream", client.stream) + monkeypatch.setattr("diffusers.commands.serve.backends.MAX_MEDIA_DOWNLOAD_BYTES", 8) + public = [(None, None, None, "", ("93.184.215.14", 443))] + monkeypatch.setattr("diffusers.commands.serve.backends.socket.getaddrinfo", lambda host, port: public) + for path, message in (("/redirect", "Redirect response"), ("/large", "larger than")): + with pytest.raises(GMSError, match=message) as error: + _load_media("image", {"type": "image", "url": f"https://example.com{path}"}, "/inputs/image", None) + assert error.value.pointer == "/inputs/image", f"{path} should be rejected as a bad input" + + @pytest.mark.parametrize( + "address, allowed_domains, message", + [ + ("169.254.169.254", None, "non-public address"), + ("127.0.0.1", None, "non-public address"), + ("93.184.215.14", ["upload.wikimedia.org"], "not in --allowed-media-domains"), + ], + ) + def test_media_urls_are_limited_to_public_or_allowed_hosts(self, monkeypatch, address, allowed_domains, message): + resolved = [(None, None, None, "", (address, 80))] + monkeypatch.setattr("diffusers.commands.serve.backends.socket.getaddrinfo", lambda host, port: resolved) + value = {"type": "image", "url": "http://media.example/cat.png"} + with pytest.raises(GMSError, match=message): + _load_media("image", value, "/inputs/image", allowed_domains) + + def test_allowed_media_domains_may_be_private(self, monkeypatch): + buffer = io.BytesIO() + Image.new("RGB", (8, 8), "blue").save(buffer, format="PNG") + client = httpx.Client( + transport=httpx.MockTransport(lambda request: httpx.Response(200, content=buffer.getvalue())) + ) + monkeypatch.setattr("diffusers.commands.serve.backends.httpx.stream", client.stream) + value = {"type": "image", "url": "http://localhost:9000/cat.png"} + image = _load_media("image", value, "/inputs/image", ["localhost"]) + assert image.size == (8, 8), "a host on --allowed-media-domains should be fetched even when it is private" + + def test_joint_video_and_audio_outputs(self, serve, monkeypatch): + monkeypatch.setattr( + "diffusers.commands.serve.backends.export_to_video", + lambda frames, path, fps: Path(path).write_bytes(b"mp4"), + ) + manifest = { + "id": MODEL_ID, + "tasks": { + "text_to_video": { + "inputs": {"prompt": {"type": "text", "required": True}}, + "outputs": {"video": {"type": "video"}, "audio": {"type": "audio"}}, + } + }, + "capabilities": ["wait"], + } + client, pipeline = serve(manifest=manifest, pipeline=FakeAudioVideoPipeline()) + pipeline.result = SimpleNamespace( + frames=[[Image.new("RGB", (32, 24)) for _ in range(16)]], + audio=np.zeros((1, 2, 8000), dtype=np.float32), + ) + body = {"model": MODEL_ID, "inputs": {"prompt": "a cat"}} + generation = client.post("/v1/generations", json=body, headers=WAIT).json() + assert generation["status"] == "complete", generation + + # Video is encoded at the pipeline's default `frame_rate`, audio at its vocoder's sample rate. + (video,) = generation["outputs"]["video"] + (audio,) = generation["outputs"]["audio"] + assert {k: v for k, v in video.items() if k != "url"} == { + "media_type": "video/mp4", + "width": 32, + "height": 24, + "duration": 0.67, + "fps": 24.0, + } + assert {k: v for k, v in audio.items() if k != "url"} == { + "media_type": "audio/wav", + "duration": 0.33, + "sample_rate": 24000, + } + assert client.get(audio["url"]).content[:4] == b"RIFF" + + def test_failed_generation_returns_error_envelope(self, serve): + client, pipeline = serve() + body = {"model": MODEL_ID, "inputs": {"prompt": "a cat"}} + envelope = {"error": {"code": "internal", "message": "RuntimeError: CUDA out of memory"}} + + pipeline.error = RuntimeError("CUDA out of memory") + created = client.post("/v1/generations", json=body) + assert created.status_code == 201 + # The event stream closes once the generation is done, so reading it to the end waits for the failure. + events = client.get(f"/v1/generations/{created.json()['id']}/events").text + assert 'event: error\ndata: {"code": "internal", "message": "RuntimeError: CUDA out of memory"}' in events + response = client.get(f"/v1/generations/{created.json()['id']}") + assert (response.status_code, response.json()) == (500, envelope) + # There is no `failed` status: a create that waited for the failure returns the envelope as well. + waited = client.post("/v1/generations", json=body, headers=WAIT) + assert (waited.status_code, waited.json()) == (500, envelope) + + # A SystemExit from the backend must not end the worker: the next generation still runs. + pipeline.error = SystemExit("unsupported") + assert client.post("/v1/generations", json=body, headers=WAIT).status_code == 500 + pipeline.error = None + assert client.post("/v1/generations", json=body, headers=WAIT).json()["status"] == "complete" + + def test_undeclared_capabilities_are_off(self, serve): + manifest = json.loads(json.dumps(LATTICE_MANIFEST)) + manifest["tasks"]["text_to_video"] = { + "inputs": {"prompt": {"type": "text", "required": True}}, + "outputs": {"image": {"type": "image"}}, + } + manifest["capabilities"] = [] + client, pipeline = serve(manifest=manifest) + # The pipeline is held, so a response that waited for the generation could never return. + pipeline.release.clear() + created = client.post("/v1/generations", json={"model": MODEL_ID, "inputs": {"prompt": "a cat"}}, headers=WAIT) + assert created.json()["status"] in ("queued", "generating"), "without `wait`, Prefer: wait is ignored" + events = client.get(f"/v1/generations/{created.json()['id']}/events") + assert (events.status_code, events.json()["error"]["code"]) == (501, "not_implemented") + pipeline.release.set() + + def test_health_checks_do_not_count_as_activity(self, serve): + client, _ = serve() + client.app.state.last_request = 0.0 + client.get("/health") + assert client.app.state.last_request == 0.0, "a health check must not keep an idle server alive" + client.get("/v1/models") + assert client.app.state.last_request > 0.0, "a request should reset the idle clock" + + def test_public_url_replaces_the_request_address_in_output_links(self, serve): + client, _ = serve(app_options={"public_url": "https://proxy.example/sandbox/proxy/8000/"}) + body = {"model": MODEL_ID, "inputs": {"prompt": "a cat"}} + generation = client.post("/v1/generations", json=body, headers=WAIT).json() + expected = f"https://proxy.example/sandbox/proxy/8000/v1/generations/{generation['id']}/files/image-0.png" + assert generation["outputs"]["image"][0]["url"] == expected + + # A reverse proxy may forward requests with the published path still attached. + for path in ("/v1/models", "/sandbox/proxy/8000/v1/models"): + assert client.get(path).status_code == 200, path + prefixed = client.post("/sandbox/proxy/8000/v1/generations", json=body, headers=WAIT) + assert prefixed.status_code == 201, prefixed.text + + def test_cancel_queued_and_running(self, serve): + client, pipeline = serve() + pipeline.release.clear() + body = {"model": MODEL_ID, "inputs": {"prompt": "a cat"}} + running = client.post("/v1/generations", json=body).json() + queued = client.post("/v1/generations", json=body).json() + assert queued["status"] == "queued", queued + assert pipeline.started.wait(timeout=30), "the worker never picked up the first generation" + + assert client.post(f"/v1/generations/{queued['id']}/cancel").json()["status"] == "canceled" + client.post(f"/v1/generations/{running['id']}/cancel") + pipeline.release.set() + + # The stream ends once the generation is done; a canceled generation emits no `complete` event. + events = client.get(f"/v1/generations/{running['id']}/events").text + assert "event: complete" not in events, events + canceled = client.get(f"/v1/generations/{running['id']}").json() + assert (canceled["status"], canceled["outputs"]) == ("canceled", None) + assert len(pipeline.calls) == 1, "the canceled queued generation must never reach the pipeline" + + def test_request_adapters_are_loaded_then_unloaded(self, serve): + client, pipeline = serve() + body = { + "model": MODEL_ID, + "inputs": {"prompt": "a cat"}, + "adapters": [ + {"type": "lora", "path": "hf:someone/style", "scale": 0.8}, + {"type": "lora", "path": "hf:someone/detail", "scale": 0.5, "targets": {"text_encoder": 0.1}}, + ], + } + generation = client.post("/v1/generations", json=body, headers=WAIT).json() + assert generation["status"] == "complete", generation + assert pipeline.lora_calls == [ + ("load", "someone/style", "request_0"), + ("load", "someone/detail", "request_1"), + ("set", ["request_0", "request_1"], [0.8, {"transformer": 0.5, "text_encoder": 0.1}]), + ("unload",), + ] + + def test_discovery_serves_manifest_bytes_verbatim(self, serve): + manifest = derive_manifest(MODEL_ID, FakePipeline, FAKE_SIGNATURE, ["sse", "wait", "adapters"]) + raw = json.dumps(manifest, indent=3).encode() + client, _ = serve(manifest=raw) + + response = client.get(f"/v1/models/{MODEL_ID}") + assert raw in response.content + discovery = response.json() + assert discovery["spec"] == "gms/0.1" + assert discovery["manifest"] == manifest + assert "server" not in discovery, "the manifest describes the server, so nothing is added to it" + assert client.get("/v1/models").json() == { + "data": [ + { + "id": MODEL_ID, + "modalities": {"in": ["image"], "out": ["image"]}, + "tasks": ["text_to_image", "image_to_image"], + } + ] + } + + +class TestModularPipeline: + def test_generation_runs_a_modular_pipeline(self, serve): + pipeline = FakeImageBlock().init_pipeline() + signature = blocks_signature(pipeline.blocks) + manifest = derive_manifest(MODEL_ID, type(pipeline), signature, ["wait"]) + assert list(manifest["tasks"]) == ["text_to_image", "image_to_image"], manifest["tasks"].keys() + assert manifest["tasks"]["text_to_image"]["inputs"] == {"prompt": {"type": "text", "required": True}} + assert manifest["tasks"]["text_to_image"]["parameters"] == { + "steps": {"type": "integer", "default": 4}, + "seed": {"type": "integer"}, + } + + client, _ = serve(manifest=manifest, pipeline=pipeline, signature=signature) + + body = {"model": MODEL_ID, "inputs": {"prompt": "a cat"}, "parameters": {"seed": 3}} + generation = client.post("/v1/generations", json=body, headers=WAIT).json() + assert generation["status"] == "complete", generation + assert generation["seed"] == 3 + (artifact,) = generation["outputs"]["image"] + assert (artifact["media_type"], artifact["width"], artifact["height"]) == ("image/png", 32, 32) + + def test_modular_pipeline_reports_steps_and_stops_on_cancel(self, monkeypatch, tmp_path): + pipeline = FakeDenoiseLoop().init_pipeline() + monkeypatch.setattr("diffusers.commands.serve.backends._load_pipeline", lambda args: pipeline) + manifest = _manifest(pipeline_cls=type(pipeline), signature=blocks_signature(pipeline.blocks)) + backend = DiffusersBackend(_args(), manifest) + + def generate(cancel): + generation = GenerationJob(manifest.resolve({"model": MODEL_ID, "inputs": {"prompt": "a cat"}})) + generation.directory = tmp_path + generation.canceled = cancel + return backend.generate(generation), [event["step"] for _, event in generation.events] + + outputs, steps = generate(cancel=False) + assert steps == [1, 2, 3], "every denoising step of a modular pipeline should be reported" + assert outputs["image"], outputs + assert generate(cancel=True) == ({}, [1]), "a canceled modular pipeline should stop at its next step" + assert generate(cancel=False)[1] == [1, 2, 3], "the pipeline should stay usable after a cancel" + + +class FakeJobsApi: + def __init__(self, logs=("server started",)): + self.logs = logs + self.run_kwargs = None + self.cancelled = False + self.telemetry = [] + + def run_job(self, **kwargs): + self.run_kwargs = kwargs + return SimpleNamespace(id="job-1", owner=SimpleNamespace(name="me")) + + def fetch_job_logs(self, **kwargs): + for line in self.logs: + if isinstance(line, BaseException): + raise line + yield line + + def inspect_job(self, **kwargs): + return SimpleNamespace(status=SimpleNamespace(stage="COMPLETED", expose_urls=None)) + + def cancel_job(self, **kwargs): + self.cancelled = True + + +class TestServeRemote: + def _run(self, monkeypatch, api, argv): + monkeypatch.setattr("diffusers.commands.serve.remote.HfApi", lambda token: api) + monkeypatch.setattr("diffusers.commands.serve.remote.get_token", lambda: "hf_x") + monkeypatch.setattr( + "diffusers.commands.serve.remote.send_telemetry", lambda **kwargs: api.telemetry.append(kwargs["topic"]) + ) + parser = ArgumentParser() + ServeCommand.register_subcommand(parser.add_subparsers()) + ServeCommand(parser.parse_args(["serve", "org/model", "--remote", *argv])).run() + + def test_remote_starts_the_server_in_a_job(self, monkeypatch): + api = FakeJobsApi() + argv = ["--dtype", "fp16", "--flavor", "a10g-large", "--port", "9000"] + self._run(monkeypatch, api, [*argv, "--dependencies", "extra-package"]) + + job = api.run_kwargs + assert (job["flavor"], job["image"]) == ("a10g-large", _DEFAULT_REMOTE_IMAGE), job + assert job["expose"] == [8000] and job["timeout"] == "24h", job + assert job["secrets"] == {"HF_TOKEN": "hf_x"}, "gated models need the HF token inside the Job" + + assert job["command"][:2] == ["sh", "-c"], job["command"] + install, start = job["command"][2].split(" && ") + assert "fastapi" in install and "extra-package" in install, install + start = shlex.split(start) + # The server runs under the interpreter the dependencies were installed into, not whatever is on PATH. + assert start[:4] == ["exec", "$(uv python find --system)", "-m", "diffusers.commands.diffusers_cli"], start + serve_argv = start[start.index("serve") + 1 :] + options = dict(zip(serve_argv[::2], serve_argv[1::2])) + assert options["--model"] == "org/model" and options["--dtype"] == "fp16", options + # The server binds the exposed port on every interface, and builds output links on the public address. + assert (options["--host"], options["--port"]) == ("0.0.0.0", "8000"), options + assert options["--public-url"] == "https://${JOB_ID}--8000.hf.jobs", options + assert options["--idle-timeout"] == "30m", "a remote server should shut itself down when idle by default" + assert not {"--remote", "--flavor", "--dependencies"} & set(serve_argv), serve_argv + assert not api.cancelled, "a Job that completed on its own needs no cancel" + assert api.telemetry == ["diffusers/cli/serve/remote"], api.telemetry + + def test_interrupt_cancels_the_job(self, monkeypatch): + api = FakeJobsApi(logs=("server started", KeyboardInterrupt())) + with pytest.raises(KeyboardInterrupt): + self._run(monkeypatch, api, []) + assert api.cancelled, "Ctrl-C must cancel the Job so it stops billing" + + +class BlockingBackend: + def __init__(self): + self.started = threading.Event() + self.release = threading.Event() + + def generate(self, generation): + self.started.set() + self.release.wait(timeout=30) + return {} + + +class TestScheduler: + def test_each_backend_runs_its_own_generation(self, tmp_path): + backends = [BlockingBackend(), BlockingBackend()] + scheduler = Scheduler(backends, str(tmp_path)) + request = _manifest().resolve({"model": MODEL_ID, "inputs": {"prompt": "a cat"}}) + jobs = [GenerationJob(request) for _ in backends] + for job in jobs: + scheduler.submit(job) + + for index, backend in enumerate(backends): + assert backend.started.wait(timeout=30), f"backend {index} never picked up a generation" + assert [job.status for job in jobs] == ["generating", "generating"], "both generations should run at once" + + for backend in backends: + backend.release.set() + for job in jobs: + asyncio.run(job.wait(timeout=30)) + assert [job.status for job in jobs] == ["complete", "complete"] + assert not scheduler.busy(), "the scheduler should be idle once every backend has finished" + + def test_generation_past_the_timeout_fails(self, tmp_path): + class SteppingBackend: + """Stops at the next step once interrupted, like a pipeline's step callback.""" + + def generate(self, generation): + while not generation.cancel_requested: + time.sleep(0.01) + return {} + + scheduler = Scheduler([SteppingBackend()], str(tmp_path), timeout=0.1) + job = GenerationJob(_manifest().resolve({"model": MODEL_ID, "inputs": {"prompt": "a cat"}})) + scheduler.submit(job) + asyncio.run(job.wait(timeout=30)) + assert job.status == "failed", job.status + assert "--generation-timeout" in job.error.message, job.error.message + + def test_generation_stuck_past_the_timeout_fails_health(self, tmp_path): + backend = BlockingBackend() + scheduler = Scheduler([backend], str(tmp_path), timeout=60) + job = GenerationJob(_manifest().resolve({"model": MODEL_ID, "inputs": {"prompt": "a cat"}})) + scheduler.submit(job) + assert backend.started.wait(timeout=30), "the worker never picked up the generation" + assert scheduler.failure() is None, "a generation within its time limit is healthy" + + # As if it had run for over twice the limit without reaching a step that could stop it. + job.deadline -= 121 + assert "did not stop when interrupted" in (scheduler.failure() or ""), scheduler.failure() + backend.release.set() + + +class FakeClock: + """Stands in for the `time` module: `sleep` advances the clock instead of waiting.""" + + def __init__(self): + self.now = 0.0 + + def monotonic(self): + return self.now + + def sleep(self, seconds): + if self.now > 1000: + raise AssertionError("the idle watcher never shut the server down") + self.now += seconds + + +class TestIdleShutdown: + def _watch(self, monkeypatch, busy_checks): + """Run the watcher with a 1m timeout, checked every 30s, until it shuts the server down; returns when it did.""" + clock = FakeClock() + monkeypatch.setattr("diffusers.commands.serve.time", clock) + busy = iter(busy_checks) + server = SimpleNamespace(should_exit=False) + app = SimpleNamespace(state=SimpleNamespace(last_request=0.0)) + _shut_down_when_idle(server, app, SimpleNamespace(busy=lambda: next(busy, False)), "1m") + assert server.should_exit, "the watcher should only return once it told the server to exit" + return clock.now + + def test_idle_server_shuts_down(self, monkeypatch): + assert self._watch(monkeypatch, []) == 60, "an idle server should exit after one timeout" + + def test_busy_server_stays_up_until_the_work_ends(self, monkeypatch): + # Busy at the checks at 30s and 60s, so the idle clock restarts at 60s. + assert self._watch(monkeypatch, [True, True]) == 120, "the server should exit one timeout after the work ends" + + +class TestServeCommand: + def _parse(self, argv): + parser = ArgumentParser() + ServeCommand.register_subcommand(parser.add_subparsers()) + return parser.parse_args(["serve", *argv]) + + def test_model_is_positional_and_loads_in_bf16(self, monkeypatch): + pytest.importorskip("fastapi", reason="`diffusers-cli serve` needs the `diffusers[serve]` extra") + resolved = {} + + def stop(args): + resolved.update(vars(args)) + raise RuntimeError("stop after the flag check") + + monkeypatch.setattr("diffusers.commands.serve._resolve_pipeline", stop) + with pytest.raises(RuntimeError, match="stop after the flag check"): + ServeCommand(self._parse(["org/model"])).run() + assert resolved["model"] == "org/model", resolved + assert resolved["dtype"] == "bf16", "serve should default to bf16 rather than `run`'s fp32 `auto`"