From 260ae4a1c2116bb2df0fef04ea8f1e254c99cbd9 Mon Sep 17 00:00:00 2001 From: DN6 Date: Fri, 9 Oct 2026 11:02:21 +0530 Subject: [PATCH 01/21] add diffusers serve and generate commands --- .ai/skills/diffusers-cli/SKILL.md | 4 + docs/source/en/using-diffusers/cli.md | 144 ++++ setup.py | 3 + src/diffusers/commands/diffusers_cli.py | 4 + src/diffusers/commands/generate.py | 238 ++++++ src/diffusers/commands/serve/__init__.py | 281 +++++++ src/diffusers/commands/serve/app.py | 218 ++++++ src/diffusers/commands/serve/backends.py | 472 +++++++++++ src/diffusers/commands/serve/generations.py | 194 +++++ src/diffusers/commands/serve/manifest.py | 779 ++++++++++++++++++ src/diffusers/dependency_versions_table.py | 2 + tests/others/test_cli_commands.py | 2 +- tests/others/test_cli_serve.py | 826 ++++++++++++++++++++ 13 files changed, 3166 insertions(+), 1 deletion(-) create mode 100644 src/diffusers/commands/generate.py create mode 100644 src/diffusers/commands/serve/__init__.py create mode 100644 src/diffusers/commands/serve/app.py create mode 100644 src/diffusers/commands/serve/backends.py create mode 100644 src/diffusers/commands/serve/generations.py create mode 100644 src/diffusers/commands/serve/manifest.py create mode 100644 tests/others/test_cli_serve.py diff --git a/.ai/skills/diffusers-cli/SKILL.md b/.ai/skills/diffusers-cli/SKILL.md index dd9ffd66dabd..9baf0540855a 100644 --- a/.ai/skills/diffusers-cli/SKILL.md +++ b/.ai/skills/diffusers-cli/SKILL.md @@ -16,6 +16,8 @@ description: > | --------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `run` | Run any `DiffusionPipeline` or `ModularPipeline`. Forwards `--pipeline-kwargs` verbatim, saves output by detecting its runtime type, optionally runs on HF Jobs via `--remote`. | | `schema` | Print the input schema for a pipeline repo (kwarg names, types, defaults, descriptions). **No weights downloaded** — only the small index file. | +| `serve` | Serve a pipeline over HTTP with the Generative Media Spec API: model discovery, queued generations, polling or SSE progress. In-process, or behind SGLang / vLLM-Omni via `--backend`. | +| `generate` | Send a request to a Generative Media Spec server (such as `serve`) and save the outputs. Loads no pipeline; takes `--url`, `--inputs` and `--parameters` with the server's declared names. | | `custom_blocks` | Package a local `ModularPipelineBlocks` subclass for the Hub. | | `env` | Print versions of diffusers + torch + transformers + accelerate + safetensors + CUDA + GPU info. Use when investigating environment issues, dtype/precision support, or building bug reports. | @@ -33,6 +35,8 @@ The other commands are small enough that `diffusers-cli --help` is the ```bash diffusers-cli schema --help +diffusers-cli serve --help +diffusers-cli generate --help diffusers-cli custom_blocks --help diffusers-cli env --help ``` diff --git a/docs/source/en/using-diffusers/cli.md b/docs/source/en/using-diffusers/cli.md index 02be2bd2fd0e..d7e8d6523b38 100644 --- a/docs/source/en/using-diffusers/cli.md +++ b/docs/source/en/using-diffusers/cli.md @@ -21,6 +21,8 @@ specific language governing permissions and limitations under the License. | [`env`](#env) | Print environment info for bug reports. | | [`schema`](#schema) | Inspect a pipeline's `__call__` signature without downloading weights. | | [`run`](#run) | Run a pipeline locally or in a Hugging Face Sandbox. | +| [`serve`](#serve) | Serve a pipeline over HTTP with the Generative Media Spec API. | +| [`generate`](#generate) | Send a generation to a Generative Media Spec server and save its outputs. | | [`custom_blocks`](#customblocks) | Package a local `ModularPipelineBlocks` subclass for the Hub. | | [`fp16_safetensors`](#fp16safetensors) | Convert a checkpoint to fp16 `.safetensors`. | | [`skills`](#skills) | Install pre-authored skill bundles into your AI coding agent. | @@ -287,6 +289,148 @@ diffusers-cli run -m black-forest-labs/FLUX.1-dev --dtype bf16 \ hf sandbox kill ``` +## `serve` + +Serve a pipeline over HTTP. The server implements the Generative Media Spec (GMS, draft `gms/0.1`): clients +discover what the model accepts, create a generation, follow it while it runs, and read the results. It needs +the `serve` extra: + +```bash +pip install "diffusers[serve]" +diffusers-cli serve --model black-forest-labs/FLUX.1-dev --dtype bf16 +``` + +`serve` takes the same loading and optimization flags as [`run`](#run) (`--dtype`, `--device-map`, +`--cpu-offload`, `--compile`, `--lora`, ...), plus `--host` (default `127.0.0.1`) and `--port` (default `8000`). +The server has no authentication, so keep it on a trusted network. + +[Modular repos](../modular_diffusers/overview) are detected the same way as in `run` and served in-process. They +report no step progress, and a cancel request only takes effect once the pipeline call returns. + +| Flag | Effect | +|---|---| +| `--default-seed` | Seed used for requests that do not set one. Without it, each such request gets a random seed. | +| `--enable-cors` | Allow cross-origin requests from any origin, so a browser app on another host can call the server. | +| `--log-level` | Logging level for diffusers and the HTTP server (`debug`, `info`, `warning`, `error`, `critical`). | + +`GET /health` returns `{"status": "ok"}`, or `503` when an SGLang or vLLM-Omni backend has exited. Every response +carries an `x-request-id` header: the one sent with the request, or a generated one. + +Generations run one at a time. Creating one returns immediately with a `queued` generation; poll it, subscribe +to its events, or ask the server to hold the connection with `Prefer: wait=`: + +```bash +# What the model accepts: tasks, inputs, parameters with types and defaults. +curl localhost:8000/v1/models/black-forest-labs/FLUX.1-dev + +# Create a generation and wait up to 120 seconds for it. +curl localhost:8000/v1/generations -H 'Prefer: wait=120' -d '{ + "model": "black-forest-labs/FLUX.1-dev", + "inputs": {"prompt": "an astronaut riding a horse"}, + "parameters": {"width": 1024, "height": 1024, "steps": 28, "seed": 42} +}' + +# Follow step progress as server-sent events, or poll the generation. +curl -N localhost:8000/v1/generations//events +curl localhost:8000/v1/generations/ + +# Cancel a queued or running generation, or delete it together with its files. +curl -X POST localhost:8000/v1/generations//cancel +curl -X DELETE localhost:8000/v1/generations/ +``` + +A completed generation lists its outputs with a download URL each (or inline base64 with +`"response_format": "b64_json"`), the seed that was used, and the model revision. Outputs are kept under +`--output-dir` and expire after one hour. Images are PNG by default (`"output_format": "image/jpeg"` or +`"image/webp"` to change it), videos MP4, audio WAV. Media inputs are passed as +`{"type": "image", "url": "https://..."}` or `{"type": "image", "base64": "...", "media_type": "image/png"}`. + +Validation is strict: an unknown parameter, a value outside its declared range, or a missing input is rejected +with a JSON pointer to the offending field. + +```json +{"error": {"code": "invalid_request", "message": "task 'text_to_image' declares no parameter 'cfg'", "pointer": "/parameters/cfg"}} +``` + +### Manifests + +What a model accepts is described by its manifest. `serve` looks for one in this order: + +1. the file passed with `--manifest `; +2. a `gms.json` published in the model repo, served byte-for-byte; +3. a manifest derived from the pipeline's `__call__` signature. + +A derived manifest is a best-effort reading of the signature: scalar arguments become parameters, prompts and +media arguments become inputs, and arguments with a core GMS name are renamed (`num_inference_steps` becomes +`steps`, `num_images_per_prompt` becomes `num_outputs`, `generator` becomes `seed`). It carries no value ranges. +Write a `gms.json` when you need ranges, presets, or a duration in seconds. + +When a manifest uses a name that differs from the pipeline argument, map it per task under `x-diffusers`: + +```json +"x-diffusers": {"inputs": {"caption": "prompt"}, "parameters": {"cfg": "guidance_scale"}} +``` + +A task is only hosted when every input it requires maps to a pipeline argument. A parameter the pipeline cannot +set is rejected with `not_implemented` unless the request sends its default value. + +### Adapters + +A request can apply LoRAs for that generation only. They are loaded before the call and unloaded after it: + +```bash +curl localhost:8000/v1/generations -d '{ + "model": "black-forest-labs/FLUX.1-dev", + "inputs": {"prompt": "a tiny cat"}, + "adapters": [{"type": "lora", "path": "hf:alvdansen/littletinies", "scale": 0.8}] +}' +``` + +Request adapters are turned off when the server starts with `--compile` or `--lora`; such requests fail with +`not_implemented`. + +### SGLang and vLLM-Omni backends + +`--backend sglang` and `--backend vllm` run the model in [SGLang Diffusion](https://docs.sglang.io/) or +[vLLM-Omni](https://docs.vllm.ai/projects/vllm-omni) instead of in-process. `serve` starts `sglang serve` or +`vllm serve --omni` as a child process on a local port, waits for it to become healthy, and translates GMS +requests to the engine's OpenAI-compatible image and video routes. Clients only ever talk to the GMS endpoints +of `serve`. The engine must be installed separately. + +```bash +diffusers-cli serve --model Qwen/Qwen-Image --backend sglang --backend-args '--num-gpus 2' +diffusers-cli serve --model Wan-AI/Wan2.2-T2V-A14B-Diffusers --backend vllm +``` + +The pipeline flags (`--dtype`, `--cpu-offload`, `--compile`, ...) do not apply to these backends; pass the +engine's own flags with `--backend-args`. Compared with the in-process backend, they serve image and video +outputs only, accept `image` as the only media input, expose the parameters both engines share (size, steps, +guidance scale, seed, number of frames, fps, number of images), report no step progress, and apply no request +adapters. + +## `generate` + +Send a generation to a server started with [`serve`](#serve), or to any other Generative Media Spec server, and +save its outputs. Unlike [`run`](#run), `generate` loads no pipeline: it reads the model's discovery document, +submits the request, polls the generation until it finishes, and downloads the files. + +```bash +diffusers-cli generate --url http://localhost:8000 --inputs '{"prompt": "a cat on the moon"}' + +diffusers-cli generate --url http://localhost:8000 \ + --inputs '{"prompt": "make the fur grey", "image": "cat.png"}' \ + --parameters '{"steps": 8, "seed": 0}' --output outputs/ +``` + +`--inputs` and `--parameters` use the names the server declares, not pipeline argument names (`steps`, not +`num_inference_steps`). An image, mask, video or audio input can be an http(s) URL or a local file path; local +files are sent inline as base64. `--task`, `--preset`, `--adapters` and `--output-format` map to the request +fields of the same name. `--model` is only needed when the server hosts more than one model. + +Outputs are saved to `~/.diffusers/cli/generate/outputs//` unless `--output` names a directory. +`--token` sends a bearer token in the `Authorization` header, for servers behind an authenticating proxy; no +token is sent by default. Interrupting the command cancels the generation on the server. + ## `custom_blocks` Package a local `ModularPipelineBlocks` subclass for upload to the Hub. Reads a Python file, AST-scans it for diff --git a/setup.py b/setup.py index 34981886d0f8..94620fb3c722 100644 --- a/setup.py +++ b/setup.py @@ -146,6 +146,8 @@ "opencv-python", "timm", "flashpack", + "fastapi", + "uvicorn", ] # this is a lookup table with items like: @@ -253,6 +255,7 @@ def run(self): extras["nvidia_modelopt"] = deps_list("nvidia_modelopt[hf]") extras["sdnq"] = deps_list("sdnq") extras["flashpack"] = deps_list("flashpack") +extras["serve"] = deps_list("fastapi", "uvicorn") extras["dev"] = extras["quality"] + extras["test"] + extras["training"] + extras["docs"] + extras["torch"] diff --git a/src/diffusers/commands/diffusers_cli.py b/src/diffusers/commands/diffusers_cli.py index 0e4f2c27fb64..1df91bd7ddb4 100644 --- a/src/diffusers/commands/diffusers_cli.py +++ b/src/diffusers/commands/diffusers_cli.py @@ -20,8 +20,10 @@ from .custom_blocks import CustomBlocksCommand from .env import EnvironmentCommand from .fp16_safetensors import FP16SafetensorsCommand +from .generate import GenerateCommand from .run import RunCommand from .schema import SchemaCommand +from .serve import ServeCommand from .skills import SkillsCommand @@ -47,8 +49,10 @@ def main(): EnvironmentCommand.register_subcommand(commands_parser) FP16SafetensorsCommand.register_subcommand(commands_parser) CustomBlocksCommand.register_subcommand(commands_parser) + GenerateCommand.register_subcommand(commands_parser) RunCommand.register_subcommand(commands_parser) SchemaCommand.register_subcommand(commands_parser) + ServeCommand.register_subcommand(commands_parser) SkillsCommand.register_subcommand(commands_parser) # Let's go diff --git a/src/diffusers/commands/generate.py b/src/diffusers/commands/generate.py new file mode 100644 index 000000000000..f3ea85d54e1d --- /dev/null +++ b/src/diffusers/commands/generate.py @@ -0,0 +1,238 @@ +# Copyright 2026 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""`diffusers-cli generate` — send a generation to a Generative Media Spec (GMS) server and save its outputs. + +The client side of `diffusers-cli serve`: it loads no pipeline. It reads the model's discovery document, submits the +request, polls the generation until it finishes, and downloads the files. +""" + +from __future__ import annotations + +import base64 +import json +import mimetypes +import time +from argparse import ArgumentParser, Namespace, RawDescriptionHelpFormatter, _SubParsersAction +from pathlib import Path +from typing import Any + +from huggingface_hub.cli._output import out +from huggingface_hub.utils import httpx + +from . import BaseDiffusersCLICommand + + +DEFAULT_OUTPUT_DIR = str(Path.home() / ".diffusers" / "cli" / "generate" / "outputs") +_MEDIA_TYPES = ("image", "mask", "video", "audio") +_POLL_SECONDS = 1.0 +_TIMEOUT_SECONDS = 60.0 + + +def _parse_json(raw: str | None, flag: str, expected: type) -> Any: + if raw is None: + return None + try: + parsed = json.loads(raw) + except json.JSONDecodeError as e: + raise SystemExit(f"{flag} must be valid JSON: {e}") from e + if not isinstance(parsed, expected): + raise SystemExit(f"{flag} must decode to a JSON {'object' if expected is dict else 'array'}.") + return parsed + + +def _media_value(name: str, media_type: str, value: Any) -> Any: + """Turn a URL or local path into a GMS media value; anything else is sent as given.""" + if isinstance(value, list): + return [_media_value(name, media_type, item) for item in value] + if not isinstance(value, str): + return value + if value.startswith(("http://", "https://")): + return {"type": media_type, "url": value} + + path = Path(value).expanduser() + if not path.is_file(): + raise SystemExit(f"Input {name!r} is neither an http(s) URL nor an existing file: {value!r}") + media = {"type": media_type, "base64": base64.b64encode(path.read_bytes()).decode()} + guessed = mimetypes.guess_type(path.name)[0] + if guessed is not None: + media["media_type"] = guessed + return media + + +class GenerateCommand(BaseDiffusersCLICommand): + task = "generate" + + @staticmethod + def register_subcommand(subparsers: _SubParsersAction) -> None: + epilog = ( + "Examples\n" + ' $ diffusers-cli generate --url http://localhost:8000 --inputs \'{"prompt": "a cat on the moon"}\'\n' + " $ diffusers-cli generate --url http://localhost:8000 \\\n" + ' --inputs \'{"prompt": "make the fur grey", "image": "cat.png"}\' \\\n' + ' --parameters \'{"steps": 8, "seed": 0}\' -o outputs/\n' + "\n" + "Learn more\n" + " Use `diffusers-cli --help` for more information about a command.\n" + " Read the documentation at https://huggingface.co/docs/diffusers\n" + ) + + parser: ArgumentParser = subparsers.add_parser( + "generate", + help="Send a generation to a Generative Media Spec server and save its outputs.", + usage="\n diffusers-cli generate --url --inputs [options]", + epilog=epilog, + formatter_class=RawDescriptionHelpFormatter, + ) + parser._optionals.title = "Options" + parser.add_argument("--url", required=True, help="Base URL of the server, e.g. http://localhost:8000.") + parser.add_argument( + "--model", "-m", default=None, help="Model id to request. Defaults to the server's only model." + ) + parser.add_argument( + "--inputs", + required=True, + metavar="JSON", + help=( + "JSON object of the task's inputs. An image, mask, video or audio input given as an http(s) URL " + "or a local file path is converted to a media value; local files are sent inline as base64." + ), + ) + parser.add_argument("--parameters", default=None, metavar="JSON", help="JSON object of task parameters.") + parser.add_argument( + "--task", default=None, help="Task to run. Defaults to the one the server infers from the inputs." + ) + parser.add_argument("--preset", default=None, help="Named preset of the task to start from.") + parser.add_argument( + "--adapters", + default=None, + metavar="JSON", + help='JSON array of adapters to apply, e.g. \'[{"type": "lora", "path": "hf:org/name", "scale": 0.8}]\'.', + ) + parser.add_argument("--output-format", default=None, help="Media type to encode outputs as, e.g. image/jpeg.") + parser.add_argument( + "--output", + "-o", + default=None, + help=f"Directory to save outputs in. Defaults to {DEFAULT_OUTPUT_DIR}//.", + ) + parser.add_argument( + "--token", + default=None, + help="Bearer token sent to the server in the Authorization header. No token is sent by default.", + ) + parser.set_defaults(func=GenerateCommand) + + def __init__(self, args: Namespace): + self.args = args + + def run(self) -> None: + args = self.args + inputs = _parse_json(args.inputs, "--inputs", dict) + parameters = _parse_json(args.parameters, "--parameters", dict) + adapters = _parse_json(args.adapters, "--adapters", list) + + with self._client() as client: + model = args.model or self._only_model(client) + document = self._request(client, "GET", f"/v1/models/{model}") + tasks = document["manifest"]["tasks"] + if args.task is not None and args.task not in tasks: + raise SystemExit(f"Model {model!r} declares no task {args.task!r}; its tasks are {list(tasks)}.") + + declared = [tasks[args.task]] if args.task is not None else list(tasks.values()) + input_types = {name: spec.get("type") for task in declared for name, spec in task["inputs"].items()} + body: dict[str, Any] = {"model": model, "inputs": {}} + for name, value in inputs.items(): + if input_types.get(name) in _MEDIA_TYPES: + value = _media_value(name, input_types[name], value) + body["inputs"][name] = value + optional = { + "task": args.task, + "preset": args.preset, + "parameters": parameters, + "adapters": adapters, + "output_format": args.output_format, + } + body.update({key: value for key, value in optional.items() if value is not None}) + + generation = self._request(client, "POST", "/v1/generations", json=body) + try: + while generation["status"] in ("queued", "generating"): + time.sleep(_POLL_SECONDS) + generation = self._request(client, "GET", f"/v1/generations/{generation['id']}") + except KeyboardInterrupt: + # Without this the server keeps generating for a client that has gone away. + self._request(client, "POST", f"/v1/generations/{generation['id']}/cancel") + raise + if generation["status"] != "complete": + raise SystemExit(f"Generation {generation['id']} ended with status {generation['status']!r}.") + + directory = Path(args.output or Path(DEFAULT_OUTPUT_DIR) / generation["id"]).expanduser() + directory.mkdir(parents=True, exist_ok=True) + saved = [] + for name, artifacts in generation["outputs"].items(): + for index, artifact in enumerate(artifacts): + path = directory / f"{name}-{index}.{artifact['media_type'].split('/')[1]}" + path.write_bytes(self._artifact_bytes(client, artifact)) + saved.append(str(path)) + + out.result( + self.task, + url=args.url, + model=model, + generation=generation["id"], + seed=generation.get("seed"), + outputs=saved, + ) + + def _client(self) -> httpx.Client: + headers = {"Authorization": f"Bearer {self.args.token}"} if self.args.token else {} + return httpx.Client(base_url=self.args.url, headers=headers, timeout=_TIMEOUT_SECONDS, follow_redirects=True) + + def _request(self, client: httpx.Client, method: str, path: str, **kwargs: Any) -> Any: + try: + response = client.request(method, path, **kwargs) + except httpx.HTTPError as e: + raise SystemExit(f"Could not reach {self.args.url}: {type(e).__name__}: {e}") from e + if response.status_code < 400: + return response.json() + + try: + error = response.json()["error"] + message = f"{error['code']}: {error['message']}" + if error.get("pointer"): + message += f" (at {error['pointer']})" + except (ValueError, KeyError, TypeError): + message = response.text + raise SystemExit(f"{method} {path} failed with HTTP {response.status_code}: {message}") + + def _only_model(self, client: httpx.Client) -> str: + models = [model["id"] for model in self._request(client, "GET", "/v1/models")["data"]] + if len(models) != 1: + raise SystemExit(f"{self.args.url} serves {len(models)} models ({models}); pick one with --model.") + return models[0] + + def _artifact_bytes(self, client: httpx.Client, artifact: dict[str, Any]) -> bytes: + if "base64" in artifact: + return base64.b64decode(artifact["base64"]) + try: + # The token is for the server; an output hosted elsewhere is fetched without it. + if httpx.URL(artifact["url"]).host == client.base_url.host: + response = client.get(artifact["url"]) + else: + response = httpx.get(artifact["url"], timeout=_TIMEOUT_SECONDS, follow_redirects=True) + response.raise_for_status() + except httpx.HTTPError as e: + raise SystemExit(f"Could not download output {artifact['url']}: {type(e).__name__}: {e}") from e + return response.content diff --git a/src/diffusers/commands/serve/__init__.py b/src/diffusers/commands/serve/__init__.py new file mode 100644 index 000000000000..e56c73a5c530 --- /dev/null +++ b/src/diffusers/commands/serve/__init__.py @@ -0,0 +1,281 @@ +# Copyright 2026 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""`diffusers-cli serve` — serve a pipeline over HTTP with the Generative Media Spec (GMS) API. + +Generations run one at a time on a background worker: a request returns immediately with a queued generation that +clients poll or follow over server-sent events. The pipeline runs in-process by default, or behind an SGLang or +vLLM-Omni server with `--backend`. +""" + +from __future__ import annotations + +import inspect +from argparse import ArgumentParser, Namespace, RawDescriptionHelpFormatter, _SubParsersAction +from pathlib import Path + +from huggingface_hub.cli._output import out + +from ...utils import logging +from .. import BaseDiffusersCLICommand +from ..run import _add_loading_arguments, _add_optimization_arguments + + +DEFAULT_OUTPUT_DIR = str(Path.home() / ".diffusers" / "cli" / "serve" / "outputs") +BACKEND_CHOICES = ("diffusers", "sglang", "vllm") + +# Flags that configure the in-process pipeline, paired with the value argparse gives them when unset. +_DIFFUSERS_ONLY_FLAGS = { + "device_map": None, + "dtype": "auto", + "variant": None, + "lora": None, + "cpu_offload": None, + "attention_backend": "default", + "vae_tiling": False, + "vae_slicing": False, + "compile": None, +} + + +def _resolve_pipeline(args: Namespace) -> tuple[type, inspect.Signature, str | None]: + """Return the pipeline class, the signature of its call, and the commit hash of a standard Hub repo.""" + import diffusers + + from .manifest import blocks_signature + + # A modular repo ships `modular_model_index.json` instead of `model_index.json`, so `load_config` raises. + try: + config, commit_hash = diffusers.DiffusionPipeline.load_config( + args.model, token=args.token, revision=args.revision, return_commit_hash=True + ) + except OSError: + config, commit_hash = None, None + + if config is not None: + class_name = config.get("_class_name") + pipeline_cls = getattr(diffusers, str(class_name), None) + if not isinstance(pipeline_cls, type): + raise SystemExit( + f"Pipeline class {class_name!r} declared in {diffusers.DiffusionPipeline.config_name} is not " + "exported by the installed diffusers." + ) + if not issubclass(pipeline_cls, diffusers.ModularPipeline): + return pipeline_cls, inspect.signature(pipeline_cls.__call__), commit_hash + + if args.backend != "diffusers": + raise SystemExit( + f"{args.model!r} is a modular pipeline, which --backend {args.backend} cannot serve. " + "Use --backend diffusers." + ) + try: + pipeline = diffusers.ModularPipeline.from_pretrained( + args.model, trust_remote_code=args.trust_remote_code, token=args.token, revision=args.revision + ) + except OSError as e: + raise SystemExit(f"Could not read a pipeline config for {args.model!r}: {e}") from e + return type(pipeline), blocks_signature(pipeline.blocks), commit_hash + + +class ServeCommand(BaseDiffusersCLICommand): + task = "serve" + + @staticmethod + def register_subcommand(subparsers: _SubParsersAction) -> None: + epilog = ( + "Examples\n" + " $ diffusers-cli serve -m black-forest-labs/FLUX.1-dev --dtype bf16\n" + " $ diffusers-cli serve -m black-forest-labs/FLUX.1-dev --dtype bf16 --compile --port 9000\n" + " $ diffusers-cli serve -m Wan-AI/Wan2.2-T2V-A14B-Diffusers --backend vllm\n" + " $ diffusers-cli serve -m Qwen/Qwen-Image --backend sglang --backend-args '--num-gpus 2'\n" + "\n" + " $ curl localhost:8000/v1/generations -H 'Prefer: wait=120' \\\n" + ' -d \'{"model": "black-forest-labs/FLUX.1-dev", "inputs": {"prompt": "a cat on the moon"}}\'\n' + "\n" + "Learn more\n" + " Use `diffusers-cli --help` for more information about a command.\n" + " Read the documentation at https://huggingface.co/docs/diffusers\n" + ) + + parser: ArgumentParser = subparsers.add_parser( + "serve", + help="Serve a diffusers pipeline over HTTP with the Generative Media Spec API.", + usage="\n diffusers-cli serve [options]", + epilog=epilog, + formatter_class=RawDescriptionHelpFormatter, + ) + parser._optionals.title = "Options" + _add_loading_arguments(parser) + _add_optimization_arguments(parser) + parser.add_argument( + "--backend", + choices=BACKEND_CHOICES, + default="diffusers", + help=( + "What runs the model. 'diffusers' loads the pipeline in this process. 'sglang' and 'vllm' start " + "`sglang serve` / `vllm serve --omni` as a child process and translate requests to its " + "OpenAI-compatible image and video routes." + ), + ) + parser.add_argument( + "--backend-args", + default=None, + metavar="ARGS", + help=( + "Extra arguments appended to the `sglang serve` / `vllm serve` command, as one quoted string " + "(e.g. '--num-gpus 2'). Only used with --backend sglang or vllm." + ), + ) + parser.add_argument("--host", default="127.0.0.1", help="Interface to bind. Defaults to 127.0.0.1.") + parser.add_argument("--port", type=int, default=8000, help="Port to bind. Defaults to 8000.") + parser.add_argument( + "--manifest", + default=None, + metavar="PATH", + help=( + "Path to a GMS manifest (`gms.json`) describing the model. Defaults to the `gms.json` published " + "in the model repo; when there is none, a manifest is derived from the pipeline's call signature." + ), + ) + parser.add_argument( + "--output-dir", + default=DEFAULT_OUTPUT_DIR, + help=f"Directory generated files are stored in until they expire. Defaults to {DEFAULT_OUTPUT_DIR}.", + ) + parser.add_argument( + "--fps", + type=int, + default=8, + help="FPS used to encode video outputs when the pipeline call takes no fps argument.", + ) + parser.add_argument( + "--sampling-rate", + type=int, + default=None, + help="Sample rate used to encode audio outputs. Defaults to 16000.", + ) + parser.add_argument( + "--default-seed", + type=int, + default=None, + help="Seed used for requests that do not set one. Defaults to a random seed per request.", + ) + parser.add_argument( + "--enable-cors", + action="store_true", + help="Allow cross-origin requests from any origin, so browser apps on other hosts can call the server.", + ) + parser.add_argument( + "--log-level", + choices=tuple(logging.get_log_levels_dict()), + default=None, + help="Logging level for diffusers and the HTTP server. Defaults to the current diffusers verbosity.", + ) + # `_load_pipeline` reads these `run` flags, which `serve` does not expose. + parser.set_defaults(func=ServeCommand, workflow=None, offload_margin=None) + + def __init__(self, args: Namespace): + self.args = args + + def run(self) -> None: + args = self.args + try: + import fastapi # noqa: F401 + import uvicorn + except ImportError as e: + raise SystemExit( + '`diffusers-cli serve` requires FastAPI and uvicorn. Install them with: pip install "diffusers[serve]"' + ) from e + + if args.log_level is not None: + logging.set_verbosity(logging.get_log_levels_dict()[args.log_level]) + + if args.context_parallel: + raise SystemExit( + "--context-parallel is not supported by `diffusers-cli serve`: under torchrun every rank would " + "start its own server on the same port." + ) + if args.backend == "diffusers" and args.backend_args is not None: + raise SystemExit("--backend-args only applies to --backend sglang or vllm.") + if args.backend != "diffusers": + rejected = [ + "--" + name.replace("_", "-") + for name, unset in _DIFFUSERS_ONLY_FLAGS.items() + if getattr(args, name) != unset + ] + if rejected: + raise SystemExit( + f"{', '.join(rejected)} configure the in-process pipeline and have no effect with " + f"--backend {args.backend}. Pass the engine's own flags through --backend-args instead." + ) + + from .app import create_app + from .backends import ( + DiffusersBackend, + EngineBackend, + backend_capabilities, + backend_kwargs, + backend_output_types, + ) + from .generations import GenerationQueue + from .manifest import ServedModel, load_manifest + + pipeline_cls, signature, commit_hash = _resolve_pipeline(args) + kwarg_names = backend_kwargs(args.backend, signature) + manifest = load_manifest( + args.model, + pipeline_cls, + signature, + kwarg_names, + manifest_path=args.manifest, + revision=args.revision, + token=args.token, + ) + served = ServedModel( + manifest, + kwarg_names, + backend_output_types(args.backend), + backend_capabilities(args, pipeline_cls), + revision=commit_hash, + ) + if not served.tasks: + raise SystemExit( + f"None of the tasks {list(manifest.data['tasks'])} in the manifest for {args.model!r} can be " + f"served by {pipeline_cls.__name__} with --backend {args.backend}." + ) + + if args.backend == "diffusers": + backend = DiffusersBackend(args, served) + else: + backend = EngineBackend.launch(args, served) + + try: + app = create_app( + served, + GenerationQueue(backend, args.output_dir), + default_seed=args.default_seed, + enable_cors=args.enable_cors, + ) + out.result( + self.task, + url=f"http://{args.host}:{args.port}", + model=served.id, + backend=args.backend, + pipeline_class=pipeline_cls.__name__, + tasks=list(served.tasks), + capabilities=served.capabilities, + ) + uvicorn.run(app, host=args.host, port=args.port, log_level=args.log_level or "info") + finally: + backend.close() diff --git a/src/diffusers/commands/serve/app.py b/src/diffusers/commands/serve/app.py new file mode 100644 index 000000000000..05213fc302ce --- /dev/null +++ b/src/diffusers/commands/serve/app.py @@ -0,0 +1,218 @@ +# Copyright 2026 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The Generative Media Spec HTTP surface, as a FastAPI app over one served model and one generation queue.""" + +from __future__ import annotations + +import asyncio +import base64 +import json +import re +import uuid +from typing import Any + +from fastapi import FastAPI, Request +from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import FileResponse, JSONResponse, Response, StreamingResponse +from starlette.exceptions import HTTPException + +from ...utils import logging +from .generations import Generation, GenerationQueue +from .manifest import GMSError, ServedModel + + +logger = logging.get_logger("diffusers-cli/serve") + +MAX_WAIT_SECONDS = 300.0 +REQUEST_ID_HEADER = "x-request-id" +_KEEP_ALIVE_SECONDS = 15.0 + + +def _timestamp(value: Any) -> str | None: + if value is None: + return None + return value.strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _render(generation: Generation, request: Request) -> dict[str, Any]: + outputs = None + if generation.status == "complete": + outputs = {} + for name, artifacts in generation.outputs.items(): + outputs[name] = [] + for artifact in artifacts: + rendered = {key: value for key, value in artifact.items() if key != "file"} + if generation.request.response_format == "b64_json": + content = (generation.directory / artifact["file"]).read_bytes() + rendered["base64"] = base64.b64encode(content).decode() + else: + url = request.url_for( + "get_generation_file", generation_id=generation.id, filename=artifact["file"] + ) + rendered["url"] = str(url) + outputs[name].append(rendered) + + return { + "id": generation.id, + "model": generation.model, + "revision": generation.revision, + "status": generation.status, + "created_at": _timestamp(generation.created_at), + "progress": generation.progress if generation.status == "generating" else None, + "outputs": outputs, + "expires_at": _timestamp(generation.expires_at), + "seed": generation.seed if generation.status == "complete" else None, + "usage": generation.usage, + } + + +def create_app( + served: ServedModel, + generations: GenerationQueue, + default_seed: int | None = None, + enable_cors: bool = False, +) -> FastAPI: + app = FastAPI(title="diffusers-cli serve", docs_url=None, redoc_url=None, openapi_url=None) + + @app.middleware("http") + async def request_id(request: Request, call_next) -> Response: + request.state.request_id = request.headers.get(REQUEST_ID_HEADER) or str(uuid.uuid4()) + response = await call_next(request) + response.headers[REQUEST_ID_HEADER] = request.state.request_id + return response + + if enable_cors: + app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_methods=["*"], + allow_headers=["*"], + expose_headers=[REQUEST_ID_HEADER], + ) + logger.warning("--enable-cors allows requests from any origin. Not recommended for production.") + + @app.exception_handler(GMSError) + async def gms_error(request: Request, error: GMSError) -> JSONResponse: + return JSONResponse(error.to_dict(), status_code=error.status) + + @app.exception_handler(HTTPException) + async def http_error(request: Request, error: HTTPException) -> JSONResponse: + if error.status_code == 404: + code = "not_found" + elif error.status_code < 500: + code = "invalid_request" + else: + code = "internal" + return JSONResponse({"error": {"code": code, "message": str(error.detail)}}, status_code=error.status_code) + + @app.exception_handler(Exception) + async def internal_error(request: Request, error: Exception) -> JSONResponse: + message = f"{type(error).__name__}: {error}" + # An unhandled error is answered outside the middleware stack, so the request id is attached here. + headers = {REQUEST_ID_HEADER: request.state.request_id} + return JSONResponse({"error": {"code": "internal", "message": message}}, status_code=500, headers=headers) + + @app.get("/health") + async def health() -> JSONResponse: + failure = generations.failure() + if failure is not None: + return JSONResponse({"status": "unhealthy", "reason": failure}, status_code=503) + return JSONResponse({"status": "ok"}) + + @app.get("/v1/models") + async def list_models() -> dict[str, Any]: + return {"data": [served.summary()]} + + @app.get("/v1/models/{model_id:path}") + async def get_model(model_id: str) -> Response: + if model_id != served.id: + raise GMSError("not_found", f"model {model_id!r} is not served here; this server hosts {served.id!r}") + return Response(served.discovery_document(), media_type="application/json") + + @app.post("/v1/generations", status_code=201) + async def create_generation(request: Request) -> dict[str, Any]: + try: + body = await request.json() + except json.JSONDecodeError as e: + raise GMSError("invalid_request", f"request body is not valid JSON: {e}") from e + resolved = served.resolve(body) + seeded = served.seeded(resolved.task) + generation = Generation(served.id, served.revision, resolved, seeded=seeded, default_seed=default_seed) + generations.submit(generation) + + wait = re.fullmatch(r"\s*wait=(\d+)\s*", request.headers.get("prefer", "")) + if wait is not None: + await asyncio.to_thread(generation.wait, min(float(wait.group(1)), MAX_WAIT_SECONDS)) + if generation.error is not None: + raise generation.error + return _render(generation, request) + + @app.get("/v1/generations/{generation_id}") + async def get_generation(generation_id: str, request: Request) -> dict[str, Any]: + generation = generations.get(generation_id) + if generation.error is not None: + raise generation.error + return _render(generation, request) + + @app.delete("/v1/generations/{generation_id}", status_code=204) + async def delete_generation(generation_id: str) -> Response: + generations.delete(generation_id) + return Response(status_code=204) + + @app.post("/v1/generations/{generation_id}/cancel") + async def cancel_generation(generation_id: str, request: Request) -> dict[str, Any]: + generation = generations.cancel(generation_id) + if generation.error is not None: + raise generation.error + return _render(generation, request) + + @app.get("/v1/generations/{generation_id}/events") + async def get_generation_events(generation_id: str, request: Request) -> StreamingResponse: + generation = generations.get(generation_id) + last_event_id = request.headers.get("last-event-id", "") + start = int(last_event_id) if last_event_id.isdigit() and len(last_event_id) < 10 else 0 + + async def stream(): + index = start + while True: + events, done = await asyncio.to_thread(generation.wait_for_events, index, _KEEP_ALIVE_SECONDS) + for event, data in events: + index += 1 + payload = _render(generation, request) if event == "complete" else data + yield f"id: {index}\nevent: {event}\ndata: {json.dumps(payload)}\n\n" + if done: + return + if not events: + yield ": keep-alive\n\n" + + return StreamingResponse(stream(), media_type="text/event-stream", headers={"Cache-Control": "no-cache"}) + + @app.get("/v1/generations/{generation_id}/files/{filename}") + async def get_generation_file(generation_id: str, filename: str) -> FileResponse: + generation = generations.get(generation_id) + artifacts = [a for group in (generation.outputs or {}).values() for a in group if a["file"] == filename] + if not artifacts: + raise GMSError("not_found", f"generation {generation_id!r} has no file {filename!r}") + return FileResponse(generation.directory / filename, media_type=artifacts[0]["media_type"]) + + @app.get("/v1/generations/{generation_id}/outputs/{name}/stream") + async def stream_generation_output(generation_id: str, name: str) -> Response: + raise GMSError("not_implemented", "this server does not advertise the `stream` capability") + + @app.post("/v1/uploads") + async def create_upload() -> Response: + raise GMSError("not_implemented", "this server does not advertise the `presigned_upload` capability") + + return app diff --git a/src/diffusers/commands/serve/backends.py b/src/diffusers/commands/serve/backends.py new file mode 100644 index 000000000000..c0a779804dad --- /dev/null +++ b/src/diffusers/commands/serve/backends.py @@ -0,0 +1,472 @@ +# Copyright 2026 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Backends that execute a resolved generation: an in-process pipeline, or an SGLang / vLLM-Omni server.""" + +from __future__ import annotations + +import base64 +import inspect +import io +import mimetypes +import os +import shlex +import shutil +import socket +import subprocess +import tempfile +import time +import wave +from argparse import Namespace +from pathlib import Path +from typing import Any +from urllib.parse import urlparse + +import numpy as np +import torch +from huggingface_hub.utils import httpx +from PIL import Image + +from ...modular_pipelines import ModularPipeline +from ...utils import export_to_video, load_image, load_video, logging, numpy_to_pil +from ...utils.constants import DIFFUSERS_REQUEST_TIMEOUT +from ..run import _as_audio_arrays, _as_pil_list, _get_generator, _load_pipeline, _save_audio_arrays +from .generations import Generation +from .manifest import GMSError, ServedModel, blocks_signature + + +logger = logging.get_logger("diffusers-cli/serve") + +# Request fields both engines accept on their OpenAI-compatible image and video routes, named as the +# pipeline kwargs they correspond to. +_ENGINE_KWARGS = frozenset( + { + "prompt", + "negative_prompt", + "image", + "width", + "height", + "num_frames", + "fps", + "num_inference_steps", + "guidance_scale", + "guidance_scale_2", + "true_cfg_scale", + "generator", + "num_images_per_prompt", + } +) +_ENGINE_FIELDS = {"num_images_per_prompt": "n"} +_ENGINE_INSTALL = {"sglang": 'pip install "sglang[diffusion]"', "vllm": "pip install vllm-omni"} +_POLL_SECONDS = 1.0 + +_OUTPUT_FIELDS = {"image": ("images",), "video": ("frames", "videos"), "audio": ("audios", "audio")} + + +def backend_kwargs(backend: str, signature: inspect.Signature) -> set[str]: + names = set(signature.parameters) - {"self"} + if backend == "diffusers": + return names + return names & _ENGINE_KWARGS + + +def backend_output_types(backend: str) -> set[str]: + if backend == "diffusers": + return {"image", "video", "audio"} + return {"image", "video"} + + +def backend_capabilities(args: Namespace, pipeline_cls: type) -> list[str]: + capabilities = ["sse", "wait"] + if args.backend != "diffusers": + return capabilities + # Swapping adapter weights retraces a compiled denoiser on every request, and LoRAs passed at + # startup are part of the deployment rather than something a request may replace. + if hasattr(pipeline_cls, "load_lora_weights") and args.compile is None and not args.lora: + capabilities.append("adapters") + return capabilities + + +def _media_bytes(name: str, value: dict[str, Any], pointer: str) -> bytes: + try: + if "base64" in value: + return base64.b64decode(value["base64"], validate=True) + response = httpx.get(value["url"], follow_redirects=True, timeout=DIFFUSERS_REQUEST_TIMEOUT) + response.raise_for_status() + return response.content + except (ValueError, httpx.HTTPError) as e: + raise GMSError("invalid_request", f"could not read input {name!r}: {e}", pointer) from e + + +def _media_suffix(value: dict[str, Any], default: str) -> str: + if value.get("media_type"): + return mimetypes.guess_extension(value["media_type"]) or default + return Path(urlparse(value.get("url", "")).path).suffix or default + + +def _load_media(name: str, value: dict[str, Any], pointer: str) -> Any: + content = _media_bytes(name, value, pointer) + try: + if value["type"] in ("image", "mask"): + return load_image(Image.open(io.BytesIO(content))) + if value["type"] == "audio": + import torchaudio + + return torchaudio.load(io.BytesIO(content)) + with tempfile.NamedTemporaryFile(suffix=_media_suffix(value, ".mp4")) as file: + file.write(content) + file.flush() + return load_video(file.name) + except Exception as e: + raise GMSError("invalid_request", f"could not decode input {name!r}: {e}", pointer) from e + + +def _save_images(images: Any, directory: Path, name: str, media_type: str, rate: float) -> list[dict[str, Any]]: + if isinstance(images, np.ndarray): + images = numpy_to_pil(images) + images = _as_pil_list(images) + if images is None: + raise GMSError("internal", f"cannot encode output {name!r} as {media_type}: not a list of images") + + artifacts = [] + for index, image in enumerate(images): + filename = f"{name}-{index}.{media_type.split('/')[1]}" + if media_type == "image/jpeg": + image = image.convert("RGB") + image.save(directory / filename) + artifacts.append({"file": filename, "media_type": media_type, "width": image.width, "height": image.height}) + return artifacts + + +def _save_videos(videos: Any, directory: Path, name: str, media_type: str, rate: float) -> list[dict[str, Any]]: + artifacts = [] + for index, frames in enumerate(videos): + frames = list(frames) + filename = f"{name}-{index}.mp4" + export_to_video(frames, str(directory / filename), fps=rate) + first = frames[0] + width, height = first.size if isinstance(first, Image.Image) else (first.shape[1], first.shape[0]) + artifacts.append( + { + "file": filename, + "media_type": media_type, + "width": width, + "height": height, + "duration": round(len(frames) / rate, 2), + "fps": rate, + } + ) + return artifacts + + +def _save_audio(audios: Any, directory: Path, name: str, media_type: str, rate: float) -> list[dict[str, Any]]: + if isinstance(audios, torch.Tensor): + audios = audios.detach().to(torch.float32).cpu().numpy() + arrays = _as_audio_arrays(audios) + if arrays is None: + raise GMSError("internal", f"cannot encode output {name!r} as {media_type}: not an audio array") + + staging = directory / f"{name}-staging" + paths = _save_audio_arrays(arrays, int(rate), Namespace(output=str(staging) + os.sep)) + artifacts = [] + for index, path in enumerate(paths): + filename = f"{name}-{index}.wav" + Path(path).rename(directory / filename) + with wave.open(str(directory / filename), "rb") as file: + duration = round(file.getnframes() / file.getframerate(), 2) + artifacts.append({"file": filename, "media_type": media_type, "duration": duration, "sample_rate": int(rate)}) + staging.rmdir() + return artifacts + + +_SAVERS = {"image": _save_images, "video": _save_videos, "audio": _save_audio} + + +class DiffusersBackend: + def __init__(self, args: Namespace, served: ServedModel): + self._args = args + self._served = served + self.pipeline = _load_pipeline(args) + self._modular = isinstance(self.pipeline, ModularPipeline) + if self._modular: + parameters = blocks_signature(self.pipeline.blocks).parameters + else: + parameters = inspect.signature(type(self.pipeline).__call__).parameters + self._defaults = { + name: parameter.default + for name, parameter in parameters.items() + if parameter.default is not inspect.Parameter.empty + } + self._reports_steps = "callback_on_step_end" in parameters + + def close(self) -> None: + pass + + def failure(self) -> str | None: + return None + + def generate(self, generation: Generation) -> dict[str, list[dict[str, Any]]]: + request = generation.request + kwargs = dict(request.kwargs) + for kwarg, (name, value) in request.media.items(): + if isinstance(value, list): + loaded = [_load_media(name, item, f"/inputs/{name}/{index}") for index, item in enumerate(value)] + else: + loaded = [_load_media(name, value, f"/inputs/{name}")] + if self._served.tasks[request.task]["inputs"][name]["type"] == "audio": + if kwarg == "initial_audio_waveforms": + kwargs.setdefault("initial_audio_sampling_rate", loaded[0][1]) + loaded = [waveform for waveform, _ in loaded] + kwargs[kwarg] = loaded if isinstance(value, list) else loaded[0] + + if generation.seed is not None: + device = self.pipeline.device.type if hasattr(self.pipeline, "device") else "cpu" + kwargs["generator"] = _get_generator(generation.seed, device) + + if self._reports_steps: + + def on_step_end(pipeline, step, timestep, callback_kwargs): + total_steps = getattr(pipeline, "num_timesteps", None) + if total_steps: + generation.report_progress(step + 1, total_steps) + if generation.cancel_requested: + pipeline._interrupt = True + return callback_kwargs + + kwargs["callback_on_step_end"] = on_step_end + + try: + self._load_adapters(request.adapters) + result = self.pipeline(**kwargs) + finally: + if request.adapters: + self.pipeline.unload_lora_weights() + if generation.cancel_requested: + return {} + + outputs = {} + for name, spec in self._served.tasks[request.task]["outputs"].items(): + kind = spec["type"] + # A modular pipeline returns its `PipelineState`; a standard one returns an output object. + if self._modular: + fields = (result.get(field) for field in _OUTPUT_FIELDS.get(kind, ())) + else: + fields = (getattr(result, field, None) for field in _OUTPUT_FIELDS.get(kind, ())) + value = next((field for field in fields if field is not None), None) + if value is None: + continue + rate = self._fps(kwargs) if kind == "video" else (self._args.sampling_rate or 16000) + outputs[name] = _SAVERS[kind](value, generation.directory, name, request.output_formats[kind], rate) + if not outputs: + declared = list(self._served.tasks[request.task]["outputs"]) + raise GMSError( + "internal", + f"{type(self.pipeline).__name__} returned none of the outputs {declared} declared for task " + f"{request.task!r}", + ) + return outputs + + def _fps(self, kwargs: dict[str, Any]) -> float: + for kwarg in ("fps", "frame_rate"): + rate = kwargs.get(kwarg) or self._defaults.get(kwarg) + if rate: + return rate + return self._args.fps + + def _load_adapters(self, adapters: list[dict[str, Any]]) -> None: + if not adapters: + return + names = [] + weights = [] + for index, adapter in enumerate(adapters): + name = f"request_{index}" + try: + self.pipeline.load_lora_weights(adapter["path"].removeprefix("hf:"), adapter_name=name) + except Exception as e: + raise GMSError( + "invalid_request", + f"could not load adapter {adapter['path']!r}: {type(e).__name__}: {e}", + f"/adapters/{index}/path", + ) from e + scale = adapter.get("scale", 1.0) + targets = adapter.get("targets") + if targets: + valid_targets = self._served.adapters[adapter["type"]].get("targets", []) + weights.append({target: targets.get(target, scale) for target in valid_targets}) + else: + weights.append(scale) + names.append(name) + self.pipeline.set_adapters(names, adapter_weights=weights) + + +class EngineBackend: + def __init__(self, engine: str, served: ServedModel, client: Any, process: subprocess.Popen | None = None): + self._engine = engine + self._served = served + self._client = client + self._process = process + + @classmethod + def launch(cls, args: Namespace, served: ServedModel) -> EngineBackend: + with socket.socket() as probe: + probe.bind(("127.0.0.1", 0)) + port = probe.getsockname()[1] + + if args.backend == "sglang": + command = ["sglang", "serve", "--model-path", args.model] + else: + command = ["vllm", "serve", args.model, "--omni"] + command += ["--host", "127.0.0.1", "--port", str(port)] + if args.revision: + command += ["--revision", args.revision] + command += shlex.split(args.backend_args or "") + + if shutil.which(command[0]) is None: + raise SystemExit( + f"--backend {args.backend} needs the `{command[0]}` executable on PATH. " + f"Install it with: {_ENGINE_INSTALL[args.backend]}" + ) + + logger.info(f"starting {args.backend} backend: {shlex.join(command)}") + process = subprocess.Popen(command) + client = httpx.Client(base_url=f"http://127.0.0.1:{port}", timeout=None) + backend = cls(args.backend, served, client, process) + try: + backend._wait_until_ready(command) + except BaseException: + backend.close() + raise + return backend + + def failure(self) -> str | None: + if self._process is None or self._process.poll() is None: + return None + return f"the {self._engine} backend exited with code {self._process.returncode}" + + def close(self) -> None: + self._client.close() + if self._process is None or self._process.poll() is not None: + return + self._process.terminate() + try: + self._process.wait(timeout=30) + except subprocess.TimeoutExpired: + self._process.kill() + self._process.wait() + + def generate(self, generation: Generation) -> dict[str, list[dict[str, Any]]]: + request = generation.request + outputs = {spec["type"]: name for name, spec in self._served.tasks[request.task]["outputs"].items()} + + fields = {_ENGINE_FIELDS.get(kwarg, kwarg): value for kwarg, value in request.kwargs.items()} + if generation.seed is not None: + fields["seed"] = generation.seed + width = fields.pop("width", None) + height = fields.pop("height", None) + if (width is None) != (height is None): + missing = "width" if width is None else "height" + raise GMSError( + "invalid_request", + f"the {self._engine} backend takes width and height together; {missing} is missing", + f"/parameters/{missing}", + ) + if width is not None: + fields["size"] = f"{width}x{height}" + + images = [] + if "image" in request.media: + name, value = request.media["image"] + for index, item in enumerate(value if isinstance(value, list) else [value]): + pointer = f"/inputs/{name}/{index}" if isinstance(value, list) else f"/inputs/{name}" + content = _media_bytes(name, item, pointer) + images.append((f"image-{index}{_media_suffix(item, '.png')}", content, item.get("media_type"))) + + if "video" in outputs: + return {outputs["video"]: self._generate_video(generation, fields, images)} + return {outputs["image"]: self._generate_images(generation, fields, images)} + + def _generate_images(self, generation: Generation, fields: dict[str, Any], images: list) -> list[dict[str, Any]]: + media_type = generation.request.output_formats["image"] + extension = media_type.split("/")[1] + fields = {**fields, "response_format": "b64_json", "output_format": extension} + if images: + response = self._client.post("/v1/images/edits", files=self._form(fields, "image", images)) + else: + response = self._client.post("/v1/images/generations", json=fields) + + artifacts = [] + for index, item in enumerate(self._json(response)["data"]): + content = base64.b64decode(item["b64_json"]) + filename = f"image-{index}.{extension}" + (generation.directory / filename).write_bytes(content) + width, height = Image.open(io.BytesIO(content)).size + artifacts.append({"file": filename, "media_type": media_type, "width": width, "height": height}) + return artifacts + + def _generate_video(self, generation: Generation, fields: dict[str, Any], images: list) -> list[dict[str, Any]]: + job = self._json(self._client.post("/v1/videos", files=self._form(fields, "input_reference", images[:1]))) + while job["status"] in ("queued", "in_progress"): + if generation.cancel_requested: + self._client.delete(f"/v1/videos/{job['id']}") + return [] + time.sleep(_POLL_SECONDS) + job = self._json(self._client.get(f"/v1/videos/{job['id']}")) + if job["status"] != "completed": + message = (job.get("error") or {}).get("message", f"video job ended with status {job['status']!r}") + raise GMSError("internal", f"{self._engine} backend: {message}") + + response = self._client.get(f"/v1/videos/{job['id']}/content") + if response.status_code >= 400: + self._json(response) + filename = "video-0.mp4" + (generation.directory / filename).write_bytes(response.content) + self._client.delete(f"/v1/videos/{job['id']}") + + artifact = {"file": filename, "media_type": "video/mp4"} + if job.get("fps"): + artifact["fps"] = job["fps"] + if job.get("duration_s"): + artifact["duration"] = job["duration_s"] + return [artifact] + + @staticmethod + def _form(fields: dict[str, Any], file_field: str, files: list) -> list: + # Both engines read these routes as multipart forms, so plain fields are sent as nameless + # parts: httpx falls back to urlencoding when a request carries no file parts. + parts = [] + for key, value in fields.items(): + text = str(value).lower() if isinstance(value, bool) else str(value) + parts.append((key, (None, text))) + for filename, content, media_type in files: + parts.append((file_field, (filename, content, media_type or "application/octet-stream"))) + return parts + + def _json(self, response: Any) -> Any: + if response.status_code >= 400: + code = "invalid_request" if response.status_code < 500 else "internal" + raise GMSError(code, f"{self._engine} backend returned {response.status_code}: {response.text[:500]}") + return response.json() + + def _wait_until_ready(self, command: list[str]) -> None: + while True: + if self._process.poll() is not None: + raise SystemExit( + f"`{shlex.join(command)}` exited with code {self._process.returncode} before becoming ready." + ) + try: + if self._client.get("/health", timeout=5.0).status_code == 200: + return + except httpx.HTTPError: + pass + time.sleep(2.0) diff --git a/src/diffusers/commands/serve/generations.py b/src/diffusers/commands/serve/generations.py new file mode 100644 index 000000000000..592ca965d54a --- /dev/null +++ b/src/diffusers/commands/serve/generations.py @@ -0,0 +1,194 @@ +# Copyright 2026 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The generation resource and the single-worker queue that runs generations one at a time.""" + +from __future__ import annotations + +import queue +import secrets +import shutil +import threading +import time +import uuid +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any + +from ...utils import logging +from .manifest import GMSError, ResolvedRequest + + +logger = logging.get_logger("diffusers-cli/serve") + +EXPIRY_SECONDS = 3600 + + +class Generation: + def __init__( + self, + model: str, + revision: str | None, + request: ResolvedRequest, + seeded: bool, + default_seed: int | None = None, + ): + self.id = f"gen_{uuid.uuid4().hex[:12]}" + self.model = model + self.revision = revision + self.request = request + # The spec has the server pick the seed when the request omits it, so the result is reproducible. + self.seed = None + if seeded and request.seed is not None: + self.seed = request.seed + elif seeded and default_seed is not None: + self.seed = default_seed + elif seeded: + self.seed = secrets.randbelow(2**32) + self.status = "queued" + self.created_at = datetime.now(timezone.utc) + self.expires_at: datetime | None = None + self.progress: dict[str, Any] | None = None + self.outputs: dict[str, list[dict[str, Any]]] | None = None + self.usage: dict[str, Any] | None = None + self.error: GMSError | None = None + self.cancel_requested = False + self.directory: Path | None = None + self.events: list[tuple[str, dict[str, Any] | None]] = [] + self.done = False + self._condition = threading.Condition() + + def report_progress(self, step: int, total_steps: int) -> None: + self.progress = {"step": step, "total_steps": total_steps, "fraction": round(step / total_steps, 4)} + self._emit("progress", self.progress) + + def wait_for_events(self, index: int, timeout: float) -> tuple[list[tuple[str, dict[str, Any] | None]], bool]: + with self._condition: + self._condition.wait_for(lambda: len(self.events) > index or self.done, timeout) + return self.events[index:], self.done + + def wait(self, timeout: float) -> None: + with self._condition: + self._condition.wait_for(lambda: self.done, timeout) + + def finish(self, status: str, error: GMSError | None = None) -> None: + with self._condition: + self.status = status + self.error = error + self.progress = None + self.expires_at = datetime.now(timezone.utc) + timedelta(seconds=EXPIRY_SECONDS) + if error is not None: + self.events.append(("error", {"code": error.code, "message": error.message})) + elif status == "complete": + self.events.append(("complete", None)) + self.done = True + self._condition.notify_all() + + def _emit(self, event: str, data: dict[str, Any] | None) -> None: + with self._condition: + self.events.append((event, data)) + self._condition.notify_all() + + +class GenerationQueue: + def __init__(self, backend: Any, output_dir: str): + self._backend = backend + self._output_dir = Path(output_dir) + self._generations: dict[str, Generation] = {} + self._lock = threading.Lock() + self._pending: queue.Queue[Generation] = queue.Queue() + threading.Thread(target=self._work, name="diffusers-serve-worker", daemon=True).start() + + def failure(self) -> str | None: + return self._backend.failure() + + def submit(self, generation: Generation) -> None: + self._prune() + with self._lock: + self._generations[generation.id] = generation + self._pending.put(generation) + + def get(self, generation_id: str) -> Generation: + self._prune() + with self._lock: + generation = self._generations.get(generation_id) + if generation is None: + raise GMSError("not_found", f"no generation with id {generation_id!r}") + return generation + + def cancel(self, generation_id: str) -> Generation: + generation = self.get(generation_id) + with self._lock: + if generation.done: + return generation + generation.cancel_requested = True + if generation.status == "queued": + generation.finish("canceled") + return generation + + def delete(self, generation_id: str) -> None: + generation = self.cancel(generation_id) + with self._lock: + del self._generations[generation.id] + # A generation still running keeps writing into its directory; the worker removes it once the backend returns. + if generation.done: + self._remove_files(generation) + + def _prune(self) -> None: + now = datetime.now(timezone.utc) + with self._lock: + expired = [g for g in self._generations.values() if g.expires_at is not None and g.expires_at <= now] + for generation in expired: + del self._generations[generation.id] + for generation in expired: + self._remove_files(generation) + + def _remove_files(self, generation: Generation) -> None: + if generation.directory is not None: + shutil.rmtree(generation.directory, ignore_errors=True) + + def _work(self) -> None: + while True: + generation = self._pending.get() + with self._lock: + if generation.done: + continue + generation.status = "generating" + generation.directory = self._output_dir / generation.id + + started = time.perf_counter() + error = None + try: + generation.directory.mkdir(parents=True, exist_ok=True) + outputs = self._backend.generate(generation) + except GMSError as e: + error = e + # The helpers shared with `run` raise SystemExit, and anything escaping here would end the + # only worker and leave every later generation queued. + except BaseException as e: + logger.error(f"generation {generation.id} failed", exc_info=True) + error = GMSError("internal", f"{type(e).__name__}: {e}") + + with self._lock: + deleted = generation.id not in self._generations + if error is not None: + generation.finish("failed", error) + elif generation.cancel_requested: + generation.finish("canceled") + else: + generation.outputs = outputs + generation.usage = {"compute_seconds": round(time.perf_counter() - started, 2)} + generation.finish("complete") + if deleted or generation.status != "complete": + self._remove_files(generation) diff --git a/src/diffusers/commands/serve/manifest.py b/src/diffusers/commands/serve/manifest.py new file mode 100644 index 000000000000..054dbb8a9c93 --- /dev/null +++ b/src/diffusers/commands/serve/manifest.py @@ -0,0 +1,779 @@ +# Copyright 2026 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Generative Media Spec (GMS) manifests: loading, derivation from a pipeline signature, and request validation.""" + +from __future__ import annotations + +import hashlib +import inspect +import json +import math +import operator +import re +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +from huggingface_hub import hf_hub_download +from huggingface_hub.errors import EntryNotFoundError + +from ...pipelines.auto_pipeline import ( + AUTO_IMAGE2IMAGE_PIPELINES_MAPPING, + AUTO_IMAGE2VIDEO_PIPELINES_MAPPING, + AUTO_INPAINT_PIPELINES_MAPPING, + AUTO_TEXT2AUDIO_PIPELINES_MAPPING, + AUTO_TEXT2IMAGE_PIPELINES_MAPPING, + AUTO_TEXT2VIDEO_PIPELINES_MAPPING, + AUTO_VIDEO2VIDEO_PIPELINES_MAPPING, +) +from ..run import _AUDIO_INPUT_KEYS, _IMAGE_INPUT_KEYS, _VIDEO_INPUT_KEYS + + +SPEC_VERSION = "gms/0.1" +MANIFEST_FILENAME = "gms.json" +EXTENSION_KEY = "x-diffusers" + +MAX_ADAPTERS = 4 +ADAPTER_SOURCES = ("hf",) + +# First entry of each tuple is the default when the request names no `output_format`. +OUTPUT_FORMATS = { + "image": ("image/png", "image/jpeg", "image/webp"), + "video": ("video/mp4",), + "audio": ("audio/wav",), +} + +_ERROR_STATUS = { + "invalid_request": 400, + "not_found": 404, + "rate_limited": 429, + "internal": 500, + "not_implemented": 501, +} + +_REQUEST_FIELDS = ( + "model", + "task", + "preset", + "inputs", + "parameters", + "output_format", + "response_format", + "adapters", +) + +# GMS name -> pipeline kwargs that carry the same meaning, tried in order before the name itself. +_PARAMETER_KWARGS = { + "seed": ("generator",), + "steps": ("num_inference_steps",), + "num_outputs": ("num_images_per_prompt", "num_videos_per_prompt", "num_waveforms_per_prompt"), + "fps": ("fps", "frame_rate"), + "duration": ("num_frames",), +} +_INPUT_KWARGS = {"mask": ("mask_image",)} + +# (task, classes registered for it, inputs the task requires, output modality) +_AUTO_TASKS = ( + ("text_to_image", AUTO_TEXT2IMAGE_PIPELINES_MAPPING, (), "image"), + ("image_to_image", AUTO_IMAGE2IMAGE_PIPELINES_MAPPING, ("image",), "image"), + ("inpaint", AUTO_INPAINT_PIPELINES_MAPPING, ("image", "mask"), "image"), + ("text_to_video", AUTO_TEXT2VIDEO_PIPELINES_MAPPING, (), "video"), + ("image_to_video", AUTO_IMAGE2VIDEO_PIPELINES_MAPPING, ("image",), "video"), + ("video_to_video", AUTO_VIDEO2VIDEO_PIPELINES_MAPPING, ("video",), "video"), + ("text_to_audio", AUTO_TEXT2AUDIO_PIPELINES_MAPPING, (), "audio"), +) + +_SKIPPED_KWARGS = frozenset({"output_type", "return_dict", "callback_steps", "callback_on_step_end_tensor_inputs"}) +_AUDIO_OUTPUT_KWARGS = frozenset({"audio_length_in_s", "audio_end_in_s", "num_waveforms_per_prompt"}) +_SCALAR_TYPES = {"float": "number", "int": "integer", "bool": "boolean", "str": "string"} +_PYTHON_TYPES = {"number": (int, float), "integer": int, "boolean": bool, "string": str} +_COMPARISONS = { + "eq": operator.eq, + "gt": operator.gt, + "gte": operator.ge, + "lt": operator.lt, + "lte": operator.le, +} +_FRAME_LATTICE = re.compile(r"(\d+)k\+(\d+)") +_HF_ADAPTER_PATH = re.compile(r"hf:[\w.-]+/[\w.-]+") + + +class GMSError(Exception): + def __init__(self, code: str, message: str, pointer: str | None = None): + super().__init__(message) + self.code = code + self.message = message + self.pointer = pointer + self.status = _ERROR_STATUS[code] + + def to_dict(self) -> dict[str, Any]: + error = {"code": self.code, "message": self.message} + if self.pointer is not None: + error["pointer"] = self.pointer + return {"error": error} + + +@dataclass +class Manifest: + raw: bytes + data: dict[str, Any] + source: str | None = None + + @property + def hash(self) -> str: + return f"sha256:{hashlib.sha256(self.raw).hexdigest()}" + + +@dataclass +class ResolvedRequest: + task: str + kwargs: dict[str, Any] + # pipeline kwarg -> (request input name, media value or list of media values) + media: dict[str, tuple[str, Any]] + seed: int | None + output_formats: dict[str, str] + response_format: str + adapters: list[dict[str, Any]] = field(default_factory=list) + + +def blocks_signature(blocks: Any) -> inspect.Signature: + """The call signature of a modular pipeline, which takes its keyword arguments from its blocks' inputs.""" + parameters = [] + for input_param in blocks.inputs: + if input_param.name is None: + continue + annotation = inspect.Parameter.empty if input_param.type_hint is None else input_param.type_hint + parameters.append( + inspect.Parameter( + input_param.name, inspect.Parameter.KEYWORD_ONLY, default=input_param.default, annotation=annotation + ) + ) + return inspect.Signature(parameters) + + +def load_manifest( + model: str, + pipeline_cls: type, + signature: inspect.Signature, + kwarg_names: set[str], + manifest_path: str | None = None, + revision: str | None = None, + token: str | None = None, +) -> Manifest: + if manifest_path is not None: + return _parse_manifest(Path(manifest_path).read_bytes(), origin=manifest_path) + + if Path(model).is_dir(): + published = Path(model) / MANIFEST_FILENAME + if published.is_file(): + return _parse_manifest(published.read_bytes(), origin=str(published)) + else: + try: + published = hf_hub_download(model, MANIFEST_FILENAME, revision=revision, token=token) + except EntryNotFoundError: + published = None + if published is not None: + manifest = _parse_manifest(Path(published).read_bytes(), origin=f"{model}/{MANIFEST_FILENAME}") + manifest.source = f"hf:{model}/{MANIFEST_FILENAME}" + return manifest + + model_id = Path(model).name if Path(model).is_dir() else model + data = derive_manifest(model_id, pipeline_cls, signature, kwarg_names) + return Manifest(raw=json.dumps(data).encode(), data=data) + + +def _parse_manifest(raw: bytes, origin: str) -> Manifest: + try: + data = json.loads(raw) + except json.JSONDecodeError as e: + raise SystemExit(f"GMS manifest {origin!r} is not valid JSON: {e}") from e + if not isinstance(data, dict) or not isinstance(data.get("id"), str): + raise SystemExit(f"GMS manifest {origin!r} must be a JSON object with a string `id`.") + if not isinstance(data.get("tasks"), dict) or not data["tasks"]: + raise SystemExit(f"GMS manifest {origin!r} declares no `tasks`, so there is nothing to serve.") + return Manifest(raw=raw, data=data) + + +def _annotation_schema(annotation: Any, default: Any) -> dict[str, Any] | None: + """Map a `__call__` annotation to a GMS parameter schema, or `None` when it is not a scalar or list of scalars.""" + if annotation is inspect.Parameter.empty: + text = type(default).__name__ if default is not None else "" + elif isinstance(annotation, str): + text = annotation + elif type(annotation) is type: + text = annotation.__name__ + else: + text = str(annotation) + text = text.replace("typing.", "") + + def scalar(fragment: str) -> str | None: + names = set(re.findall(r"[A-Za-z_][\w.]*", fragment)) + names -= {"Optional", "Union", "None", "NoneType", "list", "List"} + if not names or not names <= set(_SCALAR_TYPES): + return None + return next(gms for python, gms in _SCALAR_TYPES.items() if python in names) + + outside_lists = re.sub(r"\b[Ll]ist\[[^\[\]]*\]", "", text) + if outside_lists == text: + scalar_type = scalar(text) + return {"type": scalar_type} if scalar_type else None + + # `str | list[str]` is a scalar that also accepts a batch; only a bare list is an array parameter. + scalar_type = scalar(text) + if scalar_type is None: + return None + if scalar(outside_lists): + return {"type": scalar_type} + return {"type": "array", "items": {"type": scalar_type}} + + +def derive_manifest( + model_id: str, pipeline_cls: type, signature: inspect.Signature, kwarg_names: set[str] +) -> dict[str, Any]: + """Build a server-authored manifest from the pipeline call `signature`, keeping only kwargs in `kwarg_names`.""" + inputs: dict[str, dict[str, Any]] = {} + parameters: dict[str, dict[str, Any]] = {} + input_kwargs: dict[str, str] = {} + parameter_kwargs: dict[str, str] = {} + media_types = { + **dict.fromkeys(_IMAGE_INPUT_KEYS, "image"), + **dict.fromkeys(_VIDEO_INPUT_KEYS, "video"), + **dict.fromkeys(_AUDIO_INPUT_KEYS, "audio"), + "mask_image": "mask", + } + kwarg_parameters = {kwarg: name for name, kwargs in _PARAMETER_KWARGS.items() for kwarg in kwargs} + del kwarg_parameters["num_frames"] + + for kwarg, parameter in signature.parameters.items(): + if kwarg == "self" or kwarg in _SKIPPED_KWARGS or kwarg not in kwarg_names: + continue + if parameter.kind in (inspect.Parameter.VAR_POSITIONAL, inspect.Parameter.VAR_KEYWORD): + continue + has_default = parameter.default is not inspect.Parameter.empty + default = parameter.default if has_default else None + + if kwarg in media_types: + name = "mask" if kwarg == "mask_image" else kwarg + spec: dict[str, Any] = {"type": media_types[kwarg]} + if not has_default: + spec["required"] = True + if re.search(r"\b[Ll]ist\b", str(parameter.annotation)): + spec["multiple"] = {} + inputs[name] = spec + if name != kwarg: + input_kwargs[name] = kwarg + continue + + if kwarg == "generator": + parameters["seed"] = {"type": "integer"} + parameter_kwargs["seed"] = kwarg + continue + + schema = _annotation_schema(parameter.annotation, default) + # Modular blocks often declare `prompt` without a type hint or a default to infer one from. + if schema is None and kwarg == "prompt" and parameter.annotation is inspect.Parameter.empty: + schema = {"type": "string"} + if schema is None: + continue + if schema["type"] == "string" and re.fullmatch(r"(.*_)?prompt(_\d+)?", kwarg): + inputs[kwarg] = {"type": "text", "required": True} if kwarg == "prompt" else {"type": "text"} + continue + + name = kwarg_parameters.get(kwarg, kwarg) + if name in parameters: + continue + if default is not None and isinstance(default, (bool, int, float, str, list)): + schema["default"] = default + parameters[name] = schema + if name != kwarg: + parameter_kwargs[name] = kwarg + + registered = [entry for entry in _AUTO_TASKS if pipeline_cls in entry[1].values()] + names = set(signature.parameters) + if registered: + output = registered[0][3] + elif "num_frames" in names or "Video" in pipeline_cls.__name__: + output = "video" + elif names & _AUDIO_OUTPUT_KWARGS: + output = "audio" + else: + output = "image" + + extension: dict[str, Any] = {} + if input_kwargs: + extension["inputs"] = input_kwargs + if parameter_kwargs: + extension["parameters"] = parameter_kwargs + + def task(required: tuple[str, ...], excluded: set[str]) -> dict[str, Any]: + task_inputs = { + name: {**spec, "required": True} if name in required else spec + for name, spec in inputs.items() + if name not in excluded + } + spec = {"inputs": task_inputs, "outputs": {output: {"type": output}}, "parameters": parameters} + if extension: + spec[EXTENSION_KEY] = extension + return spec + + primary = next((name for name in ("image", "video") if name in inputs), None) + tasks = {} + if registered: + # A class registered for several tasks takes each task's conditioning input optionally; an + # input another task requires is what tells the tasks apart, so it is left out of the rest. + for name, _, required, _ in registered: + if not set(required) <= set(inputs): + continue + others = {key for other in registered if other[0] != name for key in other[2]} + tasks[name] = task(required, others - set(required)) + elif primary is None: + tasks[f"text_to_{output}"] = task((), set()) + elif inputs[primary].get("required"): + tasks[f"{primary}_to_{output}"] = task((), set()) + else: + tasks[f"text_to_{output}"] = task((), {primary}) + tasks[f"{primary}_to_{output}"] = task((primary,), set()) + + modalities_in = sorted({"image" if s["type"] == "mask" else s["type"] for s in inputs.values()} - {"text"}) + manifest: dict[str, Any] = { + "id": model_id, + "modalities": {"in": modalities_in or ["text"], "out": [output]}, + "tasks": tasks, + } + if hasattr(pipeline_cls, "load_lora_weights"): + manifest["adapters"] = {"lora": {"targets": list(getattr(pipeline_cls, "_lora_loadable_modules", []))}} + return manifest + + +def _bind(name: str, aliases: dict[str, tuple[str, ...]], declared: dict[str, str], kwarg_names: set[str]): + candidates = (declared[name],) if name in declared else (*aliases.get(name, ()), name) + return next((kwarg for kwarg in candidates if kwarg in kwarg_names), None) + + +def _frame_lattice(spec: dict[str, Any]) -> tuple[int, int, float] | None: + lattice = spec.get("lattice") or {} + match = _FRAME_LATTICE.fullmatch(str(lattice.get("frames", "")).replace(" ", "")) + if match is None or not isinstance(lattice.get("fps"), (int, float)): + return None + return int(match.group(1)), int(match.group(2)), lattice["fps"] + + +def _is_active(condition: dict[str, Any], values: dict[str, Any]) -> bool: + if "all" in condition: + return all(_is_active(item, values) for item in condition["all"]) + actual = values.get(condition["parameter"]) + for name, compare in _COMPARISONS.items(): + if name not in condition: + continue + if actual is None or isinstance(actual, str) != isinstance(condition[name], str): + return False + if not compare(actual, condition[name]): + return False + return True + + +def _check_value(label: str, value: Any, spec: dict[str, Any], pointer: str) -> None: + kind = spec.get("type") + if kind == "array": + if not isinstance(value, list): + raise GMSError("invalid_request", f"{label} must be an array, got {value!r}", pointer) + if "maxItems" in spec and len(value) > spec["maxItems"]: + raise GMSError("invalid_request", f"{label} accepts at most {spec['maxItems']} items", pointer) + for index, item in enumerate(value): + _check_value(f"{label}[{index}]", item, spec["items"], f"{pointer}/{index}") + return + if kind == "object": + if not isinstance(value, dict): + raise GMSError("invalid_request", f"{label} must be an object, got {value!r}", pointer) + for key, item in value.items(): + if key not in spec["properties"]: + raise GMSError("invalid_request", f"{label} has no property {key!r}", f"{pointer}/{key}") + _check_value(f"{label}.{key}", item, spec["properties"][key], f"{pointer}/{key}") + return + + expected = _PYTHON_TYPES.get(kind) + is_bool = isinstance(value, bool) + if expected is not None and (not isinstance(value, expected) or (is_bool and kind != "boolean")): + raise GMSError("invalid_request", f"{label} must be of type {kind}, got {value!r}", pointer) + if isinstance(value, float) and not math.isfinite(value): + raise GMSError("invalid_request", f"{label} must be a finite number, got {value}", pointer) + if "enum" in spec and value not in spec["enum"]: + raise GMSError("invalid_request", f"{label} must be one of {spec['enum']}, got {value!r}", pointer) + if "min" in spec and value < spec["min"]: + raise GMSError("invalid_request", f"{label} must be at least {spec['min']}, got {value}", pointer) + if "max" in spec and value > spec["max"]: + raise GMSError("invalid_request", f"{label} must be at most {spec['max']}, got {value}", pointer) + if "multiple_of" in spec: + quotient = value / spec["multiple_of"] + if not math.isclose(quotient, round(quotient), abs_tol=1e-9): + raise GMSError( + "invalid_request", f"{label} must be a multiple of {spec['multiple_of']}, got {value}", pointer + ) + + +def _check_media(name: str, value: Any, spec: dict[str, Any], pointer: str) -> None: + if not isinstance(value, dict): + raise GMSError("invalid_request", f"{name} must be a media value object, got {value!r}", pointer) + if value.get("type") != spec["type"]: + raise GMSError( + "invalid_request", f"{name} must have type {spec['type']!r}, got {value.get('type')!r}", f"{pointer}/type" + ) + keys = set(value) + if keys == {"type", "base64", "media_type"}: + if not isinstance(value["base64"], str) or not isinstance(value["media_type"], str): + raise GMSError("invalid_request", f"{name} base64 and media_type must be strings", pointer) + return + if "url" not in keys or not keys <= {"type", "url", "media_type"}: + raise GMSError( + "invalid_request", + f"{name} must be either {{type, url, media_type?}} or {{type, base64, media_type}}, got keys {sorted(keys)}", + pointer, + ) + # A URL media value is fetched by the server, so anything but http(s) would read the server's own filesystem. + if not isinstance(value["url"], str) or not value["url"].startswith(("http://", "https://")): + raise GMSError("invalid_request", f"{name} url must be an http(s) URL", f"{pointer}/url") + + +class ServedModel: + """One manifest bound to the kwargs a backend accepts: the discovery document and request resolution.""" + + def __init__( + self, + manifest: Manifest, + kwarg_names: set[str], + output_types: set[str], + capabilities: list[str], + revision: str | None = None, + ): + self.manifest = manifest + self.id = manifest.data["id"] + self.capabilities = capabilities + self.revision = revision + self.adapters = manifest.data.get("adapters", {}) + self._kwarg_names = kwarg_names + + self.tasks: dict[str, dict[str, Any]] = {} + self._inputs: dict[str, dict[str, str | None]] = {} + self._parameters: dict[str, dict[str, str | None]] = {} + self._presets: dict[str, list[str]] = {} + for name, task in manifest.data["tasks"].items(): + declared = task.get(EXTENSION_KEY, {}) + inputs = { + key: _bind(key, _INPUT_KWARGS, declared.get("inputs", {}), kwarg_names) for key in task["inputs"] + } + parameters = { + key: _bind(key, _PARAMETER_KWARGS, declared.get("parameters", {}), kwarg_names) + for key in task.get("parameters", {}) + } + if "duration" in parameters and _frame_lattice(task["parameters"]["duration"]) is None: + parameters["duration"] = None + if any(inputs[key] is None for key, spec in task["inputs"].items() if spec.get("required")): + continue + if not any(spec["type"] in output_types for spec in task["outputs"].values()): + continue + self.tasks[name] = task + self._inputs[name] = inputs + self._parameters[name] = parameters + self._presets[name] = [ + preset + for preset, body in task.get("presets", {}).items() + if all(parameters.get(key) is not None for key in body.get("parameters", {})) + ] + + def summary(self) -> dict[str, Any]: + return { + "id": self.id, + "modalities": self.manifest.data.get("modalities", {}), + "tasks": list(self.tasks), + } + + def overlay(self) -> dict[str, Any]: + tasks = {} + limits = {} + for name, task in self.tasks.items(): + tasks[name] = {"presets": self._presets[name]} if "presets" in task else {} + disabled = {key: False for key, kwarg in self._inputs[name].items() if kwarg is None} + if disabled: + limits[name] = {"inputs": disabled} + overlay: dict[str, Any] = {"tasks": tasks, "capabilities": self.capabilities} + if limits: + overlay["limits"] = limits + if "adapters" in self.capabilities: + overlay["adapters"] = {"max": MAX_ADAPTERS, "sources": list(ADAPTER_SOURCES)} + return overlay + + def discovery_document(self) -> bytes: + # The manifest bytes are spliced in untouched: the spec defines verification as a hash of the served bytes. + head: dict[str, Any] = {"spec": SPEC_VERSION} + if self.manifest.source is not None: + head["source"] = self.manifest.source + head["hash"] = self.manifest.hash + prefix = json.dumps(head)[:-1].encode() + return ( + prefix + + b', "manifest": ' + + self.manifest.raw + + b', "server": ' + + json.dumps(self.overlay()).encode() + + b"}" + ) + + def resolve(self, body: Any) -> ResolvedRequest: + if not isinstance(body, dict): + raise GMSError("invalid_request", "request body must be a JSON object") + for key in body: + if key not in _REQUEST_FIELDS: + raise GMSError("invalid_request", f"unknown request field {key!r}", f"/{key}") + if not isinstance(body.get("model"), str): + raise GMSError("invalid_request", "model is required and must be a string", "/model") + if body["model"] != self.id: + raise GMSError("not_found", f"model {body['model']!r} is not served here; this server hosts {self.id!r}") + inputs = body.get("inputs") + if not isinstance(inputs, dict): + raise GMSError("invalid_request", "inputs is required and must be an object", "/inputs") + + name = self._select_task(body.get("task"), inputs) + task = self.tasks[name] + values, explicit = self._resolve_parameters(name, task, body) + + kwargs: dict[str, Any] = {} + media: dict[str, tuple[str, Any]] = {} + for key, value in inputs.items(): + pointer = f"/inputs/{key}" + spec = task["inputs"].get(key) + if spec is None: + raise GMSError("invalid_request", f"task {name!r} declares no input {key!r}", pointer) + kwarg = self._inputs[name][key] + if kwarg is None: + raise GMSError("invalid_request", f"input {key!r} is disabled on this server", pointer) + if "active_when" in spec and not _is_active(spec["active_when"], values): + raise GMSError( + "invalid_request", f"input {key!r} is inactive unless {json.dumps(spec['active_when'])}", pointer + ) + if spec["type"] == "text": + if not isinstance(value, str): + raise GMSError("invalid_request", f"{key} must be a string, got {value!r}", pointer) + kwargs[kwarg] = value + continue + if isinstance(value, list): + if "multiple" not in spec: + raise GMSError("invalid_request", f"{key} accepts a single media value, not a list", pointer) + limit = spec["multiple"].get("max") + if limit is not None and len(value) > limit: + raise GMSError( + "invalid_request", f"{key} accepts at most {limit} items, got {len(value)}", pointer + ) + for index, item in enumerate(value): + _check_media(key, item, spec, f"{pointer}/{index}") + else: + _check_media(key, value, spec, pointer) + media[kwarg] = (key, value) + for key, spec in task["inputs"].items(): + if spec.get("required") and key not in inputs: + raise GMSError("invalid_request", f"task {name!r} requires input {key!r}", f"/inputs/{key}") + + seed = None + for key, value in values.items(): + spec = task["parameters"][key] + kwarg = self._parameters[name][key] + pointer = f"/parameters/{key}" + if "active_when" in spec and not _is_active(spec["active_when"], values): + if key in explicit: + raise GMSError( + "invalid_request", + f"parameter {key!r} is inactive unless {json.dumps(spec['active_when'])}", + pointer, + ) + continue + if kwarg is None: + # Clients that render every control resend defaults, so a default on a parameter this + # backend cannot set is the one supplied value that changes nothing by being skipped. + if key in explicit and value != spec.get("default"): + raise GMSError( + "not_implemented", f"parameter {key!r} is not supported by this server's backend", pointer + ) + continue + if key == "seed": + if not 0 <= value < 2**63: + raise GMSError("invalid_request", f"seed must be between 0 and 2**63 - 1, got {value}", pointer) + seed = value + continue + if key == "duration": + lattice_fps = spec["lattice"]["fps"] + if values.get("fps", lattice_fps) != lattice_fps: + raise GMSError( + "invalid_request", + f"duration is declared on a {lattice_fps} fps frame lattice; fps must be {lattice_fps} " + f"when duration is set, got {values['fps']}", + "/parameters/fps", + ) + kwargs[kwarg] = self._duration_to_frames(value, spec, pointer) + fps_kwarg = _bind("fps", _PARAMETER_KWARGS, {}, self._kwarg_names) + if fps_kwarg is not None: + kwargs[fps_kwarg] = lattice_fps + continue + kwargs[kwarg] = value + + output_formats = {kind: formats[0] for kind, formats in OUTPUT_FORMATS.items()} + output_format = body.get("output_format") + if output_format is not None: + kind = output_format.split("/")[0] if isinstance(output_format, str) else None + produced = {spec["type"] for spec in task["outputs"].values()} + if kind not in produced or output_format not in OUTPUT_FORMATS.get(kind, ()): + supported = sorted(f for k in produced for f in OUTPUT_FORMATS.get(k, ())) + raise GMSError( + "invalid_request", + f"output_format must be one of {supported} for task {name!r}, got {output_format!r}", + "/output_format", + ) + output_formats[kind] = output_format + + response_format = body.get("response_format", "url") + if response_format not in ("url", "b64_json"): + raise GMSError( + "invalid_request", + f"response_format must be 'url' or 'b64_json', got {response_format!r}", + "/response_format", + ) + + return ResolvedRequest( + task=name, + kwargs=kwargs, + media=media, + seed=seed, + output_formats=output_formats, + response_format=response_format, + adapters=self._resolve_adapters(body.get("adapters")), + ) + + def seeded(self, task: str) -> bool: + return self._parameters[task].get("seed") is not None + + def _select_task(self, requested: Any, inputs: dict[str, Any]) -> str: + if requested is not None: + if not isinstance(requested, str) or requested not in self.tasks: + raise GMSError( + "invalid_request", + f"task {requested!r} is not hosted for {self.id!r}; hosted tasks: {list(self.tasks)}", + "/task", + ) + return requested + + supplied = set(inputs) + candidates = [] + for name, task in self.tasks.items(): + accepted = {key for key, kwarg in self._inputs[name].items() if kwarg is not None} + required = {key for key, spec in task["inputs"].items() if spec.get("required")} + if required <= supplied <= accepted: + candidates.append(name) + if len(candidates) == 1: + return candidates[0] + if len(self.tasks) == 1: + return next(iter(self.tasks)) + raise GMSError( + "invalid_request", + f"could not infer the task from inputs {sorted(supplied)}; set `task` to one of {list(self.tasks)}", + "/task", + ) + + def _resolve_parameters(self, name: str, task: dict[str, Any], body: dict[str, Any]): + declared = task.get("parameters", {}) + explicit = body.get("parameters", {}) + if not isinstance(explicit, dict): + raise GMSError("invalid_request", "parameters must be an object", "/parameters") + for key, value in explicit.items(): + if key not in declared: + raise GMSError("invalid_request", f"task {name!r} declares no parameter {key!r}", f"/parameters/{key}") + _check_value(key, value, declared[key], f"/parameters/{key}") + + values = {key: spec["default"] for key, spec in declared.items() if "default" in spec} + preset = body.get("preset") + if preset is not None: + if preset not in self._presets[name]: + raise GMSError( + "invalid_request", + f"preset {preset!r} is not supported for task {name!r}; supported presets: {self._presets[name]}", + "/preset", + ) + values.update(task["presets"][preset].get("parameters", {})) + values.update(explicit) + return values, set(explicit) + + def _duration_to_frames(self, seconds: float, spec: dict[str, Any], pointer: str) -> int: + step, offset, fps = _frame_lattice(spec) + position = (seconds * fps - offset) / step + snap = spec["lattice"].get("snap", "nearest") + if snap == "reject" and not math.isclose(position, round(position), abs_tol=1e-9): + raise GMSError( + "invalid_request", + f"duration {seconds} is off the {spec['lattice']['frames']} frame lattice at {fps} fps", + pointer, + ) + index = math.ceil(position - 1e-9) if snap == "up" else round(position) + return step * max(index, 0) + offset + + def _resolve_adapters(self, adapters: Any) -> list[dict[str, Any]]: + if adapters is None: + return [] + if "adapters" not in self.capabilities: + raise GMSError("not_implemented", "this server does not apply adapters", "/adapters") + if not isinstance(adapters, list): + raise GMSError("invalid_request", "adapters must be an array", "/adapters") + if len(adapters) > MAX_ADAPTERS: + raise GMSError( + "invalid_request", f"at most {MAX_ADAPTERS} adapters per request, got {len(adapters)}", "/adapters" + ) + for index, adapter in enumerate(adapters): + pointer = f"/adapters/{index}" + if not isinstance(adapter, dict): + raise GMSError("invalid_request", "an adapter must be an object", pointer) + for key in adapter: + if key not in ("type", "path", "scale", "targets"): + raise GMSError("invalid_request", f"unknown adapter field {key!r}", f"{pointer}/{key}") + if not isinstance(adapter.get("type"), str) or adapter["type"] not in self.adapters: + raise GMSError( + "invalid_request", + f"adapter type must be one of {list(self.adapters)}, got {adapter.get('type')!r}", + f"{pointer}/type", + ) + path = adapter.get("path") + # Anything but a repo id would reach `load_lora_weights` as a path on the server's own disk. + if not isinstance(path, str) or not _HF_ADAPTER_PATH.fullmatch(path): + raise GMSError( + "invalid_request", + f"adapter path must be a Hub repo id of the form 'hf:/', got {path!r}", + f"{pointer}/path", + ) + scale = adapter.get("scale", 1.0) + if isinstance(scale, bool) or not isinstance(scale, (int, float)): + raise GMSError("invalid_request", f"adapter scale must be a number, got {scale!r}", f"{pointer}/scale") + valid_targets = self.adapters[adapter["type"]].get("targets", []) + targets = adapter.get("targets", {}) + if not isinstance(targets, dict): + raise GMSError("invalid_request", "adapter targets must be an object", f"{pointer}/targets") + for target, target_scale in targets.items(): + if target not in valid_targets: + raise GMSError( + "invalid_request", + f"adapter target must be one of {valid_targets}, got {target!r}", + f"{pointer}/targets/{target}", + ) + if isinstance(target_scale, bool) or not isinstance(target_scale, (int, float)): + raise GMSError( + "invalid_request", + f"adapter target scale must be a number, got {target_scale!r}", + f"{pointer}/targets/{target}", + ) + return adapters diff --git a/src/diffusers/dependency_versions_table.py b/src/diffusers/dependency_versions_table.py index 89d411a63a1d..a0975e4a5918 100644 --- a/src/diffusers/dependency_versions_table.py +++ b/src/diffusers/dependency_versions_table.py @@ -53,4 +53,6 @@ "opencv-python": "opencv-python", "timm": "timm", "flashpack": "flashpack", + "fastapi": "fastapi", + "uvicorn": "uvicorn", } diff --git a/tests/others/test_cli_commands.py b/tests/others/test_cli_commands.py index 5b27eaf372f4..e2d2d11f3cb6 100644 --- a/tests/others/test_cli_commands.py +++ b/tests/others/test_cli_commands.py @@ -53,7 +53,7 @@ ) -AVAILABLE_COMMANDS = ("env", "fp16_safetensors", "custom_blocks", "run", "schema", "skills") +AVAILABLE_COMMANDS = ("env", "fp16_safetensors", "custom_blocks", "generate", "run", "schema", "serve", "skills") class TestRunCommand: diff --git a/tests/others/test_cli_serve.py b/tests/others/test_cli_serve.py new file mode 100644 index 000000000000..03a31909fe5f --- /dev/null +++ b/tests/others/test_cli_serve.py @@ -0,0 +1,826 @@ +# Copyright 2026 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Unit tests for `diffusers-cli serve`. + +No weights are loaded: the diffusers backend runs a fake pipeline and the engine backend talks to a fake +SGLang / vLLM-Omni server through an httpx mock transport. +""" + +import base64 +import inspect +import io +import json +import subprocess +import sys +import threading +from argparse import ArgumentParser, Namespace +from pathlib import Path +from types import SimpleNamespace + +import numpy as np +import pytest +import torch +from huggingface_hub.utils import httpx +from PIL import Image + +from diffusers.commands.generate import GenerateCommand +from diffusers.commands.serve import ServeCommand +from diffusers.commands.serve.backends import ( + DiffusersBackend, + EngineBackend, + backend_capabilities, + backend_kwargs, + backend_output_types, +) +from diffusers.commands.serve.generations import Generation, GenerationQueue +from diffusers.commands.serve.manifest import GMSError, Manifest, ServedModel, blocks_signature, derive_manifest +from diffusers.modular_pipelines import InputParam, ModularPipelineBlocks, OutputParam + + +MODEL_ID = "test-org/fake-model" +WAIT = {"Prefer": "wait=30"} + + +class FakePipeline: + _lora_loadable_modules = ["transformer", "text_encoder"] + device = torch.device("cpu") + num_timesteps = 0 + + def __init__(self): + self.calls = [] + self.lora_calls = [] + self.release = threading.Event() + self.release.set() + self.started = threading.Event() + self.error = None + self.result = None + + def __call__( + self, + prompt: str | list[str] = None, + negative_prompt: str | list[str] | None = None, + image: Image.Image | None = None, + height: int | None = None, + width: int | None = None, + num_inference_steps: int = 4, + sigmas: list[float] | None = None, + guidance_scale: float = 3.5, + num_images_per_prompt: int | None = 1, + generator: torch.Generator | None = None, + latents: torch.Tensor | None = None, + output_type: str | None = "pil", + return_dict: bool = True, + callback_on_step_end=None, + callback_on_step_end_tensor_inputs: list[str] = ["latents"], + ): + self.calls.append( + { + "prompt": prompt, + "negative_prompt": negative_prompt, + "image": image, + "height": height, + "width": width, + "num_inference_steps": num_inference_steps, + "guidance_scale": guidance_scale, + "generator": generator, + } + ) + self.started.set() + self.release.wait(timeout=30) + if self.error is not None: + raise self.error + self.num_timesteps = num_inference_steps + self._interrupt = False + for step in range(num_inference_steps): + if self._interrupt: + break + callback_on_step_end(self, step, 0, {}) + if self.result is not None: + return self.result + size = (width or 32, height or 32) + return SimpleNamespace(images=[Image.new("RGB", size, "red") for _ in range(num_images_per_prompt)]) + + def load_lora_weights(self, repo, adapter_name=None): + self.lora_calls.append(("load", repo, adapter_name)) + + def set_adapters(self, names, adapter_weights=None): + self.lora_calls.append(("set", names, adapter_weights)) + + def unload_lora_weights(self): + self.lora_calls.append(("unload",)) + + +FAKE_SIGNATURE = inspect.signature(FakePipeline.__call__) + + +class FakeImageBlock(ModularPipelineBlocks): + """A one-block modular pipeline that declares its inputs the way the built-in blocks do, mostly untyped.""" + + model_name = "fake" + + @property + def inputs(self): + return [ + InputParam("prompt"), + InputParam("image"), + InputParam("width"), + InputParam("num_inference_steps", default=4), + InputParam("generator"), + ] + + @property + def intermediate_outputs(self): + return [OutputParam("images")] + + def __call__(self, components, state): + block_state = self.get_block_state(state) + size = (block_state.width or 32, 32) + block_state.images = [Image.new("RGB", size, "red")] + self.set_block_state(state, block_state) + return components, state + + +def _args(**overrides): + values = {"backend": "diffusers", "compile": None, "lora": None, "fps": 8, "sampling_rate": None} + return Namespace(**{**values, **overrides}) + + +def _served(backend="diffusers", manifest=None, pipeline_cls=FakePipeline, **arg_overrides): + args = _args(backend=backend, **arg_overrides) + signature = inspect.signature(pipeline_cls.__call__) + kwarg_names = backend_kwargs(backend, signature) + if manifest is None: + manifest = derive_manifest(MODEL_ID, pipeline_cls, signature, kwarg_names) + raw = manifest if isinstance(manifest, bytes) else json.dumps(manifest).encode() + return ServedModel( + Manifest(raw=raw, data=json.loads(raw)), + kwarg_names, + backend_output_types(backend), + backend_capabilities(args, pipeline_cls), + revision="abc123", + ) + + +@pytest.fixture +def serve(tmp_path, monkeypatch): + """Build a test client around a `DiffusersBackend` running `FakePipeline`. Returns `(client, pipeline)`.""" + pytest.importorskip("fastapi", reason="`diffusers-cli serve` needs the `diffusers[serve]` extra") + from fastapi.testclient import TestClient + + from diffusers.commands.serve.app import create_app + + def _build(manifest=None, app_options=None, **arg_overrides): + pipeline = FakePipeline() + monkeypatch.setattr("diffusers.commands.serve.backends._load_pipeline", lambda args: pipeline) + served = _served(manifest=manifest, **arg_overrides) + backend = DiffusersBackend(_args(**arg_overrides), served) + app = create_app(served, GenerationQueue(backend, str(tmp_path)), **(app_options or {})) + return TestClient(app), pipeline + + return _build + + +LATTICE_MANIFEST = { + "id": MODEL_ID, + "modalities": {"in": ["image"], "out": ["video"]}, + "tasks": { + "text_to_video": { + "inputs": { + "prompt": {"type": "text", "required": True}, + "negative_prompt": {"type": "text", "active_when": {"parameter": "guidance_scale", "gt": 1}}, + "last_image": {"type": "image"}, + }, + "outputs": {"video": {"type": "video"}}, + "parameters": { + "duration": {"type": "number", "min": 1, "max": 20, "lattice": {"frames": "8k+1", "fps": 24}}, + "width": {"type": "integer", "default": 768, "min": 512, "max": 1536, "multiple_of": 64}, + "steps": {"type": "integer", "default": 30, "min": 1, "max": 100}, + "fps": {"type": "number"}, + "guidance_scale": {"type": "number", "default": 3.0, "min": 1.0, "max": 10.0}, + "guidance_type": {"type": "string", "enum": ["cfg", "distilled"], "default": "cfg"}, + "seed": {"type": "integer"}, + "video_decoder": {"type": "string", "enum": ["conv", "diffusion"], "default": "conv"}, + }, + "presets": { + "default": {}, + "distilled": {"parameters": {"steps": 8, "guidance_scale": 1.0}}, + "quality": {"parameters": {"steps": 40, "video_decoder": "diffusion"}}, + }, + } + }, +} + + +class FakeVideoPipeline: + def __call__( + self, + prompt: str = None, + negative_prompt: str | None = None, + width: int = 768, + num_frames: int = 121, + frame_rate: float = 24.0, + num_inference_steps: int = 40, + guidance_scale: float = 4.0, + generator: torch.Generator | None = None, + ): + pass + + +class TestManifest: + def test_derive_manifest_from_signature(self): + manifest = derive_manifest(MODEL_ID, FakePipeline, FAKE_SIGNATURE, backend_kwargs("diffusers", FAKE_SIGNATURE)) + + # An optional `image` kwarg yields one task without it and one that requires it. + assert list(manifest["tasks"]) == ["text_to_image", "image_to_image"], manifest["tasks"].keys() + text_to_image = manifest["tasks"]["text_to_image"] + assert text_to_image["inputs"] == { + "prompt": {"type": "text", "required": True}, + "negative_prompt": {"type": "text"}, + } + assert manifest["tasks"]["image_to_image"]["inputs"]["image"] == {"type": "image", "required": True} + assert text_to_image["outputs"] == {"image": {"type": "image"}} + # Core vocabulary names replace the pipeline's own; tensors, callbacks and output plumbing are dropped. + assert text_to_image["parameters"] == { + "height": {"type": "integer"}, + "width": {"type": "integer"}, + "steps": {"type": "integer", "default": 4}, + "sigmas": {"type": "array", "items": {"type": "number"}}, + "guidance_scale": {"type": "number", "default": 3.5}, + "num_outputs": {"type": "integer", "default": 1}, + "seed": {"type": "integer"}, + } + assert text_to_image["x-diffusers"] == { + "parameters": {"steps": "num_inference_steps", "num_outputs": "num_images_per_prompt", "seed": "generator"} + } + assert manifest["adapters"] == {"lora": {"targets": ["transformer", "text_encoder"]}} + + @pytest.mark.parametrize( + "body, code, pointer", + [ + ({"parameters": {"cfg": 3}}, "invalid_request", "/parameters/cfg"), + ({"parameters": {"width": 700}}, "invalid_request", "/parameters/width"), + ({"parameters": {"steps": 500}}, "invalid_request", "/parameters/steps"), + ({"parameters": {"steps": "8"}}, "invalid_request", "/parameters/steps"), + ({"inputs": {}}, "invalid_request", "/inputs/prompt"), + ({"inputs": {"prompt": "a cat", "caption": "x"}}, "invalid_request", "/inputs/caption"), + ({"task": "image_to_video"}, "invalid_request", "/task"), + ({"task": []}, "invalid_request", "/task"), + ({"parameters": []}, "invalid_request", "/parameters"), + ({"parameters": {"seed": 10**30}}, "invalid_request", "/parameters/seed"), + ({"parameters": {"duration": float("nan")}}, "invalid_request", "/parameters/duration"), + # The lattice pins duration to 24 fps, so another fps would deliver a different length. + ({"parameters": {"duration": 4, "fps": 12}}, "invalid_request", "/parameters/fps"), + ({"steps": 8}, "invalid_request", "/steps"), + ({"preset": "quality"}, "invalid_request", "/preset"), + ({"model": "someone/else"}, "not_found", None), + ({"adapters": [{"type": "lora", "path": "hf:a/b"}]}, "not_implemented", "/adapters"), + ({"output_format": "video/webm"}, "invalid_request", "/output_format"), + # `negative_prompt` is only honored while guidance_scale > 1. + ( + {"inputs": {"prompt": "a cat", "negative_prompt": "blurry"}, "preset": "distilled"}, + "invalid_request", + "/inputs/negative_prompt", + ), + # `last_image` has no matching pipeline kwarg, so the server overlay disables it. + ( + {"inputs": {"prompt": "a cat", "last_image": {"type": "image", "url": "https://x/y.png"}}}, + "invalid_request", + "/inputs/last_image", + ), + # A parameter the pipeline cannot set is refused rather than dropped, unless it is the default. + ({"parameters": {"video_decoder": "diffusion"}}, "not_implemented", "/parameters/video_decoder"), + ], + ) + def test_resolve_rejects_invalid_requests(self, body, code, pointer): + served = _served(manifest=LATTICE_MANIFEST, pipeline_cls=FakeVideoPipeline) + with pytest.raises(GMSError) as error: + served.resolve({"model": MODEL_ID, "inputs": {"prompt": "a cat"}, **body}) + assert (error.value.code, error.value.pointer) == (code, pointer), error.value.message + + def test_resolve_rejects_non_http_media_urls(self): + served = _served() + body = {"model": MODEL_ID, "inputs": {"prompt": "a cat", "image": {"type": "image", "url": "/etc/passwd"}}} + with pytest.raises(GMSError) as error: + served.resolve(body) + assert error.value.pointer == "/inputs/image/url", error.value.message + + @pytest.mark.parametrize("path", ["hf:/etc/passwd", "hf:../loras/x", "/etc/passwd", "https://x/y.safetensors"]) + def test_resolve_rejects_adapter_paths_that_are_not_repo_ids(self, path): + body = {"model": MODEL_ID, "inputs": {"prompt": "a cat"}, "adapters": [{"type": "lora", "path": path}]} + with pytest.raises(GMSError) as error: + _served().resolve(body) + assert (error.value.code, error.value.pointer) == ("invalid_request", "/adapters/0/path"), error.value.message + + def test_resolve_maps_request_to_pipeline_kwargs(self): + served = _served(manifest=LATTICE_MANIFEST, pipeline_cls=FakeVideoPipeline) + resolved = served.resolve( + { + "model": MODEL_ID, + "preset": "distilled", + "inputs": {"prompt": "a cat"}, + "parameters": {"duration": 8, "seed": 7, "guidance_type": "cfg"}, + } + ) + # 8 s at 24 fps is 192 frames; the nearest point on the 8k+1 lattice is 193. The preset overrides the + # task defaults, and `guidance_type` at its default is accepted although the pipeline has no such kwarg. + assert resolved.kwargs == { + "prompt": "a cat", + "num_frames": 193, + "frame_rate": 24, + "width": 768, + "num_inference_steps": 8, + "guidance_scale": 1.0, + }, resolved.kwargs + assert resolved.seed == 7 + assert resolved.task == "text_to_video" + + def test_discovery_overlay(self): + served = _served(manifest=LATTICE_MANIFEST, pipeline_cls=FakeVideoPipeline) + assert served.overlay() == { + "tasks": {"text_to_video": {"presets": ["default", "distilled"]}}, + "capabilities": ["sse", "wait"], + "limits": {"text_to_video": {"inputs": {"last_image": False}}}, + } + + +class TestServeApp: + def test_generation_lifecycle(self, serve): + client, pipeline = serve() + body = { + "model": MODEL_ID, + "inputs": {"prompt": "a cat"}, + "parameters": {"width": 64, "height": 48, "steps": 3, "seed": 11}, + } + response = client.post("/v1/generations", json=body, headers=WAIT) + assert response.status_code == 201, response.text + generation = response.json() + assert generation["status"] == "complete", generation + assert generation["seed"] == 11 + assert generation["revision"] == "abc123" + assert generation["progress"] is None + assert generation["usage"]["compute_seconds"] >= 0 + + call = pipeline.calls[0] + assert (call["prompt"], call["width"], call["height"], call["num_inference_steps"]) == ("a cat", 64, 48, 3) + assert torch.equal(call["generator"].get_state(), torch.Generator().manual_seed(11).get_state()) + + (artifact,) = generation["outputs"]["image"] + assert {k: artifact[k] for k in ("media_type", "width", "height")} == { + "media_type": "image/png", + "width": 64, + "height": 48, + } + image = Image.open(io.BytesIO(client.get(artifact["url"]).content)) + assert (image.format, image.size) == ("PNG", (64, 48)) + + events = client.get(f"/v1/generations/{generation['id']}/events").text + assert events.count("event: progress") == 3, events + assert 'data: {"step": 3, "total_steps": 3, "fraction": 1.0}' in events + assert events.rstrip().split("\n\n")[-1].startswith("id: 4\nevent: complete\n") + # Resuming after the last progress event replays only what followed it. + resumed = client.get(f"/v1/generations/{generation['id']}/events", headers={"Last-Event-ID": "3"}).text + assert "event: progress" not in resumed and "event: complete" in resumed, resumed + + assert client.delete(f"/v1/generations/{generation['id']}").status_code == 204 + missing = client.get(f"/v1/generations/{generation['id']}") + assert (missing.status_code, missing.json()["error"]["code"]) == (404, "not_found") + assert client.get(artifact["url"]).status_code == 404 + + def test_inline_response_and_media_input(self, serve): + client, pipeline = serve() + buffer = io.BytesIO() + Image.new("RGB", (8, 8), "blue").save(buffer, format="PNG") + body = { + "model": MODEL_ID, + "inputs": { + "prompt": "a cat", + "image": { + "type": "image", + "base64": base64.b64encode(buffer.getvalue()).decode(), + "media_type": "image/png", + }, + }, + "output_format": "image/jpeg", + "response_format": "b64_json", + } + generation = client.post("/v1/generations", json=body, headers=WAIT).json() + assert generation["status"] == "complete", generation + # The task is inferred from the inputs: only `image_to_image` accepts an image. + assert pipeline.calls[0]["image"].size == (8, 8) + (artifact,) = generation["outputs"]["image"] + assert artifact["media_type"] == "image/jpeg" and "url" not in artifact + assert Image.open(io.BytesIO(base64.b64decode(artifact["base64"]))).format == "JPEG" + + def test_joint_video_and_audio_outputs(self, serve, monkeypatch): + monkeypatch.setattr( + "diffusers.commands.serve.backends.export_to_video", + lambda frames, path, fps: Path(path).write_bytes(b"mp4"), + ) + manifest = { + "id": MODEL_ID, + "tasks": { + "text_to_video": { + "inputs": {"prompt": {"type": "text", "required": True}}, + "outputs": {"video": {"type": "video"}, "audio": {"type": "audio"}}, + } + }, + } + client, pipeline = serve(manifest=manifest) + pipeline.result = SimpleNamespace( + frames=[[Image.new("RGB", (32, 24)) for _ in range(16)]], + audio=np.zeros((1, 2, 8000), dtype=np.float32), + ) + body = {"model": MODEL_ID, "inputs": {"prompt": "a cat"}} + generation = client.post("/v1/generations", json=body, headers=WAIT).json() + assert generation["status"] == "complete", generation + + # The fake pipeline takes no fps argument, so the video is encoded at the `--fps` default. + (video,) = generation["outputs"]["video"] + (audio,) = generation["outputs"]["audio"] + assert {k: v for k, v in video.items() if k != "url"} == { + "media_type": "video/mp4", + "width": 32, + "height": 24, + "duration": 2.0, + "fps": 8, + } + assert {k: v for k, v in audio.items() if k != "url"} == { + "media_type": "audio/wav", + "duration": 0.5, + "sample_rate": 16000, + } + assert client.get(audio["url"]).content[:4] == b"RIFF" + + def test_validation_error_envelope(self, serve): + client, _ = serve() + body = {"model": MODEL_ID, "inputs": {"prompt": "a cat"}, "parameters": {"cfg_scale": 3}} + response = client.post("/v1/generations", json=body) + assert response.status_code == 400 + assert response.json() == { + "error": { + "code": "invalid_request", + "message": "task 'text_to_image' declares no parameter 'cfg_scale'", + "pointer": "/parameters/cfg_scale", + } + } + + def test_failed_generation_returns_error_envelope(self, serve): + client, pipeline = serve() + body = {"model": MODEL_ID, "inputs": {"prompt": "a cat"}} + envelope = {"error": {"code": "internal", "message": "RuntimeError: CUDA out of memory"}} + + pipeline.error = RuntimeError("CUDA out of memory") + created = client.post("/v1/generations", json=body) + assert created.status_code == 201 + # The event stream closes once the generation is done, so reading it to the end waits for the failure. + events = client.get(f"/v1/generations/{created.json()['id']}/events").text + assert 'event: error\ndata: {"code": "internal", "message": "RuntimeError: CUDA out of memory"}' in events + response = client.get(f"/v1/generations/{created.json()['id']}") + assert (response.status_code, response.json()) == (500, envelope) + # There is no `failed` status: a create that waited for the failure returns the envelope as well. + waited = client.post("/v1/generations", json=body, headers=WAIT) + assert (waited.status_code, waited.json()) == (500, envelope) + + # A SystemExit from the backend must not end the worker: the next generation still runs. + pipeline.error = SystemExit("unsupported") + assert client.post("/v1/generations", json=body, headers=WAIT).status_code == 500 + pipeline.error = None + assert client.post("/v1/generations", json=body, headers=WAIT).json()["status"] == "complete" + + def test_health(self, serve): + client, _ = serve() + response = client.get("/health") + assert (response.status_code, response.json()) == (200, {"status": "ok"}) + + def test_request_id_is_generated_or_echoed(self, serve): + client, _ = serve() + assert client.get("/health").headers["x-request-id"], "a request without an id should be given one" + echoed = client.get("/v1/models/unknown", headers={"x-request-id": "req-1"}) + assert (echoed.status_code, echoed.headers["x-request-id"]) == (404, "req-1") + + def test_default_seed_applies_when_request_omits_seed(self, serve): + client, _ = serve(app_options={"default_seed": 7}) + body = {"model": MODEL_ID, "inputs": {"prompt": "a cat"}} + assert client.post("/v1/generations", json=body, headers=WAIT).json()["seed"] == 7 + body["parameters"] = {"seed": 11} + assert client.post("/v1/generations", json=body, headers=WAIT).json()["seed"] == 11 + + def test_cors_is_opt_in(self, serve): + origin = {"Origin": "http://example.com"} + client, _ = serve() + assert "access-control-allow-origin" not in client.get("/health", headers=origin).headers + client, _ = serve(app_options={"enable_cors": True}) + assert client.get("/health", headers=origin).headers["access-control-allow-origin"] == "*" + + def test_cancel_queued_and_running(self, serve): + client, pipeline = serve() + pipeline.release.clear() + body = {"model": MODEL_ID, "inputs": {"prompt": "a cat"}} + running = client.post("/v1/generations", json=body).json() + queued = client.post("/v1/generations", json=body).json() + assert queued["status"] == "queued", queued + assert pipeline.started.wait(timeout=30), "the worker never picked up the first generation" + + assert client.post(f"/v1/generations/{queued['id']}/cancel").json()["status"] == "canceled" + client.post(f"/v1/generations/{running['id']}/cancel") + pipeline.release.set() + + # The stream ends once the generation is done; a canceled generation emits no `complete` event. + events = client.get(f"/v1/generations/{running['id']}/events").text + assert "event: complete" not in events, events + canceled = client.get(f"/v1/generations/{running['id']}").json() + assert (canceled["status"], canceled["outputs"]) == ("canceled", None) + assert len(pipeline.calls) == 1, "the canceled queued generation must never reach the pipeline" + + def test_request_adapters_are_loaded_then_unloaded(self, serve): + client, pipeline = serve() + body = { + "model": MODEL_ID, + "inputs": {"prompt": "a cat"}, + "adapters": [ + {"type": "lora", "path": "hf:someone/style", "scale": 0.8}, + {"type": "lora", "path": "hf:someone/detail", "scale": 0.5, "targets": {"text_encoder": 0.1}}, + ], + } + generation = client.post("/v1/generations", json=body, headers=WAIT).json() + assert generation["status"] == "complete", generation + assert pipeline.lora_calls == [ + ("load", "someone/style", "request_0"), + ("load", "someone/detail", "request_1"), + ("set", ["request_0", "request_1"], [0.8, {"transformer": 0.5, "text_encoder": 0.1}]), + ("unload",), + ] + + def test_adapters_not_advertised_when_compiled(self, serve): + client, _ = serve(compile="{}") + discovery = client.get(f"/v1/models/{MODEL_ID}").json() + assert "adapters" not in discovery["server"]["capabilities"] + body = {"model": MODEL_ID, "inputs": {"prompt": "a cat"}, "adapters": [{"type": "lora", "path": "hf:a/b"}]} + response = client.post("/v1/generations", json=body) + assert (response.status_code, response.json()["error"]["code"]) == (501, "not_implemented") + + def test_discovery_serves_manifest_bytes_verbatim(self, serve): + manifest = derive_manifest(MODEL_ID, FakePipeline, FAKE_SIGNATURE, backend_kwargs("diffusers", FAKE_SIGNATURE)) + raw = json.dumps(manifest, indent=3).encode() + client, _ = serve(manifest=raw) + + response = client.get(f"/v1/models/{MODEL_ID}") + assert raw in response.content + discovery = response.json() + assert discovery["spec"] == "gms/0.1" + assert discovery["manifest"] == manifest + assert discovery["server"]["tasks"] == {"text_to_image": {}, "image_to_image": {}} + assert discovery["server"]["adapters"] == {"max": 4, "sources": ["hf"]} + assert client.get("/v1/models").json() == { + "data": [ + { + "id": MODEL_ID, + "modalities": {"in": ["image"], "out": ["image"]}, + "tasks": ["text_to_image", "image_to_image"], + } + ] + } + + def test_unadvertised_capabilities_return_not_implemented(self, serve): + client, _ = serve() + for response in ( + client.post("/v1/uploads", json={}), + client.get("/v1/generations/gen_x/outputs/video/stream"), + ): + assert (response.status_code, response.json()["error"]["code"]) == (501, "not_implemented") + + +class FakeEngine: + """The image and video routes SGLang and vLLM-Omni share, recording what the backend sends.""" + + def __init__(self): + self.requests = [] + self.polls = 0 + + def __call__(self, request): + content_type = request.headers.get("content-type", "") + self.requests.append((request.method, request.url.path, content_type.split(";")[0], request.content)) + path = request.url.path + if path in ("/v1/images/generations", "/v1/images/edits"): + buffer = io.BytesIO() + Image.new("RGB", (16, 8), "green").save(buffer, format="PNG") + return httpx.Response(200, json={"data": [{"b64_json": base64.b64encode(buffer.getvalue()).decode()}]}) + if path == "/v1/videos" and request.method == "POST": + return httpx.Response(200, json={"id": "video_1", "status": "queued"}) + if path == "/v1/videos/video_1" and request.method == "GET": + self.polls += 1 + if self.polls < 2: + return httpx.Response(200, json={"id": "video_1", "status": "in_progress"}) + return httpx.Response(200, json={"id": "video_1", "status": "completed", "fps": 16.0, "duration_s": 5.0}) + if path == "/v1/videos/video_1/content": + return httpx.Response(200, content=b"mp4-bytes") + if path == "/v1/videos/video_1" and request.method == "DELETE": + return httpx.Response(200, json={"id": "video_1", "deleted": True}) + return httpx.Response(404, text="no such route") + + +class FakeTextToVideoPipeline: + def __call__( + self, + prompt: str = None, + image: Image.Image = None, + height: int = 480, + width: int = 832, + num_frames: int = 81, + num_inference_steps: int = 50, + generator: torch.Generator | None = None, + ): + pass + + +class TestModularPipeline: + def test_generation_runs_a_modular_pipeline(self, tmp_path, monkeypatch): + pytest.importorskip("fastapi", reason="`diffusers-cli serve` needs the `diffusers[serve]` extra") + from fastapi.testclient import TestClient + + from diffusers.commands.serve.app import create_app + + pipeline = FakeImageBlock().init_pipeline() + monkeypatch.setattr("diffusers.commands.serve.backends._load_pipeline", lambda args: pipeline) + signature = blocks_signature(pipeline.blocks) + kwarg_names = backend_kwargs("diffusers", signature) + manifest = derive_manifest(MODEL_ID, type(pipeline), signature, kwarg_names) + assert list(manifest["tasks"]) == ["text_to_image", "image_to_image"], manifest["tasks"].keys() + assert manifest["tasks"]["text_to_image"]["inputs"] == {"prompt": {"type": "text", "required": True}} + assert manifest["tasks"]["text_to_image"]["parameters"] == { + "steps": {"type": "integer", "default": 4}, + "seed": {"type": "integer"}, + } + + raw = json.dumps(manifest).encode() + served = ServedModel( + Manifest(raw=raw, data=manifest), + kwarg_names, + backend_output_types("diffusers"), + backend_capabilities(_args(), type(pipeline)), + revision=None, + ) + backend = DiffusersBackend(_args(), served) + client = TestClient(create_app(served, GenerationQueue(backend, str(tmp_path)))) + + body = {"model": MODEL_ID, "inputs": {"prompt": "a cat"}, "parameters": {"seed": 3}} + generation = client.post("/v1/generations", json=body, headers=WAIT).json() + assert generation["status"] == "complete", generation + assert generation["seed"] == 3 + (artifact,) = generation["outputs"]["image"] + assert (artifact["media_type"], artifact["width"], artifact["height"]) == ("image/png", 32, 32) + + +class TestEngineBackend: + def _generate(self, tmp_path, monkeypatch, served, body): + monkeypatch.setattr("diffusers.commands.serve.backends._POLL_SECONDS", 0.0) + engine = FakeEngine() + client = httpx.Client(base_url="http://engine", transport=httpx.MockTransport(engine)) + backend = EngineBackend("vllm", served, client) + resolved = served.resolve({"model": MODEL_ID, **body}) + generation = Generation(served.id, None, resolved, seeded=served.seeded(resolved.task)) + generation.directory = tmp_path + return engine, backend.generate(generation) + + def test_exited_engine_is_unhealthy(self, tmp_path): + pytest.importorskip("fastapi", reason="`diffusers-cli serve` needs the `diffusers[serve]` extra") + from fastapi.testclient import TestClient + + from diffusers.commands.serve.app import create_app + + process = subprocess.Popen([sys.executable, "-c", "raise SystemExit(3)"]) + process.wait() + served = _served(backend="vllm") + backend = EngineBackend("vllm", served, httpx.Client(base_url="http://engine"), process) + response = TestClient(create_app(served, GenerationQueue(backend, str(tmp_path)))).get("/health") + expected = {"status": "unhealthy", "reason": "the vllm backend exited with code 3"} + assert (response.status_code, response.json()) == (503, expected) + + def test_derived_manifest_keeps_only_engine_kwargs(self): + manifest = derive_manifest(MODEL_ID, FakePipeline, FAKE_SIGNATURE, backend_kwargs("vllm", FAKE_SIGNATURE)) + assert "sigmas" not in manifest["tasks"]["text_to_image"]["parameters"] + assert "steps" in manifest["tasks"]["text_to_image"]["parameters"] + + def test_text_to_image_request_translation(self, tmp_path, monkeypatch): + body = { + "inputs": {"prompt": "a cat", "negative_prompt": "blurry"}, + "parameters": {"width": 1024, "height": 768, "steps": 20, "num_outputs": 2, "seed": 5}, + "output_format": "image/webp", + } + engine, outputs = self._generate(tmp_path, monkeypatch, _served(backend="vllm"), body) + + ((method, path, content_type, content),) = engine.requests + assert (method, path, content_type) == ("POST", "/v1/images/generations", "application/json") + assert json.loads(content) == { + "prompt": "a cat", + "negative_prompt": "blurry", + "num_inference_steps": 20, + "guidance_scale": 3.5, + "n": 2, + "seed": 5, + "size": "1024x768", + "response_format": "b64_json", + "output_format": "webp", + } + assert outputs == {"image": [{"file": "image-0.webp", "media_type": "image/webp", "width": 16, "height": 8}]} + assert (tmp_path / "image-0.webp").is_file() + + def test_image_input_uses_multipart_edits(self, tmp_path, monkeypatch): + image = {"type": "image", "base64": base64.b64encode(b"png-bytes").decode(), "media_type": "image/png"} + body = {"inputs": {"prompt": "make it blue", "image": image}, "parameters": {"seed": 5}} + engine, _ = self._generate(tmp_path, monkeypatch, _served(backend="sglang"), body) + + ((method, path, content_type, content),) = engine.requests + assert (method, path, content_type) == ("POST", "/v1/images/edits", "multipart/form-data") + assert b'name="image"; filename="image-0.png"' in content and b"png-bytes" in content + assert b'name="prompt"\r\n\r\nmake it blue' in content + + def test_video_job_is_polled_downloaded_and_deleted(self, tmp_path, monkeypatch): + served = _served(backend="vllm", pipeline_cls=FakeTextToVideoPipeline) + body = {"task": "text_to_video", "inputs": {"prompt": "a cat"}, "parameters": {"num_frames": 33, "seed": 5}} + engine, outputs = self._generate(tmp_path, monkeypatch, served, body) + + assert [(method, path) for method, path, _, _ in engine.requests] == [ + ("POST", "/v1/videos"), + ("GET", "/v1/videos/video_1"), + ("GET", "/v1/videos/video_1"), + ("GET", "/v1/videos/video_1/content"), + ("DELETE", "/v1/videos/video_1"), + ] + _, _, content_type, content = engine.requests[0] + assert content_type == "multipart/form-data" + assert b'name="num_frames"\r\n\r\n33' in content and b'name="size"\r\n\r\n832x480' in content + assert outputs == {"video": [{"file": "video-0.mp4", "media_type": "video/mp4", "fps": 16.0, "duration": 5.0}]} + assert (tmp_path / "video-0.mp4").read_bytes() == b"mp4-bytes" + + def test_audio_tasks_are_not_hosted(self): + manifest = { + "id": MODEL_ID, + "tasks": { + "text_to_audio": {"inputs": {"prompt": {"type": "text"}}, "outputs": {"audio": {"type": "audio"}}} + }, + } + assert _served(backend="vllm", manifest=manifest).tasks == {} + + +class TestGenerateCommand: + def _run(self, monkeypatch, client, argv): + parser = ArgumentParser() + GenerateCommand.register_subcommand(parser.add_subparsers()) + monkeypatch.setattr("diffusers.commands.generate._POLL_SECONDS", 0.01) + monkeypatch.setattr(GenerateCommand, "_client", lambda self: client) + GenerateCommand(parser.parse_args(["generate", "--url", "http://testserver", *argv])).run() + + def test_generate_saves_outputs(self, serve, tmp_path, monkeypatch): + client, pipeline = serve() + output = tmp_path / "saved" + argv = ["--inputs", '{"prompt": "a cat"}', "--parameters", '{"width": 64, "seed": 5}', "-o", str(output)] + self._run(monkeypatch, client, argv) + + assert pipeline.calls[0]["prompt"] == "a cat" + assert [path.name for path in output.iterdir()] == ["image-0.png"], list(output.iterdir()) + assert Image.open(output / "image-0.png").size == (64, 32) + + def test_local_file_is_sent_as_a_media_value(self, serve, tmp_path, monkeypatch): + client, pipeline = serve() + source = tmp_path / "cat.png" + Image.new("RGB", (8, 8), "blue").save(source) + inputs = json.dumps({"prompt": "make it grey", "image": str(source)}) + self._run(monkeypatch, client, ["--inputs", inputs, "-o", str(tmp_path / "saved")]) + + assert pipeline.calls[0]["image"].size == (8, 8) + + def test_server_error_is_reported(self, serve, tmp_path, monkeypatch): + client, _ = serve() + argv = ["--inputs", '{"prompt": "a cat"}', "--parameters", '{"cfg": 1}', "-o", str(tmp_path)] + with pytest.raises(SystemExit, match=r"HTTP 400: invalid_request: .*\(at /parameters/cfg\)"): + self._run(monkeypatch, client, argv) + + +class TestServeCommand: + def _parse(self, argv): + parser = ArgumentParser() + ServeCommand.register_subcommand(parser.add_subparsers()) + return parser.parse_args(["serve", *argv]) + + def test_engine_backend_rejects_pipeline_flags(self): + pytest.importorskip("fastapi", reason="`diffusers-cli serve` needs the `diffusers[serve]` extra") + args = self._parse(["-m", "org/model", "--backend", "vllm", "--dtype", "bf16", "--vae-tiling"]) + with pytest.raises(SystemExit, match="--dtype, --vae-tiling configure the in-process pipeline"): + ServeCommand(args).run() + + def test_context_parallel_is_rejected(self): + pytest.importorskip("fastapi", reason="`diffusers-cli serve` needs the `diffusers[serve]` extra") + with pytest.raises(SystemExit, match="--context-parallel is not supported"): + ServeCommand(self._parse(["-m", "org/model", "--context-parallel"])).run() From 3e4aa57c2842e1ff01edbde8dec80e2147c341be Mon Sep 17 00:00:00 2001 From: DN6 Date: Fri, 9 Oct 2026 13:07:51 +0530 Subject: [PATCH 02/21] add remote serve --- src/diffusers/commands/serve/remote.py | 156 +++++++++++++++++++++++++ 1 file changed, 156 insertions(+) create mode 100644 src/diffusers/commands/serve/remote.py diff --git a/src/diffusers/commands/serve/remote.py b/src/diffusers/commands/serve/remote.py new file mode 100644 index 000000000000..ebabe97ab57e --- /dev/null +++ b/src/diffusers/commands/serve/remote.py @@ -0,0 +1,156 @@ +# Copyright 2026 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""`diffusers-cli serve --remote` — run the server in a Hugging Face Sandbox, reachable through the sandbox proxy.""" + +from __future__ import annotations + +import shlex +import sys +import threading +from argparse import ArgumentParser, Namespace +from pathlib import Path +from typing import Any + +from huggingface_hub import Sandbox, get_token +from huggingface_hub.cli._output import out +from huggingface_hub.utils import httpx + +from ...utils import logging +from ..run import _DEFAULT_REMOTE_DEPS, _DEFAULT_REMOTE_IMAGE, _build_task_kwargs, _kwargs_to_argv + + +logger = logging.get_logger("diffusers-cli/serve") + +# The port the server binds inside the sandbox. `generate --sandbox-id` reaches it through the proxy. +SANDBOX_PORT = 8000 +_SANDBOX_MANIFEST_PATH = "/tmp/diffusers-cli/gms.json" +_SERVE_DEPS = ("fastapi", "uvicorn") +_READY_POLL_SECONDS = 5.0 + +# Flags that say how the sandbox is set up, or that the sandbox server sets itself. +_LOCAL_KEYS = ("remote", "flavor", "dependencies", "namespace", "image", "idle_timeout", "func", "format") +_SANDBOX_SET_KEYS = ("host", "port", "public_url", "output_dir") + + +def add_remote_arguments(parser: ArgumentParser) -> None: + parser.add_argument( + "--remote", + action="store_true", + help="Run the server in a Hugging Face Sandbox instead of on the local machine.", + ) + parser.add_argument( + "--flavor", + default="a10g-small", + help="HF Sandbox hardware flavor for --remote (e.g. a10g-small, a100-large).", + ) + parser.add_argument( + "--dependencies", + action="append", + default=None, + help="Extra pip dependencies to install in the sandbox. Repeat to add multiple.", + ) + parser.add_argument( + "--namespace", + default=None, + help="HF namespace to create the sandbox under (defaults to the current user).", + ) + parser.add_argument( + "--image", + default=None, + help=f"Sandbox image for --remote (defaults to {_DEFAULT_REMOTE_IMAGE!r}). Must provide torch + CUDA.", + ) + parser.add_argument( + "--idle-timeout", + default="30m", + help="Auto-shutdown the sandbox after this much inactivity (e.g. 30m, 2h). Defaults to 30m.", + ) + + +def _announce_when_ready(sbx: Any, args: Namespace, stopped: threading.Event) -> None: + health_url = sbx.proxy_url_for(SANDBOX_PORT, "/health") + while not stopped.wait(_READY_POLL_SECONDS): + try: + ready = httpx.get(health_url, headers=sbx.proxy_headers, timeout=10.0).status_code == 200 + except httpx.HTTPError: + ready = False + if not ready: + continue + out.result( + "remote-serve", + sandbox_id=sbx.id, + url=sbx.proxy_url_for(SANDBOX_PORT), + model=args.model, + hint=f'diffusers-cli generate --sandbox-id {sbx.id} --inputs \'{{"prompt": "..."}}\'', + ) + return + + +def serve_remote(args: Namespace, task: str) -> None: + if Path(args.model).exists(): + raise SystemExit( + f"--model {args.model!r} is a local path; the sandbox can't see it. " + "Pass a Hub repo id so the sandbox can download it." + ) + if args.manifest is not None and not Path(args.manifest).is_file(): + raise SystemExit(f"--manifest {args.manifest!r} is not a file.") + + hf_token = args.token or get_token() + logger.info(f"creating sandbox on flavor={args.flavor!r}...") + create_kwargs: dict[str, Any] = { + "image": args.image or _DEFAULT_REMOTE_IMAGE, + "flavor": args.flavor, + "forward_hf_token": True, + "token": hf_token, + "env": {"HF_ENABLE_PARALLEL_LOADING": "1"}, + "idle_timeout": args.idle_timeout, + # The 120s default expires while a cold node pulls the multi-GB pytorch base image or + # waits for GPU capacity, which fails the launch before it starts. + "start_timeout": 600.0, + } + if args.namespace is not None: + create_kwargs["namespace"] = args.namespace + sbx = Sandbox.create(**create_kwargs) + + def _stream(chunk: str) -> None: + sys.stderr.write(chunk) + sys.stderr.flush() + + stopped = threading.Event() + try: + dependencies = [*_DEFAULT_REMOTE_DEPS, *_SERVE_DEPS, *(args.dependencies or [])] + install_cmd = shlex.join(["uv", "pip", "install", "--system", "--break-system-packages", *dependencies]) + logger.info("installing dependencies in the sandbox...") + sbx.run(install_cmd, on_stdout=_stream, on_stderr=_stream) + + task_kwargs = _build_task_kwargs(args) + for key in (*_LOCAL_KEYS, *_SANDBOX_SET_KEYS): + task_kwargs.pop(key, None) + if args.manifest is not None: + sbx.files.upload(args.manifest, _SANDBOX_MANIFEST_PATH) + task_kwargs["manifest"] = _SANDBOX_MANIFEST_PATH + task_kwargs["port"] = SANDBOX_PORT + # Clients reach the server through the proxy, so output links must be built on the proxied address. + task_kwargs["public_url"] = sbx.proxy_url_for(SANDBOX_PORT) + cli_argv = ["diffusers-cli", "--format", "quiet", *_kwargs_to_argv(task, task_kwargs)] + + threading.Thread(target=_announce_when_ready, args=(sbx, args, stopped), daemon=True).start() + logger.info(f"starting the server in sandbox {sbx.id}; stop it with Ctrl-C.") + result = sbx.run(cli_argv, on_stdout=_stream, on_stderr=_stream, check=False, capture_output=False) + finally: + stopped.set() + sbx.kill() + + if result.exit_code != 0: + raise SystemExit(f"remote server exited with code {result.exit_code}") From 5bc39489723b18f1ce2939d31955b48fe38aa45c Mon Sep 17 00:00:00 2001 From: DN6 Date: Fri, 9 Oct 2026 13:07:55 +0530 Subject: [PATCH 03/21] update --- docs/source/en/using-diffusers/cli.md | 26 ++++++++++ src/diffusers/commands/generate.py | 34 ++++++++++--- src/diffusers/commands/serve/__init__.py | 32 +++++++++--- src/diffusers/commands/serve/app.py | 27 ++++++---- tests/others/test_cli_serve.py | 65 ++++++++++++++++++++++++ 5 files changed, 160 insertions(+), 24 deletions(-) diff --git a/docs/source/en/using-diffusers/cli.md b/docs/source/en/using-diffusers/cli.md index d7e8d6523b38..45eb53ee61d6 100644 --- a/docs/source/en/using-diffusers/cli.md +++ b/docs/source/en/using-diffusers/cli.md @@ -311,6 +311,7 @@ report no step progress, and a cancel request only takes effect once the pipelin |---|---| | `--default-seed` | Seed used for requests that do not set one. Without it, each such request gets a random seed. | | `--enable-cors` | Allow cross-origin requests from any origin, so a browser app on another host can call the server. | +| `--public-url` | Base URL clients reach the server at, used to build output file links. Set it behind a reverse proxy. | | `--log-level` | Logging level for diffusers and the HTTP server (`debug`, `info`, `warning`, `error`, `critical`). | `GET /health` returns `{"status": "ok"}`, or `503` when an SGLang or vLLM-Omni backend has exited. Every response @@ -352,6 +353,28 @@ with a JSON pointer to the offending field. {"error": {"code": "invalid_request", "message": "task 'text_to_image' declares no parameter 'cfg'", "pointer": "/parameters/cfg"}} ``` +### Remote serving (`--remote`) + +`--remote` starts the server in a [Hugging Face Sandbox](https://huggingface.co/docs/huggingface_hub/guides/sandbox) +instead of on the local machine. The command creates the sandbox, installs the dependencies, starts the server, +and streams its logs until you interrupt it; the sandbox is then deleted. + +```bash +diffusers-cli serve --model black-forest-labs/FLUX.1-dev --dtype bf16 --remote --flavor a10g-large +``` + +Once the model has loaded, the command prints the sandbox id. The server is only reachable through the sandbox +proxy, which requires your Hugging Face token, so pass the id to [`generate`](#generate) rather than calling a URL +directly: + +```bash +diffusers-cli generate --sandbox-id --inputs '{"prompt": "a cat on the moon"}' +``` + +`--flavor`, `--image`, `--dependencies`, `--namespace` and `--idle-timeout` work as they do for +[`run --remote`](#remote-execution---remote). `--model` must be a Hub repo id. A local `--manifest` file is +uploaded to the sandbox. + ### Manifests What a model accepts is described by its manifest. `serve` looks for one in this order: @@ -431,6 +454,9 @@ Outputs are saved to `~/.diffusers/cli/generate/outputs//` unless `--token` sends a bearer token in the `Authorization` header, for servers behind an authenticating proxy; no token is sent by default. Interrupting the command cancels the generation on the server. +For a server started with [`serve --remote`](#remote-serving---remote), pass `--sandbox-id ` instead of +`--url`. The request then goes through the sandbox proxy, authenticated with your Hugging Face token. + ## `custom_blocks` Package a local `ModularPipelineBlocks` subclass for upload to the Hub. Reads a Python file, AST-scans it for diff --git a/src/diffusers/commands/generate.py b/src/diffusers/commands/generate.py index f3ea85d54e1d..aeebd03c1b59 100644 --- a/src/diffusers/commands/generate.py +++ b/src/diffusers/commands/generate.py @@ -82,6 +82,7 @@ def register_subcommand(subparsers: _SubParsersAction) -> None: " $ diffusers-cli generate --url http://localhost:8000 \\\n" ' --inputs \'{"prompt": "make the fur grey", "image": "cat.png"}\' \\\n' ' --parameters \'{"steps": 8, "seed": 0}\' -o outputs/\n' + ' $ diffusers-cli generate --sandbox-id --inputs \'{"prompt": "a cat on the moon"}\'\n' "\n" "Learn more\n" " Use `diffusers-cli --help` for more information about a command.\n" @@ -91,12 +92,18 @@ def register_subcommand(subparsers: _SubParsersAction) -> None: parser: ArgumentParser = subparsers.add_parser( "generate", help="Send a generation to a Generative Media Spec server and save its outputs.", - usage="\n diffusers-cli generate --url --inputs [options]", + usage="\n diffusers-cli generate (--url | --sandbox-id ) --inputs [options]", epilog=epilog, formatter_class=RawDescriptionHelpFormatter, ) parser._optionals.title = "Options" - parser.add_argument("--url", required=True, help="Base URL of the server, e.g. http://localhost:8000.") + target = parser.add_mutually_exclusive_group(required=True) + target.add_argument("--url", default=None, help="Base URL of the server, e.g. http://localhost:8000.") + target.add_argument( + "--sandbox-id", + default=None, + help="Id of a sandbox started with `diffusers-cli serve --remote`, reached through the sandbox proxy.", + ) parser.add_argument( "--model", "-m", default=None, help="Model id to request. Defaults to the server's only model." ) @@ -130,7 +137,10 @@ def register_subcommand(subparsers: _SubParsersAction) -> None: parser.add_argument( "--token", default=None, - help="Bearer token sent to the server in the Authorization header. No token is sent by default.", + help=( + "With --url: bearer token sent in the Authorization header; none is sent by default. " + "With --sandbox-id: Hugging Face token used to reach the sandbox; defaults to the logged-in one." + ), ) parser.set_defaults(func=GenerateCommand) @@ -189,7 +199,7 @@ def run(self) -> None: out.result( self.task, - url=args.url, + url=str(client.base_url), model=model, generation=generation["id"], seed=generation.get("seed"), @@ -197,6 +207,18 @@ def run(self) -> None: ) def _client(self) -> httpx.Client: + if self.args.sandbox_id is not None: + from huggingface_hub import Sandbox + + from .serve.remote import SANDBOX_PORT + + sbx = Sandbox.connect(self.args.sandbox_id, token=self.args.token) + return httpx.Client( + base_url=sbx.proxy_url_for(SANDBOX_PORT), + headers=sbx.proxy_headers, + timeout=_TIMEOUT_SECONDS, + follow_redirects=True, + ) headers = {"Authorization": f"Bearer {self.args.token}"} if self.args.token else {} return httpx.Client(base_url=self.args.url, headers=headers, timeout=_TIMEOUT_SECONDS, follow_redirects=True) @@ -204,7 +226,7 @@ def _request(self, client: httpx.Client, method: str, path: str, **kwargs: Any) try: response = client.request(method, path, **kwargs) except httpx.HTTPError as e: - raise SystemExit(f"Could not reach {self.args.url}: {type(e).__name__}: {e}") from e + raise SystemExit(f"Could not reach {client.base_url}: {type(e).__name__}: {e}") from e if response.status_code < 400: return response.json() @@ -220,7 +242,7 @@ def _request(self, client: httpx.Client, method: str, path: str, **kwargs: Any) def _only_model(self, client: httpx.Client) -> str: models = [model["id"] for model in self._request(client, "GET", "/v1/models")["data"]] if len(models) != 1: - raise SystemExit(f"{self.args.url} serves {len(models)} models ({models}); pick one with --model.") + raise SystemExit(f"{client.base_url} serves {len(models)} models ({models}); pick one with --model.") return models[0] def _artifact_bytes(self, client: httpx.Client, artifact: dict[str, Any]) -> bytes: diff --git a/src/diffusers/commands/serve/__init__.py b/src/diffusers/commands/serve/__init__.py index e56c73a5c530..e1ddd6d8294b 100644 --- a/src/diffusers/commands/serve/__init__.py +++ b/src/diffusers/commands/serve/__init__.py @@ -30,6 +30,7 @@ from ...utils import logging from .. import BaseDiffusersCLICommand from ..run import _add_loading_arguments, _add_optimization_arguments +from .remote import add_remote_arguments, serve_remote DEFAULT_OUTPUT_DIR = str(Path.home() / ".diffusers" / "cli" / "serve" / "outputs") @@ -139,6 +140,15 @@ def register_subcommand(subparsers: _SubParsersAction) -> None: ) parser.add_argument("--host", default="127.0.0.1", help="Interface to bind. Defaults to 127.0.0.1.") parser.add_argument("--port", type=int, default=8000, help="Port to bind. Defaults to 8000.") + parser.add_argument( + "--public-url", + default=None, + metavar="URL", + help=( + "Base URL clients reach the server at, used to build output file links. Set it when the server " + "sits behind a reverse proxy; defaults to the address of each incoming request." + ), + ) parser.add_argument( "--manifest", default=None, @@ -182,6 +192,7 @@ def register_subcommand(subparsers: _SubParsersAction) -> None: default=None, help="Logging level for diffusers and the HTTP server. Defaults to the current diffusers verbosity.", ) + add_remote_arguments(parser) # `_load_pipeline` reads these `run` flags, which `serve` does not expose. parser.set_defaults(func=ServeCommand, workflow=None, offload_margin=None) @@ -190,14 +201,6 @@ def __init__(self, args: Namespace): def run(self) -> None: args = self.args - try: - import fastapi # noqa: F401 - import uvicorn - except ImportError as e: - raise SystemExit( - '`diffusers-cli serve` requires FastAPI and uvicorn. Install them with: pip install "diffusers[serve]"' - ) from e - if args.log_level is not None: logging.set_verbosity(logging.get_log_levels_dict()[args.log_level]) @@ -220,6 +223,18 @@ def run(self) -> None: f"--backend {args.backend}. Pass the engine's own flags through --backend-args instead." ) + if args.remote: + serve_remote(args, self.task) + return + + try: + import fastapi # noqa: F401 + import uvicorn + except ImportError as e: + raise SystemExit( + '`diffusers-cli serve` requires FastAPI and uvicorn. Install them with: pip install "diffusers[serve]"' + ) from e + from .app import create_app from .backends import ( DiffusersBackend, @@ -266,6 +281,7 @@ def run(self) -> None: GenerationQueue(backend, args.output_dir), default_seed=args.default_seed, enable_cors=args.enable_cors, + public_url=args.public_url, ) out.result( self.task, diff --git a/src/diffusers/commands/serve/app.py b/src/diffusers/commands/serve/app.py index 05213fc302ce..706ef4c0e4d5 100644 --- a/src/diffusers/commands/serve/app.py +++ b/src/diffusers/commands/serve/app.py @@ -22,6 +22,7 @@ import re import uuid from typing import Any +from urllib.parse import urlparse from fastapi import FastAPI, Request from fastapi.middleware.cors import CORSMiddleware @@ -46,7 +47,7 @@ def _timestamp(value: Any) -> str | None: return value.strftime("%Y-%m-%dT%H:%M:%SZ") -def _render(generation: Generation, request: Request) -> dict[str, Any]: +def _render(generation: Generation, request: Request, public_url: str | None) -> dict[str, Any]: outputs = None if generation.status == "complete": outputs = {} @@ -58,10 +59,12 @@ def _render(generation: Generation, request: Request) -> dict[str, Any]: content = (generation.directory / artifact["file"]).read_bytes() rendered["base64"] = base64.b64encode(content).decode() else: - url = request.url_for( - "get_generation_file", generation_id=generation.id, filename=artifact["file"] - ) - rendered["url"] = str(url) + route = {"generation_id": generation.id, "filename": artifact["file"]} + if public_url is None: + rendered["url"] = str(request.url_for("get_generation_file", **route)) + else: + path = request.app.url_path_for("get_generation_file", **route) + rendered["url"] = public_url.rstrip("/") + path outputs[name].append(rendered) return { @@ -83,8 +86,12 @@ def create_app( generations: GenerationQueue, default_seed: int | None = None, enable_cors: bool = False, + public_url: str | None = None, ) -> FastAPI: - app = FastAPI(title="diffusers-cli serve", docs_url=None, redoc_url=None, openapi_url=None) + # A proxy may or may not strip the path it publishes the server under before forwarding a request. + # With that path as the root path, routing accepts both forms. + root_path = urlparse(public_url).path.rstrip("/") if public_url is not None else "" + app = FastAPI(title="diffusers-cli serve", docs_url=None, redoc_url=None, openapi_url=None, root_path=root_path) @app.middleware("http") async def request_id(request: Request, call_next) -> Response: @@ -157,14 +164,14 @@ async def create_generation(request: Request) -> dict[str, Any]: await asyncio.to_thread(generation.wait, min(float(wait.group(1)), MAX_WAIT_SECONDS)) if generation.error is not None: raise generation.error - return _render(generation, request) + return _render(generation, request, public_url) @app.get("/v1/generations/{generation_id}") async def get_generation(generation_id: str, request: Request) -> dict[str, Any]: generation = generations.get(generation_id) if generation.error is not None: raise generation.error - return _render(generation, request) + return _render(generation, request, public_url) @app.delete("/v1/generations/{generation_id}", status_code=204) async def delete_generation(generation_id: str) -> Response: @@ -176,7 +183,7 @@ async def cancel_generation(generation_id: str, request: Request) -> dict[str, A generation = generations.cancel(generation_id) if generation.error is not None: raise generation.error - return _render(generation, request) + return _render(generation, request, public_url) @app.get("/v1/generations/{generation_id}/events") async def get_generation_events(generation_id: str, request: Request) -> StreamingResponse: @@ -190,7 +197,7 @@ async def stream(): events, done = await asyncio.to_thread(generation.wait_for_events, index, _KEEP_ALIVE_SECONDS) for event, data in events: index += 1 - payload = _render(generation, request) if event == "complete" else data + payload = _render(generation, request, public_url) if event == "complete" else data yield f"id: {index}\nevent: {event}\ndata: {json.dumps(payload)}\n\n" if done: return diff --git a/tests/others/test_cli_serve.py b/tests/others/test_cli_serve.py index 03a31909fe5f..380034ea42d8 100644 --- a/tests/others/test_cli_serve.py +++ b/tests/others/test_cli_serve.py @@ -516,6 +516,19 @@ def test_default_seed_applies_when_request_omits_seed(self, serve): body["parameters"] = {"seed": 11} assert client.post("/v1/generations", json=body, headers=WAIT).json()["seed"] == 11 + def test_public_url_replaces_the_request_address_in_output_links(self, serve): + client, _ = serve(app_options={"public_url": "https://proxy.example/sandbox/proxy/8000/"}) + body = {"model": MODEL_ID, "inputs": {"prompt": "a cat"}} + generation = client.post("/v1/generations", json=body, headers=WAIT).json() + expected = f"https://proxy.example/sandbox/proxy/8000/v1/generations/{generation['id']}/files/image-0.png" + assert generation["outputs"]["image"][0]["url"] == expected + + # The sandbox proxy forwards some requests with the published path still attached. + for path in ("/v1/models", "/sandbox/proxy/8000/v1/models"): + assert client.get(path).status_code == 200, path + prefixed = client.post("/sandbox/proxy/8000/v1/generations", json=body, headers=WAIT) + assert prefixed.status_code == 201, prefixed.text + def test_cors_is_opt_in(self, serve): origin = {"Origin": "http://example.com"} client, _ = serve() @@ -808,6 +821,58 @@ def test_server_error_is_reported(self, serve, tmp_path, monkeypatch): self._run(monkeypatch, client, argv) +class FakeSandbox: + id = "sbx-1" + proxy_headers = {"Authorization": "Bearer hf_x", "X-Sandbox-Token": "tok"} + + def __init__(self): + self.commands = [] + self.uploads = [] + self.killed = False + self.files = SimpleNamespace(upload=lambda local, remote: self.uploads.append((local, remote))) + + def proxy_url_for(self, port, path="/"): + return f"https://job.hf.jobs/v1/sandboxes/s/proxy/{port}{path}" + + def run(self, cmd, **kwargs): + self.commands.append(cmd) + return SimpleNamespace(exit_code=0) + + def kill(self): + self.killed = True + + +class TestServeRemote: + def test_remote_starts_the_server_in_a_sandbox(self, tmp_path, monkeypatch): + sandbox = FakeSandbox() + created = {} + monkeypatch.setattr( + "diffusers.commands.serve.remote.Sandbox", + SimpleNamespace(create=lambda **kwargs: created.update(kwargs) or sandbox), + ) + manifest = tmp_path / "gms.json" + manifest.write_text("{}") + parser = ArgumentParser() + ServeCommand.register_subcommand(parser.add_subparsers()) + argv = ["serve", "-m", "org/model", "--dtype", "bf16", "--remote", "--flavor", "a10g-large"] + argv += ["--manifest", str(manifest), "--port", "9000", "--dependencies", "extra-package"] + ServeCommand(parser.parse_args(argv)).run() + + assert created["flavor"] == "a10g-large" + install, serve_argv = sandbox.commands + assert "fastapi" in install and "extra-package" in install, install + assert serve_argv[:5] == ["diffusers-cli", "--format", "quiet", "serve", "--model"], serve_argv + options = dict(zip(serve_argv[4::2], serve_argv[5::2])) + assert options["--dtype"] == "bf16" + # The sandbox server always binds the proxied port and builds output links on the proxied address. + assert options["--port"] == "8000" + assert options["--public-url"] == "https://job.hf.jobs/v1/sandboxes/s/proxy/8000/" + assert options["--manifest"] == "/tmp/diffusers-cli/gms.json" + assert sandbox.uploads == [(str(manifest), "/tmp/diffusers-cli/gms.json")] + assert "--remote" not in serve_argv and "--flavor" not in serve_argv, serve_argv + assert sandbox.killed, "the sandbox must be killed once the server stops" + + class TestServeCommand: def _parse(self, argv): parser = ArgumentParser() From fc92295c376bf82c30c75e06608d549ffdecc5fc Mon Sep 17 00:00:00 2001 From: DN6 Date: Fri, 9 Oct 2026 23:42:55 +0530 Subject: [PATCH 04/21] update --- docs/source/en/using-diffusers/cli.md | 25 ++++++-- src/diffusers/commands/run.py | 79 +++++++++++++++++++----- src/diffusers/commands/serve/__init__.py | 11 ++-- src/diffusers/commands/serve/backends.py | 14 ++++- src/diffusers/commands/serve/remote.py | 15 ++++- tests/others/test_cli_commands.py | 37 +++++++++++ tests/others/test_cli_serve.py | 45 +++++++++++--- 7 files changed, 187 insertions(+), 39 deletions(-) diff --git a/docs/source/en/using-diffusers/cli.md b/docs/source/en/using-diffusers/cli.md index 45eb53ee61d6..36994135bdca 100644 --- a/docs/source/en/using-diffusers/cli.md +++ b/docs/source/en/using-diffusers/cli.md @@ -141,10 +141,15 @@ Configure how the CLI loads model weights and custom pipeline code. pipelines. See [Attention backends](../optimization/attention_backends). - `--vae-tiling` / `--vae-slicing` — lower VAE decode VRAM. See [VAE tiling](../optimization/memory#vae-tiling) and [VAE slicing](../optimization/memory#vae-slicing). -- `--compile [JSON]` — compile denoiser modules with [torch.compile](../optimization/fp16#torchcompile). The - CLI prefers [regional compilation](../optimization/fp16#regional-compilation) for modules with repeated - blocks. Bare `--compile` uses `fullgraph=true`. A JSON object is forwarded to `torch.compile`. Not supported - with `--context-parallel`. +- `--compile [JSON]` — compile denoiser modules and the VAE decoder with + [torch.compile](../optimization/fp16#torchcompile). The CLI prefers + [regional compilation](../optimization/fp16#regional-compilation) for denoisers with repeated blocks; the VAE + decoder is always compiled whole. Bare `--compile` uses `fullgraph=true`. A JSON object is forwarded to + `torch.compile`. Not supported with `--context-parallel`. +- `--compile-mode {regional,full}` — what `--compile` compiles. `regional` (default) compiles only the repeated + blocks, which keeps the first step fast. `full` compiles the whole denoiser. `torch.compile` modes that use CUDA + graphs (`max-autotune`, `reduce-overhead`) need `--compile-mode full`: repeated blocks share one compiled graph, + so with CUDA graphs each block's output is overwritten by the next block's run. - `--context-parallel` — Ulysses-style context parallelism on a DiT-based pipeline. Locally requires torchrun; under `--remote` the CLI wraps `torchrun --nproc-per-node=gpu` for you. See [Context parallelism](../training/distributed_inference#context-parallelism). @@ -375,6 +380,14 @@ diffusers-cli generate --sandbox-id --inputs '{"prompt": "a cat on the moon [`run --remote`](#remote-execution---remote). `--model` must be a Hub repo id. A local `--manifest` file is uploaded to the sandbox. +With `--backend sglang` or `--backend vllm`, the sandbox starts from the engine's own image +(`lmsysorg/sglang:latest` or `vllm/vllm-omni:latest`) unless `--image` names another, so the engine does not need +installing: + +```bash +diffusers-cli serve --model black-forest-labs/FLUX.2-klein-4B --backend sglang --remote --flavor a100-large +``` + ### Manifests What a model accepts is described by its manifest. `serve` looks for one in this order: @@ -425,8 +438,8 @@ diffusers-cli serve --model Qwen/Qwen-Image --backend sglang --backend-args '--n diffusers-cli serve --model Wan-AI/Wan2.2-T2V-A14B-Diffusers --backend vllm ``` -The pipeline flags (`--dtype`, `--cpu-offload`, `--compile`, ...) do not apply to these backends; pass the -engine's own flags with `--backend-args`. Compared with the in-process backend, they serve image and video +The pipeline flags (`--dtype`, `--cpu-offload`, `--compile`, ...) do not apply to these backends: they are +ignored with a warning. Pass the engine's own flags with `--backend-args`. Compared with the in-process backend, they serve image and video outputs only, accept `image` as the only media input, expose the parameters both engines share (size, steps, guidance scale, seed, number of frames, fps, number of images), report no step progress, and apply no request adapters. diff --git a/src/diffusers/commands/run.py b/src/diffusers/commands/run.py index cad580d592bc..26922842927a 100644 --- a/src/diffusers/commands/run.py +++ b/src/diffusers/commands/run.py @@ -63,6 +63,14 @@ ATTENTION_BACKEND_CHOICES = ("default", *sorted(b.value for b in _HUB_KERNELS_REGISTRY)) +COMPILE_MODE_CHOICES = ("regional", "full") +# zlib level for PNG outputs. Pillow's default of 6 takes close to four times as long to encode a +# 1024x1024 image for a file about 7% smaller; the output is lossless either way. +PNG_COMPRESS_LEVEL = 1 + +# `torch.compile` modes that replay CUDA graphs. Repeated blocks share one compiled graph, so each +# block's output is overwritten when the next block replays it. +_CUDA_GRAPH_COMPILE_MODES = ("max-autotune", "reduce-overhead") # Kwarg keys whose string value gets auto-loaded before being passed to the pipeline call. # Images resolve via `diffusers.utils.load_image` → PIL.Image.Image; videos resolve via @@ -211,13 +219,23 @@ def _add_optimization_arguments(parser: ArgumentParser) -> None: default=None, metavar="JSON", help=( - "torch.compile every denoiser submodule on the pipeline. Accepts an optional JSON " + "torch.compile every denoiser submodule on the pipeline, and the VAE decoder. Accepts an optional JSON " 'object of kwargs forwarded to `torch.compile`, e.g. \'{"mode": "max-autotune", ' '"fullgraph": true}\'. Bare `--compile` uses `fullgraph=true`. Adds a one-time ' "compilation cost on the first step but speeds up every subsequent step — worth it " "for multi-step generation (50+ steps)." ), ) + parser.add_argument( + "--compile-mode", + choices=COMPILE_MODE_CHOICES, + default="regional", + help=( + "What --compile compiles. 'regional' (default) compiles only the repeated blocks of a denoiser " + "that declares them, which keeps the first step fast. 'full' compiles the whole denoiser, which " + "`torch.compile` modes that use CUDA graphs (max-autotune, reduce-overhead) require." + ), + ) def _add_output_arguments(parser: ArgumentParser) -> None: @@ -461,26 +479,44 @@ def _apply_optimizations(pipeline: Any, args: Namespace) -> None: if args.context_parallel: logger.warning("--compile is currently not supported with --context-parallel; skipping compile.") else: - _compile_denoiser(pipeline, args.compile) + _compile_denoiser(pipeline, args.compile, args.compile_mode) + _compile_vae_decoder(pipeline, args.compile) + + +def _parse_compile_spec(compile_spec: str) -> dict[str, Any]: + try: + compile_kwargs = json.loads(compile_spec) + except json.JSONDecodeError as e: + raise SystemExit(f"--compile must be valid JSON: {e}") from e + if not isinstance(compile_kwargs, dict): + raise SystemExit("--compile must decode to a JSON object.") + return compile_kwargs -def _compile_denoiser(pipeline: Any, compile_spec: str) -> None: +def _compile_vae_decoder(pipeline: Any, compile_spec: str) -> None: + """Compile the decoder of the pipeline's VAE with the `--compile` kwargs. + + The decoder runs once per call, so it has no repeated blocks to compile regionally and is always compiled whole. + """ + decoder = getattr(getattr(pipeline, "vae", None), "decoder", None) + if not isinstance(decoder, torch.nn.Module): + return + pipeline.vae.decoder = torch.compile(decoder, **_parse_compile_spec(compile_spec)) + + +def _compile_denoiser(pipeline: Any, compile_spec: str, compile_mode: str = "regional") -> None: """Compile every `transformer*` and `unet*` submodule on the pipeline. `compile_spec` is the raw JSON string from `--compile` (`"{}"` for bare flag). Decoded into kwargs and forwarded verbatim to the compile call. - Prefers regional compilation via `module.compile_repeated_blocks(**kwargs)` — only compiles the repeated inner - blocks (the bulk of the compute), much faster first-step latency than compiling the whole module. Falls back to - full `torch.compile` if the model doesn't expose `_repeated_blocks`. + With `compile_mode="regional"`, prefers regional compilation via `module.compile_repeated_blocks(**kwargs)` — only + compiles the repeated inner blocks (the bulk of the compute), much faster first-step latency than compiling the + whole module. Falls back to full `torch.compile` if the model doesn't expose `_repeated_blocks`. With + `compile_mode="full"`, always compiles the whole module. """ - try: - compile_kwargs = json.loads(compile_spec) - except json.JSONDecodeError as e: - raise SystemExit(f"--compile must be valid JSON: {e}") from e - if not isinstance(compile_kwargs, dict): - raise SystemExit("--compile must decode to a JSON object.") + compile_kwargs = _parse_compile_spec(compile_spec) for attr in dir(pipeline): if not any(attr.startswith(key) for key in _DENOISER_COMPONENT_KEYS): @@ -489,11 +525,17 @@ def _compile_denoiser(pipeline: Any, compile_spec: str) -> None: if not isinstance(module, torch.nn.Module): continue - if getattr(module, "_repeated_blocks", None): + if compile_mode == "regional" and getattr(module, "_repeated_blocks", None): + if compile_kwargs.get("mode") in _CUDA_GRAPH_COMPILE_MODES: + raise SystemExit( + f"--compile mode {compile_kwargs['mode']!r} uses CUDA graphs, which fail when the repeated " + f"blocks of {type(module).__name__} are compiled separately. Add `--compile-mode full`, or " + "use mode 'max-autotune-no-cudagraphs'." + ) # Regional compile — only the repeated blocks. Mutates `module` in place. module.compile_repeated_blocks(**compile_kwargs) else: - # No regional metadata declared; fall back to compiling the whole module. + # Full compile was asked for, or no regional metadata is declared. setattr(pipeline, attr, torch.compile(module, **compile_kwargs)) @@ -535,6 +577,9 @@ def _load_lora(pipeline: Any, args: Namespace) -> None: def _load_pipeline(args: Namespace) -> Any: + if args.compile is None and args.compile_mode != "regional": + raise SystemExit(f"--compile-mode {args.compile_mode} only applies together with --compile.") + # Detect modular repos by trying the standard config; `ModularPipeline` repos ship # `modular_model_index.json` instead of `model_index.json`, so `load_config` OSErrors. # A repo can also ship a `model_index.json` whose `_class_name` is a modular pipeline @@ -800,7 +845,7 @@ def _save_videos(videos: list[Any], args: Namespace) -> list[str]: arr = (np.clip(arr, 0.0, 1.0) * 255).round().astype(np.uint8) frame = Image.fromarray(arr) frame_path = frames_dir / f"{i:04d}.png" - frame.save(frame_path) + frame.save(frame_path, compress_level=PNG_COMPRESS_LEVEL) saved.append(str(frame_path)) try: export_to_video(frames, str(path), fps=args.fps) @@ -837,14 +882,14 @@ def _save_output(value: Any, args: Namespace) -> list[str]: for arr, path in zip(value, paths): if arr.dtype != np.uint8: arr = (np.clip(arr, 0.0, 1.0) * 255).round().astype(np.uint8) - Image.fromarray(arr).save(path) + Image.fromarray(arr).save(path, compress_level=PNG_COMPRESS_LEVEL) return [str(p) for p in paths] pil_images = _as_pil_list(value) if pil_images is not None: paths = _resolve_output_paths(len(pil_images), args.output, ext="png") for img, path in zip(pil_images, paths): - img.save(path) + img.save(path, compress_level=PNG_COMPRESS_LEVEL) return [str(p) for p in paths] frames = _as_frame_sequence(value) diff --git a/src/diffusers/commands/serve/__init__.py b/src/diffusers/commands/serve/__init__.py index e1ddd6d8294b..ec35ea631bfa 100644 --- a/src/diffusers/commands/serve/__init__.py +++ b/src/diffusers/commands/serve/__init__.py @@ -33,6 +33,8 @@ from .remote import add_remote_arguments, serve_remote +logger = logging.get_logger("diffusers-cli/serve") + DEFAULT_OUTPUT_DIR = str(Path.home() / ".diffusers" / "cli" / "serve" / "outputs") BACKEND_CHOICES = ("diffusers", "sglang", "vllm") @@ -47,6 +49,7 @@ "vae_tiling": False, "vae_slicing": False, "compile": None, + "compile_mode": "regional", } @@ -212,14 +215,14 @@ def run(self) -> None: if args.backend == "diffusers" and args.backend_args is not None: raise SystemExit("--backend-args only applies to --backend sglang or vllm.") if args.backend != "diffusers": - rejected = [ + ignored = [ "--" + name.replace("_", "-") for name, unset in _DIFFUSERS_ONLY_FLAGS.items() if getattr(args, name) != unset ] - if rejected: - raise SystemExit( - f"{', '.join(rejected)} configure the in-process pipeline and have no effect with " + if ignored: + logger.warning( + f"{', '.join(ignored)} configure the in-process pipeline and are ignored with " f"--backend {args.backend}. Pass the engine's own flags through --backend-args instead." ) diff --git a/src/diffusers/commands/serve/backends.py b/src/diffusers/commands/serve/backends.py index c0a779804dad..bc1eb2811eaf 100644 --- a/src/diffusers/commands/serve/backends.py +++ b/src/diffusers/commands/serve/backends.py @@ -41,7 +41,14 @@ from ...modular_pipelines import ModularPipeline from ...utils import export_to_video, load_image, load_video, logging, numpy_to_pil from ...utils.constants import DIFFUSERS_REQUEST_TIMEOUT -from ..run import _as_audio_arrays, _as_pil_list, _get_generator, _load_pipeline, _save_audio_arrays +from ..run import ( + PNG_COMPRESS_LEVEL, + _as_audio_arrays, + _as_pil_list, + _get_generator, + _load_pipeline, + _save_audio_arrays, +) from .generations import Generation from .manifest import GMSError, ServedModel, blocks_signature @@ -144,7 +151,10 @@ def _save_images(images: Any, directory: Path, name: str, media_type: str, rate: filename = f"{name}-{index}.{media_type.split('/')[1]}" if media_type == "image/jpeg": image = image.convert("RGB") - image.save(directory / filename) + if media_type == "image/png": + image.save(directory / filename, compress_level=PNG_COMPRESS_LEVEL) + else: + image.save(directory / filename) artifacts.append({"file": filename, "media_type": media_type, "width": image.width, "height": image.height}) return artifacts diff --git a/src/diffusers/commands/serve/remote.py b/src/diffusers/commands/serve/remote.py index ebabe97ab57e..7743ebf0e444 100644 --- a/src/diffusers/commands/serve/remote.py +++ b/src/diffusers/commands/serve/remote.py @@ -37,6 +37,8 @@ SANDBOX_PORT = 8000 _SANDBOX_MANIFEST_PATH = "/tmp/diffusers-cli/gms.json" _SERVE_DEPS = ("fastapi", "uvicorn") +# Each engine's own image carries the engine, its kernels and the CUDA toolchain they are built with. +_ENGINE_IMAGES = {"sglang": "lmsysorg/sglang:latest", "vllm": "vllm/vllm-omni:latest"} _READY_POLL_SECONDS = 5.0 # Flags that say how the sandbox is set up, or that the sandbox server sets itself. @@ -69,7 +71,10 @@ def add_remote_arguments(parser: ArgumentParser) -> None: parser.add_argument( "--image", default=None, - help=f"Sandbox image for --remote (defaults to {_DEFAULT_REMOTE_IMAGE!r}). Must provide torch + CUDA.", + help=( + f"Sandbox image for --remote. Defaults to {_DEFAULT_REMOTE_IMAGE!r}, or to the engine's own image " + f"with --backend sglang ({_ENGINE_IMAGES['sglang']!r}) or vllm ({_ENGINE_IMAGES['vllm']!r})." + ), ) parser.add_argument( "--idle-timeout", @@ -109,7 +114,7 @@ def serve_remote(args: Namespace, task: str) -> None: hf_token = args.token or get_token() logger.info(f"creating sandbox on flavor={args.flavor!r}...") create_kwargs: dict[str, Any] = { - "image": args.image or _DEFAULT_REMOTE_IMAGE, + "image": args.image or _ENGINE_IMAGES.get(args.backend, _DEFAULT_REMOTE_IMAGE), "flavor": args.flavor, "forward_hf_token": True, "token": hf_token, @@ -133,6 +138,9 @@ def _stream(chunk: str) -> None: install_cmd = shlex.join(["uv", "pip", "install", "--system", "--break-system-packages", *dependencies]) logger.info("installing dependencies in the sandbox...") sbx.run(install_cmd, on_stdout=_stream, on_stderr=_stream) + # An image can put another environment first on PATH, with its own older `diffusers-cli` + # (the SGLang image does), so the server is started with the interpreter the install went into. + python = sbx.run(["uv", "python", "find", "--system"]).stdout.strip() task_kwargs = _build_task_kwargs(args) for key in (*_LOCAL_KEYS, *_SANDBOX_SET_KEYS): @@ -143,7 +151,8 @@ def _stream(chunk: str) -> None: task_kwargs["port"] = SANDBOX_PORT # Clients reach the server through the proxy, so output links must be built on the proxied address. task_kwargs["public_url"] = sbx.proxy_url_for(SANDBOX_PORT) - cli_argv = ["diffusers-cli", "--format", "quiet", *_kwargs_to_argv(task, task_kwargs)] + cli_module = [python, "-m", "diffusers.commands.diffusers_cli"] + cli_argv = [*cli_module, "--format", "quiet", *_kwargs_to_argv(task, task_kwargs)] threading.Thread(target=_announce_when_ready, args=(sbx, args, stopped), daemon=True).start() logger.info(f"starting the server in sandbox {sbx.id}; stop it with Ctrl-C.") diff --git a/tests/others/test_cli_commands.py b/tests/others/test_cli_commands.py index e2d2d11f3cb6..4c0fc16cbded 100644 --- a/tests/others/test_cli_commands.py +++ b/tests/others/test_cli_commands.py @@ -34,6 +34,8 @@ RunCommand, _build_task_kwargs, _collapse_frame_dirs, + _compile_denoiser, + _compile_vae_decoder, _download_outputs_from_sandbox, _kwargs_to_argv, _load_lora, @@ -241,6 +243,41 @@ def test_compile_arg(self): assert compiled_blocks assert all(m._compiled_call_impl is not None for m in compiled_blocks) + def test_compile_mode_full_compiles_the_whole_denoiser(self, monkeypatch): + class Denoiser(torch.nn.Module): + _repeated_blocks = ["Block"] + + def compile_repeated_blocks(self, **kwargs): + raise AssertionError("regional compilation must not run with compile_mode='full'") + + compiled = {} + monkeypatch.setattr(torch, "compile", lambda module, **kwargs: compiled.update(kwargs) or "compiled") + pipeline = SimpleNamespace(transformer=Denoiser()) + _compile_denoiser(pipeline, '{"mode": "max-autotune"}', "full") + assert pipeline.transformer == "compiled" + assert compiled == {"mode": "max-autotune"} + + def test_regional_compile_rejects_cuda_graph_modes(self): + class Denoiser(torch.nn.Module): + _repeated_blocks = ["Block"] + + pipeline = SimpleNamespace(transformer=Denoiser()) + with pytest.raises(SystemExit, match="uses CUDA graphs.*--compile-mode full"): + _compile_denoiser(pipeline, '{"mode": "max-autotune"}', "regional") + + def test_compile_vae_decoder_compiles_the_decoder(self, monkeypatch): + compiled = {} + monkeypatch.setattr(torch, "compile", lambda module, **kwargs: compiled.update(kwargs) or "compiled") + pipeline = SimpleNamespace(vae=SimpleNamespace(decoder=torch.nn.Conv2d(4, 3, 3))) + _compile_vae_decoder(pipeline, '{"fullgraph": true}') + assert pipeline.vae.decoder == "compiled" + assert compiled == {"fullgraph": True} + + def test_compile_vae_decoder_skips_a_pipeline_without_one(self): + pipeline = SimpleNamespace(vae=None) + _compile_vae_decoder(pipeline, "{}") + assert pipeline.vae is None + @require_torch_gpu # `--attention-backend` only exposes Hub-hosted kernels, all of which need `kernels>=0.12`. @require_kernels_version_greater_or_equal("0.12") diff --git a/tests/others/test_cli_serve.py b/tests/others/test_cli_serve.py index 380034ea42d8..8701d06a8942 100644 --- a/tests/others/test_cli_serve.py +++ b/tests/others/test_cli_serve.py @@ -46,6 +46,8 @@ from diffusers.commands.serve.generations import Generation, GenerationQueue from diffusers.commands.serve.manifest import GMSError, Manifest, ServedModel, blocks_signature, derive_manifest from diffusers.modular_pipelines import InputParam, ModularPipelineBlocks, OutputParam +from diffusers.utils import logging +from diffusers.utils.testing_utils import CaptureLogger MODEL_ID = "test-org/fake-model" @@ -836,7 +838,7 @@ def proxy_url_for(self, port, path="/"): def run(self, cmd, **kwargs): self.commands.append(cmd) - return SimpleNamespace(exit_code=0) + return SimpleNamespace(exit_code=0, stdout="/usr/bin/python3\n") def kill(self): self.killed = True @@ -859,10 +861,13 @@ def test_remote_starts_the_server_in_a_sandbox(self, tmp_path, monkeypatch): ServeCommand(parser.parse_args(argv)).run() assert created["flavor"] == "a10g-large" - install, serve_argv = sandbox.commands + assert created["image"] == "pytorch/pytorch:2.10.0-cuda12.8-cudnn9-runtime" + install, _, serve_argv = sandbox.commands assert "fastapi" in install and "extra-package" in install, install - assert serve_argv[:5] == ["diffusers-cli", "--format", "quiet", "serve", "--model"], serve_argv - options = dict(zip(serve_argv[4::2], serve_argv[5::2])) + # The server runs under the interpreter the dependencies were installed into, not whatever is on PATH. + assert serve_argv[:3] == ["/usr/bin/python3", "-m", "diffusers.commands.diffusers_cli"], serve_argv + assert serve_argv[3:7] == ["--format", "quiet", "serve", "--model"], serve_argv + options = dict(zip(serve_argv[6::2], serve_argv[7::2])) assert options["--dtype"] == "bf16" # The sandbox server always binds the proxied port and builds output links on the proxied address. assert options["--port"] == "8000" @@ -872,6 +877,25 @@ def test_remote_starts_the_server_in_a_sandbox(self, tmp_path, monkeypatch): assert "--remote" not in serve_argv and "--flavor" not in serve_argv, serve_argv assert sandbox.killed, "the sandbox must be killed once the server stops" + @pytest.mark.parametrize( + "backend, image", [("sglang", "lmsysorg/sglang:latest"), ("vllm", "vllm/vllm-omni:latest")] + ) + def test_engine_backend_defaults_to_the_engine_image(self, monkeypatch, backend, image): + created = {} + monkeypatch.setattr( + "diffusers.commands.serve.remote.Sandbox", + SimpleNamespace(create=lambda **kwargs: created.update(kwargs) or FakeSandbox()), + ) + parser = ArgumentParser() + ServeCommand.register_subcommand(parser.add_subparsers()) + ServeCommand(parser.parse_args(["serve", "-m", "org/model", "--backend", backend, "--remote"])).run() + assert created["image"] == image + + ServeCommand( + parser.parse_args(["serve", "-m", "org/model", "--backend", backend, "--remote", "--image", "mine"]) + ).run() + assert created["image"] == "mine", "an explicit --image must win over the engine default" + class TestServeCommand: def _parse(self, argv): @@ -879,11 +903,18 @@ def _parse(self, argv): ServeCommand.register_subcommand(parser.add_subparsers()) return parser.parse_args(["serve", *argv]) - def test_engine_backend_rejects_pipeline_flags(self): + def test_engine_backend_warns_about_ignored_pipeline_flags(self, monkeypatch): pytest.importorskip("fastapi", reason="`diffusers-cli serve` needs the `diffusers[serve]` extra") + + def stop(args): + raise RuntimeError("stop after the flag check") + + monkeypatch.setattr("diffusers.commands.serve._resolve_pipeline", stop) args = self._parse(["-m", "org/model", "--backend", "vllm", "--dtype", "bf16", "--vae-tiling"]) - with pytest.raises(SystemExit, match="--dtype, --vae-tiling configure the in-process pipeline"): - ServeCommand(args).run() + with CaptureLogger(logging.get_logger("diffusers-cli/serve")) as captured: + with pytest.raises(RuntimeError, match="stop after the flag check"): + ServeCommand(args).run() + assert "--dtype, --vae-tiling configure the in-process pipeline and are ignored" in captured.out def test_context_parallel_is_rejected(self): pytest.importorskip("fastapi", reason="`diffusers-cli serve` needs the `diffusers[serve]` extra") From c8967b0c46dfc1ef69ace34b74ce5d99b9e2d1f6 Mon Sep 17 00:00:00 2001 From: DN6 Date: Tue, 3 Nov 2026 14:10:22 +0530 Subject: [PATCH 05/21] Address serve review feedback and run serve --remote on HF Jobs --- .ai/skills/diffusers-cli/SKILL.md | 2 - docs/source/en/_toctree.yml | 2 + docs/source/en/using-diffusers/cli.md | 175 +------------ docs/source/en/using-diffusers/serve.md | 173 +++++++++++++ setup.py | 7 +- src/diffusers/commands/diffusers_cli.py | 2 - src/diffusers/commands/generate.py | 260 -------------------- src/diffusers/commands/run.py | 6 +- src/diffusers/commands/serve/__init__.py | 77 +++++- src/diffusers/commands/serve/app.py | 5 + src/diffusers/commands/serve/generations.py | 6 + src/diffusers/commands/serve/remote.py | 162 ++++++------ tests/others/test_cli_commands.py | 2 +- tests/others/test_cli_serve.py | 220 ++++++++++------- 14 files changed, 483 insertions(+), 616 deletions(-) create mode 100644 docs/source/en/using-diffusers/serve.md delete mode 100644 src/diffusers/commands/generate.py diff --git a/.ai/skills/diffusers-cli/SKILL.md b/.ai/skills/diffusers-cli/SKILL.md index 9baf0540855a..b4c5814865a7 100644 --- a/.ai/skills/diffusers-cli/SKILL.md +++ b/.ai/skills/diffusers-cli/SKILL.md @@ -17,7 +17,6 @@ description: > | `run` | Run any `DiffusionPipeline` or `ModularPipeline`. Forwards `--pipeline-kwargs` verbatim, saves output by detecting its runtime type, optionally runs on HF Jobs via `--remote`. | | `schema` | Print the input schema for a pipeline repo (kwarg names, types, defaults, descriptions). **No weights downloaded** — only the small index file. | | `serve` | Serve a pipeline over HTTP with the Generative Media Spec API: model discovery, queued generations, polling or SSE progress. In-process, or behind SGLang / vLLM-Omni via `--backend`. | -| `generate` | Send a request to a Generative Media Spec server (such as `serve`) and save the outputs. Loads no pipeline; takes `--url`, `--inputs` and `--parameters` with the server's declared names. | | `custom_blocks` | Package a local `ModularPipelineBlocks` subclass for the Hub. | | `env` | Print versions of diffusers + torch + transformers + accelerate + safetensors + CUDA + GPU info. Use when investigating environment issues, dtype/precision support, or building bug reports. | @@ -36,7 +35,6 @@ The other commands are small enough that `diffusers-cli --help` is the ```bash diffusers-cli schema --help diffusers-cli serve --help -diffusers-cli generate --help diffusers-cli custom_blocks --help diffusers-cli env --help ``` diff --git a/docs/source/en/_toctree.yml b/docs/source/en/_toctree.yml index ab98a5243059..391c128193ab 100644 --- a/docs/source/en/_toctree.yml +++ b/docs/source/en/_toctree.yml @@ -7,6 +7,8 @@ title: Quickstart - local: using-diffusers/cli title: Command line interface + - local: using-diffusers/serve + title: Serving title: Get started - isExpanded: true sections: diff --git a/docs/source/en/using-diffusers/cli.md b/docs/source/en/using-diffusers/cli.md index df25159a65e2..961c2a1ddbe8 100644 --- a/docs/source/en/using-diffusers/cli.md +++ b/docs/source/en/using-diffusers/cli.md @@ -12,7 +12,8 @@ specific language governing permissions and limitations under the License. # Command line interface -`diffusers-cli` is a command line client for running, inspecting, and packaging Diffusers pipelines. +`diffusers-cli` is a command line client for running, inspecting, and packaging Diffusers pipelines. It is also +installed as `diffusers`, so `diffusers serve` and `diffusers-cli serve` are the same command. ## Available commands @@ -22,7 +23,6 @@ specific language governing permissions and limitations under the License. | [`schema`](#schema) | Inspect a pipeline's `__call__` signature without downloading weights. | | [`run`](#run) | Run a pipeline locally or in a Hugging Face Sandbox. | | [`serve`](#serve) | Serve a pipeline over HTTP with the Generative Media Spec API. | -| [`generate`](#generate) | Send a generation to a Generative Media Spec server and save its outputs. | | [`custom_blocks`](#customblocks) | Package a local `ModularPipelineBlocks` subclass for the Hub. | | [`fp16_safetensors`](#fp16safetensors) | Convert a checkpoint to fp16 `.safetensors`. | | [`skills`](#skills) | Install pre-authored skill bundles into your AI coding agent. | @@ -296,180 +296,13 @@ hf sandbox kill ## `serve` -Serve a pipeline over HTTP. The server implements the Generative Media Spec (GMS, draft `gms/0.1`): clients -discover what the model accepts, create a generation, follow it while it runs, and read the results. It needs -the `serve` extra: +Serve a pipeline over HTTP with the Generative Media Spec API. See [Serving](./serve) for the full guide. ```bash pip install "diffusers[serve]" -diffusers-cli serve --model black-forest-labs/FLUX.1-dev --dtype bf16 +diffusers-cli serve black-forest-labs/FLUX.1-dev ``` -`serve` takes the same loading and optimization flags as [`run`](#run) (`--dtype`, `--device-map`, -`--cpu-offload`, `--compile`, `--lora`, ...), plus `--host` (default `127.0.0.1`) and `--port` (default `8000`). -The server has no authentication, so keep it on a trusted network. - -[Modular repos](../modular_diffusers/overview) are detected the same way as in `run` and served in-process. They -report no step progress, and a cancel request only takes effect once the pipeline call returns. - -| Flag | Effect | -|---|---| -| `--default-seed` | Seed used for requests that do not set one. Without it, each such request gets a random seed. | -| `--enable-cors` | Allow cross-origin requests from any origin, so a browser app on another host can call the server. | -| `--public-url` | Base URL clients reach the server at, used to build output file links. Set it behind a reverse proxy. | -| `--log-level` | Logging level for diffusers and the HTTP server (`debug`, `info`, `warning`, `error`, `critical`). | - -`GET /health` returns `{"status": "ok"}`, or `503` when an SGLang or vLLM-Omni backend has exited. Every response -carries an `x-request-id` header: the one sent with the request, or a generated one. - -Generations run one at a time. Creating one returns immediately with a `queued` generation; poll it, subscribe -to its events, or ask the server to hold the connection with `Prefer: wait=`: - -```bash -# What the model accepts: tasks, inputs, parameters with types and defaults. -curl localhost:8000/v1/models/black-forest-labs/FLUX.1-dev - -# Create a generation and wait up to 120 seconds for it. -curl localhost:8000/v1/generations -H 'Prefer: wait=120' -d '{ - "model": "black-forest-labs/FLUX.1-dev", - "inputs": {"prompt": "an astronaut riding a horse"}, - "parameters": {"width": 1024, "height": 1024, "steps": 28, "seed": 42} -}' - -# Follow step progress as server-sent events, or poll the generation. -curl -N localhost:8000/v1/generations//events -curl localhost:8000/v1/generations/ - -# Cancel a queued or running generation, or delete it together with its files. -curl -X POST localhost:8000/v1/generations//cancel -curl -X DELETE localhost:8000/v1/generations/ -``` - -A completed generation lists its outputs with a download URL each (or inline base64 with -`"response_format": "b64_json"`), the seed that was used, and the model revision. Outputs are kept under -`--output-dir` and expire after one hour. Images are PNG by default (`"output_format": "image/jpeg"` or -`"image/webp"` to change it), videos MP4, audio WAV. Media inputs are passed as -`{"type": "image", "url": "https://..."}` or `{"type": "image", "base64": "...", "media_type": "image/png"}`. - -Validation is strict: an unknown parameter, a value outside its declared range, or a missing input is rejected -with a JSON pointer to the offending field. - -```json -{"error": {"code": "invalid_request", "message": "task 'text_to_image' declares no parameter 'cfg'", "pointer": "/parameters/cfg"}} -``` - -### Remote serving (`--remote`) - -`--remote` starts the server in a [Hugging Face Sandbox](https://huggingface.co/docs/huggingface_hub/guides/sandbox) -instead of on the local machine. The command creates the sandbox, installs the dependencies, starts the server, -and streams its logs until you interrupt it; the sandbox is then deleted. - -```bash -diffusers-cli serve --model black-forest-labs/FLUX.1-dev --dtype bf16 --remote --flavor a10g-large -``` - -Once the model has loaded, the command prints the sandbox id. The server is only reachable through the sandbox -proxy, which requires your Hugging Face token, so pass the id to [`generate`](#generate) rather than calling a URL -directly: - -```bash -diffusers-cli generate --sandbox-id --inputs '{"prompt": "a cat on the moon"}' -``` - -`--flavor`, `--image`, `--dependencies`, `--namespace` and `--idle-timeout` work as they do for -[`run --remote`](#remote-execution---remote). `--model` must be a Hub repo id. A local `--manifest` file is -uploaded to the sandbox. - -With `--backend sglang` or `--backend vllm`, the sandbox starts from the engine's own image -(`lmsysorg/sglang:latest` or `vllm/vllm-omni:latest`) unless `--image` names another, so the engine does not need -installing: - -```bash -diffusers-cli serve --model black-forest-labs/FLUX.2-klein-4B --backend sglang --remote --flavor a100-large -``` - -### Manifests - -What a model accepts is described by its manifest. `serve` looks for one in this order: - -1. the file passed with `--manifest `; -2. a `gms.json` published in the model repo, served byte-for-byte; -3. a manifest derived from the pipeline's `__call__` signature. - -A derived manifest is a best-effort reading of the signature: scalar arguments become parameters, prompts and -media arguments become inputs, and arguments with a core GMS name are renamed (`num_inference_steps` becomes -`steps`, `num_images_per_prompt` becomes `num_outputs`, `generator` becomes `seed`). It carries no value ranges. -Write a `gms.json` when you need ranges, presets, or a duration in seconds. - -When a manifest uses a name that differs from the pipeline argument, map it per task under `x-diffusers`: - -```json -"x-diffusers": {"inputs": {"caption": "prompt"}, "parameters": {"cfg": "guidance_scale"}} -``` - -A task is only hosted when every input it requires maps to a pipeline argument. A parameter the pipeline cannot -set is rejected with `not_implemented` unless the request sends its default value. - -### Adapters - -A request can apply LoRAs for that generation only. They are loaded before the call and unloaded after it: - -```bash -curl localhost:8000/v1/generations -d '{ - "model": "black-forest-labs/FLUX.1-dev", - "inputs": {"prompt": "a tiny cat"}, - "adapters": [{"type": "lora", "path": "hf:alvdansen/littletinies", "scale": 0.8}] -}' -``` - -Request adapters are turned off when the server starts with `--compile` or `--lora`; such requests fail with -`not_implemented`. - -### SGLang and vLLM-Omni backends - -`--backend sglang` and `--backend vllm` run the model in [SGLang Diffusion](https://docs.sglang.io/) or -[vLLM-Omni](https://docs.vllm.ai/projects/vllm-omni) instead of in-process. `serve` starts `sglang serve` or -`vllm serve --omni` as a child process on a local port, waits for it to become healthy, and translates GMS -requests to the engine's OpenAI-compatible image and video routes. Clients only ever talk to the GMS endpoints -of `serve`. The engine must be installed separately. - -```bash -diffusers-cli serve --model Qwen/Qwen-Image --backend sglang --backend-args '--num-gpus 2' -diffusers-cli serve --model Wan-AI/Wan2.2-T2V-A14B-Diffusers --backend vllm -``` - -The pipeline flags (`--dtype`, `--cpu-offload`, `--compile`, ...) do not apply to these backends: they are -ignored with a warning. Pass the engine's own flags with `--backend-args`. Compared with the in-process backend, they serve image and video -outputs only, accept `image` as the only media input, expose the parameters both engines share (size, steps, -guidance scale, seed, number of frames, fps, number of images), report no step progress, and apply no request -adapters. - -## `generate` - -Send a generation to a server started with [`serve`](#serve), or to any other Generative Media Spec server, and -save its outputs. Unlike [`run`](#run), `generate` loads no pipeline: it reads the model's discovery document, -submits the request, polls the generation until it finishes, and downloads the files. - -```bash -diffusers-cli generate --url http://localhost:8000 --inputs '{"prompt": "a cat on the moon"}' - -diffusers-cli generate --url http://localhost:8000 \ - --inputs '{"prompt": "make the fur grey", "image": "cat.png"}' \ - --parameters '{"steps": 8, "seed": 0}' --output outputs/ -``` - -`--inputs` and `--parameters` use the names the server declares, not pipeline argument names (`steps`, not -`num_inference_steps`). An image, mask, video or audio input can be an http(s) URL or a local file path; local -files are sent inline as base64. `--task`, `--preset`, `--adapters` and `--output-format` map to the request -fields of the same name. `--model` is only needed when the server hosts more than one model. - -Outputs are saved to `~/.diffusers/cli/generate/outputs//` unless `--output` names a directory. -`--token` sends a bearer token in the `Authorization` header, for servers behind an authenticating proxy; no -token is sent by default. Interrupting the command cancels the generation on the server. - -For a server started with [`serve --remote`](#remote-serving---remote), pass `--sandbox-id ` instead of -`--url`. The request then goes through the sandbox proxy, authenticated with your Hugging Face token. - ## `custom_blocks` Package a local `ModularPipelineBlocks` subclass for upload to the Hub. Reads a Python file, AST-scans it for diff --git a/docs/source/en/using-diffusers/serve.md b/docs/source/en/using-diffusers/serve.md new file mode 100644 index 000000000000..aa6f3fba7b15 --- /dev/null +++ b/docs/source/en/using-diffusers/serve.md @@ -0,0 +1,173 @@ + + +# Serving + +`diffusers serve` serves a pipeline over HTTP. The server implements the Generative Media Spec (GMS, draft +`gms/0.1`): clients discover what the model accepts, create a generation, follow it while it runs, and read the +results. + +Install the `serve` extra and start a server: + +```bash +pip install "diffusers[serve]" +diffusers serve black-forest-labs/FLUX.1-dev +``` + +Then send it a request: + +```bash +curl localhost:8000/v1/generations -H 'Prefer: wait=120' -d '{ + "model": "black-forest-labs/FLUX.1-dev", + "inputs": {"prompt": "an astronaut riding a horse"} +}' +``` + +`serve` takes the same loading and optimization flags as [`diffusers-cli run`](./cli#run), and loads weights in +bf16 unless `--dtype` says otherwise. The server listens on `127.0.0.1:8000` by default and has no authentication, +so keep it on a trusted network. Run `diffusers serve --help` for every option. + +`--idle-timeout ` (for example `30m`) shuts the server down once it has received no requests for that +long and no generation is queued or running. Health checks don't count as requests. By default a local server +never shuts itself down. + +[Modular repos](../modular_diffusers/overview) are detected the same way as in `run` and served in-process. They +report no step progress, and a cancel request only takes effect once the pipeline call returns. + +## Generations + +Generations run one at a time. `POST /v1/generations` returns immediately with a `queued` generation. Poll it, +subscribe to its events, or ask the server to hold the connection with `Prefer: wait=`: + +```bash +# Create a generation and wait up to 120 seconds for it. +curl localhost:8000/v1/generations -H 'Prefer: wait=120' -d '{ + "model": "black-forest-labs/FLUX.1-dev", + "inputs": {"prompt": "an astronaut riding a horse"}, + "parameters": {"width": 1024, "height": 1024, "steps": 28, "seed": 42} +}' + +# Follow step progress as server-sent events, or poll the generation. +curl -N localhost:8000/v1/generations//events +curl localhost:8000/v1/generations/ + +# Cancel a queued or running generation, or delete it together with its files. +curl -X POST localhost:8000/v1/generations//cancel +curl -X DELETE localhost:8000/v1/generations/ +``` + +A request has two groups of fields. `inputs` holds the prompt and any media. `parameters` holds settings such as +size, steps and seed. Both use the names the server declares, which can differ from the pipeline's argument names +(`steps`, not `num_inference_steps`). Media inputs are passed as `{"type": "image", "url": "https://..."}` or +`{"type": "image", "base64": "...", "media_type": "image/png"}`. + +A completed generation lists its outputs with a download URL each (or inline base64 with +`"response_format": "b64_json"`), the seed that was used, and the model revision. Outputs expire after one hour. +Images are PNG by default (`"output_format": "image/jpeg"` or `"image/webp"` to change it), videos MP4, audio WAV. + +Validation is strict: an unknown parameter, a value outside its declared range, or a missing input is rejected +with a JSON pointer to the offending field. + +```json +{"error": {"code": "invalid_request", "message": "task 'text_to_image' declares no parameter 'cfg'", "pointer": "/parameters/cfg"}} +``` + +When a model hosts more than one task (for example `text_to_image` and `image_to_image`), the server picks the task +from the inputs. Set `"task"` in the request when the inputs match more than one. + +### Adapters + +A request can apply LoRAs for that generation only. They are loaded before the call and unloaded after it: + +```bash +curl localhost:8000/v1/generations -d '{ + "model": "black-forest-labs/FLUX.1-dev", + "inputs": {"prompt": "a tiny cat"}, + "adapters": [{"type": "lora", "path": "hf:alvdansen/littletinies", "scale": 0.8}] +}' +``` + +Request adapters are turned off when the server starts with `--compile` or `--lora`. Such requests fail with +`not_implemented`. + +## Discovery and manifests + +`GET /v1/models/` returns what the model accepts: its tasks, inputs, and parameters with types and defaults. +`GET /health` returns `{"status": "ok"}`. + +```bash +curl localhost:8000/v1/models/black-forest-labs/FLUX.1-dev +``` + +This document is the model's manifest. `serve` looks for one in this order: + +1. the file passed with `--manifest `; +2. a `gms.json` published in the model repo, served byte-for-byte; +3. a manifest derived from the pipeline's `__call__` signature. + +A derived manifest is a best-effort reading of the signature: scalar arguments become parameters, prompts and +media arguments become inputs, and arguments with a core GMS name are renamed (`num_inference_steps` becomes +`steps`, `num_images_per_prompt` becomes `num_outputs`, `generator` becomes `seed`). It carries no value ranges. +Write a `gms.json` when you need ranges, presets, or a duration in seconds. + +When a manifest uses a name that differs from the pipeline argument, map it per task under `x-diffusers`: + +```json +"x-diffusers": {"inputs": {"caption": "prompt"}, "parameters": {"cfg": "guidance_scale"}} +``` + +A task is only hosted when every input it requires maps to a pipeline argument. A parameter the pipeline cannot +set is rejected with `not_implemented` unless the request sends its default value. + +## Remote serving + +`--remote` starts the server in a [Hugging Face Job](https://huggingface.co/docs/huggingface_hub/guides/jobs) +instead of on the local machine. The command starts the Job, which installs the dependencies and runs the server, +and streams its logs until you interrupt it. Interrupting the command cancels the Job. + +```bash +diffusers serve black-forest-labs/FLUX.1-dev --remote --flavor a10g-large +``` + +Once the model has loaded, the command prints the server URL, `https://--8000.hf.jobs`. Requests +authenticate with your Hugging Face token: + +```bash +curl /v1/generations -H 'Prefer: wait=120' -H "Authorization: Bearer $HF_TOKEN" \ + -d '{"model": "black-forest-labs/FLUX.1-dev", "inputs": {"prompt": "a cat on the moon"}}' +``` + +The Job ends when the server shuts down. `--idle-timeout` defaults to `30m`, and `--timeout` (default `24h`) caps +how long the Job runs even while it is busy. `--flavor`, `--image`, `--dependencies` and `--namespace` work as they +do for [`run --remote`](./cli#remote-execution---remote). The model must be a Hub repo id. `--manifest` is not +supported with `--remote`: publish the manifest as `gms.json` in the model repo instead. + +## SGLang and vLLM-Omni backends + +`--backend sglang` and `--backend vllm` run the model in [SGLang Diffusion](https://docs.sglang.io/) or +[vLLM-Omni](https://docs.vllm.ai/projects/vllm-omni) instead of in-process. `serve` starts `sglang serve` or +`vllm serve --omni` as a child process, waits for it to become healthy, and translates GMS requests to the +engine's OpenAI-compatible image and video routes. Clients only talk to the GMS endpoints of `serve`. The engine +must be installed separately. + +```bash +diffusers serve Qwen/Qwen-Image --backend sglang --backend-args '--num-gpus 2' +diffusers serve Wan-AI/Wan2.2-T2V-A14B-Diffusers --backend vllm +``` + +The pipeline flags (`--dtype`, `--cpu-offload`, `--compile`, ...) are ignored with a warning. Pass the engine's own +flags with `--backend-args`. Compared with the in-process backend, these backends serve image and video outputs +only, accept `image` as the only media input, expose only the parameters both engines share, report no step +progress, and apply no request adapters. `GET /health` returns `503` once the engine has exited. + +With `--remote`, the Job starts from the engine's own image (`lmsysorg/sglang:latest` or +`vllm/vllm-omni:latest`) unless `--image` names another. diff --git a/setup.py b/setup.py index 494a7d0a27ba..346e6fa8974c 100644 --- a/setup.py +++ b/setup.py @@ -290,7 +290,12 @@ def run(self): python_requires=">=3.10.0", install_requires=list(install_requires), extras_require=extras, - entry_points={"console_scripts": ["diffusers-cli=diffusers.commands.diffusers_cli:main"]}, + entry_points={ + "console_scripts": [ + "diffusers-cli=diffusers.commands.diffusers_cli:main", + "diffusers=diffusers.commands.diffusers_cli:main", + ] + }, classifiers=[ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", diff --git a/src/diffusers/commands/diffusers_cli.py b/src/diffusers/commands/diffusers_cli.py index 1df91bd7ddb4..32252a386d1e 100644 --- a/src/diffusers/commands/diffusers_cli.py +++ b/src/diffusers/commands/diffusers_cli.py @@ -20,7 +20,6 @@ from .custom_blocks import CustomBlocksCommand from .env import EnvironmentCommand from .fp16_safetensors import FP16SafetensorsCommand -from .generate import GenerateCommand from .run import RunCommand from .schema import SchemaCommand from .serve import ServeCommand @@ -49,7 +48,6 @@ def main(): EnvironmentCommand.register_subcommand(commands_parser) FP16SafetensorsCommand.register_subcommand(commands_parser) CustomBlocksCommand.register_subcommand(commands_parser) - GenerateCommand.register_subcommand(commands_parser) RunCommand.register_subcommand(commands_parser) SchemaCommand.register_subcommand(commands_parser) ServeCommand.register_subcommand(commands_parser) diff --git a/src/diffusers/commands/generate.py b/src/diffusers/commands/generate.py deleted file mode 100644 index aeebd03c1b59..000000000000 --- a/src/diffusers/commands/generate.py +++ /dev/null @@ -1,260 +0,0 @@ -# Copyright 2026 The HuggingFace Team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""`diffusers-cli generate` — send a generation to a Generative Media Spec (GMS) server and save its outputs. - -The client side of `diffusers-cli serve`: it loads no pipeline. It reads the model's discovery document, submits the -request, polls the generation until it finishes, and downloads the files. -""" - -from __future__ import annotations - -import base64 -import json -import mimetypes -import time -from argparse import ArgumentParser, Namespace, RawDescriptionHelpFormatter, _SubParsersAction -from pathlib import Path -from typing import Any - -from huggingface_hub.cli._output import out -from huggingface_hub.utils import httpx - -from . import BaseDiffusersCLICommand - - -DEFAULT_OUTPUT_DIR = str(Path.home() / ".diffusers" / "cli" / "generate" / "outputs") -_MEDIA_TYPES = ("image", "mask", "video", "audio") -_POLL_SECONDS = 1.0 -_TIMEOUT_SECONDS = 60.0 - - -def _parse_json(raw: str | None, flag: str, expected: type) -> Any: - if raw is None: - return None - try: - parsed = json.loads(raw) - except json.JSONDecodeError as e: - raise SystemExit(f"{flag} must be valid JSON: {e}") from e - if not isinstance(parsed, expected): - raise SystemExit(f"{flag} must decode to a JSON {'object' if expected is dict else 'array'}.") - return parsed - - -def _media_value(name: str, media_type: str, value: Any) -> Any: - """Turn a URL or local path into a GMS media value; anything else is sent as given.""" - if isinstance(value, list): - return [_media_value(name, media_type, item) for item in value] - if not isinstance(value, str): - return value - if value.startswith(("http://", "https://")): - return {"type": media_type, "url": value} - - path = Path(value).expanduser() - if not path.is_file(): - raise SystemExit(f"Input {name!r} is neither an http(s) URL nor an existing file: {value!r}") - media = {"type": media_type, "base64": base64.b64encode(path.read_bytes()).decode()} - guessed = mimetypes.guess_type(path.name)[0] - if guessed is not None: - media["media_type"] = guessed - return media - - -class GenerateCommand(BaseDiffusersCLICommand): - task = "generate" - - @staticmethod - def register_subcommand(subparsers: _SubParsersAction) -> None: - epilog = ( - "Examples\n" - ' $ diffusers-cli generate --url http://localhost:8000 --inputs \'{"prompt": "a cat on the moon"}\'\n' - " $ diffusers-cli generate --url http://localhost:8000 \\\n" - ' --inputs \'{"prompt": "make the fur grey", "image": "cat.png"}\' \\\n' - ' --parameters \'{"steps": 8, "seed": 0}\' -o outputs/\n' - ' $ diffusers-cli generate --sandbox-id --inputs \'{"prompt": "a cat on the moon"}\'\n' - "\n" - "Learn more\n" - " Use `diffusers-cli --help` for more information about a command.\n" - " Read the documentation at https://huggingface.co/docs/diffusers\n" - ) - - parser: ArgumentParser = subparsers.add_parser( - "generate", - help="Send a generation to a Generative Media Spec server and save its outputs.", - usage="\n diffusers-cli generate (--url | --sandbox-id ) --inputs [options]", - epilog=epilog, - formatter_class=RawDescriptionHelpFormatter, - ) - parser._optionals.title = "Options" - target = parser.add_mutually_exclusive_group(required=True) - target.add_argument("--url", default=None, help="Base URL of the server, e.g. http://localhost:8000.") - target.add_argument( - "--sandbox-id", - default=None, - help="Id of a sandbox started with `diffusers-cli serve --remote`, reached through the sandbox proxy.", - ) - parser.add_argument( - "--model", "-m", default=None, help="Model id to request. Defaults to the server's only model." - ) - parser.add_argument( - "--inputs", - required=True, - metavar="JSON", - help=( - "JSON object of the task's inputs. An image, mask, video or audio input given as an http(s) URL " - "or a local file path is converted to a media value; local files are sent inline as base64." - ), - ) - parser.add_argument("--parameters", default=None, metavar="JSON", help="JSON object of task parameters.") - parser.add_argument( - "--task", default=None, help="Task to run. Defaults to the one the server infers from the inputs." - ) - parser.add_argument("--preset", default=None, help="Named preset of the task to start from.") - parser.add_argument( - "--adapters", - default=None, - metavar="JSON", - help='JSON array of adapters to apply, e.g. \'[{"type": "lora", "path": "hf:org/name", "scale": 0.8}]\'.', - ) - parser.add_argument("--output-format", default=None, help="Media type to encode outputs as, e.g. image/jpeg.") - parser.add_argument( - "--output", - "-o", - default=None, - help=f"Directory to save outputs in. Defaults to {DEFAULT_OUTPUT_DIR}//.", - ) - parser.add_argument( - "--token", - default=None, - help=( - "With --url: bearer token sent in the Authorization header; none is sent by default. " - "With --sandbox-id: Hugging Face token used to reach the sandbox; defaults to the logged-in one." - ), - ) - parser.set_defaults(func=GenerateCommand) - - def __init__(self, args: Namespace): - self.args = args - - def run(self) -> None: - args = self.args - inputs = _parse_json(args.inputs, "--inputs", dict) - parameters = _parse_json(args.parameters, "--parameters", dict) - adapters = _parse_json(args.adapters, "--adapters", list) - - with self._client() as client: - model = args.model or self._only_model(client) - document = self._request(client, "GET", f"/v1/models/{model}") - tasks = document["manifest"]["tasks"] - if args.task is not None and args.task not in tasks: - raise SystemExit(f"Model {model!r} declares no task {args.task!r}; its tasks are {list(tasks)}.") - - declared = [tasks[args.task]] if args.task is not None else list(tasks.values()) - input_types = {name: spec.get("type") for task in declared for name, spec in task["inputs"].items()} - body: dict[str, Any] = {"model": model, "inputs": {}} - for name, value in inputs.items(): - if input_types.get(name) in _MEDIA_TYPES: - value = _media_value(name, input_types[name], value) - body["inputs"][name] = value - optional = { - "task": args.task, - "preset": args.preset, - "parameters": parameters, - "adapters": adapters, - "output_format": args.output_format, - } - body.update({key: value for key, value in optional.items() if value is not None}) - - generation = self._request(client, "POST", "/v1/generations", json=body) - try: - while generation["status"] in ("queued", "generating"): - time.sleep(_POLL_SECONDS) - generation = self._request(client, "GET", f"/v1/generations/{generation['id']}") - except KeyboardInterrupt: - # Without this the server keeps generating for a client that has gone away. - self._request(client, "POST", f"/v1/generations/{generation['id']}/cancel") - raise - if generation["status"] != "complete": - raise SystemExit(f"Generation {generation['id']} ended with status {generation['status']!r}.") - - directory = Path(args.output or Path(DEFAULT_OUTPUT_DIR) / generation["id"]).expanduser() - directory.mkdir(parents=True, exist_ok=True) - saved = [] - for name, artifacts in generation["outputs"].items(): - for index, artifact in enumerate(artifacts): - path = directory / f"{name}-{index}.{artifact['media_type'].split('/')[1]}" - path.write_bytes(self._artifact_bytes(client, artifact)) - saved.append(str(path)) - - out.result( - self.task, - url=str(client.base_url), - model=model, - generation=generation["id"], - seed=generation.get("seed"), - outputs=saved, - ) - - def _client(self) -> httpx.Client: - if self.args.sandbox_id is not None: - from huggingface_hub import Sandbox - - from .serve.remote import SANDBOX_PORT - - sbx = Sandbox.connect(self.args.sandbox_id, token=self.args.token) - return httpx.Client( - base_url=sbx.proxy_url_for(SANDBOX_PORT), - headers=sbx.proxy_headers, - timeout=_TIMEOUT_SECONDS, - follow_redirects=True, - ) - headers = {"Authorization": f"Bearer {self.args.token}"} if self.args.token else {} - return httpx.Client(base_url=self.args.url, headers=headers, timeout=_TIMEOUT_SECONDS, follow_redirects=True) - - def _request(self, client: httpx.Client, method: str, path: str, **kwargs: Any) -> Any: - try: - response = client.request(method, path, **kwargs) - except httpx.HTTPError as e: - raise SystemExit(f"Could not reach {client.base_url}: {type(e).__name__}: {e}") from e - if response.status_code < 400: - return response.json() - - try: - error = response.json()["error"] - message = f"{error['code']}: {error['message']}" - if error.get("pointer"): - message += f" (at {error['pointer']})" - except (ValueError, KeyError, TypeError): - message = response.text - raise SystemExit(f"{method} {path} failed with HTTP {response.status_code}: {message}") - - def _only_model(self, client: httpx.Client) -> str: - models = [model["id"] for model in self._request(client, "GET", "/v1/models")["data"]] - if len(models) != 1: - raise SystemExit(f"{client.base_url} serves {len(models)} models ({models}); pick one with --model.") - return models[0] - - def _artifact_bytes(self, client: httpx.Client, artifact: dict[str, Any]) -> bytes: - if "base64" in artifact: - return base64.b64decode(artifact["base64"]) - try: - # The token is for the server; an output hosted elsewhere is fetched without it. - if httpx.URL(artifact["url"]).host == client.base_url.host: - response = client.get(artifact["url"]) - else: - response = httpx.get(artifact["url"], timeout=_TIMEOUT_SECONDS, follow_redirects=True) - response.raise_for_status() - except httpx.HTTPError as e: - raise SystemExit(f"Could not download output {artifact['url']}: {type(e).__name__}: {e}") from e - return response.content diff --git a/src/diffusers/commands/run.py b/src/diffusers/commands/run.py index ebb453ead4d9..31f7cadccd18 100644 --- a/src/diffusers/commands/run.py +++ b/src/diffusers/commands/run.py @@ -156,8 +156,10 @@ # --------------------------------------------------------------------------- -def _add_loading_arguments(parser: ArgumentParser) -> None: - parser.add_argument("--model", "-m", required=True, help="Model id on the Hugging Face Hub or local path.") +def _add_loading_arguments(parser: ArgumentParser, require_model: bool = True) -> None: + parser.add_argument( + "--model", "-m", required=require_model, help="Model id on the Hugging Face Hub or local path." + ) parser.add_argument( "--device-map", default=None, diff --git a/src/diffusers/commands/serve/__init__.py b/src/diffusers/commands/serve/__init__.py index ec35ea631bfa..904f6d595e28 100644 --- a/src/diffusers/commands/serve/__init__.py +++ b/src/diffusers/commands/serve/__init__.py @@ -22,14 +22,17 @@ from __future__ import annotations import inspect +import threading +import time from argparse import ArgumentParser, Namespace, RawDescriptionHelpFormatter, _SubParsersAction from pathlib import Path +from typing import Any from huggingface_hub.cli._output import out from ...utils import logging from .. import BaseDiffusersCLICommand -from ..run import _add_loading_arguments, _add_optimization_arguments +from ..run import _add_loading_arguments, _add_optimization_arguments, _duration_to_seconds from .remote import add_remote_arguments, serve_remote @@ -41,7 +44,7 @@ # Flags that configure the in-process pipeline, paired with the value argparse gives them when unset. _DIFFUSERS_ONLY_FLAGS = { "device_map": None, - "dtype": "auto", + "dtype": "bf16", "variant": None, "lora": None, "cpu_offload": None, @@ -92,6 +95,19 @@ def _resolve_pipeline(args: Namespace) -> tuple[type, inspect.Signature, str | N return type(pipeline), blocks_signature(pipeline.blocks), commit_hash +def _shut_down_when_idle(server: Any, app: Any, generations: Any, idle_timeout: str) -> None: + limit = _duration_to_seconds(idle_timeout) + while True: + time.sleep(min(limit, 30.0)) + if generations.busy(): + app.state.last_request = time.monotonic() + continue + if time.monotonic() - app.state.last_request >= limit: + logger.info(f"No requests for {idle_timeout}; shutting down.") + server.should_exit = True + return + + class ServeCommand(BaseDiffusersCLICommand): task = "serve" @@ -99,10 +115,10 @@ class ServeCommand(BaseDiffusersCLICommand): def register_subcommand(subparsers: _SubParsersAction) -> None: epilog = ( "Examples\n" - " $ diffusers-cli serve -m black-forest-labs/FLUX.1-dev --dtype bf16\n" - " $ diffusers-cli serve -m black-forest-labs/FLUX.1-dev --dtype bf16 --compile --port 9000\n" - " $ diffusers-cli serve -m Wan-AI/Wan2.2-T2V-A14B-Diffusers --backend vllm\n" - " $ diffusers-cli serve -m Qwen/Qwen-Image --backend sglang --backend-args '--num-gpus 2'\n" + " $ diffusers-cli serve black-forest-labs/FLUX.1-dev\n" + " $ diffusers-cli serve black-forest-labs/FLUX.1-dev --compile --port 9000\n" + " $ diffusers-cli serve Wan-AI/Wan2.2-T2V-A14B-Diffusers --backend vllm\n" + " $ diffusers-cli serve Qwen/Qwen-Image --backend sglang --backend-args '--num-gpus 2'\n" "\n" " $ curl localhost:8000/v1/generations -H 'Prefer: wait=120' \\\n" ' -d \'{"model": "black-forest-labs/FLUX.1-dev", "inputs": {"prompt": "a cat on the moon"}}\'\n' @@ -115,12 +131,19 @@ def register_subcommand(subparsers: _SubParsersAction) -> None: parser: ArgumentParser = subparsers.add_parser( "serve", help="Serve a diffusers pipeline over HTTP with the Generative Media Spec API.", - usage="\n diffusers-cli serve [options]", + usage="\n diffusers-cli serve [options]", epilog=epilog, formatter_class=RawDescriptionHelpFormatter, ) parser._optionals.title = "Options" - _add_loading_arguments(parser) + parser.add_argument( + "model_id", + nargs="?", + default=None, + metavar="MODEL", + help="Model id on the Hugging Face Hub or local path.", + ) + _add_loading_arguments(parser, require_model=False) _add_optimization_arguments(parser) parser.add_argument( "--backend", @@ -195,15 +218,36 @@ def register_subcommand(subparsers: _SubParsersAction) -> None: default=None, help="Logging level for diffusers and the HTTP server. Defaults to the current diffusers verbosity.", ) + parser.add_argument( + "--idle-timeout", + default=None, + help=( + "Shut the server down after this long without requests while no generation is queued or running " + "(e.g. 30m, 2h). Defaults to never, or to 30m with --remote." + ), + ) add_remote_arguments(parser) # `_load_pipeline` reads these `run` flags, which `serve` does not expose. - parser.set_defaults(func=ServeCommand, workflow=None, offload_margin=None) + # Most served models are DiTs trained in bf16; `run`'s `auto` would load them in fp32. + parser.set_defaults(func=ServeCommand, workflow=None, offload_margin=None, dtype="bf16") def __init__(self, args: Namespace): self.args = args def run(self) -> None: args = self.args + if args.model_id is not None and args.model is not None and args.model_id != args.model: + raise SystemExit(f"Got two models: {args.model_id!r} and --model {args.model!r}. Pass only one.") + args.model = args.model or args.model_id + # Cleared so `serve --remote` does not forward it to the Job as `--model-id`. + args.model_id = None + if args.model is None: + raise SystemExit("Pass the model to serve, e.g. `diffusers-cli serve black-forest-labs/FLUX.1-dev`.") + if args.idle_timeout is not None: + try: + _duration_to_seconds(args.idle_timeout) + except ValueError: + raise SystemExit(f"--idle-timeout {args.idle_timeout!r} is not a duration such as 30m or 2h.") if args.log_level is not None: logging.set_verbosity(logging.get_log_levels_dict()[args.log_level]) @@ -279,9 +323,10 @@ def run(self) -> None: backend = EngineBackend.launch(args, served) try: + generations = GenerationQueue(backend, args.output_dir) app = create_app( served, - GenerationQueue(backend, args.output_dir), + generations, default_seed=args.default_seed, enable_cors=args.enable_cors, public_url=args.public_url, @@ -295,6 +340,16 @@ def run(self) -> None: tasks=list(served.tasks), capabilities=served.capabilities, ) - uvicorn.run(app, host=args.host, port=args.port, log_level=args.log_level or "info") + server = uvicorn.Server( + uvicorn.Config(app, host=args.host, port=args.port, log_level=args.log_level or "info") + ) + if args.idle_timeout is not None: + threading.Thread( + target=_shut_down_when_idle, + args=(server, app, generations, args.idle_timeout), + name="diffusers-serve-idle", + daemon=True, + ).start() + server.run() finally: backend.close() diff --git a/src/diffusers/commands/serve/app.py b/src/diffusers/commands/serve/app.py index 706ef4c0e4d5..babb5dab972f 100644 --- a/src/diffusers/commands/serve/app.py +++ b/src/diffusers/commands/serve/app.py @@ -20,6 +20,7 @@ import base64 import json import re +import time import uuid from typing import Any from urllib.parse import urlparse @@ -92,10 +93,14 @@ def create_app( # With that path as the root path, routing accepts both forms. root_path = urlparse(public_url).path.rstrip("/") if public_url is not None else "" app = FastAPI(title="diffusers-cli serve", docs_url=None, redoc_url=None, openapi_url=None, root_path=root_path) + app.state.last_request = time.monotonic() @app.middleware("http") async def request_id(request: Request, call_next) -> Response: request.state.request_id = request.headers.get(REQUEST_ID_HEADER) or str(uuid.uuid4()) + # Health checks come from monitors rather than users, and must not keep an idle server alive. + if not request.url.path.rstrip("/").endswith("/health"): + app.state.last_request = time.monotonic() response = await call_next(request) response.headers[REQUEST_ID_HEADER] = request.state.request_id return response diff --git a/src/diffusers/commands/serve/generations.py b/src/diffusers/commands/serve/generations.py index 592ca965d54a..b33e277963ad 100644 --- a/src/diffusers/commands/serve/generations.py +++ b/src/diffusers/commands/serve/generations.py @@ -108,11 +108,15 @@ def __init__(self, backend: Any, output_dir: str): self._generations: dict[str, Generation] = {} self._lock = threading.Lock() self._pending: queue.Queue[Generation] = queue.Queue() + self._running = False threading.Thread(target=self._work, name="diffusers-serve-worker", daemon=True).start() def failure(self) -> str | None: return self._backend.failure() + def busy(self) -> bool: + return self._running or not self._pending.empty() + def submit(self, generation: Generation) -> None: self._prune() with self._lock: @@ -165,6 +169,7 @@ def _work(self) -> None: if generation.done: continue generation.status = "generating" + self._running = True generation.directory = self._output_dir / generation.id started = time.perf_counter() @@ -190,5 +195,6 @@ def _work(self) -> None: generation.outputs = outputs generation.usage = {"compute_seconds": round(time.perf_counter() - started, 2)} generation.finish("complete") + self._running = False if deleted or generation.status != "complete": self._remove_files(generation) diff --git a/src/diffusers/commands/serve/remote.py b/src/diffusers/commands/serve/remote.py index 7743ebf0e444..0cd97fd1b9ee 100644 --- a/src/diffusers/commands/serve/remote.py +++ b/src/diffusers/commands/serve/remote.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""`diffusers-cli serve --remote` — run the server in a Hugging Face Sandbox, reachable through the sandbox proxy.""" +"""`diffusers-cli serve --remote` — run the server in a Hugging Face Job, reachable on the port the Job exposes.""" from __future__ import annotations @@ -23,143 +23,159 @@ from pathlib import Path from typing import Any -from huggingface_hub import Sandbox, get_token +from huggingface_hub import HfApi, get_token from huggingface_hub.cli._output import out from huggingface_hub.utils import httpx from ...utils import logging +from ...utils.constants import DIFFUSERS_TRUST_REMOTE_KERNELS from ..run import _DEFAULT_REMOTE_DEPS, _DEFAULT_REMOTE_IMAGE, _build_task_kwargs, _kwargs_to_argv logger = logging.get_logger("diffusers-cli/serve") -# The port the server binds inside the sandbox. `generate --sandbox-id` reaches it through the proxy. -SANDBOX_PORT = 8000 -_SANDBOX_MANIFEST_PATH = "/tmp/diffusers-cli/gms.json" +# The port the server binds inside the Job. The Jobs proxy publishes it at `https://--.hf.jobs`. +JOB_PORT = 8000 _SERVE_DEPS = ("fastapi", "uvicorn") +_DEFAULT_REMOTE_IDLE_TIMEOUT = "30m" # Each engine's own image carries the engine, its kernels and the CUDA toolchain they are built with. _ENGINE_IMAGES = {"sglang": "lmsysorg/sglang:latest", "vllm": "vllm/vllm-omni:latest"} _READY_POLL_SECONDS = 5.0 -# Flags that say how the sandbox is set up, or that the sandbox server sets itself. -_LOCAL_KEYS = ("remote", "flavor", "dependencies", "namespace", "image", "idle_timeout", "func", "format") -_SANDBOX_SET_KEYS = ("host", "port", "public_url", "output_dir") +# Flags the Job command sets itself. `_build_task_kwargs` already drops the flags that say how the Job is set up. +_JOB_SET_KEYS = ("host", "port", "public_url", "output_dir") def add_remote_arguments(parser: ArgumentParser) -> None: parser.add_argument( "--remote", action="store_true", - help="Run the server in a Hugging Face Sandbox instead of on the local machine.", + help="Run the server in a Hugging Face Job instead of on the local machine.", ) parser.add_argument( "--flavor", default="a10g-small", - help="HF Sandbox hardware flavor for --remote (e.g. a10g-small, a100-large).", + help="HF Jobs hardware flavor for --remote (e.g. a10g-small, a100-large).", ) parser.add_argument( "--dependencies", action="append", default=None, - help="Extra pip dependencies to install in the sandbox. Repeat to add multiple.", + help="Extra pip dependencies to install in the Job. Repeat to add multiple.", ) parser.add_argument( "--namespace", default=None, - help="HF namespace to create the sandbox under (defaults to the current user).", + help="HF namespace to run the Job under (defaults to the current user).", ) parser.add_argument( "--image", default=None, help=( - f"Sandbox image for --remote. Defaults to {_DEFAULT_REMOTE_IMAGE!r}, or to the engine's own image " + f"Job image for --remote. Defaults to {_DEFAULT_REMOTE_IMAGE!r}, or to the engine's own image " f"with --backend sglang ({_ENGINE_IMAGES['sglang']!r}) or vllm ({_ENGINE_IMAGES['vllm']!r})." ), ) parser.add_argument( - "--idle-timeout", - default="30m", - help="Auto-shutdown the sandbox after this much inactivity (e.g. 30m, 2h). Defaults to 30m.", + "--timeout", + default="24h", + help=( + "Hard limit on how long the Job runs with --remote, busy or not (e.g. 2h, 1d). A backstop for " + "--idle-timeout. Defaults to 24h." + ), ) -def _announce_when_ready(sbx: Any, args: Namespace, stopped: threading.Event) -> None: - health_url = sbx.proxy_url_for(SANDBOX_PORT, "/health") +def _announce_when_ready(api: HfApi, job: Any, args: Namespace, stopped: threading.Event) -> None: while not stopped.wait(_READY_POLL_SECONDS): + urls = api.inspect_job(job_id=job.id, namespace=job.owner.name).status.expose_urls + if not urls: + continue try: - ready = httpx.get(health_url, headers=sbx.proxy_headers, timeout=10.0).status_code == 200 + response = httpx.get(f"{urls[0]}/health", headers={"Authorization": f"Bearer {api.token}"}, timeout=10.0) except httpx.HTTPError: - ready = False - if not ready: + continue + if response.status_code != 200: continue out.result( "remote-serve", - sandbox_id=sbx.id, - url=sbx.proxy_url_for(SANDBOX_PORT), + job_id=job.id, + url=urls[0], model=args.model, - hint=f'diffusers-cli generate --sandbox-id {sbx.id} --inputs \'{{"prompt": "..."}}\'', + # The HF token is left as a variable so it is not printed to the terminal. + hint=f'curl {urls[0]}/v1/models -H "Authorization: Bearer $HF_TOKEN"', ) return +def _job_script(args: Namespace, task: str) -> str: + """The shell command the Job runs: install the dependencies, then start the server in the foreground.""" + task_kwargs = _build_task_kwargs(args) + for key in _JOB_SET_KEYS: + task_kwargs.pop(key, None) + task_kwargs["host"] = "0.0.0.0" + task_kwargs["port"] = JOB_PORT + task_kwargs["idle_timeout"] = args.idle_timeout + + dependencies = [*_DEFAULT_REMOTE_DEPS, *_SERVE_DEPS, *(args.dependencies or [])] + install = shlex.join(["uv", "pip", "install", "--system", "--break-system-packages", *dependencies]) + # An image can put another environment first on PATH, with its own older `diffusers-cli` (the SGLang image + # does), so the server is started with the interpreter the install went into. The Job id is only known + # inside the Job, and output links must be built on the public address. + start = ( + 'exec "$(uv python find --system)" -m diffusers.commands.diffusers_cli --format quiet ' + f"{shlex.join(_kwargs_to_argv(task, task_kwargs))} " + f'--public-url "https://${{JOB_ID}}--{JOB_PORT}.hf.jobs"' + ) + return f"{install} && {start}" + + def serve_remote(args: Namespace, task: str) -> None: if Path(args.model).exists(): raise SystemExit( - f"--model {args.model!r} is a local path; the sandbox can't see it. " - "Pass a Hub repo id so the sandbox can download it." + f"--model {args.model!r} is a local path; the Job can't see it. " + "Pass a Hub repo id so the Job can download it." + ) + if args.manifest is not None: + raise SystemExit( + f"--manifest is not supported with --remote: the Job can't read {args.manifest!r}. Upload it as " + f"`gms.json` to the model repo, e.g. `hf upload {args.model} {args.manifest} gms.json`, and the " + "remote server will use it." ) - if args.manifest is not None and not Path(args.manifest).is_file(): - raise SystemExit(f"--manifest {args.manifest!r} is not a file.") + if args.idle_timeout is None: + args.idle_timeout = _DEFAULT_REMOTE_IDLE_TIMEOUT hf_token = args.token or get_token() - logger.info(f"creating sandbox on flavor={args.flavor!r}...") - create_kwargs: dict[str, Any] = { - "image": args.image or _ENGINE_IMAGES.get(args.backend, _DEFAULT_REMOTE_IMAGE), - "flavor": args.flavor, - "forward_hf_token": True, - "token": hf_token, - "env": {"HF_ENABLE_PARALLEL_LOADING": "1"}, - "idle_timeout": args.idle_timeout, - # The 120s default expires while a cold node pulls the multi-GB pytorch base image or - # waits for GPU capacity, which fails the launch before it starts. - "start_timeout": 600.0, - } - if args.namespace is not None: - create_kwargs["namespace"] = args.namespace - sbx = Sandbox.create(**create_kwargs) - - def _stream(chunk: str) -> None: - sys.stderr.write(chunk) - sys.stderr.flush() + env = {"HF_ENABLE_PARALLEL_LOADING": "1"} + if DIFFUSERS_TRUST_REMOTE_KERNELS: + env["DIFFUSERS_TRUST_REMOTE_KERNELS"] = "true" + + api = HfApi(token=hf_token) + logger.info(f"starting a Job on flavor={args.flavor!r}...") + job = api.run_job( + image=args.image or _ENGINE_IMAGES.get(args.backend, _DEFAULT_REMOTE_IMAGE), + command=["sh", "-c", _job_script(args, task)], + env=env, + secrets={"HF_TOKEN": hf_token} if hf_token else None, + flavor=args.flavor, + timeout=args.timeout, + expose=[JOB_PORT], + namespace=args.namespace, + ) + stage = None stopped = threading.Event() try: - dependencies = [*_DEFAULT_REMOTE_DEPS, *_SERVE_DEPS, *(args.dependencies or [])] - install_cmd = shlex.join(["uv", "pip", "install", "--system", "--break-system-packages", *dependencies]) - logger.info("installing dependencies in the sandbox...") - sbx.run(install_cmd, on_stdout=_stream, on_stderr=_stream) - # An image can put another environment first on PATH, with its own older `diffusers-cli` - # (the SGLang image does), so the server is started with the interpreter the install went into. - python = sbx.run(["uv", "python", "find", "--system"]).stdout.strip() - - task_kwargs = _build_task_kwargs(args) - for key in (*_LOCAL_KEYS, *_SANDBOX_SET_KEYS): - task_kwargs.pop(key, None) - if args.manifest is not None: - sbx.files.upload(args.manifest, _SANDBOX_MANIFEST_PATH) - task_kwargs["manifest"] = _SANDBOX_MANIFEST_PATH - task_kwargs["port"] = SANDBOX_PORT - # Clients reach the server through the proxy, so output links must be built on the proxied address. - task_kwargs["public_url"] = sbx.proxy_url_for(SANDBOX_PORT) - cli_module = [python, "-m", "diffusers.commands.diffusers_cli"] - cli_argv = [*cli_module, "--format", "quiet", *_kwargs_to_argv(task, task_kwargs)] - - threading.Thread(target=_announce_when_ready, args=(sbx, args, stopped), daemon=True).start() - logger.info(f"starting the server in sandbox {sbx.id}; stop it with Ctrl-C.") - result = sbx.run(cli_argv, on_stdout=_stream, on_stderr=_stream, check=False, capture_output=False) + threading.Thread(target=_announce_when_ready, args=(api, job, args, stopped), daemon=True).start() + logger.info(f"streaming the logs of Job {job.id}; stop it with Ctrl-C.") + for line in api.fetch_job_logs(job_id=job.id, namespace=job.owner.name, follow=True): + sys.stderr.write(line + "\n") + sys.stderr.flush() + stage = api.inspect_job(job_id=job.id, namespace=job.owner.name).status.stage finally: stopped.set() - sbx.kill() + if stage is None: + api.cancel_job(job_id=job.id, namespace=job.owner.name) - if result.exit_code != 0: - raise SystemExit(f"remote server exited with code {result.exit_code}") + if stage != "COMPLETED": + raise SystemExit(f"remote server Job {job.id} ended with status {stage}") diff --git a/tests/others/test_cli_commands.py b/tests/others/test_cli_commands.py index fbfe1f660dbe..c450008e9a3c 100644 --- a/tests/others/test_cli_commands.py +++ b/tests/others/test_cli_commands.py @@ -56,7 +56,7 @@ ) -AVAILABLE_COMMANDS = ("env", "fp16_safetensors", "custom_blocks", "generate", "run", "schema", "serve", "skills") +AVAILABLE_COMMANDS = ("env", "fp16_safetensors", "custom_blocks", "run", "schema", "serve", "skills") class TestRunCommand: diff --git a/tests/others/test_cli_serve.py b/tests/others/test_cli_serve.py index 8701d06a8942..e20db3132216 100644 --- a/tests/others/test_cli_serve.py +++ b/tests/others/test_cli_serve.py @@ -21,6 +21,7 @@ import inspect import io import json +import shlex import subprocess import sys import threading @@ -34,8 +35,7 @@ from huggingface_hub.utils import httpx from PIL import Image -from diffusers.commands.generate import GenerateCommand -from diffusers.commands.serve import ServeCommand +from diffusers.commands.serve import ServeCommand, _shut_down_when_idle from diffusers.commands.serve.backends import ( DiffusersBackend, EngineBackend, @@ -505,6 +505,14 @@ def test_health(self, serve): response = client.get("/health") assert (response.status_code, response.json()) == (200, {"status": "ok"}) + def test_health_checks_do_not_count_as_activity(self, serve): + client, _ = serve() + client.app.state.last_request = 0.0 + client.get("/health") + assert client.app.state.last_request == 0.0, "a health check must not keep an idle server alive" + client.get("/v1/models") + assert client.app.state.last_request > 0.0, "a request should reset the idle clock" + def test_request_id_is_generated_or_echoed(self, serve): client, _ = serve() assert client.get("/health").headers["x-request-id"], "a request without an id should be given one" @@ -525,7 +533,7 @@ def test_public_url_replaces_the_request_address_in_output_links(self, serve): expected = f"https://proxy.example/sandbox/proxy/8000/v1/generations/{generation['id']}/files/image-0.png" assert generation["outputs"]["image"][0]["url"] == expected - # The sandbox proxy forwards some requests with the published path still attached. + # A reverse proxy may forward requests with the published path still attached. for path in ("/v1/models", "/sandbox/proxy/8000/v1/models"): assert client.get(path).status_code == 200, path prefixed = client.post("/sandbox/proxy/8000/v1/generations", json=body, headers=WAIT) @@ -789,112 +797,116 @@ def test_audio_tasks_are_not_hosted(self): assert _served(backend="vllm", manifest=manifest).tasks == {} -class TestGenerateCommand: - def _run(self, monkeypatch, client, argv): - parser = ArgumentParser() - GenerateCommand.register_subcommand(parser.add_subparsers()) - monkeypatch.setattr("diffusers.commands.generate._POLL_SECONDS", 0.01) - monkeypatch.setattr(GenerateCommand, "_client", lambda self: client) - GenerateCommand(parser.parse_args(["generate", "--url", "http://testserver", *argv])).run() - - def test_generate_saves_outputs(self, serve, tmp_path, monkeypatch): - client, pipeline = serve() - output = tmp_path / "saved" - argv = ["--inputs", '{"prompt": "a cat"}', "--parameters", '{"width": 64, "seed": 5}', "-o", str(output)] - self._run(monkeypatch, client, argv) - - assert pipeline.calls[0]["prompt"] == "a cat" - assert [path.name for path in output.iterdir()] == ["image-0.png"], list(output.iterdir()) - assert Image.open(output / "image-0.png").size == (64, 32) +class FakeJobsApi: + def __init__(self, logs=("server started",), stage="COMPLETED"): + self.logs = logs + self.stage = stage + self.run_kwargs = None + self.cancelled = False - def test_local_file_is_sent_as_a_media_value(self, serve, tmp_path, monkeypatch): - client, pipeline = serve() - source = tmp_path / "cat.png" - Image.new("RGB", (8, 8), "blue").save(source) - inputs = json.dumps({"prompt": "make it grey", "image": str(source)}) - self._run(monkeypatch, client, ["--inputs", inputs, "-o", str(tmp_path / "saved")]) + def run_job(self, **kwargs): + self.run_kwargs = kwargs + return SimpleNamespace(id="job-1", owner=SimpleNamespace(name="me")) - assert pipeline.calls[0]["image"].size == (8, 8) + def fetch_job_logs(self, **kwargs): + for line in self.logs: + if isinstance(line, BaseException): + raise line + yield line - def test_server_error_is_reported(self, serve, tmp_path, monkeypatch): - client, _ = serve() - argv = ["--inputs", '{"prompt": "a cat"}', "--parameters", '{"cfg": 1}', "-o", str(tmp_path)] - with pytest.raises(SystemExit, match=r"HTTP 400: invalid_request: .*\(at /parameters/cfg\)"): - self._run(monkeypatch, client, argv) + def inspect_job(self, **kwargs): + return SimpleNamespace(status=SimpleNamespace(stage=self.stage, expose_urls=None)) - -class FakeSandbox: - id = "sbx-1" - proxy_headers = {"Authorization": "Bearer hf_x", "X-Sandbox-Token": "tok"} - - def __init__(self): - self.commands = [] - self.uploads = [] - self.killed = False - self.files = SimpleNamespace(upload=lambda local, remote: self.uploads.append((local, remote))) - - def proxy_url_for(self, port, path="/"): - return f"https://job.hf.jobs/v1/sandboxes/s/proxy/{port}{path}" - - def run(self, cmd, **kwargs): - self.commands.append(cmd) - return SimpleNamespace(exit_code=0, stdout="/usr/bin/python3\n") - - def kill(self): - self.killed = True + def cancel_job(self, **kwargs): + self.cancelled = True class TestServeRemote: - def test_remote_starts_the_server_in_a_sandbox(self, tmp_path, monkeypatch): - sandbox = FakeSandbox() - created = {} - monkeypatch.setattr( - "diffusers.commands.serve.remote.Sandbox", - SimpleNamespace(create=lambda **kwargs: created.update(kwargs) or sandbox), - ) - manifest = tmp_path / "gms.json" - manifest.write_text("{}") + def _run(self, monkeypatch, api, argv): + monkeypatch.setattr("diffusers.commands.serve.remote.HfApi", lambda token: api) + monkeypatch.setattr("diffusers.commands.serve.remote.get_token", lambda: "hf_x") parser = ArgumentParser() ServeCommand.register_subcommand(parser.add_subparsers()) - argv = ["serve", "-m", "org/model", "--dtype", "bf16", "--remote", "--flavor", "a10g-large"] - argv += ["--manifest", str(manifest), "--port", "9000", "--dependencies", "extra-package"] - ServeCommand(parser.parse_args(argv)).run() + ServeCommand(parser.parse_args(["serve", "org/model", "--remote", *argv])).run() - assert created["flavor"] == "a10g-large" - assert created["image"] == "pytorch/pytorch:2.10.0-cuda12.8-cudnn9-runtime" - install, _, serve_argv = sandbox.commands + def test_remote_starts_the_server_in_a_job(self, monkeypatch): + api = FakeJobsApi() + argv = ["--dtype", "fp16", "--flavor", "a10g-large", "--port", "9000"] + self._run(monkeypatch, api, [*argv, "--dependencies", "extra-package"]) + + job = api.run_kwargs + assert (job["flavor"], job["image"]) == ("a10g-large", "diffusers/diffusers-cli-cuda:latest"), job + assert job["expose"] == [8000] and job["timeout"] == "24h", job + assert job["secrets"] == {"HF_TOKEN": "hf_x"}, "gated models need the HF token inside the Job" + + assert job["command"][:2] == ["sh", "-c"], job["command"] + install, start = job["command"][2].split(" && ") assert "fastapi" in install and "extra-package" in install, install + start = shlex.split(start) # The server runs under the interpreter the dependencies were installed into, not whatever is on PATH. - assert serve_argv[:3] == ["/usr/bin/python3", "-m", "diffusers.commands.diffusers_cli"], serve_argv - assert serve_argv[3:7] == ["--format", "quiet", "serve", "--model"], serve_argv - options = dict(zip(serve_argv[6::2], serve_argv[7::2])) - assert options["--dtype"] == "bf16" - # The sandbox server always binds the proxied port and builds output links on the proxied address. - assert options["--port"] == "8000" - assert options["--public-url"] == "https://job.hf.jobs/v1/sandboxes/s/proxy/8000/" - assert options["--manifest"] == "/tmp/diffusers-cli/gms.json" - assert sandbox.uploads == [(str(manifest), "/tmp/diffusers-cli/gms.json")] - assert "--remote" not in serve_argv and "--flavor" not in serve_argv, serve_argv - assert sandbox.killed, "the sandbox must be killed once the server stops" + assert start[:4] == ["exec", "$(uv python find --system)", "-m", "diffusers.commands.diffusers_cli"], start + serve_argv = start[start.index("serve") + 1 :] + options = dict(zip(serve_argv[::2], serve_argv[1::2])) + assert options["--model"] == "org/model" and options["--dtype"] == "fp16", options + # The server binds the exposed port on every interface, and builds output links on the public address. + assert (options["--host"], options["--port"]) == ("0.0.0.0", "8000"), options + assert options["--public-url"] == "https://${JOB_ID}--8000.hf.jobs", options + assert options["--idle-timeout"] == "30m", "a remote server should shut itself down when idle by default" + assert not {"--remote", "--flavor", "--timeout", "--dependencies"} & set(serve_argv), serve_argv + assert not api.cancelled, "a Job that completed on its own needs no cancel" + + def test_local_manifest_is_rejected(self, tmp_path, monkeypatch): + api = FakeJobsApi() + manifest = tmp_path / "gms.json" + manifest.write_text("{}") + with pytest.raises(SystemExit, match="Upload it as `gms.json` to the model repo"): + self._run(monkeypatch, api, ["--manifest", str(manifest)]) + assert api.run_kwargs is None, "no Job should start when the manifest can't reach it" + + def test_interrupt_cancels_the_job(self, monkeypatch): + api = FakeJobsApi(logs=("server started", KeyboardInterrupt())) + with pytest.raises(KeyboardInterrupt): + self._run(monkeypatch, api, []) + assert api.cancelled, "Ctrl-C must cancel the Job so it stops billing" + + def test_failed_job_is_reported(self, monkeypatch): + with pytest.raises(SystemExit, match="Job job-1 ended with status ERROR"): + self._run(monkeypatch, FakeJobsApi(stage="ERROR"), []) @pytest.mark.parametrize( "backend, image", [("sglang", "lmsysorg/sglang:latest"), ("vllm", "vllm/vllm-omni:latest")] ) def test_engine_backend_defaults_to_the_engine_image(self, monkeypatch, backend, image): - created = {} - monkeypatch.setattr( - "diffusers.commands.serve.remote.Sandbox", - SimpleNamespace(create=lambda **kwargs: created.update(kwargs) or FakeSandbox()), - ) - parser = ArgumentParser() - ServeCommand.register_subcommand(parser.add_subparsers()) - ServeCommand(parser.parse_args(["serve", "-m", "org/model", "--backend", backend, "--remote"])).run() - assert created["image"] == image - - ServeCommand( - parser.parse_args(["serve", "-m", "org/model", "--backend", backend, "--remote", "--image", "mine"]) - ).run() - assert created["image"] == "mine", "an explicit --image must win over the engine default" + api = FakeJobsApi() + self._run(monkeypatch, api, ["--backend", backend]) + assert api.run_kwargs["image"] == image + + self._run(monkeypatch, api, ["--backend", backend, "--image", "mine"]) + assert api.run_kwargs["image"] == "mine", "an explicit --image must win over the engine default" + + +class TestIdleShutdown: + def _watch(self, busy): + server = SimpleNamespace(should_exit=False) + app = SimpleNamespace(state=SimpleNamespace(last_request=0.0)) + generations = SimpleNamespace(busy=lambda: busy[0]) + watcher = threading.Thread(target=_shut_down_when_idle, args=(server, app, generations, "0.05"), daemon=True) + watcher.start() + return server, watcher + + def test_idle_server_shuts_down(self): + server, watcher = self._watch(busy=[False]) + watcher.join(timeout=5) + assert server.should_exit, "an idle server should be told to exit" + + def test_busy_server_stays_up_until_the_work_ends(self): + busy = [True] + server, watcher = self._watch(busy) + watcher.join(timeout=0.3) + assert not server.should_exit, "a queued or running generation must keep the server up" + busy[0] = False + watcher.join(timeout=5) + assert server.should_exit, "the server should exit once the work ends and the timeout passes" class TestServeCommand: @@ -910,12 +922,34 @@ def stop(args): raise RuntimeError("stop after the flag check") monkeypatch.setattr("diffusers.commands.serve._resolve_pipeline", stop) - args = self._parse(["-m", "org/model", "--backend", "vllm", "--dtype", "bf16", "--vae-tiling"]) + args = self._parse(["-m", "org/model", "--backend", "vllm", "--dtype", "fp16", "--vae-tiling"]) with CaptureLogger(logging.get_logger("diffusers-cli/serve")) as captured: with pytest.raises(RuntimeError, match="stop after the flag check"): ServeCommand(args).run() assert "--dtype, --vae-tiling configure the in-process pipeline and are ignored" in captured.out + def test_model_is_positional_and_loads_in_bf16(self, monkeypatch): + pytest.importorskip("fastapi", reason="`diffusers-cli serve` needs the `diffusers[serve]` extra") + resolved = {} + + def stop(args): + resolved.update(vars(args)) + raise RuntimeError("stop after the flag check") + + monkeypatch.setattr("diffusers.commands.serve._resolve_pipeline", stop) + with pytest.raises(RuntimeError, match="stop after the flag check"): + ServeCommand(self._parse(["org/model"])).run() + assert resolved["model"] == "org/model", resolved + assert resolved["dtype"] == "bf16", "serve should default to bf16 rather than `run`'s fp32 `auto`" + + def test_two_different_models_are_rejected(self): + with pytest.raises(SystemExit, match="Got two models"): + ServeCommand(self._parse(["org/model", "-m", "org/other"])).run() + + def test_missing_model_is_rejected(self): + with pytest.raises(SystemExit, match="Pass the model to serve"): + ServeCommand(self._parse([])).run() + def test_context_parallel_is_rejected(self): pytest.importorskip("fastapi", reason="`diffusers-cli serve` needs the `diffusers[serve]` extra") with pytest.raises(SystemExit, match="--context-parallel is not supported"): From a5781d1d84624dece411096fc8aba9cd3063e941 Mon Sep 17 00:00:00 2001 From: DN6 Date: Wed, 4 Nov 2026 14:26:07 +0530 Subject: [PATCH 06/21] Add context-parallel serving and allow --compile with --context-parallel --- .ai/skills/diffusers-cli/run.md | 4 - docs/source/en/using-diffusers/cli.md | 2 +- docs/source/en/using-diffusers/serve.md | 17 +- src/diffusers/commands/run.py | 7 +- src/diffusers/commands/serve/__init__.py | 11 +- src/diffusers/commands/serve/backends.py | 49 +++-- src/diffusers/commands/serve/parallel.py | 235 +++++++++++++++++++++++ tests/others/test_cli_serve.py | 194 ++++++++++++++++++- 8 files changed, 483 insertions(+), 36 deletions(-) create mode 100644 src/diffusers/commands/serve/parallel.py diff --git a/.ai/skills/diffusers-cli/run.md b/.ai/skills/diffusers-cli/run.md index bd60b1212d44..7ce269f4b1cc 100644 --- a/.ai/skills/diffusers-cli/run.md +++ b/.ai/skills/diffusers-cli/run.md @@ -251,10 +251,6 @@ paid on every run — `--compile` only breaks even on very long generations. On sandbox the compile cache persists on disk, so the cost is paid once and reused: keep a warm sandbox and `--compile` pays off across runs. -`--compile` is **currently not supported with `--context-parallel`** — CP shards attention across ranks while -regional compile assumes a stable single-device graph. If both are set, the CLI logs a warning and skips the -compile step; CP still runs. - ## Context parallel `--context-parallel` enables Ulysses CP on a DiT-based pipeline. **Locally** the user must launch via torchrun: diff --git a/docs/source/en/using-diffusers/cli.md b/docs/source/en/using-diffusers/cli.md index 961c2a1ddbe8..8ccccf73c68a 100644 --- a/docs/source/en/using-diffusers/cli.md +++ b/docs/source/en/using-diffusers/cli.md @@ -136,7 +136,7 @@ Configure how the CLI loads model weights and custom pipeline code. [torch.compile](../optimization/fp16#torchcompile). The CLI prefers [regional compilation](../optimization/fp16#regional-compilation) for denoisers with repeated blocks; the VAE decoder is always compiled whole. Bare `--compile` uses `fullgraph=true`. A JSON object is forwarded to - `torch.compile`. Not supported with `--context-parallel`. + `torch.compile`. - `--compile-mode {regional,full}` — what `--compile` compiles. `regional` (default) compiles only the repeated blocks, which keeps the first step fast. `full` compiles the whole denoiser. `torch.compile` modes that use CUDA graphs (`max-autotune`, `reduce-overhead`) need `--compile-mode full`: repeated blocks share one compiled graph, diff --git a/docs/source/en/using-diffusers/serve.md b/docs/source/en/using-diffusers/serve.md index aa6f3fba7b15..10140c7007c2 100644 --- a/docs/source/en/using-diffusers/serve.md +++ b/docs/source/en/using-diffusers/serve.md @@ -128,6 +128,20 @@ When a manifest uses a name that differs from the pipeline argument, map it per A task is only hosted when every input it requires maps to a pipeline argument. A parameter the pipeline cannot set is rejected with `not_implemented` unless the request sends its default value. +## Multiple GPUs + +`--context-parallel` splits each generation across every visible GPU with +[context parallelism](../training/distributed_inference#context-parallelism). It needs a DiT-based pipeline and at +least two GPUs. `serve` starts one worker process per GPU itself, so don't launch it with `torchrun`: + +```bash +diffusers serve black-forest-labs/FLUX.1-dev --context-parallel +``` + +Every worker runs every generation, and rank 0 saves the outputs. If a worker process exits, `GET /health` returns +`503` and the server stops generating. `--device-map` and request adapters are not available with +`--context-parallel`. + ## Remote serving `--remote` starts the server in a [Hugging Face Job](https://huggingface.co/docs/huggingface_hub/guides/jobs) @@ -148,7 +162,8 @@ curl /v1/generations -H 'Prefer: wait=120' -H "Authorization: Bearer $HF_TO The Job ends when the server shuts down. `--idle-timeout` defaults to `30m`, and `--timeout` (default `24h`) caps how long the Job runs even while it is busy. `--flavor`, `--image`, `--dependencies` and `--namespace` work as they -do for [`run --remote`](./cli#remote-execution---remote). The model must be a Hub repo id. `--manifest` is not +do for [`run --remote`](./cli#remote-execution---remote). With `--context-parallel`, pick a flavor with +several GPUs. The model must be a Hub repo id. `--manifest` is not supported with `--remote`: publish the manifest as `gms.json` in the model repo instead. ## SGLang and vLLM-Omni backends diff --git a/src/diffusers/commands/run.py b/src/diffusers/commands/run.py index 31f7cadccd18..ee93f3605513 100644 --- a/src/diffusers/commands/run.py +++ b/src/diffusers/commands/run.py @@ -485,11 +485,8 @@ def _apply_optimizations(pipeline: Any, args: Namespace) -> None: if args.context_parallel: _enable_context_parallel(pipeline) if args.compile is not None: - if args.context_parallel: - logger.warning("--compile is currently not supported with --context-parallel; skipping compile.") - else: - _compile_denoiser(pipeline, args.compile, args.compile_mode) - _compile_vae_decoder(pipeline, args.compile) + _compile_denoiser(pipeline, args.compile, args.compile_mode) + _compile_vae_decoder(pipeline, args.compile) def _parse_compile_spec(compile_spec: str) -> dict[str, Any]: diff --git a/src/diffusers/commands/serve/__init__.py b/src/diffusers/commands/serve/__init__.py index 904f6d595e28..5faee6ca0416 100644 --- a/src/diffusers/commands/serve/__init__.py +++ b/src/diffusers/commands/serve/__init__.py @@ -53,6 +53,7 @@ "vae_slicing": False, "compile": None, "compile_mode": "regional", + "context_parallel": False, } @@ -251,10 +252,9 @@ def run(self) -> None: if args.log_level is not None: logging.set_verbosity(logging.get_log_levels_dict()[args.log_level]) - if args.context_parallel: + if args.context_parallel and args.device_map is not None: raise SystemExit( - "--context-parallel is not supported by `diffusers-cli serve`: under torchrun every rank would " - "start its own server on the same port." + "--device-map does not apply with --context-parallel: each worker loads the pipeline on its own GPU." ) if args.backend == "diffusers" and args.backend_args is not None: raise SystemExit("--backend-args only applies to --backend sglang or vllm.") @@ -292,6 +292,7 @@ def run(self) -> None: ) from .generations import GenerationQueue from .manifest import ServedModel, load_manifest + from .parallel import ParallelBackend pipeline_cls, signature, commit_hash = _resolve_pipeline(args) kwarg_names = backend_kwargs(args.backend, signature) @@ -317,7 +318,9 @@ def run(self) -> None: f"served by {pipeline_cls.__name__} with --backend {args.backend}." ) - if args.backend == "diffusers": + if args.backend == "diffusers" and args.context_parallel: + backend = ParallelBackend(args, served) + elif args.backend == "diffusers": backend = DiffusersBackend(args, served) else: backend = EngineBackend.launch(args, served) diff --git a/src/diffusers/commands/serve/backends.py b/src/diffusers/commands/serve/backends.py index bc1eb2811eaf..21e8b8a2b854 100644 --- a/src/diffusers/commands/serve/backends.py +++ b/src/diffusers/commands/serve/backends.py @@ -99,8 +99,14 @@ def backend_capabilities(args: Namespace, pipeline_cls: type) -> list[str]: if args.backend != "diffusers": return capabilities # Swapping adapter weights retraces a compiled denoiser on every request, and LoRAs passed at - # startup are part of the deployment rather than something a request may replace. - if hasattr(pipeline_cls, "load_lora_weights") and args.compile is None and not args.lora: + # startup are part of the deployment rather than something a request may replace. Under context + # parallelism every rank would have to swap them in lockstep, which is untested. + if ( + hasattr(pipeline_cls, "load_lora_weights") + and args.compile is None + and not args.lora + and not args.context_parallel + ): capabilities.append("adapters") return capabilities @@ -203,10 +209,28 @@ def _save_audio(audios: Any, directory: Path, name: str, media_type: str, rate: _SAVERS = {"image": _save_images, "video": _save_videos, "audio": _save_audio} +def load_request_media(served: ServedModel, request: Any) -> dict[str, Any]: + """The pipeline kwargs of a request, with its media inputs fetched and decoded.""" + kwargs = dict(request.kwargs) + for kwarg, (name, value) in request.media.items(): + if isinstance(value, list): + loaded = [_load_media(name, item, f"/inputs/{name}/{index}") for index, item in enumerate(value)] + else: + loaded = [_load_media(name, value, f"/inputs/{name}")] + if served.tasks[request.task]["inputs"][name]["type"] == "audio": + if kwarg == "initial_audio_waveforms": + kwargs.setdefault("initial_audio_sampling_rate", loaded[0][1]) + loaded = [waveform for waveform, _ in loaded] + kwargs[kwarg] = loaded if isinstance(value, list) else loaded[0] + return kwargs + + class DiffusersBackend: - def __init__(self, args: Namespace, served: ServedModel): + def __init__(self, args: Namespace, served: ServedModel, save_outputs: bool = True): self._args = args self._served = served + # Under context parallelism every rank runs the call, and only rank 0 writes the files. + self._save_outputs = save_outputs self.pipeline = _load_pipeline(args) self._modular = isinstance(self.pipeline, ModularPipeline) if self._modular: @@ -227,19 +251,11 @@ def failure(self) -> str | None: return None def generate(self, generation: Generation) -> dict[str, list[dict[str, Any]]]: - request = generation.request - kwargs = dict(request.kwargs) - for kwarg, (name, value) in request.media.items(): - if isinstance(value, list): - loaded = [_load_media(name, item, f"/inputs/{name}/{index}") for index, item in enumerate(value)] - else: - loaded = [_load_media(name, value, f"/inputs/{name}")] - if self._served.tasks[request.task]["inputs"][name]["type"] == "audio": - if kwarg == "initial_audio_waveforms": - kwargs.setdefault("initial_audio_sampling_rate", loaded[0][1]) - loaded = [waveform for waveform, _ in loaded] - kwargs[kwarg] = loaded if isinstance(value, list) else loaded[0] + return self.run(load_request_media(self._served, generation.request), generation) + def run(self, kwargs: dict[str, Any], generation: Any) -> dict[str, list[dict[str, Any]]]: + """Call the pipeline with the request's kwargs, media already loaded, and save its outputs.""" + request = generation.request if generation.seed is not None: device = self.pipeline.device.type if hasattr(self.pipeline, "device") else "cpu" kwargs["generator"] = _get_generator(generation.seed, device) @@ -262,7 +278,8 @@ def on_step_end(pipeline, step, timestep, callback_kwargs): finally: if request.adapters: self.pipeline.unload_lora_weights() - if generation.cancel_requested: + # Read first: under context parallelism `cancel_requested` is a collective every rank must join. + if generation.cancel_requested or not self._save_outputs: return {} outputs = {} diff --git a/src/diffusers/commands/serve/parallel.py b/src/diffusers/commands/serve/parallel.py new file mode 100644 index 000000000000..86029e849370 --- /dev/null +++ b/src/diffusers/commands/serve/parallel.py @@ -0,0 +1,235 @@ +# Copyright 2026 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""`diffusers-cli serve --context-parallel` — one worker process per GPU, all running every generation together. + +The server process never touches CUDA. It spawns the workers, sends each generation to all of them over a pipe, and +reads the progress and result back from rank 0. Idle workers wait on their pipe rather than inside a collective, so +the process group cannot time out between requests. +""" + +from __future__ import annotations + +import multiprocessing +import os +import signal +import socket +import threading +from argparse import Namespace +from multiprocessing.connection import Connection, wait +from pathlib import Path +from typing import Any + +from ...utils import logging +from .backends import DiffusersBackend, load_request_media +from .generations import Generation +from .manifest import GMSError, ServedModel + + +logger = logging.get_logger("diffusers-cli/serve") + +_POLL_SECONDS = 0.1 +_SHUTDOWN_SECONDS = 30.0 + + +class _WorkerGeneration: + """The parts of a generation `DiffusersBackend.run` reads, inside a worker. + + `cancel_requested` is a collective: rank 0 reads the flag the server sets and broadcasts it, so every rank + interrupts at the same step. A rank that stopped alone would leave the others waiting in the next collective. + """ + + def __init__(self, job: dict[str, Any], rank: int, conn: Connection, cancel_flag: Any): + self.request = job["request"] + self.seed = job["seed"] + self.directory = Path(job["directory"]) + self._rank = rank + self._conn = conn + self._cancel_flag = cancel_flag + + def report_progress(self, step: int, total_steps: int) -> None: + if self._rank == 0: + self._conn.send(("progress", step, total_steps)) + + @property + def cancel_requested(self) -> bool: + import torch + + flag = torch.tensor([self._cancel_flag.value if self._rank == 0 else 0], dtype=torch.uint8) + torch.distributed.broadcast(flag, src=0) + return bool(flag.item()) + + +def _serve_jobs(backend: Any, rank: int, conn: Connection, cancel_flag: Any) -> None: + while True: + job = conn.recv() + if job is None: + return + generation = _WorkerGeneration(job, rank, conn, cancel_flag) + try: + reply = ("complete", backend.run(job["kwargs"], generation)) + except GMSError as e: + reply = ("failed", e.code, e.message, e.pointer) + # The helpers shared with `run` raise SystemExit, which must not end the worker. + except (Exception, SystemExit) as e: + logger.error(f"generation failed on rank {rank}", exc_info=True) + reply = ("failed", "internal", f"{type(e).__name__}: {e}", None) + if rank == 0: + conn.send(reply) + + +def _worker_main( + rank: int, world_size: int, port: int, args: Namespace, served: ServedModel, conn: Connection, cancel_flag: Any +) -> None: + import torch + + # The server process decides when workers stop, so Ctrl-C in the terminal must not interrupt them mid-call. + signal.signal(signal.SIGINT, signal.SIG_IGN) + os.environ.update( + MASTER_ADDR="127.0.0.1", + MASTER_PORT=str(port), + RANK=str(rank), + LOCAL_RANK=str(rank), + WORLD_SIZE=str(world_size), + ) + if args.log_level is not None: + logging.set_verbosity(logging.get_log_levels_dict()[args.log_level]) + # Loading applies `--context-parallel`, which joins the process group of all ranks. + backend = DiffusersBackend(args, served, save_outputs=rank == 0) + conn.send(("ready",)) + try: + _serve_jobs(backend, rank, conn, cancel_flag) + finally: + torch.distributed.destroy_process_group() + + +class ParallelBackend: + def __init__(self, args: Namespace, served: ServedModel): + import torch + + world_size = torch.cuda.device_count() + if world_size < 2: + raise SystemExit(f"--context-parallel needs at least 2 visible GPUs, found {world_size}.") + with socket.socket() as probe: + probe.bind(("127.0.0.1", 0)) + port = probe.getsockname()[1] + + self._served = served + self._failure: str | None = None + self._closing = False + # Spawned rather than forked: a forked child inherits whatever CUDA state its parent has. + context = multiprocessing.get_context("spawn") + # Lock-free: a worker killed while holding the lock would leave the server blocked on its next write. + self._cancel_flag = context.Value("b", 0, lock=False) + self._conns: list[Connection] = [] + self._processes = [] + for rank in range(world_size): + parent_conn, child_conn = context.Pipe() + process = context.Process( + target=_worker_main, + args=(rank, world_size, port, args, served, child_conn, self._cancel_flag), + name=f"diffusers-serve-rank-{rank}", + daemon=True, + ) + process.start() + child_conn.close() + self._conns.append(parent_conn) + self._processes.append(process) + + logger.info(f"loading the pipeline on {world_size} GPUs...") + self._wait_until_ready() + threading.Thread(target=self._monitor, name="diffusers-serve-workers", daemon=True).start() + + def _wait_until_ready(self) -> None: + waiting = dict(zip(self._conns, self._processes)) + while waiting: + sentinels = {process.sentinel: process for process in waiting.values()} + for ready in wait([*waiting, *sentinels]): + process = waiting.pop(ready, None) if isinstance(ready, Connection) else sentinels[ready] + try: + if isinstance(ready, Connection) and ready.recv() == ("ready",): + continue + except EOFError: + pass + self.close() + raise SystemExit( + f"Context-parallel worker {process.name} exited with code {process.exitcode} while loading " + "the pipeline. Its error is in the log above." + ) + + def _monitor(self) -> None: + sentinels = {process.sentinel: process for process in self._processes} + dead = sentinels[wait(list(sentinels))[0]] + if self._closing: + return + self._failure = f"context-parallel worker {dead.name} exited with code {dead.exitcode}" + logger.error(f"{self._failure}; the remaining workers are stopped and the server no longer generates.") + # The other ranks would wait forever in their next collective, so the group is not reused. + self.close() + + def failure(self) -> str | None: + return self._failure + + def generate(self, generation: Generation) -> dict[str, list[dict[str, Any]]]: + if self._failure is not None: + raise GMSError("internal", self._failure) + job = { + "request": generation.request, + "kwargs": load_request_media(self._served, generation.request), + "seed": generation.seed, + "directory": str(generation.directory), + } + self._cancel_flag.value = 0 + for conn in self._conns: + conn.send(job) + + rank_0 = self._conns[0] + while True: + if generation.cancel_requested: + self._cancel_flag.value = 1 + try: + if not rank_0.poll(_POLL_SECONDS): + if self._failure is not None: + raise GMSError("internal", self._failure) + continue + message = rank_0.recv() + except (EOFError, OSError) as e: + # The monitor may not have noticed yet; rank 0's pipe only breaks once it has exited. + self._processes[0].join() + if self._failure is None: + self._failure = ( + f"context-parallel worker {self._processes[0].name} exited with code " + f"{self._processes[0].exitcode}" + ) + raise GMSError("internal", self._failure) from e + if message[0] == "progress": + generation.report_progress(*message[1:]) + elif message[0] == "complete": + return message[1] + else: + _, code, text, pointer = message + raise GMSError(code, text, pointer) + + def close(self) -> None: + self._closing = True + for conn in self._conns: + try: + conn.send(None) + except OSError: + pass + for process in self._processes: + process.join(timeout=_SHUTDOWN_SECONDS) + if process.is_alive(): + process.kill() + process.join() diff --git a/tests/others/test_cli_serve.py b/tests/others/test_cli_serve.py index e20db3132216..8ce7cda0b9e8 100644 --- a/tests/others/test_cli_serve.py +++ b/tests/others/test_cli_serve.py @@ -21,10 +21,12 @@ import inspect import io import json +import os import shlex import subprocess import sys import threading +import time from argparse import ArgumentParser, Namespace from pathlib import Path from types import SimpleNamespace @@ -154,7 +156,14 @@ def __call__(self, components, state): def _args(**overrides): - values = {"backend": "diffusers", "compile": None, "lora": None, "fps": 8, "sampling_rate": None} + values = { + "backend": "diffusers", + "compile": None, + "lora": None, + "context_parallel": False, + "fps": 8, + "sampling_rate": None, + } return Namespace(**{**values, **overrides}) @@ -950,7 +959,182 @@ def test_missing_model_is_rejected(self): with pytest.raises(SystemExit, match="Pass the model to serve"): ServeCommand(self._parse([])).run() - def test_context_parallel_is_rejected(self): - pytest.importorskip("fastapi", reason="`diffusers-cli serve` needs the `diffusers[serve]` extra") - with pytest.raises(SystemExit, match="--context-parallel is not supported"): - ServeCommand(self._parse(["-m", "org/model", "--context-parallel"])).run() + def test_context_parallel_rejects_device_map(self): + with pytest.raises(SystemExit, match="--device-map does not apply with --context-parallel"): + ServeCommand(self._parse(["-m", "org/model", "--context-parallel", "--device-map", "cuda:0"])).run() + + +def _fake_worker_main(rank, world_size, port, args, served, conn, cancel_flag): + """Stands in for `_worker_main` in forked children: no pipeline, no process group.""" + if args.fail_at_startup: + os._exit(3) + conn.send(("ready",)) + while True: + job = conn.recv() + if job is None: + return + if job["seed"] == -1: + os._exit(4) + if rank != 0: + continue + conn.send(("progress", 1, 2)) + # Give the server time to raise the cancel flag between the two steps. + time.sleep(0.3) + if cancel_flag.value: + conn.send(("complete", {})) + elif job["kwargs"]["prompt"] == "bad": + conn.send(("failed", "invalid_request", "bad prompt", "/inputs/prompt")) + else: + conn.send(("complete", {"image": [{"file": "image-0.png", "world_size": world_size}]})) + + +class FakeGeneration: + def __init__(self, prompt="a cat", seed=0): + self.request = SimpleNamespace(kwargs={"prompt": prompt}, media={}, task="text_to_image") + self.seed = seed + self.directory = Path("/tmp/unused") + self.cancel_requested = False + self.progress = [] + + def report_progress(self, step, total_steps): + self.progress.append((step, total_steps)) + + +class TestParallelBackend: + @pytest.fixture + def backend(self, monkeypatch): + import multiprocessing + + from diffusers.commands.serve import parallel + + if "fork" not in multiprocessing.get_all_start_methods(): + pytest.skip("the fake workers are forked so they can use a test-local target") + fork = multiprocessing.get_context("fork") + monkeypatch.setattr(parallel.multiprocessing, "get_context", lambda method: fork) + monkeypatch.setattr(parallel, "_worker_main", _fake_worker_main) + monkeypatch.setattr(torch.cuda, "device_count", lambda: 2) + backends = [] + + def _build(**arg_overrides): + args = Namespace(**{"fail_at_startup": False, **arg_overrides}) + backends.append(parallel.ParallelBackend(args, _served())) + return backends[-1] + + yield _build + for backend in backends: + backend.close() + + def test_generation_runs_on_every_rank_and_rank_0_answers(self, backend): + generation = FakeGeneration() + outputs = backend().generate(generation) + assert outputs == {"image": [{"file": "image-0.png", "world_size": 2}]}, outputs + assert generation.progress == [(1, 2)], "rank 0's progress should reach the generation" + + def test_worker_error_becomes_a_request_error(self, backend): + with pytest.raises(GMSError, match="bad prompt") as error: + backend().generate(FakeGeneration(prompt="bad")) + assert (error.value.code, error.value.pointer) == ("invalid_request", "/inputs/prompt") + + def test_cancel_reaches_the_workers(self, backend): + generation = FakeGeneration() + threading.Timer(0.1, lambda: setattr(generation, "cancel_requested", True)).start() + assert backend().generate(generation) == {}, "a cancelled generation returns no outputs" + + def test_dead_worker_fails_the_server(self, backend): + server = backend() + with pytest.raises(GMSError, match="exited"): + server.generate(FakeGeneration(seed=-1)) + assert "exited with code 4" in server.failure(), server.failure() + with pytest.raises(GMSError, match="exited with code 4"): + server.generate(FakeGeneration()) + + def test_worker_dying_at_startup_stops_the_server(self, backend): + with pytest.raises(SystemExit, match="exited with code 3 while loading the pipeline"): + backend(fail_at_startup=True) + + def test_worker_arguments_survive_spawn(self): + import pickle + + parser = ArgumentParser() + ServeCommand.register_subcommand(parser.add_subparsers()) + args = parser.parse_args(["serve", "org/model", "--context-parallel"]) + # Spawned workers receive the parsed flags and the served model pickled. + restored_args, restored_served = pickle.loads(pickle.dumps((args, _served()))) + assert restored_args.context_parallel and restored_served.tasks + + def test_too_few_gpus_is_rejected(self, monkeypatch): + from diffusers.commands.serve.parallel import ParallelBackend + + monkeypatch.setattr(torch.cuda, "device_count", lambda: 1) + with pytest.raises(SystemExit, match="needs at least 2 visible GPUs, found 1"): + ParallelBackend(Namespace(), _served()) + + +class TestParallelWorker: + def _jobs(self, backend, rank, *jobs): + from multiprocessing import Pipe, Value + + from diffusers.commands.serve.parallel import _serve_jobs + + server, worker = Pipe() + for job in (*jobs, None): + server.send(job) + _serve_jobs(backend, rank, worker, Value("b", 0)) + replies = [] + while server.poll(): + replies.append(server.recv()) + return replies + + def _job(self, prompt): + return {"request": None, "kwargs": {"prompt": prompt}, "seed": 0, "directory": "/tmp/unused"} + + def test_rank_0_reports_progress_and_results(self): + def run(kwargs, generation): + generation.report_progress(1, 1) + if kwargs["prompt"] == "bad": + raise GMSError("invalid_request", "bad prompt", "/inputs/prompt") + if kwargs["prompt"] == "crash": + raise SystemExit("unsupported") + return {"image": []} + + replies = self._jobs(SimpleNamespace(run=run), 0, self._job("ok"), self._job("bad"), self._job("crash")) + assert replies == [ + ("progress", 1, 1), + ("complete", {"image": []}), + ("progress", 1, 1), + ("failed", "invalid_request", "bad prompt", "/inputs/prompt"), + ("progress", 1, 1), + ("failed", "internal", "SystemExit: unsupported", None), + ], "a failed generation must not end the worker" + + def test_other_ranks_stay_silent(self): + def run(kwargs, generation): + generation.report_progress(1, 1) + return {"image": []} + + assert self._jobs(SimpleNamespace(run=run), 1, self._job("ok")) == [] + + def test_cancel_is_broadcast_from_rank_0(self, monkeypatch): + from multiprocessing import Value + + from diffusers.commands.serve.parallel import _WorkerGeneration + + monkeypatch.setenv("MASTER_ADDR", "127.0.0.1") + monkeypatch.setenv("MASTER_PORT", str(_free_port())) + torch.distributed.init_process_group("gloo", rank=0, world_size=1) + try: + flag = Value("b", 0) + generation = _WorkerGeneration(self._job("ok"), 0, None, flag) + assert not generation.cancel_requested + flag.value = 1 + assert generation.cancel_requested, "rank 0 should broadcast the flag the server raised" + finally: + torch.distributed.destroy_process_group() + + +def _free_port(): + import socket + + with socket.socket() as probe: + probe.bind(("127.0.0.1", 0)) + return probe.getsockname()[1] From a60093dacd6b3d7e0f517e7cd15a8f0eca047c52 Mon Sep 17 00:00:00 2001 From: DN6 Date: Wed, 4 Nov 2026 14:42:13 +0530 Subject: [PATCH 07/21] Restore --compile with --context-parallel guard --- .ai/skills/diffusers-cli/run.md | 4 ++++ docs/source/en/using-diffusers/cli.md | 2 +- docs/source/en/using-diffusers/serve.md | 2 +- src/diffusers/commands/run.py | 7 +++++-- 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/.ai/skills/diffusers-cli/run.md b/.ai/skills/diffusers-cli/run.md index 7ce269f4b1cc..bd60b1212d44 100644 --- a/.ai/skills/diffusers-cli/run.md +++ b/.ai/skills/diffusers-cli/run.md @@ -251,6 +251,10 @@ paid on every run — `--compile` only breaks even on very long generations. On sandbox the compile cache persists on disk, so the cost is paid once and reused: keep a warm sandbox and `--compile` pays off across runs. +`--compile` is **currently not supported with `--context-parallel`** — CP shards attention across ranks while +regional compile assumes a stable single-device graph. If both are set, the CLI logs a warning and skips the +compile step; CP still runs. + ## Context parallel `--context-parallel` enables Ulysses CP on a DiT-based pipeline. **Locally** the user must launch via torchrun: diff --git a/docs/source/en/using-diffusers/cli.md b/docs/source/en/using-diffusers/cli.md index 8ccccf73c68a..961c2a1ddbe8 100644 --- a/docs/source/en/using-diffusers/cli.md +++ b/docs/source/en/using-diffusers/cli.md @@ -136,7 +136,7 @@ Configure how the CLI loads model weights and custom pipeline code. [torch.compile](../optimization/fp16#torchcompile). The CLI prefers [regional compilation](../optimization/fp16#regional-compilation) for denoisers with repeated blocks; the VAE decoder is always compiled whole. Bare `--compile` uses `fullgraph=true`. A JSON object is forwarded to - `torch.compile`. + `torch.compile`. Not supported with `--context-parallel`. - `--compile-mode {regional,full}` — what `--compile` compiles. `regional` (default) compiles only the repeated blocks, which keeps the first step fast. `full` compiles the whole denoiser. `torch.compile` modes that use CUDA graphs (`max-autotune`, `reduce-overhead`) need `--compile-mode full`: repeated blocks share one compiled graph, diff --git a/docs/source/en/using-diffusers/serve.md b/docs/source/en/using-diffusers/serve.md index 10140c7007c2..822db93c480b 100644 --- a/docs/source/en/using-diffusers/serve.md +++ b/docs/source/en/using-diffusers/serve.md @@ -140,7 +140,7 @@ diffusers serve black-forest-labs/FLUX.1-dev --context-parallel Every worker runs every generation, and rank 0 saves the outputs. If a worker process exits, `GET /health` returns `503` and the server stops generating. `--device-map` and request adapters are not available with -`--context-parallel`. +`--context-parallel`, and `--compile` is skipped. ## Remote serving diff --git a/src/diffusers/commands/run.py b/src/diffusers/commands/run.py index ee93f3605513..31f7cadccd18 100644 --- a/src/diffusers/commands/run.py +++ b/src/diffusers/commands/run.py @@ -485,8 +485,11 @@ def _apply_optimizations(pipeline: Any, args: Namespace) -> None: if args.context_parallel: _enable_context_parallel(pipeline) if args.compile is not None: - _compile_denoiser(pipeline, args.compile, args.compile_mode) - _compile_vae_decoder(pipeline, args.compile) + if args.context_parallel: + logger.warning("--compile is currently not supported with --context-parallel; skipping compile.") + else: + _compile_denoiser(pipeline, args.compile, args.compile_mode) + _compile_vae_decoder(pipeline, args.compile) def _parse_compile_spec(compile_spec: str) -> dict[str, Any]: From 3df7581b420bcdeb6ed97e3a4d1b8ed30a3b95d1 Mon Sep 17 00:00:00 2001 From: DN6 Date: Wed, 4 Nov 2026 14:56:01 +0530 Subject: [PATCH 08/21] clean up --- tests/others/test_cli_commands.py | 5 ----- tests/others/test_cli_serve.py | 24 ------------------------ 2 files changed, 29 deletions(-) diff --git a/tests/others/test_cli_commands.py b/tests/others/test_cli_commands.py index c450008e9a3c..c2a6cc27153f 100644 --- a/tests/others/test_cli_commands.py +++ b/tests/others/test_cli_commands.py @@ -274,11 +274,6 @@ def test_compile_vae_decoder_compiles_the_decoder(self, monkeypatch): assert pipeline.vae.decoder == "compiled" assert compiled == {"fullgraph": True} - def test_compile_vae_decoder_skips_a_pipeline_without_one(self): - pipeline = SimpleNamespace(vae=None) - _compile_vae_decoder(pipeline, "{}") - assert pipeline.vae is None - @require_torch_gpu # `--attention-backend` only exposes Hub-hosted kernels, all of which need `kernels>=0.12`. @require_kernels_version_greater_or_equal("0.12") diff --git a/tests/others/test_cli_serve.py b/tests/others/test_cli_serve.py index 8ce7cda0b9e8..bfbc9c4c7d90 100644 --- a/tests/others/test_cli_serve.py +++ b/tests/others/test_cli_serve.py @@ -509,11 +509,6 @@ def test_failed_generation_returns_error_envelope(self, serve): pipeline.error = None assert client.post("/v1/generations", json=body, headers=WAIT).json()["status"] == "complete" - def test_health(self, serve): - client, _ = serve() - response = client.get("/health") - assert (response.status_code, response.json()) == (200, {"status": "ok"}) - def test_health_checks_do_not_count_as_activity(self, serve): client, _ = serve() client.app.state.last_request = 0.0 @@ -951,18 +946,6 @@ def stop(args): assert resolved["model"] == "org/model", resolved assert resolved["dtype"] == "bf16", "serve should default to bf16 rather than `run`'s fp32 `auto`" - def test_two_different_models_are_rejected(self): - with pytest.raises(SystemExit, match="Got two models"): - ServeCommand(self._parse(["org/model", "-m", "org/other"])).run() - - def test_missing_model_is_rejected(self): - with pytest.raises(SystemExit, match="Pass the model to serve"): - ServeCommand(self._parse([])).run() - - def test_context_parallel_rejects_device_map(self): - with pytest.raises(SystemExit, match="--device-map does not apply with --context-parallel"): - ServeCommand(self._parse(["-m", "org/model", "--context-parallel", "--device-map", "cuda:0"])).run() - def _fake_worker_main(rank, world_size, port, args, served, conn, cancel_flag): """Stands in for `_worker_main` in forked children: no pipeline, no process group.""" @@ -1062,13 +1045,6 @@ def test_worker_arguments_survive_spawn(self): restored_args, restored_served = pickle.loads(pickle.dumps((args, _served()))) assert restored_args.context_parallel and restored_served.tasks - def test_too_few_gpus_is_rejected(self, monkeypatch): - from diffusers.commands.serve.parallel import ParallelBackend - - monkeypatch.setattr(torch.cuda, "device_count", lambda: 1) - with pytest.raises(SystemExit, match="needs at least 2 visible GPUs, found 1"): - ParallelBackend(Namespace(), _served()) - class TestParallelWorker: def _jobs(self, backend, rank, *jobs): From b81768c22923fe990f6ce14f5fef0fc29cea1e5d Mon Sep 17 00:00:00 2001 From: DN6 Date: Wed, 4 Nov 2026 16:53:33 +0530 Subject: [PATCH 09/21] Remove the SGLang and vLLM-Omni serve backends and clean up the serve tests --- .ai/skills/diffusers-cli/SKILL.md | 2 +- docs/source/en/using-diffusers/serve.md | 25 +- src/diffusers/commands/serve/__init__.py | 80 +---- src/diffusers/commands/serve/backends.py | 233 +------------- src/diffusers/commands/serve/manifest.py | 16 +- src/diffusers/commands/serve/parallel.py | 14 +- src/diffusers/commands/serve/remote.py | 13 +- tests/others/test_cli_serve.py | 370 +++++------------------ 8 files changed, 108 insertions(+), 645 deletions(-) diff --git a/.ai/skills/diffusers-cli/SKILL.md b/.ai/skills/diffusers-cli/SKILL.md index b4c5814865a7..204f683e1413 100644 --- a/.ai/skills/diffusers-cli/SKILL.md +++ b/.ai/skills/diffusers-cli/SKILL.md @@ -16,7 +16,7 @@ description: > | --------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `run` | Run any `DiffusionPipeline` or `ModularPipeline`. Forwards `--pipeline-kwargs` verbatim, saves output by detecting its runtime type, optionally runs on HF Jobs via `--remote`. | | `schema` | Print the input schema for a pipeline repo (kwarg names, types, defaults, descriptions). **No weights downloaded** — only the small index file. | -| `serve` | Serve a pipeline over HTTP with the Generative Media Spec API: model discovery, queued generations, polling or SSE progress. In-process, or behind SGLang / vLLM-Omni via `--backend`. | +| `serve` | Serve a pipeline over HTTP with the Generative Media Spec API: model discovery, queued generations, polling or SSE progress. | | `custom_blocks` | Package a local `ModularPipelineBlocks` subclass for the Hub. | | `env` | Print versions of diffusers + torch + transformers + accelerate + safetensors + CUDA + GPU info. Use when investigating environment issues, dtype/precision support, or building bug reports. | diff --git a/docs/source/en/using-diffusers/serve.md b/docs/source/en/using-diffusers/serve.md index 822db93c480b..9103815b3d6d 100644 --- a/docs/source/en/using-diffusers/serve.md +++ b/docs/source/en/using-diffusers/serve.md @@ -40,8 +40,8 @@ so keep it on a trusted network. Run `diffusers serve --help` for every option. long and no generation is queued or running. Health checks don't count as requests. By default a local server never shuts itself down. -[Modular repos](../modular_diffusers/overview) are detected the same way as in `run` and served in-process. They -report no step progress, and a cancel request only takes effect once the pipeline call returns. +[Modular repos](../modular_diffusers/overview) are detected the same way as in `run`. They report no step +progress, and a cancel request only takes effect once the pipeline call returns. ## Generations @@ -165,24 +165,3 @@ how long the Job runs even while it is busy. `--flavor`, `--image`, `--dependenc do for [`run --remote`](./cli#remote-execution---remote). With `--context-parallel`, pick a flavor with several GPUs. The model must be a Hub repo id. `--manifest` is not supported with `--remote`: publish the manifest as `gms.json` in the model repo instead. - -## SGLang and vLLM-Omni backends - -`--backend sglang` and `--backend vllm` run the model in [SGLang Diffusion](https://docs.sglang.io/) or -[vLLM-Omni](https://docs.vllm.ai/projects/vllm-omni) instead of in-process. `serve` starts `sglang serve` or -`vllm serve --omni` as a child process, waits for it to become healthy, and translates GMS requests to the -engine's OpenAI-compatible image and video routes. Clients only talk to the GMS endpoints of `serve`. The engine -must be installed separately. - -```bash -diffusers serve Qwen/Qwen-Image --backend sglang --backend-args '--num-gpus 2' -diffusers serve Wan-AI/Wan2.2-T2V-A14B-Diffusers --backend vllm -``` - -The pipeline flags (`--dtype`, `--cpu-offload`, `--compile`, ...) are ignored with a warning. Pass the engine's own -flags with `--backend-args`. Compared with the in-process backend, these backends serve image and video outputs -only, accept `image` as the only media input, expose only the parameters both engines share, report no step -progress, and apply no request adapters. `GET /health` returns `503` once the engine has exited. - -With `--remote`, the Job starts from the engine's own image (`lmsysorg/sglang:latest` or -`vllm/vllm-omni:latest`) unless `--image` names another. diff --git a/src/diffusers/commands/serve/__init__.py b/src/diffusers/commands/serve/__init__.py index 5faee6ca0416..c1c53b672c0d 100644 --- a/src/diffusers/commands/serve/__init__.py +++ b/src/diffusers/commands/serve/__init__.py @@ -15,8 +15,7 @@ """`diffusers-cli serve` — serve a pipeline over HTTP with the Generative Media Spec (GMS) API. Generations run one at a time on a background worker: a request returns immediately with a queued generation that -clients poll or follow over server-sent events. The pipeline runs in-process by default, or behind an SGLang or -vLLM-Omni server with `--backend`. +clients poll or follow over server-sent events. """ from __future__ import annotations @@ -39,22 +38,6 @@ logger = logging.get_logger("diffusers-cli/serve") DEFAULT_OUTPUT_DIR = str(Path.home() / ".diffusers" / "cli" / "serve" / "outputs") -BACKEND_CHOICES = ("diffusers", "sglang", "vllm") - -# Flags that configure the in-process pipeline, paired with the value argparse gives them when unset. -_DIFFUSERS_ONLY_FLAGS = { - "device_map": None, - "dtype": "bf16", - "variant": None, - "lora": None, - "cpu_offload": None, - "attention_backend": "default", - "vae_tiling": False, - "vae_slicing": False, - "compile": None, - "compile_mode": "regional", - "context_parallel": False, -} def _resolve_pipeline(args: Namespace) -> tuple[type, inspect.Signature, str | None]: @@ -82,11 +65,6 @@ def _resolve_pipeline(args: Namespace) -> tuple[type, inspect.Signature, str | N if not issubclass(pipeline_cls, diffusers.ModularPipeline): return pipeline_cls, inspect.signature(pipeline_cls.__call__), commit_hash - if args.backend != "diffusers": - raise SystemExit( - f"{args.model!r} is a modular pipeline, which --backend {args.backend} cannot serve. " - "Use --backend diffusers." - ) try: pipeline = diffusers.ModularPipeline.from_pretrained( args.model, trust_remote_code=args.trust_remote_code, token=args.token, revision=args.revision @@ -118,8 +96,6 @@ def register_subcommand(subparsers: _SubParsersAction) -> None: "Examples\n" " $ diffusers-cli serve black-forest-labs/FLUX.1-dev\n" " $ diffusers-cli serve black-forest-labs/FLUX.1-dev --compile --port 9000\n" - " $ diffusers-cli serve Wan-AI/Wan2.2-T2V-A14B-Diffusers --backend vllm\n" - " $ diffusers-cli serve Qwen/Qwen-Image --backend sglang --backend-args '--num-gpus 2'\n" "\n" " $ curl localhost:8000/v1/generations -H 'Prefer: wait=120' \\\n" ' -d \'{"model": "black-forest-labs/FLUX.1-dev", "inputs": {"prompt": "a cat on the moon"}}\'\n' @@ -146,25 +122,6 @@ def register_subcommand(subparsers: _SubParsersAction) -> None: ) _add_loading_arguments(parser, require_model=False) _add_optimization_arguments(parser) - parser.add_argument( - "--backend", - choices=BACKEND_CHOICES, - default="diffusers", - help=( - "What runs the model. 'diffusers' loads the pipeline in this process. 'sglang' and 'vllm' start " - "`sglang serve` / `vllm serve --omni` as a child process and translate requests to its " - "OpenAI-compatible image and video routes." - ), - ) - parser.add_argument( - "--backend-args", - default=None, - metavar="ARGS", - help=( - "Extra arguments appended to the `sglang serve` / `vllm serve` command, as one quoted string " - "(e.g. '--num-gpus 2'). Only used with --backend sglang or vllm." - ), - ) parser.add_argument("--host", default="127.0.0.1", help="Interface to bind. Defaults to 127.0.0.1.") parser.add_argument("--port", type=int, default=8000, help="Port to bind. Defaults to 8000.") parser.add_argument( @@ -256,19 +213,6 @@ def run(self) -> None: raise SystemExit( "--device-map does not apply with --context-parallel: each worker loads the pipeline on its own GPU." ) - if args.backend == "diffusers" and args.backend_args is not None: - raise SystemExit("--backend-args only applies to --backend sglang or vllm.") - if args.backend != "diffusers": - ignored = [ - "--" + name.replace("_", "-") - for name, unset in _DIFFUSERS_ONLY_FLAGS.items() - if getattr(args, name) != unset - ] - if ignored: - logger.warning( - f"{', '.join(ignored)} configure the in-process pipeline and are ignored with " - f"--backend {args.backend}. Pass the engine's own flags through --backend-args instead." - ) if args.remote: serve_remote(args, self.task) @@ -283,47 +227,36 @@ def run(self) -> None: ) from e from .app import create_app - from .backends import ( - DiffusersBackend, - EngineBackend, - backend_capabilities, - backend_kwargs, - backend_output_types, - ) + from .backends import DiffusersBackend, backend_capabilities from .generations import GenerationQueue from .manifest import ServedModel, load_manifest from .parallel import ParallelBackend pipeline_cls, signature, commit_hash = _resolve_pipeline(args) - kwarg_names = backend_kwargs(args.backend, signature) manifest = load_manifest( args.model, pipeline_cls, signature, - kwarg_names, manifest_path=args.manifest, revision=args.revision, token=args.token, ) served = ServedModel( manifest, - kwarg_names, - backend_output_types(args.backend), + set(signature.parameters) - {"self"}, backend_capabilities(args, pipeline_cls), revision=commit_hash, ) if not served.tasks: raise SystemExit( f"None of the tasks {list(manifest.data['tasks'])} in the manifest for {args.model!r} can be " - f"served by {pipeline_cls.__name__} with --backend {args.backend}." + f"served by {pipeline_cls.__name__}." ) - if args.backend == "diffusers" and args.context_parallel: + if args.context_parallel: backend = ParallelBackend(args, served) - elif args.backend == "diffusers": - backend = DiffusersBackend(args, served) else: - backend = EngineBackend.launch(args, served) + backend = DiffusersBackend(args, served) try: generations = GenerationQueue(backend, args.output_dir) @@ -338,7 +271,6 @@ def run(self) -> None: self.task, url=f"http://{args.host}:{args.port}", model=served.id, - backend=args.backend, pipeline_class=pipeline_cls.__name__, tasks=list(served.tasks), capabilities=served.capabilities, diff --git a/src/diffusers/commands/serve/backends.py b/src/diffusers/commands/serve/backends.py index 21e8b8a2b854..f9f891bb15f0 100644 --- a/src/diffusers/commands/serve/backends.py +++ b/src/diffusers/commands/serve/backends.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Backends that execute a resolved generation: an in-process pipeline, or an SGLang / vLLM-Omni server.""" +"""The backend that executes a resolved generation with an in-process pipeline.""" from __future__ import annotations @@ -21,12 +21,7 @@ import io import mimetypes import os -import shlex -import shutil -import socket -import subprocess import tempfile -import time import wave from argparse import Namespace from pathlib import Path @@ -55,49 +50,11 @@ logger = logging.get_logger("diffusers-cli/serve") -# Request fields both engines accept on their OpenAI-compatible image and video routes, named as the -# pipeline kwargs they correspond to. -_ENGINE_KWARGS = frozenset( - { - "prompt", - "negative_prompt", - "image", - "width", - "height", - "num_frames", - "fps", - "num_inference_steps", - "guidance_scale", - "guidance_scale_2", - "true_cfg_scale", - "generator", - "num_images_per_prompt", - } -) -_ENGINE_FIELDS = {"num_images_per_prompt": "n"} -_ENGINE_INSTALL = {"sglang": 'pip install "sglang[diffusion]"', "vllm": "pip install vllm-omni"} -_POLL_SECONDS = 1.0 - _OUTPUT_FIELDS = {"image": ("images",), "video": ("frames", "videos"), "audio": ("audios", "audio")} -def backend_kwargs(backend: str, signature: inspect.Signature) -> set[str]: - names = set(signature.parameters) - {"self"} - if backend == "diffusers": - return names - return names & _ENGINE_KWARGS - - -def backend_output_types(backend: str) -> set[str]: - if backend == "diffusers": - return {"image", "video", "audio"} - return {"image", "video"} - - def backend_capabilities(args: Namespace, pipeline_cls: type) -> list[str]: capabilities = ["sse", "wait"] - if args.backend != "diffusers": - return capabilities # Swapping adapter weights retraces a compiled denoiser on every request, and LoRAs passed at # startup are part of the deployment rather than something a request may replace. Under context # parallelism every rank would have to swap them in lockstep, which is untested. @@ -111,25 +68,17 @@ def backend_capabilities(args: Namespace, pipeline_cls: type) -> list[str]: return capabilities -def _media_bytes(name: str, value: dict[str, Any], pointer: str) -> bytes: +def _load_media(name: str, value: dict[str, Any], pointer: str) -> Any: try: if "base64" in value: - return base64.b64decode(value["base64"], validate=True) - response = httpx.get(value["url"], follow_redirects=True, timeout=DIFFUSERS_REQUEST_TIMEOUT) - response.raise_for_status() - return response.content + content = base64.b64decode(value["base64"], validate=True) + else: + response = httpx.get(value["url"], follow_redirects=True, timeout=DIFFUSERS_REQUEST_TIMEOUT) + response.raise_for_status() + content = response.content except (ValueError, httpx.HTTPError) as e: raise GMSError("invalid_request", f"could not read input {name!r}: {e}", pointer) from e - -def _media_suffix(value: dict[str, Any], default: str) -> str: - if value.get("media_type"): - return mimetypes.guess_extension(value["media_type"]) or default - return Path(urlparse(value.get("url", "")).path).suffix or default - - -def _load_media(name: str, value: dict[str, Any], pointer: str) -> Any: - content = _media_bytes(name, value, pointer) try: if value["type"] in ("image", "mask"): return load_image(Image.open(io.BytesIO(content))) @@ -137,7 +86,11 @@ def _load_media(name: str, value: dict[str, Any], pointer: str) -> Any: import torchaudio return torchaudio.load(io.BytesIO(content)) - with tempfile.NamedTemporaryFile(suffix=_media_suffix(value, ".mp4")) as file: + if value.get("media_type"): + suffix = mimetypes.guess_extension(value["media_type"]) or ".mp4" + else: + suffix = Path(urlparse(value.get("url", "")).path).suffix or ".mp4" + with tempfile.NamedTemporaryFile(suffix=suffix) as file: file.write(content) file.flush() return load_video(file.name) @@ -335,165 +288,3 @@ def _load_adapters(self, adapters: list[dict[str, Any]]) -> None: weights.append(scale) names.append(name) self.pipeline.set_adapters(names, adapter_weights=weights) - - -class EngineBackend: - def __init__(self, engine: str, served: ServedModel, client: Any, process: subprocess.Popen | None = None): - self._engine = engine - self._served = served - self._client = client - self._process = process - - @classmethod - def launch(cls, args: Namespace, served: ServedModel) -> EngineBackend: - with socket.socket() as probe: - probe.bind(("127.0.0.1", 0)) - port = probe.getsockname()[1] - - if args.backend == "sglang": - command = ["sglang", "serve", "--model-path", args.model] - else: - command = ["vllm", "serve", args.model, "--omni"] - command += ["--host", "127.0.0.1", "--port", str(port)] - if args.revision: - command += ["--revision", args.revision] - command += shlex.split(args.backend_args or "") - - if shutil.which(command[0]) is None: - raise SystemExit( - f"--backend {args.backend} needs the `{command[0]}` executable on PATH. " - f"Install it with: {_ENGINE_INSTALL[args.backend]}" - ) - - logger.info(f"starting {args.backend} backend: {shlex.join(command)}") - process = subprocess.Popen(command) - client = httpx.Client(base_url=f"http://127.0.0.1:{port}", timeout=None) - backend = cls(args.backend, served, client, process) - try: - backend._wait_until_ready(command) - except BaseException: - backend.close() - raise - return backend - - def failure(self) -> str | None: - if self._process is None or self._process.poll() is None: - return None - return f"the {self._engine} backend exited with code {self._process.returncode}" - - def close(self) -> None: - self._client.close() - if self._process is None or self._process.poll() is not None: - return - self._process.terminate() - try: - self._process.wait(timeout=30) - except subprocess.TimeoutExpired: - self._process.kill() - self._process.wait() - - def generate(self, generation: Generation) -> dict[str, list[dict[str, Any]]]: - request = generation.request - outputs = {spec["type"]: name for name, spec in self._served.tasks[request.task]["outputs"].items()} - - fields = {_ENGINE_FIELDS.get(kwarg, kwarg): value for kwarg, value in request.kwargs.items()} - if generation.seed is not None: - fields["seed"] = generation.seed - width = fields.pop("width", None) - height = fields.pop("height", None) - if (width is None) != (height is None): - missing = "width" if width is None else "height" - raise GMSError( - "invalid_request", - f"the {self._engine} backend takes width and height together; {missing} is missing", - f"/parameters/{missing}", - ) - if width is not None: - fields["size"] = f"{width}x{height}" - - images = [] - if "image" in request.media: - name, value = request.media["image"] - for index, item in enumerate(value if isinstance(value, list) else [value]): - pointer = f"/inputs/{name}/{index}" if isinstance(value, list) else f"/inputs/{name}" - content = _media_bytes(name, item, pointer) - images.append((f"image-{index}{_media_suffix(item, '.png')}", content, item.get("media_type"))) - - if "video" in outputs: - return {outputs["video"]: self._generate_video(generation, fields, images)} - return {outputs["image"]: self._generate_images(generation, fields, images)} - - def _generate_images(self, generation: Generation, fields: dict[str, Any], images: list) -> list[dict[str, Any]]: - media_type = generation.request.output_formats["image"] - extension = media_type.split("/")[1] - fields = {**fields, "response_format": "b64_json", "output_format": extension} - if images: - response = self._client.post("/v1/images/edits", files=self._form(fields, "image", images)) - else: - response = self._client.post("/v1/images/generations", json=fields) - - artifacts = [] - for index, item in enumerate(self._json(response)["data"]): - content = base64.b64decode(item["b64_json"]) - filename = f"image-{index}.{extension}" - (generation.directory / filename).write_bytes(content) - width, height = Image.open(io.BytesIO(content)).size - artifacts.append({"file": filename, "media_type": media_type, "width": width, "height": height}) - return artifacts - - def _generate_video(self, generation: Generation, fields: dict[str, Any], images: list) -> list[dict[str, Any]]: - job = self._json(self._client.post("/v1/videos", files=self._form(fields, "input_reference", images[:1]))) - while job["status"] in ("queued", "in_progress"): - if generation.cancel_requested: - self._client.delete(f"/v1/videos/{job['id']}") - return [] - time.sleep(_POLL_SECONDS) - job = self._json(self._client.get(f"/v1/videos/{job['id']}")) - if job["status"] != "completed": - message = (job.get("error") or {}).get("message", f"video job ended with status {job['status']!r}") - raise GMSError("internal", f"{self._engine} backend: {message}") - - response = self._client.get(f"/v1/videos/{job['id']}/content") - if response.status_code >= 400: - self._json(response) - filename = "video-0.mp4" - (generation.directory / filename).write_bytes(response.content) - self._client.delete(f"/v1/videos/{job['id']}") - - artifact = {"file": filename, "media_type": "video/mp4"} - if job.get("fps"): - artifact["fps"] = job["fps"] - if job.get("duration_s"): - artifact["duration"] = job["duration_s"] - return [artifact] - - @staticmethod - def _form(fields: dict[str, Any], file_field: str, files: list) -> list: - # Both engines read these routes as multipart forms, so plain fields are sent as nameless - # parts: httpx falls back to urlencoding when a request carries no file parts. - parts = [] - for key, value in fields.items(): - text = str(value).lower() if isinstance(value, bool) else str(value) - parts.append((key, (None, text))) - for filename, content, media_type in files: - parts.append((file_field, (filename, content, media_type or "application/octet-stream"))) - return parts - - def _json(self, response: Any) -> Any: - if response.status_code >= 400: - code = "invalid_request" if response.status_code < 500 else "internal" - raise GMSError(code, f"{self._engine} backend returned {response.status_code}: {response.text[:500]}") - return response.json() - - def _wait_until_ready(self, command: list[str]) -> None: - while True: - if self._process.poll() is not None: - raise SystemExit( - f"`{shlex.join(command)}` exited with code {self._process.returncode} before becoming ready." - ) - try: - if self._client.get("/health", timeout=5.0).status_code == 200: - return - except httpx.HTTPError: - pass - time.sleep(2.0) diff --git a/src/diffusers/commands/serve/manifest.py b/src/diffusers/commands/serve/manifest.py index 054dbb8a9c93..89fbce794ac0 100644 --- a/src/diffusers/commands/serve/manifest.py +++ b/src/diffusers/commands/serve/manifest.py @@ -167,7 +167,6 @@ def load_manifest( model: str, pipeline_cls: type, signature: inspect.Signature, - kwarg_names: set[str], manifest_path: str | None = None, revision: str | None = None, token: str | None = None, @@ -190,7 +189,7 @@ def load_manifest( return manifest model_id = Path(model).name if Path(model).is_dir() else model - data = derive_manifest(model_id, pipeline_cls, signature, kwarg_names) + data = derive_manifest(model_id, pipeline_cls, signature) return Manifest(raw=json.dumps(data).encode(), data=data) @@ -239,10 +238,8 @@ def scalar(fragment: str) -> str | None: return {"type": "array", "items": {"type": scalar_type}} -def derive_manifest( - model_id: str, pipeline_cls: type, signature: inspect.Signature, kwarg_names: set[str] -) -> dict[str, Any]: - """Build a server-authored manifest from the pipeline call `signature`, keeping only kwargs in `kwarg_names`.""" +def derive_manifest(model_id: str, pipeline_cls: type, signature: inspect.Signature) -> dict[str, Any]: + """Build a server-authored manifest from the pipeline call `signature`.""" inputs: dict[str, dict[str, Any]] = {} parameters: dict[str, dict[str, Any]] = {} input_kwargs: dict[str, str] = {} @@ -257,7 +254,7 @@ def derive_manifest( del kwarg_parameters["num_frames"] for kwarg, parameter in signature.parameters.items(): - if kwarg == "self" or kwarg in _SKIPPED_KWARGS or kwarg not in kwarg_names: + if kwarg == "self" or kwarg in _SKIPPED_KWARGS: continue if parameter.kind in (inspect.Parameter.VAR_POSITIONAL, inspect.Parameter.VAR_KEYWORD): continue @@ -447,13 +444,12 @@ def _check_media(name: str, value: Any, spec: dict[str, Any], pointer: str) -> N class ServedModel: - """One manifest bound to the kwargs a backend accepts: the discovery document and request resolution.""" + """One manifest bound to the kwargs the pipeline accepts: the discovery document and request resolution.""" def __init__( self, manifest: Manifest, kwarg_names: set[str], - output_types: set[str], capabilities: list[str], revision: str | None = None, ): @@ -481,7 +477,7 @@ def __init__( parameters["duration"] = None if any(inputs[key] is None for key, spec in task["inputs"].items() if spec.get("required")): continue - if not any(spec["type"] in output_types for spec in task["outputs"].values()): + if not any(spec["type"] in OUTPUT_FORMATS for spec in task["outputs"].values()): continue self.tasks[name] = task self._inputs[name] = inputs diff --git a/src/diffusers/commands/serve/parallel.py b/src/diffusers/commands/serve/parallel.py index 86029e849370..fdbfd1278525 100644 --- a/src/diffusers/commands/serve/parallel.py +++ b/src/diffusers/commands/serve/parallel.py @@ -127,6 +127,7 @@ def __init__(self, args: Namespace, served: ServedModel): self._served = served self._failure: str | None = None + self._failed = threading.Event() self._closing = False # Spawned rather than forked: a forked child inherits whatever CUDA state its parent has. context = multiprocessing.get_context("spawn") @@ -173,7 +174,10 @@ def _monitor(self) -> None: dead = sentinels[wait(list(sentinels))[0]] if self._closing: return + # `exitcode` polls without waiting, so it can still be None just after the sentinel fires. + dead.join() self._failure = f"context-parallel worker {dead.name} exited with code {dead.exitcode}" + self._failed.set() logger.error(f"{self._failure}; the remaining workers are stopped and the server no longer generates.") # The other ranks would wait forever in their next collective, so the group is not reused. self.close() @@ -205,13 +209,9 @@ def generate(self, generation: Generation) -> dict[str, list[dict[str, Any]]]: continue message = rank_0.recv() except (EOFError, OSError) as e: - # The monitor may not have noticed yet; rank 0's pipe only breaks once it has exited. - self._processes[0].join() - if self._failure is None: - self._failure = ( - f"context-parallel worker {self._processes[0].name} exited with code " - f"{self._processes[0].exitcode}" - ) + # Rank 0's pipe only breaks once it has exited. Only the monitor reaps workers: a second + # thread joining the same process can read its exit code before it is recorded. + self._failed.wait() raise GMSError("internal", self._failure) from e if message[0] == "progress": generation.report_progress(*message[1:]) diff --git a/src/diffusers/commands/serve/remote.py b/src/diffusers/commands/serve/remote.py index 0cd97fd1b9ee..aedeb56114a7 100644 --- a/src/diffusers/commands/serve/remote.py +++ b/src/diffusers/commands/serve/remote.py @@ -38,8 +38,6 @@ JOB_PORT = 8000 _SERVE_DEPS = ("fastapi", "uvicorn") _DEFAULT_REMOTE_IDLE_TIMEOUT = "30m" -# Each engine's own image carries the engine, its kernels and the CUDA toolchain they are built with. -_ENGINE_IMAGES = {"sglang": "lmsysorg/sglang:latest", "vllm": "vllm/vllm-omni:latest"} _READY_POLL_SECONDS = 5.0 # Flags the Job command sets itself. `_build_task_kwargs` already drops the flags that say how the Job is set up. @@ -71,10 +69,7 @@ def add_remote_arguments(parser: ArgumentParser) -> None: parser.add_argument( "--image", default=None, - help=( - f"Job image for --remote. Defaults to {_DEFAULT_REMOTE_IMAGE!r}, or to the engine's own image " - f"with --backend sglang ({_ENGINE_IMAGES['sglang']!r}) or vllm ({_ENGINE_IMAGES['vllm']!r})." - ), + help=f"Job image for --remote. Defaults to {_DEFAULT_REMOTE_IMAGE!r}.", ) parser.add_argument( "--timeout", @@ -119,8 +114,8 @@ def _job_script(args: Namespace, task: str) -> str: dependencies = [*_DEFAULT_REMOTE_DEPS, *_SERVE_DEPS, *(args.dependencies or [])] install = shlex.join(["uv", "pip", "install", "--system", "--break-system-packages", *dependencies]) - # An image can put another environment first on PATH, with its own older `diffusers-cli` (the SGLang image - # does), so the server is started with the interpreter the install went into. The Job id is only known + # An `--image` can put another environment first on PATH, with its own older `diffusers-cli`, so the server + # is started with the interpreter the install went into. The Job id is only known # inside the Job, and output links must be built on the public address. start = ( 'exec "$(uv python find --system)" -m diffusers.commands.diffusers_cli --format quiet ' @@ -153,7 +148,7 @@ def serve_remote(args: Namespace, task: str) -> None: api = HfApi(token=hf_token) logger.info(f"starting a Job on flavor={args.flavor!r}...") job = api.run_job( - image=args.image or _ENGINE_IMAGES.get(args.backend, _DEFAULT_REMOTE_IMAGE), + image=args.image or _DEFAULT_REMOTE_IMAGE, command=["sh", "-c", _job_script(args, task)], env=env, secrets={"HF_TOKEN": hf_token} if hf_token else None, diff --git a/tests/others/test_cli_serve.py b/tests/others/test_cli_serve.py index bfbc9c4c7d90..0b2823340b25 100644 --- a/tests/others/test_cli_serve.py +++ b/tests/others/test_cli_serve.py @@ -13,8 +13,7 @@ # limitations under the License. """Unit tests for `diffusers-cli serve`. -No weights are loaded: the diffusers backend runs a fake pipeline and the engine backend talks to a fake -SGLang / vLLM-Omni server through an httpx mock transport. +No weights are loaded: the backend runs a fake pipeline. """ import base64 @@ -23,8 +22,6 @@ import json import os import shlex -import subprocess -import sys import threading import time from argparse import ArgumentParser, Namespace @@ -34,22 +31,13 @@ import numpy as np import pytest import torch -from huggingface_hub.utils import httpx from PIL import Image from diffusers.commands.serve import ServeCommand, _shut_down_when_idle -from diffusers.commands.serve.backends import ( - DiffusersBackend, - EngineBackend, - backend_capabilities, - backend_kwargs, - backend_output_types, -) -from diffusers.commands.serve.generations import Generation, GenerationQueue +from diffusers.commands.serve.backends import DiffusersBackend, backend_capabilities +from diffusers.commands.serve.generations import GenerationQueue from diffusers.commands.serve.manifest import GMSError, Manifest, ServedModel, blocks_signature, derive_manifest from diffusers.modular_pipelines import InputParam, ModularPipelineBlocks, OutputParam -from diffusers.utils import logging -from diffusers.utils.testing_utils import CaptureLogger MODEL_ID = "test-org/fake-model" @@ -157,7 +145,6 @@ def __call__(self, components, state): def _args(**overrides): values = { - "backend": "diffusers", "compile": None, "lora": None, "context_parallel": False, @@ -167,35 +154,34 @@ def _args(**overrides): return Namespace(**{**values, **overrides}) -def _served(backend="diffusers", manifest=None, pipeline_cls=FakePipeline, **arg_overrides): - args = _args(backend=backend, **arg_overrides) - signature = inspect.signature(pipeline_cls.__call__) - kwarg_names = backend_kwargs(backend, signature) +def _served(manifest=None, pipeline_cls=FakePipeline, signature=None, args=None): + signature = signature or inspect.signature(pipeline_cls.__call__) if manifest is None: - manifest = derive_manifest(MODEL_ID, pipeline_cls, signature, kwarg_names) + manifest = derive_manifest(MODEL_ID, pipeline_cls, signature) raw = manifest if isinstance(manifest, bytes) else json.dumps(manifest).encode() return ServedModel( Manifest(raw=raw, data=json.loads(raw)), - kwarg_names, - backend_output_types(backend), - backend_capabilities(args, pipeline_cls), + set(signature.parameters) - {"self"}, + backend_capabilities(args or _args(), pipeline_cls), revision="abc123", ) @pytest.fixture def serve(tmp_path, monkeypatch): - """Build a test client around a `DiffusersBackend` running `FakePipeline`. Returns `(client, pipeline)`.""" + """Build a test client around a `DiffusersBackend` running `FakePipeline`, or around the given pipeline or + backend. Returns `(client, pipeline)`.""" pytest.importorskip("fastapi", reason="`diffusers-cli serve` needs the `diffusers[serve]` extra") from fastapi.testclient import TestClient from diffusers.commands.serve.app import create_app - def _build(manifest=None, app_options=None, **arg_overrides): - pipeline = FakePipeline() + def _build(manifest=None, app_options=None, pipeline=None, signature=None, backend=None, **arg_overrides): + pipeline = pipeline or FakePipeline() monkeypatch.setattr("diffusers.commands.serve.backends._load_pipeline", lambda args: pipeline) - served = _served(manifest=manifest, **arg_overrides) - backend = DiffusersBackend(_args(**arg_overrides), served) + args = _args(**arg_overrides) + served = _served(manifest=manifest, pipeline_cls=type(pipeline), signature=signature, args=args) + backend = backend or DiffusersBackend(args, served) app = create_app(served, GenerationQueue(backend, str(tmp_path)), **(app_options or {})) return TestClient(app), pipeline @@ -250,7 +236,7 @@ def __call__( class TestManifest: def test_derive_manifest_from_signature(self): - manifest = derive_manifest(MODEL_ID, FakePipeline, FAKE_SIGNATURE, backend_kwargs("diffusers", FAKE_SIGNATURE)) + manifest = derive_manifest(MODEL_ID, FakePipeline, FAKE_SIGNATURE) # An optional `image` kwarg yields one task without it and one that requires it. assert list(manifest["tasks"]) == ["text_to_image", "image_to_image"], manifest["tasks"].keys() @@ -280,35 +266,18 @@ def test_derive_manifest_from_signature(self): "body, code, pointer", [ ({"parameters": {"cfg": 3}}, "invalid_request", "/parameters/cfg"), - ({"parameters": {"width": 700}}, "invalid_request", "/parameters/width"), ({"parameters": {"steps": 500}}, "invalid_request", "/parameters/steps"), ({"parameters": {"steps": "8"}}, "invalid_request", "/parameters/steps"), ({"inputs": {}}, "invalid_request", "/inputs/prompt"), - ({"inputs": {"prompt": "a cat", "caption": "x"}}, "invalid_request", "/inputs/caption"), ({"task": "image_to_video"}, "invalid_request", "/task"), - ({"task": []}, "invalid_request", "/task"), - ({"parameters": []}, "invalid_request", "/parameters"), - ({"parameters": {"seed": 10**30}}, "invalid_request", "/parameters/seed"), - ({"parameters": {"duration": float("nan")}}, "invalid_request", "/parameters/duration"), # The lattice pins duration to 24 fps, so another fps would deliver a different length. ({"parameters": {"duration": 4, "fps": 12}}, "invalid_request", "/parameters/fps"), - ({"steps": 8}, "invalid_request", "/steps"), - ({"preset": "quality"}, "invalid_request", "/preset"), - ({"model": "someone/else"}, "not_found", None), - ({"adapters": [{"type": "lora", "path": "hf:a/b"}]}, "not_implemented", "/adapters"), - ({"output_format": "video/webm"}, "invalid_request", "/output_format"), # `negative_prompt` is only honored while guidance_scale > 1. ( {"inputs": {"prompt": "a cat", "negative_prompt": "blurry"}, "preset": "distilled"}, "invalid_request", "/inputs/negative_prompt", ), - # `last_image` has no matching pipeline kwarg, so the server overlay disables it. - ( - {"inputs": {"prompt": "a cat", "last_image": {"type": "image", "url": "https://x/y.png"}}}, - "invalid_request", - "/inputs/last_image", - ), # A parameter the pipeline cannot set is refused rather than dropped, unless it is the default. ({"parameters": {"video_decoder": "diffusion"}}, "not_implemented", "/parameters/video_decoder"), ], @@ -473,19 +442,6 @@ def test_joint_video_and_audio_outputs(self, serve, monkeypatch): } assert client.get(audio["url"]).content[:4] == b"RIFF" - def test_validation_error_envelope(self, serve): - client, _ = serve() - body = {"model": MODEL_ID, "inputs": {"prompt": "a cat"}, "parameters": {"cfg_scale": 3}} - response = client.post("/v1/generations", json=body) - assert response.status_code == 400 - assert response.json() == { - "error": { - "code": "invalid_request", - "message": "task 'text_to_image' declares no parameter 'cfg_scale'", - "pointer": "/parameters/cfg_scale", - } - } - def test_failed_generation_returns_error_envelope(self, serve): client, pipeline = serve() body = {"model": MODEL_ID, "inputs": {"prompt": "a cat"}} @@ -509,6 +465,13 @@ def test_failed_generation_returns_error_envelope(self, serve): pipeline.error = None assert client.post("/v1/generations", json=body, headers=WAIT).json()["status"] == "complete" + def test_failed_backend_is_unhealthy(self, serve): + backend = SimpleNamespace(failure=lambda: "context-parallel worker rank-1 exited with code 3") + client, _ = serve(backend=backend) + response = client.get("/health") + expected = {"status": "unhealthy", "reason": "context-parallel worker rank-1 exited with code 3"} + assert (response.status_code, response.json()) == (503, expected) + def test_health_checks_do_not_count_as_activity(self, serve): client, _ = serve() client.app.state.last_request = 0.0 @@ -517,19 +480,6 @@ def test_health_checks_do_not_count_as_activity(self, serve): client.get("/v1/models") assert client.app.state.last_request > 0.0, "a request should reset the idle clock" - def test_request_id_is_generated_or_echoed(self, serve): - client, _ = serve() - assert client.get("/health").headers["x-request-id"], "a request without an id should be given one" - echoed = client.get("/v1/models/unknown", headers={"x-request-id": "req-1"}) - assert (echoed.status_code, echoed.headers["x-request-id"]) == (404, "req-1") - - def test_default_seed_applies_when_request_omits_seed(self, serve): - client, _ = serve(app_options={"default_seed": 7}) - body = {"model": MODEL_ID, "inputs": {"prompt": "a cat"}} - assert client.post("/v1/generations", json=body, headers=WAIT).json()["seed"] == 7 - body["parameters"] = {"seed": 11} - assert client.post("/v1/generations", json=body, headers=WAIT).json()["seed"] == 11 - def test_public_url_replaces_the_request_address_in_output_links(self, serve): client, _ = serve(app_options={"public_url": "https://proxy.example/sandbox/proxy/8000/"}) body = {"model": MODEL_ID, "inputs": {"prompt": "a cat"}} @@ -543,13 +493,6 @@ def test_public_url_replaces_the_request_address_in_output_links(self, serve): prefixed = client.post("/sandbox/proxy/8000/v1/generations", json=body, headers=WAIT) assert prefixed.status_code == 201, prefixed.text - def test_cors_is_opt_in(self, serve): - origin = {"Origin": "http://example.com"} - client, _ = serve() - assert "access-control-allow-origin" not in client.get("/health", headers=origin).headers - client, _ = serve(app_options={"enable_cors": True}) - assert client.get("/health", headers=origin).headers["access-control-allow-origin"] == "*" - def test_cancel_queued_and_running(self, serve): client, pipeline = serve() pipeline.release.clear() @@ -598,7 +541,7 @@ def test_adapters_not_advertised_when_compiled(self, serve): assert (response.status_code, response.json()["error"]["code"]) == (501, "not_implemented") def test_discovery_serves_manifest_bytes_verbatim(self, serve): - manifest = derive_manifest(MODEL_ID, FakePipeline, FAKE_SIGNATURE, backend_kwargs("diffusers", FAKE_SIGNATURE)) + manifest = derive_manifest(MODEL_ID, FakePipeline, FAKE_SIGNATURE) raw = json.dumps(manifest, indent=3).encode() client, _ = serve(manifest=raw) @@ -619,70 +562,12 @@ def test_discovery_serves_manifest_bytes_verbatim(self, serve): ] } - def test_unadvertised_capabilities_return_not_implemented(self, serve): - client, _ = serve() - for response in ( - client.post("/v1/uploads", json={}), - client.get("/v1/generations/gen_x/outputs/video/stream"), - ): - assert (response.status_code, response.json()["error"]["code"]) == (501, "not_implemented") - - -class FakeEngine: - """The image and video routes SGLang and vLLM-Omni share, recording what the backend sends.""" - - def __init__(self): - self.requests = [] - self.polls = 0 - - def __call__(self, request): - content_type = request.headers.get("content-type", "") - self.requests.append((request.method, request.url.path, content_type.split(";")[0], request.content)) - path = request.url.path - if path in ("/v1/images/generations", "/v1/images/edits"): - buffer = io.BytesIO() - Image.new("RGB", (16, 8), "green").save(buffer, format="PNG") - return httpx.Response(200, json={"data": [{"b64_json": base64.b64encode(buffer.getvalue()).decode()}]}) - if path == "/v1/videos" and request.method == "POST": - return httpx.Response(200, json={"id": "video_1", "status": "queued"}) - if path == "/v1/videos/video_1" and request.method == "GET": - self.polls += 1 - if self.polls < 2: - return httpx.Response(200, json={"id": "video_1", "status": "in_progress"}) - return httpx.Response(200, json={"id": "video_1", "status": "completed", "fps": 16.0, "duration_s": 5.0}) - if path == "/v1/videos/video_1/content": - return httpx.Response(200, content=b"mp4-bytes") - if path == "/v1/videos/video_1" and request.method == "DELETE": - return httpx.Response(200, json={"id": "video_1", "deleted": True}) - return httpx.Response(404, text="no such route") - - -class FakeTextToVideoPipeline: - def __call__( - self, - prompt: str = None, - image: Image.Image = None, - height: int = 480, - width: int = 832, - num_frames: int = 81, - num_inference_steps: int = 50, - generator: torch.Generator | None = None, - ): - pass - class TestModularPipeline: - def test_generation_runs_a_modular_pipeline(self, tmp_path, monkeypatch): - pytest.importorskip("fastapi", reason="`diffusers-cli serve` needs the `diffusers[serve]` extra") - from fastapi.testclient import TestClient - - from diffusers.commands.serve.app import create_app - + def test_generation_runs_a_modular_pipeline(self, serve): pipeline = FakeImageBlock().init_pipeline() - monkeypatch.setattr("diffusers.commands.serve.backends._load_pipeline", lambda args: pipeline) signature = blocks_signature(pipeline.blocks) - kwarg_names = backend_kwargs("diffusers", signature) - manifest = derive_manifest(MODEL_ID, type(pipeline), signature, kwarg_names) + manifest = derive_manifest(MODEL_ID, type(pipeline), signature) assert list(manifest["tasks"]) == ["text_to_image", "image_to_image"], manifest["tasks"].keys() assert manifest["tasks"]["text_to_image"]["inputs"] == {"prompt": {"type": "text", "required": True}} assert manifest["tasks"]["text_to_image"]["parameters"] == { @@ -690,16 +575,7 @@ def test_generation_runs_a_modular_pipeline(self, tmp_path, monkeypatch): "seed": {"type": "integer"}, } - raw = json.dumps(manifest).encode() - served = ServedModel( - Manifest(raw=raw, data=manifest), - kwarg_names, - backend_output_types("diffusers"), - backend_capabilities(_args(), type(pipeline)), - revision=None, - ) - backend = DiffusersBackend(_args(), served) - client = TestClient(create_app(served, GenerationQueue(backend, str(tmp_path)))) + client, _ = serve(manifest=manifest, pipeline=pipeline, signature=signature) body = {"model": MODEL_ID, "inputs": {"prompt": "a cat"}, "parameters": {"seed": 3}} generation = client.post("/v1/generations", json=body, headers=WAIT).json() @@ -709,98 +585,6 @@ def test_generation_runs_a_modular_pipeline(self, tmp_path, monkeypatch): assert (artifact["media_type"], artifact["width"], artifact["height"]) == ("image/png", 32, 32) -class TestEngineBackend: - def _generate(self, tmp_path, monkeypatch, served, body): - monkeypatch.setattr("diffusers.commands.serve.backends._POLL_SECONDS", 0.0) - engine = FakeEngine() - client = httpx.Client(base_url="http://engine", transport=httpx.MockTransport(engine)) - backend = EngineBackend("vllm", served, client) - resolved = served.resolve({"model": MODEL_ID, **body}) - generation = Generation(served.id, None, resolved, seeded=served.seeded(resolved.task)) - generation.directory = tmp_path - return engine, backend.generate(generation) - - def test_exited_engine_is_unhealthy(self, tmp_path): - pytest.importorskip("fastapi", reason="`diffusers-cli serve` needs the `diffusers[serve]` extra") - from fastapi.testclient import TestClient - - from diffusers.commands.serve.app import create_app - - process = subprocess.Popen([sys.executable, "-c", "raise SystemExit(3)"]) - process.wait() - served = _served(backend="vllm") - backend = EngineBackend("vllm", served, httpx.Client(base_url="http://engine"), process) - response = TestClient(create_app(served, GenerationQueue(backend, str(tmp_path)))).get("/health") - expected = {"status": "unhealthy", "reason": "the vllm backend exited with code 3"} - assert (response.status_code, response.json()) == (503, expected) - - def test_derived_manifest_keeps_only_engine_kwargs(self): - manifest = derive_manifest(MODEL_ID, FakePipeline, FAKE_SIGNATURE, backend_kwargs("vllm", FAKE_SIGNATURE)) - assert "sigmas" not in manifest["tasks"]["text_to_image"]["parameters"] - assert "steps" in manifest["tasks"]["text_to_image"]["parameters"] - - def test_text_to_image_request_translation(self, tmp_path, monkeypatch): - body = { - "inputs": {"prompt": "a cat", "negative_prompt": "blurry"}, - "parameters": {"width": 1024, "height": 768, "steps": 20, "num_outputs": 2, "seed": 5}, - "output_format": "image/webp", - } - engine, outputs = self._generate(tmp_path, monkeypatch, _served(backend="vllm"), body) - - ((method, path, content_type, content),) = engine.requests - assert (method, path, content_type) == ("POST", "/v1/images/generations", "application/json") - assert json.loads(content) == { - "prompt": "a cat", - "negative_prompt": "blurry", - "num_inference_steps": 20, - "guidance_scale": 3.5, - "n": 2, - "seed": 5, - "size": "1024x768", - "response_format": "b64_json", - "output_format": "webp", - } - assert outputs == {"image": [{"file": "image-0.webp", "media_type": "image/webp", "width": 16, "height": 8}]} - assert (tmp_path / "image-0.webp").is_file() - - def test_image_input_uses_multipart_edits(self, tmp_path, monkeypatch): - image = {"type": "image", "base64": base64.b64encode(b"png-bytes").decode(), "media_type": "image/png"} - body = {"inputs": {"prompt": "make it blue", "image": image}, "parameters": {"seed": 5}} - engine, _ = self._generate(tmp_path, monkeypatch, _served(backend="sglang"), body) - - ((method, path, content_type, content),) = engine.requests - assert (method, path, content_type) == ("POST", "/v1/images/edits", "multipart/form-data") - assert b'name="image"; filename="image-0.png"' in content and b"png-bytes" in content - assert b'name="prompt"\r\n\r\nmake it blue' in content - - def test_video_job_is_polled_downloaded_and_deleted(self, tmp_path, monkeypatch): - served = _served(backend="vllm", pipeline_cls=FakeTextToVideoPipeline) - body = {"task": "text_to_video", "inputs": {"prompt": "a cat"}, "parameters": {"num_frames": 33, "seed": 5}} - engine, outputs = self._generate(tmp_path, monkeypatch, served, body) - - assert [(method, path) for method, path, _, _ in engine.requests] == [ - ("POST", "/v1/videos"), - ("GET", "/v1/videos/video_1"), - ("GET", "/v1/videos/video_1"), - ("GET", "/v1/videos/video_1/content"), - ("DELETE", "/v1/videos/video_1"), - ] - _, _, content_type, content = engine.requests[0] - assert content_type == "multipart/form-data" - assert b'name="num_frames"\r\n\r\n33' in content and b'name="size"\r\n\r\n832x480' in content - assert outputs == {"video": [{"file": "video-0.mp4", "media_type": "video/mp4", "fps": 16.0, "duration": 5.0}]} - assert (tmp_path / "video-0.mp4").read_bytes() == b"mp4-bytes" - - def test_audio_tasks_are_not_hosted(self): - manifest = { - "id": MODEL_ID, - "tasks": { - "text_to_audio": {"inputs": {"prompt": {"type": "text"}}, "outputs": {"audio": {"type": "audio"}}} - }, - } - assert _served(backend="vllm", manifest=manifest).tasks == {} - - class FakeJobsApi: def __init__(self, logs=("server started",), stage="COMPLETED"): self.logs = logs @@ -877,40 +661,40 @@ def test_failed_job_is_reported(self, monkeypatch): with pytest.raises(SystemExit, match="Job job-1 ended with status ERROR"): self._run(monkeypatch, FakeJobsApi(stage="ERROR"), []) - @pytest.mark.parametrize( - "backend, image", [("sglang", "lmsysorg/sglang:latest"), ("vllm", "vllm/vllm-omni:latest")] - ) - def test_engine_backend_defaults_to_the_engine_image(self, monkeypatch, backend, image): - api = FakeJobsApi() - self._run(monkeypatch, api, ["--backend", backend]) - assert api.run_kwargs["image"] == image - self._run(monkeypatch, api, ["--backend", backend, "--image", "mine"]) - assert api.run_kwargs["image"] == "mine", "an explicit --image must win over the engine default" +class FakeClock: + """Stands in for the `time` module: `sleep` advances the clock instead of waiting.""" + + def __init__(self): + self.now = 0.0 + + def monotonic(self): + return self.now + + def sleep(self, seconds): + if self.now > 1000: + raise AssertionError("the idle watcher never shut the server down") + self.now += seconds class TestIdleShutdown: - def _watch(self, busy): + def _watch(self, monkeypatch, busy_checks): + """Run the watcher with a 1m timeout, checked every 30s, until it shuts the server down; returns when it did.""" + clock = FakeClock() + monkeypatch.setattr("diffusers.commands.serve.time", clock) + busy = iter(busy_checks) server = SimpleNamespace(should_exit=False) app = SimpleNamespace(state=SimpleNamespace(last_request=0.0)) - generations = SimpleNamespace(busy=lambda: busy[0]) - watcher = threading.Thread(target=_shut_down_when_idle, args=(server, app, generations, "0.05"), daemon=True) - watcher.start() - return server, watcher - - def test_idle_server_shuts_down(self): - server, watcher = self._watch(busy=[False]) - watcher.join(timeout=5) - assert server.should_exit, "an idle server should be told to exit" - - def test_busy_server_stays_up_until_the_work_ends(self): - busy = [True] - server, watcher = self._watch(busy) - watcher.join(timeout=0.3) - assert not server.should_exit, "a queued or running generation must keep the server up" - busy[0] = False - watcher.join(timeout=5) - assert server.should_exit, "the server should exit once the work ends and the timeout passes" + _shut_down_when_idle(server, app, SimpleNamespace(busy=lambda: next(busy, False)), "1m") + assert server.should_exit, "the watcher should only return once it told the server to exit" + return clock.now + + def test_idle_server_shuts_down(self, monkeypatch): + assert self._watch(monkeypatch, []) == 60, "an idle server should exit after one timeout" + + def test_busy_server_stays_up_until_the_work_ends(self, monkeypatch): + # Busy at the checks at 30s and 60s, so the idle clock restarts at 60s. + assert self._watch(monkeypatch, [True, True]) == 120, "the server should exit one timeout after the work ends" class TestServeCommand: @@ -919,19 +703,6 @@ def _parse(self, argv): ServeCommand.register_subcommand(parser.add_subparsers()) return parser.parse_args(["serve", *argv]) - def test_engine_backend_warns_about_ignored_pipeline_flags(self, monkeypatch): - pytest.importorskip("fastapi", reason="`diffusers-cli serve` needs the `diffusers[serve]` extra") - - def stop(args): - raise RuntimeError("stop after the flag check") - - monkeypatch.setattr("diffusers.commands.serve._resolve_pipeline", stop) - args = self._parse(["-m", "org/model", "--backend", "vllm", "--dtype", "fp16", "--vae-tiling"]) - with CaptureLogger(logging.get_logger("diffusers-cli/serve")) as captured: - with pytest.raises(RuntimeError, match="stop after the flag check"): - ServeCommand(args).run() - assert "--dtype, --vae-tiling configure the in-process pipeline and are ignored" in captured.out - def test_model_is_positional_and_loads_in_bf16(self, monkeypatch): pytest.importorskip("fastapi", reason="`diffusers-cli serve` needs the `diffusers[serve]` extra") resolved = {} @@ -961,10 +732,13 @@ def _fake_worker_main(rank, world_size, port, args, served, conn, cancel_flag): if rank != 0: continue conn.send(("progress", 1, 2)) - # Give the server time to raise the cancel flag between the two steps. - time.sleep(0.3) - if cancel_flag.value: - conn.send(("complete", {})) + if job["kwargs"]["prompt"] == "cancel me": + # The server raises the flag once it has read the progress above. + for _ in range(1000): + if cancel_flag.value: + break + time.sleep(0.01) + conn.send(("complete", {} if cancel_flag.value else {"image": []})) elif job["kwargs"]["prompt"] == "bad": conn.send(("failed", "invalid_request", "bad prompt", "/inputs/prompt")) else: @@ -981,6 +755,8 @@ def __init__(self, prompt="a cat", seed=0): def report_progress(self, step, total_steps): self.progress.append((step, total_steps)) + if self.request.kwargs["prompt"] == "cancel me": + self.cancel_requested = True class TestParallelBackend: @@ -1019,9 +795,8 @@ def test_worker_error_becomes_a_request_error(self, backend): assert (error.value.code, error.value.pointer) == ("invalid_request", "/inputs/prompt") def test_cancel_reaches_the_workers(self, backend): - generation = FakeGeneration() - threading.Timer(0.1, lambda: setattr(generation, "cancel_requested", True)).start() - assert backend().generate(generation) == {}, "a cancelled generation returns no outputs" + outputs = backend().generate(FakeGeneration(prompt="cancel me")) + assert outputs == {}, "a cancel raised mid-generation should reach the workers" def test_dead_worker_fails_the_server(self, backend): server = backend() @@ -1064,7 +839,7 @@ def _jobs(self, backend, rank, *jobs): def _job(self, prompt): return {"request": None, "kwargs": {"prompt": prompt}, "seed": 0, "directory": "/tmp/unused"} - def test_rank_0_reports_progress_and_results(self): + def test_only_rank_0_reports_progress_and_results(self): def run(kwargs, generation): generation.report_progress(1, 1) if kwargs["prompt"] == "bad": @@ -1073,7 +848,8 @@ def run(kwargs, generation): raise SystemExit("unsupported") return {"image": []} - replies = self._jobs(SimpleNamespace(run=run), 0, self._job("ok"), self._job("bad"), self._job("crash")) + backend = SimpleNamespace(run=run) + replies = self._jobs(backend, 0, self._job("ok"), self._job("bad"), self._job("crash")) assert replies == [ ("progress", 1, 1), ("complete", {"image": []}), @@ -1082,13 +858,7 @@ def run(kwargs, generation): ("progress", 1, 1), ("failed", "internal", "SystemExit: unsupported", None), ], "a failed generation must not end the worker" - - def test_other_ranks_stay_silent(self): - def run(kwargs, generation): - generation.report_progress(1, 1) - return {"image": []} - - assert self._jobs(SimpleNamespace(run=run), 1, self._job("ok")) == [] + assert self._jobs(backend, 1, self._job("ok"), self._job("bad")) == [], "only rank 0 talks to the server" def test_cancel_is_broadcast_from_rank_0(self, monkeypatch): from multiprocessing import Value From a7e8ea0221ad51fbe7e6527d31461d102c16d091 Mon Sep 17 00:00:00 2001 From: DN6 Date: Wed, 4 Nov 2026 18:15:47 +0530 Subject: [PATCH 10/21] Reorganize serve modules and validate manifests at startup --- docs/source/en/using-diffusers/serve.md | 19 +- src/diffusers/commands/serve/__init__.py | 40 +- src/diffusers/commands/serve/app.py | 121 +-- src/diffusers/commands/serve/backends.py | 22 +- src/diffusers/commands/serve/manifest.py | 875 ++++++++++-------- src/diffusers/commands/serve/parallel.py | 22 +- .../serve/{generations.py => scheduler.py} | 25 +- tests/others/test_cli_serve.py | 121 ++- 8 files changed, 687 insertions(+), 558 deletions(-) rename src/diffusers/commands/serve/{generations.py => scheduler.py} (89%) diff --git a/docs/source/en/using-diffusers/serve.md b/docs/source/en/using-diffusers/serve.md index 9103815b3d6d..353502182fcb 100644 --- a/docs/source/en/using-diffusers/serve.md +++ b/docs/source/en/using-diffusers/serve.md @@ -96,8 +96,9 @@ curl localhost:8000/v1/generations -d '{ }' ``` -Request adapters are turned off when the server starts with `--compile` or `--lora`. Such requests fail with -`not_implemented`. +Adapters are a capability the manifest declares (see below). They are unavailable when the server starts with +`--compile`, `--lora` or `--context-parallel`: a derived manifest leaves them out, and a published manifest that +declares them is refused at startup. ## Discovery and manifests @@ -125,8 +126,18 @@ When a manifest uses a name that differs from the pipeline argument, map it per "x-diffusers": {"inputs": {"caption": "prompt"}, "parameters": {"cfg": "guidance_scale"}} ``` -A task is only hosted when every input it requires maps to a pipeline argument. A parameter the pipeline cannot -set is rejected with `not_implemented` unless the request sends its default value. +The manifest describes the server, so `serve` checks it at startup and refuses to start when it declares +something the server can't honor: an input or parameter that maps to no pipeline argument, a `duration` without a +frame lattice, an output type other than image, video or audio, or a capability the server doesn't provide. + +The server's capabilities are listed in the manifest under `capabilities`, from `sse` (event streams), `wait` +(`Prefer: wait`) and `adapters` (per-request LoRAs). A capability the manifest leaves out is off: event streams +return `not_implemented` and `Prefer: wait` is ignored. A derived manifest declares every capability the server +can provide. + +```json +"capabilities": ["sse", "wait", "adapters"] +``` ## Multiple GPUs diff --git a/src/diffusers/commands/serve/__init__.py b/src/diffusers/commands/serve/__init__.py index c1c53b672c0d..313dfb537747 100644 --- a/src/diffusers/commands/serve/__init__.py +++ b/src/diffusers/commands/serve/__init__.py @@ -74,11 +74,11 @@ def _resolve_pipeline(args: Namespace) -> tuple[type, inspect.Signature, str | N return type(pipeline), blocks_signature(pipeline.blocks), commit_hash -def _shut_down_when_idle(server: Any, app: Any, generations: Any, idle_timeout: str) -> None: +def _shut_down_when_idle(server: Any, app: Any, scheduler: Any, idle_timeout: str) -> None: limit = _duration_to_seconds(idle_timeout) while True: time.sleep(min(limit, 30.0)) - if generations.busy(): + if scheduler.busy(): app.state.last_request = time.monotonic() continue if time.monotonic() - app.state.last_request >= limit: @@ -228,41 +228,33 @@ def run(self) -> None: from .app import create_app from .backends import DiffusersBackend, backend_capabilities - from .generations import GenerationQueue - from .manifest import ServedModel, load_manifest + from .manifest import load_manifest from .parallel import ParallelBackend + from .scheduler import Scheduler pipeline_cls, signature, commit_hash = _resolve_pipeline(args) + capabilities = backend_capabilities(args, pipeline_cls) manifest = load_manifest( args.model, pipeline_cls, signature, + capabilities, manifest_path=args.manifest, revision=args.revision, + commit_hash=commit_hash, token=args.token, ) - served = ServedModel( - manifest, - set(signature.parameters) - {"self"}, - backend_capabilities(args, pipeline_cls), - revision=commit_hash, - ) - if not served.tasks: - raise SystemExit( - f"None of the tasks {list(manifest.data['tasks'])} in the manifest for {args.model!r} can be " - f"served by {pipeline_cls.__name__}." - ) if args.context_parallel: - backend = ParallelBackend(args, served) + backend = ParallelBackend(args, manifest) else: - backend = DiffusersBackend(args, served) + backend = DiffusersBackend(args, manifest) try: - generations = GenerationQueue(backend, args.output_dir) + scheduler = Scheduler(backend, args.output_dir) app = create_app( - served, - generations, + manifest, + scheduler, default_seed=args.default_seed, enable_cors=args.enable_cors, public_url=args.public_url, @@ -270,10 +262,10 @@ def run(self) -> None: out.result( self.task, url=f"http://{args.host}:{args.port}", - model=served.id, + model=manifest.id, pipeline_class=pipeline_cls.__name__, - tasks=list(served.tasks), - capabilities=served.capabilities, + tasks=list(manifest.tasks), + capabilities=manifest.capabilities, ) server = uvicorn.Server( uvicorn.Config(app, host=args.host, port=args.port, log_level=args.log_level or "info") @@ -281,7 +273,7 @@ def run(self) -> None: if args.idle_timeout is not None: threading.Thread( target=_shut_down_when_idle, - args=(server, app, generations, args.idle_timeout), + args=(server, app, scheduler, args.idle_timeout), name="diffusers-serve-idle", daemon=True, ).start() diff --git a/src/diffusers/commands/serve/app.py b/src/diffusers/commands/serve/app.py index babb5dab972f..302d88e1a562 100644 --- a/src/diffusers/commands/serve/app.py +++ b/src/diffusers/commands/serve/app.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""The Generative Media Spec HTTP surface, as a FastAPI app over one served model and one generation queue.""" +"""The Generative Media Spec HTTP surface, as a FastAPI app over one model's manifest and its scheduler.""" from __future__ import annotations @@ -31,8 +31,8 @@ from starlette.exceptions import HTTPException from ...utils import logging -from .generations import Generation, GenerationQueue -from .manifest import GMSError, ServedModel +from .manifest import GMSError, Manifest +from .scheduler import GenerationJob, Scheduler logger = logging.get_logger("diffusers-cli/serve") @@ -40,6 +40,7 @@ MAX_WAIT_SECONDS = 300.0 REQUEST_ID_HEADER = "x-request-id" _KEEP_ALIVE_SECONDS = 15.0 +_FILE_ROUTE = "/v1/generations/{generation_id}/files/{filename}" def _timestamp(value: Any) -> str | None: @@ -48,43 +49,44 @@ def _timestamp(value: Any) -> str | None: return value.strftime("%Y-%m-%dT%H:%M:%SZ") -def _render(generation: Generation, request: Request, public_url: str | None) -> dict[str, Any]: +def _job_response(job: GenerationJob, base_url: str) -> dict[str, Any]: + """The response body for a generation: outputs are linked under `base_url`, or inlined as base64. + + Raises the generation's error once it failed, which the error handler turns into the error envelope. + """ + if job.error is not None: + raise job.error + outputs = None - if generation.status == "complete": + if job.status == "complete": outputs = {} - for name, artifacts in generation.outputs.items(): + for name, artifacts in job.outputs.items(): outputs[name] = [] for artifact in artifacts: - rendered = {key: value for key, value in artifact.items() if key != "file"} - if generation.request.response_format == "b64_json": - content = (generation.directory / artifact["file"]).read_bytes() - rendered["base64"] = base64.b64encode(content).decode() + entry = {key: value for key, value in artifact.items() if key != "file"} + if job.request.response_format == "b64_json": + entry["base64"] = base64.b64encode((job.directory / artifact["file"]).read_bytes()).decode() else: - route = {"generation_id": generation.id, "filename": artifact["file"]} - if public_url is None: - rendered["url"] = str(request.url_for("get_generation_file", **route)) - else: - path = request.app.url_path_for("get_generation_file", **route) - rendered["url"] = public_url.rstrip("/") + path - outputs[name].append(rendered) + entry["url"] = base_url + _FILE_ROUTE.format(generation_id=job.id, filename=artifact["file"]) + outputs[name].append(entry) return { - "id": generation.id, - "model": generation.model, - "revision": generation.revision, - "status": generation.status, - "created_at": _timestamp(generation.created_at), - "progress": generation.progress if generation.status == "generating" else None, + "id": job.id, + "model": job.model, + "revision": job.revision, + "status": job.status, + "created_at": _timestamp(job.created_at), + "progress": job.progress if job.status == "generating" else None, "outputs": outputs, - "expires_at": _timestamp(generation.expires_at), - "seed": generation.seed if generation.status == "complete" else None, - "usage": generation.usage, + "expires_at": _timestamp(job.expires_at), + "seed": job.seed if job.status == "complete" else None, + "usage": job.usage, } def create_app( - served: ServedModel, - generations: GenerationQueue, + manifest: Manifest, + scheduler: Scheduler, default_seed: int | None = None, enable_cors: bool = False, public_url: str | None = None, @@ -95,16 +97,24 @@ def create_app( app = FastAPI(title="diffusers-cli serve", docs_url=None, redoc_url=None, openapi_url=None, root_path=root_path) app.state.last_request = time.monotonic() + def base_url(request: Request) -> str: + # Output links must work for the client, so they are built on the address it reached the server at. + return (public_url or str(request.base_url)).rstrip("/") + @app.middleware("http") async def request_id(request: Request, call_next) -> Response: request.state.request_id = request.headers.get(REQUEST_ID_HEADER) or str(uuid.uuid4()) - # Health checks come from monitors rather than users, and must not keep an idle server alive. - if not request.url.path.rstrip("/").endswith("/health"): - app.state.last_request = time.monotonic() response = await call_next(request) response.headers[REQUEST_ID_HEADER] = request.state.request_id return response + @app.middleware("http") + async def track_activity(request: Request, call_next) -> Response: + # Health checks come from monitors rather than users, and must not keep an idle server alive. + if not request.url.path.rstrip("/").endswith("/health"): + app.state.last_request = time.monotonic() + return await call_next(request) + if enable_cors: app.add_middleware( CORSMiddleware, @@ -138,20 +148,20 @@ async def internal_error(request: Request, error: Exception) -> JSONResponse: @app.get("/health") async def health() -> JSONResponse: - failure = generations.failure() + failure = scheduler.failure() if failure is not None: return JSONResponse({"status": "unhealthy", "reason": failure}, status_code=503) return JSONResponse({"status": "ok"}) @app.get("/v1/models") async def list_models() -> dict[str, Any]: - return {"data": [served.summary()]} + return {"data": [manifest.summary()]} @app.get("/v1/models/{model_id:path}") async def get_model(model_id: str) -> Response: - if model_id != served.id: - raise GMSError("not_found", f"model {model_id!r} is not served here; this server hosts {served.id!r}") - return Response(served.discovery_document(), media_type="application/json") + if model_id != manifest.id: + raise GMSError("not_found", f"model {model_id!r} is not served here; this server hosts {manifest.id!r}") + return Response(manifest.discovery_document(), media_type="application/json") @app.post("/v1/generations", status_code=201) async def create_generation(request: Request) -> dict[str, Any]: @@ -159,40 +169,37 @@ async def create_generation(request: Request) -> dict[str, Any]: body = await request.json() except json.JSONDecodeError as e: raise GMSError("invalid_request", f"request body is not valid JSON: {e}") from e - resolved = served.resolve(body) - seeded = served.seeded(resolved.task) - generation = Generation(served.id, served.revision, resolved, seeded=seeded, default_seed=default_seed) - generations.submit(generation) + resolved = manifest.resolve(body) + seeded = manifest.seeded(resolved.task) + generation = GenerationJob(manifest.id, manifest.revision, resolved, seeded=seeded, default_seed=default_seed) + scheduler.submit(generation) wait = re.fullmatch(r"\s*wait=(\d+)\s*", request.headers.get("prefer", "")) - if wait is not None: + # `Prefer` is a hint, so a server without the `wait` capability answers right away instead of failing. + if wait is not None and "wait" in manifest.capabilities: await asyncio.to_thread(generation.wait, min(float(wait.group(1)), MAX_WAIT_SECONDS)) - if generation.error is not None: - raise generation.error - return _render(generation, request, public_url) + return _job_response(generation, base_url(request)) @app.get("/v1/generations/{generation_id}") async def get_generation(generation_id: str, request: Request) -> dict[str, Any]: - generation = generations.get(generation_id) - if generation.error is not None: - raise generation.error - return _render(generation, request, public_url) + generation = scheduler.get(generation_id) + return _job_response(generation, base_url(request)) @app.delete("/v1/generations/{generation_id}", status_code=204) async def delete_generation(generation_id: str) -> Response: - generations.delete(generation_id) + scheduler.delete(generation_id) return Response(status_code=204) @app.post("/v1/generations/{generation_id}/cancel") async def cancel_generation(generation_id: str, request: Request) -> dict[str, Any]: - generation = generations.cancel(generation_id) - if generation.error is not None: - raise generation.error - return _render(generation, request, public_url) + generation = scheduler.cancel(generation_id) + return _job_response(generation, base_url(request)) @app.get("/v1/generations/{generation_id}/events") async def get_generation_events(generation_id: str, request: Request) -> StreamingResponse: - generation = generations.get(generation_id) + if "sse" not in manifest.capabilities: + raise GMSError("not_implemented", "this server does not advertise the `sse` capability") + generation = scheduler.get(generation_id) last_event_id = request.headers.get("last-event-id", "") start = int(last_event_id) if last_event_id.isdigit() and len(last_event_id) < 10 else 0 @@ -202,7 +209,7 @@ async def stream(): events, done = await asyncio.to_thread(generation.wait_for_events, index, _KEEP_ALIVE_SECONDS) for event, data in events: index += 1 - payload = _render(generation, request, public_url) if event == "complete" else data + payload = _job_response(generation, base_url(request)) if event == "complete" else data yield f"id: {index}\nevent: {event}\ndata: {json.dumps(payload)}\n\n" if done: return @@ -211,9 +218,9 @@ async def stream(): return StreamingResponse(stream(), media_type="text/event-stream", headers={"Cache-Control": "no-cache"}) - @app.get("/v1/generations/{generation_id}/files/{filename}") + @app.get(_FILE_ROUTE) async def get_generation_file(generation_id: str, filename: str) -> FileResponse: - generation = generations.get(generation_id) + generation = scheduler.get(generation_id) artifacts = [a for group in (generation.outputs or {}).values() for a in group if a["file"] == filename] if not artifacts: raise GMSError("not_found", f"generation {generation_id!r} has no file {filename!r}") diff --git a/src/diffusers/commands/serve/backends.py b/src/diffusers/commands/serve/backends.py index f9f891bb15f0..4c41dc7ba205 100644 --- a/src/diffusers/commands/serve/backends.py +++ b/src/diffusers/commands/serve/backends.py @@ -44,8 +44,8 @@ _load_pipeline, _save_audio_arrays, ) -from .generations import Generation -from .manifest import GMSError, ServedModel, blocks_signature +from .manifest import GMSError, Manifest, blocks_signature +from .scheduler import GenerationJob logger = logging.get_logger("diffusers-cli/serve") @@ -162,7 +162,7 @@ def _save_audio(audios: Any, directory: Path, name: str, media_type: str, rate: _SAVERS = {"image": _save_images, "video": _save_videos, "audio": _save_audio} -def load_request_media(served: ServedModel, request: Any) -> dict[str, Any]: +def load_request_media(manifest: Manifest, request: Any) -> dict[str, Any]: """The pipeline kwargs of a request, with its media inputs fetched and decoded.""" kwargs = dict(request.kwargs) for kwarg, (name, value) in request.media.items(): @@ -170,7 +170,7 @@ def load_request_media(served: ServedModel, request: Any) -> dict[str, Any]: loaded = [_load_media(name, item, f"/inputs/{name}/{index}") for index, item in enumerate(value)] else: loaded = [_load_media(name, value, f"/inputs/{name}")] - if served.tasks[request.task]["inputs"][name]["type"] == "audio": + if manifest.tasks[request.task]["inputs"][name]["type"] == "audio": if kwarg == "initial_audio_waveforms": kwargs.setdefault("initial_audio_sampling_rate", loaded[0][1]) loaded = [waveform for waveform, _ in loaded] @@ -179,9 +179,9 @@ def load_request_media(served: ServedModel, request: Any) -> dict[str, Any]: class DiffusersBackend: - def __init__(self, args: Namespace, served: ServedModel, save_outputs: bool = True): + def __init__(self, args: Namespace, manifest: Manifest, save_outputs: bool = True): self._args = args - self._served = served + self._manifest = manifest # Under context parallelism every rank runs the call, and only rank 0 writes the files. self._save_outputs = save_outputs self.pipeline = _load_pipeline(args) @@ -203,8 +203,8 @@ def close(self) -> None: def failure(self) -> str | None: return None - def generate(self, generation: Generation) -> dict[str, list[dict[str, Any]]]: - return self.run(load_request_media(self._served, generation.request), generation) + def generate(self, generation: GenerationJob) -> dict[str, list[dict[str, Any]]]: + return self.run(load_request_media(self._manifest, generation.request), generation) def run(self, kwargs: dict[str, Any], generation: Any) -> dict[str, list[dict[str, Any]]]: """Call the pipeline with the request's kwargs, media already loaded, and save its outputs.""" @@ -236,7 +236,7 @@ def on_step_end(pipeline, step, timestep, callback_kwargs): return {} outputs = {} - for name, spec in self._served.tasks[request.task]["outputs"].items(): + for name, spec in self._manifest.tasks[request.task]["outputs"].items(): kind = spec["type"] # A modular pipeline returns its `PipelineState`; a standard one returns an output object. if self._modular: @@ -249,7 +249,7 @@ def on_step_end(pipeline, step, timestep, callback_kwargs): rate = self._fps(kwargs) if kind == "video" else (self._args.sampling_rate or 16000) outputs[name] = _SAVERS[kind](value, generation.directory, name, request.output_formats[kind], rate) if not outputs: - declared = list(self._served.tasks[request.task]["outputs"]) + declared = list(self._manifest.tasks[request.task]["outputs"]) raise GMSError( "internal", f"{type(self.pipeline).__name__} returned none of the outputs {declared} declared for task " @@ -282,7 +282,7 @@ def _load_adapters(self, adapters: list[dict[str, Any]]) -> None: scale = adapter.get("scale", 1.0) targets = adapter.get("targets") if targets: - valid_targets = self._served.adapters[adapter["type"]].get("targets", []) + valid_targets = self._manifest.adapters[adapter["type"]].get("targets", []) weights.append({target: targets.get(target, scale) for target in valid_targets}) else: weights.append(scale) diff --git a/src/diffusers/commands/serve/manifest.py b/src/diffusers/commands/serve/manifest.py index 89fbce794ac0..ab8c62835ef2 100644 --- a/src/diffusers/commands/serve/manifest.py +++ b/src/diffusers/commands/serve/manifest.py @@ -12,7 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Generative Media Spec (GMS) manifests: loading, derivation from a pipeline signature, and request validation.""" +"""Generative Media Spec (GMS) manifests. + +A manifest describes what this server offers: its tasks with their inputs and parameters, and its capabilities. It is +loaded from a published `gms.json` or derived from the pipeline signature, checked against the pipeline at startup, +served for discovery, and used to validate each request and map it to the pipeline call. +""" from __future__ import annotations @@ -41,12 +46,17 @@ from ..run import _AUDIO_INPUT_KEYS, _IMAGE_INPUT_KEYS, _VIDEO_INPUT_KEYS +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + SPEC_VERSION = "gms/0.1" MANIFEST_FILENAME = "gms.json" EXTENSION_KEY = "x-diffusers" MAX_ADAPTERS = 4 -ADAPTER_SOURCES = ("hf",) +# What a manifest may declare under `capabilities`; the server checks each one it declares at startup. +SUPPORTED_CAPABILITIES = ("sse", "wait", "adapters") # First entry of each tuple is the default when the request names no `output_format`. OUTPUT_FORMATS = { @@ -62,17 +72,8 @@ "internal": 500, "not_implemented": 501, } - -_REQUEST_FIELDS = ( - "model", - "task", - "preset", - "inputs", - "parameters", - "output_format", - "response_format", - "adapters", -) +_REQUEST_FIELDS = ("model", "task", "preset", "inputs", "parameters", "output_format", "response_format", "adapters") +_ADAPTER_FIELDS = ("type", "path", "scale", "targets") # GMS name -> pipeline kwargs that carry the same meaning, tried in order before the name itself. _PARAMETER_KWARGS = { @@ -95,6 +96,12 @@ ("text_to_audio", AUTO_TEXT2AUDIO_PIPELINES_MAPPING, (), "audio"), ) +_MEDIA_KWARGS = { + **dict.fromkeys(_IMAGE_INPUT_KEYS, "image"), + **dict.fromkeys(_VIDEO_INPUT_KEYS, "video"), + **dict.fromkeys(_AUDIO_INPUT_KEYS, "audio"), + "mask_image": "mask", +} _SKIPPED_KWARGS = frozenset({"output_type", "return_dict", "callback_steps", "callback_on_step_end_tensor_inputs"}) _AUDIO_OUTPUT_KWARGS = frozenset({"audio_length_in_s", "audio_end_in_s", "num_waveforms_per_prompt"}) _SCALAR_TYPES = {"float": "number", "int": "integer", "bool": "boolean", "str": "string"} @@ -110,6 +117,11 @@ _HF_ADAPTER_PATH = re.compile(r"hf:[\w.-]+/[\w.-]+") +# --------------------------------------------------------------------------- +# Types +# --------------------------------------------------------------------------- + + class GMSError(Exception): def __init__(self, code: str, message: str, pointer: str | None = None): super().__init__(message) @@ -125,17 +137,6 @@ def to_dict(self) -> dict[str, Any]: return {"error": error} -@dataclass -class Manifest: - raw: bytes - data: dict[str, Any] - source: str | None = None - - @property - def hash(self) -> str: - return f"sha256:{hashlib.sha256(self.raw).hexdigest()}" - - @dataclass class ResolvedRequest: task: str @@ -148,52 +149,53 @@ class ResolvedRequest: adapters: list[dict[str, Any]] = field(default_factory=list) -def blocks_signature(blocks: Any) -> inspect.Signature: - """The call signature of a modular pipeline, which takes its keyword arguments from its blocks' inputs.""" - parameters = [] - for input_param in blocks.inputs: - if input_param.name is None: - continue - annotation = inspect.Parameter.empty if input_param.type_hint is None else input_param.type_hint - parameters.append( - inspect.Parameter( - input_param.name, inspect.Parameter.KEYWORD_ONLY, default=input_param.default, annotation=annotation - ) - ) - return inspect.Signature(parameters) +# --------------------------------------------------------------------------- +# Loading a manifest +# --------------------------------------------------------------------------- def load_manifest( model: str, pipeline_cls: type, signature: inspect.Signature, + capabilities: list[str], manifest_path: str | None = None, revision: str | None = None, + commit_hash: str | None = None, token: str | None = None, ) -> Manifest: - if manifest_path is not None: - return _parse_manifest(Path(manifest_path).read_bytes(), origin=manifest_path) + """Load the model's manifest and bind it to the pipeline: `--manifest`, else the repo's `gms.json`, else derived. - if Path(model).is_dir(): + `revision` is the one requested for the download; `commit_hash` is the one the pipeline was resolved at, which + generations report. + """ + source = None + raw = None + if manifest_path is not None: + raw, origin = Path(manifest_path).read_bytes(), manifest_path + elif Path(model).is_dir(): published = Path(model) / MANIFEST_FILENAME if published.is_file(): - return _parse_manifest(published.read_bytes(), origin=str(published)) + raw, origin = published.read_bytes(), str(published) else: try: published = hf_hub_download(model, MANIFEST_FILENAME, revision=revision, token=token) except EntryNotFoundError: published = None if published is not None: - manifest = _parse_manifest(Path(published).read_bytes(), origin=f"{model}/{MANIFEST_FILENAME}") - manifest.source = f"hf:{model}/{MANIFEST_FILENAME}" - return manifest + raw, origin = Path(published).read_bytes(), f"{model}/{MANIFEST_FILENAME}" + source = f"hf:{model}/{MANIFEST_FILENAME}" - model_id = Path(model).name if Path(model).is_dir() else model - data = derive_manifest(model_id, pipeline_cls, signature) - return Manifest(raw=json.dumps(data).encode(), data=data) + if raw is not None: + data = _parse_manifest(raw, origin) + else: + model_id = Path(model).name if Path(model).is_dir() else model + data = derive_manifest(model_id, pipeline_cls, signature, capabilities) + raw = json.dumps(data).encode() + return Manifest(raw, data, pipeline_cls, signature, capabilities, source=source, revision=commit_hash) -def _parse_manifest(raw: bytes, origin: str) -> Manifest: +def _parse_manifest(raw: bytes, origin: str) -> dict[str, Any]: try: data = json.loads(raw) except json.JSONDecodeError as e: @@ -202,55 +204,86 @@ def _parse_manifest(raw: bytes, origin: str) -> Manifest: raise SystemExit(f"GMS manifest {origin!r} must be a JSON object with a string `id`.") if not isinstance(data.get("tasks"), dict) or not data["tasks"]: raise SystemExit(f"GMS manifest {origin!r} declares no `tasks`, so there is nothing to serve.") - return Manifest(raw=raw, data=data) + return data -def _annotation_schema(annotation: Any, default: Any) -> dict[str, Any] | None: - """Map a `__call__` annotation to a GMS parameter schema, or `None` when it is not a scalar or list of scalars.""" - if annotation is inspect.Parameter.empty: - text = type(default).__name__ if default is not None else "" - elif isinstance(annotation, str): - text = annotation - elif type(annotation) is type: - text = annotation.__name__ - else: - text = str(annotation) - text = text.replace("typing.", "") +# --------------------------------------------------------------------------- +# Deriving a manifest from the pipeline signature +# --------------------------------------------------------------------------- - def scalar(fragment: str) -> str | None: - names = set(re.findall(r"[A-Za-z_][\w.]*", fragment)) - names -= {"Optional", "Union", "None", "NoneType", "list", "List"} - if not names or not names <= set(_SCALAR_TYPES): - return None - return next(gms for python, gms in _SCALAR_TYPES.items() if python in names) - outside_lists = re.sub(r"\b[Ll]ist\[[^\[\]]*\]", "", text) - if outside_lists == text: - scalar_type = scalar(text) - return {"type": scalar_type} if scalar_type else None +def blocks_signature(blocks: Any) -> inspect.Signature: + """The call signature of a modular pipeline, which takes its keyword arguments from its blocks' inputs.""" + parameters = [] + for input_param in blocks.inputs: + if input_param.name is None: + continue + annotation = inspect.Parameter.empty if input_param.type_hint is None else input_param.type_hint + parameters.append( + inspect.Parameter( + input_param.name, inspect.Parameter.KEYWORD_ONLY, default=input_param.default, annotation=annotation + ) + ) + return inspect.Signature(parameters) - # `str | list[str]` is a scalar that also accepts a batch; only a bare list is an array parameter. - scalar_type = scalar(text) - if scalar_type is None: - return None - if scalar(outside_lists): - return {"type": scalar_type} - return {"type": "array", "items": {"type": scalar_type}} + +def derive_manifest( + model_id: str, pipeline_cls: type, signature: inspect.Signature, capabilities: list[str] +) -> dict[str, Any]: + """Build a server-authored manifest from the pipeline call `signature`, declaring the server's `capabilities`.""" + inputs, parameters, extension = _describe_kwargs(signature) + registered = [entry for entry in _AUTO_TASKS if pipeline_cls in entry[1].values()] + output = _output_modality(pipeline_cls, signature, registered) + + def task(required: tuple[str, ...], excluded: set[str]) -> dict[str, Any]: + task_inputs = { + name: {**spec, "required": True} if name in required else spec + for name, spec in inputs.items() + if name not in excluded + } + spec = {"inputs": task_inputs, "outputs": {output: {"type": output}}, "parameters": parameters} + if extension: + spec[EXTENSION_KEY] = extension + return spec + + primary = next((name for name in ("image", "video") if name in inputs), None) + tasks = {} + if registered: + # A class registered for several tasks takes each task's conditioning input optionally; an + # input another task requires is what tells the tasks apart, so it is left out of the rest. + for name, _, required, _ in registered: + if not set(required) <= set(inputs): + continue + others = {key for other in registered if other[0] != name for key in other[2]} + tasks[name] = task(required, others - set(required)) + elif primary is None: + tasks[f"text_to_{output}"] = task((), set()) + elif inputs[primary].get("required"): + tasks[f"{primary}_to_{output}"] = task((), set()) + else: + tasks[f"text_to_{output}"] = task((), {primary}) + tasks[f"{primary}_to_{output}"] = task((primary,), set()) + + modalities_in = sorted({"image" if s["type"] == "mask" else s["type"] for s in inputs.values()} - {"text"}) + manifest: dict[str, Any] = { + "id": model_id, + "modalities": {"in": modalities_in or ["text"], "out": [output]}, + "tasks": tasks, + "capabilities": list(capabilities), + } + if "adapters" in capabilities: + manifest["adapters"] = {"lora": {"targets": list(getattr(pipeline_cls, "_lora_loadable_modules", []))}} + return manifest -def derive_manifest(model_id: str, pipeline_cls: type, signature: inspect.Signature) -> dict[str, Any]: - """Build a server-authored manifest from the pipeline call `signature`.""" +def _describe_kwargs(signature: inspect.Signature) -> tuple[dict, dict, dict]: + """Sort the call kwargs into GMS inputs and parameters, and record the ones whose GMS name differs.""" inputs: dict[str, dict[str, Any]] = {} parameters: dict[str, dict[str, Any]] = {} input_kwargs: dict[str, str] = {} parameter_kwargs: dict[str, str] = {} - media_types = { - **dict.fromkeys(_IMAGE_INPUT_KEYS, "image"), - **dict.fromkeys(_VIDEO_INPUT_KEYS, "video"), - **dict.fromkeys(_AUDIO_INPUT_KEYS, "audio"), - "mask_image": "mask", - } kwarg_parameters = {kwarg: name for name, kwargs in _PARAMETER_KWARGS.items() for kwarg in kwargs} + # `num_frames` keeps its name: `duration` is in seconds, which only a written manifest's frame lattice converts. del kwarg_parameters["num_frames"] for kwarg, parameter in signature.parameters.items(): @@ -261,9 +294,9 @@ def derive_manifest(model_id: str, pipeline_cls: type, signature: inspect.Signat has_default = parameter.default is not inspect.Parameter.empty default = parameter.default if has_default else None - if kwarg in media_types: + if kwarg in _MEDIA_KWARGS: name = "mask" if kwarg == "mask_image" else kwarg - spec: dict[str, Any] = {"type": media_types[kwarg]} + spec: dict[str, Any] = {"type": _MEDIA_KWARGS[kwarg]} if not has_default: spec["required"] = True if re.search(r"\b[Ll]ist\b", str(parameter.annotation)): @@ -297,236 +330,213 @@ def derive_manifest(model_id: str, pipeline_cls: type, signature: inspect.Signat if name != kwarg: parameter_kwargs[name] = kwarg - registered = [entry for entry in _AUTO_TASKS if pipeline_cls in entry[1].values()] - names = set(signature.parameters) - if registered: - output = registered[0][3] - elif "num_frames" in names or "Video" in pipeline_cls.__name__: - output = "video" - elif names & _AUDIO_OUTPUT_KWARGS: - output = "audio" - else: - output = "image" - extension: dict[str, Any] = {} if input_kwargs: extension["inputs"] = input_kwargs if parameter_kwargs: extension["parameters"] = parameter_kwargs + return inputs, parameters, extension - def task(required: tuple[str, ...], excluded: set[str]) -> dict[str, Any]: - task_inputs = { - name: {**spec, "required": True} if name in required else spec - for name, spec in inputs.items() - if name not in excluded - } - spec = {"inputs": task_inputs, "outputs": {output: {"type": output}}, "parameters": parameters} - if extension: - spec[EXTENSION_KEY] = extension - return spec - primary = next((name for name in ("image", "video") if name in inputs), None) - tasks = {} +def _output_modality(pipeline_cls: type, signature: inspect.Signature, registered: list) -> str: if registered: - # A class registered for several tasks takes each task's conditioning input optionally; an - # input another task requires is what tells the tasks apart, so it is left out of the rest. - for name, _, required, _ in registered: - if not set(required) <= set(inputs): - continue - others = {key for other in registered if other[0] != name for key in other[2]} - tasks[name] = task(required, others - set(required)) - elif primary is None: - tasks[f"text_to_{output}"] = task((), set()) - elif inputs[primary].get("required"): - tasks[f"{primary}_to_{output}"] = task((), set()) - else: - tasks[f"text_to_{output}"] = task((), {primary}) - tasks[f"{primary}_to_{output}"] = task((primary,), set()) + return registered[0][3] + names = set(signature.parameters) + if "num_frames" in names or "Video" in pipeline_cls.__name__: + return "video" + if names & _AUDIO_OUTPUT_KWARGS: + return "audio" + return "image" - modalities_in = sorted({"image" if s["type"] == "mask" else s["type"] for s in inputs.values()} - {"text"}) - manifest: dict[str, Any] = { - "id": model_id, - "modalities": {"in": modalities_in or ["text"], "out": [output]}, - "tasks": tasks, - } - if hasattr(pipeline_cls, "load_lora_weights"): - manifest["adapters"] = {"lora": {"targets": list(getattr(pipeline_cls, "_lora_loadable_modules", []))}} - return manifest +def _annotation_schema(annotation: Any, default: Any) -> dict[str, Any] | None: + """Map a `__call__` annotation to a GMS parameter schema, or `None` when it is not a scalar or list of scalars.""" + if annotation is inspect.Parameter.empty: + text = type(default).__name__ if default is not None else "" + elif isinstance(annotation, str): + text = annotation + elif type(annotation) is type: + text = annotation.__name__ + else: + text = str(annotation) + text = text.replace("typing.", "") -def _bind(name: str, aliases: dict[str, tuple[str, ...]], declared: dict[str, str], kwarg_names: set[str]): - candidates = (declared[name],) if name in declared else (*aliases.get(name, ()), name) - return next((kwarg for kwarg in candidates if kwarg in kwarg_names), None) + def scalar(fragment: str) -> str | None: + names = set(re.findall(r"[A-Za-z_][\w.]*", fragment)) + names -= {"Optional", "Union", "None", "NoneType", "list", "List"} + if not names or not names <= set(_SCALAR_TYPES): + return None + return next(gms for python, gms in _SCALAR_TYPES.items() if python in names) + outside_lists = re.sub(r"\b[Ll]ist\[[^\[\]]*\]", "", text) + if outside_lists == text: + scalar_type = scalar(text) + return {"type": scalar_type} if scalar_type else None -def _frame_lattice(spec: dict[str, Any]) -> tuple[int, int, float] | None: - lattice = spec.get("lattice") or {} - match = _FRAME_LATTICE.fullmatch(str(lattice.get("frames", "")).replace(" ", "")) - if match is None or not isinstance(lattice.get("fps"), (int, float)): + # `str | list[str]` is a scalar that also accepts a batch; only a bare list is an array parameter. + scalar_type = scalar(text) + if scalar_type is None: return None - return int(match.group(1)), int(match.group(2)), lattice["fps"] - - -def _is_active(condition: dict[str, Any], values: dict[str, Any]) -> bool: - if "all" in condition: - return all(_is_active(item, values) for item in condition["all"]) - actual = values.get(condition["parameter"]) - for name, compare in _COMPARISONS.items(): - if name not in condition: - continue - if actual is None or isinstance(actual, str) != isinstance(condition[name], str): - return False - if not compare(actual, condition[name]): - return False - return True + if scalar(outside_lists): + return {"type": scalar_type} + return {"type": "array", "items": {"type": scalar_type}} -def _check_value(label: str, value: Any, spec: dict[str, Any], pointer: str) -> None: - kind = spec.get("type") - if kind == "array": - if not isinstance(value, list): - raise GMSError("invalid_request", f"{label} must be an array, got {value!r}", pointer) - if "maxItems" in spec and len(value) > spec["maxItems"]: - raise GMSError("invalid_request", f"{label} accepts at most {spec['maxItems']} items", pointer) - for index, item in enumerate(value): - _check_value(f"{label}[{index}]", item, spec["items"], f"{pointer}/{index}") - return - if kind == "object": - if not isinstance(value, dict): - raise GMSError("invalid_request", f"{label} must be an object, got {value!r}", pointer) - for key, item in value.items(): - if key not in spec["properties"]: - raise GMSError("invalid_request", f"{label} has no property {key!r}", f"{pointer}/{key}") - _check_value(f"{label}.{key}", item, spec["properties"][key], f"{pointer}/{key}") - return - - expected = _PYTHON_TYPES.get(kind) - is_bool = isinstance(value, bool) - if expected is not None and (not isinstance(value, expected) or (is_bool and kind != "boolean")): - raise GMSError("invalid_request", f"{label} must be of type {kind}, got {value!r}", pointer) - if isinstance(value, float) and not math.isfinite(value): - raise GMSError("invalid_request", f"{label} must be a finite number, got {value}", pointer) - if "enum" in spec and value not in spec["enum"]: - raise GMSError("invalid_request", f"{label} must be one of {spec['enum']}, got {value!r}", pointer) - if "min" in spec and value < spec["min"]: - raise GMSError("invalid_request", f"{label} must be at least {spec['min']}, got {value}", pointer) - if "max" in spec and value > spec["max"]: - raise GMSError("invalid_request", f"{label} must be at most {spec['max']}, got {value}", pointer) - if "multiple_of" in spec: - quotient = value / spec["multiple_of"] - if not math.isclose(quotient, round(quotient), abs_tol=1e-9): - raise GMSError( - "invalid_request", f"{label} must be a multiple of {spec['multiple_of']}, got {value}", pointer - ) +# --------------------------------------------------------------------------- +# The manifest bound to the pipeline: startup checks, discovery and request resolution +# --------------------------------------------------------------------------- -def _check_media(name: str, value: Any, spec: dict[str, Any], pointer: str) -> None: - if not isinstance(value, dict): - raise GMSError("invalid_request", f"{name} must be a media value object, got {value!r}", pointer) - if value.get("type") != spec["type"]: - raise GMSError( - "invalid_request", f"{name} must have type {spec['type']!r}, got {value.get('type')!r}", f"{pointer}/type" - ) - keys = set(value) - if keys == {"type", "base64", "media_type"}: - if not isinstance(value["base64"], str) or not isinstance(value["media_type"], str): - raise GMSError("invalid_request", f"{name} base64 and media_type must be strings", pointer) - return - if "url" not in keys or not keys <= {"type", "url", "media_type"}: - raise GMSError( - "invalid_request", - f"{name} must be either {{type, url, media_type?}} or {{type, base64, media_type}}, got keys {sorted(keys)}", - pointer, - ) - # A URL media value is fetched by the server, so anything but http(s) would read the server's own filesystem. - if not isinstance(value["url"], str) or not value["url"].startswith(("http://", "https://")): - raise GMSError("invalid_request", f"{name} url must be an http(s) URL", f"{pointer}/url") - +class Manifest: + """A model's manifest, bound to the pipeline that serves it: discovery and request resolution. -class ServedModel: - """One manifest bound to the kwargs the pipeline accepts: the discovery document and request resolution.""" + The manifest describes this server, so everything it declares must be servable: each input and parameter maps + to a kwarg of the pipeline call, and each capability is one this server provides. Anything else is an error at + startup rather than a request that fails later. + """ def __init__( self, - manifest: Manifest, - kwarg_names: set[str], - capabilities: list[str], + raw: bytes, + data: dict[str, Any], + pipeline_cls: type, + signature: inspect.Signature, + supported_capabilities: list[str], + source: str | None = None, revision: str | None = None, ): - self.manifest = manifest - self.id = manifest.data["id"] - self.capabilities = capabilities + # The bytes are kept as published: discovery serves them unchanged, and `hash` is computed over them. + self.raw = raw + self.data = data + self.source = source + self.id = data["id"] self.revision = revision - self.adapters = manifest.data.get("adapters", {}) - self._kwarg_names = kwarg_names - - self.tasks: dict[str, dict[str, Any]] = {} - self._inputs: dict[str, dict[str, str | None]] = {} - self._parameters: dict[str, dict[str, str | None]] = {} - self._presets: dict[str, list[str]] = {} - for name, task in manifest.data["tasks"].items(): + self.tasks: dict[str, dict[str, Any]] = data["tasks"] + self.capabilities: list[str] = data.get("capabilities", []) + self.adapters = data.get("adapters", {}) + self._kwarg_names = set(signature.parameters) - {"self"} + self._pipeline_name = pipeline_cls.__name__ + + for capability in self.capabilities: + if capability not in supported_capabilities: + raise SystemExit(self._unsupported_capability(capability)) + if "adapters" in self.capabilities and not self.adapters: + raise SystemExit( + f"The GMS manifest for {self.id!r} declares the `adapters` capability but no top-level `adapters` " + 'section naming the adapter types, e.g. {"lora": {"targets": ["transformer"]}}.' + ) + + # Per task: GMS input or parameter name -> the pipeline kwarg it sets. + self._input_kwargs: dict[str, dict[str, str]] = {} + self._parameter_kwargs: dict[str, dict[str, str]] = {} + for name, task in self.tasks.items(): declared = task.get(EXTENSION_KEY, {}) - inputs = { - key: _bind(key, _INPUT_KWARGS, declared.get("inputs", {}), kwarg_names) for key in task["inputs"] + self._input_kwargs[name] = { + key: self._require_kwarg(name, "input", key, _INPUT_KWARGS, declared.get("inputs", {})) + for key in task["inputs"] } - parameters = { - key: _bind(key, _PARAMETER_KWARGS, declared.get("parameters", {}), kwarg_names) + self._parameter_kwargs[name] = { + key: self._require_kwarg(name, "parameter", key, _PARAMETER_KWARGS, declared.get("parameters", {})) for key in task.get("parameters", {}) } - if "duration" in parameters and _frame_lattice(task["parameters"]["duration"]) is None: - parameters["duration"] = None - if any(inputs[key] is None for key, spec in task["inputs"].items() if spec.get("required")): - continue - if not any(spec["type"] in OUTPUT_FORMATS for spec in task["outputs"].values()): - continue - self.tasks[name] = task - self._inputs[name] = inputs - self._parameters[name] = parameters - self._presets[name] = [ - preset - for preset, body in task.get("presets", {}).items() - if all(parameters.get(key) is not None for key in body.get("parameters", {})) - ] + self._check_task(name, task) + + def _pipeline_kwarg(self, name: str, aliases: dict[str, tuple[str, ...]], declared: dict[str, str]) -> str | None: + candidates = (declared[name],) if name in declared else (*aliases.get(name, ()), name) + return next((kwarg for kwarg in candidates if kwarg in self._kwarg_names), None) + + def _require_kwarg( + self, task: str, kind: str, name: str, aliases: dict[str, tuple[str, ...]], declared: dict[str, str] + ) -> str: + kwarg = self._pipeline_kwarg(name, aliases, declared) + if kwarg is None: + raise SystemExit( + f"The GMS manifest for {self.id!r} declares {kind} {name!r} on task {task!r}, which " + f"{self._pipeline_name} does not take. Remove it from the manifest, or map it to a pipeline kwarg " + f'under `{EXTENSION_KEY}`, e.g. "{EXTENSION_KEY}": {{"{kind}s": {{"{name}": ""}}}}.' + ) + return kwarg + + def _check_task(self, name: str, task: dict[str, Any]) -> None: + unsupported = [spec["type"] for spec in task["outputs"].values() if spec["type"] not in OUTPUT_FORMATS] + if unsupported: + raise SystemExit( + f"The GMS manifest for {self.id!r} declares {unsupported} outputs on task {name!r}; this server " + f"produces {list(OUTPUT_FORMATS)} outputs." + ) + parameters = task.get("parameters", {}) + if "duration" in parameters and _frame_lattice(parameters["duration"]) is None: + raise SystemExit( + f"The GMS manifest for {self.id!r} declares `duration` on task {name!r} without a frame lattice, so " + 'seconds cannot be converted to frames. Add one, e.g. "lattice": {"frames": "8k+1", "fps": 24}.' + ) + for preset, body in task.get("presets", {}).items(): + for key in body.get("parameters", {}): + if key not in parameters: + raise SystemExit( + f"The GMS manifest for {self.id!r} sets parameter {key!r} in preset {preset!r} of task " + f"{name!r}, which the task does not declare." + ) + + def _unsupported_capability(self, capability: str) -> str: + message = f"The GMS manifest for {self.id!r} declares the {capability!r} capability, " + if capability == "adapters": + return message + ( + f"but request adapters are off: {self._pipeline_name} cannot load LoRAs, or the server was started " + "with --compile, --lora or --context-parallel." + ) + return message + f"which this server does not provide. It provides: {list(SUPPORTED_CAPABILITIES)}." + + @property + def hash(self) -> str: + return f"sha256:{hashlib.sha256(self.raw).hexdigest()}" def summary(self) -> dict[str, Any]: return { "id": self.id, - "modalities": self.manifest.data.get("modalities", {}), + "modalities": self.data.get("modalities", {}), "tasks": list(self.tasks), } - def overlay(self) -> dict[str, Any]: - tasks = {} - limits = {} - for name, task in self.tasks.items(): - tasks[name] = {"presets": self._presets[name]} if "presets" in task else {} - disabled = {key: False for key, kwarg in self._inputs[name].items() if kwarg is None} - if disabled: - limits[name] = {"inputs": disabled} - overlay: dict[str, Any] = {"tasks": tasks, "capabilities": self.capabilities} - if limits: - overlay["limits"] = limits - if "adapters" in self.capabilities: - overlay["adapters"] = {"max": MAX_ADAPTERS, "sources": list(ADAPTER_SOURCES)} - return overlay - def discovery_document(self) -> bytes: # The manifest bytes are spliced in untouched: the spec defines verification as a hash of the served bytes. head: dict[str, Any] = {"spec": SPEC_VERSION} - if self.manifest.source is not None: - head["source"] = self.manifest.source - head["hash"] = self.manifest.hash - prefix = json.dumps(head)[:-1].encode() - return ( - prefix - + b', "manifest": ' - + self.manifest.raw - + b', "server": ' - + json.dumps(self.overlay()).encode() - + b"}" - ) + if self.source is not None: + head["source"] = self.source + head["hash"] = self.hash + return json.dumps(head)[:-1].encode() + b', "manifest": ' + self.raw + b"}" + + def seeded(self, task: str) -> bool: + return "seed" in self._parameter_kwargs[task] def resolve(self, body: Any) -> ResolvedRequest: + """Validate a create-generation request body and map it to the pipeline call.""" + inputs = self._check_body(body) + task = self._select_task(body.get("task"), inputs) + values, explicit = self._parameter_values(task, body) + input_kwargs, media = self._resolve_inputs(task, inputs, values) + parameter_kwargs, seed = self._resolve_parameters(task, values, explicit) + output_formats = self._resolve_output_formats(task, body.get("output_format")) + + response_format = body.get("response_format", "url") + if response_format not in ("url", "b64_json"): + raise GMSError( + "invalid_request", + f"response_format must be 'url' or 'b64_json', got {response_format!r}", + "/response_format", + ) + + return ResolvedRequest( + task=task, + kwargs={**input_kwargs, **parameter_kwargs}, + media=media, + seed=seed, + output_formats=output_formats, + response_format=response_format, + adapters=self._resolve_adapters(body.get("adapters")), + ) + + def _check_body(self, body: Any) -> dict[str, Any]: if not isinstance(body, dict): raise GMSError("invalid_request", "request body must be a JSON object") for key in body: @@ -536,24 +546,77 @@ def resolve(self, body: Any) -> ResolvedRequest: raise GMSError("invalid_request", "model is required and must be a string", "/model") if body["model"] != self.id: raise GMSError("not_found", f"model {body['model']!r} is not served here; this server hosts {self.id!r}") - inputs = body.get("inputs") - if not isinstance(inputs, dict): + if not isinstance(body.get("inputs"), dict): raise GMSError("invalid_request", "inputs is required and must be an object", "/inputs") + return body["inputs"] - name = self._select_task(body.get("task"), inputs) - task = self.tasks[name] - values, explicit = self._resolve_parameters(name, task, body) + def _select_task(self, requested: Any, inputs: dict[str, Any]) -> str: + if requested is not None: + if not isinstance(requested, str) or requested not in self.tasks: + raise GMSError( + "invalid_request", + f"task {requested!r} is not hosted for {self.id!r}; hosted tasks: {list(self.tasks)}", + "/task", + ) + return requested + supplied = set(inputs) + candidates = [] + for name, task in self.tasks.items(): + required = {key for key, spec in task["inputs"].items() if spec.get("required")} + if required <= supplied <= set(task["inputs"]): + candidates.append(name) + if len(candidates) == 1: + return candidates[0] + if len(self.tasks) == 1: + return next(iter(self.tasks)) + raise GMSError( + "invalid_request", + f"could not infer the task from inputs {sorted(supplied)}; set `task` to one of {list(self.tasks)}", + "/task", + ) + + def _parameter_values(self, task: str, body: dict[str, Any]) -> tuple[dict[str, Any], set[str]]: + """The parameter values a request runs with: declared defaults, then the preset, then the request's own. + + Also returns the names the request set itself, which are held to stricter rules than defaults. + """ + declared = self.tasks[task].get("parameters", {}) + explicit = body.get("parameters", {}) + if not isinstance(explicit, dict): + raise GMSError("invalid_request", "parameters must be an object", "/parameters") + for key, value in explicit.items(): + if key not in declared: + raise GMSError("invalid_request", f"task {task!r} declares no parameter {key!r}", f"/parameters/{key}") + _check_value(key, value, declared[key], f"/parameters/{key}") + + values = {key: spec["default"] for key, spec in declared.items() if "default" in spec} + preset = body.get("preset") + if preset is not None: + presets = self.tasks[task].get("presets", {}) + if preset not in presets: + raise GMSError( + "invalid_request", + f"preset {preset!r} is not declared for task {task!r}; declared presets: {list(presets)}", + "/preset", + ) + values.update(self.tasks[task]["presets"][preset].get("parameters", {})) + values.update(explicit) + return values, set(explicit) + + def _resolve_inputs( + self, task: str, inputs: dict[str, Any], values: dict[str, Any] + ) -> tuple[dict[str, Any], dict[str, tuple[str, Any]]]: + """Map the request inputs to pipeline kwargs. Text is passed as is; media is checked and left to load.""" + declared = self.tasks[task]["inputs"] kwargs: dict[str, Any] = {} media: dict[str, tuple[str, Any]] = {} for key, value in inputs.items(): pointer = f"/inputs/{key}" - spec = task["inputs"].get(key) + spec = declared.get(key) if spec is None: - raise GMSError("invalid_request", f"task {name!r} declares no input {key!r}", pointer) - kwarg = self._inputs[name][key] - if kwarg is None: - raise GMSError("invalid_request", f"input {key!r} is disabled on this server", pointer) + raise GMSError("invalid_request", f"task {task!r} declares no input {key!r}", pointer) + kwarg = self._input_kwargs[task][key] if "active_when" in spec and not _is_active(spec["active_when"], values): raise GMSError( "invalid_request", f"input {key!r} is inactive unless {json.dumps(spec['active_when'])}", pointer @@ -576,14 +639,21 @@ def resolve(self, body: Any) -> ResolvedRequest: else: _check_media(key, value, spec, pointer) media[kwarg] = (key, value) - for key, spec in task["inputs"].items(): + + for key, spec in declared.items(): if spec.get("required") and key not in inputs: - raise GMSError("invalid_request", f"task {name!r} requires input {key!r}", f"/inputs/{key}") + raise GMSError("invalid_request", f"task {task!r} requires input {key!r}", f"/inputs/{key}") + return kwargs, media + def _resolve_parameters( + self, task: str, values: dict[str, Any], explicit: set[str] + ) -> tuple[dict[str, Any], int | None]: + """Map the parameter values to pipeline kwargs, and pull out the seed, which seeds a generator instead.""" + kwargs: dict[str, Any] = {} seed = None for key, value in values.items(): - spec = task["parameters"][key] - kwarg = self._parameters[name][key] + spec = self.tasks[task]["parameters"][key] + kwarg = self._parameter_kwargs[task][key] pointer = f"/parameters/{key}" if "active_when" in spec and not _is_active(spec["active_when"], values): if key in explicit: @@ -593,14 +663,6 @@ def resolve(self, body: Any) -> ResolvedRequest: pointer, ) continue - if kwarg is None: - # Clients that render every control resend defaults, so a default on a parameter this - # backend cannot set is the one supplied value that changes nothing by being skipped. - if key in explicit and value != spec.get("default"): - raise GMSError( - "not_implemented", f"parameter {key!r} is not supported by this server's backend", pointer - ) - continue if key == "seed": if not 0 <= value < 2**63: raise GMSError("invalid_request", f"seed must be between 0 and 2**63 - 1, got {value}", pointer) @@ -615,110 +677,29 @@ def resolve(self, body: Any) -> ResolvedRequest: f"when duration is set, got {values['fps']}", "/parameters/fps", ) - kwargs[kwarg] = self._duration_to_frames(value, spec, pointer) - fps_kwarg = _bind("fps", _PARAMETER_KWARGS, {}, self._kwarg_names) + kwargs[kwarg] = _duration_to_frames(value, spec, pointer) + fps_kwarg = self._pipeline_kwarg("fps", _PARAMETER_KWARGS, {}) if fps_kwarg is not None: kwargs[fps_kwarg] = lattice_fps continue kwargs[kwarg] = value + return kwargs, seed + def _resolve_output_formats(self, task: str, output_format: Any) -> dict[str, str]: output_formats = {kind: formats[0] for kind, formats in OUTPUT_FORMATS.items()} - output_format = body.get("output_format") - if output_format is not None: - kind = output_format.split("/")[0] if isinstance(output_format, str) else None - produced = {spec["type"] for spec in task["outputs"].values()} - if kind not in produced or output_format not in OUTPUT_FORMATS.get(kind, ()): - supported = sorted(f for k in produced for f in OUTPUT_FORMATS.get(k, ())) - raise GMSError( - "invalid_request", - f"output_format must be one of {supported} for task {name!r}, got {output_format!r}", - "/output_format", - ) - output_formats[kind] = output_format - - response_format = body.get("response_format", "url") - if response_format not in ("url", "b64_json"): + if output_format is None: + return output_formats + kind = output_format.split("/")[0] if isinstance(output_format, str) else None + produced = {spec["type"] for spec in self.tasks[task]["outputs"].values()} + if kind not in produced or output_format not in OUTPUT_FORMATS.get(kind, ()): + supported = sorted(f for k in produced for f in OUTPUT_FORMATS.get(k, ())) raise GMSError( "invalid_request", - f"response_format must be 'url' or 'b64_json', got {response_format!r}", - "/response_format", + f"output_format must be one of {supported} for task {task!r}, got {output_format!r}", + "/output_format", ) - - return ResolvedRequest( - task=name, - kwargs=kwargs, - media=media, - seed=seed, - output_formats=output_formats, - response_format=response_format, - adapters=self._resolve_adapters(body.get("adapters")), - ) - - def seeded(self, task: str) -> bool: - return self._parameters[task].get("seed") is not None - - def _select_task(self, requested: Any, inputs: dict[str, Any]) -> str: - if requested is not None: - if not isinstance(requested, str) or requested not in self.tasks: - raise GMSError( - "invalid_request", - f"task {requested!r} is not hosted for {self.id!r}; hosted tasks: {list(self.tasks)}", - "/task", - ) - return requested - - supplied = set(inputs) - candidates = [] - for name, task in self.tasks.items(): - accepted = {key for key, kwarg in self._inputs[name].items() if kwarg is not None} - required = {key for key, spec in task["inputs"].items() if spec.get("required")} - if required <= supplied <= accepted: - candidates.append(name) - if len(candidates) == 1: - return candidates[0] - if len(self.tasks) == 1: - return next(iter(self.tasks)) - raise GMSError( - "invalid_request", - f"could not infer the task from inputs {sorted(supplied)}; set `task` to one of {list(self.tasks)}", - "/task", - ) - - def _resolve_parameters(self, name: str, task: dict[str, Any], body: dict[str, Any]): - declared = task.get("parameters", {}) - explicit = body.get("parameters", {}) - if not isinstance(explicit, dict): - raise GMSError("invalid_request", "parameters must be an object", "/parameters") - for key, value in explicit.items(): - if key not in declared: - raise GMSError("invalid_request", f"task {name!r} declares no parameter {key!r}", f"/parameters/{key}") - _check_value(key, value, declared[key], f"/parameters/{key}") - - values = {key: spec["default"] for key, spec in declared.items() if "default" in spec} - preset = body.get("preset") - if preset is not None: - if preset not in self._presets[name]: - raise GMSError( - "invalid_request", - f"preset {preset!r} is not supported for task {name!r}; supported presets: {self._presets[name]}", - "/preset", - ) - values.update(task["presets"][preset].get("parameters", {})) - values.update(explicit) - return values, set(explicit) - - def _duration_to_frames(self, seconds: float, spec: dict[str, Any], pointer: str) -> int: - step, offset, fps = _frame_lattice(spec) - position = (seconds * fps - offset) / step - snap = spec["lattice"].get("snap", "nearest") - if snap == "reject" and not math.isclose(position, round(position), abs_tol=1e-9): - raise GMSError( - "invalid_request", - f"duration {seconds} is off the {spec['lattice']['frames']} frame lattice at {fps} fps", - pointer, - ) - index = math.ceil(position - 1e-9) if snap == "up" else round(position) - return step * max(index, 0) + offset + output_formats[kind] = output_format + return output_formats def _resolve_adapters(self, adapters: Any) -> list[dict[str, Any]]: if adapters is None: @@ -736,7 +717,7 @@ def _resolve_adapters(self, adapters: Any) -> list[dict[str, Any]]: if not isinstance(adapter, dict): raise GMSError("invalid_request", "an adapter must be an object", pointer) for key in adapter: - if key not in ("type", "path", "scale", "targets"): + if key not in _ADAPTER_FIELDS: raise GMSError("invalid_request", f"unknown adapter field {key!r}", f"{pointer}/{key}") if not isinstance(adapter.get("type"), str) or adapter["type"] not in self.adapters: raise GMSError( @@ -773,3 +754,109 @@ def _resolve_adapters(self, adapters: Any) -> list[dict[str, Any]]: f"{pointer}/targets/{target}", ) return adapters + + +# --------------------------------------------------------------------------- +# Value checks +# --------------------------------------------------------------------------- + + +def _check_value(label: str, value: Any, spec: dict[str, Any], pointer: str) -> None: + """Check a parameter value against its declared schema: type, enum, range and `multiple_of`.""" + kind = spec.get("type") + if kind == "array": + if not isinstance(value, list): + raise GMSError("invalid_request", f"{label} must be an array, got {value!r}", pointer) + if "maxItems" in spec and len(value) > spec["maxItems"]: + raise GMSError("invalid_request", f"{label} accepts at most {spec['maxItems']} items", pointer) + for index, item in enumerate(value): + _check_value(f"{label}[{index}]", item, spec["items"], f"{pointer}/{index}") + return + if kind == "object": + if not isinstance(value, dict): + raise GMSError("invalid_request", f"{label} must be an object, got {value!r}", pointer) + for key, item in value.items(): + if key not in spec["properties"]: + raise GMSError("invalid_request", f"{label} has no property {key!r}", f"{pointer}/{key}") + _check_value(f"{label}.{key}", item, spec["properties"][key], f"{pointer}/{key}") + return + + expected = _PYTHON_TYPES.get(kind) + is_bool = isinstance(value, bool) + if expected is not None and (not isinstance(value, expected) or (is_bool and kind != "boolean")): + raise GMSError("invalid_request", f"{label} must be of type {kind}, got {value!r}", pointer) + if isinstance(value, float) and not math.isfinite(value): + raise GMSError("invalid_request", f"{label} must be a finite number, got {value}", pointer) + if "enum" in spec and value not in spec["enum"]: + raise GMSError("invalid_request", f"{label} must be one of {spec['enum']}, got {value!r}", pointer) + if "min" in spec and value < spec["min"]: + raise GMSError("invalid_request", f"{label} must be at least {spec['min']}, got {value}", pointer) + if "max" in spec and value > spec["max"]: + raise GMSError("invalid_request", f"{label} must be at most {spec['max']}, got {value}", pointer) + if "multiple_of" in spec: + quotient = value / spec["multiple_of"] + if not math.isclose(quotient, round(quotient), abs_tol=1e-9): + raise GMSError( + "invalid_request", f"{label} must be a multiple of {spec['multiple_of']}, got {value}", pointer + ) + + +def _check_media(name: str, value: Any, spec: dict[str, Any], pointer: str) -> None: + if not isinstance(value, dict): + raise GMSError("invalid_request", f"{name} must be a media value object, got {value!r}", pointer) + if value.get("type") != spec["type"]: + raise GMSError( + "invalid_request", f"{name} must have type {spec['type']!r}, got {value.get('type')!r}", f"{pointer}/type" + ) + keys = set(value) + if keys == {"type", "base64", "media_type"}: + if not isinstance(value["base64"], str) or not isinstance(value["media_type"], str): + raise GMSError("invalid_request", f"{name} base64 and media_type must be strings", pointer) + return + if "url" not in keys or not keys <= {"type", "url", "media_type"}: + raise GMSError( + "invalid_request", + f"{name} must be either {{type, url, media_type?}} or {{type, base64, media_type}}, got keys {sorted(keys)}", + pointer, + ) + # A URL media value is fetched by the server, so anything but http(s) would read the server's own filesystem. + if not isinstance(value["url"], str) or not value["url"].startswith(("http://", "https://")): + raise GMSError("invalid_request", f"{name} url must be an http(s) URL", f"{pointer}/url") + + +def _is_active(condition: dict[str, Any], values: dict[str, Any]) -> bool: + """Evaluate an `active_when` condition against the parameter values.""" + if "all" in condition: + return all(_is_active(item, values) for item in condition["all"]) + actual = values.get(condition["parameter"]) + for name, compare in _COMPARISONS.items(): + if name not in condition: + continue + if actual is None or isinstance(actual, str) != isinstance(condition[name], str): + return False + if not compare(actual, condition[name]): + return False + return True + + +def _frame_lattice(spec: dict[str, Any]) -> tuple[int, int, float] | None: + """Parse a `duration` parameter's frame lattice, e.g. `8k+1` at 24 fps, into `(8, 1, 24)`.""" + lattice = spec.get("lattice") or {} + match = _FRAME_LATTICE.fullmatch(str(lattice.get("frames", "")).replace(" ", "")) + if match is None or not isinstance(lattice.get("fps"), (int, float)): + return None + return int(match.group(1)), int(match.group(2)), lattice["fps"] + + +def _duration_to_frames(seconds: float, spec: dict[str, Any], pointer: str) -> int: + step, offset, fps = _frame_lattice(spec) + position = (seconds * fps - offset) / step + snap = spec["lattice"].get("snap", "nearest") + if snap == "reject" and not math.isclose(position, round(position), abs_tol=1e-9): + raise GMSError( + "invalid_request", + f"duration {seconds} is off the {spec['lattice']['frames']} frame lattice at {fps} fps", + pointer, + ) + index = math.ceil(position - 1e-9) if snap == "up" else round(position) + return step * max(index, 0) + offset diff --git a/src/diffusers/commands/serve/parallel.py b/src/diffusers/commands/serve/parallel.py index fdbfd1278525..77dc25c7de9a 100644 --- a/src/diffusers/commands/serve/parallel.py +++ b/src/diffusers/commands/serve/parallel.py @@ -33,8 +33,8 @@ from ...utils import logging from .backends import DiffusersBackend, load_request_media -from .generations import Generation -from .manifest import GMSError, ServedModel +from .manifest import GMSError, Manifest +from .scheduler import GenerationJob logger = logging.get_logger("diffusers-cli/serve") @@ -43,7 +43,7 @@ _SHUTDOWN_SECONDS = 30.0 -class _WorkerGeneration: +class _WorkerJob: """The parts of a generation `DiffusersBackend.run` reads, inside a worker. `cancel_requested` is a collective: rank 0 reads the flag the server sets and broadcasts it, so every rank @@ -76,7 +76,7 @@ def _serve_jobs(backend: Any, rank: int, conn: Connection, cancel_flag: Any) -> job = conn.recv() if job is None: return - generation = _WorkerGeneration(job, rank, conn, cancel_flag) + generation = _WorkerJob(job, rank, conn, cancel_flag) try: reply = ("complete", backend.run(job["kwargs"], generation)) except GMSError as e: @@ -90,7 +90,7 @@ def _serve_jobs(backend: Any, rank: int, conn: Connection, cancel_flag: Any) -> def _worker_main( - rank: int, world_size: int, port: int, args: Namespace, served: ServedModel, conn: Connection, cancel_flag: Any + rank: int, world_size: int, port: int, args: Namespace, manifest: Manifest, conn: Connection, cancel_flag: Any ) -> None: import torch @@ -106,7 +106,7 @@ def _worker_main( if args.log_level is not None: logging.set_verbosity(logging.get_log_levels_dict()[args.log_level]) # Loading applies `--context-parallel`, which joins the process group of all ranks. - backend = DiffusersBackend(args, served, save_outputs=rank == 0) + backend = DiffusersBackend(args, manifest, save_outputs=rank == 0) conn.send(("ready",)) try: _serve_jobs(backend, rank, conn, cancel_flag) @@ -115,7 +115,7 @@ def _worker_main( class ParallelBackend: - def __init__(self, args: Namespace, served: ServedModel): + def __init__(self, args: Namespace, manifest: Manifest): import torch world_size = torch.cuda.device_count() @@ -125,7 +125,7 @@ def __init__(self, args: Namespace, served: ServedModel): probe.bind(("127.0.0.1", 0)) port = probe.getsockname()[1] - self._served = served + self._manifest = manifest self._failure: str | None = None self._failed = threading.Event() self._closing = False @@ -139,7 +139,7 @@ def __init__(self, args: Namespace, served: ServedModel): parent_conn, child_conn = context.Pipe() process = context.Process( target=_worker_main, - args=(rank, world_size, port, args, served, child_conn, self._cancel_flag), + args=(rank, world_size, port, args, manifest, child_conn, self._cancel_flag), name=f"diffusers-serve-rank-{rank}", daemon=True, ) @@ -185,12 +185,12 @@ def _monitor(self) -> None: def failure(self) -> str | None: return self._failure - def generate(self, generation: Generation) -> dict[str, list[dict[str, Any]]]: + def generate(self, generation: GenerationJob) -> dict[str, list[dict[str, Any]]]: if self._failure is not None: raise GMSError("internal", self._failure) job = { "request": generation.request, - "kwargs": load_request_media(self._served, generation.request), + "kwargs": load_request_media(self._manifest, generation.request), "seed": generation.seed, "directory": str(generation.directory), } diff --git a/src/diffusers/commands/serve/generations.py b/src/diffusers/commands/serve/scheduler.py similarity index 89% rename from src/diffusers/commands/serve/generations.py rename to src/diffusers/commands/serve/scheduler.py index b33e277963ad..4508f3114125 100644 --- a/src/diffusers/commands/serve/generations.py +++ b/src/diffusers/commands/serve/scheduler.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""The generation resource and the single-worker queue that runs generations one at a time.""" +"""Generation jobs, and the scheduler that runs them through the backend in arrival order, one at a time.""" from __future__ import annotations @@ -35,7 +35,9 @@ EXPIRY_SECONDS = 3600 -class Generation: +class GenerationJob: + """One generation, from the request that created it to its outputs: the resource `/v1/generations/{id}` returns.""" + def __init__( self, model: str, @@ -101,13 +103,18 @@ def _emit(self, event: str, data: dict[str, Any] | None) -> None: self._condition.notify_all() -class GenerationQueue: +class Scheduler: + """Runs generation jobs through the backend on a single worker thread, first in, first out. + + It also keeps every job until it expires, so the API can look jobs up, cancel them and delete them. + """ + def __init__(self, backend: Any, output_dir: str): self._backend = backend self._output_dir = Path(output_dir) - self._generations: dict[str, Generation] = {} + self._generations: dict[str, GenerationJob] = {} self._lock = threading.Lock() - self._pending: queue.Queue[Generation] = queue.Queue() + self._pending: queue.Queue[GenerationJob] = queue.Queue() self._running = False threading.Thread(target=self._work, name="diffusers-serve-worker", daemon=True).start() @@ -117,13 +124,13 @@ def failure(self) -> str | None: def busy(self) -> bool: return self._running or not self._pending.empty() - def submit(self, generation: Generation) -> None: + def submit(self, generation: GenerationJob) -> None: self._prune() with self._lock: self._generations[generation.id] = generation self._pending.put(generation) - def get(self, generation_id: str) -> Generation: + def get(self, generation_id: str) -> GenerationJob: self._prune() with self._lock: generation = self._generations.get(generation_id) @@ -131,7 +138,7 @@ def get(self, generation_id: str) -> Generation: raise GMSError("not_found", f"no generation with id {generation_id!r}") return generation - def cancel(self, generation_id: str) -> Generation: + def cancel(self, generation_id: str) -> GenerationJob: generation = self.get(generation_id) with self._lock: if generation.done: @@ -158,7 +165,7 @@ def _prune(self) -> None: for generation in expired: self._remove_files(generation) - def _remove_files(self, generation: Generation) -> None: + def _remove_files(self, generation: GenerationJob) -> None: if generation.directory is not None: shutil.rmtree(generation.directory, ignore_errors=True) diff --git a/tests/others/test_cli_serve.py b/tests/others/test_cli_serve.py index 0b2823340b25..97e4922ceb42 100644 --- a/tests/others/test_cli_serve.py +++ b/tests/others/test_cli_serve.py @@ -21,6 +21,7 @@ import io import json import os +import re import shlex import threading import time @@ -35,8 +36,8 @@ from diffusers.commands.serve import ServeCommand, _shut_down_when_idle from diffusers.commands.serve.backends import DiffusersBackend, backend_capabilities -from diffusers.commands.serve.generations import GenerationQueue -from diffusers.commands.serve.manifest import GMSError, Manifest, ServedModel, blocks_signature, derive_manifest +from diffusers.commands.serve.manifest import GMSError, Manifest, blocks_signature, derive_manifest +from diffusers.commands.serve.scheduler import Scheduler from diffusers.modular_pipelines import InputParam, ModularPipelineBlocks, OutputParam @@ -154,17 +155,14 @@ def _args(**overrides): return Namespace(**{**values, **overrides}) -def _served(manifest=None, pipeline_cls=FakePipeline, signature=None, args=None): +def _manifest(manifest=None, pipeline_cls=FakePipeline, signature=None, args=None): + """A `Manifest` bound to `pipeline_cls`, from the given manifest dict or bytes, or derived from the pipeline.""" signature = signature or inspect.signature(pipeline_cls.__call__) + capabilities = backend_capabilities(args or _args(), pipeline_cls) if manifest is None: - manifest = derive_manifest(MODEL_ID, pipeline_cls, signature) + manifest = derive_manifest(MODEL_ID, pipeline_cls, signature, capabilities) raw = manifest if isinstance(manifest, bytes) else json.dumps(manifest).encode() - return ServedModel( - Manifest(raw=raw, data=json.loads(raw)), - set(signature.parameters) - {"self"}, - backend_capabilities(args or _args(), pipeline_cls), - revision="abc123", - ) + return Manifest(raw, json.loads(raw), pipeline_cls, signature, capabilities, revision="abc123") @pytest.fixture @@ -180,9 +178,9 @@ def _build(manifest=None, app_options=None, pipeline=None, signature=None, backe pipeline = pipeline or FakePipeline() monkeypatch.setattr("diffusers.commands.serve.backends._load_pipeline", lambda args: pipeline) args = _args(**arg_overrides) - served = _served(manifest=manifest, pipeline_cls=type(pipeline), signature=signature, args=args) - backend = backend or DiffusersBackend(args, served) - app = create_app(served, GenerationQueue(backend, str(tmp_path)), **(app_options or {})) + bound = _manifest(manifest=manifest, pipeline_cls=type(pipeline), signature=signature, args=args) + backend = backend or DiffusersBackend(args, bound) + app = create_app(bound, Scheduler(backend, str(tmp_path)), **(app_options or {})) return TestClient(app), pipeline return _build @@ -196,7 +194,6 @@ def _build(manifest=None, app_options=None, pipeline=None, signature=None, backe "inputs": { "prompt": {"type": "text", "required": True}, "negative_prompt": {"type": "text", "active_when": {"parameter": "guidance_scale", "gt": 1}}, - "last_image": {"type": "image"}, }, "outputs": {"video": {"type": "video"}}, "parameters": { @@ -205,17 +202,15 @@ def _build(manifest=None, app_options=None, pipeline=None, signature=None, backe "steps": {"type": "integer", "default": 30, "min": 1, "max": 100}, "fps": {"type": "number"}, "guidance_scale": {"type": "number", "default": 3.0, "min": 1.0, "max": 10.0}, - "guidance_type": {"type": "string", "enum": ["cfg", "distilled"], "default": "cfg"}, "seed": {"type": "integer"}, - "video_decoder": {"type": "string", "enum": ["conv", "diffusion"], "default": "conv"}, }, "presets": { "default": {}, "distilled": {"parameters": {"steps": 8, "guidance_scale": 1.0}}, - "quality": {"parameters": {"steps": 40, "video_decoder": "diffusion"}}, }, } }, + "capabilities": ["sse", "wait"], } @@ -236,7 +231,7 @@ def __call__( class TestManifest: def test_derive_manifest_from_signature(self): - manifest = derive_manifest(MODEL_ID, FakePipeline, FAKE_SIGNATURE) + manifest = derive_manifest(MODEL_ID, FakePipeline, FAKE_SIGNATURE, ["sse", "wait", "adapters"]) # An optional `image` kwarg yields one task without it and one that requires it. assert list(manifest["tasks"]) == ["text_to_image", "image_to_image"], manifest["tasks"].keys() @@ -260,6 +255,7 @@ def test_derive_manifest_from_signature(self): assert text_to_image["x-diffusers"] == { "parameters": {"steps": "num_inference_steps", "num_outputs": "num_images_per_prompt", "seed": "generator"} } + assert manifest["capabilities"] == ["sse", "wait", "adapters"] assert manifest["adapters"] == {"lora": {"targets": ["transformer", "text_encoder"]}} @pytest.mark.parametrize( @@ -278,42 +274,40 @@ def test_derive_manifest_from_signature(self): "invalid_request", "/inputs/negative_prompt", ), - # A parameter the pipeline cannot set is refused rather than dropped, unless it is the default. - ({"parameters": {"video_decoder": "diffusion"}}, "not_implemented", "/parameters/video_decoder"), ], ) def test_resolve_rejects_invalid_requests(self, body, code, pointer): - served = _served(manifest=LATTICE_MANIFEST, pipeline_cls=FakeVideoPipeline) + bound = _manifest(manifest=LATTICE_MANIFEST, pipeline_cls=FakeVideoPipeline) with pytest.raises(GMSError) as error: - served.resolve({"model": MODEL_ID, "inputs": {"prompt": "a cat"}, **body}) + bound.resolve({"model": MODEL_ID, "inputs": {"prompt": "a cat"}, **body}) assert (error.value.code, error.value.pointer) == (code, pointer), error.value.message def test_resolve_rejects_non_http_media_urls(self): - served = _served() + bound = _manifest() body = {"model": MODEL_ID, "inputs": {"prompt": "a cat", "image": {"type": "image", "url": "/etc/passwd"}}} with pytest.raises(GMSError) as error: - served.resolve(body) + bound.resolve(body) assert error.value.pointer == "/inputs/image/url", error.value.message @pytest.mark.parametrize("path", ["hf:/etc/passwd", "hf:../loras/x", "/etc/passwd", "https://x/y.safetensors"]) def test_resolve_rejects_adapter_paths_that_are_not_repo_ids(self, path): body = {"model": MODEL_ID, "inputs": {"prompt": "a cat"}, "adapters": [{"type": "lora", "path": path}]} with pytest.raises(GMSError) as error: - _served().resolve(body) + _manifest().resolve(body) assert (error.value.code, error.value.pointer) == ("invalid_request", "/adapters/0/path"), error.value.message def test_resolve_maps_request_to_pipeline_kwargs(self): - served = _served(manifest=LATTICE_MANIFEST, pipeline_cls=FakeVideoPipeline) - resolved = served.resolve( + bound = _manifest(manifest=LATTICE_MANIFEST, pipeline_cls=FakeVideoPipeline) + resolved = bound.resolve( { "model": MODEL_ID, "preset": "distilled", "inputs": {"prompt": "a cat"}, - "parameters": {"duration": 8, "seed": 7, "guidance_type": "cfg"}, + "parameters": {"duration": 8, "seed": 7}, } ) # 8 s at 24 fps is 192 frames; the nearest point on the 8k+1 lattice is 193. The preset overrides the - # task defaults, and `guidance_type` at its default is accepted although the pipeline has no such kwarg. + # task defaults. assert resolved.kwargs == { "prompt": "a cat", "num_frames": 193, @@ -325,13 +319,28 @@ def test_resolve_maps_request_to_pipeline_kwargs(self): assert resolved.seed == 7 assert resolved.task == "text_to_video" - def test_discovery_overlay(self): - served = _served(manifest=LATTICE_MANIFEST, pipeline_cls=FakeVideoPipeline) - assert served.overlay() == { - "tasks": {"text_to_video": {"presets": ["default", "distilled"]}}, - "capabilities": ["sse", "wait"], - "limits": {"text_to_video": {"inputs": {"last_image": False}}}, - } + @pytest.mark.parametrize( + "change, args, message", + [ + # flux2-klein's published manifest declares `guidance_type`, which its pipeline does not take. + ( + {"parameters": {"guidance_type": {"type": "string", "enum": ["distilled"]}}}, + {}, + "declares parameter 'guidance_type' on task 'text_to_video', which FakeVideoPipeline does not take", + ), + ({"capabilities": ["adapters"]}, {}, "request adapters are off"), + ({"capabilities": ["stream"]}, {}, "declares the 'stream' capability, which this server does not provide"), + ({"parameters": {"duration": {"type": "number"}}}, {}, "without a frame lattice"), + ], + ) + def test_manifest_the_server_cannot_honor_fails_startup(self, change, args, message): + manifest = json.loads(json.dumps(LATTICE_MANIFEST)) + if "parameters" in change: + manifest["tasks"]["text_to_video"]["parameters"].update(change["parameters"]) + else: + manifest.update(change) + with pytest.raises(SystemExit, match=re.escape(message)): + _manifest(manifest=manifest, pipeline_cls=FakeVideoPipeline, args=_args(**args)) class TestServeApp: @@ -415,6 +424,7 @@ def test_joint_video_and_audio_outputs(self, serve, monkeypatch): "outputs": {"video": {"type": "video"}, "audio": {"type": "audio"}}, } }, + "capabilities": ["wait"], } client, pipeline = serve(manifest=manifest) pipeline.result = SimpleNamespace( @@ -465,6 +475,22 @@ def test_failed_generation_returns_error_envelope(self, serve): pipeline.error = None assert client.post("/v1/generations", json=body, headers=WAIT).json()["status"] == "complete" + def test_undeclared_capabilities_are_off(self, serve): + manifest = json.loads(json.dumps(LATTICE_MANIFEST)) + manifest["tasks"]["text_to_video"] = { + "inputs": {"prompt": {"type": "text", "required": True}}, + "outputs": {"image": {"type": "image"}}, + } + manifest["capabilities"] = [] + client, pipeline = serve(manifest=manifest) + # The pipeline is held, so a response that waited for the generation could never return. + pipeline.release.clear() + created = client.post("/v1/generations", json={"model": MODEL_ID, "inputs": {"prompt": "a cat"}}, headers=WAIT) + assert created.json()["status"] in ("queued", "generating"), "without `wait`, Prefer: wait is ignored" + events = client.get(f"/v1/generations/{created.json()['id']}/events") + assert (events.status_code, events.json()["error"]["code"]) == (501, "not_implemented") + pipeline.release.set() + def test_failed_backend_is_unhealthy(self, serve): backend = SimpleNamespace(failure=lambda: "context-parallel worker rank-1 exited with code 3") client, _ = serve(backend=backend) @@ -535,13 +561,13 @@ def test_request_adapters_are_loaded_then_unloaded(self, serve): def test_adapters_not_advertised_when_compiled(self, serve): client, _ = serve(compile="{}") discovery = client.get(f"/v1/models/{MODEL_ID}").json() - assert "adapters" not in discovery["server"]["capabilities"] + assert "adapters" not in discovery["manifest"]["capabilities"] body = {"model": MODEL_ID, "inputs": {"prompt": "a cat"}, "adapters": [{"type": "lora", "path": "hf:a/b"}]} response = client.post("/v1/generations", json=body) assert (response.status_code, response.json()["error"]["code"]) == (501, "not_implemented") def test_discovery_serves_manifest_bytes_verbatim(self, serve): - manifest = derive_manifest(MODEL_ID, FakePipeline, FAKE_SIGNATURE) + manifest = derive_manifest(MODEL_ID, FakePipeline, FAKE_SIGNATURE, ["sse", "wait", "adapters"]) raw = json.dumps(manifest, indent=3).encode() client, _ = serve(manifest=raw) @@ -550,8 +576,7 @@ def test_discovery_serves_manifest_bytes_verbatim(self, serve): discovery = response.json() assert discovery["spec"] == "gms/0.1" assert discovery["manifest"] == manifest - assert discovery["server"]["tasks"] == {"text_to_image": {}, "image_to_image": {}} - assert discovery["server"]["adapters"] == {"max": 4, "sources": ["hf"]} + assert "server" not in discovery, "the manifest describes the server, so nothing is added to it" assert client.get("/v1/models").json() == { "data": [ { @@ -567,7 +592,7 @@ class TestModularPipeline: def test_generation_runs_a_modular_pipeline(self, serve): pipeline = FakeImageBlock().init_pipeline() signature = blocks_signature(pipeline.blocks) - manifest = derive_manifest(MODEL_ID, type(pipeline), signature) + manifest = derive_manifest(MODEL_ID, type(pipeline), signature, ["wait"]) assert list(manifest["tasks"]) == ["text_to_image", "image_to_image"], manifest["tasks"].keys() assert manifest["tasks"]["text_to_image"]["inputs"] == {"prompt": {"type": "text", "required": True}} assert manifest["tasks"]["text_to_image"]["parameters"] == { @@ -718,7 +743,7 @@ def stop(args): assert resolved["dtype"] == "bf16", "serve should default to bf16 rather than `run`'s fp32 `auto`" -def _fake_worker_main(rank, world_size, port, args, served, conn, cancel_flag): +def _fake_worker_main(rank, world_size, port, args, manifest, conn, cancel_flag): """Stands in for `_worker_main` in forked children: no pipeline, no process group.""" if args.fail_at_startup: os._exit(3) @@ -776,7 +801,7 @@ def backend(self, monkeypatch): def _build(**arg_overrides): args = Namespace(**{"fail_at_startup": False, **arg_overrides}) - backends.append(parallel.ParallelBackend(args, _served())) + backends.append(parallel.ParallelBackend(args, _manifest())) return backends[-1] yield _build @@ -816,9 +841,9 @@ def test_worker_arguments_survive_spawn(self): parser = ArgumentParser() ServeCommand.register_subcommand(parser.add_subparsers()) args = parser.parse_args(["serve", "org/model", "--context-parallel"]) - # Spawned workers receive the parsed flags and the served model pickled. - restored_args, restored_served = pickle.loads(pickle.dumps((args, _served()))) - assert restored_args.context_parallel and restored_served.tasks + # Spawned workers receive the parsed flags and the manifest pickled. + restored_args, restored_manifest = pickle.loads(pickle.dumps((args, _manifest()))) + assert restored_args.context_parallel and restored_manifest.tasks class TestParallelWorker: @@ -863,14 +888,14 @@ def run(kwargs, generation): def test_cancel_is_broadcast_from_rank_0(self, monkeypatch): from multiprocessing import Value - from diffusers.commands.serve.parallel import _WorkerGeneration + from diffusers.commands.serve.parallel import _WorkerJob monkeypatch.setenv("MASTER_ADDR", "127.0.0.1") monkeypatch.setenv("MASTER_PORT", str(_free_port())) torch.distributed.init_process_group("gloo", rank=0, world_size=1) try: flag = Value("b", 0) - generation = _WorkerGeneration(self._job("ok"), 0, None, flag) + generation = _WorkerJob(self._job("ok"), 0, None, flag) assert not generation.cancel_requested flag.value = 1 assert generation.cancel_requested, "rank 0 should broadcast the flag the server raised" From ccdcc3643352d2483b2a39774f6cd3297ba6353d Mon Sep 17 00:00:00 2001 From: DN6 Date: Thu, 5 Nov 2026 10:24:02 +0530 Subject: [PATCH 11/21] Flatten serve response formatting into _format_job_response and _format_output --- src/diffusers/commands/serve/app.py | 34 ++++++++++++++++------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/src/diffusers/commands/serve/app.py b/src/diffusers/commands/serve/app.py index 302d88e1a562..a759bd73e95b 100644 --- a/src/diffusers/commands/serve/app.py +++ b/src/diffusers/commands/serve/app.py @@ -49,7 +49,17 @@ def _timestamp(value: Any) -> str | None: return value.strftime("%Y-%m-%dT%H:%M:%SZ") -def _job_response(job: GenerationJob, base_url: str) -> dict[str, Any]: +def _format_output(job: GenerationJob, artifact: dict[str, Any], base_url: str) -> dict[str, Any]: + """One output file as the response lists it: its metadata, plus a download link or its bytes inline.""" + entry = {key: value for key, value in artifact.items() if key != "file"} + if job.request.response_format == "b64_json": + entry["base64"] = base64.b64encode((job.directory / artifact["file"]).read_bytes()).decode() + return entry + entry["url"] = base_url + _FILE_ROUTE.format(generation_id=job.id, filename=artifact["file"]) + return entry + + +def _format_job_response(job: GenerationJob, base_url: str) -> dict[str, Any]: """The response body for a generation: outputs are linked under `base_url`, or inlined as base64. Raises the generation's error once it failed, which the error handler turns into the error envelope. @@ -59,16 +69,10 @@ def _job_response(job: GenerationJob, base_url: str) -> dict[str, Any]: outputs = None if job.status == "complete": - outputs = {} - for name, artifacts in job.outputs.items(): - outputs[name] = [] - for artifact in artifacts: - entry = {key: value for key, value in artifact.items() if key != "file"} - if job.request.response_format == "b64_json": - entry["base64"] = base64.b64encode((job.directory / artifact["file"]).read_bytes()).decode() - else: - entry["url"] = base_url + _FILE_ROUTE.format(generation_id=job.id, filename=artifact["file"]) - outputs[name].append(entry) + outputs = { + name: [_format_output(job, artifact, base_url) for artifact in artifacts] + for name, artifacts in job.outputs.items() + } return { "id": job.id, @@ -178,12 +182,12 @@ async def create_generation(request: Request) -> dict[str, Any]: # `Prefer` is a hint, so a server without the `wait` capability answers right away instead of failing. if wait is not None and "wait" in manifest.capabilities: await asyncio.to_thread(generation.wait, min(float(wait.group(1)), MAX_WAIT_SECONDS)) - return _job_response(generation, base_url(request)) + return _format_job_response(generation, base_url(request)) @app.get("/v1/generations/{generation_id}") async def get_generation(generation_id: str, request: Request) -> dict[str, Any]: generation = scheduler.get(generation_id) - return _job_response(generation, base_url(request)) + return _format_job_response(generation, base_url(request)) @app.delete("/v1/generations/{generation_id}", status_code=204) async def delete_generation(generation_id: str) -> Response: @@ -193,7 +197,7 @@ async def delete_generation(generation_id: str) -> Response: @app.post("/v1/generations/{generation_id}/cancel") async def cancel_generation(generation_id: str, request: Request) -> dict[str, Any]: generation = scheduler.cancel(generation_id) - return _job_response(generation, base_url(request)) + return _format_job_response(generation, base_url(request)) @app.get("/v1/generations/{generation_id}/events") async def get_generation_events(generation_id: str, request: Request) -> StreamingResponse: @@ -209,7 +213,7 @@ async def stream(): events, done = await asyncio.to_thread(generation.wait_for_events, index, _KEEP_ALIVE_SECONDS) for event, data in events: index += 1 - payload = _job_response(generation, base_url(request)) if event == "complete" else data + payload = _format_job_response(generation, base_url(request)) if event == "complete" else data yield f"id: {index}\nevent: {event}\ndata: {json.dumps(payload)}\n\n" if done: return From 2fd5dcb15daffcbeb6dc365c97783cecdd19cf69 Mon Sep 17 00:00:00 2001 From: DN6 Date: Thu, 5 Nov 2026 14:39:37 +0530 Subject: [PATCH 12/21] Derive serve fps and sample rate from the model and merge parallel.py into backends.py --- docs/source/en/using-diffusers/serve.md | 12 +- src/diffusers/commands/serve/__init__.py | 62 ++++-- src/diffusers/commands/serve/app.py | 10 +- src/diffusers/commands/serve/backends.py | 243 +++++++++++++++++++++- src/diffusers/commands/serve/manifest.py | 26 ++- src/diffusers/commands/serve/parallel.py | 235 --------------------- src/diffusers/commands/serve/scheduler.py | 51 ++--- tests/others/test_cli_serve.py | 85 ++++++-- 8 files changed, 396 insertions(+), 328 deletions(-) delete mode 100644 src/diffusers/commands/serve/parallel.py diff --git a/docs/source/en/using-diffusers/serve.md b/docs/source/en/using-diffusers/serve.md index 353502182fcb..c9f68f454cc8 100644 --- a/docs/source/en/using-diffusers/serve.md +++ b/docs/source/en/using-diffusers/serve.md @@ -45,7 +45,7 @@ progress, and a cancel request only takes effect once the pipeline call returns. ## Generations -Generations run one at a time. `POST /v1/generations` returns immediately with a `queued` generation. Poll it, +Generations run one at a time, or one per GPU with [`--data-parallel`](#multiple-gpus). `POST /v1/generations` returns immediately with a `queued` generation. Poll it, subscribe to its events, or ask the server to hold the connection with `Prefer: wait=`: ```bash @@ -141,6 +141,14 @@ can provide. ## Multiple GPUs +`--data-parallel N` loads a copy of the pipeline on each of the first N GPUs and runs up to N generations at +once. Generations still start in the order they arrive: each GPU takes the next queued generation when it finishes +its current one. + +```bash +diffusers serve black-forest-labs/FLUX.1-dev --data-parallel 4 +``` + `--context-parallel` splits each generation across every visible GPU with [context parallelism](../training/distributed_inference#context-parallelism). It needs a DiT-based pipeline and at least two GPUs. `serve` starts one worker process per GPU itself, so don't launch it with `torchrun`: @@ -151,7 +159,7 @@ diffusers serve black-forest-labs/FLUX.1-dev --context-parallel Every worker runs every generation, and rank 0 saves the outputs. If a worker process exits, `GET /health` returns `503` and the server stops generating. `--device-map` and request adapters are not available with -`--context-parallel`, and `--compile` is skipped. +`--context-parallel`, and `--compile` is skipped. `--context-parallel` and `--data-parallel` can't be combined. ## Remote serving diff --git a/src/diffusers/commands/serve/__init__.py b/src/diffusers/commands/serve/__init__.py index 313dfb537747..2b229bc03256 100644 --- a/src/diffusers/commands/serve/__init__.py +++ b/src/diffusers/commands/serve/__init__.py @@ -21,12 +21,14 @@ from __future__ import annotations import inspect +import secrets import threading import time from argparse import ArgumentParser, Namespace, RawDescriptionHelpFormatter, _SubParsersAction from pathlib import Path from typing import Any +import torch from huggingface_hub.cli._output import out from ...utils import logging @@ -148,22 +150,23 @@ def register_subcommand(subparsers: _SubParsersAction) -> None: help=f"Directory generated files are stored in until they expire. Defaults to {DEFAULT_OUTPUT_DIR}.", ) parser.add_argument( - "--fps", + "--data-parallel", type=int, - default=8, - help="FPS used to encode video outputs when the pipeline call takes no fps argument.", - ) - parser.add_argument( - "--sampling-rate", - type=int, - default=None, - help="Sample rate used to encode audio outputs. Defaults to 16000.", + default=1, + metavar="N", + help=( + "Load a copy of the pipeline on each of the first N GPUs and run up to N generations at once. " + "Defaults to 1." + ), ) parser.add_argument( - "--default-seed", + "--seed", type=int, default=None, - help="Seed used for requests that do not set one. Defaults to a random seed per request.", + help=( + "Seed for the generator that picks the seed of each request that does not set one. " + "Defaults to a random seed chosen at startup." + ), ) parser.add_argument( "--enable-cors", @@ -213,6 +216,22 @@ def run(self) -> None: raise SystemExit( "--device-map does not apply with --context-parallel: each worker loads the pipeline on its own GPU." ) + if args.data_parallel < 1: + raise SystemExit(f"--data-parallel must be at least 1, got {args.data_parallel}.") + if args.data_parallel > 1 and args.context_parallel: + raise SystemExit( + "--data-parallel and --context-parallel cannot be combined: --context-parallel already uses " + "every visible GPU for each generation." + ) + if args.data_parallel > 1 and args.device_map is not None: + raise SystemExit( + "--device-map does not apply with --data-parallel: each copy of the pipeline is loaded on its own GPU." + ) + if args.data_parallel > 1 and not args.remote and torch.cuda.device_count() < args.data_parallel: + raise SystemExit( + f"--data-parallel {args.data_parallel} needs {args.data_parallel} GPUs, but " + f"{torch.cuda.device_count()} are visible." + ) if args.remote: serve_remote(args, self.task) @@ -227,9 +246,8 @@ def run(self) -> None: ) from e from .app import create_app - from .backends import DiffusersBackend, backend_capabilities + from .backends import DiffusersBackend, ParallelBackend, backend_capabilities from .manifest import load_manifest - from .parallel import ParallelBackend from .scheduler import Scheduler pipeline_cls, signature, commit_hash = _resolve_pipeline(args) @@ -246,16 +264,22 @@ def run(self) -> None: ) if args.context_parallel: - backend = ParallelBackend(args, manifest) + backends = [ParallelBackend(args, manifest)] + elif args.data_parallel > 1: + backends = [ + DiffusersBackend(Namespace(**{**vars(args), "device_map": f"cuda:{index}"}), manifest) + for index in range(args.data_parallel) + ] else: - backend = DiffusersBackend(args, manifest) + backends = [DiffusersBackend(args, manifest)] try: - scheduler = Scheduler(backend, args.output_dir) + seed = args.seed if args.seed is not None else secrets.randbelow(2**32) + scheduler = Scheduler(backends, args.output_dir) app = create_app( manifest, scheduler, - default_seed=args.default_seed, + seed=seed, enable_cors=args.enable_cors, public_url=args.public_url, ) @@ -266,6 +290,7 @@ def run(self) -> None: pipeline_class=pipeline_cls.__name__, tasks=list(manifest.tasks), capabilities=manifest.capabilities, + seed=seed, ) server = uvicorn.Server( uvicorn.Config(app, host=args.host, port=args.port, log_level=args.log_level or "info") @@ -279,4 +304,5 @@ def run(self) -> None: ).start() server.run() finally: - backend.close() + for backend in backends: + backend.close() diff --git a/src/diffusers/commands/serve/app.py b/src/diffusers/commands/serve/app.py index a759bd73e95b..f4d5ef4f5cef 100644 --- a/src/diffusers/commands/serve/app.py +++ b/src/diffusers/commands/serve/app.py @@ -19,6 +19,7 @@ import asyncio import base64 import json +import random import re import time import uuid @@ -83,7 +84,7 @@ def _format_job_response(job: GenerationJob, base_url: str) -> dict[str, Any]: "progress": job.progress if job.status == "generating" else None, "outputs": outputs, "expires_at": _timestamp(job.expires_at), - "seed": job.seed if job.status == "complete" else None, + "seed": job.request.seed if job.status == "complete" else None, "usage": job.usage, } @@ -91,7 +92,7 @@ def _format_job_response(job: GenerationJob, base_url: str) -> dict[str, Any]: def create_app( manifest: Manifest, scheduler: Scheduler, - default_seed: int | None = None, + seed: int | None = None, enable_cors: bool = False, public_url: str | None = None, ) -> FastAPI: @@ -100,6 +101,7 @@ def create_app( root_path = urlparse(public_url).path.rstrip("/") if public_url is not None else "" app = FastAPI(title="diffusers-cli serve", docs_url=None, redoc_url=None, openapi_url=None, root_path=root_path) app.state.last_request = time.monotonic() + seeds = random.Random(seed) def base_url(request: Request) -> str: # Output links must work for the client, so they are built on the address it reached the server at. @@ -173,9 +175,7 @@ async def create_generation(request: Request) -> dict[str, Any]: body = await request.json() except json.JSONDecodeError as e: raise GMSError("invalid_request", f"request body is not valid JSON: {e}") from e - resolved = manifest.resolve(body) - seeded = manifest.seeded(resolved.task) - generation = GenerationJob(manifest.id, manifest.revision, resolved, seeded=seeded, default_seed=default_seed) + generation = GenerationJob(manifest.id, manifest.revision, manifest.resolve(body, seeds)) scheduler.submit(generation) wait = re.fullmatch(r"\s*wait=(\d+)\s*", request.headers.get("prefer", "")) diff --git a/src/diffusers/commands/serve/backends.py b/src/diffusers/commands/serve/backends.py index 4c41dc7ba205..d9d3781c7f56 100644 --- a/src/diffusers/commands/serve/backends.py +++ b/src/diffusers/commands/serve/backends.py @@ -12,7 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""The backend that executes a resolved generation with an in-process pipeline.""" +"""The backends that execute a resolved generation: in the server process, or in one worker process per GPU. + +`DiffusersBackend` runs the pipeline in the server process. `ParallelBackend` serves `--context-parallel`: one worker +process per GPU, all running every generation together. The server process never touches CUDA. It spawns the workers, +sends each generation to all of them over a pipe, and reads the progress and result back from rank 0. Idle workers +wait on their pipe rather than inside a collective, so the process group cannot time out between requests. +""" from __future__ import annotations @@ -20,10 +26,15 @@ import inspect import io import mimetypes +import multiprocessing import os +import signal +import socket import tempfile +import threading import wave from argparse import Namespace +from multiprocessing.connection import Connection, wait from pathlib import Path from typing import Any from urllib.parse import urlparse @@ -44,13 +55,21 @@ _load_pipeline, _save_audio_arrays, ) -from .manifest import GMSError, Manifest, blocks_signature +from .manifest import _PARAMETER_KWARGS, GMSError, Manifest, _frame_lattice, blocks_signature from .scheduler import GenerationJob logger = logging.get_logger("diffusers-cli/serve") _OUTPUT_FIELDS = {"image": ("images",), "video": ("frames", "videos"), "audio": ("audios", "audio")} +_SAMPLE_RATE_ATTRIBUTES = ( + "sample_rate", + "vocoder.config.output_sampling_rate", + "vocoder.config.sampling_rate", + "audio_sampling_rate", + "sound_sampling_rate", + "vae.config.sampling_rate", +) def backend_capabilities(args: Namespace, pipeline_cls: type) -> list[str]: @@ -162,6 +181,19 @@ def _save_audio(audios: Any, directory: Path, name: str, media_type: str, rate: _SAVERS = {"image": _save_images, "video": _save_videos, "audio": _save_audio} +def _sample_rate(pipeline: Any) -> int: + for path in _SAMPLE_RATE_ATTRIBUTES: + value = pipeline + for attribute in path.split("."): + value = getattr(value, attribute, None) + if value: + return int(value) + raise SystemExit( + f"Cannot serve audio outputs: {type(pipeline).__name__} exposes no output sample rate. Looked for " + f"{list(_SAMPLE_RATE_ATTRIBUTES)} on the pipeline." + ) + + def load_request_media(manifest: Manifest, request: Any) -> dict[str, Any]: """The pipeline kwargs of a request, with its media inputs fetched and decoded.""" kwargs = dict(request.kwargs) @@ -180,7 +212,6 @@ def load_request_media(manifest: Manifest, request: Any) -> dict[str, Any]: class DiffusersBackend: def __init__(self, args: Namespace, manifest: Manifest, save_outputs: bool = True): - self._args = args self._manifest = manifest # Under context parallelism every rank runs the call, and only rank 0 writes the files. self._save_outputs = save_outputs @@ -196,6 +227,8 @@ def __init__(self, args: Namespace, manifest: Manifest, save_outputs: bool = Tru if parameter.default is not inspect.Parameter.empty } self._reports_steps = "callback_on_step_end" in parameters + outputs = (spec["type"] for task in manifest.tasks.values() for spec in task["outputs"].values()) + self._sample_rate = _sample_rate(self.pipeline) if "audio" in outputs else None def close(self) -> None: pass @@ -209,9 +242,10 @@ def generate(self, generation: GenerationJob) -> dict[str, list[dict[str, Any]]] def run(self, kwargs: dict[str, Any], generation: Any) -> dict[str, list[dict[str, Any]]]: """Call the pipeline with the request's kwargs, media already loaded, and save its outputs.""" request = generation.request - if generation.seed is not None: - device = self.pipeline.device.type if hasattr(self.pipeline, "device") else "cpu" - kwargs["generator"] = _get_generator(generation.seed, device) + if request.seed is not None: + device = self.pipeline.device if hasattr(self.pipeline, "device") else torch.device("cpu") + # A CUDA generator must sit on the pipeline's own GPU, which is `cuda:1` and up with --data-parallel. + kwargs["generator"] = _get_generator(request.seed, str(device) if device.type == "cuda" else device.type) if self._reports_steps: @@ -246,7 +280,7 @@ def on_step_end(pipeline, step, timestep, callback_kwargs): value = next((field for field in fields if field is not None), None) if value is None: continue - rate = self._fps(kwargs) if kind == "video" else (self._args.sampling_rate or 16000) + rate = self._fps(request.task, kwargs) if kind == "video" else self._sample_rate outputs[name] = _SAVERS[kind](value, generation.directory, name, request.output_formats[kind], rate) if not outputs: declared = list(self._manifest.tasks[request.task]["outputs"]) @@ -257,12 +291,13 @@ def on_step_end(pipeline, step, timestep, callback_kwargs): ) return outputs - def _fps(self, kwargs: dict[str, Any]) -> float: - for kwarg in ("fps", "frame_rate"): + def _fps(self, task: str, kwargs: dict[str, Any]) -> float: + for kwarg in _PARAMETER_KWARGS["fps"]: rate = kwargs.get(kwarg) or self._defaults.get(kwarg) if rate: return rate - return self._args.fps + # The manifest check at startup guarantees a frame lattice when the pipeline has no default fps. + return _frame_lattice(self._manifest.tasks[task]["parameters"]["duration"])[2] def _load_adapters(self, adapters: list[dict[str, Any]]) -> None: if not adapters: @@ -288,3 +323,191 @@ def _load_adapters(self, adapters: list[dict[str, Any]]) -> None: weights.append(scale) names.append(name) self.pipeline.set_adapters(names, adapter_weights=weights) + + +_POLL_SECONDS = 0.1 +_SHUTDOWN_SECONDS = 30.0 + + +class _WorkerJob: + """The parts of a generation `DiffusersBackend.run` reads, inside a worker. + + `cancel_requested` is a collective: rank 0 reads the flag the server sets and broadcasts it, so every rank + interrupts at the same step. A rank that stopped alone would leave the others waiting in the next collective. + """ + + def __init__(self, job: dict[str, Any], rank: int, conn: Connection, cancel_flag: Any): + self.request = job["request"] + self.directory = Path(job["directory"]) + self._rank = rank + self._conn = conn + self._cancel_flag = cancel_flag + + def report_progress(self, step: int, total_steps: int) -> None: + if self._rank == 0: + self._conn.send(("progress", step, total_steps)) + + @property + def cancel_requested(self) -> bool: + flag = torch.tensor([self._cancel_flag.value if self._rank == 0 else 0], dtype=torch.uint8) + torch.distributed.broadcast(flag, src=0) + return bool(flag.item()) + + +def _serve_jobs(backend: Any, rank: int, conn: Connection, cancel_flag: Any) -> None: + while True: + job = conn.recv() + if job is None: + return + generation = _WorkerJob(job, rank, conn, cancel_flag) + try: + reply = ("complete", backend.run(job["kwargs"], generation)) + except GMSError as e: + reply = ("failed", e.code, e.message, e.pointer) + # The helpers shared with `run` raise SystemExit, which must not end the worker. + except (Exception, SystemExit) as e: + logger.error(f"generation failed on rank {rank}", exc_info=True) + reply = ("failed", "internal", f"{type(e).__name__}: {e}", None) + if rank == 0: + conn.send(reply) + + +def _worker_main( + rank: int, world_size: int, port: int, args: Namespace, manifest: Manifest, conn: Connection, cancel_flag: Any +) -> None: + # The server process decides when workers stop, so Ctrl-C in the terminal must not interrupt them mid-call. + signal.signal(signal.SIGINT, signal.SIG_IGN) + os.environ.update( + MASTER_ADDR="127.0.0.1", + MASTER_PORT=str(port), + RANK=str(rank), + LOCAL_RANK=str(rank), + WORLD_SIZE=str(world_size), + ) + if args.log_level is not None: + logging.set_verbosity(logging.get_log_levels_dict()[args.log_level]) + # Loading applies `--context-parallel`, which joins the process group of all ranks. + backend = DiffusersBackend(args, manifest, save_outputs=rank == 0) + conn.send(("ready",)) + try: + _serve_jobs(backend, rank, conn, cancel_flag) + finally: + torch.distributed.destroy_process_group() + + +class ParallelBackend: + def __init__(self, args: Namespace, manifest: Manifest): + world_size = torch.cuda.device_count() + if world_size < 2: + raise SystemExit(f"--context-parallel needs at least 2 visible GPUs, found {world_size}.") + with socket.socket() as probe: + probe.bind(("127.0.0.1", 0)) + port = probe.getsockname()[1] + + self._manifest = manifest + self._failure: str | None = None + self._failed = threading.Event() + self._closing = False + # Spawned rather than forked: a forked child inherits whatever CUDA state its parent has. + context = multiprocessing.get_context("spawn") + # Lock-free: a worker killed while holding the lock would leave the server blocked on its next write. + self._cancel_flag = context.Value("b", 0, lock=False) + self._conns: list[Connection] = [] + self._processes = [] + for rank in range(world_size): + parent_conn, child_conn = context.Pipe() + process = context.Process( + target=_worker_main, + args=(rank, world_size, port, args, manifest, child_conn, self._cancel_flag), + name=f"diffusers-serve-rank-{rank}", + daemon=True, + ) + process.start() + child_conn.close() + self._conns.append(parent_conn) + self._processes.append(process) + + logger.info(f"loading the pipeline on {world_size} GPUs...") + self._wait_until_ready() + threading.Thread(target=self._monitor, name="diffusers-serve-workers", daemon=True).start() + + def _wait_until_ready(self) -> None: + waiting = dict(zip(self._conns, self._processes)) + while waiting: + sentinels = {process.sentinel: process for process in waiting.values()} + for ready in wait([*waiting, *sentinels]): + process = waiting.pop(ready, None) if isinstance(ready, Connection) else sentinels[ready] + try: + if isinstance(ready, Connection) and ready.recv() == ("ready",): + continue + except EOFError: + pass + self.close() + raise SystemExit( + f"Context-parallel worker {process.name} exited with code {process.exitcode} while loading " + "the pipeline. Its error is in the log above." + ) + + def _monitor(self) -> None: + sentinels = {process.sentinel: process for process in self._processes} + dead = sentinels[wait(list(sentinels))[0]] + if self._closing: + return + # `exitcode` polls without waiting, so it can still be None just after the sentinel fires. + dead.join() + self._failure = f"context-parallel worker {dead.name} exited with code {dead.exitcode}" + self._failed.set() + logger.error(f"{self._failure}; the remaining workers are stopped and the server no longer generates.") + # The other ranks would wait forever in their next collective, so the group is not reused. + self.close() + + def failure(self) -> str | None: + return self._failure + + def generate(self, generation: GenerationJob) -> dict[str, list[dict[str, Any]]]: + if self._failure is not None: + raise GMSError("internal", self._failure) + job = { + "request": generation.request, + "kwargs": load_request_media(self._manifest, generation.request), + "directory": str(generation.directory), + } + self._cancel_flag.value = 0 + for conn in self._conns: + conn.send(job) + + rank_0 = self._conns[0] + while True: + if generation.cancel_requested: + self._cancel_flag.value = 1 + try: + if not rank_0.poll(_POLL_SECONDS): + if self._failure is not None: + raise GMSError("internal", self._failure) + continue + message = rank_0.recv() + except (EOFError, OSError) as e: + # Rank 0's pipe only breaks once it has exited. Only the monitor reaps workers: a second + # thread joining the same process can read its exit code before it is recorded. + self._failed.wait() + raise GMSError("internal", self._failure) from e + if message[0] == "progress": + generation.report_progress(*message[1:]) + elif message[0] == "complete": + return message[1] + else: + _, code, text, pointer = message + raise GMSError(code, text, pointer) + + def close(self) -> None: + self._closing = True + for conn in self._conns: + try: + conn.send(None) + except OSError: + pass + for process in self._processes: + process.join(timeout=_SHUTDOWN_SECONDS) + if process.is_alive(): + process.kill() + process.join() diff --git a/src/diffusers/commands/serve/manifest.py b/src/diffusers/commands/serve/manifest.py index ab8c62835ef2..6323df256df5 100644 --- a/src/diffusers/commands/serve/manifest.py +++ b/src/diffusers/commands/serve/manifest.py @@ -26,6 +26,7 @@ import json import math import operator +import random import re from dataclasses import dataclass, field from pathlib import Path @@ -416,6 +417,11 @@ def __init__( self.adapters = data.get("adapters", {}) self._kwarg_names = set(signature.parameters) - {"self"} self._pipeline_name = pipeline_cls.__name__ + self._has_default_fps = any( + signature.parameters[kwarg].default not in (None, inspect.Parameter.empty) + for kwarg in _PARAMETER_KWARGS["fps"] + if kwarg in signature.parameters + ) for capability in self.capabilities: if capability not in supported_capabilities: @@ -470,6 +476,15 @@ def _check_task(self, name: str, task: dict[str, Any]) -> None: f"The GMS manifest for {self.id!r} declares `duration` on task {name!r} without a frame lattice, so " 'seconds cannot be converted to frames. Add one, e.g. "lattice": {"frames": "8k+1", "fps": 24}.' ) + produces_video = any(spec["type"] == "video" for spec in task["outputs"].values()) + if produces_video and not self._has_default_fps and "duration" not in parameters: + raise SystemExit( + f"The GMS manifest for {self.id!r} declares a video output on task {name!r}, but " + f"{self._pipeline_name} has no default `fps` or `frame_rate` argument, so the frame rate to encode " + "at is unknown. Declare `duration` with a frame lattice at the model's native fps, e.g. " + '"duration": {"type": "number", "lattice": {"frames": "4k+1", "fps": 16}}, and pass the manifest ' + "with --manifest." + ) for preset, body in task.get("presets", {}).items(): for key in body.get("parameters", {}): if key not in parameters: @@ -506,16 +521,19 @@ def discovery_document(self) -> bytes: head["hash"] = self.hash return json.dumps(head)[:-1].encode() + b', "manifest": ' + self.raw + b"}" - def seeded(self, task: str) -> bool: - return "seed" in self._parameter_kwargs[task] + def resolve(self, body: Any, seeds: random.Random | None = None) -> ResolvedRequest: + """Validate a create-generation request body and map it to the pipeline call. - def resolve(self, body: Any) -> ResolvedRequest: - """Validate a create-generation request body and map it to the pipeline call.""" + A request that omits the seed gets one drawn from `seeds`. + """ inputs = self._check_body(body) task = self._select_task(body.get("task"), inputs) values, explicit = self._parameter_values(task, body) input_kwargs, media = self._resolve_inputs(task, inputs, values) parameter_kwargs, seed = self._resolve_parameters(task, values, explicit) + # The spec has the server pick the seed when the request omits it, so the result is reproducible. + if seed is None and "seed" in self._parameter_kwargs[task]: + seed = (seeds or random.Random()).randrange(2**32) output_formats = self._resolve_output_formats(task, body.get("output_format")) response_format = body.get("response_format", "url") diff --git a/src/diffusers/commands/serve/parallel.py b/src/diffusers/commands/serve/parallel.py deleted file mode 100644 index 77dc25c7de9a..000000000000 --- a/src/diffusers/commands/serve/parallel.py +++ /dev/null @@ -1,235 +0,0 @@ -# Copyright 2026 The HuggingFace Team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""`diffusers-cli serve --context-parallel` — one worker process per GPU, all running every generation together. - -The server process never touches CUDA. It spawns the workers, sends each generation to all of them over a pipe, and -reads the progress and result back from rank 0. Idle workers wait on their pipe rather than inside a collective, so -the process group cannot time out between requests. -""" - -from __future__ import annotations - -import multiprocessing -import os -import signal -import socket -import threading -from argparse import Namespace -from multiprocessing.connection import Connection, wait -from pathlib import Path -from typing import Any - -from ...utils import logging -from .backends import DiffusersBackend, load_request_media -from .manifest import GMSError, Manifest -from .scheduler import GenerationJob - - -logger = logging.get_logger("diffusers-cli/serve") - -_POLL_SECONDS = 0.1 -_SHUTDOWN_SECONDS = 30.0 - - -class _WorkerJob: - """The parts of a generation `DiffusersBackend.run` reads, inside a worker. - - `cancel_requested` is a collective: rank 0 reads the flag the server sets and broadcasts it, so every rank - interrupts at the same step. A rank that stopped alone would leave the others waiting in the next collective. - """ - - def __init__(self, job: dict[str, Any], rank: int, conn: Connection, cancel_flag: Any): - self.request = job["request"] - self.seed = job["seed"] - self.directory = Path(job["directory"]) - self._rank = rank - self._conn = conn - self._cancel_flag = cancel_flag - - def report_progress(self, step: int, total_steps: int) -> None: - if self._rank == 0: - self._conn.send(("progress", step, total_steps)) - - @property - def cancel_requested(self) -> bool: - import torch - - flag = torch.tensor([self._cancel_flag.value if self._rank == 0 else 0], dtype=torch.uint8) - torch.distributed.broadcast(flag, src=0) - return bool(flag.item()) - - -def _serve_jobs(backend: Any, rank: int, conn: Connection, cancel_flag: Any) -> None: - while True: - job = conn.recv() - if job is None: - return - generation = _WorkerJob(job, rank, conn, cancel_flag) - try: - reply = ("complete", backend.run(job["kwargs"], generation)) - except GMSError as e: - reply = ("failed", e.code, e.message, e.pointer) - # The helpers shared with `run` raise SystemExit, which must not end the worker. - except (Exception, SystemExit) as e: - logger.error(f"generation failed on rank {rank}", exc_info=True) - reply = ("failed", "internal", f"{type(e).__name__}: {e}", None) - if rank == 0: - conn.send(reply) - - -def _worker_main( - rank: int, world_size: int, port: int, args: Namespace, manifest: Manifest, conn: Connection, cancel_flag: Any -) -> None: - import torch - - # The server process decides when workers stop, so Ctrl-C in the terminal must not interrupt them mid-call. - signal.signal(signal.SIGINT, signal.SIG_IGN) - os.environ.update( - MASTER_ADDR="127.0.0.1", - MASTER_PORT=str(port), - RANK=str(rank), - LOCAL_RANK=str(rank), - WORLD_SIZE=str(world_size), - ) - if args.log_level is not None: - logging.set_verbosity(logging.get_log_levels_dict()[args.log_level]) - # Loading applies `--context-parallel`, which joins the process group of all ranks. - backend = DiffusersBackend(args, manifest, save_outputs=rank == 0) - conn.send(("ready",)) - try: - _serve_jobs(backend, rank, conn, cancel_flag) - finally: - torch.distributed.destroy_process_group() - - -class ParallelBackend: - def __init__(self, args: Namespace, manifest: Manifest): - import torch - - world_size = torch.cuda.device_count() - if world_size < 2: - raise SystemExit(f"--context-parallel needs at least 2 visible GPUs, found {world_size}.") - with socket.socket() as probe: - probe.bind(("127.0.0.1", 0)) - port = probe.getsockname()[1] - - self._manifest = manifest - self._failure: str | None = None - self._failed = threading.Event() - self._closing = False - # Spawned rather than forked: a forked child inherits whatever CUDA state its parent has. - context = multiprocessing.get_context("spawn") - # Lock-free: a worker killed while holding the lock would leave the server blocked on its next write. - self._cancel_flag = context.Value("b", 0, lock=False) - self._conns: list[Connection] = [] - self._processes = [] - for rank in range(world_size): - parent_conn, child_conn = context.Pipe() - process = context.Process( - target=_worker_main, - args=(rank, world_size, port, args, manifest, child_conn, self._cancel_flag), - name=f"diffusers-serve-rank-{rank}", - daemon=True, - ) - process.start() - child_conn.close() - self._conns.append(parent_conn) - self._processes.append(process) - - logger.info(f"loading the pipeline on {world_size} GPUs...") - self._wait_until_ready() - threading.Thread(target=self._monitor, name="diffusers-serve-workers", daemon=True).start() - - def _wait_until_ready(self) -> None: - waiting = dict(zip(self._conns, self._processes)) - while waiting: - sentinels = {process.sentinel: process for process in waiting.values()} - for ready in wait([*waiting, *sentinels]): - process = waiting.pop(ready, None) if isinstance(ready, Connection) else sentinels[ready] - try: - if isinstance(ready, Connection) and ready.recv() == ("ready",): - continue - except EOFError: - pass - self.close() - raise SystemExit( - f"Context-parallel worker {process.name} exited with code {process.exitcode} while loading " - "the pipeline. Its error is in the log above." - ) - - def _monitor(self) -> None: - sentinels = {process.sentinel: process for process in self._processes} - dead = sentinels[wait(list(sentinels))[0]] - if self._closing: - return - # `exitcode` polls without waiting, so it can still be None just after the sentinel fires. - dead.join() - self._failure = f"context-parallel worker {dead.name} exited with code {dead.exitcode}" - self._failed.set() - logger.error(f"{self._failure}; the remaining workers are stopped and the server no longer generates.") - # The other ranks would wait forever in their next collective, so the group is not reused. - self.close() - - def failure(self) -> str | None: - return self._failure - - def generate(self, generation: GenerationJob) -> dict[str, list[dict[str, Any]]]: - if self._failure is not None: - raise GMSError("internal", self._failure) - job = { - "request": generation.request, - "kwargs": load_request_media(self._manifest, generation.request), - "seed": generation.seed, - "directory": str(generation.directory), - } - self._cancel_flag.value = 0 - for conn in self._conns: - conn.send(job) - - rank_0 = self._conns[0] - while True: - if generation.cancel_requested: - self._cancel_flag.value = 1 - try: - if not rank_0.poll(_POLL_SECONDS): - if self._failure is not None: - raise GMSError("internal", self._failure) - continue - message = rank_0.recv() - except (EOFError, OSError) as e: - # Rank 0's pipe only breaks once it has exited. Only the monitor reaps workers: a second - # thread joining the same process can read its exit code before it is recorded. - self._failed.wait() - raise GMSError("internal", self._failure) from e - if message[0] == "progress": - generation.report_progress(*message[1:]) - elif message[0] == "complete": - return message[1] - else: - _, code, text, pointer = message - raise GMSError(code, text, pointer) - - def close(self) -> None: - self._closing = True - for conn in self._conns: - try: - conn.send(None) - except OSError: - pass - for process in self._processes: - process.join(timeout=_SHUTDOWN_SECONDS) - if process.is_alive(): - process.kill() - process.join() diff --git a/src/diffusers/commands/serve/scheduler.py b/src/diffusers/commands/serve/scheduler.py index 4508f3114125..e2867c87042e 100644 --- a/src/diffusers/commands/serve/scheduler.py +++ b/src/diffusers/commands/serve/scheduler.py @@ -12,12 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Generation jobs, and the scheduler that runs them through the backend in arrival order, one at a time.""" +"""Generation jobs, and the scheduler that runs them in arrival order, one at a time on each backend.""" from __future__ import annotations import queue -import secrets import shutil import threading import time @@ -38,26 +37,11 @@ class GenerationJob: """One generation, from the request that created it to its outputs: the resource `/v1/generations/{id}` returns.""" - def __init__( - self, - model: str, - revision: str | None, - request: ResolvedRequest, - seeded: bool, - default_seed: int | None = None, - ): + def __init__(self, model: str, revision: str | None, request: ResolvedRequest): self.id = f"gen_{uuid.uuid4().hex[:12]}" self.model = model self.revision = revision self.request = request - # The spec has the server pick the seed when the request omits it, so the result is reproducible. - self.seed = None - if seeded and request.seed is not None: - self.seed = request.seed - elif seeded and default_seed is not None: - self.seed = default_seed - elif seeded: - self.seed = secrets.randbelow(2**32) self.status = "queued" self.created_at = datetime.now(timezone.utc) self.expires_at: datetime | None = None @@ -104,25 +88,29 @@ def _emit(self, event: str, data: dict[str, Any] | None) -> None: class Scheduler: - """Runs generation jobs through the backend on a single worker thread, first in, first out. + """Runs generation jobs first in, first out, on one worker thread per backend. It also keeps every job until it expires, so the API can look jobs up, cancel them and delete them. """ - def __init__(self, backend: Any, output_dir: str): - self._backend = backend + def __init__(self, backends: list[Any], output_dir: str): + self._backends = backends self._output_dir = Path(output_dir) self._generations: dict[str, GenerationJob] = {} self._lock = threading.Lock() self._pending: queue.Queue[GenerationJob] = queue.Queue() - self._running = False - threading.Thread(target=self._work, name="diffusers-serve-worker", daemon=True).start() + self._running = 0 + # Every worker takes the next job from the same queue, so no job waits behind a long one on a busy backend. + for index, backend in enumerate(backends): + threading.Thread( + target=self._work, args=(backend,), name=f"diffusers-serve-worker-{index}", daemon=True + ).start() def failure(self) -> str | None: - return self._backend.failure() + return next(filter(None, (backend.failure() for backend in self._backends)), None) def busy(self) -> bool: - return self._running or not self._pending.empty() + return self._running > 0 or not self._pending.empty() def submit(self, generation: GenerationJob) -> None: self._prune() @@ -169,31 +157,33 @@ def _remove_files(self, generation: GenerationJob) -> None: if generation.directory is not None: shutil.rmtree(generation.directory, ignore_errors=True) - def _work(self) -> None: + def _work(self, backend: Any) -> None: while True: generation = self._pending.get() with self._lock: if generation.done: continue generation.status = "generating" - self._running = True + self._running += 1 generation.directory = self._output_dir / generation.id started = time.perf_counter() error = None try: generation.directory.mkdir(parents=True, exist_ok=True) - outputs = self._backend.generate(generation) + outputs = backend.generate(generation) except GMSError as e: error = e - # The helpers shared with `run` raise SystemExit, and anything escaping here would end the - # only worker and leave every later generation queued. + # The helpers shared with `run` raise SystemExit, and anything escaping here would end this + # worker and leave its backend idle for good. except BaseException as e: logger.error(f"generation {generation.id} failed", exc_info=True) error = GMSError("internal", f"{type(e).__name__}: {e}") with self._lock: deleted = generation.id not in self._generations + # Before `finish` wakes any waiter, so a finished job is never reported while the server looks busy. + self._running -= 1 if error is not None: generation.finish("failed", error) elif generation.cancel_requested: @@ -202,6 +192,5 @@ def _work(self) -> None: generation.outputs = outputs generation.usage = {"compute_seconds": round(time.perf_counter() - started, 2)} generation.finish("complete") - self._running = False if deleted or generation.status != "complete": self._remove_files(generation) diff --git a/tests/others/test_cli_serve.py b/tests/others/test_cli_serve.py index 97e4922ceb42..0e52de3fcaea 100644 --- a/tests/others/test_cli_serve.py +++ b/tests/others/test_cli_serve.py @@ -37,7 +37,7 @@ from diffusers.commands.serve import ServeCommand, _shut_down_when_idle from diffusers.commands.serve.backends import DiffusersBackend, backend_capabilities from diffusers.commands.serve.manifest import GMSError, Manifest, blocks_signature, derive_manifest -from diffusers.commands.serve.scheduler import Scheduler +from diffusers.commands.serve.scheduler import GenerationJob, Scheduler from diffusers.modular_pipelines import InputParam, ModularPipelineBlocks, OutputParam @@ -149,8 +149,6 @@ def _args(**overrides): "compile": None, "lora": None, "context_parallel": False, - "fps": 8, - "sampling_rate": None, } return Namespace(**{**values, **overrides}) @@ -180,7 +178,7 @@ def _build(manifest=None, app_options=None, pipeline=None, signature=None, backe args = _args(**arg_overrides) bound = _manifest(manifest=manifest, pipeline_cls=type(pipeline), signature=signature, args=args) backend = backend or DiffusersBackend(args, bound) - app = create_app(bound, Scheduler(backend, str(tmp_path)), **(app_options or {})) + app = create_app(bound, Scheduler([backend], str(tmp_path)), **(app_options or {})) return TestClient(app), pipeline return _build @@ -229,6 +227,13 @@ def __call__( pass +class FakeAudioVideoPipeline(FakePipeline): + vocoder = SimpleNamespace(config=SimpleNamespace(sampling_rate=24000)) + + def __call__(self, prompt: str = None, frame_rate: float = 24.0): + return self.result + + class TestManifest: def test_derive_manifest_from_signature(self): manifest = derive_manifest(MODEL_ID, FakePipeline, FAKE_SIGNATURE, ["sse", "wait", "adapters"]) @@ -426,7 +431,7 @@ def test_joint_video_and_audio_outputs(self, serve, monkeypatch): }, "capabilities": ["wait"], } - client, pipeline = serve(manifest=manifest) + client, pipeline = serve(manifest=manifest, pipeline=FakeAudioVideoPipeline()) pipeline.result = SimpleNamespace( frames=[[Image.new("RGB", (32, 24)) for _ in range(16)]], audio=np.zeros((1, 2, 8000), dtype=np.float32), @@ -435,20 +440,20 @@ def test_joint_video_and_audio_outputs(self, serve, monkeypatch): generation = client.post("/v1/generations", json=body, headers=WAIT).json() assert generation["status"] == "complete", generation - # The fake pipeline takes no fps argument, so the video is encoded at the `--fps` default. + # Video is encoded at the pipeline's default `frame_rate`, audio at its vocoder's sample rate. (video,) = generation["outputs"]["video"] (audio,) = generation["outputs"]["audio"] assert {k: v for k, v in video.items() if k != "url"} == { "media_type": "video/mp4", "width": 32, "height": 24, - "duration": 2.0, - "fps": 8, + "duration": 0.67, + "fps": 24.0, } assert {k: v for k, v in audio.items() if k != "url"} == { "media_type": "audio/wav", - "duration": 0.5, - "sample_rate": 16000, + "duration": 0.33, + "sample_rate": 24000, } assert client.get(audio["url"]).content[:4] == b"RIFF" @@ -687,6 +692,41 @@ def test_failed_job_is_reported(self, monkeypatch): self._run(monkeypatch, FakeJobsApi(stage="ERROR"), []) +class BlockingBackend: + def __init__(self): + self.started = threading.Event() + self.release = threading.Event() + + def failure(self): + return None + + def generate(self, generation): + self.started.set() + self.release.wait(timeout=30) + return {} + + +class TestScheduler: + def test_each_backend_runs_its_own_generation(self, tmp_path): + backends = [BlockingBackend(), BlockingBackend()] + scheduler = Scheduler(backends, str(tmp_path)) + request = _manifest().resolve({"model": MODEL_ID, "inputs": {"prompt": "a cat"}}) + jobs = [GenerationJob(MODEL_ID, None, request) for _ in backends] + for job in jobs: + scheduler.submit(job) + + for index, backend in enumerate(backends): + assert backend.started.wait(timeout=30), f"backend {index} never picked up a generation" + assert [job.status for job in jobs] == ["generating", "generating"], "both generations should run at once" + + for backend in backends: + backend.release.set() + for job in jobs: + job.wait(timeout=30) + assert [job.status for job in jobs] == ["complete", "complete"] + assert not scheduler.busy(), "the scheduler should be idle once every backend has finished" + + class FakeClock: """Stands in for the `time` module: `sleep` advances the clock instead of waiting.""" @@ -752,7 +792,7 @@ def _fake_worker_main(rank, world_size, port, args, manifest, conn, cancel_flag) job = conn.recv() if job is None: return - if job["seed"] == -1: + if job["request"].seed == -1: os._exit(4) if rank != 0: continue @@ -772,8 +812,7 @@ def _fake_worker_main(rank, world_size, port, args, manifest, conn, cancel_flag) class FakeGeneration: def __init__(self, prompt="a cat", seed=0): - self.request = SimpleNamespace(kwargs={"prompt": prompt}, media={}, task="text_to_image") - self.seed = seed + self.request = SimpleNamespace(kwargs={"prompt": prompt}, media={}, task="text_to_image", seed=seed) self.directory = Path("/tmp/unused") self.cancel_requested = False self.progress = [] @@ -789,23 +828,23 @@ class TestParallelBackend: def backend(self, monkeypatch): import multiprocessing - from diffusers.commands.serve import parallel + from diffusers.commands.serve import backends if "fork" not in multiprocessing.get_all_start_methods(): pytest.skip("the fake workers are forked so they can use a test-local target") fork = multiprocessing.get_context("fork") - monkeypatch.setattr(parallel.multiprocessing, "get_context", lambda method: fork) - monkeypatch.setattr(parallel, "_worker_main", _fake_worker_main) + monkeypatch.setattr(backends.multiprocessing, "get_context", lambda method: fork) + monkeypatch.setattr(backends, "_worker_main", _fake_worker_main) monkeypatch.setattr(torch.cuda, "device_count", lambda: 2) - backends = [] + built = [] def _build(**arg_overrides): args = Namespace(**{"fail_at_startup": False, **arg_overrides}) - backends.append(parallel.ParallelBackend(args, _manifest())) - return backends[-1] + built.append(backends.ParallelBackend(args, _manifest())) + return built[-1] yield _build - for backend in backends: + for backend in built: backend.close() def test_generation_runs_on_every_rank_and_rank_0_answers(self, backend): @@ -850,7 +889,7 @@ class TestParallelWorker: def _jobs(self, backend, rank, *jobs): from multiprocessing import Pipe, Value - from diffusers.commands.serve.parallel import _serve_jobs + from diffusers.commands.serve.backends import _serve_jobs server, worker = Pipe() for job in (*jobs, None): @@ -862,7 +901,7 @@ def _jobs(self, backend, rank, *jobs): return replies def _job(self, prompt): - return {"request": None, "kwargs": {"prompt": prompt}, "seed": 0, "directory": "/tmp/unused"} + return {"request": None, "kwargs": {"prompt": prompt}, "directory": "/tmp/unused"} def test_only_rank_0_reports_progress_and_results(self): def run(kwargs, generation): @@ -888,7 +927,7 @@ def run(kwargs, generation): def test_cancel_is_broadcast_from_rank_0(self, monkeypatch): from multiprocessing import Value - from diffusers.commands.serve.parallel import _WorkerJob + from diffusers.commands.serve.backends import _WorkerJob monkeypatch.setenv("MASTER_ADDR", "127.0.0.1") monkeypatch.setenv("MASTER_PORT", str(_free_port())) From 8e8f6c311a38465cd487f4f8a6066ba878a8e004 Mon Sep 17 00:00:00 2001 From: DN6 Date: Thu, 5 Nov 2026 15:16:05 +0530 Subject: [PATCH 13/21] Seed request seeds from the global RNG and build job responses inside create_app --- src/diffusers/commands/serve/__init__.py | 4 +- src/diffusers/commands/serve/app.py | 74 +++++++++++------------ src/diffusers/commands/serve/manifest.py | 9 +-- src/diffusers/commands/serve/scheduler.py | 4 +- tests/others/test_cli_serve.py | 2 +- 5 files changed, 42 insertions(+), 51 deletions(-) diff --git a/src/diffusers/commands/serve/__init__.py b/src/diffusers/commands/serve/__init__.py index 2b229bc03256..81db02ce2512 100644 --- a/src/diffusers/commands/serve/__init__.py +++ b/src/diffusers/commands/serve/__init__.py @@ -21,6 +21,7 @@ from __future__ import annotations import inspect +import random import secrets import threading import time @@ -275,11 +276,12 @@ def run(self) -> None: try: seed = args.seed if args.seed is not None else secrets.randbelow(2**32) + # Seeds the generator `Manifest.resolve` draws request seeds from, like vLLM's `--seed`. + random.seed(seed) scheduler = Scheduler(backends, args.output_dir) app = create_app( manifest, scheduler, - seed=seed, enable_cors=args.enable_cors, public_url=args.public_url, ) diff --git a/src/diffusers/commands/serve/app.py b/src/diffusers/commands/serve/app.py index f4d5ef4f5cef..5e6089d31967 100644 --- a/src/diffusers/commands/serve/app.py +++ b/src/diffusers/commands/serve/app.py @@ -19,7 +19,6 @@ import asyncio import base64 import json -import random import re import time import uuid @@ -60,39 +59,9 @@ def _format_output(job: GenerationJob, artifact: dict[str, Any], base_url: str) return entry -def _format_job_response(job: GenerationJob, base_url: str) -> dict[str, Any]: - """The response body for a generation: outputs are linked under `base_url`, or inlined as base64. - - Raises the generation's error once it failed, which the error handler turns into the error envelope. - """ - if job.error is not None: - raise job.error - - outputs = None - if job.status == "complete": - outputs = { - name: [_format_output(job, artifact, base_url) for artifact in artifacts] - for name, artifacts in job.outputs.items() - } - - return { - "id": job.id, - "model": job.model, - "revision": job.revision, - "status": job.status, - "created_at": _timestamp(job.created_at), - "progress": job.progress if job.status == "generating" else None, - "outputs": outputs, - "expires_at": _timestamp(job.expires_at), - "seed": job.request.seed if job.status == "complete" else None, - "usage": job.usage, - } - - def create_app( manifest: Manifest, scheduler: Scheduler, - seed: int | None = None, enable_cors: bool = False, public_url: str | None = None, ) -> FastAPI: @@ -101,11 +70,36 @@ def create_app( root_path = urlparse(public_url).path.rstrip("/") if public_url is not None else "" app = FastAPI(title="diffusers-cli serve", docs_url=None, redoc_url=None, openapi_url=None, root_path=root_path) app.state.last_request = time.monotonic() - seeds = random.Random(seed) - def base_url(request: Request) -> str: - # Output links must work for the client, so they are built on the address it reached the server at. - return (public_url or str(request.base_url)).rstrip("/") + def format_job_response(job: GenerationJob, request: Request) -> dict[str, Any]: + """The response body for a generation: outputs are linked under the server address, or inlined as base64. + + Raises the generation's error once it failed, which the error handler turns into the error envelope. + """ + if job.error is not None: + raise job.error + + outputs = None + if job.status == "complete": + # Output links must work for the client, so they are built on the address it reached the server at. + base_url = (public_url or str(request.base_url)).rstrip("/") + outputs = { + name: [_format_output(job, artifact, base_url) for artifact in artifacts] + for name, artifacts in job.outputs.items() + } + + return { + "id": job.id, + "model": manifest.id, + "revision": manifest.revision, + "status": job.status, + "created_at": _timestamp(job.created_at), + "progress": job.progress if job.status == "generating" else None, + "outputs": outputs, + "expires_at": _timestamp(job.expires_at), + "seed": job.request.seed if job.status == "complete" else None, + "usage": job.usage, + } @app.middleware("http") async def request_id(request: Request, call_next) -> Response: @@ -175,19 +169,19 @@ async def create_generation(request: Request) -> dict[str, Any]: body = await request.json() except json.JSONDecodeError as e: raise GMSError("invalid_request", f"request body is not valid JSON: {e}") from e - generation = GenerationJob(manifest.id, manifest.revision, manifest.resolve(body, seeds)) + generation = GenerationJob(manifest.resolve(body)) scheduler.submit(generation) wait = re.fullmatch(r"\s*wait=(\d+)\s*", request.headers.get("prefer", "")) # `Prefer` is a hint, so a server without the `wait` capability answers right away instead of failing. if wait is not None and "wait" in manifest.capabilities: await asyncio.to_thread(generation.wait, min(float(wait.group(1)), MAX_WAIT_SECONDS)) - return _format_job_response(generation, base_url(request)) + return format_job_response(generation, request) @app.get("/v1/generations/{generation_id}") async def get_generation(generation_id: str, request: Request) -> dict[str, Any]: generation = scheduler.get(generation_id) - return _format_job_response(generation, base_url(request)) + return format_job_response(generation, request) @app.delete("/v1/generations/{generation_id}", status_code=204) async def delete_generation(generation_id: str) -> Response: @@ -197,7 +191,7 @@ async def delete_generation(generation_id: str) -> Response: @app.post("/v1/generations/{generation_id}/cancel") async def cancel_generation(generation_id: str, request: Request) -> dict[str, Any]: generation = scheduler.cancel(generation_id) - return _format_job_response(generation, base_url(request)) + return format_job_response(generation, request) @app.get("/v1/generations/{generation_id}/events") async def get_generation_events(generation_id: str, request: Request) -> StreamingResponse: @@ -213,7 +207,7 @@ async def stream(): events, done = await asyncio.to_thread(generation.wait_for_events, index, _KEEP_ALIVE_SECONDS) for event, data in events: index += 1 - payload = _format_job_response(generation, base_url(request)) if event == "complete" else data + payload = format_job_response(generation, request) if event == "complete" else data yield f"id: {index}\nevent: {event}\ndata: {json.dumps(payload)}\n\n" if done: return diff --git a/src/diffusers/commands/serve/manifest.py b/src/diffusers/commands/serve/manifest.py index 6323df256df5..f972a5af9931 100644 --- a/src/diffusers/commands/serve/manifest.py +++ b/src/diffusers/commands/serve/manifest.py @@ -521,11 +521,8 @@ def discovery_document(self) -> bytes: head["hash"] = self.hash return json.dumps(head)[:-1].encode() + b', "manifest": ' + self.raw + b"}" - def resolve(self, body: Any, seeds: random.Random | None = None) -> ResolvedRequest: - """Validate a create-generation request body and map it to the pipeline call. - - A request that omits the seed gets one drawn from `seeds`. - """ + def resolve(self, body: Any) -> ResolvedRequest: + """Validate a create-generation request body and map it to the pipeline call.""" inputs = self._check_body(body) task = self._select_task(body.get("task"), inputs) values, explicit = self._parameter_values(task, body) @@ -533,7 +530,7 @@ def resolve(self, body: Any, seeds: random.Random | None = None) -> ResolvedRequ parameter_kwargs, seed = self._resolve_parameters(task, values, explicit) # The spec has the server pick the seed when the request omits it, so the result is reproducible. if seed is None and "seed" in self._parameter_kwargs[task]: - seed = (seeds or random.Random()).randrange(2**32) + seed = random.randrange(2**32) output_formats = self._resolve_output_formats(task, body.get("output_format")) response_format = body.get("response_format", "url") diff --git a/src/diffusers/commands/serve/scheduler.py b/src/diffusers/commands/serve/scheduler.py index e2867c87042e..d179092e83b7 100644 --- a/src/diffusers/commands/serve/scheduler.py +++ b/src/diffusers/commands/serve/scheduler.py @@ -37,10 +37,8 @@ class GenerationJob: """One generation, from the request that created it to its outputs: the resource `/v1/generations/{id}` returns.""" - def __init__(self, model: str, revision: str | None, request: ResolvedRequest): + def __init__(self, request: ResolvedRequest): self.id = f"gen_{uuid.uuid4().hex[:12]}" - self.model = model - self.revision = revision self.request = request self.status = "queued" self.created_at = datetime.now(timezone.utc) diff --git a/tests/others/test_cli_serve.py b/tests/others/test_cli_serve.py index 0e52de3fcaea..0245061934fd 100644 --- a/tests/others/test_cli_serve.py +++ b/tests/others/test_cli_serve.py @@ -711,7 +711,7 @@ def test_each_backend_runs_its_own_generation(self, tmp_path): backends = [BlockingBackend(), BlockingBackend()] scheduler = Scheduler(backends, str(tmp_path)) request = _manifest().resolve({"model": MODEL_ID, "inputs": {"prompt": "a cat"}}) - jobs = [GenerationJob(MODEL_ID, None, request) for _ in backends] + jobs = [GenerationJob(request) for _ in backends] for job in jobs: scheduler.submit(job) From c746c0390005bed3e348be411343e8d1f4dd806e Mon Sep 17 00:00:00 2001 From: DN6 Date: Thu, 5 Nov 2026 15:50:58 +0530 Subject: [PATCH 14/21] Resolve outputs, fps and adapter weights in the manifest so the backend no longer reads it per request --- src/diffusers/commands/serve/backends.py | 47 +++------ src/diffusers/commands/serve/manifest.py | 111 ++++++++++++---------- src/diffusers/commands/serve/scheduler.py | 11 +-- 3 files changed, 77 insertions(+), 92 deletions(-) diff --git a/src/diffusers/commands/serve/backends.py b/src/diffusers/commands/serve/backends.py index d9d3781c7f56..7ca27522ad5d 100644 --- a/src/diffusers/commands/serve/backends.py +++ b/src/diffusers/commands/serve/backends.py @@ -55,7 +55,7 @@ _load_pipeline, _save_audio_arrays, ) -from .manifest import _PARAMETER_KWARGS, GMSError, Manifest, _frame_lattice, blocks_signature +from .manifest import GMSError, Manifest, ResolvedRequest, blocks_signature from .scheduler import GenerationJob @@ -194,15 +194,15 @@ def _sample_rate(pipeline: Any) -> int: ) -def load_request_media(manifest: Manifest, request: Any) -> dict[str, Any]: +def load_request_media(request: ResolvedRequest) -> dict[str, Any]: """The pipeline kwargs of a request, with its media inputs fetched and decoded.""" kwargs = dict(request.kwargs) - for kwarg, (name, value) in request.media.items(): + for kwarg, (name, kind, value) in request.media.items(): if isinstance(value, list): loaded = [_load_media(name, item, f"/inputs/{name}/{index}") for index, item in enumerate(value)] else: loaded = [_load_media(name, value, f"/inputs/{name}")] - if manifest.tasks[request.task]["inputs"][name]["type"] == "audio": + if kind == "audio": if kwarg == "initial_audio_waveforms": kwargs.setdefault("initial_audio_sampling_rate", loaded[0][1]) loaded = [waveform for waveform, _ in loaded] @@ -212,7 +212,6 @@ def load_request_media(manifest: Manifest, request: Any) -> dict[str, Any]: class DiffusersBackend: def __init__(self, args: Namespace, manifest: Manifest, save_outputs: bool = True): - self._manifest = manifest # Under context parallelism every rank runs the call, and only rank 0 writes the files. self._save_outputs = save_outputs self.pipeline = _load_pipeline(args) @@ -221,11 +220,6 @@ def __init__(self, args: Namespace, manifest: Manifest, save_outputs: bool = Tru parameters = blocks_signature(self.pipeline.blocks).parameters else: parameters = inspect.signature(type(self.pipeline).__call__).parameters - self._defaults = { - name: parameter.default - for name, parameter in parameters.items() - if parameter.default is not inspect.Parameter.empty - } self._reports_steps = "callback_on_step_end" in parameters outputs = (spec["type"] for task in manifest.tasks.values() for spec in task["outputs"].values()) self._sample_rate = _sample_rate(self.pipeline) if "audio" in outputs else None @@ -237,7 +231,7 @@ def failure(self) -> str | None: return None def generate(self, generation: GenerationJob) -> dict[str, list[dict[str, Any]]]: - return self.run(load_request_media(self._manifest, generation.request), generation) + return self.run(load_request_media(generation.request), generation) def run(self, kwargs: dict[str, Any], generation: Any) -> dict[str, list[dict[str, Any]]]: """Call the pipeline with the request's kwargs, media already loaded, and save its outputs.""" @@ -270,8 +264,7 @@ def on_step_end(pipeline, step, timestep, callback_kwargs): return {} outputs = {} - for name, spec in self._manifest.tasks[request.task]["outputs"].items(): - kind = spec["type"] + for name, (kind, media_type) in request.outputs.items(): # A modular pipeline returns its `PipelineState`; a standard one returns an output object. if self._modular: fields = (result.get(field) for field in _OUTPUT_FIELDS.get(kind, ())) @@ -280,25 +273,16 @@ def on_step_end(pipeline, step, timestep, callback_kwargs): value = next((field for field in fields if field is not None), None) if value is None: continue - rate = self._fps(request.task, kwargs) if kind == "video" else self._sample_rate - outputs[name] = _SAVERS[kind](value, generation.directory, name, request.output_formats[kind], rate) + rate = request.fps if kind == "video" else self._sample_rate + outputs[name] = _SAVERS[kind](value, generation.directory, name, media_type, rate) if not outputs: - declared = list(self._manifest.tasks[request.task]["outputs"]) raise GMSError( "internal", - f"{type(self.pipeline).__name__} returned none of the outputs {declared} declared for task " - f"{request.task!r}", + f"{type(self.pipeline).__name__} returned none of the outputs {list(request.outputs)} declared for " + f"task {request.task!r}", ) return outputs - def _fps(self, task: str, kwargs: dict[str, Any]) -> float: - for kwarg in _PARAMETER_KWARGS["fps"]: - rate = kwargs.get(kwarg) or self._defaults.get(kwarg) - if rate: - return rate - # The manifest check at startup guarantees a frame lattice when the pipeline has no default fps. - return _frame_lattice(self._manifest.tasks[task]["parameters"]["duration"])[2] - def _load_adapters(self, adapters: list[dict[str, Any]]) -> None: if not adapters: return @@ -314,13 +298,7 @@ def _load_adapters(self, adapters: list[dict[str, Any]]) -> None: f"could not load adapter {adapter['path']!r}: {type(e).__name__}: {e}", f"/adapters/{index}/path", ) from e - scale = adapter.get("scale", 1.0) - targets = adapter.get("targets") - if targets: - valid_targets = self._manifest.adapters[adapter["type"]].get("targets", []) - weights.append({target: targets.get(target, scale) for target in valid_targets}) - else: - weights.append(scale) + weights.append(adapter["weight"]) names.append(name) self.pipeline.set_adapters(names, adapter_weights=weights) @@ -404,7 +382,6 @@ def __init__(self, args: Namespace, manifest: Manifest): probe.bind(("127.0.0.1", 0)) port = probe.getsockname()[1] - self._manifest = manifest self._failure: str | None = None self._failed = threading.Event() self._closing = False @@ -469,7 +446,7 @@ def generate(self, generation: GenerationJob) -> dict[str, list[dict[str, Any]]] raise GMSError("internal", self._failure) job = { "request": generation.request, - "kwargs": load_request_media(self._manifest, generation.request), + "kwargs": load_request_media(generation.request), "directory": str(generation.directory), } self._cancel_flag.value = 0 diff --git a/src/diffusers/commands/serve/manifest.py b/src/diffusers/commands/serve/manifest.py index f972a5af9931..a07cf7c1a788 100644 --- a/src/diffusers/commands/serve/manifest.py +++ b/src/diffusers/commands/serve/manifest.py @@ -142,10 +142,13 @@ def to_dict(self) -> dict[str, Any]: class ResolvedRequest: task: str kwargs: dict[str, Any] - # pipeline kwarg -> (request input name, media value or list of media values) - media: dict[str, tuple[str, Any]] + # pipeline kwarg -> (request input name, media type, media value or list of media values) + media: dict[str, tuple[str, str, Any]] seed: int | None - output_formats: dict[str, str] + # output name -> (output type, media type to encode it as) + outputs: dict[str, tuple[str, str]] + # The frame rate video outputs are encoded at, or None when the task produces no video. + fps: float | None response_format: str adapters: list[dict[str, Any]] = field(default_factory=list) @@ -417,15 +420,22 @@ def __init__( self.adapters = data.get("adapters", {}) self._kwarg_names = set(signature.parameters) - {"self"} self._pipeline_name = pipeline_cls.__name__ - self._has_default_fps = any( - signature.parameters[kwarg].default not in (None, inspect.Parameter.empty) - for kwarg in _PARAMETER_KWARGS["fps"] - if kwarg in signature.parameters - ) + fps_kwarg = self._pipeline_kwarg("fps", _PARAMETER_KWARGS, {}) + default_fps = signature.parameters[fps_kwarg].default if fps_kwarg is not None else None + self._default_fps = None if default_fps is inspect.Parameter.empty else default_fps for capability in self.capabilities: - if capability not in supported_capabilities: - raise SystemExit(self._unsupported_capability(capability)) + if capability in supported_capabilities: + continue + message = f"The GMS manifest for {self.id!r} declares the {capability!r} capability, " + if capability == "adapters": + raise SystemExit( + message + f"but request adapters are off: {self._pipeline_name} cannot load LoRAs, or the server " + "was started with --compile, --lora or --context-parallel." + ) + raise SystemExit( + message + f"which this server does not provide. It provides: {list(SUPPORTED_CAPABILITIES)}." + ) if "adapters" in self.capabilities and not self.adapters: raise SystemExit( f"The GMS manifest for {self.id!r} declares the `adapters` capability but no top-level `adapters` " @@ -435,6 +445,8 @@ def __init__( # Per task: GMS input or parameter name -> the pipeline kwarg it sets. self._input_kwargs: dict[str, dict[str, str]] = {} self._parameter_kwargs: dict[str, dict[str, str]] = {} + # Per task: the pipeline kwarg that sets the frame rate, if the pipeline takes one. + self._fps_kwargs: dict[str, str | None] = {} for name, task in self.tasks.items(): declared = task.get(EXTENSION_KEY, {}) self._input_kwargs[name] = { @@ -445,6 +457,7 @@ def __init__( key: self._require_kwarg(name, "parameter", key, _PARAMETER_KWARGS, declared.get("parameters", {})) for key in task.get("parameters", {}) } + self._fps_kwargs[name] = self._parameter_kwargs[name].get("fps", fps_kwarg) self._check_task(name, task) def _pipeline_kwarg(self, name: str, aliases: dict[str, tuple[str, ...]], declared: dict[str, str]) -> str | None: @@ -477,7 +490,7 @@ def _check_task(self, name: str, task: dict[str, Any]) -> None: 'seconds cannot be converted to frames. Add one, e.g. "lattice": {"frames": "8k+1", "fps": 24}.' ) produces_video = any(spec["type"] == "video" for spec in task["outputs"].values()) - if produces_video and not self._has_default_fps and "duration" not in parameters: + if produces_video and self._default_fps is None and "duration" not in parameters: raise SystemExit( f"The GMS manifest for {self.id!r} declares a video output on task {name!r}, but " f"{self._pipeline_name} has no default `fps` or `frame_rate` argument, so the frame rate to encode " @@ -493,19 +506,6 @@ def _check_task(self, name: str, task: dict[str, Any]) -> None: f"{name!r}, which the task does not declare." ) - def _unsupported_capability(self, capability: str) -> str: - message = f"The GMS manifest for {self.id!r} declares the {capability!r} capability, " - if capability == "adapters": - return message + ( - f"but request adapters are off: {self._pipeline_name} cannot load LoRAs, or the server was started " - "with --compile, --lora or --context-parallel." - ) - return message + f"which this server does not provide. It provides: {list(SUPPORTED_CAPABILITIES)}." - - @property - def hash(self) -> str: - return f"sha256:{hashlib.sha256(self.raw).hexdigest()}" - def summary(self) -> dict[str, Any]: return { "id": self.id, @@ -518,7 +518,7 @@ def discovery_document(self) -> bytes: head: dict[str, Any] = {"spec": SPEC_VERSION} if self.source is not None: head["source"] = self.source - head["hash"] = self.hash + head["hash"] = f"sha256:{hashlib.sha256(self.raw).hexdigest()}" return json.dumps(head)[:-1].encode() + b', "manifest": ' + self.raw + b"}" def resolve(self, body: Any) -> ResolvedRequest: @@ -531,7 +531,13 @@ def resolve(self, body: Any) -> ResolvedRequest: # The spec has the server pick the seed when the request omits it, so the result is reproducible. if seed is None and "seed" in self._parameter_kwargs[task]: seed = random.randrange(2**32) - output_formats = self._resolve_output_formats(task, body.get("output_format")) + kwargs = {**input_kwargs, **parameter_kwargs} + outputs = self._resolve_outputs(task, body.get("output_format")) + fps = None + if any(kind == "video" for kind, _ in outputs.values()): + # The startup check guarantees a frame lattice when the pipeline has no default frame rate. + duration = self.tasks[task].get("parameters", {}).get("duration") + fps = kwargs.get(self._fps_kwargs[task]) or self._default_fps or _frame_lattice(duration)[2] response_format = body.get("response_format", "url") if response_format not in ("url", "b64_json"): @@ -543,10 +549,11 @@ def resolve(self, body: Any) -> ResolvedRequest: return ResolvedRequest( task=task, - kwargs={**input_kwargs, **parameter_kwargs}, + kwargs=kwargs, media=media, seed=seed, - output_formats=output_formats, + outputs=outputs, + fps=fps, response_format=response_format, adapters=self._resolve_adapters(body.get("adapters")), ) @@ -615,17 +622,17 @@ def _parameter_values(self, task: str, body: dict[str, Any]) -> tuple[dict[str, f"preset {preset!r} is not declared for task {task!r}; declared presets: {list(presets)}", "/preset", ) - values.update(self.tasks[task]["presets"][preset].get("parameters", {})) + values.update(presets[preset].get("parameters", {})) values.update(explicit) return values, set(explicit) def _resolve_inputs( self, task: str, inputs: dict[str, Any], values: dict[str, Any] - ) -> tuple[dict[str, Any], dict[str, tuple[str, Any]]]: + ) -> tuple[dict[str, Any], dict[str, tuple[str, str, Any]]]: """Map the request inputs to pipeline kwargs. Text is passed as is; media is checked and left to load.""" declared = self.tasks[task]["inputs"] kwargs: dict[str, Any] = {} - media: dict[str, tuple[str, Any]] = {} + media: dict[str, tuple[str, str, Any]] = {} for key, value in inputs.items(): pointer = f"/inputs/{key}" spec = declared.get(key) @@ -653,7 +660,7 @@ def _resolve_inputs( _check_media(key, item, spec, f"{pointer}/{index}") else: _check_media(key, value, spec, pointer) - media[kwarg] = (key, value) + media[kwarg] = (key, spec["type"], value) for key, spec in declared.items(): if spec.get("required") and key not in inputs: @@ -693,30 +700,31 @@ def _resolve_parameters( "/parameters/fps", ) kwargs[kwarg] = _duration_to_frames(value, spec, pointer) - fps_kwarg = self._pipeline_kwarg("fps", _PARAMETER_KWARGS, {}) - if fps_kwarg is not None: - kwargs[fps_kwarg] = lattice_fps + if self._fps_kwargs[task] is not None: + kwargs[self._fps_kwargs[task]] = lattice_fps continue kwargs[kwarg] = value return kwargs, seed - def _resolve_output_formats(self, task: str, output_format: Any) -> dict[str, str]: + def _resolve_outputs(self, task: str, output_format: Any) -> dict[str, tuple[str, str]]: output_formats = {kind: formats[0] for kind, formats in OUTPUT_FORMATS.items()} - if output_format is None: - return output_formats - kind = output_format.split("/")[0] if isinstance(output_format, str) else None - produced = {spec["type"] for spec in self.tasks[task]["outputs"].values()} - if kind not in produced or output_format not in OUTPUT_FORMATS.get(kind, ()): - supported = sorted(f for k in produced for f in OUTPUT_FORMATS.get(k, ())) - raise GMSError( - "invalid_request", - f"output_format must be one of {supported} for task {task!r}, got {output_format!r}", - "/output_format", - ) - output_formats[kind] = output_format - return output_formats + if output_format is not None: + kind = output_format.split("/")[0] if isinstance(output_format, str) else None + produced = {spec["type"] for spec in self.tasks[task]["outputs"].values()} + if kind not in produced or output_format not in OUTPUT_FORMATS.get(kind, ()): + supported = sorted(f for k in produced for f in OUTPUT_FORMATS.get(k, ())) + raise GMSError( + "invalid_request", + f"output_format must be one of {supported} for task {task!r}, got {output_format!r}", + "/output_format", + ) + output_formats[kind] = output_format + return { + name: (spec["type"], output_formats[spec["type"]]) for name, spec in self.tasks[task]["outputs"].items() + } def _resolve_adapters(self, adapters: Any) -> list[dict[str, Any]]: + """Validate the request adapters, and resolve each to its path and the weight `set_adapters` takes.""" if adapters is None: return [] if "adapters" not in self.capabilities: @@ -727,6 +735,7 @@ def _resolve_adapters(self, adapters: Any) -> list[dict[str, Any]]: raise GMSError( "invalid_request", f"at most {MAX_ADAPTERS} adapters per request, got {len(adapters)}", "/adapters" ) + resolved = [] for index, adapter in enumerate(adapters): pointer = f"/adapters/{index}" if not isinstance(adapter, dict): @@ -768,7 +777,9 @@ def _resolve_adapters(self, adapters: Any) -> list[dict[str, Any]]: f"adapter target scale must be a number, got {target_scale!r}", f"{pointer}/targets/{target}", ) - return adapters + weight = {target: targets.get(target, scale) for target in valid_targets} if targets else scale + resolved.append({"path": path, "weight": weight}) + return resolved # --------------------------------------------------------------------------- diff --git a/src/diffusers/commands/serve/scheduler.py b/src/diffusers/commands/serve/scheduler.py index d179092e83b7..fb81dd9e628c 100644 --- a/src/diffusers/commands/serve/scheduler.py +++ b/src/diffusers/commands/serve/scheduler.py @@ -54,8 +54,10 @@ def __init__(self, request: ResolvedRequest): self._condition = threading.Condition() def report_progress(self, step: int, total_steps: int) -> None: - self.progress = {"step": step, "total_steps": total_steps, "fraction": round(step / total_steps, 4)} - self._emit("progress", self.progress) + with self._condition: + self.progress = {"step": step, "total_steps": total_steps, "fraction": round(step / total_steps, 4)} + self.events.append(("progress", self.progress)) + self._condition.notify_all() def wait_for_events(self, index: int, timeout: float) -> tuple[list[tuple[str, dict[str, Any] | None]], bool]: with self._condition: @@ -79,11 +81,6 @@ def finish(self, status: str, error: GMSError | None = None) -> None: self.done = True self._condition.notify_all() - def _emit(self, event: str, data: dict[str, Any] | None) -> None: - with self._condition: - self.events.append((event, data)) - self._condition.notify_all() - class Scheduler: """Runs generation jobs first in, first out, on one worker thread per backend. From 71798581d3e15f80b5af75e393cda1f00a7bd911 Mon Sep 17 00:00:00 2001 From: DN6 Date: Thu, 5 Nov 2026 16:18:34 +0530 Subject: [PATCH 15/21] Address review: validate adapter repo ids, add --generation-timeout --- docs/source/en/using-diffusers/cli.md | 13 +-- docs/source/en/using-diffusers/serve.md | 13 ++- src/diffusers/commands/run.py | 111 ++++++++-------------- src/diffusers/commands/serve/__init__.py | 24 ++++- src/diffusers/commands/serve/backends.py | 16 ++-- src/diffusers/commands/serve/manifest.py | 14 ++- src/diffusers/commands/serve/remote.py | 13 +-- src/diffusers/commands/serve/scheduler.py | 37 +++++++- tests/others/test_cli_commands.py | 32 ------- tests/others/test_cli_serve.py | 53 ++++++++++- 10 files changed, 179 insertions(+), 147 deletions(-) diff --git a/docs/source/en/using-diffusers/cli.md b/docs/source/en/using-diffusers/cli.md index 961c2a1ddbe8..7dd6de84ca68 100644 --- a/docs/source/en/using-diffusers/cli.md +++ b/docs/source/en/using-diffusers/cli.md @@ -132,15 +132,10 @@ Configure how the CLI loads model weights and custom pipeline code. - `--attention-backend` — Hub-hosted attention kernels, auto-downloaded on first use. Choices are `default` plus the Hub backends registered in Diffusers (for example `flash_hub`, `flash_varlen_hub`, `flash_4_hub`, `sage_hub`, `sage_blackwell_hub`). Run `diffusers-cli run --help` for the current list. It only supports Transformer-based pipelines only, and is ignored with a warning on legacy UNet pipelines. See [Attention backends](../optimization/attention_backends). - `--vae-tiling` / `--vae-slicing` — lower VAE decode VRAM. See [VAE tiling](../optimization/memory#vae-tiling) and [VAE slicing](../optimization/memory#vae-slicing). -- `--compile [JSON]` — compile denoiser modules and the VAE decoder with - [torch.compile](../optimization/fp16#torchcompile). The CLI prefers - [regional compilation](../optimization/fp16#regional-compilation) for denoisers with repeated blocks; the VAE - decoder is always compiled whole. Bare `--compile` uses `fullgraph=true`. A JSON object is forwarded to - `torch.compile`. Not supported with `--context-parallel`. -- `--compile-mode {regional,full}` — what `--compile` compiles. `regional` (default) compiles only the repeated - blocks, which keeps the first step fast. `full` compiles the whole denoiser. `torch.compile` modes that use CUDA - graphs (`max-autotune`, `reduce-overhead`) need `--compile-mode full`: repeated blocks share one compiled graph, - so with CUDA graphs each block's output is overwritten by the next block's run. +- `--compile [JSON]` — compile denoiser modules with [torch.compile](../optimization/fp16#torchcompile). The + CLI prefers [regional compilation](../optimization/fp16#regional-compilation) for modules with repeated + blocks. Bare `--compile` uses `fullgraph=true`. A JSON object is forwarded to `torch.compile`. Not supported + with `--context-parallel`. - `--context-parallel` — Ulysses-style context parallelism on a DiT-based pipeline. Locally requires torchrun, but under `--remote` the CLI wraps `torchrun --nproc-per-node=gpu` for you. See [Context parallelism](../training/distributed_inference#context-parallelism). diff --git a/docs/source/en/using-diffusers/serve.md b/docs/source/en/using-diffusers/serve.md index c9f68f454cc8..7d8a75f8afbd 100644 --- a/docs/source/en/using-diffusers/serve.md +++ b/docs/source/en/using-diffusers/serve.md @@ -34,7 +34,12 @@ curl localhost:8000/v1/generations -H 'Prefer: wait=120' -d '{ `serve` takes the same loading and optimization flags as [`diffusers-cli run`](./cli#run), and loads weights in bf16 unless `--dtype` says otherwise. The server listens on `127.0.0.1:8000` by default and has no authentication, -so keep it on a trusted network. Run `diffusers serve --help` for every option. +so keep it on a trusted network. It also downloads any http(s) URL a request passes as a media input, so a client +can make it fetch addresses on its own network. Run `diffusers serve --help` for every option. + +`--generation-timeout ` (for example `10m`) fails a generation that runs longer than that. The pipeline +is interrupted at its next step. A generation that is still running at twice the limit is stuck, for example in a +modular pipeline that has no step callbacks, and `GET /health` then returns `503`. `--idle-timeout ` (for example `30m`) shuts the server down once it has received no requests for that long and no generation is queued or running. Health checks don't count as requests. By default a local server @@ -149,6 +154,8 @@ its current one. diffusers serve black-forest-labs/FLUX.1-dev --data-parallel 4 ``` +`--compile` and `--device-map` are not available with `--data-parallel`. + `--context-parallel` splits each generation across every visible GPU with [context parallelism](../training/distributed_inference#context-parallelism). It needs a DiT-based pipeline and at least two GPUs. `serve` starts one worker process per GPU itself, so don't launch it with `torchrun`: @@ -179,8 +186,8 @@ curl /v1/generations -H 'Prefer: wait=120' -H "Authorization: Bearer $HF_TO -d '{"model": "black-forest-labs/FLUX.1-dev", "inputs": {"prompt": "a cat on the moon"}}' ``` -The Job ends when the server shuts down. `--idle-timeout` defaults to `30m`, and `--timeout` (default `24h`) caps -how long the Job runs even while it is busy. `--flavor`, `--image`, `--dependencies` and `--namespace` work as they +The Job ends when the server shuts down, after `--idle-timeout` (default `30m`) without requests. A Job that never +goes idle is stopped after 24 hours. `--flavor`, `--image`, `--dependencies` and `--namespace` work as they do for [`run --remote`](./cli#remote-execution---remote). With `--context-parallel`, pick a flavor with several GPUs. The model must be a Hub repo id. `--manifest` is not supported with `--remote`: publish the manifest as `gms.json` in the model repo instead. diff --git a/src/diffusers/commands/run.py b/src/diffusers/commands/run.py index 31f7cadccd18..d8c66578340c 100644 --- a/src/diffusers/commands/run.py +++ b/src/diffusers/commands/run.py @@ -63,15 +63,10 @@ ATTENTION_BACKEND_CHOICES = ("default", *sorted(b.value for b in _HUB_KERNELS_REGISTRY)) -COMPILE_MODE_CHOICES = ("regional", "full") # zlib level for PNG outputs. Pillow's default of 6 takes close to four times as long to encode a # 1024x1024 image for a file about 7% smaller; the output is lossless either way. PNG_COMPRESS_LEVEL = 1 -# `torch.compile` modes that replay CUDA graphs. Repeated blocks share one compiled graph, so each -# block's output is overwritten when the next block replays it. -_CUDA_GRAPH_COMPILE_MODES = ("max-autotune", "reduce-overhead") - # Kwarg keys whose string value gets auto-loaded before being passed to the pipeline call. # Images resolve via `diffusers.utils.load_image` → PIL.Image.Image; videos resolve via # `diffusers.utils.load_video` → list[PIL.Image.Image]. @@ -227,23 +222,13 @@ def _add_optimization_arguments(parser: ArgumentParser) -> None: default=None, metavar="JSON", help=( - "torch.compile every denoiser submodule on the pipeline, and the VAE decoder. Accepts an optional JSON " + "torch.compile every denoiser submodule on the pipeline. Accepts an optional JSON " 'object of kwargs forwarded to `torch.compile`, e.g. \'{"mode": "max-autotune", ' '"fullgraph": true}\'. Bare `--compile` uses `fullgraph=true`. Adds a one-time ' "compilation cost on the first step but speeds up every subsequent step — worth it " "for multi-step generation (50+ steps)." ), ) - parser.add_argument( - "--compile-mode", - choices=COMPILE_MODE_CHOICES, - default="regional", - help=( - "What --compile compiles. 'regional' (default) compiles only the repeated blocks of a denoiser " - "that declares them, which keeps the first step fast. 'full' compiles the whole denoiser, which " - "`torch.compile` modes that use CUDA graphs (max-autotune, reduce-overhead) require." - ), - ) def _add_output_arguments(parser: ArgumentParser) -> None: @@ -488,44 +473,26 @@ def _apply_optimizations(pipeline: Any, args: Namespace) -> None: if args.context_parallel: logger.warning("--compile is currently not supported with --context-parallel; skipping compile.") else: - _compile_denoiser(pipeline, args.compile, args.compile_mode) - _compile_vae_decoder(pipeline, args.compile) - - -def _parse_compile_spec(compile_spec: str) -> dict[str, Any]: - try: - compile_kwargs = json.loads(compile_spec) - except json.JSONDecodeError as e: - raise SystemExit(f"--compile must be valid JSON: {e}") from e - if not isinstance(compile_kwargs, dict): - raise SystemExit("--compile must decode to a JSON object.") - return compile_kwargs - - -def _compile_vae_decoder(pipeline: Any, compile_spec: str) -> None: - """Compile the decoder of the pipeline's VAE with the `--compile` kwargs. - - The decoder runs once per call, so it has no repeated blocks to compile regionally and is always compiled whole. - """ - decoder = getattr(getattr(pipeline, "vae", None), "decoder", None) - if not isinstance(decoder, torch.nn.Module): - return - pipeline.vae.decoder = torch.compile(decoder, **_parse_compile_spec(compile_spec)) + _compile_denoiser(pipeline, args.compile) -def _compile_denoiser(pipeline: Any, compile_spec: str, compile_mode: str = "regional") -> None: +def _compile_denoiser(pipeline: Any, compile_spec: str) -> None: """Compile every `transformer*` and `unet*` submodule on the pipeline. `compile_spec` is the raw JSON string from `--compile` (`"{}"` for bare flag). Decoded into kwargs and forwarded verbatim to the compile call. - With `compile_mode="regional"`, prefers regional compilation via `module.compile_repeated_blocks(**kwargs)` — only - compiles the repeated inner blocks (the bulk of the compute), much faster first-step latency than compiling the - whole module. Falls back to full `torch.compile` if the model doesn't expose `_repeated_blocks`. With - `compile_mode="full"`, always compiles the whole module. + Prefers regional compilation via `module.compile_repeated_blocks(**kwargs)` — only compiles the repeated inner + blocks (the bulk of the compute), much faster first-step latency than compiling the whole module. Falls back to + full `torch.compile` if the model doesn't expose `_repeated_blocks`. """ - compile_kwargs = _parse_compile_spec(compile_spec) + try: + compile_kwargs = json.loads(compile_spec) + except json.JSONDecodeError as e: + raise SystemExit(f"--compile must be valid JSON: {e}") from e + if not isinstance(compile_kwargs, dict): + raise SystemExit("--compile must decode to a JSON object.") for attr in dir(pipeline): if not any(attr.startswith(key) for key in _DENOISER_COMPONENT_KEYS): @@ -534,17 +501,11 @@ def _compile_denoiser(pipeline: Any, compile_spec: str, compile_mode: str = "reg if not isinstance(module, torch.nn.Module): continue - if compile_mode == "regional" and getattr(module, "_repeated_blocks", None): - if compile_kwargs.get("mode") in _CUDA_GRAPH_COMPILE_MODES: - raise SystemExit( - f"--compile mode {compile_kwargs['mode']!r} uses CUDA graphs, which fail when the repeated " - f"blocks of {type(module).__name__} are compiled separately. Add `--compile-mode full`, or " - "use mode 'max-autotune-no-cudagraphs'." - ) + if getattr(module, "_repeated_blocks", None): # Regional compile — only the repeated blocks. Mutates `module` in place. module.compile_repeated_blocks(**compile_kwargs) else: - # Full compile was asked for, or no regional metadata is declared. + # No regional metadata declared; fall back to compiling the whole module. setattr(pipeline, attr, torch.compile(module, **compile_kwargs)) @@ -586,9 +547,6 @@ def _load_lora(pipeline: Any, args: Namespace) -> None: def _load_pipeline(args: Namespace) -> Any: - if args.compile is None and args.compile_mode != "regional": - raise SystemExit(f"--compile-mode {args.compile_mode} only applies together with --compile.") - # Detect modular repos by trying the standard config; `ModularPipeline` repos ship # `modular_model_index.json` instead of `model_index.json`, so `load_config` OSErrors. # A repo can also ship a `model_index.json` whose `_class_name` is a modular pipeline @@ -803,6 +761,28 @@ def _as_audio_arrays(value: Any): return None +def _write_wav(audio: Any, sampling_rate: int, path: str | Path) -> None: + """Write one numpy audio array to a 16-bit PCM WAV at `sampling_rate` Hz.""" + data = np.asarray(audio) + if data.dtype.kind == "f": + data = (np.clip(data, -1.0, 1.0) * 32767).astype(np.int16) + else: + data = data.astype(np.int16) + if data.ndim == 1: + n_channels = 1 + else: + # Heuristic: shorter axis is channels (interleaved layout for `wave` is + # samples × channels, so transpose if needed). + if data.shape[0] < data.shape[-1]: + data = data.T + n_channels = data.shape[1] + with wave.open(str(path), "wb") as w: + w.setnchannels(n_channels) + w.setsampwidth(2) # 16-bit PCM + w.setframerate(sampling_rate) + w.writeframes(data.tobytes()) + + def _save_audio_arrays(audios, sampling_rate: int, args: Namespace) -> list[str]: """Write each numpy audio array to a 16-bit PCM WAV at `sampling_rate` Hz. @@ -812,24 +792,7 @@ def _save_audio_arrays(audios, sampling_rate: int, args: Namespace) -> list[str] paths = _resolve_output_paths(len(audios), args.output, ext="wav") saved: list[str] = [] for audio, path in zip(audios, paths): - data = np.asarray(audio) - if data.dtype.kind == "f": - data = (np.clip(data, -1.0, 1.0) * 32767).astype(np.int16) - else: - data = data.astype(np.int16) - if data.ndim == 1: - n_channels = 1 - else: - # Heuristic: shorter axis is channels (interleaved layout for `wave` is - # samples × channels, so transpose if needed). - if data.shape[0] < data.shape[-1]: - data = data.T - n_channels = data.shape[1] - with wave.open(str(path), "wb") as w: - w.setnchannels(n_channels) - w.setsampwidth(2) # 16-bit PCM - w.setframerate(sampling_rate) - w.writeframes(data.tobytes()) + _write_wav(audio, sampling_rate, path) saved.append(str(path)) return saved diff --git a/src/diffusers/commands/serve/__init__.py b/src/diffusers/commands/serve/__init__.py index 81db02ce2512..b44288fa42fb 100644 --- a/src/diffusers/commands/serve/__init__.py +++ b/src/diffusers/commands/serve/__init__.py @@ -180,6 +180,14 @@ def register_subcommand(subparsers: _SubParsersAction) -> None: default=None, help="Logging level for diffusers and the HTTP server. Defaults to the current diffusers verbosity.", ) + parser.add_argument( + "--generation-timeout", + default=None, + help=( + "Fail a generation that runs longer than this (e.g. 10m). It is interrupted at its next step; " + "/health reports a generation still running at twice the limit. Defaults to no limit." + ), + ) parser.add_argument( "--idle-timeout", default=None, @@ -205,11 +213,13 @@ def run(self) -> None: args.model_id = None if args.model is None: raise SystemExit("Pass the model to serve, e.g. `diffusers-cli serve black-forest-labs/FLUX.1-dev`.") - if args.idle_timeout is not None: + for flag, value in (("--idle-timeout", args.idle_timeout), ("--generation-timeout", args.generation_timeout)): + if value is None: + continue try: - _duration_to_seconds(args.idle_timeout) + _duration_to_seconds(value) except ValueError: - raise SystemExit(f"--idle-timeout {args.idle_timeout!r} is not a duration such as 30m or 2h.") + raise SystemExit(f"{flag} {value!r} is not a duration such as 30m or 2h.") if args.log_level is not None: logging.set_verbosity(logging.get_log_levels_dict()[args.log_level]) @@ -224,6 +234,11 @@ def run(self) -> None: "--data-parallel and --context-parallel cannot be combined: --context-parallel already uses " "every visible GPU for each generation." ) + if args.data_parallel > 1 and args.compile is not None: + raise SystemExit( + "--compile is not supported with --data-parallel yet: the copies of the pipeline would compile " + "concurrently on separate threads, which torch.compile does not support." + ) if args.data_parallel > 1 and args.device_map is not None: raise SystemExit( "--device-map does not apply with --data-parallel: each copy of the pipeline is loaded on its own GPU." @@ -278,7 +293,8 @@ def run(self) -> None: seed = args.seed if args.seed is not None else secrets.randbelow(2**32) # Seeds the generator `Manifest.resolve` draws request seeds from, like vLLM's `--seed`. random.seed(seed) - scheduler = Scheduler(backends, args.output_dir) + timeout = None if args.generation_timeout is None else _duration_to_seconds(args.generation_timeout) + scheduler = Scheduler(backends, args.output_dir, timeout=timeout) app = create_app( manifest, scheduler, diff --git a/src/diffusers/commands/serve/backends.py b/src/diffusers/commands/serve/backends.py index 7ca27522ad5d..de16a2cadda9 100644 --- a/src/diffusers/commands/serve/backends.py +++ b/src/diffusers/commands/serve/backends.py @@ -23,6 +23,7 @@ from __future__ import annotations import base64 +import contextlib import inspect import io import mimetypes @@ -53,7 +54,7 @@ _as_pil_list, _get_generator, _load_pipeline, - _save_audio_arrays, + _write_wav, ) from .manifest import GMSError, Manifest, ResolvedRequest, blocks_signature from .scheduler import GenerationJob @@ -165,16 +166,13 @@ def _save_audio(audios: Any, directory: Path, name: str, media_type: str, rate: if arrays is None: raise GMSError("internal", f"cannot encode output {name!r} as {media_type}: not an audio array") - staging = directory / f"{name}-staging" - paths = _save_audio_arrays(arrays, int(rate), Namespace(output=str(staging) + os.sep)) artifacts = [] - for index, path in enumerate(paths): + for index, audio in enumerate(arrays): filename = f"{name}-{index}.wav" - Path(path).rename(directory / filename) + _write_wav(audio, int(rate), directory / filename) with wave.open(str(directory / filename), "rb") as file: duration = round(file.getnframes() / file.getframerate(), 2) artifacts.append({"file": filename, "media_type": media_type, "duration": duration, "sample_rate": int(rate)}) - staging.rmdir() return artifacts @@ -236,8 +234,8 @@ def generate(self, generation: GenerationJob) -> dict[str, list[dict[str, Any]]] def run(self, kwargs: dict[str, Any], generation: Any) -> dict[str, list[dict[str, Any]]]: """Call the pipeline with the request's kwargs, media already loaded, and save its outputs.""" request = generation.request + device = self.pipeline.device if hasattr(self.pipeline, "device") else torch.device("cpu") if request.seed is not None: - device = self.pipeline.device if hasattr(self.pipeline, "device") else torch.device("cpu") # A CUDA generator must sit on the pipeline's own GPU, which is `cuda:1` and up with --data-parallel. kwargs["generator"] = _get_generator(request.seed, str(device) if device.type == "cuda" else device.type) @@ -255,7 +253,9 @@ def on_step_end(pipeline, step, timestep, callback_kwargs): try: self._load_adapters(request.adapters) - result = self.pipeline(**kwargs) + # Worker threads start on GPU 0, and with --data-parallel this pipeline may sit on another one. + with torch.cuda.device(device) if device.type == "cuda" else contextlib.nullcontext(): + result = self.pipeline(**kwargs) finally: if request.adapters: self.pipeline.unload_lora_weights() diff --git a/src/diffusers/commands/serve/manifest.py b/src/diffusers/commands/serve/manifest.py index a07cf7c1a788..8627a823b9ca 100644 --- a/src/diffusers/commands/serve/manifest.py +++ b/src/diffusers/commands/serve/manifest.py @@ -26,6 +26,7 @@ import json import math import operator +import os import random import re from dataclasses import dataclass, field @@ -34,6 +35,7 @@ from huggingface_hub import hf_hub_download from huggingface_hub.errors import EntryNotFoundError +from huggingface_hub.utils import HFValidationError, validate_repo_id from ...pipelines.auto_pipeline import ( AUTO_IMAGE2IMAGE_PIPELINES_MAPPING, @@ -115,7 +117,6 @@ "lte": operator.le, } _FRAME_LATTICE = re.compile(r"(\d+)k\+(\d+)") -_HF_ADAPTER_PATH = re.compile(r"hf:[\w.-]+/[\w.-]+") # --------------------------------------------------------------------------- @@ -750,8 +751,15 @@ def _resolve_adapters(self, adapters: Any) -> list[dict[str, Any]]: f"{pointer}/type", ) path = adapter.get("path") - # Anything but a repo id would reach `load_lora_weights` as a path on the server's own disk. - if not isinstance(path, str) or not _HF_ADAPTER_PATH.fullmatch(path): + repo_id = path.removeprefix("hf:") if isinstance(path, str) and path.startswith("hf:") else None + if repo_id is not None: + try: + validate_repo_id(repo_id) + except HFValidationError: + repo_id = None + # `load_lora_weights` prefers a local path over a Hub repo of the same name, so a repo id that also + # names a path on the server's disk is refused as well. + if repo_id is None or "/" not in repo_id or os.path.exists(repo_id): raise GMSError( "invalid_request", f"adapter path must be a Hub repo id of the form 'hf:/', got {path!r}", diff --git a/src/diffusers/commands/serve/remote.py b/src/diffusers/commands/serve/remote.py index aedeb56114a7..4eeeadd7ed49 100644 --- a/src/diffusers/commands/serve/remote.py +++ b/src/diffusers/commands/serve/remote.py @@ -38,6 +38,9 @@ JOB_PORT = 8000 _SERVE_DEPS = ("fastapi", "uvicorn") _DEFAULT_REMOTE_IDLE_TIMEOUT = "30m" +# The Job ends when the server shuts down after --idle-timeout. This cap only stops a server that never goes idle, +# so it can't run, and bill, without end. +_JOB_TIMEOUT = "24h" _READY_POLL_SECONDS = 5.0 # Flags the Job command sets itself. `_build_task_kwargs` already drops the flags that say how the Job is set up. @@ -71,14 +74,6 @@ def add_remote_arguments(parser: ArgumentParser) -> None: default=None, help=f"Job image for --remote. Defaults to {_DEFAULT_REMOTE_IMAGE!r}.", ) - parser.add_argument( - "--timeout", - default="24h", - help=( - "Hard limit on how long the Job runs with --remote, busy or not (e.g. 2h, 1d). A backstop for " - "--idle-timeout. Defaults to 24h." - ), - ) def _announce_when_ready(api: HfApi, job: Any, args: Namespace, stopped: threading.Event) -> None: @@ -153,7 +148,7 @@ def serve_remote(args: Namespace, task: str) -> None: env=env, secrets={"HF_TOKEN": hf_token} if hf_token else None, flavor=args.flavor, - timeout=args.timeout, + timeout=_JOB_TIMEOUT, expose=[JOB_PORT], namespace=args.namespace, ) diff --git a/src/diffusers/commands/serve/scheduler.py b/src/diffusers/commands/serve/scheduler.py index fb81dd9e628c..db82f190d329 100644 --- a/src/diffusers/commands/serve/scheduler.py +++ b/src/diffusers/commands/serve/scheduler.py @@ -48,6 +48,8 @@ def __init__(self, request: ResolvedRequest): self.usage: dict[str, Any] | None = None self.error: GMSError | None = None self.cancel_requested = False + self.timed_out = False + self.started_at: float | None = None self.directory: Path | None = None self.events: list[tuple[str, dict[str, Any] | None]] = [] self.done = False @@ -88,9 +90,10 @@ class Scheduler: It also keeps every job until it expires, so the API can look jobs up, cancel them and delete them. """ - def __init__(self, backends: list[Any], output_dir: str): + def __init__(self, backends: list[Any], output_dir: str, timeout: float | None = None): self._backends = backends self._output_dir = Path(output_dir) + self._timeout = timeout self._generations: dict[str, GenerationJob] = {} self._lock = threading.Lock() self._pending: queue.Queue[GenerationJob] = queue.Queue() @@ -102,7 +105,20 @@ def __init__(self, backends: list[Any], output_dir: str): ).start() def failure(self) -> str | None: - return next(filter(None, (backend.failure() for backend in self._backends)), None) + failure = next(filter(None, (backend.failure() for backend in self._backends)), None) + if failure is not None or self._timeout is None: + return failure + # A timed out generation stops at its next step. One still running long after that is stuck inside a step, + # or in a pipeline without step callbacks, and its worker takes no more jobs. + now = time.monotonic() + with self._lock: + for generation in self._generations.values(): + if generation.status == "generating" and now - generation.started_at > 2 * self._timeout: + return ( + f"generation {generation.id} has run for over twice the --generation-timeout of " + f"{self._timeout:g}s and did not stop when interrupted" + ) + return None def busy(self) -> bool: return self._running > 0 or not self._pending.empty() @@ -159,8 +175,14 @@ def _work(self, backend: Any) -> None: if generation.done: continue generation.status = "generating" + generation.started_at = time.monotonic() self._running += 1 generation.directory = self._output_dir / generation.id + timer = None + if self._timeout is not None: + timer = threading.Timer(self._timeout, self._time_out, args=(generation,)) + timer.daemon = True + timer.start() started = time.perf_counter() error = None @@ -174,6 +196,8 @@ def _work(self, backend: Any) -> None: except BaseException as e: logger.error(f"generation {generation.id} failed", exc_info=True) error = GMSError("internal", f"{type(e).__name__}: {e}") + if timer is not None: + timer.cancel() with self._lock: deleted = generation.id not in self._generations @@ -181,6 +205,9 @@ def _work(self, backend: Any) -> None: self._running -= 1 if error is not None: generation.finish("failed", error) + elif generation.timed_out: + message = f"generation exceeded the --generation-timeout of {self._timeout:g}s" + generation.finish("failed", GMSError("internal", message)) elif generation.cancel_requested: generation.finish("canceled") else: @@ -189,3 +216,9 @@ def _work(self, backend: Any) -> None: generation.finish("complete") if deleted or generation.status != "complete": self._remove_files(generation) + + def _time_out(self, generation: GenerationJob) -> None: + # Interrupts the pipeline at its next step, the same way a cancel does. + with self._lock: + generation.timed_out = True + generation.cancel_requested = True diff --git a/tests/others/test_cli_commands.py b/tests/others/test_cli_commands.py index c2a6cc27153f..cedccbcd3a0b 100644 --- a/tests/others/test_cli_commands.py +++ b/tests/others/test_cli_commands.py @@ -34,8 +34,6 @@ RunCommand, _build_task_kwargs, _collapse_frame_dirs, - _compile_denoiser, - _compile_vae_decoder, _download_outputs_from_sandbox, _kwargs_to_argv, _load_lora, @@ -244,36 +242,6 @@ def test_compile_arg(self): assert compiled_blocks assert all(m._compiled_call_impl is not None for m in compiled_blocks) - def test_compile_mode_full_compiles_the_whole_denoiser(self, monkeypatch): - class Denoiser(torch.nn.Module): - _repeated_blocks = ["Block"] - - def compile_repeated_blocks(self, **kwargs): - raise AssertionError("regional compilation must not run with compile_mode='full'") - - compiled = {} - monkeypatch.setattr(torch, "compile", lambda module, **kwargs: compiled.update(kwargs) or "compiled") - pipeline = SimpleNamespace(transformer=Denoiser()) - _compile_denoiser(pipeline, '{"mode": "max-autotune"}', "full") - assert pipeline.transformer == "compiled" - assert compiled == {"mode": "max-autotune"} - - def test_regional_compile_rejects_cuda_graph_modes(self): - class Denoiser(torch.nn.Module): - _repeated_blocks = ["Block"] - - pipeline = SimpleNamespace(transformer=Denoiser()) - with pytest.raises(SystemExit, match="uses CUDA graphs.*--compile-mode full"): - _compile_denoiser(pipeline, '{"mode": "max-autotune"}', "regional") - - def test_compile_vae_decoder_compiles_the_decoder(self, monkeypatch): - compiled = {} - monkeypatch.setattr(torch, "compile", lambda module, **kwargs: compiled.update(kwargs) or "compiled") - pipeline = SimpleNamespace(vae=SimpleNamespace(decoder=torch.nn.Conv2d(4, 3, 3))) - _compile_vae_decoder(pipeline, '{"fullgraph": true}') - assert pipeline.vae.decoder == "compiled" - assert compiled == {"fullgraph": True} - @require_torch_gpu # `--attention-backend` only exposes Hub-hosted kernels, all of which need `kernels>=0.12`. @require_kernels_version_greater_or_equal("0.12") diff --git a/tests/others/test_cli_serve.py b/tests/others/test_cli_serve.py index 0245061934fd..f26662afef13 100644 --- a/tests/others/test_cli_serve.py +++ b/tests/others/test_cli_serve.py @@ -34,6 +34,7 @@ import torch from PIL import Image +from diffusers.commands.run import _DEFAULT_REMOTE_IMAGE from diffusers.commands.serve import ServeCommand, _shut_down_when_idle from diffusers.commands.serve.backends import DiffusersBackend, backend_capabilities from diffusers.commands.serve.manifest import GMSError, Manifest, blocks_signature, derive_manifest @@ -294,13 +295,27 @@ def test_resolve_rejects_non_http_media_urls(self): bound.resolve(body) assert error.value.pointer == "/inputs/image/url", error.value.message - @pytest.mark.parametrize("path", ["hf:/etc/passwd", "hf:../loras/x", "/etc/passwd", "https://x/y.safetensors"]) + @pytest.mark.parametrize( + "path", + ["hf:/etc/passwd", "hf:../loras/x", "hf:../outputs", "hf:detail", "/etc/passwd", "https://x/y.safetensors"], + ) def test_resolve_rejects_adapter_paths_that_are_not_repo_ids(self, path): body = {"model": MODEL_ID, "inputs": {"prompt": "a cat"}, "adapters": [{"type": "lora", "path": path}]} with pytest.raises(GMSError) as error: _manifest().resolve(body) assert (error.value.code, error.value.pointer) == ("invalid_request", "/adapters/0/path"), error.value.message + def test_resolve_rejects_adapter_repo_ids_that_name_a_local_path(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + (tmp_path / "someone" / "detail").mkdir(parents=True) + body = { + "model": MODEL_ID, + "inputs": {"prompt": "a cat"}, + "adapters": [{"type": "lora", "path": "hf:someone/detail"}], + } + with pytest.raises(GMSError, match="adapter path"): + _manifest().resolve(body) + def test_resolve_maps_request_to_pipeline_kwargs(self): bound = _manifest(manifest=LATTICE_MANIFEST, pipeline_cls=FakeVideoPipeline) resolved = bound.resolve( @@ -653,7 +668,7 @@ def test_remote_starts_the_server_in_a_job(self, monkeypatch): self._run(monkeypatch, api, [*argv, "--dependencies", "extra-package"]) job = api.run_kwargs - assert (job["flavor"], job["image"]) == ("a10g-large", "diffusers/diffusers-cli-cuda:latest"), job + assert (job["flavor"], job["image"]) == ("a10g-large", _DEFAULT_REMOTE_IMAGE), job assert job["expose"] == [8000] and job["timeout"] == "24h", job assert job["secrets"] == {"HF_TOKEN": "hf_x"}, "gated models need the HF token inside the Job" @@ -670,7 +685,7 @@ def test_remote_starts_the_server_in_a_job(self, monkeypatch): assert (options["--host"], options["--port"]) == ("0.0.0.0", "8000"), options assert options["--public-url"] == "https://${JOB_ID}--8000.hf.jobs", options assert options["--idle-timeout"] == "30m", "a remote server should shut itself down when idle by default" - assert not {"--remote", "--flavor", "--timeout", "--dependencies"} & set(serve_argv), serve_argv + assert not {"--remote", "--flavor", "--dependencies"} & set(serve_argv), serve_argv assert not api.cancelled, "a Job that completed on its own needs no cancel" def test_local_manifest_is_rejected(self, tmp_path, monkeypatch): @@ -726,6 +741,38 @@ def test_each_backend_runs_its_own_generation(self, tmp_path): assert [job.status for job in jobs] == ["complete", "complete"] assert not scheduler.busy(), "the scheduler should be idle once every backend has finished" + def test_generation_past_the_timeout_fails(self, tmp_path): + class SteppingBackend: + """Stops at the next step once interrupted, like a pipeline's step callback.""" + + def failure(self): + return None + + def generate(self, generation): + while not generation.cancel_requested: + time.sleep(0.01) + return {} + + scheduler = Scheduler([SteppingBackend()], str(tmp_path), timeout=0.1) + job = GenerationJob(_manifest().resolve({"model": MODEL_ID, "inputs": {"prompt": "a cat"}})) + scheduler.submit(job) + job.wait(timeout=30) + assert job.status == "failed", job.status + assert "--generation-timeout" in job.error.message, job.error.message + + def test_generation_stuck_past_the_timeout_fails_health(self, tmp_path): + backend = BlockingBackend() + scheduler = Scheduler([backend], str(tmp_path), timeout=60) + job = GenerationJob(_manifest().resolve({"model": MODEL_ID, "inputs": {"prompt": "a cat"}})) + scheduler.submit(job) + assert backend.started.wait(timeout=30), "the worker never picked up the generation" + assert scheduler.failure() is None, "a generation within its time limit is healthy" + + # As if it had run for over twice the limit without reaching a step that could stop it. + job.started_at -= 121 + assert "did not stop when interrupted" in (scheduler.failure() or ""), scheduler.failure() + backend.release.set() + class FakeClock: """Stands in for the `time` module: `sleep` advances the clock instead of waiting.""" From 88303052c456329997329a25cea3c6f852e4d360 Mon Sep 17 00:00:00 2001 From: DN6 Date: Thu, 5 Nov 2026 16:40:48 +0530 Subject: [PATCH 16/21] Wait on the event loop, time out by deadline, and drop low-value serve tests --- docs/source/en/using-diffusers/serve.md | 4 +- src/diffusers/commands/run.py | 9 +-- src/diffusers/commands/serve/__init__.py | 4 +- src/diffusers/commands/serve/app.py | 5 +- src/diffusers/commands/serve/backends.py | 4 +- src/diffusers/commands/serve/scheduler.py | 98 ++++++++++++++--------- tests/others/test_cli_serve.py | 32 ++------ 7 files changed, 80 insertions(+), 76 deletions(-) diff --git a/docs/source/en/using-diffusers/serve.md b/docs/source/en/using-diffusers/serve.md index 7d8a75f8afbd..7968ad2ef371 100644 --- a/docs/source/en/using-diffusers/serve.md +++ b/docs/source/en/using-diffusers/serve.md @@ -38,8 +38,8 @@ so keep it on a trusted network. It also downloads any http(s) URL a request pas can make it fetch addresses on its own network. Run `diffusers serve --help` for every option. `--generation-timeout ` (for example `10m`) fails a generation that runs longer than that. The pipeline -is interrupted at its next step. A generation that is still running at twice the limit is stuck, for example in a -modular pipeline that has no step callbacks, and `GET /health` then returns `503`. +is interrupted at its next step. Modular pipelines have no step callbacks, so they run to the end and their result +is discarded. A generation that is still running at twice the limit is stuck, and `GET /health` then returns `503`. `--idle-timeout ` (for example `30m`) shuts the server down once it has received no requests for that long and no generation is queued or running. Health checks don't count as requests. By default a local server diff --git a/src/diffusers/commands/run.py b/src/diffusers/commands/run.py index d8c66578340c..89d2cb23bef5 100644 --- a/src/diffusers/commands/run.py +++ b/src/diffusers/commands/run.py @@ -63,9 +63,6 @@ ATTENTION_BACKEND_CHOICES = ("default", *sorted(b.value for b in _HUB_KERNELS_REGISTRY)) -# zlib level for PNG outputs. Pillow's default of 6 takes close to four times as long to encode a -# 1024x1024 image for a file about 7% smaller; the output is lossless either way. -PNG_COMPRESS_LEVEL = 1 # Kwarg keys whose string value gets auto-loaded before being passed to the pipeline call. # Images resolve via `diffusers.utils.load_image` → PIL.Image.Image; videos resolve via @@ -817,7 +814,7 @@ def _save_videos(videos: list[Any], args: Namespace) -> list[str]: arr = (np.clip(arr, 0.0, 1.0) * 255).round().astype(np.uint8) frame = Image.fromarray(arr) frame_path = frames_dir / f"{i:04d}.png" - frame.save(frame_path, compress_level=PNG_COMPRESS_LEVEL) + frame.save(frame_path) saved.append(str(frame_path)) try: export_to_video(frames, str(path), fps=args.fps) @@ -854,14 +851,14 @@ def _save_output(value: Any, args: Namespace) -> list[str]: for arr, path in zip(value, paths): if arr.dtype != np.uint8: arr = (np.clip(arr, 0.0, 1.0) * 255).round().astype(np.uint8) - Image.fromarray(arr).save(path, compress_level=PNG_COMPRESS_LEVEL) + Image.fromarray(arr).save(path) return [str(p) for p in paths] pil_images = _as_pil_list(value) if pil_images is not None: paths = _resolve_output_paths(len(pil_images), args.output, ext="png") for img, path in zip(pil_images, paths): - img.save(path, compress_level=PNG_COMPRESS_LEVEL) + img.save(path) return [str(p) for p in paths] frames = _as_frame_sequence(value) diff --git a/src/diffusers/commands/serve/__init__.py b/src/diffusers/commands/serve/__init__.py index b44288fa42fb..d9ad0e97ab88 100644 --- a/src/diffusers/commands/serve/__init__.py +++ b/src/diffusers/commands/serve/__init__.py @@ -217,9 +217,11 @@ def run(self) -> None: if value is None: continue try: - _duration_to_seconds(value) + seconds = _duration_to_seconds(value) except ValueError: raise SystemExit(f"{flag} {value!r} is not a duration such as 30m or 2h.") + if seconds <= 0: + raise SystemExit(f"{flag} must be a positive duration such as 30m or 2h, got {value!r}.") if args.log_level is not None: logging.set_verbosity(logging.get_log_levels_dict()[args.log_level]) diff --git a/src/diffusers/commands/serve/app.py b/src/diffusers/commands/serve/app.py index 5e6089d31967..d64afbdb4f97 100644 --- a/src/diffusers/commands/serve/app.py +++ b/src/diffusers/commands/serve/app.py @@ -16,7 +16,6 @@ from __future__ import annotations -import asyncio import base64 import json import re @@ -175,7 +174,7 @@ async def create_generation(request: Request) -> dict[str, Any]: wait = re.fullmatch(r"\s*wait=(\d+)\s*", request.headers.get("prefer", "")) # `Prefer` is a hint, so a server without the `wait` capability answers right away instead of failing. if wait is not None and "wait" in manifest.capabilities: - await asyncio.to_thread(generation.wait, min(float(wait.group(1)), MAX_WAIT_SECONDS)) + await generation.wait(min(float(wait.group(1)), MAX_WAIT_SECONDS)) return format_job_response(generation, request) @app.get("/v1/generations/{generation_id}") @@ -204,7 +203,7 @@ async def get_generation_events(generation_id: str, request: Request) -> Streami async def stream(): index = start while True: - events, done = await asyncio.to_thread(generation.wait_for_events, index, _KEEP_ALIVE_SECONDS) + events, done = await generation.wait_for_events(index, _KEEP_ALIVE_SECONDS) for event, data in events: index += 1 payload = format_job_response(generation, request) if event == "complete" else data diff --git a/src/diffusers/commands/serve/backends.py b/src/diffusers/commands/serve/backends.py index de16a2cadda9..ca386c900867 100644 --- a/src/diffusers/commands/serve/backends.py +++ b/src/diffusers/commands/serve/backends.py @@ -49,7 +49,6 @@ from ...utils import export_to_video, load_image, load_video, logging, numpy_to_pil from ...utils.constants import DIFFUSERS_REQUEST_TIMEOUT from ..run import ( - PNG_COMPRESS_LEVEL, _as_audio_arrays, _as_pil_list, _get_generator, @@ -62,6 +61,9 @@ logger = logging.get_logger("diffusers-cli/serve") +# zlib level for PNG outputs. Pillow's default of 6 takes close to four times as long to encode a +# 1024x1024 image for a file about 7% smaller; the output is lossless either way. +PNG_COMPRESS_LEVEL = 1 _OUTPUT_FIELDS = {"image": ("images",), "video": ("frames", "videos"), "audio": ("audios", "audio")} _SAMPLE_RATE_ATTRIBUTES = ( "sample_rate", diff --git a/src/diffusers/commands/serve/scheduler.py b/src/diffusers/commands/serve/scheduler.py index db82f190d329..e74e4f804c8c 100644 --- a/src/diffusers/commands/serve/scheduler.py +++ b/src/diffusers/commands/serve/scheduler.py @@ -16,6 +16,7 @@ from __future__ import annotations +import asyncio import queue import shutil import threading @@ -47,31 +48,40 @@ def __init__(self, request: ResolvedRequest): self.outputs: dict[str, list[dict[str, Any]]] | None = None self.usage: dict[str, Any] | None = None self.error: GMSError | None = None - self.cancel_requested = False - self.timed_out = False - self.started_at: float | None = None + self.canceled = False + # When a running generation must stop, from `--generation-timeout`; None when it has no limit. + self.deadline: float | None = None self.directory: Path | None = None self.events: list[tuple[str, dict[str, Any] | None]] = [] self.done = False - self._condition = threading.Condition() + self._lock = threading.Lock() + # Handlers waiting on the event loop for the next event. The worker thread wakes them through their loop. + self._waiters: list[tuple[asyncio.AbstractEventLoop, asyncio.Event]] = [] + + @property + def cancel_requested(self) -> bool: + """Whether the pipeline should stop at its next step: a client canceled it, or it ran past its deadline.""" + return self.canceled or (self.deadline is not None and time.monotonic() > self.deadline) def report_progress(self, step: int, total_steps: int) -> None: - with self._condition: + with self._lock: self.progress = {"step": step, "total_steps": total_steps, "fraction": round(step / total_steps, 4)} self.events.append(("progress", self.progress)) - self._condition.notify_all() + self._notify_waiting_requests() - def wait_for_events(self, index: int, timeout: float) -> tuple[list[tuple[str, dict[str, Any] | None]], bool]: - with self._condition: - self._condition.wait_for(lambda: len(self.events) > index or self.done, timeout) + async def wait_for_events( + self, index: int, timeout: float + ) -> tuple[list[tuple[str, dict[str, Any] | None]], bool]: + await self._wait_until(lambda: len(self.events) > index or self.done, timeout) + # Read together, so a stream never sees `done` without the final event that came with it. + with self._lock: return self.events[index:], self.done - def wait(self, timeout: float) -> None: - with self._condition: - self._condition.wait_for(lambda: self.done, timeout) + async def wait(self, timeout: float) -> None: + await self._wait_until(lambda: self.done, timeout) def finish(self, status: str, error: GMSError | None = None) -> None: - with self._condition: + with self._lock: self.status = status self.error = error self.progress = None @@ -81,7 +91,31 @@ def finish(self, status: str, error: GMSError | None = None) -> None: elif status == "complete": self.events.append(("complete", None)) self.done = True - self._condition.notify_all() + self._notify_waiting_requests() + + async def _wait_until(self, condition: Any, timeout: float) -> None: + # Waiting on the event loop, not a pool thread, so any number of clients can wait at once. + loop = asyncio.get_running_loop() + give_up_at = loop.time() + timeout + while True: + event = asyncio.Event() + with self._lock: + if condition(): + return + self._waiters.append((loop, event)) + try: + await asyncio.wait_for(event.wait(), give_up_at - loop.time()) + except asyncio.TimeoutError: + return + finally: + with self._lock: + self._waiters.remove((loop, event)) + + def _notify_waiting_requests(self) -> None: + for loop, event in self._waiters: + # A loop closed at shutdown has no handler left to wake, and raising here would end the worker. + if not loop.is_closed(): + loop.call_soon_threadsafe(event.set) class Scheduler: @@ -97,7 +131,8 @@ def __init__(self, backends: list[Any], output_dir: str, timeout: float | None = self._generations: dict[str, GenerationJob] = {} self._lock = threading.Lock() self._pending: queue.Queue[GenerationJob] = queue.Queue() - self._running = 0 + # The generations a worker is running now. + self._running: set[GenerationJob] = set() # Every worker takes the next job from the same queue, so no job waits behind a long one on a busy backend. for index, backend in enumerate(backends): threading.Thread( @@ -112,8 +147,8 @@ def failure(self) -> str | None: # or in a pipeline without step callbacks, and its worker takes no more jobs. now = time.monotonic() with self._lock: - for generation in self._generations.values(): - if generation.status == "generating" and now - generation.started_at > 2 * self._timeout: + for generation in self._running: + if now > generation.deadline + self._timeout: return ( f"generation {generation.id} has run for over twice the --generation-timeout of " f"{self._timeout:g}s and did not stop when interrupted" @@ -121,7 +156,7 @@ def failure(self) -> str | None: return None def busy(self) -> bool: - return self._running > 0 or not self._pending.empty() + return bool(self._running) or not self._pending.empty() def submit(self, generation: GenerationJob) -> None: self._prune() @@ -142,7 +177,7 @@ def cancel(self, generation_id: str) -> GenerationJob: with self._lock: if generation.done: return generation - generation.cancel_requested = True + generation.canceled = True if generation.status == "queued": generation.finish("canceled") return generation @@ -175,14 +210,10 @@ def _work(self, backend: Any) -> None: if generation.done: continue generation.status = "generating" - generation.started_at = time.monotonic() - self._running += 1 + if self._timeout is not None: + generation.deadline = time.monotonic() + self._timeout + self._running.add(generation) generation.directory = self._output_dir / generation.id - timer = None - if self._timeout is not None: - timer = threading.Timer(self._timeout, self._time_out, args=(generation,)) - timer.daemon = True - timer.start() started = time.perf_counter() error = None @@ -196,19 +227,18 @@ def _work(self, backend: Any) -> None: except BaseException as e: logger.error(f"generation {generation.id} failed", exc_info=True) error = GMSError("internal", f"{type(e).__name__}: {e}") - if timer is not None: - timer.cancel() + returned_at = time.monotonic() with self._lock: deleted = generation.id not in self._generations # Before `finish` wakes any waiter, so a finished job is never reported while the server looks busy. - self._running -= 1 + self._running.discard(generation) if error is not None: generation.finish("failed", error) - elif generation.timed_out: + elif generation.deadline is not None and returned_at > generation.deadline: message = f"generation exceeded the --generation-timeout of {self._timeout:g}s" generation.finish("failed", GMSError("internal", message)) - elif generation.cancel_requested: + elif generation.canceled: generation.finish("canceled") else: generation.outputs = outputs @@ -216,9 +246,3 @@ def _work(self, backend: Any) -> None: generation.finish("complete") if deleted or generation.status != "complete": self._remove_files(generation) - - def _time_out(self, generation: GenerationJob) -> None: - # Interrupts the pipeline at its next step, the same way a cancel does. - with self._lock: - generation.timed_out = True - generation.cancel_requested = True diff --git a/tests/others/test_cli_serve.py b/tests/others/test_cli_serve.py index f26662afef13..75be9580ae93 100644 --- a/tests/others/test_cli_serve.py +++ b/tests/others/test_cli_serve.py @@ -16,6 +16,7 @@ No weights are loaded: the backend runs a fake pipeline. """ +import asyncio import base64 import inspect import io @@ -578,14 +579,6 @@ def test_request_adapters_are_loaded_then_unloaded(self, serve): ("unload",), ] - def test_adapters_not_advertised_when_compiled(self, serve): - client, _ = serve(compile="{}") - discovery = client.get(f"/v1/models/{MODEL_ID}").json() - assert "adapters" not in discovery["manifest"]["capabilities"] - body = {"model": MODEL_ID, "inputs": {"prompt": "a cat"}, "adapters": [{"type": "lora", "path": "hf:a/b"}]} - response = client.post("/v1/generations", json=body) - assert (response.status_code, response.json()["error"]["code"]) == (501, "not_implemented") - def test_discovery_serves_manifest_bytes_verbatim(self, serve): manifest = derive_manifest(MODEL_ID, FakePipeline, FAKE_SIGNATURE, ["sse", "wait", "adapters"]) raw = json.dumps(manifest, indent=3).encode() @@ -631,9 +624,8 @@ def test_generation_runs_a_modular_pipeline(self, serve): class FakeJobsApi: - def __init__(self, logs=("server started",), stage="COMPLETED"): + def __init__(self, logs=("server started",)): self.logs = logs - self.stage = stage self.run_kwargs = None self.cancelled = False @@ -648,7 +640,7 @@ def fetch_job_logs(self, **kwargs): yield line def inspect_job(self, **kwargs): - return SimpleNamespace(status=SimpleNamespace(stage=self.stage, expose_urls=None)) + return SimpleNamespace(status=SimpleNamespace(stage="COMPLETED", expose_urls=None)) def cancel_job(self, **kwargs): self.cancelled = True @@ -688,24 +680,12 @@ def test_remote_starts_the_server_in_a_job(self, monkeypatch): assert not {"--remote", "--flavor", "--dependencies"} & set(serve_argv), serve_argv assert not api.cancelled, "a Job that completed on its own needs no cancel" - def test_local_manifest_is_rejected(self, tmp_path, monkeypatch): - api = FakeJobsApi() - manifest = tmp_path / "gms.json" - manifest.write_text("{}") - with pytest.raises(SystemExit, match="Upload it as `gms.json` to the model repo"): - self._run(monkeypatch, api, ["--manifest", str(manifest)]) - assert api.run_kwargs is None, "no Job should start when the manifest can't reach it" - def test_interrupt_cancels_the_job(self, monkeypatch): api = FakeJobsApi(logs=("server started", KeyboardInterrupt())) with pytest.raises(KeyboardInterrupt): self._run(monkeypatch, api, []) assert api.cancelled, "Ctrl-C must cancel the Job so it stops billing" - def test_failed_job_is_reported(self, monkeypatch): - with pytest.raises(SystemExit, match="Job job-1 ended with status ERROR"): - self._run(monkeypatch, FakeJobsApi(stage="ERROR"), []) - class BlockingBackend: def __init__(self): @@ -737,7 +717,7 @@ def test_each_backend_runs_its_own_generation(self, tmp_path): for backend in backends: backend.release.set() for job in jobs: - job.wait(timeout=30) + asyncio.run(job.wait(timeout=30)) assert [job.status for job in jobs] == ["complete", "complete"] assert not scheduler.busy(), "the scheduler should be idle once every backend has finished" @@ -756,7 +736,7 @@ def generate(self, generation): scheduler = Scheduler([SteppingBackend()], str(tmp_path), timeout=0.1) job = GenerationJob(_manifest().resolve({"model": MODEL_ID, "inputs": {"prompt": "a cat"}})) scheduler.submit(job) - job.wait(timeout=30) + asyncio.run(job.wait(timeout=30)) assert job.status == "failed", job.status assert "--generation-timeout" in job.error.message, job.error.message @@ -769,7 +749,7 @@ def test_generation_stuck_past_the_timeout_fails_health(self, tmp_path): assert scheduler.failure() is None, "a generation within its time limit is healthy" # As if it had run for over twice the limit without reaching a step that could stop it. - job.started_at -= 121 + job.deadline -= 121 assert "did not stop when interrupted" in (scheduler.failure() or ""), scheduler.failure() backend.release.set() From 29ad3580cfa4de5933564f386352690311f7277a Mon Sep 17 00:00:00 2001 From: DN6 Date: Thu, 5 Nov 2026 21:14:45 +0530 Subject: [PATCH 17/21] Simplify serve: inline single-use helpers, drop 501 stubs, shorten docs --- docs/source/en/using-diffusers/serve.md | 210 ++++---- src/diffusers/commands/run.py | 41 +- src/diffusers/commands/serve/__init__.py | 10 +- src/diffusers/commands/serve/app.py | 47 +- src/diffusers/commands/serve/backends.py | 212 ++++---- src/diffusers/commands/serve/manifest.py | 604 +++++++++++----------- src/diffusers/commands/serve/remote.py | 40 +- src/diffusers/commands/serve/scheduler.py | 19 +- 8 files changed, 564 insertions(+), 619 deletions(-) diff --git a/docs/source/en/using-diffusers/serve.md b/docs/source/en/using-diffusers/serve.md index 7968ad2ef371..8a24b66637f6 100644 --- a/docs/source/en/using-diffusers/serve.md +++ b/docs/source/en/using-diffusers/serve.md @@ -12,86 +12,91 @@ specific language governing permissions and limitations under the License. # Serving -`diffusers serve` serves a pipeline over HTTP. The server implements the Generative Media Spec (GMS, draft -`gms/0.1`): clients discover what the model accepts, create a generation, follow it while it runs, and read the -results. - -Install the `serve` extra and start a server: +`diffusers serve` serves a pipeline over HTTP with the Generative Media Spec (GMS) API: clients discover what the +model accepts, create a generation, follow it while it runs, and download the results. ```bash pip install "diffusers[serve]" diffusers serve black-forest-labs/FLUX.1-dev ``` -Then send it a request: +`serve` takes the same loading and optimization flags as [`diffusers-cli run`](./cli#run). The server listens on +`127.0.0.1:8000` and has no authentication, so keep it on a trusted network. Run `diffusers serve --help` for every +option. + +## Generations + +Create a generation. `Prefer: wait` holds the response until it finishes, for up to 120 seconds here: ```bash curl localhost:8000/v1/generations -H 'Prefer: wait=120' -d '{ "model": "black-forest-labs/FLUX.1-dev", - "inputs": {"prompt": "an astronaut riding a horse"} + "inputs": {"prompt": "an astronaut riding a horse"}, + "parameters": {"width": 1024, "height": 1024, "steps": 28, "seed": 42} }' ``` -`serve` takes the same loading and optimization flags as [`diffusers-cli run`](./cli#run), and loads weights in -bf16 unless `--dtype` says otherwise. The server listens on `127.0.0.1:8000` by default and has no authentication, -so keep it on a trusted network. It also downloads any http(s) URL a request passes as a media input, so a client -can make it fetch addresses on its own network. Run `diffusers serve --help` for every option. +```json +{ + "id": "gen_3f9a1c2b7d4e", + "model": "black-forest-labs/FLUX.1-dev", + "revision": "", + "status": "complete", + "created_at": "2026-11-05T10:00:00Z", + "progress": null, + "outputs": { + "image": [ + { + "media_type": "image/png", + "width": 1024, + "height": 1024, + "url": "http://localhost:8000/v1/generations/gen_3f9a1c2b7d4e/files/image-0.png" + } + ] + }, + "expires_at": "2026-11-05T11:00:12Z", + "seed": 42, + "usage": {"compute_seconds": 11.84} +} +``` -`--generation-timeout ` (for example `10m`) fails a generation that runs longer than that. The pipeline -is interrupted at its next step. Modular pipelines have no step callbacks, so they run to the end and their result -is discarded. A generation that is still running at twice the limit is stuck, and `GET /health` then returns `503`. +Without `Prefer: wait`, the response comes back at once with `"status": "queued"`. Follow its progress as +server-sent events, or poll it: -`--idle-timeout ` (for example `30m`) shuts the server down once it has received no requests for that -long and no generation is queued or running. Health checks don't count as requests. By default a local server -never shuts itself down. +```bash +curl -N localhost:8000/v1/generations/gen_3f9a1c2b7d4e/events +``` -[Modular repos](../modular_diffusers/overview) are detected the same way as in `run`. They report no step -progress, and a cancel request only takes effect once the pipeline call returns. +```text +id: 1 +event: progress +data: {"step": 1, "total_steps": 28, "fraction": 0.0357} -## Generations +... -Generations run one at a time, or one per GPU with [`--data-parallel`](#multiple-gpus). `POST /v1/generations` returns immediately with a `queued` generation. Poll it, -subscribe to its events, or ask the server to hold the connection with `Prefer: wait=`: +id: 29 +event: complete +data: {"id": "gen_3f9a1c2b7d4e", "status": "complete", "outputs": {...}, ...} +``` ```bash -# Create a generation and wait up to 120 seconds for it. -curl localhost:8000/v1/generations -H 'Prefer: wait=120' -d '{ - "model": "black-forest-labs/FLUX.1-dev", - "inputs": {"prompt": "an astronaut riding a horse"}, - "parameters": {"width": 1024, "height": 1024, "steps": 28, "seed": 42} -}' - -# Follow step progress as server-sent events, or poll the generation. -curl -N localhost:8000/v1/generations//events -curl localhost:8000/v1/generations/ - -# Cancel a queued or running generation, or delete it together with its files. -curl -X POST localhost:8000/v1/generations//cancel -curl -X DELETE localhost:8000/v1/generations/ +curl localhost:8000/v1/generations/gen_3f9a1c2b7d4e # poll +curl -X POST localhost:8000/v1/generations/gen_3f9a1c2b7d4e/cancel # cancel +curl -X DELETE localhost:8000/v1/generations/gen_3f9a1c2b7d4e # delete it and its files +curl -O http://localhost:8000/v1/generations/gen_3f9a1c2b7d4e/files/image-0.png ``` -A request has two groups of fields. `inputs` holds the prompt and any media. `parameters` holds settings such as -size, steps and seed. Both use the names the server declares, which can differ from the pipeline's argument names -(`steps`, not `num_inference_steps`). Media inputs are passed as `{"type": "image", "url": "https://..."}` or -`{"type": "image", "base64": "...", "media_type": "image/png"}`. +`inputs` holds the prompt and media (`{"type": "image", "url": "https://..."}`), and `parameters` holds settings. +Both use the names the model's manifest declares. A completed generation lists a download URL for each output, or +inline base64 with `"response_format": "b64_json"`. Outputs expire after one hour. -A completed generation lists its outputs with a download URL each (or inline base64 with -`"response_format": "b64_json"`), the seed that was used, and the model revision. Outputs expire after one hour. -Images are PNG by default (`"output_format": "image/jpeg"` or `"image/webp"` to change it), videos MP4, audio WAV. - -Validation is strict: an unknown parameter, a value outside its declared range, or a missing input is rejected -with a JSON pointer to the offending field. +Invalid requests are rejected with a pointer to the offending field: ```json {"error": {"code": "invalid_request", "message": "task 'text_to_image' declares no parameter 'cfg'", "pointer": "/parameters/cfg"}} ``` -When a model hosts more than one task (for example `text_to_image` and `image_to_image`), the server picks the task -from the inputs. Set `"task"` in the request when the inputs match more than one. - -### Adapters - -A request can apply LoRAs for that generation only. They are loaded before the call and unloaded after it: +A request can also apply LoRAs for that generation only: ```bash curl localhost:8000/v1/generations -d '{ @@ -101,93 +106,74 @@ curl localhost:8000/v1/generations -d '{ }' ``` -Adapters are a capability the manifest declares (see below). They are unavailable when the server starts with -`--compile`, `--lora` or `--context-parallel`: a derived manifest leaves them out, and a published manifest that -declares them is refused at startup. +## Manifests -## Discovery and manifests - -`GET /v1/models/` returns what the model accepts: its tasks, inputs, and parameters with types and defaults. -`GET /health` returns `{"status": "ok"}`. +`GET /v1/models/` returns the model's manifest: its tasks, inputs, parameters and capabilities. ```bash curl localhost:8000/v1/models/black-forest-labs/FLUX.1-dev ``` -This document is the model's manifest. `serve` looks for one in this order: - -1. the file passed with `--manifest `; -2. a `gms.json` published in the model repo, served byte-for-byte; -3. a manifest derived from the pipeline's `__call__` signature. - -A derived manifest is a best-effort reading of the signature: scalar arguments become parameters, prompts and -media arguments become inputs, and arguments with a core GMS name are renamed (`num_inference_steps` becomes -`steps`, `num_images_per_prompt` becomes `num_outputs`, `generator` becomes `seed`). It carries no value ranges. -Write a `gms.json` when you need ranges, presets, or a duration in seconds. - -When a manifest uses a name that differs from the pipeline argument, map it per task under `x-diffusers`: - ```json -"x-diffusers": {"inputs": {"caption": "prompt"}, "parameters": {"cfg": "guidance_scale"}} +{ + "spec": "gms/0.1", + "manifest": { + "id": "black-forest-labs/FLUX.1-dev", + "modalities": {"in": ["image"], "out": ["image"]}, + "tasks": { + "text_to_image": { + "inputs": {"prompt": {"type": "text", "required": true}, "prompt_2": {"type": "text"}}, + "outputs": {"image": {"type": "image"}}, + "parameters": { + "height": {"type": "integer"}, + "width": {"type": "integer"}, + "steps": {"type": "integer", "default": 28}, + "guidance_scale": {"type": "number", "default": 3.5}, + "seed": {"type": "integer"} + }, + "x-diffusers": {"parameters": {"steps": "num_inference_steps", "seed": "generator"}} + } + }, + "capabilities": ["sse", "wait", "adapters"], + "adapters": {"lora": {"targets": ["transformer", "text_encoder"]}} + } +} ``` -The manifest describes the server, so `serve` checks it at startup and refuses to start when it declares -something the server can't honor: an input or parameter that maps to no pipeline argument, a `duration` without a -frame lattice, an output type other than image, video or audio, or a capability the server doesn't provide. +`serve` uses the file passed with `--manifest`, else a `gms.json` in the model repo, else a manifest derived from the +pipeline's `__call__` signature. Publish a `gms.json` to add value ranges, presets, or a duration in seconds. The +server checks the manifest at startup and refuses to start if it declares something the pipeline can't do. -The server's capabilities are listed in the manifest under `capabilities`, from `sse` (event streams), `wait` -(`Prefer: wait`) and `adapters` (per-request LoRAs). A capability the manifest leaves out is off: event streams -return `not_implemented` and `Prefer: wait` is ignored. A derived manifest declares every capability the server -can provide. +`x-diffusers` maps a manifest's names to pipeline arguments, and `capabilities` lists what the server supports: +`sse` (event streams), `wait` (`Prefer: wait`) and `adapters` (per-request LoRAs). The manifest above is abridged. -```json -"capabilities": ["sse", "wait", "adapters"] -``` +## Timeouts + +- `--generation-timeout 10m` fails a generation that runs longer than 10 minutes. +- `--idle-timeout 30m` shuts the server down after 30 minutes without requests. ## Multiple GPUs -`--data-parallel N` loads a copy of the pipeline on each of the first N GPUs and runs up to N generations at -once. Generations still start in the order they arrive: each GPU takes the next queued generation when it finishes -its current one. +`--data-parallel N` loads a copy of the pipeline on each of N GPUs and runs up to N generations at once. +`--context-parallel` splits each generation across all visible GPUs with +[context parallelism](../training/distributed_inference#context-parallelism). Don't launch it with `torchrun`. ```bash diffusers serve black-forest-labs/FLUX.1-dev --data-parallel 4 -``` - -`--compile` and `--device-map` are not available with `--data-parallel`. - -`--context-parallel` splits each generation across every visible GPU with -[context parallelism](../training/distributed_inference#context-parallelism). It needs a DiT-based pipeline and at -least two GPUs. `serve` starts one worker process per GPU itself, so don't launch it with `torchrun`: - -```bash diffusers serve black-forest-labs/FLUX.1-dev --context-parallel ``` -Every worker runs every generation, and rank 0 saves the outputs. If a worker process exits, `GET /health` returns -`503` and the server stops generating. `--device-map` and request adapters are not available with -`--context-parallel`, and `--compile` is skipped. `--context-parallel` and `--data-parallel` can't be combined. - ## Remote serving -`--remote` starts the server in a [Hugging Face Job](https://huggingface.co/docs/huggingface_hub/guides/jobs) -instead of on the local machine. The command starts the Job, which installs the dependencies and runs the server, -and streams its logs until you interrupt it. Interrupting the command cancels the Job. +`--remote` runs the server in a [Hugging Face Job](https://huggingface.co/docs/huggingface_hub/guides/jobs) and +streams its logs. Once the model loads, it prints the server URL. Requests authenticate with your Hugging Face token. ```bash diffusers serve black-forest-labs/FLUX.1-dev --remote --flavor a10g-large -``` -Once the model has loaded, the command prints the server URL, `https://--8000.hf.jobs`. Requests -authenticate with your Hugging Face token: - -```bash curl /v1/generations -H 'Prefer: wait=120' -H "Authorization: Bearer $HF_TOKEN" \ -d '{"model": "black-forest-labs/FLUX.1-dev", "inputs": {"prompt": "a cat on the moon"}}' ``` -The Job ends when the server shuts down, after `--idle-timeout` (default `30m`) without requests. A Job that never -goes idle is stopped after 24 hours. `--flavor`, `--image`, `--dependencies` and `--namespace` work as they -do for [`run --remote`](./cli#remote-execution---remote). With `--context-parallel`, pick a flavor with -several GPUs. The model must be a Hub repo id. `--manifest` is not -supported with `--remote`: publish the manifest as `gms.json` in the model repo instead. +The Job stops when the server goes idle (`--idle-timeout` defaults to `30m`), and `--flavor`, `--image`, +`--dependencies` and `--namespace` work as in [`run --remote`](./cli#remote-execution---remote). diff --git a/src/diffusers/commands/run.py b/src/diffusers/commands/run.py index 89d2cb23bef5..c0a85ab550ec 100644 --- a/src/diffusers/commands/run.py +++ b/src/diffusers/commands/run.py @@ -758,28 +758,6 @@ def _as_audio_arrays(value: Any): return None -def _write_wav(audio: Any, sampling_rate: int, path: str | Path) -> None: - """Write one numpy audio array to a 16-bit PCM WAV at `sampling_rate` Hz.""" - data = np.asarray(audio) - if data.dtype.kind == "f": - data = (np.clip(data, -1.0, 1.0) * 32767).astype(np.int16) - else: - data = data.astype(np.int16) - if data.ndim == 1: - n_channels = 1 - else: - # Heuristic: shorter axis is channels (interleaved layout for `wave` is - # samples × channels, so transpose if needed). - if data.shape[0] < data.shape[-1]: - data = data.T - n_channels = data.shape[1] - with wave.open(str(path), "wb") as w: - w.setnchannels(n_channels) - w.setsampwidth(2) # 16-bit PCM - w.setframerate(sampling_rate) - w.writeframes(data.tobytes()) - - def _save_audio_arrays(audios, sampling_rate: int, args: Namespace) -> list[str]: """Write each numpy audio array to a 16-bit PCM WAV at `sampling_rate` Hz. @@ -789,7 +767,24 @@ def _save_audio_arrays(audios, sampling_rate: int, args: Namespace) -> list[str] paths = _resolve_output_paths(len(audios), args.output, ext="wav") saved: list[str] = [] for audio, path in zip(audios, paths): - _write_wav(audio, sampling_rate, path) + data = np.asarray(audio) + if data.dtype.kind == "f": + data = (np.clip(data, -1.0, 1.0) * 32767).astype(np.int16) + else: + data = data.astype(np.int16) + if data.ndim == 1: + n_channels = 1 + else: + # Heuristic: shorter axis is channels (interleaved layout for `wave` is + # samples × channels, so transpose if needed). + if data.shape[0] < data.shape[-1]: + data = data.T + n_channels = data.shape[1] + with wave.open(str(path), "wb") as w: + w.setnchannels(n_channels) + w.setsampwidth(2) # 16-bit PCM + w.setframerate(sampling_rate) + w.writeframes(data.tobytes()) saved.append(str(path)) return saved diff --git a/src/diffusers/commands/serve/__init__.py b/src/diffusers/commands/serve/__init__.py index d9ad0e97ab88..c39d4aafa7f2 100644 --- a/src/diffusers/commands/serve/__init__.py +++ b/src/diffusers/commands/serve/__init__.py @@ -20,6 +20,7 @@ from __future__ import annotations +import copy import inspect import random import secrets @@ -284,10 +285,11 @@ def run(self) -> None: if args.context_parallel: backends = [ParallelBackend(args, manifest)] elif args.data_parallel > 1: - backends = [ - DiffusersBackend(Namespace(**{**vars(args), "device_map": f"cuda:{index}"}), manifest) - for index in range(args.data_parallel) - ] + backends = [] + for index in range(args.data_parallel): + replica_args = copy.copy(args) + replica_args.device_map = f"cuda:{index}" + backends.append(DiffusersBackend(replica_args, manifest)) else: backends = [DiffusersBackend(args, manifest)] diff --git a/src/diffusers/commands/serve/app.py b/src/diffusers/commands/serve/app.py index d64afbdb4f97..d816fe53a4c5 100644 --- a/src/diffusers/commands/serve/app.py +++ b/src/diffusers/commands/serve/app.py @@ -48,16 +48,6 @@ def _timestamp(value: Any) -> str | None: return value.strftime("%Y-%m-%dT%H:%M:%SZ") -def _format_output(job: GenerationJob, artifact: dict[str, Any], base_url: str) -> dict[str, Any]: - """One output file as the response lists it: its metadata, plus a download link or its bytes inline.""" - entry = {key: value for key, value in artifact.items() if key != "file"} - if job.request.response_format == "b64_json": - entry["base64"] = base64.b64encode((job.directory / artifact["file"]).read_bytes()).decode() - return entry - entry["url"] = base_url + _FILE_ROUTE.format(generation_id=job.id, filename=artifact["file"]) - return entry - - def create_app( manifest: Manifest, scheduler: Scheduler, @@ -78,14 +68,21 @@ def format_job_response(job: GenerationJob, request: Request) -> dict[str, Any]: if job.error is not None: raise job.error + # Each output file is listed with its metadata, plus a download link or its bytes inline. outputs = None if job.status == "complete": # Output links must work for the client, so they are built on the address it reached the server at. base_url = (public_url or str(request.base_url)).rstrip("/") - outputs = { - name: [_format_output(job, artifact, base_url) for artifact in artifacts] - for name, artifacts in job.outputs.items() - } + outputs = {} + for name, artifacts in job.outputs.items(): + outputs[name] = [] + for artifact in artifacts: + entry = {key: value for key, value in artifact.items() if key != "file"} + if job.request.response_format == "b64_json": + entry["base64"] = base64.b64encode((job.directory / artifact["file"]).read_bytes()).decode() + else: + entry["url"] = base_url + _FILE_ROUTE.format(generation_id=job.id, filename=artifact["file"]) + outputs[name].append(entry) return { "id": job.id, @@ -154,7 +151,7 @@ async def health() -> JSONResponse: @app.get("/v1/models") async def list_models() -> dict[str, Any]: - return {"data": [manifest.summary()]} + return {"data": [{"id": manifest.id, "modalities": manifest.modalities, "tasks": list(manifest.tasks)}]} @app.get("/v1/models/{model_id:path}") async def get_model(model_id: str) -> Response: @@ -198,7 +195,7 @@ async def get_generation_events(generation_id: str, request: Request) -> Streami raise GMSError("not_implemented", "this server does not advertise the `sse` capability") generation = scheduler.get(generation_id) last_event_id = request.headers.get("last-event-id", "") - start = int(last_event_id) if last_event_id.isdigit() and len(last_event_id) < 10 else 0 + start = int(last_event_id) if last_event_id.isdigit() else 0 async def stream(): index = start @@ -218,17 +215,11 @@ async def stream(): @app.get(_FILE_ROUTE) async def get_generation_file(generation_id: str, filename: str) -> FileResponse: generation = scheduler.get(generation_id) - artifacts = [a for group in (generation.outputs or {}).values() for a in group if a["file"] == filename] - if not artifacts: - raise GMSError("not_found", f"generation {generation_id!r} has no file {filename!r}") - return FileResponse(generation.directory / filename, media_type=artifacts[0]["media_type"]) - - @app.get("/v1/generations/{generation_id}/outputs/{name}/stream") - async def stream_generation_output(generation_id: str, name: str) -> Response: - raise GMSError("not_implemented", "this server does not advertise the `stream` capability") - - @app.post("/v1/uploads") - async def create_upload() -> Response: - raise GMSError("not_implemented", "this server does not advertise the `presigned_upload` capability") + # Only files the backend saved are served, so a filename can't reach anything else on disk. + for artifacts in (generation.outputs or {}).values(): + for artifact in artifacts: + if artifact["file"] == filename: + return FileResponse(generation.directory / filename, media_type=artifact["media_type"]) + raise GMSError("not_found", f"generation {generation_id!r} has no file {filename!r}") return app diff --git a/src/diffusers/commands/serve/backends.py b/src/diffusers/commands/serve/backends.py index ca386c900867..5c035c87a83e 100644 --- a/src/diffusers/commands/serve/backends.py +++ b/src/diffusers/commands/serve/backends.py @@ -53,7 +53,6 @@ _as_pil_list, _get_generator, _load_pipeline, - _write_wav, ) from .manifest import GMSError, Manifest, ResolvedRequest, blocks_signature from .scheduler import GenerationJob @@ -120,67 +119,6 @@ def _load_media(name: str, value: dict[str, Any], pointer: str) -> Any: raise GMSError("invalid_request", f"could not decode input {name!r}: {e}", pointer) from e -def _save_images(images: Any, directory: Path, name: str, media_type: str, rate: float) -> list[dict[str, Any]]: - if isinstance(images, np.ndarray): - images = numpy_to_pil(images) - images = _as_pil_list(images) - if images is None: - raise GMSError("internal", f"cannot encode output {name!r} as {media_type}: not a list of images") - - artifacts = [] - for index, image in enumerate(images): - filename = f"{name}-{index}.{media_type.split('/')[1]}" - if media_type == "image/jpeg": - image = image.convert("RGB") - if media_type == "image/png": - image.save(directory / filename, compress_level=PNG_COMPRESS_LEVEL) - else: - image.save(directory / filename) - artifacts.append({"file": filename, "media_type": media_type, "width": image.width, "height": image.height}) - return artifacts - - -def _save_videos(videos: Any, directory: Path, name: str, media_type: str, rate: float) -> list[dict[str, Any]]: - artifacts = [] - for index, frames in enumerate(videos): - frames = list(frames) - filename = f"{name}-{index}.mp4" - export_to_video(frames, str(directory / filename), fps=rate) - first = frames[0] - width, height = first.size if isinstance(first, Image.Image) else (first.shape[1], first.shape[0]) - artifacts.append( - { - "file": filename, - "media_type": media_type, - "width": width, - "height": height, - "duration": round(len(frames) / rate, 2), - "fps": rate, - } - ) - return artifacts - - -def _save_audio(audios: Any, directory: Path, name: str, media_type: str, rate: float) -> list[dict[str, Any]]: - if isinstance(audios, torch.Tensor): - audios = audios.detach().to(torch.float32).cpu().numpy() - arrays = _as_audio_arrays(audios) - if arrays is None: - raise GMSError("internal", f"cannot encode output {name!r} as {media_type}: not an audio array") - - artifacts = [] - for index, audio in enumerate(arrays): - filename = f"{name}-{index}.wav" - _write_wav(audio, int(rate), directory / filename) - with wave.open(str(directory / filename), "rb") as file: - duration = round(file.getnframes() / file.getframerate(), 2) - artifacts.append({"file": filename, "media_type": media_type, "duration": duration, "sample_rate": int(rate)}) - return artifacts - - -_SAVERS = {"image": _save_images, "video": _save_videos, "audio": _save_audio} - - def _sample_rate(pipeline: Any) -> int: for path in _SAMPLE_RATE_ATTRIBUTES: value = pipeline @@ -221,8 +159,11 @@ def __init__(self, args: Namespace, manifest: Manifest, save_outputs: bool = Tru else: parameters = inspect.signature(type(self.pipeline).__call__).parameters self._reports_steps = "callback_on_step_end" in parameters - outputs = (spec["type"] for task in manifest.tasks.values() for spec in task["outputs"].values()) - self._sample_rate = _sample_rate(self.pipeline) if "audio" in outputs else None + self._sample_rate = None + for task in manifest.tasks.values(): + for spec in task["outputs"].values(): + if spec["type"] == "audio": + self._sample_rate = _sample_rate(self.pipeline) def close(self) -> None: pass @@ -253,8 +194,19 @@ def on_step_end(pipeline, step, timestep, callback_kwargs): kwargs["callback_on_step_end"] = on_step_end + names = [f"request_{index}" for index in range(len(request.adapters))] try: - self._load_adapters(request.adapters) + for index, adapter in enumerate(request.adapters): + try: + self.pipeline.load_lora_weights(adapter["path"].removeprefix("hf:"), adapter_name=names[index]) + except Exception as e: + raise GMSError( + "invalid_request", + f"could not load adapter {adapter['path']!r}: {type(e).__name__}: {e}", + f"/adapters/{index}/path", + ) from e + if names: + self.pipeline.set_adapters(names, adapter_weights=[adapter["weight"] for adapter in request.adapters]) # Worker threads start on GPU 0, and with --data-parallel this pipeline may sit on another one. with torch.cuda.device(device) if device.type == "cuda" else contextlib.nullcontext(): result = self.pipeline(**kwargs) @@ -267,16 +219,81 @@ def on_step_end(pipeline, step, timestep, callback_kwargs): outputs = {} for name, (kind, media_type) in request.outputs.items(): - # A modular pipeline returns its `PipelineState`; a standard one returns an output object. - if self._modular: - fields = (result.get(field) for field in _OUTPUT_FIELDS.get(kind, ())) - else: - fields = (getattr(result, field, None) for field in _OUTPUT_FIELDS.get(kind, ())) - value = next((field for field in fields if field is not None), None) + value = None + for field in _OUTPUT_FIELDS[kind]: + # A modular pipeline returns its `PipelineState`; a standard one returns an output object. + value = result.get(field) if self._modular else getattr(result, field, None) + if value is not None: + break if value is None: continue - rate = request.fps if kind == "video" else self._sample_rate - outputs[name] = _SAVERS[kind](value, generation.directory, name, media_type, rate) + + artifacts = [] + if kind == "image": + images = _as_pil_list(numpy_to_pil(value) if isinstance(value, np.ndarray) else value) + if images is None: + raise GMSError("internal", f"cannot encode output {name!r} as {media_type}: not a list of images") + for index, image in enumerate(images): + filename = f"{name}-{index}.{media_type.split('/')[1]}" + if media_type == "image/jpeg": + image = image.convert("RGB") + if media_type == "image/png": + image.save(generation.directory / filename, compress_level=PNG_COMPRESS_LEVEL) + else: + image.save(generation.directory / filename) + artifacts.append( + {"file": filename, "media_type": media_type, "width": image.width, "height": image.height} + ) + elif kind == "video": + for index, frames in enumerate(value): + frames = list(frames) + filename = f"{name}-{index}.mp4" + export_to_video(frames, str(generation.directory / filename), fps=request.fps) + first = frames[0] + width, height = first.size if isinstance(first, Image.Image) else (first.shape[1], first.shape[0]) + artifacts.append( + { + "file": filename, + "media_type": media_type, + "width": width, + "height": height, + "duration": round(len(frames) / request.fps, 2), + "fps": request.fps, + } + ) + else: + if isinstance(value, torch.Tensor): + value = value.detach().to(torch.float32).cpu().numpy() + arrays = _as_audio_arrays(value) + if arrays is None: + raise GMSError("internal", f"cannot encode output {name!r} as {media_type}: not an audio array") + # TODO(DN6): write the files with `export_to_wav` once it is added to `diffusers.utils`. + for index, audio in enumerate(arrays): + data = np.asarray(audio) + if data.dtype.kind == "f": + data = (np.clip(data, -1.0, 1.0) * 32767).astype(np.int16) + else: + data = data.astype(np.int16) + # `wave` takes interleaved samples x channels; the shorter axis is taken to be the channels. + if data.ndim == 1: + data = data[:, None] + elif data.shape[0] < data.shape[1]: + data = data.T + filename = f"{name}-{index}.wav" + with wave.open(str(generation.directory / filename), "wb") as file: + file.setnchannels(data.shape[1]) + file.setsampwidth(2) + file.setframerate(self._sample_rate) + file.writeframes(data.tobytes()) + artifacts.append( + { + "file": filename, + "media_type": media_type, + "duration": round(data.shape[0] / self._sample_rate, 2), + "sample_rate": self._sample_rate, + } + ) + outputs[name] = artifacts if not outputs: raise GMSError( "internal", @@ -285,25 +302,6 @@ def on_step_end(pipeline, step, timestep, callback_kwargs): ) return outputs - def _load_adapters(self, adapters: list[dict[str, Any]]) -> None: - if not adapters: - return - names = [] - weights = [] - for index, adapter in enumerate(adapters): - name = f"request_{index}" - try: - self.pipeline.load_lora_weights(adapter["path"].removeprefix("hf:"), adapter_name=name) - except Exception as e: - raise GMSError( - "invalid_request", - f"could not load adapter {adapter['path']!r}: {type(e).__name__}: {e}", - f"/adapters/{index}/path", - ) from e - weights.append(adapter["weight"]) - names.append(name) - self.pipeline.set_adapters(names, adapter_weights=weights) - _POLL_SECONDS = 0.1 _SHUTDOWN_SECONDS = 30.0 @@ -407,25 +405,27 @@ def __init__(self, args: Namespace, manifest: Manifest): self._processes.append(process) logger.info(f"loading the pipeline on {world_size} GPUs...") - self._wait_until_ready() - threading.Thread(target=self._monitor, name="diffusers-serve-workers", daemon=True).start() - - def _wait_until_ready(self) -> None: - waiting = dict(zip(self._conns, self._processes)) - while waiting: - sentinels = {process.sentinel: process for process in waiting.values()} - for ready in wait([*waiting, *sentinels]): - process = waiting.pop(ready, None) if isinstance(ready, Connection) else sentinels[ready] - try: - if isinstance(ready, Connection) and ready.recv() == ("ready",): - continue - except EOFError: - pass + # Every rank is watched at once: a rank that dies while loading leaves the others stuck in the process + # group setup, so waiting on one rank at a time could hang forever. + loading = dict(zip(self._conns, self._processes)) + while loading: + sentinels = {process.sentinel: process for process in loading.values()} + for ready in wait([*loading, *sentinels]): + if ready in sentinels: + process = sentinels[ready] + else: + process = loading.pop(ready) + try: + if ready.recv() == ("ready",): + continue + except EOFError: + pass self.close() raise SystemExit( f"Context-parallel worker {process.name} exited with code {process.exitcode} while loading " "the pipeline. Its error is in the log above." ) + threading.Thread(target=self._monitor, name="diffusers-serve-workers", daemon=True).start() def _monitor(self) -> None: sentinels = {process.sentinel: process for process in self._processes} diff --git a/src/diffusers/commands/serve/manifest.py b/src/diffusers/commands/serve/manifest.py index 8627a823b9ca..e2b115009d7f 100644 --- a/src/diffusers/commands/serve/manifest.py +++ b/src/diffusers/commands/serve/manifest.py @@ -29,7 +29,7 @@ import os import random import re -from dataclasses import dataclass, field +from dataclasses import dataclass from pathlib import Path from typing import Any @@ -58,8 +58,6 @@ EXTENSION_KEY = "x-diffusers" MAX_ADAPTERS = 4 -# What a manifest may declare under `capabilities`; the server checks each one it declares at startup. -SUPPORTED_CAPABILITIES = ("sse", "wait", "adapters") # First entry of each tuple is the default when the request names no `output_format`. OUTPUT_FORMATS = { @@ -71,7 +69,6 @@ _ERROR_STATUS = { "invalid_request": 400, "not_found": 404, - "rate_limited": 429, "internal": 500, "not_implemented": 501, } @@ -151,7 +148,7 @@ class ResolvedRequest: # The frame rate video outputs are encoded at, or None when the task produces no video. fps: float | None response_format: str - adapters: list[dict[str, Any]] = field(default_factory=list) + adapters: list[dict[str, Any]] # --------------------------------------------------------------------------- @@ -192,7 +189,14 @@ def load_manifest( source = f"hf:{model}/{MANIFEST_FILENAME}" if raw is not None: - data = _parse_manifest(raw, origin) + try: + data = json.loads(raw) + except json.JSONDecodeError as e: + raise SystemExit(f"GMS manifest {origin!r} is not valid JSON: {e}") from e + if not isinstance(data, dict) or not isinstance(data.get("id"), str): + raise SystemExit(f"GMS manifest {origin!r} must be a JSON object with a string `id`.") + if not isinstance(data.get("tasks"), dict) or not data["tasks"]: + raise SystemExit(f"GMS manifest {origin!r} declares no `tasks`, so there is nothing to serve.") else: model_id = Path(model).name if Path(model).is_dir() else model data = derive_manifest(model_id, pipeline_cls, signature, capabilities) @@ -200,18 +204,6 @@ def load_manifest( return Manifest(raw, data, pipeline_cls, signature, capabilities, source=source, revision=commit_hash) -def _parse_manifest(raw: bytes, origin: str) -> dict[str, Any]: - try: - data = json.loads(raw) - except json.JSONDecodeError as e: - raise SystemExit(f"GMS manifest {origin!r} is not valid JSON: {e}") from e - if not isinstance(data, dict) or not isinstance(data.get("id"), str): - raise SystemExit(f"GMS manifest {origin!r} must be a JSON object with a string `id`.") - if not isinstance(data.get("tasks"), dict) or not data["tasks"]: - raise SystemExit(f"GMS manifest {origin!r} declares no `tasks`, so there is nothing to serve.") - return data - - # --------------------------------------------------------------------------- # Deriving a manifest from the pipeline signature # --------------------------------------------------------------------------- @@ -236,60 +228,19 @@ def derive_manifest( model_id: str, pipeline_cls: type, signature: inspect.Signature, capabilities: list[str] ) -> dict[str, Any]: """Build a server-authored manifest from the pipeline call `signature`, declaring the server's `capabilities`.""" - inputs, parameters, extension = _describe_kwargs(signature) - registered = [entry for entry in _AUTO_TASKS if pipeline_cls in entry[1].values()] - output = _output_modality(pipeline_cls, signature, registered) - - def task(required: tuple[str, ...], excluded: set[str]) -> dict[str, Any]: - task_inputs = { - name: {**spec, "required": True} if name in required else spec - for name, spec in inputs.items() - if name not in excluded - } - spec = {"inputs": task_inputs, "outputs": {output: {"type": output}}, "parameters": parameters} - if extension: - spec[EXTENSION_KEY] = extension - return spec - - primary = next((name for name in ("image", "video") if name in inputs), None) - tasks = {} - if registered: - # A class registered for several tasks takes each task's conditioning input optionally; an - # input another task requires is what tells the tasks apart, so it is left out of the rest. - for name, _, required, _ in registered: - if not set(required) <= set(inputs): - continue - others = {key for other in registered if other[0] != name for key in other[2]} - tasks[name] = task(required, others - set(required)) - elif primary is None: - tasks[f"text_to_{output}"] = task((), set()) - elif inputs[primary].get("required"): - tasks[f"{primary}_to_{output}"] = task((), set()) - else: - tasks[f"text_to_{output}"] = task((), {primary}) - tasks[f"{primary}_to_{output}"] = task((primary,), set()) - - modalities_in = sorted({"image" if s["type"] == "mask" else s["type"] for s in inputs.values()} - {"text"}) - manifest: dict[str, Any] = { - "id": model_id, - "modalities": {"in": modalities_in or ["text"], "out": [output]}, - "tasks": tasks, - "capabilities": list(capabilities), - } - if "adapters" in capabilities: - manifest["adapters"] = {"lora": {"targets": list(getattr(pipeline_cls, "_lora_loadable_modules", []))}} - return manifest - - -def _describe_kwargs(signature: inspect.Signature) -> tuple[dict, dict, dict]: - """Sort the call kwargs into GMS inputs and parameters, and record the ones whose GMS name differs.""" + # Sort the call kwargs into GMS inputs and parameters, and record the ones whose GMS name differs. inputs: dict[str, dict[str, Any]] = {} parameters: dict[str, dict[str, Any]] = {} input_kwargs: dict[str, str] = {} parameter_kwargs: dict[str, str] = {} - kwarg_parameters = {kwarg: name for name, kwargs in _PARAMETER_KWARGS.items() for kwarg in kwargs} - # `num_frames` keeps its name: `duration` is in seconds, which only a written manifest's frame lattice converts. - del kwarg_parameters["num_frames"] + # Pipeline kwarg -> its GMS parameter name. `num_frames` keeps its name: `duration` is in seconds, which only a + # written manifest's frame lattice converts. + kwarg_parameters = {} + for name, kwargs in _PARAMETER_KWARGS.items(): + if name == "duration": + continue + for kwarg in kwargs: + kwarg_parameters[kwarg] = name for kwarg, parameter in signature.parameters.items(): if kwarg == "self" or kwarg in _SKIPPED_KWARGS: @@ -316,9 +267,40 @@ def _describe_kwargs(signature: inspect.Signature) -> tuple[dict, dict, dict]: parameter_kwargs["seed"] = kwarg continue - schema = _annotation_schema(parameter.annotation, default) + # The GMS schema of the annotation, if it is a scalar or a list of scalars. + annotation = parameter.annotation + if annotation is inspect.Parameter.empty: + text = type(default).__name__ if default is not None else "" + elif isinstance(annotation, str): + text = annotation + elif type(annotation) is type: + text = annotation.__name__ + else: + text = str(annotation) + text = text.replace("typing.", "") + # The type names in the annotation, without the wrappers around them: {"str"} for `Optional[List[str]]`. + type_names = set(re.findall(r"[A-Za-z_][\w.]*", text)) - { + "Optional", + "Union", + "None", + "NoneType", + "list", + "List", + } + schema = None + if type_names and type_names <= set(_SCALAR_TYPES): + for python_type, gms_type in _SCALAR_TYPES.items(): + if python_type in type_names: + scalar_type = gms_type + break + # `str | list[str]` is a scalar that also accepts a batch; only a bare list is an array parameter. + outside_lists = re.sub(r"\b[Ll]ist\[[^\[\]]*\]", "", text) + if outside_lists == text or set(re.findall(r"[A-Za-z_][\w.]*", outside_lists)) & set(_SCALAR_TYPES): + schema = {"type": scalar_type} + else: + schema = {"type": "array", "items": {"type": scalar_type}} # Modular blocks often declare `prompt` without a type hint or a default to infer one from. - if schema is None and kwarg == "prompt" and parameter.annotation is inspect.Parameter.empty: + if schema is None and kwarg == "prompt" and annotation is inspect.Parameter.empty: schema = {"type": "string"} if schema is None: continue @@ -340,51 +322,63 @@ def _describe_kwargs(signature: inspect.Signature) -> tuple[dict, dict, dict]: extension["inputs"] = input_kwargs if parameter_kwargs: extension["parameters"] = parameter_kwargs - return inputs, parameters, extension + registered = [entry for entry in _AUTO_TASKS if pipeline_cls in entry[1].values()] + if registered: + output = registered[0][3] + elif "num_frames" in signature.parameters or "Video" in pipeline_cls.__name__: + output = "video" + elif set(signature.parameters) & _AUDIO_OUTPUT_KWARGS: + output = "audio" + else: + output = "image" -def _output_modality(pipeline_cls: type, signature: inspect.Signature, registered: list) -> str: + def task(required: tuple[str, ...], excluded: set[str]) -> dict[str, Any]: + task_inputs = { + name: {**spec, "required": True} if name in required else spec + for name, spec in inputs.items() + if name not in excluded + } + spec = {"inputs": task_inputs, "outputs": {output: {"type": output}}, "parameters": parameters} + if extension: + spec[EXTENSION_KEY] = extension + return spec + + primary = None + if "image" in inputs: + primary = "image" + elif "video" in inputs: + primary = "video" + tasks = {} if registered: - return registered[0][3] - names = set(signature.parameters) - if "num_frames" in names or "Video" in pipeline_cls.__name__: - return "video" - if names & _AUDIO_OUTPUT_KWARGS: - return "audio" - return "image" - - -def _annotation_schema(annotation: Any, default: Any) -> dict[str, Any] | None: - """Map a `__call__` annotation to a GMS parameter schema, or `None` when it is not a scalar or list of scalars.""" - if annotation is inspect.Parameter.empty: - text = type(default).__name__ if default is not None else "" - elif isinstance(annotation, str): - text = annotation - elif type(annotation) is type: - text = annotation.__name__ + # A class registered for several tasks takes each task's conditioning input optionally; an + # input another task requires is what tells the tasks apart, so it is left out of the rest. + for name, _, required, _ in registered: + if not set(required) <= set(inputs): + continue + others = set() + for other_name, _, other_required, _ in registered: + if other_name != name: + others.update(other_required) + tasks[name] = task(required, others - set(required)) + elif primary is None: + tasks[f"text_to_{output}"] = task((), set()) + elif inputs[primary].get("required"): + tasks[f"{primary}_to_{output}"] = task((), set()) else: - text = str(annotation) - text = text.replace("typing.", "") - - def scalar(fragment: str) -> str | None: - names = set(re.findall(r"[A-Za-z_][\w.]*", fragment)) - names -= {"Optional", "Union", "None", "NoneType", "list", "List"} - if not names or not names <= set(_SCALAR_TYPES): - return None - return next(gms for python, gms in _SCALAR_TYPES.items() if python in names) - - outside_lists = re.sub(r"\b[Ll]ist\[[^\[\]]*\]", "", text) - if outside_lists == text: - scalar_type = scalar(text) - return {"type": scalar_type} if scalar_type else None - - # `str | list[str]` is a scalar that also accepts a batch; only a bare list is an array parameter. - scalar_type = scalar(text) - if scalar_type is None: - return None - if scalar(outside_lists): - return {"type": scalar_type} - return {"type": "array", "items": {"type": scalar_type}} + tasks[f"text_to_{output}"] = task((), {primary}) + tasks[f"{primary}_to_{output}"] = task((primary,), set()) + + modalities_in = sorted({"image" if s["type"] == "mask" else s["type"] for s in inputs.values()} - {"text"}) + manifest: dict[str, Any] = { + "id": model_id, + "modalities": {"in": modalities_in or ["text"], "out": [output]}, + "tasks": tasks, + "capabilities": list(capabilities), + } + if "adapters" in capabilities: + manifest["adapters"] = {"lora": {"targets": list(getattr(pipeline_cls, "_lora_loadable_modules", []))}} + return manifest # --------------------------------------------------------------------------- @@ -410,13 +404,13 @@ def __init__( source: str | None = None, revision: str | None = None, ): - # The bytes are kept as published: discovery serves them unchanged, and `hash` is computed over them. + # The bytes are kept as published: discovery serves them unchanged, and the hash is computed over them. self.raw = raw - self.data = data self.source = source self.id = data["id"] self.revision = revision self.tasks: dict[str, dict[str, Any]] = data["tasks"] + self.modalities = data.get("modalities", {}) self.capabilities: list[str] = data.get("capabilities", []) self.adapters = data.get("adapters", {}) self._kwarg_names = set(signature.parameters) - {"self"} @@ -435,7 +429,7 @@ def __init__( "was started with --compile, --lora or --context-parallel." ) raise SystemExit( - message + f"which this server does not provide. It provides: {list(SUPPORTED_CAPABILITIES)}." + message + f"which this server does not provide. It provides: {list(supported_capabilities)}." ) if "adapters" in self.capabilities and not self.adapters: raise SystemExit( @@ -459,11 +453,44 @@ def __init__( for key in task.get("parameters", {}) } self._fps_kwargs[name] = self._parameter_kwargs[name].get("fps", fps_kwarg) - self._check_task(name, task) + unsupported = [spec["type"] for spec in task["outputs"].values() if spec["type"] not in OUTPUT_FORMATS] + if unsupported: + raise SystemExit( + f"The GMS manifest for {self.id!r} declares {unsupported} outputs on task {name!r}; this server " + f"produces {list(OUTPUT_FORMATS)} outputs." + ) + parameters = task.get("parameters", {}) + if "duration" in parameters and _frame_lattice(parameters["duration"]) is None: + raise SystemExit( + f"The GMS manifest for {self.id!r} declares `duration` on task {name!r} without a frame lattice, so " + 'seconds cannot be converted to frames. Add one, e.g. "lattice": {"frames": "8k+1", "fps": 24}.' + ) + produces_video = any(spec["type"] == "video" for spec in task["outputs"].values()) + if produces_video and self._default_fps is None and "duration" not in parameters: + raise SystemExit( + f"The GMS manifest for {self.id!r} declares a video output on task {name!r}, but " + f"{self._pipeline_name} has no default `fps` or `frame_rate` argument, so the frame rate to encode " + "at is unknown. Declare `duration` with a frame lattice at the model's native fps, e.g. " + '"duration": {"type": "number", "lattice": {"frames": "4k+1", "fps": 16}}, and pass the manifest ' + "with --manifest." + ) + for preset, body in task.get("presets", {}).items(): + for key in body.get("parameters", {}): + if key not in parameters: + raise SystemExit( + f"The GMS manifest for {self.id!r} sets parameter {key!r} in preset {preset!r} of task " + f"{name!r}, which the task does not declare." + ) def _pipeline_kwarg(self, name: str, aliases: dict[str, tuple[str, ...]], declared: dict[str, str]) -> str | None: - candidates = (declared[name],) if name in declared else (*aliases.get(name, ()), name) - return next((kwarg for kwarg in candidates if kwarg in self._kwarg_names), None) + if name in declared: + candidates = [declared[name]] + else: + candidates = [*aliases.get(name, ()), name] + for kwarg in candidates: + if kwarg in self._kwarg_names: + return kwarg + return None def _require_kwarg( self, task: str, kind: str, name: str, aliases: dict[str, tuple[str, ...]], declared: dict[str, str] @@ -477,89 +504,17 @@ def _require_kwarg( ) return kwarg - def _check_task(self, name: str, task: dict[str, Any]) -> None: - unsupported = [spec["type"] for spec in task["outputs"].values() if spec["type"] not in OUTPUT_FORMATS] - if unsupported: - raise SystemExit( - f"The GMS manifest for {self.id!r} declares {unsupported} outputs on task {name!r}; this server " - f"produces {list(OUTPUT_FORMATS)} outputs." - ) - parameters = task.get("parameters", {}) - if "duration" in parameters and _frame_lattice(parameters["duration"]) is None: - raise SystemExit( - f"The GMS manifest for {self.id!r} declares `duration` on task {name!r} without a frame lattice, so " - 'seconds cannot be converted to frames. Add one, e.g. "lattice": {"frames": "8k+1", "fps": 24}.' - ) - produces_video = any(spec["type"] == "video" for spec in task["outputs"].values()) - if produces_video and self._default_fps is None and "duration" not in parameters: - raise SystemExit( - f"The GMS manifest for {self.id!r} declares a video output on task {name!r}, but " - f"{self._pipeline_name} has no default `fps` or `frame_rate` argument, so the frame rate to encode " - "at is unknown. Declare `duration` with a frame lattice at the model's native fps, e.g. " - '"duration": {"type": "number", "lattice": {"frames": "4k+1", "fps": 16}}, and pass the manifest ' - "with --manifest." - ) - for preset, body in task.get("presets", {}).items(): - for key in body.get("parameters", {}): - if key not in parameters: - raise SystemExit( - f"The GMS manifest for {self.id!r} sets parameter {key!r} in preset {preset!r} of task " - f"{name!r}, which the task does not declare." - ) - - def summary(self) -> dict[str, Any]: - return { - "id": self.id, - "modalities": self.data.get("modalities", {}), - "tasks": list(self.tasks), - } - def discovery_document(self) -> bytes: # The manifest bytes are spliced in untouched: the spec defines verification as a hash of the served bytes. - head: dict[str, Any] = {"spec": SPEC_VERSION} + document = '{"spec": ' + json.dumps(SPEC_VERSION) if self.source is not None: - head["source"] = self.source - head["hash"] = f"sha256:{hashlib.sha256(self.raw).hexdigest()}" - return json.dumps(head)[:-1].encode() + b', "manifest": ' + self.raw + b"}" + document += ', "source": ' + json.dumps(self.source) + document += f', "hash": "sha256:{hashlib.sha256(self.raw).hexdigest()}"' + document += ', "manifest": ' + return document.encode() + self.raw + b"}" def resolve(self, body: Any) -> ResolvedRequest: """Validate a create-generation request body and map it to the pipeline call.""" - inputs = self._check_body(body) - task = self._select_task(body.get("task"), inputs) - values, explicit = self._parameter_values(task, body) - input_kwargs, media = self._resolve_inputs(task, inputs, values) - parameter_kwargs, seed = self._resolve_parameters(task, values, explicit) - # The spec has the server pick the seed when the request omits it, so the result is reproducible. - if seed is None and "seed" in self._parameter_kwargs[task]: - seed = random.randrange(2**32) - kwargs = {**input_kwargs, **parameter_kwargs} - outputs = self._resolve_outputs(task, body.get("output_format")) - fps = None - if any(kind == "video" for kind, _ in outputs.values()): - # The startup check guarantees a frame lattice when the pipeline has no default frame rate. - duration = self.tasks[task].get("parameters", {}).get("duration") - fps = kwargs.get(self._fps_kwargs[task]) or self._default_fps or _frame_lattice(duration)[2] - - response_format = body.get("response_format", "url") - if response_format not in ("url", "b64_json"): - raise GMSError( - "invalid_request", - f"response_format must be 'url' or 'b64_json', got {response_format!r}", - "/response_format", - ) - - return ResolvedRequest( - task=task, - kwargs=kwargs, - media=media, - seed=seed, - outputs=outputs, - fps=fps, - response_format=response_format, - adapters=self._resolve_adapters(body.get("adapters")), - ) - - def _check_body(self, body: Any) -> dict[str, Any]: if not isinstance(body, dict): raise GMSError("invalid_request", "request body must be a JSON object") for key in body: @@ -569,51 +524,48 @@ def _check_body(self, body: Any) -> dict[str, Any]: raise GMSError("invalid_request", "model is required and must be a string", "/model") if body["model"] != self.id: raise GMSError("not_found", f"model {body['model']!r} is not served here; this server hosts {self.id!r}") - if not isinstance(body.get("inputs"), dict): + inputs = body.get("inputs") + if not isinstance(inputs, dict): raise GMSError("invalid_request", "inputs is required and must be an object", "/inputs") - return body["inputs"] - def _select_task(self, requested: Any, inputs: dict[str, Any]) -> str: - if requested is not None: - if not isinstance(requested, str) or requested not in self.tasks: + # The task: the one requested, else the only one whose inputs match what was supplied. + task = body.get("task") + if task is not None: + if not isinstance(task, str) or task not in self.tasks: raise GMSError( "invalid_request", - f"task {requested!r} is not hosted for {self.id!r}; hosted tasks: {list(self.tasks)}", + f"task {task!r} is not hosted for {self.id!r}; hosted tasks: {list(self.tasks)}", "/task", ) - return requested - - supplied = set(inputs) - candidates = [] - for name, task in self.tasks.items(): - required = {key for key, spec in task["inputs"].items() if spec.get("required")} - if required <= supplied <= set(task["inputs"]): - candidates.append(name) - if len(candidates) == 1: - return candidates[0] - if len(self.tasks) == 1: - return next(iter(self.tasks)) - raise GMSError( - "invalid_request", - f"could not infer the task from inputs {sorted(supplied)}; set `task` to one of {list(self.tasks)}", - "/task", - ) - - def _parameter_values(self, task: str, body: dict[str, Any]) -> tuple[dict[str, Any], set[str]]: - """The parameter values a request runs with: declared defaults, then the preset, then the request's own. + else: + candidates = [] + for name, spec in self.tasks.items(): + required = {key for key, input_spec in spec["inputs"].items() if input_spec.get("required")} + if required <= set(inputs) <= set(spec["inputs"]): + candidates.append(name) + if len(candidates) == 1: + task = candidates[0] + elif len(self.tasks) == 1: + task = list(self.tasks)[0] + else: + raise GMSError( + "invalid_request", + f"could not infer the task from inputs {sorted(inputs)}; set `task` to one of {list(self.tasks)}", + "/task", + ) + declared_inputs = self.tasks[task]["inputs"] + declared_parameters = self.tasks[task].get("parameters", {}) - Also returns the names the request set itself, which are held to stricter rules than defaults. - """ - declared = self.tasks[task].get("parameters", {}) + # The parameter values: declared defaults, then the preset, then the request's own. Values the request set + # itself are held to stricter rules than defaults. explicit = body.get("parameters", {}) if not isinstance(explicit, dict): raise GMSError("invalid_request", "parameters must be an object", "/parameters") for key, value in explicit.items(): - if key not in declared: + if key not in declared_parameters: raise GMSError("invalid_request", f"task {task!r} declares no parameter {key!r}", f"/parameters/{key}") - _check_value(key, value, declared[key], f"/parameters/{key}") - - values = {key: spec["default"] for key, spec in declared.items() if "default" in spec} + _check_value(key, value, declared_parameters[key], f"/parameters/{key}") + values = {key: spec["default"] for key, spec in declared_parameters.items() if "default" in spec} preset = body.get("preset") if preset is not None: presets = self.tasks[task].get("presets", {}) @@ -625,18 +577,13 @@ def _parameter_values(self, task: str, body: dict[str, Any]) -> tuple[dict[str, ) values.update(presets[preset].get("parameters", {})) values.update(explicit) - return values, set(explicit) - def _resolve_inputs( - self, task: str, inputs: dict[str, Any], values: dict[str, Any] - ) -> tuple[dict[str, Any], dict[str, tuple[str, str, Any]]]: - """Map the request inputs to pipeline kwargs. Text is passed as is; media is checked and left to load.""" - declared = self.tasks[task]["inputs"] + # Inputs: text is passed as is; media is checked here and loaded by the backend. kwargs: dict[str, Any] = {} media: dict[str, tuple[str, str, Any]] = {} for key, value in inputs.items(): pointer = f"/inputs/{key}" - spec = declared.get(key) + spec = declared_inputs.get(key) if spec is None: raise GMSError("invalid_request", f"task {task!r} declares no input {key!r}", pointer) kwarg = self._input_kwargs[task][key] @@ -662,20 +609,14 @@ def _resolve_inputs( else: _check_media(key, value, spec, pointer) media[kwarg] = (key, spec["type"], value) - - for key, spec in declared.items(): + for key, spec in declared_inputs.items(): if spec.get("required") and key not in inputs: raise GMSError("invalid_request", f"task {task!r} requires input {key!r}", f"/inputs/{key}") - return kwargs, media - def _resolve_parameters( - self, task: str, values: dict[str, Any], explicit: set[str] - ) -> tuple[dict[str, Any], int | None]: - """Map the parameter values to pipeline kwargs, and pull out the seed, which seeds a generator instead.""" - kwargs: dict[str, Any] = {} + # Parameters. The seed is pulled out, since it seeds a generator instead of being passed as is. seed = None for key, value in values.items(): - spec = self.tasks[task]["parameters"][key] + spec = declared_parameters[key] kwarg = self._parameter_kwargs[task][key] pointer = f"/parameters/{key}" if "active_when" in spec and not _is_active(spec["active_when"], values): @@ -700,94 +641,139 @@ def _resolve_parameters( f"when duration is set, got {values['fps']}", "/parameters/fps", ) - kwargs[kwarg] = _duration_to_frames(value, spec, pointer) + step, offset, _ = _frame_lattice(spec) + position = (value * lattice_fps - offset) / step + snap = spec["lattice"].get("snap", "nearest") + if snap == "reject" and not math.isclose(position, round(position), abs_tol=1e-9): + raise GMSError( + "invalid_request", + f"duration {value} is off the {spec['lattice']['frames']} frame lattice at {lattice_fps} fps", + pointer, + ) + index = math.ceil(position - 1e-9) if snap == "up" else round(position) + kwargs[kwarg] = step * max(index, 0) + offset if self._fps_kwargs[task] is not None: kwargs[self._fps_kwargs[task]] = lattice_fps continue kwargs[kwarg] = value - return kwargs, seed + # The spec has the server pick the seed when the request omits it, so the result is reproducible. + if seed is None and "seed" in self._parameter_kwargs[task]: + seed = random.randrange(2**32) - def _resolve_outputs(self, task: str, output_format: Any) -> dict[str, tuple[str, str]]: + # Outputs, each with the media type it is encoded as. output_formats = {kind: formats[0] for kind, formats in OUTPUT_FORMATS.items()} + output_format = body.get("output_format") if output_format is not None: kind = output_format.split("/")[0] if isinstance(output_format, str) else None produced = {spec["type"] for spec in self.tasks[task]["outputs"].values()} - if kind not in produced or output_format not in OUTPUT_FORMATS.get(kind, ()): - supported = sorted(f for k in produced for f in OUTPUT_FORMATS.get(k, ())) + # The startup check limits output types to those in OUTPUT_FORMATS. + if kind not in produced or output_format not in OUTPUT_FORMATS[kind]: + supported = [] + for produced_kind in produced: + supported.extend(OUTPUT_FORMATS[produced_kind]) + supported.sort() raise GMSError( "invalid_request", f"output_format must be one of {supported} for task {task!r}, got {output_format!r}", "/output_format", ) output_formats[kind] = output_format - return { + outputs = { name: (spec["type"], output_formats[spec["type"]]) for name, spec in self.tasks[task]["outputs"].items() } + fps = None + if any(kind == "video" for kind, _ in outputs.values()): + # The startup check guarantees a frame lattice when the pipeline has no default frame rate. + fps = ( + kwargs.get(self._fps_kwargs[task]) + or self._default_fps + or _frame_lattice(declared_parameters.get("duration"))[2] + ) - def _resolve_adapters(self, adapters: Any) -> list[dict[str, Any]]: - """Validate the request adapters, and resolve each to its path and the weight `set_adapters` takes.""" - if adapters is None: - return [] - if "adapters" not in self.capabilities: - raise GMSError("not_implemented", "this server does not apply adapters", "/adapters") - if not isinstance(adapters, list): - raise GMSError("invalid_request", "adapters must be an array", "/adapters") - if len(adapters) > MAX_ADAPTERS: + response_format = body.get("response_format", "url") + if response_format not in ("url", "b64_json"): raise GMSError( - "invalid_request", f"at most {MAX_ADAPTERS} adapters per request, got {len(adapters)}", "/adapters" + "invalid_request", + f"response_format must be 'url' or 'b64_json', got {response_format!r}", + "/response_format", ) - resolved = [] - for index, adapter in enumerate(adapters): - pointer = f"/adapters/{index}" - if not isinstance(adapter, dict): - raise GMSError("invalid_request", "an adapter must be an object", pointer) - for key in adapter: - if key not in _ADAPTER_FIELDS: - raise GMSError("invalid_request", f"unknown adapter field {key!r}", f"{pointer}/{key}") - if not isinstance(adapter.get("type"), str) or adapter["type"] not in self.adapters: - raise GMSError( - "invalid_request", - f"adapter type must be one of {list(self.adapters)}, got {adapter.get('type')!r}", - f"{pointer}/type", - ) - path = adapter.get("path") - repo_id = path.removeprefix("hf:") if isinstance(path, str) and path.startswith("hf:") else None - if repo_id is not None: - try: - validate_repo_id(repo_id) - except HFValidationError: - repo_id = None - # `load_lora_weights` prefers a local path over a Hub repo of the same name, so a repo id that also - # names a path on the server's disk is refused as well. - if repo_id is None or "/" not in repo_id or os.path.exists(repo_id): + + # Adapters, each resolved to its path and the weight `set_adapters` takes. + adapters = [] + if body.get("adapters") is not None: + if "adapters" not in self.capabilities: + raise GMSError("not_implemented", "this server does not apply adapters", "/adapters") + if not isinstance(body["adapters"], list): + raise GMSError("invalid_request", "adapters must be an array", "/adapters") + if len(body["adapters"]) > MAX_ADAPTERS: raise GMSError( "invalid_request", - f"adapter path must be a Hub repo id of the form 'hf:/', got {path!r}", - f"{pointer}/path", + f"at most {MAX_ADAPTERS} adapters per request, got {len(body['adapters'])}", + "/adapters", ) - scale = adapter.get("scale", 1.0) - if isinstance(scale, bool) or not isinstance(scale, (int, float)): - raise GMSError("invalid_request", f"adapter scale must be a number, got {scale!r}", f"{pointer}/scale") - valid_targets = self.adapters[adapter["type"]].get("targets", []) - targets = adapter.get("targets", {}) - if not isinstance(targets, dict): - raise GMSError("invalid_request", "adapter targets must be an object", f"{pointer}/targets") - for target, target_scale in targets.items(): - if target not in valid_targets: + for index, adapter in enumerate(body["adapters"]): + pointer = f"/adapters/{index}" + if not isinstance(adapter, dict): + raise GMSError("invalid_request", "an adapter must be an object", pointer) + for key in adapter: + if key not in _ADAPTER_FIELDS: + raise GMSError("invalid_request", f"unknown adapter field {key!r}", f"{pointer}/{key}") + if not isinstance(adapter.get("type"), str) or adapter["type"] not in self.adapters: raise GMSError( "invalid_request", - f"adapter target must be one of {valid_targets}, got {target!r}", - f"{pointer}/targets/{target}", + f"adapter type must be one of {list(self.adapters)}, got {adapter.get('type')!r}", + f"{pointer}/type", ) - if isinstance(target_scale, bool) or not isinstance(target_scale, (int, float)): + path = adapter.get("path") + repo_id = path.removeprefix("hf:") if isinstance(path, str) and path.startswith("hf:") else None + if repo_id is not None: + try: + validate_repo_id(repo_id) + except HFValidationError: + repo_id = None + # `load_lora_weights` prefers a local path over a Hub repo of the same name, so a repo id that also + # names a path on the server's disk is refused as well. + if repo_id is None or "/" not in repo_id or os.path.exists(repo_id): raise GMSError( "invalid_request", - f"adapter target scale must be a number, got {target_scale!r}", - f"{pointer}/targets/{target}", + f"adapter path must be a Hub repo id of the form 'hf:/', got {path!r}", + f"{pointer}/path", + ) + scale = adapter.get("scale", 1.0) + if isinstance(scale, bool) or not isinstance(scale, (int, float)): + raise GMSError( + "invalid_request", f"adapter scale must be a number, got {scale!r}", f"{pointer}/scale" ) - weight = {target: targets.get(target, scale) for target in valid_targets} if targets else scale - resolved.append({"path": path, "weight": weight}) - return resolved + valid_targets = self.adapters[adapter["type"]].get("targets", []) + targets = adapter.get("targets", {}) + if not isinstance(targets, dict): + raise GMSError("invalid_request", "adapter targets must be an object", f"{pointer}/targets") + for target, target_scale in targets.items(): + if target not in valid_targets: + raise GMSError( + "invalid_request", + f"adapter target must be one of {valid_targets}, got {target!r}", + f"{pointer}/targets/{target}", + ) + if isinstance(target_scale, bool) or not isinstance(target_scale, (int, float)): + raise GMSError( + "invalid_request", + f"adapter target scale must be a number, got {target_scale!r}", + f"{pointer}/targets/{target}", + ) + weight = {target: targets.get(target, scale) for target in valid_targets} if targets else scale + adapters.append({"path": path, "weight": weight}) + + return ResolvedRequest( + task=task, + kwargs=kwargs, + media=media, + seed=seed, + outputs=outputs, + fps=fps, + response_format=response_format, + adapters=adapters, + ) # --------------------------------------------------------------------------- @@ -880,17 +866,3 @@ def _frame_lattice(spec: dict[str, Any]) -> tuple[int, int, float] | None: if match is None or not isinstance(lattice.get("fps"), (int, float)): return None return int(match.group(1)), int(match.group(2)), lattice["fps"] - - -def _duration_to_frames(seconds: float, spec: dict[str, Any], pointer: str) -> int: - step, offset, fps = _frame_lattice(spec) - position = (seconds * fps - offset) / step - snap = spec["lattice"].get("snap", "nearest") - if snap == "reject" and not math.isclose(position, round(position), abs_tol=1e-9): - raise GMSError( - "invalid_request", - f"duration {seconds} is off the {spec['lattice']['frames']} frame lattice at {fps} fps", - pointer, - ) - index = math.ceil(position - 1e-9) if snap == "up" else round(position) - return step * max(index, 0) + offset diff --git a/src/diffusers/commands/serve/remote.py b/src/diffusers/commands/serve/remote.py index 4eeeadd7ed49..eec740417d9f 100644 --- a/src/diffusers/commands/serve/remote.py +++ b/src/diffusers/commands/serve/remote.py @@ -98,15 +98,29 @@ def _announce_when_ready(api: HfApi, job: Any, args: Namespace, stopped: threadi return -def _job_script(args: Namespace, task: str) -> str: - """The shell command the Job runs: install the dependencies, then start the server in the foreground.""" +def serve_remote(args: Namespace, task: str) -> None: + if Path(args.model).exists(): + raise SystemExit( + f"--model {args.model!r} is a local path; the Job can't see it. " + "Pass a Hub repo id so the Job can download it." + ) + if args.manifest is not None: + raise SystemExit( + f"--manifest is not supported with --remote: the Job can't read {args.manifest!r}. Upload it as " + f"`gms.json` to the model repo, e.g. `hf upload {args.model} {args.manifest} gms.json`, and the " + "remote server will use it." + ) + + if args.idle_timeout is None: + args.idle_timeout = _DEFAULT_REMOTE_IDLE_TIMEOUT + + # The Job installs the dependencies, then starts the server in the foreground. task_kwargs = _build_task_kwargs(args) for key in _JOB_SET_KEYS: task_kwargs.pop(key, None) task_kwargs["host"] = "0.0.0.0" task_kwargs["port"] = JOB_PORT task_kwargs["idle_timeout"] = args.idle_timeout - dependencies = [*_DEFAULT_REMOTE_DEPS, *_SERVE_DEPS, *(args.dependencies or [])] install = shlex.join(["uv", "pip", "install", "--system", "--break-system-packages", *dependencies]) # An `--image` can put another environment first on PATH, with its own older `diffusers-cli`, so the server @@ -117,24 +131,6 @@ def _job_script(args: Namespace, task: str) -> str: f"{shlex.join(_kwargs_to_argv(task, task_kwargs))} " f'--public-url "https://${{JOB_ID}}--{JOB_PORT}.hf.jobs"' ) - return f"{install} && {start}" - - -def serve_remote(args: Namespace, task: str) -> None: - if Path(args.model).exists(): - raise SystemExit( - f"--model {args.model!r} is a local path; the Job can't see it. " - "Pass a Hub repo id so the Job can download it." - ) - if args.manifest is not None: - raise SystemExit( - f"--manifest is not supported with --remote: the Job can't read {args.manifest!r}. Upload it as " - f"`gms.json` to the model repo, e.g. `hf upload {args.model} {args.manifest} gms.json`, and the " - "remote server will use it." - ) - - if args.idle_timeout is None: - args.idle_timeout = _DEFAULT_REMOTE_IDLE_TIMEOUT hf_token = args.token or get_token() env = {"HF_ENABLE_PARALLEL_LOADING": "1"} if DIFFUSERS_TRUST_REMOTE_KERNELS: @@ -144,7 +140,7 @@ def serve_remote(args: Namespace, task: str) -> None: logger.info(f"starting a Job on flavor={args.flavor!r}...") job = api.run_job( image=args.image or _DEFAULT_REMOTE_IMAGE, - command=["sh", "-c", _job_script(args, task)], + command=["sh", "-c", f"{install} && {start}"], env=env, secrets={"HF_TOKEN": hf_token} if hf_token else None, flavor=args.flavor, diff --git a/src/diffusers/commands/serve/scheduler.py b/src/diffusers/commands/serve/scheduler.py index e74e4f804c8c..131c27c8991b 100644 --- a/src/diffusers/commands/serve/scheduler.py +++ b/src/diffusers/commands/serve/scheduler.py @@ -24,7 +24,7 @@ import uuid from datetime import datetime, timedelta, timezone from pathlib import Path -from typing import Any +from typing import Any, Callable from ...utils import logging from .manifest import GMSError, ResolvedRequest @@ -55,7 +55,7 @@ def __init__(self, request: ResolvedRequest): self.events: list[tuple[str, dict[str, Any] | None]] = [] self.done = False self._lock = threading.Lock() - # Handlers waiting on the event loop for the next event. The worker thread wakes them through their loop. + # Requests waiting on the event loop for the job to change. The worker thread wakes them through their loop. self._waiters: list[tuple[asyncio.AbstractEventLoop, asyncio.Event]] = [] @property @@ -93,8 +93,8 @@ def finish(self, status: str, error: GMSError | None = None) -> None: self.done = True self._notify_waiting_requests() - async def _wait_until(self, condition: Any, timeout: float) -> None: - # Waiting on the event loop, not a pool thread, so any number of clients can wait at once. + async def _wait_until(self, condition: Callable[[], bool], timeout: float) -> None: + # Waiting on the event loop, not a pool thread, so any number of requests can wait at once. loop = asyncio.get_running_loop() give_up_at = loop.time() + timeout while True: @@ -113,7 +113,7 @@ async def _wait_until(self, condition: Any, timeout: float) -> None: def _notify_waiting_requests(self) -> None: for loop, event in self._waiters: - # A loop closed at shutdown has no handler left to wake, and raising here would end the worker. + # A loop closed at shutdown has no request left to wake, and raising here would end the worker. if not loop.is_closed(): loop.call_soon_threadsafe(event.set) @@ -140,9 +140,12 @@ def __init__(self, backends: list[Any], output_dir: str, timeout: float | None = ).start() def failure(self) -> str | None: - failure = next(filter(None, (backend.failure() for backend in self._backends)), None) - if failure is not None or self._timeout is None: - return failure + for backend in self._backends: + failure = backend.failure() + if failure is not None: + return failure + if self._timeout is None: + return None # A timed out generation stops at its next step. One still running long after that is stuck inside a step, # or in a pipeline without step callbacks, and its worker takes no more jobs. now = time.monotonic() From e66798ce55b34f8afba6aab11a56c00c8aec549f Mon Sep 17 00:00:00 2001 From: DN6 Date: Tue, 17 Nov 2026 16:15:42 +0530 Subject: [PATCH 18/21] Remove ParallelBackend --- docs/source/en/using-diffusers/serve.md | 4 +- src/diffusers/commands/serve/__init__.py | 83 +++---- src/diffusers/commands/serve/backends.py | 261 ++-------------------- src/diffusers/commands/serve/manifest.py | 2 +- src/diffusers/commands/serve/remote.py | 4 +- src/diffusers/commands/serve/scheduler.py | 8 +- tests/others/test_cli_serve.py | 187 +--------------- 7 files changed, 68 insertions(+), 481 deletions(-) diff --git a/docs/source/en/using-diffusers/serve.md b/docs/source/en/using-diffusers/serve.md index 8a24b66637f6..1254feab3e55 100644 --- a/docs/source/en/using-diffusers/serve.md +++ b/docs/source/en/using-diffusers/serve.md @@ -155,12 +155,10 @@ server checks the manifest at startup and refuses to start if it declares someth ## Multiple GPUs `--data-parallel N` loads a copy of the pipeline on each of N GPUs and runs up to N generations at once. -`--context-parallel` splits each generation across all visible GPUs with -[context parallelism](../training/distributed_inference#context-parallelism). Don't launch it with `torchrun`. +`--context-parallel` is not supported by `serve` yet. ```bash diffusers serve black-forest-labs/FLUX.1-dev --data-parallel 4 -diffusers serve black-forest-labs/FLUX.1-dev --context-parallel ``` ## Remote serving diff --git a/src/diffusers/commands/serve/__init__.py b/src/diffusers/commands/serve/__init__.py index c39d4aafa7f2..42331d64e40d 100644 --- a/src/diffusers/commands/serve/__init__.py +++ b/src/diffusers/commands/serve/__init__.py @@ -226,17 +226,14 @@ def run(self) -> None: if args.log_level is not None: logging.set_verbosity(logging.get_log_levels_dict()[args.log_level]) - if args.context_parallel and args.device_map is not None: + # TODO(DN6): support --context-parallel once the encoders and the VAE decode are not repeated on every rank. + if args.context_parallel: raise SystemExit( - "--device-map does not apply with --context-parallel: each worker loads the pipeline on its own GPU." + "--context-parallel is not supported by `diffusers-cli serve` yet. Use --data-parallel N to serve " + "on multiple GPUs." ) if args.data_parallel < 1: raise SystemExit(f"--data-parallel must be at least 1, got {args.data_parallel}.") - if args.data_parallel > 1 and args.context_parallel: - raise SystemExit( - "--data-parallel and --context-parallel cannot be combined: --context-parallel already uses " - "every visible GPU for each generation." - ) if args.data_parallel > 1 and args.compile is not None: raise SystemExit( "--compile is not supported with --data-parallel yet: the copies of the pipeline would compile " @@ -265,7 +262,7 @@ def run(self) -> None: ) from e from .app import create_app - from .backends import DiffusersBackend, ParallelBackend, backend_capabilities + from .backends import DiffusersBackend, backend_capabilities from .manifest import load_manifest from .scheduler import Scheduler @@ -282,9 +279,7 @@ def run(self) -> None: token=args.token, ) - if args.context_parallel: - backends = [ParallelBackend(args, manifest)] - elif args.data_parallel > 1: + if args.data_parallel > 1: backends = [] for index in range(args.data_parallel): replica_args = copy.copy(args) @@ -293,38 +288,34 @@ def run(self) -> None: else: backends = [DiffusersBackend(args, manifest)] - try: - seed = args.seed if args.seed is not None else secrets.randbelow(2**32) - # Seeds the generator `Manifest.resolve` draws request seeds from, like vLLM's `--seed`. - random.seed(seed) - timeout = None if args.generation_timeout is None else _duration_to_seconds(args.generation_timeout) - scheduler = Scheduler(backends, args.output_dir, timeout=timeout) - app = create_app( - manifest, - scheduler, - enable_cors=args.enable_cors, - public_url=args.public_url, - ) - out.result( - self.task, - url=f"http://{args.host}:{args.port}", - model=manifest.id, - pipeline_class=pipeline_cls.__name__, - tasks=list(manifest.tasks), - capabilities=manifest.capabilities, - seed=seed, - ) - server = uvicorn.Server( - uvicorn.Config(app, host=args.host, port=args.port, log_level=args.log_level or "info") - ) - if args.idle_timeout is not None: - threading.Thread( - target=_shut_down_when_idle, - args=(server, app, scheduler, args.idle_timeout), - name="diffusers-serve-idle", - daemon=True, - ).start() - server.run() - finally: - for backend in backends: - backend.close() + seed = args.seed if args.seed is not None else secrets.randbelow(2**32) + # Seeds the generator `Manifest.resolve` draws request seeds from, like vLLM's `--seed`. + random.seed(seed) + timeout = None if args.generation_timeout is None else _duration_to_seconds(args.generation_timeout) + scheduler = Scheduler(backends, args.output_dir, timeout=timeout) + app = create_app( + manifest, + scheduler, + enable_cors=args.enable_cors, + public_url=args.public_url, + ) + out.result( + self.task, + url=f"http://{args.host}:{args.port}", + model=manifest.id, + pipeline_class=pipeline_cls.__name__, + tasks=list(manifest.tasks), + capabilities=manifest.capabilities, + seed=seed, + ) + server = uvicorn.Server( + uvicorn.Config(app, host=args.host, port=args.port, log_level=args.log_level or "info") + ) + if args.idle_timeout is not None: + threading.Thread( + target=_shut_down_when_idle, + args=(server, app, scheduler, args.idle_timeout), + name="diffusers-serve-idle", + daemon=True, + ).start() + server.run() diff --git a/src/diffusers/commands/serve/backends.py b/src/diffusers/commands/serve/backends.py index 5c035c87a83e..e36be30f8b26 100644 --- a/src/diffusers/commands/serve/backends.py +++ b/src/diffusers/commands/serve/backends.py @@ -12,13 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""The backends that execute a resolved generation: in the server process, or in one worker process per GPU. - -`DiffusersBackend` runs the pipeline in the server process. `ParallelBackend` serves `--context-parallel`: one worker -process per GPU, all running every generation together. The server process never touches CUDA. It spawns the workers, -sends each generation to all of them over a pipe, and reads the progress and result back from rank 0. Idle workers -wait on their pipe rather than inside a collective, so the process group cannot time out between requests. -""" +"""The backend that executes a resolved generation in the server process.""" from __future__ import annotations @@ -27,15 +21,9 @@ import inspect import io import mimetypes -import multiprocessing -import os -import signal -import socket import tempfile -import threading import wave from argparse import Namespace -from multiprocessing.connection import Connection, wait from pathlib import Path from typing import Any from urllib.parse import urlparse @@ -54,7 +42,7 @@ _get_generator, _load_pipeline, ) -from .manifest import GMSError, Manifest, ResolvedRequest, blocks_signature +from .manifest import GMSError, Manifest, blocks_signature from .scheduler import GenerationJob @@ -77,14 +65,8 @@ def backend_capabilities(args: Namespace, pipeline_cls: type) -> list[str]: capabilities = ["sse", "wait"] # Swapping adapter weights retraces a compiled denoiser on every request, and LoRAs passed at - # startup are part of the deployment rather than something a request may replace. Under context - # parallelism every rank would have to swap them in lockstep, which is untested. - if ( - hasattr(pipeline_cls, "load_lora_weights") - and args.compile is None - and not args.lora - and not args.context_parallel - ): + # startup are part of the deployment rather than something a request may replace. + if hasattr(pipeline_cls, "load_lora_weights") and args.compile is None and not args.lora: capabilities.append("adapters") return capabilities @@ -132,26 +114,8 @@ def _sample_rate(pipeline: Any) -> int: ) -def load_request_media(request: ResolvedRequest) -> dict[str, Any]: - """The pipeline kwargs of a request, with its media inputs fetched and decoded.""" - kwargs = dict(request.kwargs) - for kwarg, (name, kind, value) in request.media.items(): - if isinstance(value, list): - loaded = [_load_media(name, item, f"/inputs/{name}/{index}") for index, item in enumerate(value)] - else: - loaded = [_load_media(name, value, f"/inputs/{name}")] - if kind == "audio": - if kwarg == "initial_audio_waveforms": - kwargs.setdefault("initial_audio_sampling_rate", loaded[0][1]) - loaded = [waveform for waveform, _ in loaded] - kwargs[kwarg] = loaded if isinstance(value, list) else loaded[0] - return kwargs - - class DiffusersBackend: - def __init__(self, args: Namespace, manifest: Manifest, save_outputs: bool = True): - # Under context parallelism every rank runs the call, and only rank 0 writes the files. - self._save_outputs = save_outputs + def __init__(self, args: Namespace, manifest: Manifest): self.pipeline = _load_pipeline(args) self._modular = isinstance(self.pipeline, ModularPipeline) if self._modular: @@ -165,18 +129,21 @@ def __init__(self, args: Namespace, manifest: Manifest, save_outputs: bool = Tru if spec["type"] == "audio": self._sample_rate = _sample_rate(self.pipeline) - def close(self) -> None: - pass - - def failure(self) -> str | None: - return None - def generate(self, generation: GenerationJob) -> dict[str, list[dict[str, Any]]]: - return self.run(load_request_media(generation.request), generation) - - def run(self, kwargs: dict[str, Any], generation: Any) -> dict[str, list[dict[str, Any]]]: - """Call the pipeline with the request's kwargs, media already loaded, and save its outputs.""" + """Call the pipeline with the request's kwargs and media, and save its outputs.""" request = generation.request + kwargs = dict(request.kwargs) + for kwarg, (name, kind, value) in request.media.items(): + if isinstance(value, list): + loaded = [_load_media(name, item, f"/inputs/{name}/{index}") for index, item in enumerate(value)] + else: + loaded = [_load_media(name, value, f"/inputs/{name}")] + if kind == "audio": + if kwarg == "initial_audio_waveforms": + kwargs.setdefault("initial_audio_sampling_rate", loaded[0][1]) + loaded = [waveform for waveform, _ in loaded] + kwargs[kwarg] = loaded if isinstance(value, list) else loaded[0] + device = self.pipeline.device if hasattr(self.pipeline, "device") else torch.device("cpu") if request.seed is not None: # A CUDA generator must sit on the pipeline's own GPU, which is `cuda:1` and up with --data-parallel. @@ -213,8 +180,7 @@ def on_step_end(pipeline, step, timestep, callback_kwargs): finally: if request.adapters: self.pipeline.unload_lora_weights() - # Read first: under context parallelism `cancel_requested` is a collective every rank must join. - if generation.cancel_requested or not self._save_outputs: + if generation.cancel_requested: return {} outputs = {} @@ -301,192 +267,3 @@ def on_step_end(pipeline, step, timestep, callback_kwargs): f"task {request.task!r}", ) return outputs - - -_POLL_SECONDS = 0.1 -_SHUTDOWN_SECONDS = 30.0 - - -class _WorkerJob: - """The parts of a generation `DiffusersBackend.run` reads, inside a worker. - - `cancel_requested` is a collective: rank 0 reads the flag the server sets and broadcasts it, so every rank - interrupts at the same step. A rank that stopped alone would leave the others waiting in the next collective. - """ - - def __init__(self, job: dict[str, Any], rank: int, conn: Connection, cancel_flag: Any): - self.request = job["request"] - self.directory = Path(job["directory"]) - self._rank = rank - self._conn = conn - self._cancel_flag = cancel_flag - - def report_progress(self, step: int, total_steps: int) -> None: - if self._rank == 0: - self._conn.send(("progress", step, total_steps)) - - @property - def cancel_requested(self) -> bool: - flag = torch.tensor([self._cancel_flag.value if self._rank == 0 else 0], dtype=torch.uint8) - torch.distributed.broadcast(flag, src=0) - return bool(flag.item()) - - -def _serve_jobs(backend: Any, rank: int, conn: Connection, cancel_flag: Any) -> None: - while True: - job = conn.recv() - if job is None: - return - generation = _WorkerJob(job, rank, conn, cancel_flag) - try: - reply = ("complete", backend.run(job["kwargs"], generation)) - except GMSError as e: - reply = ("failed", e.code, e.message, e.pointer) - # The helpers shared with `run` raise SystemExit, which must not end the worker. - except (Exception, SystemExit) as e: - logger.error(f"generation failed on rank {rank}", exc_info=True) - reply = ("failed", "internal", f"{type(e).__name__}: {e}", None) - if rank == 0: - conn.send(reply) - - -def _worker_main( - rank: int, world_size: int, port: int, args: Namespace, manifest: Manifest, conn: Connection, cancel_flag: Any -) -> None: - # The server process decides when workers stop, so Ctrl-C in the terminal must not interrupt them mid-call. - signal.signal(signal.SIGINT, signal.SIG_IGN) - os.environ.update( - MASTER_ADDR="127.0.0.1", - MASTER_PORT=str(port), - RANK=str(rank), - LOCAL_RANK=str(rank), - WORLD_SIZE=str(world_size), - ) - if args.log_level is not None: - logging.set_verbosity(logging.get_log_levels_dict()[args.log_level]) - # Loading applies `--context-parallel`, which joins the process group of all ranks. - backend = DiffusersBackend(args, manifest, save_outputs=rank == 0) - conn.send(("ready",)) - try: - _serve_jobs(backend, rank, conn, cancel_flag) - finally: - torch.distributed.destroy_process_group() - - -class ParallelBackend: - def __init__(self, args: Namespace, manifest: Manifest): - world_size = torch.cuda.device_count() - if world_size < 2: - raise SystemExit(f"--context-parallel needs at least 2 visible GPUs, found {world_size}.") - with socket.socket() as probe: - probe.bind(("127.0.0.1", 0)) - port = probe.getsockname()[1] - - self._failure: str | None = None - self._failed = threading.Event() - self._closing = False - # Spawned rather than forked: a forked child inherits whatever CUDA state its parent has. - context = multiprocessing.get_context("spawn") - # Lock-free: a worker killed while holding the lock would leave the server blocked on its next write. - self._cancel_flag = context.Value("b", 0, lock=False) - self._conns: list[Connection] = [] - self._processes = [] - for rank in range(world_size): - parent_conn, child_conn = context.Pipe() - process = context.Process( - target=_worker_main, - args=(rank, world_size, port, args, manifest, child_conn, self._cancel_flag), - name=f"diffusers-serve-rank-{rank}", - daemon=True, - ) - process.start() - child_conn.close() - self._conns.append(parent_conn) - self._processes.append(process) - - logger.info(f"loading the pipeline on {world_size} GPUs...") - # Every rank is watched at once: a rank that dies while loading leaves the others stuck in the process - # group setup, so waiting on one rank at a time could hang forever. - loading = dict(zip(self._conns, self._processes)) - while loading: - sentinels = {process.sentinel: process for process in loading.values()} - for ready in wait([*loading, *sentinels]): - if ready in sentinels: - process = sentinels[ready] - else: - process = loading.pop(ready) - try: - if ready.recv() == ("ready",): - continue - except EOFError: - pass - self.close() - raise SystemExit( - f"Context-parallel worker {process.name} exited with code {process.exitcode} while loading " - "the pipeline. Its error is in the log above." - ) - threading.Thread(target=self._monitor, name="diffusers-serve-workers", daemon=True).start() - - def _monitor(self) -> None: - sentinels = {process.sentinel: process for process in self._processes} - dead = sentinels[wait(list(sentinels))[0]] - if self._closing: - return - # `exitcode` polls without waiting, so it can still be None just after the sentinel fires. - dead.join() - self._failure = f"context-parallel worker {dead.name} exited with code {dead.exitcode}" - self._failed.set() - logger.error(f"{self._failure}; the remaining workers are stopped and the server no longer generates.") - # The other ranks would wait forever in their next collective, so the group is not reused. - self.close() - - def failure(self) -> str | None: - return self._failure - - def generate(self, generation: GenerationJob) -> dict[str, list[dict[str, Any]]]: - if self._failure is not None: - raise GMSError("internal", self._failure) - job = { - "request": generation.request, - "kwargs": load_request_media(generation.request), - "directory": str(generation.directory), - } - self._cancel_flag.value = 0 - for conn in self._conns: - conn.send(job) - - rank_0 = self._conns[0] - while True: - if generation.cancel_requested: - self._cancel_flag.value = 1 - try: - if not rank_0.poll(_POLL_SECONDS): - if self._failure is not None: - raise GMSError("internal", self._failure) - continue - message = rank_0.recv() - except (EOFError, OSError) as e: - # Rank 0's pipe only breaks once it has exited. Only the monitor reaps workers: a second - # thread joining the same process can read its exit code before it is recorded. - self._failed.wait() - raise GMSError("internal", self._failure) from e - if message[0] == "progress": - generation.report_progress(*message[1:]) - elif message[0] == "complete": - return message[1] - else: - _, code, text, pointer = message - raise GMSError(code, text, pointer) - - def close(self) -> None: - self._closing = True - for conn in self._conns: - try: - conn.send(None) - except OSError: - pass - for process in self._processes: - process.join(timeout=_SHUTDOWN_SECONDS) - if process.is_alive(): - process.kill() - process.join() diff --git a/src/diffusers/commands/serve/manifest.py b/src/diffusers/commands/serve/manifest.py index e2b115009d7f..c3e1043a9c71 100644 --- a/src/diffusers/commands/serve/manifest.py +++ b/src/diffusers/commands/serve/manifest.py @@ -426,7 +426,7 @@ def __init__( if capability == "adapters": raise SystemExit( message + f"but request adapters are off: {self._pipeline_name} cannot load LoRAs, or the server " - "was started with --compile, --lora or --context-parallel." + "was started with --compile or --lora." ) raise SystemExit( message + f"which this server does not provide. It provides: {list(supported_capabilities)}." diff --git a/src/diffusers/commands/serve/remote.py b/src/diffusers/commands/serve/remote.py index eec740417d9f..6f073e79b65e 100644 --- a/src/diffusers/commands/serve/remote.py +++ b/src/diffusers/commands/serve/remote.py @@ -25,8 +25,9 @@ from huggingface_hub import HfApi, get_token from huggingface_hub.cli._output import out -from huggingface_hub.utils import httpx +from huggingface_hub.utils import httpx, send_telemetry +from ... import __version__ from ...utils import logging from ...utils.constants import DIFFUSERS_TRUST_REMOTE_KERNELS from ..run import _DEFAULT_REMOTE_DEPS, _DEFAULT_REMOTE_IMAGE, _build_task_kwargs, _kwargs_to_argv @@ -148,6 +149,7 @@ def serve_remote(args: Namespace, task: str) -> None: expose=[JOB_PORT], namespace=args.namespace, ) + send_telemetry(topic="diffusers/cli/serve/remote", library_name="diffusers", library_version=__version__) stage = None stopped = threading.Event() diff --git a/src/diffusers/commands/serve/scheduler.py b/src/diffusers/commands/serve/scheduler.py index 131c27c8991b..c2436992d888 100644 --- a/src/diffusers/commands/serve/scheduler.py +++ b/src/diffusers/commands/serve/scheduler.py @@ -49,7 +49,8 @@ def __init__(self, request: ResolvedRequest): self.usage: dict[str, Any] | None = None self.error: GMSError | None = None self.canceled = False - # When a running generation must stop, from `--generation-timeout`; None when it has no limit. + # The `time.monotonic()` time a running generation must stop by, from `--generation-timeout`; None when it has + # no limit. self.deadline: float | None = None self.directory: Path | None = None self.events: list[tuple[str, dict[str, Any] | None]] = [] @@ -125,7 +126,6 @@ class Scheduler: """ def __init__(self, backends: list[Any], output_dir: str, timeout: float | None = None): - self._backends = backends self._output_dir = Path(output_dir) self._timeout = timeout self._generations: dict[str, GenerationJob] = {} @@ -140,10 +140,6 @@ def __init__(self, backends: list[Any], output_dir: str, timeout: float | None = ).start() def failure(self) -> str | None: - for backend in self._backends: - failure = backend.failure() - if failure is not None: - return failure if self._timeout is None: return None # A timed out generation stops at its next step. One still running long after that is stuck inside a step, diff --git a/tests/others/test_cli_serve.py b/tests/others/test_cli_serve.py index 75be9580ae93..f4205df4f0d0 100644 --- a/tests/others/test_cli_serve.py +++ b/tests/others/test_cli_serve.py @@ -21,7 +21,6 @@ import inspect import io import json -import os import re import shlex import threading @@ -150,7 +149,6 @@ def _args(**overrides): values = { "compile": None, "lora": None, - "context_parallel": False, } return Namespace(**{**values, **overrides}) @@ -512,13 +510,6 @@ def test_undeclared_capabilities_are_off(self, serve): assert (events.status_code, events.json()["error"]["code"]) == (501, "not_implemented") pipeline.release.set() - def test_failed_backend_is_unhealthy(self, serve): - backend = SimpleNamespace(failure=lambda: "context-parallel worker rank-1 exited with code 3") - client, _ = serve(backend=backend) - response = client.get("/health") - expected = {"status": "unhealthy", "reason": "context-parallel worker rank-1 exited with code 3"} - assert (response.status_code, response.json()) == (503, expected) - def test_health_checks_do_not_count_as_activity(self, serve): client, _ = serve() client.app.state.last_request = 0.0 @@ -628,6 +619,7 @@ def __init__(self, logs=("server started",)): self.logs = logs self.run_kwargs = None self.cancelled = False + self.telemetry = [] def run_job(self, **kwargs): self.run_kwargs = kwargs @@ -650,6 +642,9 @@ class TestServeRemote: def _run(self, monkeypatch, api, argv): monkeypatch.setattr("diffusers.commands.serve.remote.HfApi", lambda token: api) monkeypatch.setattr("diffusers.commands.serve.remote.get_token", lambda: "hf_x") + monkeypatch.setattr( + "diffusers.commands.serve.remote.send_telemetry", lambda **kwargs: api.telemetry.append(kwargs["topic"]) + ) parser = ArgumentParser() ServeCommand.register_subcommand(parser.add_subparsers()) ServeCommand(parser.parse_args(["serve", "org/model", "--remote", *argv])).run() @@ -679,6 +674,7 @@ def test_remote_starts_the_server_in_a_job(self, monkeypatch): assert options["--idle-timeout"] == "30m", "a remote server should shut itself down when idle by default" assert not {"--remote", "--flavor", "--dependencies"} & set(serve_argv), serve_argv assert not api.cancelled, "a Job that completed on its own needs no cancel" + assert api.telemetry == ["diffusers/cli/serve/remote"], api.telemetry def test_interrupt_cancels_the_job(self, monkeypatch): api = FakeJobsApi(logs=("server started", KeyboardInterrupt())) @@ -692,9 +688,6 @@ def __init__(self): self.started = threading.Event() self.release = threading.Event() - def failure(self): - return None - def generate(self, generation): self.started.set() self.release.wait(timeout=30) @@ -725,9 +718,6 @@ def test_generation_past_the_timeout_fails(self, tmp_path): class SteppingBackend: """Stops at the next step once interrupted, like a pipeline's step callback.""" - def failure(self): - return None - def generate(self, generation): while not generation.cancel_requested: time.sleep(0.01) @@ -808,170 +798,3 @@ def stop(args): ServeCommand(self._parse(["org/model"])).run() assert resolved["model"] == "org/model", resolved assert resolved["dtype"] == "bf16", "serve should default to bf16 rather than `run`'s fp32 `auto`" - - -def _fake_worker_main(rank, world_size, port, args, manifest, conn, cancel_flag): - """Stands in for `_worker_main` in forked children: no pipeline, no process group.""" - if args.fail_at_startup: - os._exit(3) - conn.send(("ready",)) - while True: - job = conn.recv() - if job is None: - return - if job["request"].seed == -1: - os._exit(4) - if rank != 0: - continue - conn.send(("progress", 1, 2)) - if job["kwargs"]["prompt"] == "cancel me": - # The server raises the flag once it has read the progress above. - for _ in range(1000): - if cancel_flag.value: - break - time.sleep(0.01) - conn.send(("complete", {} if cancel_flag.value else {"image": []})) - elif job["kwargs"]["prompt"] == "bad": - conn.send(("failed", "invalid_request", "bad prompt", "/inputs/prompt")) - else: - conn.send(("complete", {"image": [{"file": "image-0.png", "world_size": world_size}]})) - - -class FakeGeneration: - def __init__(self, prompt="a cat", seed=0): - self.request = SimpleNamespace(kwargs={"prompt": prompt}, media={}, task="text_to_image", seed=seed) - self.directory = Path("/tmp/unused") - self.cancel_requested = False - self.progress = [] - - def report_progress(self, step, total_steps): - self.progress.append((step, total_steps)) - if self.request.kwargs["prompt"] == "cancel me": - self.cancel_requested = True - - -class TestParallelBackend: - @pytest.fixture - def backend(self, monkeypatch): - import multiprocessing - - from diffusers.commands.serve import backends - - if "fork" not in multiprocessing.get_all_start_methods(): - pytest.skip("the fake workers are forked so they can use a test-local target") - fork = multiprocessing.get_context("fork") - monkeypatch.setattr(backends.multiprocessing, "get_context", lambda method: fork) - monkeypatch.setattr(backends, "_worker_main", _fake_worker_main) - monkeypatch.setattr(torch.cuda, "device_count", lambda: 2) - built = [] - - def _build(**arg_overrides): - args = Namespace(**{"fail_at_startup": False, **arg_overrides}) - built.append(backends.ParallelBackend(args, _manifest())) - return built[-1] - - yield _build - for backend in built: - backend.close() - - def test_generation_runs_on_every_rank_and_rank_0_answers(self, backend): - generation = FakeGeneration() - outputs = backend().generate(generation) - assert outputs == {"image": [{"file": "image-0.png", "world_size": 2}]}, outputs - assert generation.progress == [(1, 2)], "rank 0's progress should reach the generation" - - def test_worker_error_becomes_a_request_error(self, backend): - with pytest.raises(GMSError, match="bad prompt") as error: - backend().generate(FakeGeneration(prompt="bad")) - assert (error.value.code, error.value.pointer) == ("invalid_request", "/inputs/prompt") - - def test_cancel_reaches_the_workers(self, backend): - outputs = backend().generate(FakeGeneration(prompt="cancel me")) - assert outputs == {}, "a cancel raised mid-generation should reach the workers" - - def test_dead_worker_fails_the_server(self, backend): - server = backend() - with pytest.raises(GMSError, match="exited"): - server.generate(FakeGeneration(seed=-1)) - assert "exited with code 4" in server.failure(), server.failure() - with pytest.raises(GMSError, match="exited with code 4"): - server.generate(FakeGeneration()) - - def test_worker_dying_at_startup_stops_the_server(self, backend): - with pytest.raises(SystemExit, match="exited with code 3 while loading the pipeline"): - backend(fail_at_startup=True) - - def test_worker_arguments_survive_spawn(self): - import pickle - - parser = ArgumentParser() - ServeCommand.register_subcommand(parser.add_subparsers()) - args = parser.parse_args(["serve", "org/model", "--context-parallel"]) - # Spawned workers receive the parsed flags and the manifest pickled. - restored_args, restored_manifest = pickle.loads(pickle.dumps((args, _manifest()))) - assert restored_args.context_parallel and restored_manifest.tasks - - -class TestParallelWorker: - def _jobs(self, backend, rank, *jobs): - from multiprocessing import Pipe, Value - - from diffusers.commands.serve.backends import _serve_jobs - - server, worker = Pipe() - for job in (*jobs, None): - server.send(job) - _serve_jobs(backend, rank, worker, Value("b", 0)) - replies = [] - while server.poll(): - replies.append(server.recv()) - return replies - - def _job(self, prompt): - return {"request": None, "kwargs": {"prompt": prompt}, "directory": "/tmp/unused"} - - def test_only_rank_0_reports_progress_and_results(self): - def run(kwargs, generation): - generation.report_progress(1, 1) - if kwargs["prompt"] == "bad": - raise GMSError("invalid_request", "bad prompt", "/inputs/prompt") - if kwargs["prompt"] == "crash": - raise SystemExit("unsupported") - return {"image": []} - - backend = SimpleNamespace(run=run) - replies = self._jobs(backend, 0, self._job("ok"), self._job("bad"), self._job("crash")) - assert replies == [ - ("progress", 1, 1), - ("complete", {"image": []}), - ("progress", 1, 1), - ("failed", "invalid_request", "bad prompt", "/inputs/prompt"), - ("progress", 1, 1), - ("failed", "internal", "SystemExit: unsupported", None), - ], "a failed generation must not end the worker" - assert self._jobs(backend, 1, self._job("ok"), self._job("bad")) == [], "only rank 0 talks to the server" - - def test_cancel_is_broadcast_from_rank_0(self, monkeypatch): - from multiprocessing import Value - - from diffusers.commands.serve.backends import _WorkerJob - - monkeypatch.setenv("MASTER_ADDR", "127.0.0.1") - monkeypatch.setenv("MASTER_PORT", str(_free_port())) - torch.distributed.init_process_group("gloo", rank=0, world_size=1) - try: - flag = Value("b", 0) - generation = _WorkerJob(self._job("ok"), 0, None, flag) - assert not generation.cancel_requested - flag.value = 1 - assert generation.cancel_requested, "rank 0 should broadcast the flag the server raised" - finally: - torch.distributed.destroy_process_group() - - -def _free_port(): - import socket - - with socket.socket() as probe: - probe.bind(("127.0.0.1", 0)) - return probe.getsockname()[1] From 196fb0fb191fa54b3dd28c708b4f87cf47a05145 Mon Sep 17 00:00:00 2001 From: DN6 Date: Wed, 18 Nov 2026 16:28:39 +0530 Subject: [PATCH 19/21] Harden serve media inputs, support modular step callbacks, and simplify manifest derivation --- docs/source/en/using-diffusers/serve.md | 32 +++++- src/diffusers/commands/serve/__init__.py | 22 +++- src/diffusers/commands/serve/backends.py | 101 +++++++++++++---- src/diffusers/commands/serve/manifest.py | 95 +++++++--------- src/diffusers/commands/serve/remote.py | 17 +-- tests/others/test_cli_serve.py | 133 ++++++++++++++++++++++- 6 files changed, 307 insertions(+), 93 deletions(-) diff --git a/docs/source/en/using-diffusers/serve.md b/docs/source/en/using-diffusers/serve.md index 1254feab3e55..aa6bad0160f7 100644 --- a/docs/source/en/using-diffusers/serve.md +++ b/docs/source/en/using-diffusers/serve.md @@ -26,7 +26,7 @@ option. ## Generations -Create a generation. `Prefer: wait` holds the response until it finishes, for up to 120 seconds here: +Create a generation. `Prefer: wait` holds the response until it finishes, for up to 120 seconds here (the server caps it at 300): ```bash curl localhost:8000/v1/generations -H 'Prefer: wait=120' -d '{ @@ -60,6 +60,22 @@ curl localhost:8000/v1/generations -H 'Prefer: wait=120' -d '{ } ``` +Query the server from Python with [httpx](https://www.python-httpx.org/): + +```py +import httpx + +response = httpx.post( + "http://localhost:8000/v1/generations", + headers={"Prefer": "wait=120"}, + json={"model": "black-forest-labs/FLUX.1-dev", "inputs": {"prompt": "an astronaut riding a horse"}}, + timeout=130, +) +image_url = response.json()["outputs"]["image"][0]["url"] +with open("image.png", "wb") as f: + f.write(httpx.get(image_url).content) +``` + Without `Prefer: wait`, the response comes back at once with `"status": "queued"`. Follow its progress as server-sent events, or poll it: @@ -152,6 +168,20 @@ server checks the manifest at startup and refuses to start if it declares someth - `--generation-timeout 10m` fails a generation that runs longer than 10 minutes. - `--idle-timeout 30m` shuts the server down after 30 minutes without requests. +## Security + +The server has no authentication and listens on `127.0.0.1`. Before exposing it, put it behind a proxy that +authenticates requests. With `--remote`, requests already need your Hugging Face token. + +The server downloads media inputs given as URLs. By default it only fetches public addresses, up to 256 MB, and doesn't +follow redirects. To allow only specific hosts, list them: + +```bash +diffusers serve black-forest-labs/FLUX.1-dev --allowed-media-domains upload.wikimedia.org +``` + +Listed hosts are fetched even when they're private, such as `localhost` while testing. + ## Multiple GPUs `--data-parallel N` loads a copy of the pipeline on each of N GPUs and runs up to N generations at once. diff --git a/src/diffusers/commands/serve/__init__.py b/src/diffusers/commands/serve/__init__.py index 42331d64e40d..8e65f3754980 100644 --- a/src/diffusers/commands/serve/__init__.py +++ b/src/diffusers/commands/serve/__init__.py @@ -14,8 +14,8 @@ """`diffusers-cli serve` — serve a pipeline over HTTP with the Generative Media Spec (GMS) API. -Generations run one at a time on a background worker: a request returns immediately with a queued generation that -clients poll or follow over server-sent events. +Generations run on background workers, one per pipeline copy: a request returns immediately with a queued generation +that clients poll or follow over server-sent events. """ from __future__ import annotations @@ -67,7 +67,13 @@ def _resolve_pipeline(args: Namespace) -> tuple[type, inspect.Signature, str | N "exported by the installed diffusers." ) if not issubclass(pipeline_cls, diffusers.ModularPipeline): - return pipeline_cls, inspect.signature(pipeline_cls.__call__), commit_hash + # Evaluated, so a module with `from __future__ import annotations` gives types rather than strings. A + # kwarg whose annotation doesn't evaluate is left out of a derived manifest. + try: + signature = inspect.signature(pipeline_cls.__call__, eval_str=True) + except (NameError, TypeError): + signature = inspect.signature(pipeline_cls.__call__) + return pipeline_cls, signature, commit_hash try: pipeline = diffusers.ModularPipeline.from_pretrained( @@ -170,6 +176,16 @@ def register_subcommand(subparsers: _SubParsersAction) -> None: "Defaults to a random seed chosen at startup." ), ) + parser.add_argument( + "--allowed-media-domains", + action="append", + default=None, + metavar="HOST", + help=( + "Only fetch media input URLs from this host (e.g. upload.wikimedia.org). Repeat to allow several. " + "Defaults to any host with a public address." + ), + ) parser.add_argument( "--enable-cors", action="store_true", diff --git a/src/diffusers/commands/serve/backends.py b/src/diffusers/commands/serve/backends.py index e36be30f8b26..111c4b5dd8b6 100644 --- a/src/diffusers/commands/serve/backends.py +++ b/src/diffusers/commands/serve/backends.py @@ -20,12 +20,14 @@ import contextlib import inspect import io +import ipaddress import mimetypes +import socket import tempfile import wave from argparse import Namespace from pathlib import Path -from typing import Any +from typing import Any, Callable from urllib.parse import urlparse import numpy as np @@ -33,15 +35,10 @@ from huggingface_hub.utils import httpx from PIL import Image -from ...modular_pipelines import ModularPipeline +from ...modular_pipelines import InputParam, LoopSequentialPipelineBlocks, ModularPipeline, ModularPipelineBlocks from ...utils import export_to_video, load_image, load_video, logging, numpy_to_pil from ...utils.constants import DIFFUSERS_REQUEST_TIMEOUT -from ..run import ( - _as_audio_arrays, - _as_pil_list, - _get_generator, - _load_pipeline, -) +from ..run import _as_audio_arrays, _as_pil_list, _get_generator, _load_pipeline from .manifest import GMSError, Manifest, blocks_signature from .scheduler import GenerationJob @@ -51,6 +48,9 @@ # zlib level for PNG outputs. Pillow's default of 6 takes close to four times as long to encode a # 1024x1024 image for a file about 7% smaller; the output is lossless either way. PNG_COMPRESS_LEVEL = 1 +# Caps each media URL download, like vLLM's `VLLM_MAX_MEDIA_DOWNLOAD_SIZE_MB`, so a request can't make the server read +# an unbounded file into memory. +MAX_MEDIA_DOWNLOAD_BYTES = 256 * 2**20 _OUTPUT_FIELDS = {"image": ("images",), "video": ("frames", "videos"), "audio": ("audios", "audio")} _SAMPLE_RATE_ATTRIBUTES = ( "sample_rate", @@ -71,15 +71,36 @@ def backend_capabilities(args: Namespace, pipeline_cls: type) -> list[str]: return capabilities -def _load_media(name: str, value: dict[str, Any], pointer: str) -> Any: +def _load_media(name: str, value: dict[str, Any], pointer: str, allowed_domains: list[str] | None) -> Any: try: if "base64" in value: content = base64.b64decode(value["base64"], validate=True) else: - response = httpx.get(value["url"], follow_redirects=True, timeout=DIFFUSERS_REQUEST_TIMEOUT) - response.raise_for_status() - content = response.content - except (ValueError, httpx.HTTPError) as e: + url = urlparse(value["url"]) + if allowed_domains is not None: + if url.hostname not in allowed_domains: + raise ValueError(f"host {url.hostname!r} is not in --allowed-media-domains {allowed_domains}") + else: + # Only public addresses are fetched, so a request can't reach the server's own network, e.g. a cloud + # metadata endpoint. The operator vouches for the hosts on an allowlist. + default_port = 443 if url.scheme == "https" else 80 + for info in socket.getaddrinfo(url.hostname, url.port or default_port): + if not ipaddress.ip_address(info[4][0].split("%")[0]).is_global: + raise ValueError( + f"host {url.hostname!r} resolves to the non-public address {info[4][0]}. Pass " + "--allowed-media-domains to fetch from it." + ) + # Redirects are not followed, so a public URL can't send the server to an address on its own network. + with httpx.stream( + "GET", value["url"], follow_redirects=False, timeout=DIFFUSERS_REQUEST_TIMEOUT + ) as response: + response.raise_for_status() + content = bytearray() + for chunk in response.iter_bytes(): + content += chunk + if len(content) > MAX_MEDIA_DOWNLOAD_BYTES: + raise ValueError(f"the download is larger than {MAX_MEDIA_DOWNLOAD_BYTES // 2**20} MB") + except (ValueError, OSError, httpx.HTTPError) as e: raise GMSError("invalid_request", f"could not read input {name!r}: {e}", pointer) from e try: @@ -114,20 +135,52 @@ def _sample_rate(pipeline: Any) -> int: ) +class _Canceled(BaseException): + """Stops a modular pipeline from its step callback. Not an `Exception`, so the blocks don't log it as an error.""" + + +# TODO(DN6): remove once modular pipelines stream their steps with `pipe.stream()` (#14159). +class _StepCallback(ModularPipelineBlocks): + """Calls `callback_on_step_end` after each denoising step of a modular pipeline, as a `DiffusionPipeline` does. + + Modular pipelines have no `num_timesteps` attribute, so the step count is passed in `callback_kwargs`. + """ + + @property + def inputs(self): + return [InputParam("callback_on_step_end", type_hint=Callable), InputParam("timesteps")] + + def __call__(self, components, block_state, i, t): + if block_state.callback_on_step_end is not None: + block_state.callback_on_step_end(components, i, t, {"num_timesteps": len(block_state.timesteps)}) + return components, block_state + + +def _denoise_loops(block: ModularPipelineBlocks) -> list[LoopSequentialPipelineBlocks]: + loops = [] + if isinstance(block, LoopSequentialPipelineBlocks) and "denoiser" in block.sub_blocks: + loops.append(block) + for sub_block in getattr(block, "sub_blocks", {}).values(): + loops.extend(_denoise_loops(sub_block)) + return loops + + class DiffusersBackend: def __init__(self, args: Namespace, manifest: Manifest): self.pipeline = _load_pipeline(args) + self._allowed_media_domains = args.allowed_media_domains self._modular = isinstance(self.pipeline, ModularPipeline) if self._modular: + # `pipeline.blocks` returns a copy, and the pipeline is already built, so the loops are edited in place. + for loop in _denoise_loops(self.pipeline._blocks): + loop.sub_blocks["step_callback"] = _StepCallback() parameters = blocks_signature(self.pipeline.blocks).parameters else: parameters = inspect.signature(type(self.pipeline).__call__).parameters self._reports_steps = "callback_on_step_end" in parameters self._sample_rate = None - for task in manifest.tasks.values(): - for spec in task["outputs"].values(): - if spec["type"] == "audio": - self._sample_rate = _sample_rate(self.pipeline) + if any(spec["type"] == "audio" for task in manifest.tasks.values() for spec in task["outputs"].values()): + self._sample_rate = _sample_rate(self.pipeline) def generate(self, generation: GenerationJob) -> dict[str, list[dict[str, Any]]]: """Call the pipeline with the request's kwargs and media, and save its outputs.""" @@ -135,9 +188,12 @@ def generate(self, generation: GenerationJob) -> dict[str, list[dict[str, Any]]] kwargs = dict(request.kwargs) for kwarg, (name, kind, value) in request.media.items(): if isinstance(value, list): - loaded = [_load_media(name, item, f"/inputs/{name}/{index}") for index, item in enumerate(value)] + loaded = [ + _load_media(name, item, f"/inputs/{name}/{index}", self._allowed_media_domains) + for index, item in enumerate(value) + ] else: - loaded = [_load_media(name, value, f"/inputs/{name}")] + loaded = [_load_media(name, value, f"/inputs/{name}", self._allowed_media_domains)] if kind == "audio": if kwarg == "initial_audio_waveforms": kwargs.setdefault("initial_audio_sampling_rate", loaded[0][1]) @@ -152,10 +208,13 @@ def generate(self, generation: GenerationJob) -> dict[str, list[dict[str, Any]]] if self._reports_steps: def on_step_end(pipeline, step, timestep, callback_kwargs): - total_steps = getattr(pipeline, "num_timesteps", None) + total_steps = callback_kwargs.get("num_timesteps", getattr(pipeline, "num_timesteps", None)) if total_steps: generation.report_progress(step + 1, total_steps) if generation.cancel_requested: + # A block can't end the loop it runs in, so a modular pipeline is stopped by raising. + if self._modular: + raise _Canceled pipeline._interrupt = True return callback_kwargs @@ -177,6 +236,8 @@ def on_step_end(pipeline, step, timestep, callback_kwargs): # Worker threads start on GPU 0, and with --data-parallel this pipeline may sit on another one. with torch.cuda.device(device) if device.type == "cuda" else contextlib.nullcontext(): result = self.pipeline(**kwargs) + except _Canceled: + return {} finally: if request.adapters: self.pipeline.unload_lora_weights() diff --git a/src/diffusers/commands/serve/manifest.py b/src/diffusers/commands/serve/manifest.py index c3e1043a9c71..ecfcf7fad1c9 100644 --- a/src/diffusers/commands/serve/manifest.py +++ b/src/diffusers/commands/serve/manifest.py @@ -29,6 +29,8 @@ import os import random import re +import types +import typing from dataclasses import dataclass from pathlib import Path from typing import Any @@ -104,7 +106,8 @@ } _SKIPPED_KWARGS = frozenset({"output_type", "return_dict", "callback_steps", "callback_on_step_end_tensor_inputs"}) _AUDIO_OUTPUT_KWARGS = frozenset({"audio_length_in_s", "audio_end_in_s", "num_waveforms_per_prompt"}) -_SCALAR_TYPES = {"float": "number", "int": "integer", "bool": "boolean", "str": "string"} +# In order of preference when an annotation allows several, so `int | float` is a number. +_SCALAR_TYPES = {float: "number", int: "integer", bool: "boolean", str: "string"} _PYTHON_TYPES = {"number": (int, float), "integer": int, "boolean": bool, "string": str} _COMPARISONS = { "eq": operator.eq, @@ -216,10 +219,10 @@ def blocks_signature(blocks: Any) -> inspect.Signature: if input_param.name is None: continue annotation = inspect.Parameter.empty if input_param.type_hint is None else input_param.type_hint + # A required input is given no default, as a required argument of a `__call__` has none. + default = inspect.Parameter.empty if input_param.required else input_param.default parameters.append( - inspect.Parameter( - input_param.name, inspect.Parameter.KEYWORD_ONLY, default=input_param.default, annotation=annotation - ) + inspect.Parameter(input_param.name, inspect.Parameter.KEYWORD_ONLY, default=default, annotation=annotation) ) return inspect.Signature(parameters) @@ -235,12 +238,9 @@ def derive_manifest( parameter_kwargs: dict[str, str] = {} # Pipeline kwarg -> its GMS parameter name. `num_frames` keeps its name: `duration` is in seconds, which only a # written manifest's frame lattice converts. - kwarg_parameters = {} - for name, kwargs in _PARAMETER_KWARGS.items(): - if name == "duration": - continue - for kwarg in kwargs: - kwarg_parameters[kwarg] = name + kwarg_parameters = { + kwarg: name for name, kwargs in _PARAMETER_KWARGS.items() if name != "duration" for kwarg in kwargs + } for kwarg, parameter in signature.parameters.items(): if kwarg == "self" or kwarg in _SKIPPED_KWARGS: @@ -267,38 +267,10 @@ def derive_manifest( parameter_kwargs["seed"] = kwarg continue - # The GMS schema of the annotation, if it is a scalar or a list of scalars. annotation = parameter.annotation - if annotation is inspect.Parameter.empty: - text = type(default).__name__ if default is not None else "" - elif isinstance(annotation, str): - text = annotation - elif type(annotation) is type: - text = annotation.__name__ - else: - text = str(annotation) - text = text.replace("typing.", "") - # The type names in the annotation, without the wrappers around them: {"str"} for `Optional[List[str]]`. - type_names = set(re.findall(r"[A-Za-z_][\w.]*", text)) - { - "Optional", - "Union", - "None", - "NoneType", - "list", - "List", - } - schema = None - if type_names and type_names <= set(_SCALAR_TYPES): - for python_type, gms_type in _SCALAR_TYPES.items(): - if python_type in type_names: - scalar_type = gms_type - break - # `str | list[str]` is a scalar that also accepts a batch; only a bare list is an array parameter. - outside_lists = re.sub(r"\b[Ll]ist\[[^\[\]]*\]", "", text) - if outside_lists == text or set(re.findall(r"[A-Za-z_][\w.]*", outside_lists)) & set(_SCALAR_TYPES): - schema = {"type": scalar_type} - else: - schema = {"type": "array", "items": {"type": scalar_type}} + if annotation is inspect.Parameter.empty and default is not None: + annotation = type(default) + schema = _parameter_schema(annotation) # Modular blocks often declare `prompt` without a type hint or a default to infer one from. if schema is None and kwarg == "prompt" and annotation is inspect.Parameter.empty: schema = {"type": "string"} @@ -381,6 +353,26 @@ def task(required: tuple[str, ...], excluded: set[str]) -> dict[str, Any]: return manifest +def _parameter_schema(annotation: Any) -> dict[str, Any] | None: + """The GMS schema of a scalar annotation, a list of scalars, or a union of them; None for anything else.""" + if typing.get_origin(annotation) in (typing.Union, types.UnionType): + # Any member's values are valid, so members that aren't scalars or lists of them, like a tensor, are left out. + options = [schema for schema in map(_parameter_schema, typing.get_args(annotation)) if schema is not None] + # `str | list[str]` is a scalar that also accepts a batch; only a bare list is an array parameter. + options = [option for option in options if option["type"] != "array"] or options + order = list(_SCALAR_TYPES.values()) + return min(options, key=lambda option: order.index(option.get("items", option)["type"]), default=None) + if typing.get_origin(annotation) is list: + args = typing.get_args(annotation) + items = _parameter_schema(args[0]) if args else None + if items is None or items["type"] == "array": + return None + return {"type": "array", "items": items} + if annotation in _SCALAR_TYPES: + return {"type": _SCALAR_TYPES[annotation]} + return None + + # --------------------------------------------------------------------------- # The manifest bound to the pipeline: startup checks, discovery and request resolution # --------------------------------------------------------------------------- @@ -483,14 +475,8 @@ def __init__( ) def _pipeline_kwarg(self, name: str, aliases: dict[str, tuple[str, ...]], declared: dict[str, str]) -> str | None: - if name in declared: - candidates = [declared[name]] - else: - candidates = [*aliases.get(name, ()), name] - for kwarg in candidates: - if kwarg in self._kwarg_names: - return kwarg - return None + candidates = [declared[name]] if name in declared else [*aliases.get(name, ()), name] + return next((kwarg for kwarg in candidates if kwarg in self._kwarg_names), None) def _require_kwarg( self, task: str, kind: str, name: str, aliases: dict[str, tuple[str, ...]], declared: dict[str, str] @@ -537,6 +523,8 @@ def resolve(self, body: Any) -> ResolvedRequest: f"task {task!r} is not hosted for {self.id!r}; hosted tasks: {list(self.tasks)}", "/task", ) + elif len(self.tasks) == 1: + task = next(iter(self.tasks)) else: candidates = [] for name, spec in self.tasks.items(): @@ -545,8 +533,6 @@ def resolve(self, body: Any) -> ResolvedRequest: candidates.append(name) if len(candidates) == 1: task = candidates[0] - elif len(self.tasks) == 1: - task = list(self.tasks)[0] else: raise GMSError( "invalid_request", @@ -668,10 +654,9 @@ def resolve(self, body: Any) -> ResolvedRequest: produced = {spec["type"] for spec in self.tasks[task]["outputs"].values()} # The startup check limits output types to those in OUTPUT_FORMATS. if kind not in produced or output_format not in OUTPUT_FORMATS[kind]: - supported = [] - for produced_kind in produced: - supported.extend(OUTPUT_FORMATS[produced_kind]) - supported.sort() + supported = sorted( + media_type for produced_kind in produced for media_type in OUTPUT_FORMATS[produced_kind] + ) raise GMSError( "invalid_request", f"output_format must be one of {supported} for task {task!r}, got {output_format!r}", diff --git a/src/diffusers/commands/serve/remote.py b/src/diffusers/commands/serve/remote.py index 6f073e79b65e..e7b6a4f7ff61 100644 --- a/src/diffusers/commands/serve/remote.py +++ b/src/diffusers/commands/serve/remote.py @@ -44,9 +44,6 @@ _JOB_TIMEOUT = "24h" _READY_POLL_SECONDS = 5.0 -# Flags the Job command sets itself. `_build_task_kwargs` already drops the flags that say how the Job is set up. -_JOB_SET_KEYS = ("host", "port", "public_url", "output_dir") - def add_remote_arguments(parser: ArgumentParser) -> None: parser.add_argument( @@ -112,16 +109,12 @@ def serve_remote(args: Namespace, task: str) -> None: "remote server will use it." ) - if args.idle_timeout is None: - args.idle_timeout = _DEFAULT_REMOTE_IDLE_TIMEOUT - - # The Job installs the dependencies, then starts the server in the foreground. + # The Job installs the dependencies, then starts the server in the foreground. A local --output-dir means nothing + # inside the Job, and --public-url is set from the Job id below. task_kwargs = _build_task_kwargs(args) - for key in _JOB_SET_KEYS: - task_kwargs.pop(key, None) - task_kwargs["host"] = "0.0.0.0" - task_kwargs["port"] = JOB_PORT - task_kwargs["idle_timeout"] = args.idle_timeout + task_kwargs.pop("output_dir", None) + task_kwargs.pop("public_url", None) + task_kwargs.update(host="0.0.0.0", port=JOB_PORT, idle_timeout=args.idle_timeout or _DEFAULT_REMOTE_IDLE_TIMEOUT) dependencies = [*_DEFAULT_REMOTE_DEPS, *_SERVE_DEPS, *(args.dependencies or [])] install = shlex.join(["uv", "pip", "install", "--system", "--break-system-packages", *dependencies]) # An `--image` can put another environment first on PATH, with its own older `diffusers-cli`, so the server diff --git a/tests/others/test_cli_serve.py b/tests/others/test_cli_serve.py index f4205df4f0d0..3a13a48ce0b7 100644 --- a/tests/others/test_cli_serve.py +++ b/tests/others/test_cli_serve.py @@ -32,14 +32,15 @@ import numpy as np import pytest import torch +from huggingface_hub.utils import httpx from PIL import Image from diffusers.commands.run import _DEFAULT_REMOTE_IMAGE from diffusers.commands.serve import ServeCommand, _shut_down_when_idle -from diffusers.commands.serve.backends import DiffusersBackend, backend_capabilities +from diffusers.commands.serve.backends import DiffusersBackend, _load_media, backend_capabilities from diffusers.commands.serve.manifest import GMSError, Manifest, blocks_signature, derive_manifest from diffusers.commands.serve.scheduler import GenerationJob, Scheduler -from diffusers.modular_pipelines import InputParam, ModularPipelineBlocks, OutputParam +from diffusers.modular_pipelines import InputParam, LoopSequentialPipelineBlocks, ModularPipelineBlocks, OutputParam MODEL_ID = "test-org/fake-model" @@ -145,10 +146,49 @@ def __call__(self, components, state): return components, state +class FakeDenoiser(ModularPipelineBlocks): + @property + def inputs(self): + return [] + + def __call__(self, components, block_state, i, t): + return components, block_state + + +class FakeDenoiseLoop(LoopSequentialPipelineBlocks): + """A modular pipeline whose denoising loop, like the built-in ones, has no step callback of its own.""" + + model_name = "fake" + block_classes = [FakeDenoiser] + block_names = ["denoiser"] + + @property + def description(self): + return "Denoises for `num_inference_steps` steps." + + @property + def loop_inputs(self): + return [InputParam("prompt"), InputParam("num_inference_steps", default=3)] + + @property + def loop_intermediate_outputs(self): + return [OutputParam("images")] + + def __call__(self, components, state): + block_state = self.get_block_state(state) + block_state.timesteps = list(range(block_state.num_inference_steps)) + for i, t in enumerate(block_state.timesteps): + components, block_state = self.loop_step(components, block_state, i=i, t=t) + block_state.images = [Image.new("RGB", (32, 32), "red")] + self.set_block_state(state, block_state) + return components, state + + def _args(**overrides): values = { "compile": None, "lora": None, + "allowed_media_domains": None, } return Namespace(**{**values, **overrides}) @@ -263,6 +303,35 @@ def test_derive_manifest_from_signature(self): assert manifest["capabilities"] == ["sse", "wait", "adapters"] assert manifest["adapters"] == {"lora": {"targets": ["transformer", "text_encoder"]}} + def test_derive_manifest_keeps_the_scalar_members_of_unions(self): + def __call__( + self, + prompt: str | list[str], + strength: float | torch.Tensor = 0.5, + control_mode: int | list[int] | list[list[int]] | None = None, + boxes: list[list[float]] | None = None, + latents: torch.Tensor | None = None, + ): + pass + + manifest = derive_manifest(MODEL_ID, FakePipeline, inspect.signature(__call__), []) + assert manifest["tasks"]["text_to_image"]["parameters"] == { + "strength": {"type": "number", "default": 0.5}, + "control_mode": {"type": "integer"}, + }, "a union should keep its scalar members, and nested lists and tensors should be left out" + + def test_derive_manifest_marks_required_modular_inputs(self): + class RequiredImageBlock(FakeImageBlock): + @property + def inputs(self): + return [InputParam("prompt", required=True), InputParam("image", required=True)] + + pipeline = RequiredImageBlock().init_pipeline() + manifest = derive_manifest(MODEL_ID, type(pipeline), blocks_signature(pipeline.blocks), []) + assert list(manifest["tasks"]) == ["image_to_image"], ( + "a required modular `image` should not yield text_to_image" + ) + @pytest.mark.parametrize( "body, code, pointer", [ @@ -430,6 +499,48 @@ def test_inline_response_and_media_input(self, serve): assert artifact["media_type"] == "image/jpeg" and "url" not in artifact assert Image.open(io.BytesIO(base64.b64decode(artifact["base64"]))).format == "JPEG" + def test_media_urls_are_capped_and_not_redirected(self, monkeypatch): + def respond(request): + if request.url.path == "/redirect": + return httpx.Response(302, headers={"location": "http://169.254.169.254/"}) + return httpx.Response(200, content=b"x" * 16) + + client = httpx.Client(transport=httpx.MockTransport(respond)) + monkeypatch.setattr("diffusers.commands.serve.backends.httpx.stream", client.stream) + monkeypatch.setattr("diffusers.commands.serve.backends.MAX_MEDIA_DOWNLOAD_BYTES", 8) + public = [(None, None, None, "", ("93.184.215.14", 443))] + monkeypatch.setattr("diffusers.commands.serve.backends.socket.getaddrinfo", lambda host, port: public) + for path, message in (("/redirect", "Redirect response"), ("/large", "larger than")): + with pytest.raises(GMSError, match=message) as error: + _load_media("image", {"type": "image", "url": f"https://example.com{path}"}, "/inputs/image", None) + assert error.value.pointer == "/inputs/image", f"{path} should be rejected as a bad input" + + @pytest.mark.parametrize( + "address, allowed_domains, message", + [ + ("169.254.169.254", None, "non-public address"), + ("127.0.0.1", None, "non-public address"), + ("93.184.215.14", ["upload.wikimedia.org"], "not in --allowed-media-domains"), + ], + ) + def test_media_urls_are_limited_to_public_or_allowed_hosts(self, monkeypatch, address, allowed_domains, message): + resolved = [(None, None, None, "", (address, 80))] + monkeypatch.setattr("diffusers.commands.serve.backends.socket.getaddrinfo", lambda host, port: resolved) + value = {"type": "image", "url": "http://media.example/cat.png"} + with pytest.raises(GMSError, match=message): + _load_media("image", value, "/inputs/image", allowed_domains) + + def test_allowed_media_domains_may_be_private(self, monkeypatch): + buffer = io.BytesIO() + Image.new("RGB", (8, 8), "blue").save(buffer, format="PNG") + client = httpx.Client( + transport=httpx.MockTransport(lambda request: httpx.Response(200, content=buffer.getvalue())) + ) + monkeypatch.setattr("diffusers.commands.serve.backends.httpx.stream", client.stream) + value = {"type": "image", "url": "http://localhost:9000/cat.png"} + image = _load_media("image", value, "/inputs/image", ["localhost"]) + assert image.size == (8, 8), "a host on --allowed-media-domains should be fetched even when it is private" + def test_joint_video_and_audio_outputs(self, serve, monkeypatch): monkeypatch.setattr( "diffusers.commands.serve.backends.export_to_video", @@ -613,6 +724,24 @@ def test_generation_runs_a_modular_pipeline(self, serve): (artifact,) = generation["outputs"]["image"] assert (artifact["media_type"], artifact["width"], artifact["height"]) == ("image/png", 32, 32) + def test_modular_pipeline_reports_steps_and_stops_on_cancel(self, monkeypatch, tmp_path): + pipeline = FakeDenoiseLoop().init_pipeline() + monkeypatch.setattr("diffusers.commands.serve.backends._load_pipeline", lambda args: pipeline) + manifest = _manifest(pipeline_cls=type(pipeline), signature=blocks_signature(pipeline.blocks)) + backend = DiffusersBackend(_args(), manifest) + + def generate(cancel): + generation = GenerationJob(manifest.resolve({"model": MODEL_ID, "inputs": {"prompt": "a cat"}})) + generation.directory = tmp_path + generation.canceled = cancel + return backend.generate(generation), [event["step"] for _, event in generation.events] + + outputs, steps = generate(cancel=False) + assert steps == [1, 2, 3], "every denoising step of a modular pipeline should be reported" + assert outputs["image"], outputs + assert generate(cancel=True) == ({}, [1]), "a canceled modular pipeline should stop at its next step" + assert generate(cancel=False)[1] == [1, 2, 3], "the pipeline should stay usable after a cancel" + class FakeJobsApi: def __init__(self, logs=("server started",)): From aa2a41c41cbac878699a5090f1b61e4d154e3c13 Mon Sep 17 00:00:00 2001 From: DN6 Date: Wed, 18 Nov 2026 16:44:12 +0530 Subject: [PATCH 20/21] Drop fraction from progress events --- docs/source/en/using-diffusers/serve.md | 2 +- src/diffusers/commands/serve/scheduler.py | 2 +- tests/others/test_cli_serve.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/en/using-diffusers/serve.md b/docs/source/en/using-diffusers/serve.md index aa6bad0160f7..0d0ecad19305 100644 --- a/docs/source/en/using-diffusers/serve.md +++ b/docs/source/en/using-diffusers/serve.md @@ -86,7 +86,7 @@ curl -N localhost:8000/v1/generations/gen_3f9a1c2b7d4e/events ```text id: 1 event: progress -data: {"step": 1, "total_steps": 28, "fraction": 0.0357} +data: {"step": 1, "total_steps": 28} ... diff --git a/src/diffusers/commands/serve/scheduler.py b/src/diffusers/commands/serve/scheduler.py index c2436992d888..6816fd676088 100644 --- a/src/diffusers/commands/serve/scheduler.py +++ b/src/diffusers/commands/serve/scheduler.py @@ -66,7 +66,7 @@ def cancel_requested(self) -> bool: def report_progress(self, step: int, total_steps: int) -> None: with self._lock: - self.progress = {"step": step, "total_steps": total_steps, "fraction": round(step / total_steps, 4)} + self.progress = {"step": step, "total_steps": total_steps} self.events.append(("progress", self.progress)) self._notify_waiting_requests() diff --git a/tests/others/test_cli_serve.py b/tests/others/test_cli_serve.py index 3a13a48ce0b7..6bdfeab68258 100644 --- a/tests/others/test_cli_serve.py +++ b/tests/others/test_cli_serve.py @@ -463,7 +463,7 @@ def test_generation_lifecycle(self, serve): events = client.get(f"/v1/generations/{generation['id']}/events").text assert events.count("event: progress") == 3, events - assert 'data: {"step": 3, "total_steps": 3, "fraction": 1.0}' in events + assert 'data: {"step": 3, "total_steps": 3}' in events assert events.rstrip().split("\n\n")[-1].startswith("id: 4\nevent: complete\n") # Resuming after the last progress event replays only what followed it. resumed = client.get(f"/v1/generations/{generation['id']}/events", headers={"Last-Event-ID": "3"}).text From 52f2be375dd08fefdc5ed025dbcb46d3d63b3cb0 Mon Sep 17 00:00:00 2001 From: DN6 Date: Wed, 18 Nov 2026 17:22:24 +0530 Subject: [PATCH 21/21] Clean up --- src/diffusers/commands/serve/__init__.py | 20 ++++--- src/diffusers/commands/serve/backends.py | 5 +- src/diffusers/commands/serve/manifest.py | 68 +++++++++++++++++++----- tests/others/test_cli_serve.py | 10 ++++ 4 files changed, 81 insertions(+), 22 deletions(-) diff --git a/src/diffusers/commands/serve/__init__.py b/src/diffusers/commands/serve/__init__.py index 8e65f3754980..f340452f55f1 100644 --- a/src/diffusers/commands/serve/__init__.py +++ b/src/diffusers/commands/serve/__init__.py @@ -21,11 +21,13 @@ from __future__ import annotations import copy +import dataclasses import inspect import random import secrets import threading import time +import typing from argparse import ArgumentParser, Namespace, RawDescriptionHelpFormatter, _SubParsersAction from pathlib import Path from typing import Any @@ -33,7 +35,7 @@ import torch from huggingface_hub.cli._output import out -from ...utils import logging +from ...utils import BaseOutput, logging from .. import BaseDiffusersCLICommand from ..run import _add_loading_arguments, _add_optimization_arguments, _duration_to_seconds from .remote import add_remote_arguments, serve_remote @@ -44,8 +46,9 @@ DEFAULT_OUTPUT_DIR = str(Path.home() / ".diffusers" / "cli" / "serve" / "outputs") -def _resolve_pipeline(args: Namespace) -> tuple[type, inspect.Signature, str | None]: - """Return the pipeline class, the signature of its call, and the commit hash of a standard Hub repo.""" +def _resolve_pipeline(args: Namespace) -> tuple[type, inspect.Signature, set[str] | None, str | None]: + """Return the pipeline class, the signature of its call, the fields of its output, and the commit hash of a + standard Hub repo. The output fields are None when the pipeline doesn't annotate what it returns.""" import diffusers from .manifest import blocks_signature @@ -73,7 +76,10 @@ def _resolve_pipeline(args: Namespace) -> tuple[type, inspect.Signature, str | N signature = inspect.signature(pipeline_cls.__call__, eval_str=True) except (NameError, TypeError): signature = inspect.signature(pipeline_cls.__call__) - return pipeline_cls, signature, commit_hash + returned = typing.get_args(signature.return_annotation) or (signature.return_annotation,) + outputs = [cls for cls in returned if isinstance(cls, type) and issubclass(cls, BaseOutput)] + output_fields = {field.name for field in dataclasses.fields(outputs[0])} if outputs else None + return pipeline_cls, signature, output_fields, commit_hash try: pipeline = diffusers.ModularPipeline.from_pretrained( @@ -81,7 +87,8 @@ def _resolve_pipeline(args: Namespace) -> tuple[type, inspect.Signature, str | N ) except OSError as e: raise SystemExit(f"Could not read a pipeline config for {args.model!r}: {e}") from e - return type(pipeline), blocks_signature(pipeline.blocks), commit_hash + output_fields = {output.name for output in pipeline.blocks.intermediate_outputs} + return type(pipeline), blocks_signature(pipeline.blocks), output_fields, commit_hash def _shut_down_when_idle(server: Any, app: Any, scheduler: Any, idle_timeout: str) -> None: @@ -282,7 +289,7 @@ def run(self) -> None: from .manifest import load_manifest from .scheduler import Scheduler - pipeline_cls, signature, commit_hash = _resolve_pipeline(args) + pipeline_cls, signature, output_fields, commit_hash = _resolve_pipeline(args) capabilities = backend_capabilities(args, pipeline_cls) manifest = load_manifest( args.model, @@ -293,6 +300,7 @@ def run(self) -> None: revision=args.revision, commit_hash=commit_hash, token=args.token, + output_fields=output_fields, ) if args.data_parallel > 1: diff --git a/src/diffusers/commands/serve/backends.py b/src/diffusers/commands/serve/backends.py index 111c4b5dd8b6..778a5dc8ae67 100644 --- a/src/diffusers/commands/serve/backends.py +++ b/src/diffusers/commands/serve/backends.py @@ -39,7 +39,7 @@ from ...utils import export_to_video, load_image, load_video, logging, numpy_to_pil from ...utils.constants import DIFFUSERS_REQUEST_TIMEOUT from ..run import _as_audio_arrays, _as_pil_list, _get_generator, _load_pipeline -from .manifest import GMSError, Manifest, blocks_signature +from .manifest import OUTPUT_FIELDS, GMSError, Manifest, blocks_signature from .scheduler import GenerationJob @@ -51,7 +51,6 @@ # Caps each media URL download, like vLLM's `VLLM_MAX_MEDIA_DOWNLOAD_SIZE_MB`, so a request can't make the server read # an unbounded file into memory. MAX_MEDIA_DOWNLOAD_BYTES = 256 * 2**20 -_OUTPUT_FIELDS = {"image": ("images",), "video": ("frames", "videos"), "audio": ("audios", "audio")} _SAMPLE_RATE_ATTRIBUTES = ( "sample_rate", "vocoder.config.output_sampling_rate", @@ -247,7 +246,7 @@ def on_step_end(pipeline, step, timestep, callback_kwargs): outputs = {} for name, (kind, media_type) in request.outputs.items(): value = None - for field in _OUTPUT_FIELDS[kind]: + for field in OUTPUT_FIELDS[kind]: # A modular pipeline returns its `PipelineState`; a standard one returns an output object. value = result.get(field) if self._modular else getattr(result, field, None) if value is not None: diff --git a/src/diffusers/commands/serve/manifest.py b/src/diffusers/commands/serve/manifest.py index ecfcf7fad1c9..cf45eb635075 100644 --- a/src/diffusers/commands/serve/manifest.py +++ b/src/diffusers/commands/serve/manifest.py @@ -48,9 +48,13 @@ AUTO_TEXT2VIDEO_PIPELINES_MAPPING, AUTO_VIDEO2VIDEO_PIPELINES_MAPPING, ) +from ...utils import logging from ..run import _AUDIO_INPUT_KEYS, _IMAGE_INPUT_KEYS, _VIDEO_INPUT_KEYS +logger = logging.get_logger("diffusers-cli/serve") + + # --------------------------------------------------------------------------- # Constants # --------------------------------------------------------------------------- @@ -60,8 +64,17 @@ EXTENSION_KEY = "x-diffusers" MAX_ADAPTERS = 4 +# The frame rate videos are encoded at when neither the pipeline nor the manifest gives one. Native rates range from 8 +# fps (CogVideoX) to 30 (Mochi), so this is a guess; a `duration` frame lattice in a published manifest sets the real one. +DEFAULT_FPS = 16 # First entry of each tuple is the default when the request names no `output_format`. +# The fields of a pipeline's output that hold each output type, tried in order. +OUTPUT_FIELDS = { + "image": ("images", "image"), + "video": ("frames", "videos", "video"), + "audio": ("audios", "audio", "sound"), +} OUTPUT_FORMATS = { "image": ("image/png", "image/jpeg", "image/webp"), "video": ("video/mp4",), @@ -168,6 +181,7 @@ def load_manifest( revision: str | None = None, commit_hash: str | None = None, token: str | None = None, + output_fields: set[str] | None = None, ) -> Manifest: """Load the model's manifest and bind it to the pipeline: `--manifest`, else the repo's `gms.json`, else derived. @@ -202,9 +216,18 @@ def load_manifest( raise SystemExit(f"GMS manifest {origin!r} declares no `tasks`, so there is nothing to serve.") else: model_id = Path(model).name if Path(model).is_dir() else model - data = derive_manifest(model_id, pipeline_cls, signature, capabilities) + data = derive_manifest(model_id, pipeline_cls, signature, capabilities, output_fields) raw = json.dumps(data).encode() - return Manifest(raw, data, pipeline_cls, signature, capabilities, source=source, revision=commit_hash) + return Manifest( + raw, + data, + pipeline_cls, + signature, + capabilities, + source=source, + revision=commit_hash, + output_fields=output_fields, + ) # --------------------------------------------------------------------------- @@ -228,9 +251,16 @@ def blocks_signature(blocks: Any) -> inspect.Signature: def derive_manifest( - model_id: str, pipeline_cls: type, signature: inspect.Signature, capabilities: list[str] + model_id: str, + pipeline_cls: type, + signature: inspect.Signature, + capabilities: list[str], + output_fields: set[str] | None = None, ) -> dict[str, Any]: - """Build a server-authored manifest from the pipeline call `signature`, declaring the server's `capabilities`.""" + """Build a server-authored manifest from the pipeline call `signature`, declaring the server's `capabilities`. + + `output_fields`, the fields of the pipeline's output when it annotates them, decides the output type. + """ # Sort the call kwargs into GMS inputs and parameters, and record the ones whose GMS name differs. inputs: dict[str, dict[str, Any]] = {} parameters: dict[str, dict[str, Any]] = {} @@ -296,7 +326,10 @@ def derive_manifest( extension["parameters"] = parameter_kwargs registered = [entry for entry in _AUTO_TASKS if pipeline_cls in entry[1].values()] - if registered: + returned = [kind for kind, fields in OUTPUT_FIELDS.items() if output_fields and set(fields) & output_fields] + if returned: + output = returned[0] + elif registered: output = registered[0][3] elif "num_frames" in signature.parameters or "Video" in pipeline_cls.__name__: output = "video" @@ -395,6 +428,7 @@ def __init__( supported_capabilities: list[str], source: str | None = None, revision: str | None = None, + output_fields: set[str] | None = None, ): # The bytes are kept as published: discovery serves them unchanged, and the hash is computed over them. self.raw = raw @@ -451,6 +485,15 @@ def __init__( f"The GMS manifest for {self.id!r} declares {unsupported} outputs on task {name!r}; this server " f"produces {list(OUTPUT_FORMATS)} outputs." ) + # `output_fields` is None when the pipeline doesn't annotate what it returns. + if output_fields is not None: + for spec in task["outputs"].values(): + if not set(OUTPUT_FIELDS[spec["type"]]) & output_fields: + raise SystemExit( + f"Task {name!r} outputs {spec['type']}, read from one of " + f"{list(OUTPUT_FIELDS[spec['type']])}, but {self._pipeline_name} returns " + f"{sorted(output_fields)}. `diffusers serve` only serves image, video and audio outputs." + ) parameters = task.get("parameters", {}) if "duration" in parameters and _frame_lattice(parameters["duration"]) is None: raise SystemExit( @@ -459,12 +502,11 @@ def __init__( ) produces_video = any(spec["type"] == "video" for spec in task["outputs"].values()) if produces_video and self._default_fps is None and "duration" not in parameters: - raise SystemExit( - f"The GMS manifest for {self.id!r} declares a video output on task {name!r}, but " - f"{self._pipeline_name} has no default `fps` or `frame_rate` argument, so the frame rate to encode " - "at is unknown. Declare `duration` with a frame lattice at the model's native fps, e.g. " - '"duration": {"type": "number", "lattice": {"frames": "4k+1", "fps": 16}}, and pass the manifest ' - "with --manifest." + logger.warning( + f"{self._pipeline_name} has no default `fps` or `frame_rate` argument, so task {name!r} encodes " + f"videos at {DEFAULT_FPS} fps, which may not be the model's native rate. To set it, declare " + '`duration` with a frame lattice in a manifest, e.g. "duration": {"type": "number", "lattice": ' + '{"frames": "4k+1", "fps": 24}}, and pass it with --manifest.' ) for preset, body in task.get("presets", {}).items(): for key in body.get("parameters", {}): @@ -668,11 +710,11 @@ def resolve(self, body: Any) -> ResolvedRequest: } fps = None if any(kind == "video" for kind, _ in outputs.values()): - # The startup check guarantees a frame lattice when the pipeline has no default frame rate. + duration = declared_parameters.get("duration") fps = ( kwargs.get(self._fps_kwargs[task]) or self._default_fps - or _frame_lattice(declared_parameters.get("duration"))[2] + or (_frame_lattice(duration)[2] if duration is not None else DEFAULT_FPS) ) response_format = body.get("response_format", "url") diff --git a/tests/others/test_cli_serve.py b/tests/others/test_cli_serve.py index 6bdfeab68258..d4757f385f88 100644 --- a/tests/others/test_cli_serve.py +++ b/tests/others/test_cli_serve.py @@ -303,6 +303,16 @@ def test_derive_manifest_from_signature(self): assert manifest["capabilities"] == ["sse", "wait", "adapters"] assert manifest["adapters"] == {"lora": {"targets": ["transformer", "text_encoder"]}} + def test_output_type_comes_from_the_pipeline_output_fields(self): + manifest = derive_manifest(MODEL_ID, FakePipeline, FAKE_SIGNATURE, [], output_fields={"frames"}) + assert list(manifest["modalities"]["out"]) == ["video"], "a pipeline returning `frames` should output video" + + manifest = derive_manifest(MODEL_ID, FakePipeline, FAKE_SIGNATURE, [], output_fields={"coords_nm"}) + with pytest.raises(SystemExit, match=re.escape("but FakePipeline returns ['coords_nm']")): + Manifest( + json.dumps(manifest).encode(), manifest, FakePipeline, FAKE_SIGNATURE, [], output_fields={"coords_nm"} + ) + def test_derive_manifest_keeps_the_scalar_members_of_unions(self): def __call__( self,