From 70a1993f24aed44eaaae0ba86e5a726cd17888c4 Mon Sep 17 00:00:00 2001 From: asaiyume Date: Tue, 17 Nov 2026 19:31:45 +0800 Subject: [PATCH] Add Comfy checkpoint quantization support --- docs/source/en/_toctree.yml | 2 + docs/source/en/api/quantization.md | 7 + docs/source/en/quantization/comfy.md | 84 ++++ src/diffusers/__init__.py | 20 + src/diffusers/quantizers/auto.py | 4 + src/diffusers/quantizers/comfy/__init__.py | 15 + .../quantizers/comfy/comfy_quantizer.py | 234 ++++++++++ src/diffusers/quantizers/comfy/utils.py | 207 +++++++++ .../quantizers/quantization_config.py | 37 ++ src/diffusers/utils/__init__.py | 1 + src/diffusers/utils/dummy_comfy_objects.py | 17 + src/diffusers/utils/import_utils.py | 5 + tests/quantization/comfy/__init__.py | 0 tests/quantization/comfy/test_comfy.py | 430 ++++++++++++++++++ 14 files changed, 1063 insertions(+) create mode 100644 docs/source/en/quantization/comfy.md create mode 100644 src/diffusers/quantizers/comfy/__init__.py create mode 100644 src/diffusers/quantizers/comfy/comfy_quantizer.py create mode 100644 src/diffusers/quantizers/comfy/utils.py create mode 100644 src/diffusers/utils/dummy_comfy_objects.py create mode 100644 tests/quantization/comfy/__init__.py create mode 100644 tests/quantization/comfy/test_comfy.py diff --git a/docs/source/en/_toctree.yml b/docs/source/en/_toctree.yml index adb0f0bf991b..7ffb827f4d72 100644 --- a/docs/source/en/_toctree.yml +++ b/docs/source/en/_toctree.yml @@ -93,6 +93,8 @@ title: bitsandbytes - local: quantization/bfl title: BFL (fp8r / nvfp4) + - local: quantization/comfy + title: Comfy Quantization - local: quantization/gguf title: gguf - local: quantization/nunchaku diff --git a/docs/source/en/api/quantization.md b/docs/source/en/api/quantization.md index 415762fb1d7b..e23a4102d3d9 100644 --- a/docs/source/en/api/quantization.md +++ b/docs/source/en/api/quantization.md @@ -34,6 +34,13 @@ Quantization techniques reduce memory and computational costs by representing we [[autodoc]] quantizers.quantization_config.GGUFQuantizationConfig +## ComfyQuantizationConfig + +Load prequantized single-file checkpoints with Comfy-Kitchen. See the [Comfy quantization guide](../quantization/comfy) +for usage, validated formats, and checkpoint limitations. + +[[autodoc]] quantizers.quantization_config.ComfyQuantizationConfig + ## NunchakuLiteQuantizationConfig [[autodoc]] quantizers.quantization_config.NunchakuLiteQuantizationConfig diff --git a/docs/source/en/quantization/comfy.md b/docs/source/en/quantization/comfy.md new file mode 100644 index 000000000000..47012847f81f --- /dev/null +++ b/docs/source/en/quantization/comfy.md @@ -0,0 +1,84 @@ + + +# Comfy Quantization + +[Comfy Quants](https://github.com/Comfy-Org/comfy-quants) exports prequantized checkpoints with per-layer +quantization metadata. Diffusers reads each layer's format and uses +[Comfy-Kitchen](https://github.com/Comfy-Org/comfy-kitchen) for inference. A checkpoint can mix quantization +formats and floating-point layers. + +Install the optional dependencies: + +```bash +pip install "comfy-kitchen>=0.2.41" accelerate +``` + +## Load a checkpoint + +Pass [`ComfyQuantizationConfig`] to `from_single_file`. Set `compute_dtype` to BF16 or FP16; the checkpoint +determines the quantization format of each layer. + +```python +import torch +from diffusers import ComfyQuantizationConfig, ZImagePipeline, ZImageTransformer2DModel + +transformer = ZImageTransformer2DModel.from_single_file( + "https://huggingface.co/Comfy-Org/Z-Image-Turbo/blob/main/split_files/diffusion_models/z_image_turbo_int8_convrot.safetensors", + config="Tongyi-MAI/Z-Image-Turbo", + subfolder="transformer", + quantization_config=ComfyQuantizationConfig(compute_dtype=torch.bfloat16), +) +pipe = ZImagePipeline.from_pretrained( + "Tongyi-MAI/Z-Image-Turbo", transformer=transformer, dtype=torch.bfloat16 +) +pipe.enable_model_cpu_offload() +image = pipe( + "A small orange cat sitting on a mossy stone wall in a mountain village", + height=1024, + width=1024, + num_inference_steps=8, + guidance_scale=0.0, + generator=torch.Generator("cpu").manual_seed(0), +).images[0] +``` + +## Validated formats + +The quantizer reads each layer's uint8 JSON `comfy_quant` marker and bundles the packed weight with its scales and other sidecar tensors before checkpoint conversion. + + +| Marker `format` | Stored weights | Associated tensors | +| --- | --- | --- | +| `float8_e4m3fn`, `float8_e5m2` | FP8 | Weight scale, optional activation scale | +| `int8_tensorwise` | INT8 | Scalar or row scales, optional ConvRot metadata | +| `mxfp8` | FP8 | E8M0 block scales | +| `convrot_w4a4` | Packed INT4 | Row scales and rotation metadata | +| `asym_w4a8_int8`, `w6a8_int8` | Packed INT4 or INT6 | Group and channel scales, optional codebook and correction | + +Checkpoint converters retain their usual weight splitting and concatenation. The quantized tensor keeps row +and block scales aligned without dequantizing the stored weights. Concatenation requires compatible +layouts and shared metadata. Input-feature splitting is unsupported. + +One `ComfyLinear` dispatches through the weight's Kitchen layout. Layers marked `full_precision_matrix_mult` +use dequantized matrix multiplication. Unquantized layers use the compute dtype, and model FP32 exclusions +receive dequantized FP32 weights. + +Kitchen selects an available execution backend. Hardware requirements differ by format; loading a checkpoint +does not guarantee native quantized acceleration on the current device. + +### INT8 ConvRot + +[ConvRot](https://arxiv.org/abs/2512.03673) rotates weights offline and activations at inference time to spread +activation outliers. Native `int8_tensorwise` checkpoints retain INT8 weights and FP32 scales. Rotated layers +include `convrot` and `convrot_groupsize` metadata. Both rotated and unrotated INT8 layers are supported. diff --git a/src/diffusers/__init__.py b/src/diffusers/__init__.py index 048f37f4061f..915955c69d8d 100644 --- a/src/diffusers/__init__.py +++ b/src/diffusers/__init__.py @@ -146,6 +146,18 @@ else: _import_structure["quantizers.quantization_config"].append("BFLQuantizationConfig") +try: + if not is_torch_available(): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from .utils import dummy_comfy_objects + + _import_structure["utils.dummy_comfy_objects"] = [ + name for name in dir(dummy_comfy_objects) if not name.startswith("_") + ] +else: + _import_structure["quantizers.quantization_config"].append("ComfyQuantizationConfig") + try: if not is_auto_round_available(): raise OptionalDependencyNotAvailable() @@ -1078,6 +1090,14 @@ else: from .quantizers.quantization_config import BFLQuantizationConfig + try: + if not is_torch_available(): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + from .utils.dummy_comfy_objects import * + else: + from .quantizers.quantization_config import ComfyQuantizationConfig + try: if not is_auto_round_available(): raise OptionalDependencyNotAvailable() diff --git a/src/diffusers/quantizers/auto.py b/src/diffusers/quantizers/auto.py index 377b46b7b361..0d6403990892 100644 --- a/src/diffusers/quantizers/auto.py +++ b/src/diffusers/quantizers/auto.py @@ -21,6 +21,7 @@ from .autoround import AutoRoundQuantizer from .bfl import BFLQuantizer from .bitsandbytes import BnB4BitDiffusersQuantizer, BnB8BitDiffusersQuantizer +from .comfy import ComfyQuantizer from .gguf import GGUFQuantizer from .modelopt import NVIDIAModelOptQuantizer from .nunchaku import NunchakuLiteQuantizer @@ -28,6 +29,7 @@ AutoRoundConfig, BFLQuantizationConfig, BitsAndBytesConfig, + ComfyQuantizationConfig, GGUFQuantizationConfig, NunchakuLiteQuantizationConfig, NVIDIAModelOptConfig, @@ -43,6 +45,7 @@ AUTO_QUANTIZER_MAPPING = { + "comfy": ComfyQuantizer, "bfl": BFLQuantizer, "bitsandbytes_4bit": BnB4BitDiffusersQuantizer, "bitsandbytes_8bit": BnB8BitDiffusersQuantizer, @@ -56,6 +59,7 @@ } AUTO_QUANTIZATION_CONFIG_MAPPING = { + "comfy": ComfyQuantizationConfig, "bfl": BFLQuantizationConfig, "bitsandbytes_4bit": BitsAndBytesConfig, "bitsandbytes_8bit": BitsAndBytesConfig, diff --git a/src/diffusers/quantizers/comfy/__init__.py b/src/diffusers/quantizers/comfy/__init__.py new file mode 100644 index 000000000000..5767537ba2d1 --- /dev/null +++ b/src/diffusers/quantizers/comfy/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2026 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .comfy_quantizer import ComfyQuantizer diff --git a/src/diffusers/quantizers/comfy/comfy_quantizer.py b/src/diffusers/quantizers/comfy/comfy_quantizer.py new file mode 100644 index 000000000000..e5403c941991 --- /dev/null +++ b/src/diffusers/quantizers/comfy/comfy_quantizer.py @@ -0,0 +1,234 @@ +# Copyright 2026 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import importlib.metadata +import json + +import torch +from packaging.version import Version +from torch import nn + +from ...utils import get_module_from_name, is_accelerate_available, is_comfy_kitchen_available +from ..base import DiffusersQuantizer + + +if is_comfy_kitchen_available(): + from comfy_kitchen.tensor import get_layout_class + + from .utils import ComfyLinear, ComfyQuantizedTensor + + +COMFY_FORMATS = { + "float8_e4m3fn": ("TensorCoreFP8Layout", torch.float8_e4m3fn, 8), + "float8_e5m2": ("TensorCoreFP8Layout", torch.float8_e5m2, 8), + "int8_tensorwise": ("TensorWiseINT8Layout", torch.int8, 8), + "nvfp4": ("TensorCoreNVFP4Layout", torch.uint8, 4), + "mxfp8": ("TensorCoreMXFP8Layout", torch.float8_e4m3fn, 8), + "awq_w4a16": ("TensorCoreAWQW4A16Layout", torch.int8, 4), + "svdquant_w4a4": ("TensorCoreSVDQuantW4A4Layout", torch.int8, 4), + "convrot_w4a4": ("TensorCoreConvRotW4A4Layout", torch.int8, 4), + "asym_w4a8_int8": ("AsymW4A8Int8Layout", torch.int8, 4), + "w6a8_int8": ("AsymW4A8Int8Layout", torch.int8, 6), +} + + +class ComfyQuantizer(DiffusersQuantizer): + use_keep_in_fp32_modules = True + + def __init__(self, quantization_config, **kwargs): + quantization_config.post_init() + super().__init__(quantization_config, **kwargs) + self.compute_dtype = quantization_config.compute_dtype + self.pre_quantized = True + self.keep_in_fp32_modules = [] + + def validate_environment(self, *args, **kwargs): + if not is_accelerate_available(): + raise ImportError("Loading Comfy checkpoints requires accelerate: `pip install accelerate`.") + if not is_comfy_kitchen_available(): + raise ImportError("Comfy inference requires comfy-kitchen: `pip install 'comfy-kitchen>=0.2.41'`.") + if Version(importlib.metadata.version("comfy-kitchen")) < Version("0.2.41"): + raise ImportError("Comfy inference requires comfy-kitchen>=0.2.41.") + + def update_torch_dtype(self, torch_dtype): + if torch_dtype is not None and torch_dtype != self.compute_dtype: + raise ValueError("`dtype` must match ComfyQuantizationConfig.compute_dtype.") + return self.compute_dtype + + def maybe_update_state_dict(self, state_dict): + marker_keys = [key for key in state_dict if key.endswith(".comfy_quant")] + if not marker_keys: + raise ValueError("The checkpoint has no native Comfy quantization markers.") + for key in marker_keys: + prefix = key.removesuffix(".comfy_quant") + payload = state_dict.pop(key) + if payload.dtype != torch.uint8 or payload.ndim != 1: + raise ValueError(f"{key} must be a one-dimensional uint8 JSON marker.") + try: + marker = json.loads(bytes(payload.tolist())) + except (ValueError, UnicodeDecodeError) as exc: + raise ValueError(f"{key} is not valid JSON.") from exc + if not isinstance(marker, dict) or marker.get("format") not in COMFY_FORMATS: + raise ValueError(f"{key} must declare a supported Comfy format: {list(COMFY_FORMATS)}.") + quant_format = marker["format"] + layout_name, storage_dtype, bits = COMFY_FORMATS[quant_format] + if f"{prefix}.weight" not in state_dict: + raise ValueError(f"{prefix} requires weight.") + weight = state_dict[f"{prefix}.weight"] + if weight.dtype != storage_dtype or weight.ndim not in (2, 4): + raise ValueError(f"{prefix}.weight has an invalid shape or dtype for {quant_format}.") + if weight.ndim == 4: + if quant_format != "svdquant_w4a4" or tuple(weight.shape[2:]) != (32, 128): + raise ValueError(f"{prefix}.weight has an invalid tile-packed layout.") + shape = (weight.shape[0] * 128, weight.shape[1] * 64) + else: + if weight.shape[1] * 8 % bits: + raise ValueError(f"{prefix}.weight has an invalid packed input width.") + shape = (weight.shape[0], weight.shape[1] * 8 // bits) + + def pop_tensor(suffix, dtype=None): + name = f"{prefix}.{suffix}" + if name not in state_dict: + raise ValueError(f"{prefix} requires {suffix}.") + value = state_dict.pop(name) + return value.view(dtype) if dtype is not None else value + + if not isinstance(marker.get("params", {}), dict): + raise ValueError(f"{key}.params must be an object.") + options = {**marker.get("params", {}), **marker} + params = {"orig_dtype": self.compute_dtype, "orig_shape": shape} + if quant_format == "nvfp4": + params.update( + scale=pop_tensor("weight_scale_2"), block_scale=pop_tensor("weight_scale", torch.float8_e4m3fn) + ) + elif quant_format == "mxfp8": + params["scale"] = pop_tensor("weight_scale", torch.float8_e8m0fnu) + elif quant_format in ("asym_w4a8_int8", "w6a8_int8"): + params.update( + scale=pop_tensor("weight_s_rel", torch.float8_e4m3fn), + s_channel=pop_tensor("weight_s_channel"), + codebook=state_dict.pop(f"{prefix}.weight_codebook", None), + correction=state_dict.pop(f"{prefix}.weight_correction", None), + group_size=options.get("group_size", 16), + convrot_groupsize=options.get("convrot_groupsize", 256), + ) + else: + params["scale"] = pop_tensor("weight_scale") + if quant_format == "int8_tensorwise": + scale = params["scale"] + rows, columns = shape + if scale.dtype != torch.float32 or tuple(scale.shape) not in ((), (1,), (rows,), (rows, 1)): + raise ValueError(f"{prefix}.weight_scale must be FP32 with a scalar or per-output-row shape.") + if not torch.isfinite(scale).all() or not (scale > 0).all(): + raise ValueError(f"{prefix}.weight_scale must contain finite positive values.") + convrot = options.get("convrot", False) + group_size = options.get("convrot_groupsize", 256) + if not isinstance(convrot, bool): + raise ValueError(f"{prefix}.convrot must be a boolean.") + if convrot: + if ( + type(group_size) is not int + or group_size < 4 + or group_size & (group_size - 1) + or (group_size.bit_length() - 1) % 2 + ): + raise ValueError(f"{prefix}.convrot_groupsize must be a power of four, at least 4.") + if columns % group_size or scale.numel() != rows: + raise ValueError(f"{prefix} requires group-aligned input features and per-output-row scales.") + params.update(convrot=convrot, convrot_groupsize=group_size) + if scale.numel() > 1: + params["scale"] = scale.reshape(rows, 1) + elif quant_format == "awq_w4a16": + params.update(zeros=pop_tensor("weight_zero"), group_size=options.get("group_size", 64)) + elif quant_format == "svdquant_w4a4": + params.update( + proj_down=pop_tensor("proj_down"), + proj_up=pop_tensor("proj_up"), + smooth_factor=pop_tensor("smooth_factor"), + act_unsigned=options.get("act_unsigned", False), + ) + elif quant_format == "convrot_w4a4": + params.update( + convrot_groupsize=options.get("convrot_groupsize", 256), + linear_dtype=options.get("linear_dtype", "int4"), + ) + sidecars = { + name: state_dict.pop(f"{prefix}.{name}") + for name in ("input_scale", "pre_quant_scale") + if f"{prefix}.{name}" in state_dict + } + layout = get_layout_class(layout_name) + state_dict[f"{prefix}.weight"] = ComfyQuantizedTensor( + weight, layout_name, layout.Params(**params), marker, sidecars + ) + unmarked = [ + key + for key, value in state_dict.items() + if key.endswith(".weight") and value.dtype in {value[1] for value in COMFY_FORMATS.values()} + ] + if unmarked: + raise ValueError(f"Quantized weights require native Comfy quantization markers: {unmarked}.") + return state_dict + + def _process_model_before_weight_loading( + self, model, device_map, state_dict=None, keep_in_fp32_modules=(), **kwargs + ): + if state_dict is None: + raise ValueError("Comfy checkpoints must be loaded with from_single_file().") + self.keep_in_fp32_modules = [name for name in keep_in_fp32_modules if name is not None] + for prefix, module in list(model.named_modules()): + weight = state_dict.get(f"{prefix}.weight") + if not isinstance(weight, ComfyQuantizedTensor): + continue + if not isinstance(module, nn.Linear) or tuple(weight.shape) != (module.out_features, module.in_features): + raise ValueError(f"{prefix} does not match a Linear layer with shape {tuple(weight.shape)}.") + if any(name in prefix.split(".") for name in self.keep_in_fp32_modules): + continue + parent, name = get_module_from_name(model, prefix) + linear = ComfyLinear( + module.in_features, + module.out_features, + bias=module.bias is not None, + device="meta", + dtype=self.compute_dtype, + ) + linear.bias = module.bias + setattr(parent, name, linear) + + def check_if_quantized_param(self, model, param_value, param_name, state_dict, **kwargs): + return isinstance(param_value, ComfyQuantizedTensor) + + def create_quantized_param( + self, model, param_value, param_name, target_device, state_dict, unexpected_keys=None, **kwargs + ): + module, name = get_module_from_name(model, param_name) + if isinstance(module, ComfyLinear): + value = param_value.to(target_device) + else: + value = param_value.to(torch.float32).dequantize().to(target_device) + module._parameters[name] = nn.Parameter(value, requires_grad=False) + + def _process_model_after_weight_loading(self, model, **kwargs): + missing = [name for name, tensor in model.state_dict().items() if tensor.is_meta] + if missing: + raise ValueError(f"The Comfy checkpoint is missing model tensors: {missing}.") + return model + + @property + def is_serializable(self): + return False + + @property + def is_trainable(self): + return False diff --git a/src/diffusers/quantizers/comfy/utils.py b/src/diffusers/quantizers/comfy/utils.py new file mode 100644 index 000000000000..030fb1e0d401 --- /dev/null +++ b/src/diffusers/quantizers/comfy/utils.py @@ -0,0 +1,207 @@ +# Copyright 2026 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import dataclasses +import math + +import torch +from torch import nn +from torch.nn import functional as F + +from ...utils import is_comfy_kitchen_available + + +if is_comfy_kitchen_available(): + from comfy_kitchen.float_utils import from_blocked, to_blocked + from comfy_kitchen.tensor import QuantizedTensor + + class ComfyQuantizedTensor(QuantizedTensor): + """Packed Comfy weights whose sidecar tensors follow checkpoint row conversions.""" + + def __new__(cls, qdata, layout_cls, params, metadata=None, sidecars=None, logical_shape=None): + return torch.Tensor._make_wrapper_subclass( + cls, + logical_shape or params.orig_shape, + device=qdata.device, + dtype=params.orig_dtype, + requires_grad=False, + ) + + def __init__(self, qdata, layout_cls, params, metadata=None, sidecars=None, logical_shape=None): + super().__init__(qdata, layout_cls, params) + self.metadata = metadata or {} + self.sidecars = sidecars or {} + + def _copy_with(self, qdata=None, params=None, clone_params=True): + if params is None: + params = self.params.clone() if clone_params else self.params + data = self._qdata if qdata is None else qdata + return type(self)( + data, + self._layout_cls, + params, + self.metadata.copy(), + {name: value.to(device=data.device).clone() for name, value in self.sidecars.items()}, + tuple(self.shape), + ) + + def dequantize(self): + return super().dequantize().reshape(self.shape) + + def _row_components(self): + rows, columns = self.params.orig_shape + data = self._qdata + fields = {} + layout = self._layout_cls + if layout == "TensorCoreNVFP4Layout": + fields["block_scale"] = (from_blocked(self.params.block_scale, rows, columns // 16), 0) + elif layout == "TensorCoreMXFP8Layout": + fields["scale"] = (from_blocked(self.params.scale, rows, columns // 32), 0) + elif layout in ("TensorWiseINT8Layout", "TensorCoreConvRotW4A4Layout"): + fields["scale"] = (self.params.scale.reshape(-1, 1).expand(rows, 1), 0) + elif layout in ("TensorCoreAWQW4A16Layout", "TensorCoreSVDQuantW4A4Layout"): + scale = self.params.scale + if layout == "TensorCoreSVDQuantW4A4Layout" and data.ndim == 4: + data = ( + data.reshape(data.shape[0], data.shape[1], 32, 4, 32) + .permute(0, 2, 3, 1, 4) + .reshape(rows, columns // 2) + ) + scale = scale.permute(1, 0, 2).reshape(columns // 64, rows) + fields["proj_up"] = (self.params.proj_up.permute(0, 2, 1).reshape(rows, -1), 0) + elif layout == "TensorCoreSVDQuantW4A4Layout": + fields["proj_up"] = (self.params.proj_up, 0) + fields["scale"] = (scale, 1) + if layout == "TensorCoreAWQW4A16Layout": + fields["zeros"] = (self.params.zeros, 1) + elif layout == "AsymW4A8Int8Layout": + fields["scale"] = (self.params.scale, 0) + fields["s_channel"] = (self.params.s_channel, 0) + if self.params.correction is not None: + fields["correction"] = (self.params.correction, 1) + return data[:rows], fields + + def _from_rows(self, data, fields, shape): + values = {name: value for name, (value, _) in fields.items()} + if self._layout_cls == "TensorCoreNVFP4Layout": + values["block_scale"] = to_blocked(values["block_scale"]) + elif self._layout_cls == "TensorCoreMXFP8Layout": + values["scale"] = to_blocked(values["scale"]) + elif self._layout_cls == "TensorCoreConvRotW4A4Layout": + values["scale"] = values["scale"].flatten() + params = dataclasses.replace(self.params, orig_shape=(math.prod(shape[:-1]), shape[-1]), **values) + return type(self)(data, self._layout_cls, params, self.metadata.copy(), self.sidecars.copy(), tuple(shape)) + + def _select_rows(self, indices, shape): + data, fields = self._row_components() + + def select(tensor, dim): + if tensor.element_size() == 1: + return tensor.view(torch.uint8).index_select(dim, indices).view(tensor.dtype) + return tensor.index_select(dim, indices) + + return self._from_rows( + select(data, 0), {name: (select(value, dim), dim) for name, (value, dim) in fields.items()}, shape + ) + + @classmethod + def __torch_dispatch__(cls, func, types, args=(), kwargs=None): + kwargs = kwargs or {} + if func in (torch.ops.aten.split.Tensor, torch.ops.aten.split_with_sizes.default): + source, sizes = args[:2] + dim = (args[2] if len(args) > 2 else kwargs.get("dim", 0)) % source.ndim + if dim == source.ndim - 1: + raise NotImplementedError("Splitting quantized input features is not supported.") + indices = torch.arange(math.prod(source.shape[:-1]), device=source.device).reshape(source.shape[:-1]) + return [ + source._select_rows(part.flatten(), (*part.shape, source.shape[-1])) + for part in func(indices, sizes, dim) + ] + if func == torch.ops.aten.cat.default: + tensors = args[0] + dim = (args[1] if len(args) > 1 else kwargs.get("dim", 0)) % tensors[0].ndim + if dim != 0 or any(not isinstance(t, cls) or t._layout_cls != tensors[0]._layout_cls for t in tensors): + raise NotImplementedError("Concatenation requires matching Comfy layouts along output rows.") + source = tensors[0] + components = [tensor._row_components() for tensor in tensors] + row_fields = components[0][1] + for tensor, (_, fields) in zip(tensors[1:], components[1:]): + if tensor.shape[1:] != source.shape[1:] or fields.keys() != row_fields.keys(): + raise ValueError("Cannot concatenate incompatible quantized weight shapes or scales.") + if tensor.metadata != source.metadata or tensor.sidecars.keys() != source.sidecars.keys(): + raise ValueError("Cannot concatenate weights with different quantization metadata.") + for name in source.sidecars: + if not torch.equal(source.sidecars[name], tensor.sidecars[name]): + raise ValueError(f"Cannot concatenate weights with different {name}.") + for field in dataclasses.fields(source.params): + if field.name in row_fields or field.name == "orig_shape": + continue + left, right = getattr(source.params, field.name), getattr(tensor.params, field.name) + equal = ( + torch.equal(left, right) + if isinstance(left, torch.Tensor) and isinstance(right, torch.Tensor) + else left == right + ) + if not equal: + raise ValueError(f"Cannot concatenate weights with different {field.name}.") + data = torch.cat([part[0] for part in components], dim=0) + fields = { + name: (torch.cat([part[1][name][0] for part in components], dim=axis), axis) + for name, (_, axis) in row_fields.items() + } + return source._from_rows(data, fields, (sum(t.shape[0] for t in tensors), *source.shape[1:])) + if func in (torch.ops.aten.view.default, torch.ops.aten._unsafe_view.default): + source, shape = args[:2] + shape = list(shape) + if -1 in shape: + shape[shape.index(-1)] = source.numel() // -math.prod(shape) + if shape[-1] != source.shape[-1] or math.prod(shape) != source.numel(): + raise NotImplementedError("Reshaping quantized input features is not supported.") + return cls( + source._qdata, source._layout_cls, source.params, source.metadata, source.sidecars, tuple(shape) + ) + if func in (torch.ops.aten.transpose.int, torch.ops.aten.unbind.int): + source = args[0] + indices = torch.arange(math.prod(source.shape[:-1]), device=source.device).reshape(source.shape[:-1]) + dims = [dim % source.ndim for dim in args[1:]] + if any(dim == source.ndim - 1 for dim in dims): + raise NotImplementedError("Reordering quantized input features is not supported.") + transformed = func(indices, *args[1:], **kwargs) + if isinstance(transformed, torch.Tensor): + return source._select_rows(transformed.flatten(), (*transformed.shape, source.shape[-1])) + return [source._select_rows(part.flatten(), (*part.shape, source.shape[-1])) for part in transformed] + return super().__torch_dispatch__(func, types, args, kwargs) + + +class ComfyLinear(nn.Linear): + """Linear inference dispatched through the weight's Comfy-Kitchen layout.""" + + def forward(self, inputs: torch.Tensor) -> torch.Tensor: + weight = self.weight + inputs = inputs.to(weight.dtype) + shape = inputs.shape + inputs = inputs.reshape(-1, shape[-1]) + pre_quant_scale = weight.sidecars.get("pre_quant_scale") + if pre_quant_scale is not None: + inputs = inputs * pre_quant_scale.to(inputs.dtype) + if weight.metadata.get("full_precision_matrix_mult", False): + output = F.linear(inputs, weight.dequantize().to(weight.dtype), self.bias) + else: + if weight._layout_cls in ("TensorCoreFP8Layout", "TensorCoreNVFP4Layout", "TensorCoreMXFP8Layout"): + quant_kwargs = {"scale": weight.sidecars.get("input_scale")} + if weight._layout_cls == "TensorCoreFP8Layout": + quant_kwargs["dtype"] = weight.storage_dtype + inputs = QuantizedTensor.from_float(inputs, weight._layout_cls, **quant_kwargs) + output = F.linear(inputs, weight, self.bias) + return output.to(weight.dtype).reshape(*shape[:-1], self.out_features) diff --git a/src/diffusers/quantizers/quantization_config.py b/src/diffusers/quantizers/quantization_config.py index 8d7c0c343b19..c63ea256f605 100644 --- a/src/diffusers/quantizers/quantization_config.py +++ b/src/diffusers/quantizers/quantization_config.py @@ -43,6 +43,7 @@ class QuantizationMethod(str, Enum): BITS_AND_BYTES = "bitsandbytes" BFL = "bfl" + COMFY = "comfy" GGUF = "gguf" NUNCHAKU_LITE = "nunchaku_lite" TORCHAO = "torchao" @@ -423,6 +424,42 @@ def __init__(self, compute_dtype: "torch.dtype" | None = None): self.compute_dtype = torch.float32 +@dataclass +class ComfyQuantizationConfig(QuantizationConfigMixin): + """Load native Comfy checkpoints with Comfy-Kitchen kernels. + + Quantization formats are read from each layer's `comfy_quant` metadata in the prequantized checkpoint. + See the [Comfy quantization guide](https://huggingface.co/docs/diffusers/main/en/quantization/comfy) for validated + formats and checkpoint limitations. + + Args: + compute_dtype (`torch.dtype`, defaults to `torch.bfloat16`): + Precision of activations and unquantized layers. Supports BF16 and FP16. + """ + + def __init__(self, compute_dtype: "torch.dtype" | str | None = None, **kwargs): + self.quant_method = QuantizationMethod.COMFY + self.compute_dtype = compute_dtype if compute_dtype is not None else torch.bfloat16 + self.pre_quantized = True + self.post_init() + + def post_init(self): + self.pre_quantized = True + if isinstance(self.compute_dtype, str): + self.compute_dtype = {"bfloat16": torch.bfloat16, "float16": torch.float16}.get(self.compute_dtype) + if self.compute_dtype not in (torch.bfloat16, torch.float16): + raise ValueError("Comfy compute_dtype must be torch.bfloat16 or torch.float16.") + + def to_dict(self): + result = super().to_dict() + result["compute_dtype"] = str(self.compute_dtype).replace("torch.", "") + return result + + def to_diff_dict(self): + default = ComfyQuantizationConfig().to_dict() + return {key: value for key, value in self.to_dict().items() if key == "quant_method" or value != default[key]} + + @dataclass class BFLQuantizationConfig(QuantizationConfigMixin): """Config for loading checkpoints quantized with the Black Forest Labs `fp8r` and `nvfp4` schemes. diff --git a/src/diffusers/utils/__init__.py b/src/diffusers/utils/__init__.py index 1fe67478be24..187944f8a9fc 100644 --- a/src/diffusers/utils/__init__.py +++ b/src/diffusers/utils/__init__.py @@ -78,6 +78,7 @@ is_bitsandbytes_available, is_bitsandbytes_version, is_bs4_available, + is_comfy_kitchen_available, is_cosmos_guardrail_available, is_flash_attn_3_available, is_flash_attn_available, diff --git a/src/diffusers/utils/dummy_comfy_objects.py b/src/diffusers/utils/dummy_comfy_objects.py new file mode 100644 index 000000000000..6b70697eafed --- /dev/null +++ b/src/diffusers/utils/dummy_comfy_objects.py @@ -0,0 +1,17 @@ +# This file is autogenerated by the command `make fix-copies`, do not edit. +from ..utils import DummyObject, requires_backends + + +class ComfyQuantizationConfig(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) diff --git a/src/diffusers/utils/import_utils.py b/src/diffusers/utils/import_utils.py index 020edbd5a4fa..ec94c574f16f 100644 --- a/src/diffusers/utils/import_utils.py +++ b/src/diffusers/utils/import_utils.py @@ -255,6 +255,7 @@ def _is_distribution_installed(name: str) -> tuple[bool, str]: _accelerate_available, _accelerate_version = _is_package_available("accelerate") _xformers_available, _xformers_version = _is_package_available("xformers") _gguf_available, _gguf_version = _is_package_available("gguf") +_comfy_kitchen_available, _comfy_kitchen_version = _is_package_available("comfy_kitchen") _torchao_available, _torchao_version = _is_package_available("torchao") _optimum_quanto_available, _optimum_quanto_version = _is_package_available("optimum", get_dist_name=True) _pytorch_retinaface_available, _pytorch_retinaface_version = _is_package_available("pytorch_retinaface") @@ -410,6 +411,10 @@ def is_gguf_available(): return _gguf_available +def is_comfy_kitchen_available(): + return _comfy_kitchen_available + + def is_flashinfer_available(): return _flashinfer_available diff --git a/tests/quantization/comfy/__init__.py b/tests/quantization/comfy/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/quantization/comfy/test_comfy.py b/tests/quantization/comfy/test_comfy.py new file mode 100644 index 000000000000..819318f6ff2c --- /dev/null +++ b/tests/quantization/comfy/test_comfy.py @@ -0,0 +1,430 @@ +# Copyright 2026 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import importlib.metadata +import json +from unittest.mock import patch + +import pytest +import torch +from packaging.version import Version +from torch import nn +from torch.nn import functional as F + +from diffusers import ComfyQuantizationConfig, QwenImage21Transformer2DModel +from diffusers.loaders.single_file_utils import ( + convert_minimax_h3_transformer_checkpoint_to_diffusers, + convert_qwen_image21_transformer_checkpoint_to_diffusers, + convert_z_image_transformer_checkpoint_to_diffusers, +) +from diffusers.quantizers import DiffusersAutoQuantizer +from diffusers.quantizers.comfy import ComfyQuantizer +from diffusers.utils import is_comfy_kitchen_available + + +if is_comfy_kitchen_available() and Version(importlib.metadata.version("comfy-kitchen")) >= Version("0.2.41"): + import comfy_kitchen as kitchen + from comfy_kitchen.float_utils import from_blocked, to_blocked + from comfy_kitchen.tensor import QuantizedTensor + + from diffusers.quantizers.comfy.comfy_quantizer import COMFY_FORMATS + from diffusers.quantizers.comfy.utils import ComfyLinear, ComfyQuantizedTensor + +else: + pytest.skip("Requires comfy-kitchen>=0.2.41", allow_module_level=True) + + +FORMATS = list(COMFY_FORMATS) + + +def marker(**metadata): + return torch.tensor(list(json.dumps(metadata).encode()), dtype=torch.uint8) + + +def checkpoint(quant_format="int8_tensorwise", prefix="linear", rows=96, columns=256, **metadata): + torch.manual_seed(10) + layout, dtype, bits = COMFY_FORMATS[quant_format] + weight = torch.randint(0, 64, (rows, columns * bits // 8), dtype=torch.uint8).to(dtype) + state = {"weight": weight} + if quant_format == "nvfp4": + state.update( + weight_scale=to_blocked(torch.ones(rows, columns // 16).to(torch.float8_e4m3fn)), + weight_scale_2=torch.tensor(0.01), + ) + elif quant_format == "mxfp8": + state["weight_scale"] = to_blocked(torch.full((rows, columns // 32), 127, dtype=torch.uint8)) + elif quant_format in ("asym_w4a8_int8", "w6a8_int8"): + state.update( + weight_s_rel=torch.ones(rows, columns // 16).to(torch.float8_e4m3fn), + weight_s_channel=torch.full((rows,), 0.01), + ) + elif quant_format == "convrot_w4a4": + state["weight_scale"] = torch.full((rows,), 0.01) + elif quant_format in ("awq_w4a16", "svdquant_w4a4"): + state["weight_scale"] = torch.full((columns // 64, rows), 0.01) + if quant_format == "awq_w4a16": + state["weight_zero"] = torch.randn(columns // 64, rows) * 0.001 + elif quant_format == "svdquant_w4a4": + state.update( + proj_down=torch.randn(columns, 4) * 0.01, + proj_up=torch.randn(rows, 4) * 0.01, + smooth_factor=torch.ones(columns), + ) + else: + state["weight_scale"] = ( + torch.linspace(0.001, 0.004, rows).reshape(rows, 1) + if quant_format == "int8_tensorwise" + else torch.tensor(0.01) + ) + state["comfy_quant"] = marker(format=quant_format, **metadata) + return {f"{prefix}.{name}": value for name, value in state.items()} + + +def bundle(state): + return ComfyQuantizer(ComfyQuantizationConfig()).maybe_update_state_dict(state) + + +@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16]) +def test_config_and_registration(dtype): + config = ComfyQuantizationConfig(compute_dtype=dtype, pre_quantized=False) + assert config.pre_quantized is True + assert not hasattr(config, "quant_type") + quantizer = DiffusersAutoQuantizer.from_config(config.to_dict()) + assert isinstance(quantizer, ComfyQuantizer) + assert quantizer.compute_dtype == dtype + assert quantizer.pre_quantized is True + assert json.loads(config.to_json_string(use_diff=False))["compute_dtype"] == str(dtype).removeprefix("torch.") + with pytest.raises(ValueError, match="compute_dtype"): + ComfyQuantizationConfig(compute_dtype=torch.float32) + + +@pytest.mark.parametrize("quant_format", FORMATS) +def test_layer_metadata_and_tensor_lifecycle(quant_format): + state = checkpoint(quant_format) + original = state["linear.weight"].clone() + q = bundle(state)["linear.weight"] + assert list(state) == ["linear.weight"] + assert isinstance(q, QuantizedTensor) + assert q.shape == (96, 256) + assert q.dtype == torch.bfloat16 + torch.testing.assert_close(q._qdata, original, rtol=0, atol=0) + for copy in (q.detach(), q.clone(), q.to(torch.float16), nn.Parameter(q, requires_grad=False)): + assert isinstance(copy, ComfyQuantizedTensor) + assert copy.storage_dtype == original.dtype + torch.testing.assert_close(copy._qdata, original, rtol=0, atol=0) + + +@pytest.mark.parametrize("quant_format", FORMATS) +def test_split_and_cat_keep_codes_and_sidecars_aligned(quant_format): + q = bundle(checkpoint(quant_format))["linear.weight"] + original_data, original_fields = q._row_components() + parts = q.split([16, 48, 32], dim=0) + offset = 0 + for part, rows in zip(parts, (16, 48, 32)): + assert isinstance(part, ComfyQuantizedTensor) + assert part.shape == (rows, 256) + data, fields = part._row_components() + torch.testing.assert_close(data, original_data[offset : offset + rows], rtol=0, atol=0) + for name, (value, axis) in fields.items(): + expected = original_fields[name][0].narrow(axis, offset, rows) + torch.testing.assert_close(value.float(), expected.float(), rtol=0, atol=0) + offset += rows + joined = torch.cat(parts[::-1], dim=0) + data, fields = joined._row_components() + expected_order = torch.cat([torch.arange(64, 96), torch.arange(16, 64), torch.arange(16)]) + torch.testing.assert_close(data.float(), original_data.float()[expected_order], rtol=0, atol=0) + for name, (value, axis) in fields.items(): + torch.testing.assert_close( + value.float(), original_fields[name][0].float().index_select(axis, expected_order), rtol=0, atol=0 + ) + + +@pytest.mark.parametrize("quant_format", ["nvfp4", "mxfp8"]) +def test_blocked_scale_conversion_preserves_distinct_rows(quant_format): + state = checkpoint(quant_format) + block = 16 if quant_format == "nvfp4" else 32 + scales = torch.arange(96 * (256 // block)).reshape(96, -1).remainder(20).add(1) + stored = scales.to(torch.float8_e4m3fn) if block == 16 else scales.add(110).to(torch.uint8) + state["linear.weight_scale"] = to_blocked(stored) + q = bundle(state)["linear.weight"] + parts = q.chunk(3) + for index, part in enumerate(parts): + scale = part.params.block_scale if block == 16 else part.params.scale + decoded = from_blocked(scale, 32, 256 // block) + torch.testing.assert_close( + decoded.view(torch.uint8), stored[index * 32 : (index + 1) * 32].view(torch.uint8), rtol=0, atol=0 + ) + + +@pytest.mark.parametrize("quant_format", FORMATS) +@pytest.mark.parametrize( + "conversion,source,targets", + [ + ( + convert_z_image_transformer_checkpoint_to_diffusers, + "layers.0.attention.qkv", + [f"layers.0.attention.to_{x}" for x in "qkv"], + ), + ( + convert_qwen_image21_transformer_checkpoint_to_diffusers, + "transformer_blocks.0.img_mlp.gate_up", + ["transformer_blocks.0.img_mlp.gate_layer", "transformer_blocks.0.img_mlp.proj"], + ), + ], +) +def test_existing_converters_preserve_quantized_weights(quant_format, conversion, source, targets): + state = bundle(checkpoint(quant_format, source, rows=32 * len(targets))) + q = state[f"{source}.weight"] + original_data, original_fields = q._row_components() + converted = conversion(state) + for index, target in enumerate(targets): + part = converted[f"{target}.weight"] + assert isinstance(part, ComfyQuantizedTensor) + data, fields = part._row_components() + torch.testing.assert_close(data.float(), original_data[index * 32 : (index + 1) * 32].float(), rtol=0, atol=0) + for name, (value, axis) in fields.items(): + torch.testing.assert_close( + value.float(), original_fields[name][0].narrow(axis, index * 32, 32).float(), rtol=0, atol=0 + ) + + +@pytest.mark.parametrize("layout", ["stacked", "interleaved"]) +def test_existing_h3_converter_row_reordering(layout): + state = bundle(checkpoint(prefix="blocks.0.attn.qkv_proj", rows=96)) + q = state["blocks.0.attn.qkv_proj.weight"] + converted = convert_minimax_h3_transformer_checkpoint_to_diffusers( + state, config={"num_attention_heads": 4, "attention_head_dim": 8}, qkv_layout=layout + ) + order = torch.arange(96) + if layout == "interleaved": + order = order.reshape(4, 3, 8).transpose(0, 1).flatten() + for index, name in enumerate("qkv"): + part = converted[f"transformer_blocks.0.attn.to_{name}.weight"] + assert isinstance(part, ComfyQuantizedTensor) + rows = order[index * 32 : (index + 1) * 32] + torch.testing.assert_close(part._qdata, q._qdata[rows], rtol=0, atol=0) + torch.testing.assert_close(part.params.scale, q.params.scale[rows], rtol=0, atol=0) + + +def test_cat_rejects_different_shared_scales_and_metadata(): + first = bundle(checkpoint("float8_e4m3fn"))["linear.weight"] + second = first.clone() + second._params = type(first.params)(scale=torch.tensor(0.5), orig_dtype=first.dtype, orig_shape=tuple(first.shape)) + with pytest.raises(ValueError, match="different scale"): + torch.cat([first, second]) + with pytest.raises(NotImplementedError, match="input features"): + first.chunk(2, dim=1) + + +@pytest.mark.parametrize("metadata", [{"format": "unknown"}, {"convrot": True}]) +def test_invalid_format_is_rejected(metadata): + state = checkpoint() + state["linear.comfy_quant"] = marker(**metadata) + with pytest.raises(ValueError, match="supported Comfy format"): + bundle(state) + + +def test_missing_scale_and_marker_are_rejected(): + state = checkpoint() + state.pop("linear.weight_scale") + with pytest.raises(ValueError, match="requires weight_scale"): + bundle(state) + with pytest.raises(ValueError, match="no native Comfy"): + bundle({"linear.weight": torch.zeros(32, 256, dtype=torch.int8)}) + + +def test_missing_optional_dependency(): + with patch("diffusers.quantizers.comfy.comfy_quantizer.is_comfy_kitchen_available", return_value=False): + with pytest.raises(ImportError, match="comfy-kitchen"): + ComfyQuantizer(ComfyQuantizationConfig()).validate_environment() + + +def test_mixed_formats_load_through_public_api(tmp_path): + model = QwenImage21Transformer2DModel( + in_channels=4, + out_channels=4, + num_layers=1, + attention_head_dim=64, + num_attention_heads=4, + context_in_dim=8, + mlp_ratio=1, + axes_dims_rope=(16, 24, 24), + ).to(torch.bfloat16) + model.save_config(tmp_path) + state = model.state_dict() + state.update(checkpoint(prefix="transformer_blocks.0.attn.to_q", rows=256)) + state.update( + checkpoint("float8_e4m3fn", prefix="transformer_blocks.0.attn.to_k", rows=256, full_precision_matrix_mult=True) + ) + fused = checkpoint("nvfp4", prefix="transformer_blocks.0.img_mlp.gate_up", rows=512) + state.pop("transformer_blocks.0.img_mlp.gate_layer.weight") + state.pop("transformer_blocks.0.img_mlp.proj.weight") + state.update(fused) + loaded = QwenImage21Transformer2DModel.from_single_file( + state, config=str(tmp_path), quantization_config=ComfyQuantizationConfig() + ) + assert isinstance(loaded.transformer_blocks[0].attn.to_q, ComfyLinear) + assert loaded.transformer_blocks[0].attn.to_q.weight.storage_dtype == torch.int8 + assert loaded.transformer_blocks[0].attn.to_k.weight.storage_dtype == torch.float8_e4m3fn + assert loaded.transformer_blocks[0].img_mlp.proj.weight.storage_dtype == torch.uint8 + torch.testing.assert_close(loaded.img_in.weight, model.img_in.weight) + assert all(not value.is_meta for value in loaded.state_dict().values()) + + +def test_fp32_exclusion_dequantizes_inside_quantizer(): + model = nn.Sequential(nn.Linear(256, 96, bias=False, device="meta")) + state = bundle(checkpoint(prefix="0")) + expected = state["0.weight"].to(torch.float32).dequantize() + quantizer = ComfyQuantizer(ComfyQuantizationConfig()) + quantizer.preprocess_model(model, device_map=None, state_dict=state, keep_in_fp32_modules=["0"]) + quantizer.create_quantized_param(model, state["0.weight"].to(torch.float32), "0.weight", "cpu", state) + assert type(model[0]) is nn.Linear + assert model[0].weight.dtype == torch.float32 + torch.testing.assert_close(model[0].weight, expected) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="Requires CUDA") +@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16]) +@pytest.mark.parametrize("rotated", [False, True]) +def test_int8_runtime_against_independent_reference(dtype, rotated): + torch.manual_seed(42) + state = checkpoint(rows=32, convrot=rotated, convrot_groupsize=64) + q = bundle(state)["linear.weight"].to(device="cuda", dtype=dtype) + linear = ComfyLinear(256, 32, bias=True, device="cuda", dtype=dtype) + linear.weight = nn.Parameter(q, requires_grad=False) + x = torch.randn(2, 7, 512, device="cuda", dtype=dtype)[..., ::2] + rotated_x = x.float() + if rotated: + h4 = torch.tensor( + [[1, 1, 1, -1], [1, 1, -1, 1], [1, -1, 1, 1], [-1, 1, 1, 1]], device="cuda", dtype=torch.float32 + ) + h = torch.kron(torch.kron(h4, h4), h4) / 8 + rotated_x = (rotated_x.reshape(2, 7, -1, 64) @ h).reshape(2, 7, 256) + expected = F.linear(rotated_x, q._qdata.float() * q.params.scale, linear.bias.float()) + with torch.inference_mode(), kitchen.use_backend("cuda"): + actual = linear(x) + assert actual.dtype == dtype + assert (actual.float() - expected).norm() / expected.norm() < 0.025 + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="Requires CUDA") +def test_cpu_offload_and_dtype_move_keep_sidecars(): + from accelerate import cpu_offload_with_hook + + q = bundle(checkpoint(rows=32, convrot=True, convrot_groupsize=64))["linear.weight"] + q.sidecars["input_scale"] = torch.tensor(0.1234567) + scale = q.params.scale.clone() + linear = ComfyLinear(256, 32, bias=False, dtype=torch.bfloat16) + linear.weight = nn.Parameter(q, requires_grad=False) + model, hook = cpu_offload_with_hook(nn.Sequential(linear), execution_device="cuda:0") + inputs = torch.randn(2, 256, device="cuda", dtype=torch.bfloat16) + with torch.inference_mode(): + first = model(inputs) + hook.offload() + assert linear.weight.device.type == "cpu" + assert linear.weight.sidecars["input_scale"].device.type == "cpu" + second = model(inputs) + torch.testing.assert_close(first, second, rtol=0, atol=0) + model.to(dtype=torch.float16) + assert linear.weight.dtype == torch.float16 + assert linear.weight.params.scale.dtype == torch.float32 + torch.testing.assert_close(linear.weight.params.scale.cpu(), scale, rtol=0, atol=0) + + +@pytest.mark.parametrize("quant_format", FORMATS) +def test_linear_runtime_matches_layout_reference(quant_format): + q = bundle(checkpoint(quant_format, rows=128))["linear.weight"] + linear = ComfyLinear(256, 128, bias=False, dtype=torch.bfloat16) + linear.weight = nn.Parameter(q, requires_grad=False) + torch.manual_seed(42) + inputs = torch.randn(2, 3, 256, dtype=torch.bfloat16) + with torch.inference_mode(), kitchen.use_backend("eager"): + actual = linear(inputs) + reference_input = inputs.reshape(-1, 256) + if q._layout_cls in ("TensorCoreFP8Layout", "TensorCoreNVFP4Layout", "TensorCoreMXFP8Layout"): + kwargs = {"dtype": q.storage_dtype} if q._layout_cls == "TensorCoreFP8Layout" else {} + reference_input = QuantizedTensor.from_float(reference_input, q._layout_cls, **kwargs).dequantize() + if q._layout_cls == "TensorCoreSVDQuantW4A4Layout": + reference = F.linear(inputs, QuantizedTensor(q._qdata, q._layout_cls, q.params)) + else: + reference = F.linear(reference_input, q.dequantize().to(inputs.dtype)).reshape(2, 3, 128) + assert torch.isfinite(actual).all() + assert actual.dtype == torch.bfloat16 + relative_error = (actual.float() - reference.float()).norm() / reference.float().norm() + assert relative_error < (0.25 if quant_format == "convrot_w4a4" else 0.04), relative_error.item() + + +def test_svdquant_tile_packed_qkv_conversion(): + from comfy_kitchen.backends.eager.svdquant import _tile_packed_weight_to_row_major + + rows, columns = 384, 256 + state = checkpoint("svdquant_w4a4", rows=rows) + expected = state["linear.weight"].clone() + expected_scale = state["linear.weight_scale"].clone() + expected_up = state["linear.proj_up"].clone() + state["linear.weight"] = ( + expected.reshape(3, 32, 4, columns // 64, 32).permute(0, 3, 1, 2, 4).reshape(3, columns // 64, 32, 128) + ) + state["linear.weight_scale"] = expected_scale.reshape(columns // 64, 3, 128).permute(1, 0, 2) + state["linear.proj_up"] = expected_up.reshape(3, 128, 4).transpose(1, 2) + torch.testing.assert_close(_tile_packed_weight_to_row_major(state["linear.weight"]), expected) + q = bundle(state)["linear.weight"] + for index, part in enumerate(q.chunk(3)): + torch.testing.assert_close(part._qdata, expected[index * 128 : (index + 1) * 128], rtol=0, atol=0) + torch.testing.assert_close( + part.params.scale, expected_scale[:, index * 128 : (index + 1) * 128], rtol=0, atol=0 + ) + torch.testing.assert_close(part.params.proj_up, expected_up[index * 128 : (index + 1) * 128], rtol=0, atol=0) + + +def test_full_precision_mm_keeps_input_scaling(): + state = checkpoint("float8_e4m3fn", full_precision_matrix_mult=True) + state["linear.pre_quant_scale"] = torch.full((256,), 0.5) + q = bundle(state)["linear.weight"] + linear = ComfyLinear(256, 96, bias=False, dtype=torch.bfloat16) + linear.weight = nn.Parameter(q, requires_grad=False) + inputs = torch.randn(2, 256, dtype=torch.bfloat16) + torch.testing.assert_close(linear(inputs), F.linear(inputs * 0.5, q.dequantize())) + + +@pytest.mark.parametrize( + "suffix,value,message", + [ + ("weight_scale", torch.ones(2), "per-output-row shape"), + ("weight_scale", torch.ones(96, 1, dtype=torch.float16), "FP32"), + ("weight_scale", torch.full((96, 1), float("nan")), "finite positive"), + ("weight_scale", torch.zeros(96, 1), "finite positive"), + ("weight", torch.ones(96, 256), "shape or dtype"), + ("comfy_quant", torch.tensor([255], dtype=torch.uint8), "valid JSON"), + ("comfy_quant", torch.tensor(1), "one-dimensional uint8"), + ], +) +def test_malformed_checkpoint_is_rejected(suffix, value, message): + state = checkpoint() + state[f"linear.{suffix}"] = value + with pytest.raises(ValueError, match=message): + bundle(state) + + +@pytest.mark.parametrize("group_size", [True, 0, 2, 8, 63, 1024]) +def test_invalid_rotation_group_is_rejected(group_size): + with pytest.raises(ValueError, match="power of four|group-aligned"): + bundle(checkpoint(convrot=True, convrot_groupsize=group_size)) + + +def test_unmarked_fp8_weight_is_rejected(): + state = checkpoint() + state["other.weight"] = torch.ones(32, 256).to(torch.float8_e4m3fn) + with pytest.raises(ValueError, match="markers"): + bundle(state)