diff --git a/src/diffusers/modular_pipelines/minimax_h3/before_encoder.py b/src/diffusers/modular_pipelines/minimax_h3/before_encoder.py index dea178af4cb7..58ac846295c3 100644 --- a/src/diffusers/modular_pipelines/minimax_h3/before_encoder.py +++ b/src/diffusers/modular_pipelines/minimax_h3/before_encoder.py @@ -22,6 +22,7 @@ from ...configuration_utils import FrozenDict from ...image_processor import VaeImageProcessor from ...utils import logging +from ...utils.torch_utils import resample_audio from ..modular_pipeline import ModularPipelineBlocks, PipelineState from ..modular_pipeline_utils import ComponentSpec, ConfigSpec, InputParam, OutputParam from .modular_pipeline import ( @@ -352,7 +353,7 @@ def _normalize_audio_condition( The reference implementation extracts a soundtrack at a native rate, truncates it there and resamples it once, in torch, which this mirrors: the truncation is applied at `sample_rate` and the resampling is a single - `torchaudio` pass. A mono waveform is upmixed by repeating its channel. + windowed-sinc pass (`torchaudio`'s default). A mono waveform is upmixed by repeating its channel. Args: waveform (`torch.Tensor` of shape `(channels, num_samples)`): The soundtrack, mono or stereo. @@ -372,17 +373,7 @@ def _normalize_audio_condition( waveform = waveform.to(torch.float32)[:, : int(max_duration * sample_rate)] if waveform.shape[0] != 2: waveform = waveform.expand(2, -1).contiguous() - if sample_rate == target_sample_rate: - return waveform - - try: - import torchaudio - except ImportError as error: - raise ImportError( - f"Resampling a MiniMax-H3 reference soundtrack from {sample_rate} Hz to {target_sample_rate} Hz " - "needs `torchaudio`. Pass a waveform already at the audio VAE's sample rate to do without it." - ) from error - return torchaudio.transforms.Resample(sample_rate, target_sample_rate)(waveform) + return resample_audio(waveform, sample_rate, target_sample_rate) @torch.no_grad() def __call__( diff --git a/src/diffusers/modular_pipelines/pika_h3_flex/before_encoder.py b/src/diffusers/modular_pipelines/pika_h3_flex/before_encoder.py index f091cd67bfba..e8d2091cac90 100644 --- a/src/diffusers/modular_pipelines/pika_h3_flex/before_encoder.py +++ b/src/diffusers/modular_pipelines/pika_h3_flex/before_encoder.py @@ -22,6 +22,7 @@ from ...configuration_utils import FrozenDict from ...image_processor import VaeImageProcessor from ...utils import logging +from ...utils.torch_utils import resample_audio from ..modular_pipeline import ModularPipelineBlocks, PipelineState from ..modular_pipeline_utils import ComponentSpec, ConfigSpec, InputParam, OutputParam from .modular_pipeline import ( @@ -344,8 +345,9 @@ def _normalize_audio_condition( Normalize a reference soundtrack onto the audio VAE's sample rate, as a stereo waveform. The released package resamples a soundtrack onto the audio VAE's rate first and truncates it there, to the - sample count that covers the generated frames, which this mirrors: the resampling is a single `torchaudio` pass - and the truncation is applied at `target_sample_rate`. A mono waveform is upmixed by repeating its channel. + sample count that covers the generated frames, which this mirrors: the resampling is a single windowed-sinc pass + (`torchaudio`'s default) and the truncation is applied at `target_sample_rate`. A mono waveform is upmixed by + repeating its channel. Args: waveform (`torch.Tensor` of shape `(channels, num_samples)`): The soundtrack, mono or stereo. @@ -365,15 +367,7 @@ def _normalize_audio_condition( waveform = waveform.to(torch.float32) if waveform.shape[0] != 2: waveform = waveform.expand(2, -1) - if sample_rate != target_sample_rate: - try: - import torchaudio - except ImportError as error: - raise ImportError( - f"Resampling a Pika H3 Flex reference soundtrack from {sample_rate} Hz to {target_sample_rate} " - "Hz needs `torchaudio`. Pass a waveform already at the audio VAE's sample rate to do without it." - ) from error - waveform = torchaudio.transforms.Resample(sample_rate, target_sample_rate)(waveform) + waveform = resample_audio(waveform, sample_rate, target_sample_rate) return waveform[:, :max_samples].contiguous() @torch.no_grad() diff --git a/src/diffusers/utils/torch_utils.py b/src/diffusers/utils/torch_utils.py index 801044c58106..d34a4c923409 100644 --- a/src/diffusers/utils/torch_utils.py +++ b/src/diffusers/utils/torch_utils.py @@ -19,6 +19,7 @@ import functools import itertools +import math import os from typing import Callable, ParamSpec, TypeVar @@ -188,6 +189,55 @@ def fourier_filter(x_in: "torch.Tensor", threshold: int, scale: int) -> "torch.T return x_filtered.to(dtype=x_in.dtype) +def resample_audio(waveform: "torch.Tensor", orig_freq: int, new_freq: int) -> "torch.Tensor": + r""" + Resample a waveform along its last dimension with a windowed-sinc filter, in plain PyTorch. + + This is the default band-limited interpolation of `torchaudio.functional.resample` (`sinc_interp_hann`, + `lowpass_filter_width=6`, `rolloff=0.99`) and matches it to float precision, so pipelines can resample a + conditioning soundtrack without depending on `torchaudio`. + + Args: + waveform (`torch.Tensor` of shape `(..., num_samples)`): The floating-point waveform to resample. + orig_freq (`int`): The sample rate `waveform` carries its samples at. + new_freq (`int`): The sample rate to resample to. + + Returns: + `torch.Tensor` of shape `(..., ceil(num_samples * new_freq / orig_freq))`: the resampled waveform, in the + dtype and on the device of `waveform`. + """ + if orig_freq <= 0 or new_freq <= 0: + raise ValueError(f"Sample rates must be positive, got {orig_freq} and {new_freq}.") + if not torch.is_floating_point(waveform): + raise TypeError(f"`waveform` must be a floating-point tensor, got {waveform.dtype}.") + if orig_freq == new_freq: + return waveform + + gcd = math.gcd(int(orig_freq), int(new_freq)) + orig_freq, new_freq = int(orig_freq) // gcd, int(new_freq) // gcd + lowpass_filter_width, rolloff = 6, 0.99 + + # Hann-windowed sinc kernel, one row per output phase, low-passed below the lower of the two Nyquist rates. + base_freq = min(orig_freq, new_freq) * rolloff + width = math.ceil(lowpass_filter_width * orig_freq / base_freq) + idx = torch.arange(-width, width + orig_freq, dtype=torch.float64, device=waveform.device)[None, None] / orig_freq + t = torch.arange(0, -new_freq, -1, dtype=torch.float64, device=waveform.device)[:, None, None] / new_freq + idx + t = (t * base_freq).clamp(-lowpass_filter_width, lowpass_filter_width) + window = torch.cos(t * math.pi / lowpass_filter_width / 2) ** 2 + t = t * math.pi + kernel = torch.where(t == 0, torch.ones_like(t), torch.sin(t) / t) * window * (base_freq / orig_freq) + kernel = kernel.to(waveform.dtype) + + shape = waveform.shape + waveform = waveform.reshape(-1, shape[-1]) + num_waveforms, length = waveform.shape + waveform = torch.nn.functional.pad(waveform, (width, width + orig_freq)) + resampled = torch.nn.functional.conv1d(waveform[:, None], kernel, stride=orig_freq) + resampled = resampled.transpose(1, 2).reshape(num_waveforms, -1) + resampled = resampled[:, : math.ceil(new_freq * length / orig_freq)] + return resampled.reshape(shape[:-1] + resampled.shape[-1:]) + + def apply_freeu( resolution_idx: int, hidden_states: "torch.Tensor", res_hidden_states: "torch.Tensor", **freeu_kwargs ) -> tuple["torch.Tensor", "torch.Tensor"]: diff --git a/tests/modular_pipelines/minimax_h3/test_modular_pipeline_minimax_h3.py b/tests/modular_pipelines/minimax_h3/test_modular_pipeline_minimax_h3.py index dd7f04f1c7f1..0bf3f48d9268 100644 --- a/tests/modular_pipelines/minimax_h3/test_modular_pipeline_minimax_h3.py +++ b/tests/modular_pipelines/minimax_h3/test_modular_pipeline_minimax_h3.py @@ -14,6 +14,7 @@ import os +import sys import numpy as np import pytest @@ -1084,3 +1085,19 @@ def test_video_reference_resampled_to_the_model_frame_rate(self): frames, fps=float(MINIMAX_H3_FPS), num_frames=124, **canvas ) assert np.shares_memory(untouched, frames) + + def test_audio_reference_resampled_without_torchaudio(self, monkeypatch): + r""" + A reference soundtrack at another rate than the audio VAE's is truncated to `max_duration` at its own rate, + upmixed to stereo and resampled onto the VAE's rate, with no dependency on `torchaudio`. + """ + monkeypatch.setitem(sys.modules, "torchaudio", None) + waveform = torch.randn(1, 48000 * 3, generator=torch.Generator("cpu").manual_seed(0)) + + normalized = MiniMaxH3Ref2VASetupStep._normalize_audio_condition( + waveform, sample_rate=48000, target_sample_rate=32000, max_duration=2.0 + ) + + assert normalized.shape == (2, 64000) + assert normalized.dtype == torch.float32 + assert torch.equal(normalized[0], normalized[1]) diff --git a/tests/others/test_utils.py b/tests/others/test_utils.py index 0341623d8783..b5861113321c 100755 --- a/tests/others/test_utils.py +++ b/tests/others/test_utils.py @@ -14,6 +14,7 @@ # limitations under the License. import importlib +import math import os import warnings @@ -242,6 +243,43 @@ def test_fourier_filter_preserves_dtype_and_shape(self): assert out.shape == x.shape +class TestResampleAudio: + """Tests for :func:`diffusers.utils.torch_utils.resample_audio`.""" + + @pytest.mark.parametrize("orig_freq, new_freq", [(48000, 32000), (44100, 32000), (16000, 48000), (22050, 44100)]) + def test_matches_torchaudio(self, orig_freq, new_freq): + torchaudio = pytest.importorskip("torchaudio") + + waveform = torch.randn(2, 4801, generator=torch.Generator("cpu").manual_seed(0)) + + expected = torchaudio.transforms.Resample(orig_freq, new_freq)(waveform) + resampled = torch_utils.resample_audio(waveform, orig_freq, new_freq) + + # torchaudio's transform and functional forms already differ by ~3e-5 at 44.1 kHz from float32 rounding. + assert resampled.shape == expected.shape + assert torch.allclose(resampled, expected, atol=1e-4) + + def test_preserves_a_tone_below_the_new_nyquist(self): + # A 1 kHz tone resampled from 48 kHz to 32 kHz is the same tone at the new rate, away from the padded edges. + orig_freq, new_freq, tone = 48000, 32000, 1000.0 + waveform = torch.sin(2 * math.pi * tone * torch.arange(orig_freq, dtype=torch.float64) / orig_freq)[None] + + resampled = torch_utils.resample_audio(waveform.float(), orig_freq, new_freq) + + expected = torch.sin(2 * math.pi * tone * torch.arange(new_freq, dtype=torch.float64) / new_freq) + assert resampled.shape == (1, new_freq) + assert torch.allclose(resampled[0, 100:-100].double(), expected[100:-100], atol=1e-3) + + def test_shape_dtype_and_identity(self): + waveform = torch.randn(3, 2, 1000, dtype=torch.float64) + + assert torch_utils.resample_audio(waveform, 48000, 32000).shape == (3, 2, 667) + assert torch_utils.resample_audio(waveform, 48000, 32000).dtype == torch.float64 + assert torch_utils.resample_audio(waveform, 32000, 32000) is waveform + with pytest.raises(TypeError, match="floating-point"): + torch_utils.resample_audio(torch.zeros(1, 10, dtype=torch.int16), 48000, 32000) + + class TestRandnTensor: """Tests for :func:`diffusers.utils.torch_utils.randn_tensor`."""