Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 3 additions & 12 deletions src/diffusers/modular_pipelines/minimax_h3/before_encoder.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
from ...configuration_utils import FrozenDict
from ...image_processor import VaeImageProcessor
from ...utils import logging
from ...utils.torch_utils import resample_audio
from ..modular_pipeline import ModularPipelineBlocks, PipelineState
from ..modular_pipeline_utils import ComponentSpec, ConfigSpec, InputParam, OutputParam
from .modular_pipeline import (
Expand Down Expand Up @@ -352,7 +353,7 @@ def _normalize_audio_condition(
The reference implementation extracts a soundtrack at a native rate, truncates it there and resamples it once,
in torch, which this mirrors: the truncation is applied at `sample_rate` and the resampling is a single
`torchaudio` pass. A mono waveform is upmixed by repeating its channel.
windowed-sinc pass (`torchaudio`'s default). A mono waveform is upmixed by repeating its channel.
Args:
waveform (`torch.Tensor` of shape `(channels, num_samples)`): The soundtrack, mono or stereo.
Expand All @@ -372,17 +373,7 @@ def _normalize_audio_condition(
waveform = waveform.to(torch.float32)[:, : int(max_duration * sample_rate)]
if waveform.shape[0] != 2:
waveform = waveform.expand(2, -1).contiguous()
if sample_rate == target_sample_rate:
return waveform

try:
import torchaudio
except ImportError as error:
raise ImportError(
f"Resampling a MiniMax-H3 reference soundtrack from {sample_rate} Hz to {target_sample_rate} Hz "
"needs `torchaudio`. Pass a waveform already at the audio VAE's sample rate to do without it."
) from error
return torchaudio.transforms.Resample(sample_rate, target_sample_rate)(waveform)
return resample_audio(waveform, sample_rate, target_sample_rate)

@torch.no_grad()
def __call__(
Expand Down
16 changes: 5 additions & 11 deletions src/diffusers/modular_pipelines/pika_h3_flex/before_encoder.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
from ...configuration_utils import FrozenDict
from ...image_processor import VaeImageProcessor
from ...utils import logging
from ...utils.torch_utils import resample_audio
from ..modular_pipeline import ModularPipelineBlocks, PipelineState
from ..modular_pipeline_utils import ComponentSpec, ConfigSpec, InputParam, OutputParam
from .modular_pipeline import (
Expand Down Expand Up @@ -344,8 +345,9 @@ def _normalize_audio_condition(
Normalize a reference soundtrack onto the audio VAE's sample rate, as a stereo waveform.

The released package resamples a soundtrack onto the audio VAE's rate first and truncates it there, to the
sample count that covers the generated frames, which this mirrors: the resampling is a single `torchaudio` pass
and the truncation is applied at `target_sample_rate`. A mono waveform is upmixed by repeating its channel.
sample count that covers the generated frames, which this mirrors: the resampling is a single windowed-sinc pass
(`torchaudio`'s default) and the truncation is applied at `target_sample_rate`. A mono waveform is upmixed by
repeating its channel.

Args:
waveform (`torch.Tensor` of shape `(channels, num_samples)`): The soundtrack, mono or stereo.
Expand All @@ -365,15 +367,7 @@ def _normalize_audio_condition(
waveform = waveform.to(torch.float32)
if waveform.shape[0] != 2:
waveform = waveform.expand(2, -1)
if sample_rate != target_sample_rate:
try:
import torchaudio
except ImportError as error:
raise ImportError(
f"Resampling a Pika H3 Flex reference soundtrack from {sample_rate} Hz to {target_sample_rate} "
"Hz needs `torchaudio`. Pass a waveform already at the audio VAE's sample rate to do without it."
) from error
waveform = torchaudio.transforms.Resample(sample_rate, target_sample_rate)(waveform)
waveform = resample_audio(waveform, sample_rate, target_sample_rate)
return waveform[:, :max_samples].contiguous()

@torch.no_grad()
Expand Down
50 changes: 50 additions & 0 deletions src/diffusers/utils/torch_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

import functools
import itertools
import math
import os
from typing import Callable, ParamSpec, TypeVar

Expand Down Expand Up @@ -188,6 +189,55 @@ def fourier_filter(x_in: "torch.Tensor", threshold: int, scale: int) -> "torch.T
return x_filtered.to(dtype=x_in.dtype)


def resample_audio(waveform: "torch.Tensor", orig_freq: int, new_freq: int) -> "torch.Tensor":
r"""
Resample a waveform along its last dimension with a windowed-sinc filter, in plain PyTorch.
This is the default band-limited interpolation of `torchaudio.functional.resample` (`sinc_interp_hann`,
`lowpass_filter_width=6`, `rolloff=0.99`) and matches it to float precision, so pipelines can resample a
conditioning soundtrack without depending on `torchaudio`.
Args:
waveform (`torch.Tensor` of shape `(..., num_samples)`): The floating-point waveform to resample.
orig_freq (`int`): The sample rate `waveform` carries its samples at.
new_freq (`int`): The sample rate to resample to.
Returns:
`torch.Tensor` of shape `(..., ceil(num_samples * new_freq / orig_freq))`: the resampled waveform, in the
dtype and on the device of `waveform`.
"""
if orig_freq <= 0 or new_freq <= 0:
raise ValueError(f"Sample rates must be positive, got {orig_freq} and {new_freq}.")
if not torch.is_floating_point(waveform):
raise TypeError(f"`waveform` must be a floating-point tensor, got {waveform.dtype}.")
if orig_freq == new_freq:
return waveform

gcd = math.gcd(int(orig_freq), int(new_freq))
orig_freq, new_freq = int(orig_freq) // gcd, int(new_freq) // gcd
lowpass_filter_width, rolloff = 6, 0.99

# Hann-windowed sinc kernel, one row per output phase, low-passed below the lower of the two Nyquist rates.
base_freq = min(orig_freq, new_freq) * rolloff
width = math.ceil(lowpass_filter_width * orig_freq / base_freq)
idx = torch.arange(-width, width + orig_freq, dtype=torch.float64, device=waveform.device)[None, None] / orig_freq
t = torch.arange(0, -new_freq, -1, dtype=torch.float64, device=waveform.device)[:, None, None] / new_freq + idx
t = (t * base_freq).clamp(-lowpass_filter_width, lowpass_filter_width)
window = torch.cos(t * math.pi / lowpass_filter_width / 2) ** 2
t = t * math.pi
kernel = torch.where(t == 0, torch.ones_like(t), torch.sin(t) / t) * window * (base_freq / orig_freq)
kernel = kernel.to(waveform.dtype)

shape = waveform.shape
waveform = waveform.reshape(-1, shape[-1])
num_waveforms, length = waveform.shape
waveform = torch.nn.functional.pad(waveform, (width, width + orig_freq))
resampled = torch.nn.functional.conv1d(waveform[:, None], kernel, stride=orig_freq)
resampled = resampled.transpose(1, 2).reshape(num_waveforms, -1)
resampled = resampled[:, : math.ceil(new_freq * length / orig_freq)]
return resampled.reshape(shape[:-1] + resampled.shape[-1:])


def apply_freeu(
resolution_idx: int, hidden_states: "torch.Tensor", res_hidden_states: "torch.Tensor", **freeu_kwargs
) -> tuple["torch.Tensor", "torch.Tensor"]:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@


import os
import sys

import numpy as np
import pytest
Expand Down Expand Up @@ -1084,3 +1085,19 @@ def test_video_reference_resampled_to_the_model_frame_rate(self):
frames, fps=float(MINIMAX_H3_FPS), num_frames=124, **canvas
)
assert np.shares_memory(untouched, frames)

def test_audio_reference_resampled_without_torchaudio(self, monkeypatch):
r"""
A reference soundtrack at another rate than the audio VAE's is truncated to `max_duration` at its own rate,
upmixed to stereo and resampled onto the VAE's rate, with no dependency on `torchaudio`.
"""
monkeypatch.setitem(sys.modules, "torchaudio", None)
waveform = torch.randn(1, 48000 * 3, generator=torch.Generator("cpu").manual_seed(0))

normalized = MiniMaxH3Ref2VASetupStep._normalize_audio_condition(
waveform, sample_rate=48000, target_sample_rate=32000, max_duration=2.0
)

assert normalized.shape == (2, 64000)
assert normalized.dtype == torch.float32
assert torch.equal(normalized[0], normalized[1])
38 changes: 38 additions & 0 deletions tests/others/test_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
# limitations under the License.

import importlib
import math
import os
import warnings

Expand Down Expand Up @@ -242,6 +243,43 @@ def test_fourier_filter_preserves_dtype_and_shape(self):
assert out.shape == x.shape


class TestResampleAudio:
"""Tests for :func:`diffusers.utils.torch_utils.resample_audio`."""

@pytest.mark.parametrize("orig_freq, new_freq", [(48000, 32000), (44100, 32000), (16000, 48000), (22050, 44100)])
def test_matches_torchaudio(self, orig_freq, new_freq):
torchaudio = pytest.importorskip("torchaudio")

waveform = torch.randn(2, 4801, generator=torch.Generator("cpu").manual_seed(0))

expected = torchaudio.transforms.Resample(orig_freq, new_freq)(waveform)
resampled = torch_utils.resample_audio(waveform, orig_freq, new_freq)

# torchaudio's transform and functional forms already differ by ~3e-5 at 44.1 kHz from float32 rounding.
assert resampled.shape == expected.shape
assert torch.allclose(resampled, expected, atol=1e-4)

def test_preserves_a_tone_below_the_new_nyquist(self):
# A 1 kHz tone resampled from 48 kHz to 32 kHz is the same tone at the new rate, away from the padded edges.
orig_freq, new_freq, tone = 48000, 32000, 1000.0
waveform = torch.sin(2 * math.pi * tone * torch.arange(orig_freq, dtype=torch.float64) / orig_freq)[None]

resampled = torch_utils.resample_audio(waveform.float(), orig_freq, new_freq)

expected = torch.sin(2 * math.pi * tone * torch.arange(new_freq, dtype=torch.float64) / new_freq)
assert resampled.shape == (1, new_freq)
assert torch.allclose(resampled[0, 100:-100].double(), expected[100:-100], atol=1e-3)

def test_shape_dtype_and_identity(self):
waveform = torch.randn(3, 2, 1000, dtype=torch.float64)

assert torch_utils.resample_audio(waveform, 48000, 32000).shape == (3, 2, 667)
assert torch_utils.resample_audio(waveform, 48000, 32000).dtype == torch.float64
assert torch_utils.resample_audio(waveform, 32000, 32000) is waveform
with pytest.raises(TypeError, match="floating-point"):
torch_utils.resample_audio(torch.zeros(1, 10, dtype=torch.int16), 48000, 32000)


class TestRandnTensor:
"""Tests for :func:`diffusers.utils.torch_utils.randn_tensor`."""

Expand Down
Loading