Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 25 additions & 8 deletions src/diffusers/pipelines/shap_e/renderer.py
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,9 @@ def encode_position(position):

def encode_direction(position, direction=None):
if direction is None:
return torch.zeros_like(posenc_nerf(position, min_deg=0, max_deg=8))
# Same shape as posenc_nerf(position, min_deg=0, max_deg=8), without computing an encoding that is discarded.
dim = position.shape[-1]
return position.new_zeros(*position.shape[:-1], dim + 2 * dim * 8)
else:
return posenc_nerf(direction, min_deg=0, max_deg=8)

Expand Down Expand Up @@ -603,6 +605,7 @@ class MLPNeRFModelOutput(BaseOutput):
signed_distance: torch.Tensor
channels: torch.Tensor
ts: torch.Tensor
raw_output: torch.Tensor | None = None


class MLPNeRSTFModel(ModelMixin, ConfigMixin):
Expand Down Expand Up @@ -680,6 +683,9 @@ def forward(self, *, position, direction, ts, nerf_level="coarse", rendering_mod
if h_directionless is None:
h_directionless = h_preact

return self._outputs_from_raw(h_final, ts, nerf_level=nerf_level, rendering_mode=rendering_mode)

def _outputs_from_raw(self, h_final, ts, nerf_level="coarse", rendering_mode="nerf"):
activation = self.map_indices_to_keys(h_final)

if nerf_level == "coarse":
Expand All @@ -701,7 +707,9 @@ def forward(self, *, position, direction, ts, nerf_level="coarse", rendering_mod
channels = self.channel_activation(h_channels)

# yiyi notes: I think signed_distance is not used
return MLPNeRFModelOutput(density=density, signed_distance=signed_distance, channels=channels, ts=ts)
return MLPNeRFModelOutput(
density=density, signed_distance=signed_distance, channels=channels, ts=ts, raw_output=h_final
)


class ChannelsProj(nn.Module):
Expand Down Expand Up @@ -846,6 +854,8 @@ def render_rays(self, rays, sampler, n_samples, prev_model_out=None, render_with
Args:
rays: [batch_size x ... x 2 x 3] origin and direction. sampler: disjoint volume integrals. n_samples:
number of ts to sample. prev_model_outputs: model outputs from the previous rendering step, including
`ts` and `raw_output`. They must come from the previous `render_rays` call of this renderer on the same
`rays` with the same `render_with_direction`; their raw MLP outputs are reused at their `ts`.

:return: A tuple of
- `channels`
Expand All @@ -861,17 +871,17 @@ def render_rays(self, rays, sampler, n_samples, prev_model_out=None, render_with
ts = sampler.sample(vrange.t0, vrange.t1, n_samples)
ts = ts.to(rays.dtype)

# The MLP runs on the new ts only; the raw output at `prev_model_out.ts` is reused.
query_ts = ts
if prev_model_out is not None:
# Append the previous ts now before fprop because previous
# rendering used a different model and we can't reuse the output.
ts = torch.sort(torch.cat([ts, prev_model_out.ts], dim=-2), dim=-2).values
ts, order = torch.sort(torch.cat([ts, prev_model_out.ts], dim=-2), dim=-2)

batch_size, *_shape, _t0_dim = vrange.t0.shape
_, *ts_shape, _ts_dim = ts.shape
_, *ts_shape, _ts_dim = query_ts.shape

# 2. Get the points along the ray and query the model
directions = torch.broadcast_to(direction.unsqueeze(-2), [batch_size, *ts_shape, 3])
positions = origin.unsqueeze(-2) + ts * directions
positions = origin.unsqueeze(-2) + query_ts * directions

directions = directions.to(self.mlp.dtype)
positions = positions.to(self.mlp.dtype)
Expand All @@ -881,10 +891,17 @@ def render_rays(self, rays, sampler, n_samples, prev_model_out=None, render_with
model_out = self.mlp(
position=positions,
direction=optional_directions,
ts=ts,
ts=query_ts,
nerf_level="coarse" if prev_model_out is None else "fine",
)

if prev_model_out is not None:
# Only `raw_output` of the call above is used: merge it with the previous raw output in the sorted order
# of `ts` and apply the fine heads to the result.
raw = torch.cat([model_out.raw_output, prev_model_out.raw_output], dim=-2)
raw = torch.gather(raw, -2, order.expand(*order.shape[:-1], raw.shape[-1]))
model_out = self.mlp._outputs_from_raw(raw, ts, nerf_level="fine")

# 3. Integrate the model results
channels, weights, transmittance = integrate_samples(
vrange, model_out.ts, model_out.density, model_out.channels
Expand Down
48 changes: 47 additions & 1 deletion tests/pipelines/shap_e/test_shap_e.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,8 @@
from transformers import CLIPTextConfig, CLIPTextModelWithProjection, CLIPTokenizer

from diffusers import HeunDiscreteScheduler, PriorTransformer, ShapEPipeline
from diffusers.pipelines.shap_e import ShapERenderer
from diffusers.pipelines.shap_e import MLPNeRFModelOutput, ShapERenderer, StratifiedRaySampler, create_pan_cameras
from diffusers.pipelines.shap_e.renderer import encode_direction, posenc_nerf

from ...testing_utils import (
backend_empty_cache,
Expand Down Expand Up @@ -214,6 +215,51 @@ def test_num_images_per_prompt(self):
def test_save_load_local(self, tmp_path, base_pipe_output):
super().test_save_load_local(tmp_path, base_pipe_output, expected_max_difference=5e-3)

def test_renderer_fine_pass_reuse_matches_full_recompute(self):
# The fine pass runs the MLP only on the new samples and reuses the coarse raw MLP outputs at the coarse ts.
# It must match running the MLP on all merged ts. Exact equality relies on the MLP matmul giving the same
# result for a row whatever the number of rows. That held on CPU in our runs but not for a GPU GEMM kernel at
# this tiny size, so the check runs on CPU.
renderer = self.dummy_renderer.to("cpu")
rays = create_pan_cameras(4).camera_rays.to("cpu")
origin, direction = rays[..., 0, :], rays[..., 1, :]

_, fine_sampler, coarse_out = renderer.render_rays(rays, StratifiedRaySampler(), 8)
torch.manual_seed(0)
_, _, fine_out = renderer.render_rays(rays, fine_sampler, 16, prev_model_out=coarse_out)

torch.manual_seed(0)
vrange = renderer.volume.intersect(origin, direction, t0_lower=None)
new_ts = fine_sampler.sample(vrange.t0, vrange.t1, 16).to(rays.dtype)
expected_ts = torch.sort(torch.cat([new_ts, coarse_out.ts], dim=-2), dim=-2).values
positions = origin.unsqueeze(-2) + expected_ts * direction.unsqueeze(-2)
expected = renderer.mlp(position=positions, direction=None, ts=expected_ts, nerf_level="fine")

assert torch.equal(fine_out.ts, expected_ts)
for name in ("density", "channels", "signed_distance", "raw_output"):
assert torch.equal(getattr(fine_out, name), getattr(expected, name)), name

# The coarse raw outputs are used: changing them changes the fine pass.
changed_coarse_out = MLPNeRFModelOutput(
density=coarse_out.density,
signed_distance=coarse_out.signed_distance,
channels=coarse_out.channels,
ts=coarse_out.ts,
raw_output=coarse_out.raw_output + 1,
)
torch.manual_seed(0)
_, _, changed_fine_out = renderer.render_rays(rays, fine_sampler, 16, prev_model_out=changed_coarse_out)
assert not torch.equal(changed_fine_out.raw_output, fine_out.raw_output)

def test_encode_direction_without_direction(self):
position = torch.randn(2, 5, 3, device=torch_device)
expected = torch.zeros_like(posenc_nerf(position, min_deg=0, max_deg=8))
output = encode_direction(position)
assert output.device == expected.device
assert output.dtype == expected.dtype
assert output.shape == expected.shape
assert torch.equal(output, expected)


class TestShapEPipelineMemory(ShapEPipelineTesterConfig, MemoryTesterMixin):
"""Memory optimization tests (CPU offload, group offload, layerwise casting) for the Shap-E pipeline."""
Expand Down
Loading