From a21c9c913f35985d02fdbe5e4adef2d0379064d8 Mon Sep 17 00:00:00 2001 From: "Hsu, Chan Fu" Date: Thu, 12 Nov 2026 14:41:38 +0800 Subject: [PATCH] Shap-E: reuse the coarse MLP outputs in the renderer fine pass ShapERenderer.decode_to_image renders each ray batch twice: a coarse pass on 64 stratified samples per ray, then a fine pass on 128 importance samples merged with the 64 coarse ones. The fine pass ran self.mlp on all 192 merged samples, so 64 of them repeated rows the coarse pass had already computed with the same MLP on the same rays (the trunk always emits all 12 outputs; coarse and fine heads are slices of it). MLPNeRFModelOutput gets an optional raw_output field (the 12-wide MLP output), and the head activations of MLPNeRSTFModel.forward move to _outputs_from_raw. In the fine pass render_rays keeps the torch.sort indices of the merged ts, runs the MLP only on the new samples, gathers the new and coarse raw outputs in the sorted order and applies the fine heads. The render_rays docstring states that prev_model_out must come from the previous call on the same rays. encode_direction(direction=None) returns zeros of the same shape without computing the position encoding it discarded. Add a test that checks the fine pass against running the MLP on all merged ts and that it uses the coarse raw outputs, and one for encode_direction without a direction. The fine-pass test runs on CPU: its exact compare needs the matmul to give a row the same result whatever the number of rows, which did not hold on CUDA at the tiny test size. openai/shap-e at the model card settings, fp32, RTX 5090, eager: output bit-identical to main. On other GPUs/BLAS backends the output can differ from main by rounding. Under torch.compile the output differs from compiled main by at most 2.4e-7 (fp32 rounding). Signed-off-by: Hsu, Chan Fu --- src/diffusers/pipelines/shap_e/renderer.py | 33 +++++++++++---- tests/pipelines/shap_e/test_shap_e.py | 48 +++++++++++++++++++++- 2 files changed, 72 insertions(+), 9 deletions(-) diff --git a/src/diffusers/pipelines/shap_e/renderer.py b/src/diffusers/pipelines/shap_e/renderer.py index 0c2058c887fc..186b8ba35c71 100644 --- a/src/diffusers/pipelines/shap_e/renderer.py +++ b/src/diffusers/pipelines/shap_e/renderer.py @@ -72,7 +72,9 @@ def encode_position(position): def encode_direction(position, direction=None): if direction is None: - return torch.zeros_like(posenc_nerf(position, min_deg=0, max_deg=8)) + # Same shape as posenc_nerf(position, min_deg=0, max_deg=8), without computing an encoding that is discarded. + dim = position.shape[-1] + return position.new_zeros(*position.shape[:-1], dim + 2 * dim * 8) else: return posenc_nerf(direction, min_deg=0, max_deg=8) @@ -603,6 +605,7 @@ class MLPNeRFModelOutput(BaseOutput): signed_distance: torch.Tensor channels: torch.Tensor ts: torch.Tensor + raw_output: torch.Tensor | None = None class MLPNeRSTFModel(ModelMixin, ConfigMixin): @@ -680,6 +683,9 @@ def forward(self, *, position, direction, ts, nerf_level="coarse", rendering_mod if h_directionless is None: h_directionless = h_preact + return self._outputs_from_raw(h_final, ts, nerf_level=nerf_level, rendering_mode=rendering_mode) + + def _outputs_from_raw(self, h_final, ts, nerf_level="coarse", rendering_mode="nerf"): activation = self.map_indices_to_keys(h_final) if nerf_level == "coarse": @@ -701,7 +707,9 @@ def forward(self, *, position, direction, ts, nerf_level="coarse", rendering_mod channels = self.channel_activation(h_channels) # yiyi notes: I think signed_distance is not used - return MLPNeRFModelOutput(density=density, signed_distance=signed_distance, channels=channels, ts=ts) + return MLPNeRFModelOutput( + density=density, signed_distance=signed_distance, channels=channels, ts=ts, raw_output=h_final + ) class ChannelsProj(nn.Module): @@ -846,6 +854,8 @@ def render_rays(self, rays, sampler, n_samples, prev_model_out=None, render_with Args: rays: [batch_size x ... x 2 x 3] origin and direction. sampler: disjoint volume integrals. n_samples: number of ts to sample. prev_model_outputs: model outputs from the previous rendering step, including + `ts` and `raw_output`. They must come from the previous `render_rays` call of this renderer on the same + `rays` with the same `render_with_direction`; their raw MLP outputs are reused at their `ts`. :return: A tuple of - `channels` @@ -861,17 +871,17 @@ def render_rays(self, rays, sampler, n_samples, prev_model_out=None, render_with ts = sampler.sample(vrange.t0, vrange.t1, n_samples) ts = ts.to(rays.dtype) + # The MLP runs on the new ts only; the raw output at `prev_model_out.ts` is reused. + query_ts = ts if prev_model_out is not None: - # Append the previous ts now before fprop because previous - # rendering used a different model and we can't reuse the output. - ts = torch.sort(torch.cat([ts, prev_model_out.ts], dim=-2), dim=-2).values + ts, order = torch.sort(torch.cat([ts, prev_model_out.ts], dim=-2), dim=-2) batch_size, *_shape, _t0_dim = vrange.t0.shape - _, *ts_shape, _ts_dim = ts.shape + _, *ts_shape, _ts_dim = query_ts.shape # 2. Get the points along the ray and query the model directions = torch.broadcast_to(direction.unsqueeze(-2), [batch_size, *ts_shape, 3]) - positions = origin.unsqueeze(-2) + ts * directions + positions = origin.unsqueeze(-2) + query_ts * directions directions = directions.to(self.mlp.dtype) positions = positions.to(self.mlp.dtype) @@ -881,10 +891,17 @@ def render_rays(self, rays, sampler, n_samples, prev_model_out=None, render_with model_out = self.mlp( position=positions, direction=optional_directions, - ts=ts, + ts=query_ts, nerf_level="coarse" if prev_model_out is None else "fine", ) + if prev_model_out is not None: + # Only `raw_output` of the call above is used: merge it with the previous raw output in the sorted order + # of `ts` and apply the fine heads to the result. + raw = torch.cat([model_out.raw_output, prev_model_out.raw_output], dim=-2) + raw = torch.gather(raw, -2, order.expand(*order.shape[:-1], raw.shape[-1])) + model_out = self.mlp._outputs_from_raw(raw, ts, nerf_level="fine") + # 3. Integrate the model results channels, weights, transmittance = integrate_samples( vrange, model_out.ts, model_out.density, model_out.channels diff --git a/tests/pipelines/shap_e/test_shap_e.py b/tests/pipelines/shap_e/test_shap_e.py index aa8cc23aa8e3..10e54def85de 100644 --- a/tests/pipelines/shap_e/test_shap_e.py +++ b/tests/pipelines/shap_e/test_shap_e.py @@ -20,7 +20,8 @@ from transformers import CLIPTextConfig, CLIPTextModelWithProjection, CLIPTokenizer from diffusers import HeunDiscreteScheduler, PriorTransformer, ShapEPipeline -from diffusers.pipelines.shap_e import ShapERenderer +from diffusers.pipelines.shap_e import MLPNeRFModelOutput, ShapERenderer, StratifiedRaySampler, create_pan_cameras +from diffusers.pipelines.shap_e.renderer import encode_direction, posenc_nerf from ...testing_utils import ( backend_empty_cache, @@ -214,6 +215,51 @@ def test_num_images_per_prompt(self): def test_save_load_local(self, tmp_path, base_pipe_output): super().test_save_load_local(tmp_path, base_pipe_output, expected_max_difference=5e-3) + def test_renderer_fine_pass_reuse_matches_full_recompute(self): + # The fine pass runs the MLP only on the new samples and reuses the coarse raw MLP outputs at the coarse ts. + # It must match running the MLP on all merged ts. Exact equality relies on the MLP matmul giving the same + # result for a row whatever the number of rows. That held on CPU in our runs but not for a GPU GEMM kernel at + # this tiny size, so the check runs on CPU. + renderer = self.dummy_renderer.to("cpu") + rays = create_pan_cameras(4).camera_rays.to("cpu") + origin, direction = rays[..., 0, :], rays[..., 1, :] + + _, fine_sampler, coarse_out = renderer.render_rays(rays, StratifiedRaySampler(), 8) + torch.manual_seed(0) + _, _, fine_out = renderer.render_rays(rays, fine_sampler, 16, prev_model_out=coarse_out) + + torch.manual_seed(0) + vrange = renderer.volume.intersect(origin, direction, t0_lower=None) + new_ts = fine_sampler.sample(vrange.t0, vrange.t1, 16).to(rays.dtype) + expected_ts = torch.sort(torch.cat([new_ts, coarse_out.ts], dim=-2), dim=-2).values + positions = origin.unsqueeze(-2) + expected_ts * direction.unsqueeze(-2) + expected = renderer.mlp(position=positions, direction=None, ts=expected_ts, nerf_level="fine") + + assert torch.equal(fine_out.ts, expected_ts) + for name in ("density", "channels", "signed_distance", "raw_output"): + assert torch.equal(getattr(fine_out, name), getattr(expected, name)), name + + # The coarse raw outputs are used: changing them changes the fine pass. + changed_coarse_out = MLPNeRFModelOutput( + density=coarse_out.density, + signed_distance=coarse_out.signed_distance, + channels=coarse_out.channels, + ts=coarse_out.ts, + raw_output=coarse_out.raw_output + 1, + ) + torch.manual_seed(0) + _, _, changed_fine_out = renderer.render_rays(rays, fine_sampler, 16, prev_model_out=changed_coarse_out) + assert not torch.equal(changed_fine_out.raw_output, fine_out.raw_output) + + def test_encode_direction_without_direction(self): + position = torch.randn(2, 5, 3, device=torch_device) + expected = torch.zeros_like(posenc_nerf(position, min_deg=0, max_deg=8)) + output = encode_direction(position) + assert output.device == expected.device + assert output.dtype == expected.dtype + assert output.shape == expected.shape + assert torch.equal(output, expected) + class TestShapEPipelineMemory(ShapEPipelineTesterConfig, MemoryTesterMixin): """Memory optimization tests (CPU offload, group offload, layerwise casting) for the Shap-E pipeline."""