diff --git a/src/diffusers/pipelines/shap_e/renderer.py b/src/diffusers/pipelines/shap_e/renderer.py index 0c2058c887fc..186b8ba35c71 100644 --- a/src/diffusers/pipelines/shap_e/renderer.py +++ b/src/diffusers/pipelines/shap_e/renderer.py @@ -72,7 +72,9 @@ def encode_position(position): def encode_direction(position, direction=None): if direction is None: - return torch.zeros_like(posenc_nerf(position, min_deg=0, max_deg=8)) + # Same shape as posenc_nerf(position, min_deg=0, max_deg=8), without computing an encoding that is discarded. + dim = position.shape[-1] + return position.new_zeros(*position.shape[:-1], dim + 2 * dim * 8) else: return posenc_nerf(direction, min_deg=0, max_deg=8) @@ -603,6 +605,7 @@ class MLPNeRFModelOutput(BaseOutput): signed_distance: torch.Tensor channels: torch.Tensor ts: torch.Tensor + raw_output: torch.Tensor | None = None class MLPNeRSTFModel(ModelMixin, ConfigMixin): @@ -680,6 +683,9 @@ def forward(self, *, position, direction, ts, nerf_level="coarse", rendering_mod if h_directionless is None: h_directionless = h_preact + return self._outputs_from_raw(h_final, ts, nerf_level=nerf_level, rendering_mode=rendering_mode) + + def _outputs_from_raw(self, h_final, ts, nerf_level="coarse", rendering_mode="nerf"): activation = self.map_indices_to_keys(h_final) if nerf_level == "coarse": @@ -701,7 +707,9 @@ def forward(self, *, position, direction, ts, nerf_level="coarse", rendering_mod channels = self.channel_activation(h_channels) # yiyi notes: I think signed_distance is not used - return MLPNeRFModelOutput(density=density, signed_distance=signed_distance, channels=channels, ts=ts) + return MLPNeRFModelOutput( + density=density, signed_distance=signed_distance, channels=channels, ts=ts, raw_output=h_final + ) class ChannelsProj(nn.Module): @@ -846,6 +854,8 @@ def render_rays(self, rays, sampler, n_samples, prev_model_out=None, render_with Args: rays: [batch_size x ... x 2 x 3] origin and direction. sampler: disjoint volume integrals. n_samples: number of ts to sample. prev_model_outputs: model outputs from the previous rendering step, including + `ts` and `raw_output`. They must come from the previous `render_rays` call of this renderer on the same + `rays` with the same `render_with_direction`; their raw MLP outputs are reused at their `ts`. :return: A tuple of - `channels` @@ -861,17 +871,17 @@ def render_rays(self, rays, sampler, n_samples, prev_model_out=None, render_with ts = sampler.sample(vrange.t0, vrange.t1, n_samples) ts = ts.to(rays.dtype) + # The MLP runs on the new ts only; the raw output at `prev_model_out.ts` is reused. + query_ts = ts if prev_model_out is not None: - # Append the previous ts now before fprop because previous - # rendering used a different model and we can't reuse the output. - ts = torch.sort(torch.cat([ts, prev_model_out.ts], dim=-2), dim=-2).values + ts, order = torch.sort(torch.cat([ts, prev_model_out.ts], dim=-2), dim=-2) batch_size, *_shape, _t0_dim = vrange.t0.shape - _, *ts_shape, _ts_dim = ts.shape + _, *ts_shape, _ts_dim = query_ts.shape # 2. Get the points along the ray and query the model directions = torch.broadcast_to(direction.unsqueeze(-2), [batch_size, *ts_shape, 3]) - positions = origin.unsqueeze(-2) + ts * directions + positions = origin.unsqueeze(-2) + query_ts * directions directions = directions.to(self.mlp.dtype) positions = positions.to(self.mlp.dtype) @@ -881,10 +891,17 @@ def render_rays(self, rays, sampler, n_samples, prev_model_out=None, render_with model_out = self.mlp( position=positions, direction=optional_directions, - ts=ts, + ts=query_ts, nerf_level="coarse" if prev_model_out is None else "fine", ) + if prev_model_out is not None: + # Only `raw_output` of the call above is used: merge it with the previous raw output in the sorted order + # of `ts` and apply the fine heads to the result. + raw = torch.cat([model_out.raw_output, prev_model_out.raw_output], dim=-2) + raw = torch.gather(raw, -2, order.expand(*order.shape[:-1], raw.shape[-1])) + model_out = self.mlp._outputs_from_raw(raw, ts, nerf_level="fine") + # 3. Integrate the model results channels, weights, transmittance = integrate_samples( vrange, model_out.ts, model_out.density, model_out.channels diff --git a/tests/pipelines/shap_e/test_shap_e.py b/tests/pipelines/shap_e/test_shap_e.py index aa8cc23aa8e3..10e54def85de 100644 --- a/tests/pipelines/shap_e/test_shap_e.py +++ b/tests/pipelines/shap_e/test_shap_e.py @@ -20,7 +20,8 @@ from transformers import CLIPTextConfig, CLIPTextModelWithProjection, CLIPTokenizer from diffusers import HeunDiscreteScheduler, PriorTransformer, ShapEPipeline -from diffusers.pipelines.shap_e import ShapERenderer +from diffusers.pipelines.shap_e import MLPNeRFModelOutput, ShapERenderer, StratifiedRaySampler, create_pan_cameras +from diffusers.pipelines.shap_e.renderer import encode_direction, posenc_nerf from ...testing_utils import ( backend_empty_cache, @@ -214,6 +215,51 @@ def test_num_images_per_prompt(self): def test_save_load_local(self, tmp_path, base_pipe_output): super().test_save_load_local(tmp_path, base_pipe_output, expected_max_difference=5e-3) + def test_renderer_fine_pass_reuse_matches_full_recompute(self): + # The fine pass runs the MLP only on the new samples and reuses the coarse raw MLP outputs at the coarse ts. + # It must match running the MLP on all merged ts. Exact equality relies on the MLP matmul giving the same + # result for a row whatever the number of rows. That held on CPU in our runs but not for a GPU GEMM kernel at + # this tiny size, so the check runs on CPU. + renderer = self.dummy_renderer.to("cpu") + rays = create_pan_cameras(4).camera_rays.to("cpu") + origin, direction = rays[..., 0, :], rays[..., 1, :] + + _, fine_sampler, coarse_out = renderer.render_rays(rays, StratifiedRaySampler(), 8) + torch.manual_seed(0) + _, _, fine_out = renderer.render_rays(rays, fine_sampler, 16, prev_model_out=coarse_out) + + torch.manual_seed(0) + vrange = renderer.volume.intersect(origin, direction, t0_lower=None) + new_ts = fine_sampler.sample(vrange.t0, vrange.t1, 16).to(rays.dtype) + expected_ts = torch.sort(torch.cat([new_ts, coarse_out.ts], dim=-2), dim=-2).values + positions = origin.unsqueeze(-2) + expected_ts * direction.unsqueeze(-2) + expected = renderer.mlp(position=positions, direction=None, ts=expected_ts, nerf_level="fine") + + assert torch.equal(fine_out.ts, expected_ts) + for name in ("density", "channels", "signed_distance", "raw_output"): + assert torch.equal(getattr(fine_out, name), getattr(expected, name)), name + + # The coarse raw outputs are used: changing them changes the fine pass. + changed_coarse_out = MLPNeRFModelOutput( + density=coarse_out.density, + signed_distance=coarse_out.signed_distance, + channels=coarse_out.channels, + ts=coarse_out.ts, + raw_output=coarse_out.raw_output + 1, + ) + torch.manual_seed(0) + _, _, changed_fine_out = renderer.render_rays(rays, fine_sampler, 16, prev_model_out=changed_coarse_out) + assert not torch.equal(changed_fine_out.raw_output, fine_out.raw_output) + + def test_encode_direction_without_direction(self): + position = torch.randn(2, 5, 3, device=torch_device) + expected = torch.zeros_like(posenc_nerf(position, min_deg=0, max_deg=8)) + output = encode_direction(position) + assert output.device == expected.device + assert output.dtype == expected.dtype + assert output.shape == expected.shape + assert torch.equal(output, expected) + class TestShapEPipelineMemory(ShapEPipelineTesterConfig, MemoryTesterMixin): """Memory optimization tests (CPU offload, group offload, layerwise casting) for the Shap-E pipeline."""