Get Depth Map

[OliviaDumbledore]

Thank you for sharing the code with us!
I notice that in your paper, you show us the depth and normal map

But I cannot reproduce it. Can you figure me out?

[tobias-kirschstein]

Hi,
I added it to the inference notebook now.
You can obtain a depth rendering like this

from gaussian_splatting.arguments import PipelineParams2
from gaussian_splatting.gaussian_renderer import render
from gaussian_splatting.scene.cameras import pose_to_rendercam

gaussian_model = model._setup_gaussian_model(output.gaussian_attribute_output.gaussian_attributes, 0)
gs_camera = pose_to_rendercam(pose_front, DEFAULT_INTRINSICS.rescale(render_resolution, inplace=False), render_resolution, render_resolution, device=device)
render_output = render(gs_camera, gaussian_model, PipelineParams2(), torch.tensor([1., 1., 1.]).cuda(), return_depth=True)
depth_map = render_output['depth'].permute(1, 2, 0).detach().cpu().numpy()

alpha_masks = ((output.masks[0] + 1)/2).permute(1, 2, 0).detach().cpu().numpy()
fixed_depth = depth_map / alpha_masks  # Account for the fact that semi-transparent pixels will have wrong depth since Gaussian contributions don't sum to 1

plt.imshow(fixed_depth, cmap='turbo')
plt.show()

And the normals can be derived from the rendered depth like this:

import cv2


def convert_depth_to_normal_map(depth_map: np.ndarray, ksize: int = 5, scale: float = 5) -> np.ndarray:
    rows, cols = depth_map.shape

    # Calculate the partial derivatives of depth with respect to x and y
    dx = cv2.Sobel(depth_map, cv2.CV_32F, 1, 0, ksize=ksize, scale=scale)
    dy = cv2.Sobel(depth_map, cv2.CV_32F, 0, 1, ksize=ksize, scale=scale)

    # Compute the normal vector for each pixel
    normal = np.dstack((-dx, -dy, np.ones((rows, cols))))
    norm = np.sqrt(np.sum(normal ** 2, axis=2, keepdims=True))
    normal = np.divide(normal, norm, out=np.zeros_like(normal), where=norm != 0)

    # Map the normal vectors to the [0, 255] range and convert to uint8
    normal = (normal + 1) * 127.5
    normal = normal.clip(0, 255).astype(np.uint8)

    return normal

normal_map = convert_depth_to_normal_map(fixed_depth[..., 0])
plt.imshow(normal_map)