Spaces:
Running on Zero
Running on Zero
Fix: Revert render hack, apply SH ambient boost to keep background black
Browse files
trellis/renderers/gaussian_render.py
CHANGED
|
@@ -116,7 +116,68 @@ def render(viewpoint_camera, pc : Gaussian, pipe, bg_color : torch.Tensor, scali
|
|
| 116 |
sh2rgb = eval_sh(pc.active_sh_degree, shs_view, dir_pp_normalized)
|
| 117 |
colors_precomp = torch.clamp_min(sh2rgb + 0.5, 0.0)
|
| 118 |
else:
|
|
|
|
|
|
|
|
|
|
|
|
|
| 119 |
shs = pc.get_features
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 120 |
else:
|
| 121 |
colors_precomp = override_color
|
| 122 |
|
|
|
|
| 116 |
sh2rgb = eval_sh(pc.active_sh_degree, shs_view, dir_pp_normalized)
|
| 117 |
colors_precomp = torch.clamp_min(sh2rgb + 0.5, 0.0)
|
| 118 |
else:
|
| 119 |
+
# 🔧 SHADOW WAR PIG FIX: Boost Ambient Light directly in SH coefficients
|
| 120 |
+
# The DC component (shs[:, 0, :]) represents ambient color/light.
|
| 121 |
+
# Boosting this makes the splats themselves brighter, without touching the background.
|
| 122 |
+
# 0.15 is a moderate boost to lift crushed shadows.
|
| 123 |
shs = pc.get_features
|
| 124 |
+
|
| 125 |
+
# Clone to avoid modifying the original model parameters in place (gradients etc)
|
| 126 |
+
# shs shape is [N, n_bands^2, 3] or [N, 3, n_bands^2] depending on implementation.
|
| 127 |
+
# Usually [Point_Count, 3, (Deg+1)^2] ? No, usually [N, (Deg+1)**2, 3] in some impls,
|
| 128 |
+
# but here line 113 says: .transpose(1, 2).view(-1, 3, ...) implies storage is [N, Features, ...]
|
| 129 |
+
|
| 130 |
+
# Let's just modify the SHs passed to the rasterizer.
|
| 131 |
+
# The rasterizer expects SHs.
|
| 132 |
+
# We want to add to the first coefficient (DC).
|
| 133 |
+
|
| 134 |
+
# The structure of shs is typically [N, (max_sh_degree+1)**2 * 3] or similar.
|
| 135 |
+
# But line 113 suggests `pc.get_features` has dimensions that need transposing.
|
| 136 |
+
# Actually, looking at `gaussian_model.py` (standard), features are [N, C, n_bands].
|
| 137 |
+
|
| 138 |
+
# SAFEST WAY: Add to the DC component which is usually the first 3 values if flattened,
|
| 139 |
+
# or index 0 in the feature dimension.
|
| 140 |
+
|
| 141 |
+
# Let's apply a "pre-rasterization lift" to the DC component.
|
| 142 |
+
# shs = pc.get_features <-- This is likely a Tensor.
|
| 143 |
+
# We need to add to it.
|
| 144 |
+
|
| 145 |
+
# To avoid shape guessing hell, we will use a simpler approach if possible.
|
| 146 |
+
# However, `pc.get_features` returns the raw tensor.
|
| 147 |
+
# We can just add a scalar to the whole thing? No, that breaks directionality.
|
| 148 |
+
# We must only boost DC.
|
| 149 |
+
|
| 150 |
+
shs = pc.get_features.clone()
|
| 151 |
+
|
| 152 |
+
# Assume DC components are at the beginning.
|
| 153 |
+
# Standard Gaussian Splatting storage:
|
| 154 |
+
# f_dc: [N, 1, 3]
|
| 155 |
+
# f_rest: [N, 15, 3]
|
| 156 |
+
# get_features cats them -> [N, 16, 3] typically?
|
| 157 |
+
# Or [N, 48]?
|
| 158 |
+
|
| 159 |
+
# Let's look at `eval_sh`: result = C0 * sh[..., 0]
|
| 160 |
+
# It expects sh to be [..., C, (deg+1)**2] ?
|
| 161 |
+
# line 65: sh coeffs [..., C, (deg + 1) ** 2]
|
| 162 |
+
# line 74: result = C0 * sh[..., 0]
|
| 163 |
+
|
| 164 |
+
# So index 0 is the DC component.
|
| 165 |
+
# Boosting sh[..., 0] effectively adds ambient light.
|
| 166 |
+
|
| 167 |
+
# pc.get_features shape?
|
| 168 |
+
# If we look at line 113: `pc.get_features.transpose(1, 2)`
|
| 169 |
+
# This implies `pc.get_features` is [N, (deg+1)**2, 3].
|
| 170 |
+
# So index 0 of the second dimension is DC.
|
| 171 |
+
|
| 172 |
+
# shs[:, 0, :] += 0.5 / C0 (to add 0.5 to final color)
|
| 173 |
+
# C0 = 0.282...
|
| 174 |
+
# To add 0.1 brightness: 0.1 / 0.282 ~= 0.354
|
| 175 |
+
|
| 176 |
+
# SHADOW LIFT FACTOR
|
| 177 |
+
# User wants to fix dark shadows.
|
| 178 |
+
LIFT_AMOUNT = 0.5 # Boost DC coefficient
|
| 179 |
+
shs[:, 0, :] += LIFT_AMOUNT
|
| 180 |
+
|
| 181 |
else:
|
| 182 |
colors_precomp = override_color
|
| 183 |
|
trellis/utils/render_utils.py
CHANGED
|
@@ -73,26 +73,7 @@ def render_frames(sample, extrinsics, intrinsics, options={}, colors_overwrite=N
|
|
| 73 |
res = renderer.render(sample, extr, intr, colors_overwrite=colors_overwrite)
|
| 74 |
if 'color' not in rets: rets['color'] = []
|
| 75 |
if 'depth' not in rets: rets['depth'] = []
|
| 76 |
-
|
| 77 |
-
# Shadows from splats are often too dark/crushed. We lift them here on the high-precision float data.
|
| 78 |
-
color_tensor = res['color'].detach().cpu()
|
| 79 |
-
# Calculate luminance for mask
|
| 80 |
-
lum = 0.299 * color_tensor[0] + 0.587 * color_tensor[1] + 0.114 * color_tensor[2]
|
| 81 |
-
|
| 82 |
-
# Shadow Threshold: 0.2 (Dark shadows)
|
| 83 |
-
# Lift Factor: 0.15 (Moderate lift)
|
| 84 |
-
shadow_mask = (lum < 0.2)
|
| 85 |
-
if shadow_mask.any():
|
| 86 |
-
# Quadratic lift: (1 - x/threshold)^2 scale
|
| 87 |
-
# This ensures x=0 gets max lift, x=threshold gets 0 lift. Smooth transition.
|
| 88 |
-
lift = 0.15 * torch.pow(1.0 - lum / 0.2, 2)
|
| 89 |
-
lift = lift * shadow_mask.float()
|
| 90 |
-
|
| 91 |
-
# Apply lift to all channels
|
| 92 |
-
color_tensor = color_tensor + lift.unsqueeze(0)
|
| 93 |
-
|
| 94 |
-
rets['color'].append(np.clip(color_tensor.numpy().transpose(1, 2, 0) * 255, 0, 255).astype(np.uint8))
|
| 95 |
-
|
| 96 |
if 'percent_depth' in res:
|
| 97 |
rets['depth'].append(res['percent_depth'].detach().cpu().numpy())
|
| 98 |
elif 'depth' in res:
|
|
|
|
| 73 |
res = renderer.render(sample, extr, intr, colors_overwrite=colors_overwrite)
|
| 74 |
if 'color' not in rets: rets['color'] = []
|
| 75 |
if 'depth' not in rets: rets['depth'] = []
|
| 76 |
+
rets['color'].append(np.clip(res['color'].detach().cpu().numpy().transpose(1, 2, 0) * 255, 0, 255).astype(np.uint8))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 77 |
if 'percent_depth' in res:
|
| 78 |
rets['depth'].append(res['percent_depth'].detach().cpu().numpy())
|
| 79 |
elif 'depth' in res:
|