import os import sys import random import math import torch import numpy as np import torch.nn.functional as F from argparse import ArgumentParser from core.registry import register_method from core.base_method import BaseMethod sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../../ContextGS_offy'))) from utils.loss_utils import l1_loss, ssim from gaussian_renderer import prefilter_voxel, render as native_render from scene import Scene, GaussianModel from arguments import ModelParams, PipelineParams, OptimizationParams @register_method("contextgs") class ContextGS_offyWrapper(BaseMethod): def __init__(self, dataset_config, hyperparams): self.parser = ArgumentParser() self.lp = ModelParams(self.parser) self.op = OptimizationParams(self.parser) self.pp = PipelineParams(self.parser) self.parser.add_argument('--level_num', type=int, default=3) self.parser.add_argument('--level_scale', type=int, default=10) self.parser.add_argument("--n_features", type=int, default=4) self.parser.add_argument("--lmbda", type=float, default=0.001) self.parser.add_argument("--lmbda_rec", type=float, default=1) self.parser.add_argument("--disable_hyper", default=False, action="store_true") self.args = self.parser.parse_args([]) self.args.source_path = dataset_config["source_path"] self.args.model_path = dataset_config["model_path"] self.args.eval = True self.args.resolution = dataset_config.get("resolution", 1) self.track_decoupling = hyperparams.get("track_decoupling", False) self.dataset = self.lp.extract(self.args) self.opt = self.op.extract(self.args) self.pipe = self.pp.extract(self.args) self.args_param = self.args self.gaussians = GaussianModel( self.dataset.feat_dim, self.dataset.n_offsets, self.dataset.voxel_size, self.dataset.update_depth, self.dataset.update_init_factor, self.dataset.update_hierachy_factor, self.dataset.use_feat_bank, n_features_per_level=self.args_param.n_features, level_num=self.args_param.level_num, hyper_divisor=self.dataset.hyper_divisor, target_ratio=self.dataset.target_ratio, disable_hyper=self.args_param.disable_hyper ) # INJECTED_RES_FIX begin import sys as _sys _scene, _explicit_res = None, None for _i, _a in enumerate(_sys.argv[:-1]): _v = _sys.argv[_i + 1] if _a == "--scene": _scene = _v elif _a == "--source_path": _scene = _v.rstrip("/").split("/")[-1] elif _a == "--resolution": try: _explicit_res = int(_v) except: pass _OUTDOOR_360 = {"bicycle", "flowers", "garden", "stump", "treehill"} if _explicit_res is not None and _explicit_res > 0: _res = _explicit_res elif _scene is not None: _res = 4 if _scene in _OUTDOOR_360 else 2 else: _res = None try: if _res is not None: self.dataset.resolution = _res print("[res-fix] scene=%s explicit=%s -> res=%s (%s)" % (_scene, _explicit_res, _res, __file__)) except Exception as _e: print("[res-fix] FAILED:", _e) # INJECTED_RES_FIX end self.scene = Scene(self.dataset, self.gaussians) self.gaussians.update_anchor_bound() self.gaussians.training_setup(self.opt) bg_color = [1, 1, 1] if self.dataset.white_background else [0, 0, 0] self.background = torch.tensor(bg_color, dtype=torch.float32, device="cuda") self.viewpoint_stack = self.scene.getTrainCameras().copy() self.last_n_gaussians = self.gaussians.get_anchor.shape[0] def train_iteration(self, step): self.gaussians.update_learning_rate(step) if not self.viewpoint_stack: self.viewpoint_stack = self.scene.getTrainCameras().copy() viewpoint_cam = self.viewpoint_stack.pop(random.randint(0, len(self.viewpoint_stack) - 1)) retain_grad = (step < self.opt.update_until and step >= 0) voxel_visible_mask = prefilter_voxel(viewpoint_cam, self.gaussians, self.pipe, self.background) render_pkg = native_render(viewpoint_cam, self.gaussians, self.pipe, self.background, visible_mask=voxel_visible_mask, retain_grad=retain_grad, step=step) image = render_pkg["render"] viewspace_point_tensor = render_pkg["viewspace_points"] visibility_filter = render_pkg["visibility_filter"] offset_selection_mask = render_pkg["selection_mask"] opacity = render_pkg["neural_opacity"] scaling = render_pkg["scaling"] bit_per_param = render_pkg.get("bit_per_param", None) gt_image = viewpoint_cam.original_image.cuda() Ll1 = l1_loss(image, gt_image) ssim_value = ssim(image, gt_image) scaling_reg = scaling.prod(dim=1).mean() loss_target = self.args_param.lmbda_rec * ((1.0 - self.opt.lambda_dssim) * Ll1 + self.opt.lambda_dssim * (1.0 - ssim_value)) loss_parasitic = 0.01 * scaling_reg if bit_per_param is not None: loss_parasitic = loss_parasitic + self.args_param.lmbda * bit_per_param loss_parasitic = loss_parasitic + 5e-4 * torch.mean(torch.sigmoid(self.gaussians._mask)) loss = loss_target + loss_parasitic grad_cos_sim = 0.0 parasitic_ratio = 0.0 stats = {} if self.track_decoupling and step % 100 == 0: param_groups_map = { "spatial": [self.gaussians._anchor, self.gaussians._offset], "geometry": [self.gaussians._scaling, self.gaussians._rotation], "opacity": [self.gaussians._opacity, self.gaussians._mask], "appearance": [self.gaussians._anchor_feat, self.gaussians._hyper_latent], } self.gaussians.optimizer.zero_grad(set_to_none=True) loss_target.backward(retain_graph=True) grads_target = {} for group_name, params in param_groups_map.items(): grads_target[group_name] = [] for p in params: if p.grad is not None: grads_target[group_name].append(p.grad.clone()) else: grads_target[group_name].append(torch.zeros_like(p)) self.gaussians.optimizer.zero_grad(set_to_none=True) loss_parasitic.backward(retain_graph=True) grads_parasitic = {} for group_name, params in param_groups_map.items(): grads_parasitic[group_name] = [] for p in params: if p.grad is not None: grads_parasitic[group_name].append(p.grad.clone()) else: grads_parasitic[group_name].append(torch.zeros_like(p)) for group_name, params in param_groups_map.items(): u_t_list = [] u_p_list = [] for i, p in enumerate(params): state = self.gaussians.optimizer.state.get(p, None) if state is not None and "exp_avg_sq" in state: v_t = state["exp_avg_sq"] else: v_t = torch.ones_like(p) for param_group in self.gaussians.optimizer.param_groups: if id(p) in [id(opt_p) for opt_p in param_group['params']]: lr = param_group['lr'] break else: lr = 1e-3 u_t = (lr / (torch.sqrt(v_t) + 1e-8)) * grads_target[group_name][i] u_p = (lr / (torch.sqrt(v_t) + 1e-8)) * grads_parasitic[group_name][i] u_t_list.append(u_t.reshape(-1)) u_p_list.append(u_p.reshape(-1)) gt = torch.cat(u_t_list) gp = torch.cat(u_p_list) if gt.norm() > 0 and gp.norm() > 0: cos = float(F.cosine_similarity(gt.unsqueeze(0), gp.unsqueeze(0))) r = float(gp.norm() / (gt.norm() + gp.norm() + 1e-7)) ti = r * max(0.0, -cos) else: ti = 0.0 stats[f"sti_{group_name}"] = ti all_gt = torch.cat([torch.cat(u_t_list) for group_name, params in param_groups_map.items() for i, p in enumerate(params)]) all_gp = torch.cat([torch.cat(u_p_list) for group_name, params in param_groups_map.items() for i, p in enumerate(params)]) if all_gt.norm() > 0 and all_gp.norm() > 0: grad_cos_sim = float(F.cosine_similarity(all_gt.unsqueeze(0), all_gp.unsqueeze(0))) parasitic_ratio = float(all_gp.norm() / (all_gt.norm() + 1e-7)) self.gaussians.optimizer.zero_grad(set_to_none=True) loss.backward() else: loss.backward() with torch.no_grad(): if step < self.opt.update_until and step > self.opt.start_stat: self.gaussians.training_statis(viewspace_point_tensor, opacity, visibility_filter, offset_selection_mask, voxel_visible_mask) if step not in range(3000, 4000): if step > self.opt.update_from and step % self.opt.update_interval == 0: self.gaussians.adjust_anchor(check_interval=self.opt.update_interval, success_threshold=self.opt.success_threshold, grad_threshold=self.opt.densify_grad_threshold, min_opacity=self.opt.min_opacity) elif step == self.opt.update_until: del self.gaussians.opacity_accum del self.gaussians.offset_gradient_accum del self.gaussians.offset_denom torch.cuda.empty_cache() if step < self.opt.iterations: self.gaussians.optimizer.step() self.gaussians.optimizer.zero_grad(set_to_none=True) num_gaussians = self.gaussians.get_anchor.shape[0] metrics = { "loss": float(loss), "loss_l1": float(loss_target), "loss_ssim": float(loss_parasitic), "num_gaussians": int(num_gaussians), "delta_N": int(num_gaussians - self.last_n_gaussians), "peak_vram_GB": float(torch.cuda.max_memory_allocated() / (1024 ** 3)), "grad_cos_sim": float(grad_cos_sim), "parasitic_ratio": float(parasitic_ratio) } metrics.update(stats) if bit_per_param is not None: metrics["bit_per_param"] = float(bit_per_param) self.last_n_gaussians = num_gaussians histograms = {} if step % 1000 == 0: histograms["opacity"] = torch.sigmoid(self.gaussians._opacity).clone().detach() scales = self.gaussians.get_scaling.clone().detach() histograms["scaling"] = scales scales_2d = scales[:, :2] if scales.shape[1] >= 2 else scales gamma = scales_2d.max(dim=-1)[0] / (scales_2d.min(dim=-1)[0] + 1e-7) histograms["anisotropy"] = gamma histograms["anchor_feat_mag"] = self.gaussians._anchor_feat.detach().norm(dim=-1) return metrics, histograms def render(self, camera): with torch.no_grad(): voxel_visible_mask = prefilter_voxel(camera, self.gaussians, self.pipe, self.background) render_pkg = native_render(camera, self.gaussians, self.pipe, self.background, visible_mask=voxel_visible_mask) return {"image": render_pkg["render"], "depth": render_pkg.get("depth", None)} def save(self, save_dir, step): self.scene.save(step) def load(self, model_path, iteration): self.gaussians.load_ply_sparse_gaussian(os.path.join(model_path, 'point_cloud', f'iteration_{iteration}', 'point_cloud.ply')) def get_spatial_centers(self): return self.gaussians.get_anchor def compute_physical_metrics(self, cameras=None): metrics = {} with torch.no_grad(): scales = self.gaussians.get_scaling scales_2d = scales[:, :2] if scales.dim() > 1 and scales.shape[1] >= 2 else scales.unsqueeze(-1).expand(-1, 2) max_S, _ = torch.max(scales_2d, dim=1) min_S, _ = torch.min(scales_2d, dim=1) gamma = max_S / (min_S + 1e-7) metrics["gamma_median"] = float(torch.median(gamma)) metrics["gamma_90th_percentile"] = float(torch.quantile(gamma, 0.90)) metrics["scale_mean"] = float(torch.mean(scales_2d)) metrics["alpha_mean"] = float(torch.mean(torch.sigmoid(self.gaussians._opacity))) feat, hyper = self.gaussians._anchor_feat, self.gaussians._hyper_latent if hyper is not None and hyper.shape[1] > 0: metrics["hyper_energy_ratio"] = float(hyper.norm(dim=-1).mean() / (feat.norm(dim=-1).mean() + 1e-7)) if cameras is not None and len(cameras) > 0: view_dirs = [] for c in cameras: view_dirs.append(c.world_view_transform[:3, 2].tolist()) view_dirs = F.normalize(torch.tensor(view_dirs, dtype=torch.float32, device="cuda"), dim=1) rots = F.normalize(self.gaussians._rotation.clone(), dim=1) w, x, y, z = rots.unbind(dim=-1) normals = F.normalize(torch.stack([2*(x*z + w*y), 2*(y*z - w*x), 1-2*(x*x + y*y)], dim=-1), dim=1) max_cos, _ = torch.max(torch.abs(torch.matmul(normals, view_dirs.T)), dim=1) metrics["billboard_bias_ratio"] = float((max_cos > 0.90).float().mean()) return metrics def evaluate_spatial_field(self, query_points: torch.Tensor, cameras=None) -> torch.Tensor: with torch.no_grad(): V = query_points.shape[0] densities = torch.zeros(V, device="cuda") xyz = self.gaussians.get_anchor opacities = torch.sigmoid(self.gaussians._opacity).squeeze() scales = self.gaussians.get_scaling sigma_sq = (scales[:, :2].max(dim=1)[0].pow(2)) if scales.shape[1] >= 2 else scales.squeeze().pow(2) N_gaussians = xyz.shape[0] chunk_size = max(1, 30_000_000 // (N_gaussians + 1)) for i in range(0, V, chunk_size): end = min(i + chunk_size, V) dist_sq = torch.cdist(query_points[i:end], xyz, p=2).pow(2) weights = torch.exp(-0.5 * dist_sq / (sigma_sq.unsqueeze(0) + 1e-7)) densities[i:end] = torch.sum(weights * opacities.unsqueeze(0), dim=1) return densities