| | from typing import * |
| | import torch |
| | import torch.nn as nn |
| | import torch.nn.functional as F |
| | import numpy as np |
| | from ...modules import sparse as sp |
| | from .base import SparseTransformerBase |
| | from ...representations import Strivec |
| |
|
| |
|
| | class SLatRadianceFieldDecoder(SparseTransformerBase): |
| | def __init__( |
| | self, |
| | resolution: int, |
| | model_channels: int, |
| | latent_channels: int, |
| | num_blocks: int, |
| | num_heads: Optional[int] = None, |
| | num_head_channels: Optional[int] = 64, |
| | mlp_ratio: float = 4, |
| | attn_mode: Literal[ |
| | "full", "shift_window", "shift_sequence", "shift_order", "swin" |
| | ] = "swin", |
| | window_size: int = 8, |
| | pe_mode: Literal["ape", "rope"] = "ape", |
| | use_fp16: bool = False, |
| | use_checkpoint: bool = False, |
| | qk_rms_norm: bool = False, |
| | representation_config: dict = None, |
| | ): |
| | super().__init__( |
| | in_channels=latent_channels, |
| | model_channels=model_channels, |
| | num_blocks=num_blocks, |
| | num_heads=num_heads, |
| | num_head_channels=num_head_channels, |
| | mlp_ratio=mlp_ratio, |
| | attn_mode=attn_mode, |
| | window_size=window_size, |
| | pe_mode=pe_mode, |
| | use_fp16=use_fp16, |
| | use_checkpoint=use_checkpoint, |
| | qk_rms_norm=qk_rms_norm, |
| | ) |
| | self.resolution = resolution |
| | self.rep_config = representation_config |
| | self._calc_layout() |
| | self.out_layer = sp.SparseLinear(model_channels, self.out_channels) |
| |
|
| | self.initialize_weights() |
| | if use_fp16: |
| | self.convert_to_fp16() |
| |
|
| | def initialize_weights(self) -> None: |
| | super().initialize_weights() |
| | |
| | nn.init.constant_(self.out_layer.weight, 0) |
| | nn.init.constant_(self.out_layer.bias, 0) |
| |
|
| | def _calc_layout(self) -> None: |
| | self.layout = { |
| | "trivec": { |
| | "shape": (self.rep_config["rank"], 3, self.rep_config["dim"]), |
| | "size": self.rep_config["rank"] * 3 * self.rep_config["dim"], |
| | }, |
| | "density": { |
| | "shape": (self.rep_config["rank"],), |
| | "size": self.rep_config["rank"], |
| | }, |
| | "features_dc": { |
| | "shape": (self.rep_config["rank"], 1, 3), |
| | "size": self.rep_config["rank"] * 3, |
| | }, |
| | } |
| | start = 0 |
| | for k, v in self.layout.items(): |
| | v["range"] = (start, start + v["size"]) |
| | start += v["size"] |
| | self.out_channels = start |
| |
|
| | def to_representation(self, x: sp.SparseTensor) -> List[Strivec]: |
| | """ |
| | Convert a batch of network outputs to 3D representations. |
| | |
| | Args: |
| | x: The [N x * x C] sparse tensor output by the network. |
| | |
| | Returns: |
| | list of representations |
| | """ |
| | ret = [] |
| | for i in range(x.shape[0]): |
| | representation = Strivec( |
| | sh_degree=0, |
| | resolution=self.resolution, |
| | aabb=[-0.5, -0.5, -0.5, 1, 1, 1], |
| | rank=self.rep_config["rank"], |
| | dim=self.rep_config["dim"], |
| | device="cuda", |
| | ) |
| | representation.density_shift = 0.0 |
| | representation.position = ( |
| | x.coords[x.layout[i]][:, 1:].float() + 0.5 |
| | ) / self.resolution |
| | representation.depth = torch.full( |
| | (representation.position.shape[0], 1), |
| | int(np.log2(self.resolution)), |
| | dtype=torch.uint8, |
| | device="cuda", |
| | ) |
| | for k, v in self.layout.items(): |
| | setattr( |
| | representation, |
| | k, |
| | x.feats[x.layout[i]][:, v["range"][0] : v["range"][1]].reshape( |
| | -1, *v["shape"] |
| | ), |
| | ) |
| | representation.trivec = representation.trivec + 1 |
| | ret.append(representation) |
| | return ret |
| |
|
| | def forward(self, x: sp.SparseTensor) -> List[Strivec]: |
| | h = super().forward(x) |
| | h = h.type(x.dtype) |
| | h = h.replace(F.layer_norm(h.feats, h.feats.shape[-1:])) |
| | h = self.out_layer(h) |
| | return self.to_representation(h) |
| |
|
