| |
| |
| |
| |
| |
| |
| \documentclass[10pt,twocolumn]{article} |
|
|
| \usepackage[utf8]{inputenc} |
| \usepackage{amsmath,amssymb,amsfonts} |
| \usepackage{algorithm} |
| \usepackage{algorithmic} |
| \usepackage{booktabs} |
| \usepackage{hyperref} |
| \usepackage{geometry} |
| \geometry{margin=1in} |
|
|
| \newcommand{\R}{\mathbb{R}} |
| \newcommand{\calL}{\mathcal{L}} |
| \newcommand{\calP}{\mathcal{P}} |
| \newcommand{\calG}{\mathcal{G}} |
| \newcommand{\calF}{\mathcal{F}} |
| \newcommand{\calV}{\mathcal{V}} |
| \newcommand{\bmu}{\boldsymbol{\mu}} |
| \newcommand{\bq}{\mathbf{q}} |
| \newcommand{\bs}{\mathbf{s}} |
| \newcommand{\bx}{\mathbf{x}} |
| \newcommand{\bp}{\mathbf{p}} |
| \newcommand{\bDelta}{\boldsymbol{\Delta}} |
|
|
| \title{\textbf{EndoGaussian-4D: Mathematical Formalization} \\ |
| \large Physics-Informed Deformation Field for Real-Time \\ |
| 4D Gaussian Splatting in Endoscopic Surgery} |
| \author{EndoGaussian-4D Research Team} |
| \date{Week 1 Sprint --- Phase 3 Deliverable} |
|
|
| \begin{document} |
| \maketitle |
|
|
| |
| \section{Preliminary: 3D Gaussian Splatting} |
| |
|
|
| A scene is represented as a set of $N$ anisotropic 3D Gaussians: |
| \begin{equation} |
| \calG = \{ G_i \}_{i=1}^{N}, \quad G_i = (\bmu_i, \bq_i, \bs_i, \alpha_i, \mathbf{c}_i) |
| \end{equation} |
| where each Gaussian $G_i$ is parameterized by: |
| \begin{itemize} |
| \item $\bmu_i \in \R^3$: Mean (center position) |
| \item $\bq_i \in \R^4$: Unit quaternion encoding rotation |
| \item $\bs_i \in \R^3$: Log-scale vector (anisotropic scaling) |
| \item $\alpha_i \in \R$: Logit-opacity |
| \item $\mathbf{c}_i \in \R^{K \times 3}$: Spherical harmonics coefficients ($K = (\ell+1)^2$ for degree $\ell$) |
| \end{itemize} |
|
|
| The covariance matrix of each Gaussian in world space is: |
| \begin{equation} |
| \Sigma_i = R(\bq_i) \, S(\bs_i) \, S(\bs_i)^\top \, R(\bq_i)^\top |
| \end{equation} |
| where $R(\bq_i) \in SO(3)$ is the rotation matrix from quaternion $\bq_i$, and $S(\bs_i) = \text{diag}(e^{s_{i,1}}, e^{s_{i,2}}, e^{s_{i,3}})$ is the diagonal scaling matrix. |
|
|
| The influence of Gaussian $G_i$ at a 3D point $\bx$ is: |
| \begin{equation} |
| g_i(\bx) = \exp\!\left( -\tfrac{1}{2} (\bx - \bmu_i)^\top \Sigma_i^{-1} (\bx - \bmu_i) \right) |
| \end{equation} |
|
|
| \paragraph{Differentiable Rasterization.} |
| For rendering, Gaussians are projected onto the image plane via the EWA splatting approximation. Given camera extrinsic $[R_c | \mathbf{t}_c]$ and intrinsic $K$, the 2D projected covariance is: |
| \begin{equation} |
| \Sigma'_i = J \, W \, \Sigma_i \, W^\top \, J^\top |
| \end{equation} |
| where $W = R_c$ is the world-to-camera rotation and $J$ is the Jacobian of the perspective projection. Pixel color is computed via $\alpha$-compositing in front-to-back depth order: |
| \begin{equation} |
| C(\bp) = \sum_{i \in \mathcal{N}} c_i \, \sigma_i \prod_{j=1}^{i-1} (1 - \sigma_j), \quad \sigma_i = \alpha_i \, g'_i(\bp) |
| \end{equation} |
| where $g'_i(\bp)$ is the 2D projected Gaussian evaluated at pixel $\bp$. |
|
|
| Depth is rendered analogously: |
| \begin{equation} |
| D(\bp) = \sum_{i \in \mathcal{N}} d_i \, \sigma_i \prod_{j=1}^{i-1} (1 - \sigma_j) |
| \end{equation} |
| where $d_i$ is the depth of Gaussian $i$ along the camera ray. |
|
|
|
|
| |
| \section{Physics-Informed Deformation Field} |
| \label{sec:deformation} |
| |
|
|
| \subsection{Core Formulation} |
|
|
| The fundamental limitation of static 3DGS in endoscopic settings is that tissue deforms over time. We introduce a \textbf{temporal deformation field} that transforms canonical Gaussians to their time-specific configurations: |
| \begin{equation} |
| \boxed{G_i(t) = G_i^{(0)} + \bDelta_\theta(\bmu_i, t)} |
| \label{eq:deformation} |
| \end{equation} |
| where $G_i^{(0)}$ is the canonical (rest-state) Gaussian, and $\bDelta_\theta$ is a learned deformation field parameterized by $\theta$. |
|
|
| Concretely, the deformation produces per-Gaussian displacements: |
| \begin{equation} |
| \bDelta_\theta(\bmu_i, t) = \begin{pmatrix} |
| \Delta\bmu_i(t) \in \R^3 \\ |
| \Delta\bq_i(t) \in \R^4 \\ |
| \Delta\bs_i(t) \in \R^3 \\ |
| \Delta\alpha_i(t) \in \R |
| \end{pmatrix} |
| \end{equation} |
|
|
| The deformed parameters at time $t$ are: |
| \begin{align} |
| \bmu_i(t) &= \bmu_i^{(0)} + \Delta\bmu_i(t) \label{eq:deform_mean} \\ |
| \bq_i(t) &= \text{normalize}(\bq_i^{(0)} + \Delta\bq_i(t)) \label{eq:deform_quat} \\ |
| \bs_i(t) &= \bs_i^{(0)} + \Delta\bs_i(t) \label{eq:deform_scale} \\ |
| \alpha_i(t) &= \alpha_i^{(0)} + \Delta\alpha_i(t) \label{eq:deform_opacity} |
| \end{align} |
|
|
| \subsection{HexPlane Spatio-Temporal Encoding} |
|
|
| We encode the 4D input $(\bmu_i, t) \in \R^4$ via a HexPlane factorization \cite{hexplane} that decomposes the 4D space into 6 learnable 2D feature planes: |
| \begin{equation} |
| \calP = \{(d_1, d_2) : d_1, d_2 \in \{x, y, z, t\}, \; d_1 < d_2 \} |
| \end{equation} |
| yielding planes: $F^{(xy)}, F^{(xz)}, F^{(yz)}, F^{(xt)}, F^{(yt)}, F^{(zt)}$. |
|
|
| Each plane $F_l^{(d_1, d_2)} \in \R^{C \times R_{d_1} \times R_{d_2}}$ stores $C$-dimensional features at resolution $R_{d_1} \times R_{d_2}$, with $l$ indexing the resolution level. |
|
|
| The encoded feature for Gaussian $i$ at time $t$ is: |
| \begin{equation} |
| E(\bmu_i, t) = \bigoplus_{l=1}^{L} \bigoplus_{(d_1, d_2) \in \calP} \text{BilinearSample}\!\left(F_l^{(d_1, d_2)}, [\bmu_i^{d_1}, \bmu_i^{d_2}]\right) |
| \label{eq:hexplane} |
| \end{equation} |
| where $\bigoplus$ denotes concatenation, $\bmu_i^{d}$ is the $d$-th coordinate of the normalized position, and $L$ is the number of resolution levels. |
|
|
| \paragraph{Memory complexity.} The HexPlane factorization reduces memory from $O(R^4)$ (dense 4D grid) to $O(6 L R^2)$, enabling real-time deformation of $>$100K Gaussians. With $R=64$ spatial and $R_t=75$ temporal resolution at $L=2$ levels: |
| \begin{equation} |
| \text{Memory} = 2 \times (3 \times 64^2 + 3 \times 64 \times 75) \times C \times 4 \text{ bytes} |
| \end{equation} |
| which is approximately 12 MB for $C=32$, vastly smaller than the $64^3 \times 75 \approx 20$M entries of a dense grid. |
|
|
| \subsection{Deformation Decoder} |
|
|
| The concatenated feature $E(\bmu_i, t) \in \R^{6LC}$ is decoded by an MLP: |
| \begin{align} |
| \mathbf{h}_0 &= E(\bmu_i, t) \\ |
| \mathbf{h}_{k} &= \text{ReLU}(W_k \mathbf{h}_{k-1} + \mathbf{b}_k), \quad k = 1, \ldots, K \\ |
| \bDelta_\theta(\bmu_i, t) &= \begin{pmatrix} |
| W_\mu \mathbf{h}_K \\ |
| W_q \mathbf{h}_K \\ |
| W_s \mathbf{h}_K \\ |
| W_\alpha \mathbf{h}_K |
| \end{pmatrix} |
| \end{align} |
|
|
| \paragraph{Zero initialization.} All output heads $(W_\mu, W_q, W_s, W_\alpha)$ are initialized with zeros: |
| \begin{equation} |
| W_\mu = W_q = W_s = W_\alpha = \mathbf{0}, \quad \mathbf{b}_\mu = \mathbf{b}_q = \mathbf{b}_s = \mathbf{b}_\alpha = \mathbf{0} |
| \end{equation} |
| This ensures $\bDelta_\theta(\cdot, \cdot) = \mathbf{0}$ at initialization, making the deformation an identity map. The model starts from the canonical Gaussians and \textit{gradually} learns displacements, preventing early-training instability. |
|
|
|
|
| |
| \section{Physics-Informed Priors} |
| \label{sec:physics} |
| |
|
|
| Soft tissue deformation in endoscopic surgery obeys biomechanical constraints that inform our regularization design. |
|
|
| \subsection{Tissue Biomechanics Motivation} |
|
|
| Soft tissue (e.g., liver, colon, uterus) is approximately: |
| \begin{enumerate} |
| \item \textbf{Locally smooth}: Tissue displacement fields are continuous and slowly varying in space ($\nabla_x \Delta\bmu$ small). |
| \item \textbf{Temporally coherent}: Tissue velocity changes smoothly over time ($\partial_t \Delta\bmu$ is Lipschitz). |
| \item \textbf{Volume-preserving}: Biological tissue is nearly incompressible ($\det(\nabla \Delta\bmu + I) \approx 1$). |
| \item \textbf{Bounded strain}: Tissue can only stretch/compress within physiological limits. |
| \end{enumerate} |
|
|
| These physical constraints motivate the following regularization terms. |
|
|
| \subsection{Temporal Smoothness Prior} |
|
|
| We penalize rapid changes in deformation between adjacent timesteps: |
| \begin{equation} |
| \calL_{\text{smooth}} = \frac{1}{N} \sum_{i=1}^{N} \left\| \bDelta_\theta(\bmu_i, t) - \bDelta_\theta(\bmu_i, t + \delta) \right\|_2^2 |
| \label{eq:smooth} |
| \end{equation} |
| where $\delta$ is a small time step. This enforces temporal coherence of the deformation field, preventing the model from producing discontinuous tissue jumps between frames. |
|
|
| \paragraph{Physical interpretation.} Eq.~\eqref{eq:smooth} approximates a penalty on tissue acceleration: |
| \begin{equation} |
| \calL_{\text{smooth}} \approx \delta^2 \cdot \frac{1}{N} \sum_{i=1}^{N} \left\| \frac{\partial \bDelta_\theta}{\partial t}(\bmu_i, t) \right\|_2^2 |
| \end{equation} |
| This is analogous to Tikhonov regularization on the velocity field, which is standard in biomechanical tissue simulation. |
|
|
| \subsection{Total Variation on HexPlane Features} |
|
|
| We regularize the learned feature planes to be spatially smooth: |
| \begin{equation} |
| \calL_{\text{TV}} = \frac{1}{6L} \sum_{l=1}^{L} \sum_{(d_1,d_2) \in \calP} \left( \left\| \nabla_{h} F_l^{(d_1,d_2)} \right\|_1 + \left\| \nabla_{v} F_l^{(d_1,d_2)} \right\|_1 \right) |
| \label{eq:tv} |
| \end{equation} |
| where $\nabla_h, \nabla_v$ are horizontal and vertical finite difference operators on the feature planes. |
|
|
| \paragraph{Physical interpretation.} TV on the \textit{temporal} planes (XT, YT, ZT) enforces that nearby Gaussians undergo similar deformations over time, encoding the physical constraint that tissue is a continuum (connected medium), not a collection of independent particles. |
|
|
|
|
| |
| \section{Loss Functions} |
| \label{sec:loss} |
| |
|
|
| \subsection{Complete Objective} |
|
|
| The full training objective combines appearance reconstruction, geometric accuracy, and physics-informed regularization: |
| \begin{equation} |
| \boxed{ |
| \calL = \underbrace{(1-\lambda_1)\calL_1 + \lambda_1 \calL_{\text{D-SSIM}}}_{\text{appearance}} + \underbrace{\lambda_2 \calL_{\text{depth}}}_{\text{geometry}} + \underbrace{\lambda_3 \calL_{\text{smooth}} + \lambda_4 \calL_{\text{TV}}}_{\text{regularization}} |
| } |
| \label{eq:total_loss} |
| \end{equation} |
|
|
| with hyperparameters $\lambda_1 = 0.2$, $\lambda_2 = 0.1$, $\lambda_3 = 0.01$, $\lambda_4 = 0.001$ (from EndoGaussian~\cite{endogaussian}). |
|
|
| \subsection{Appearance Loss} |
|
|
| \paragraph{L1 Photometric Loss.} |
| \begin{equation} |
| \calL_1 = \frac{1}{|\calV_t|} \sum_{\bp \in \calV_t} \left| \hat{I}(\bp) - I^*(\bp) \right| |
| \end{equation} |
| where $\calV_t = \{\bp : M_t(\bp) = 1\}$ is the set of valid pixels (tissue region, excluding tools), $\hat{I}$ is the rendered image, and $I^*$ is the ground truth. |
|
|
| \paragraph{D-SSIM Loss.} |
| \begin{equation} |
| \calL_{\text{D-SSIM}} = \frac{1 - \text{SSIM}(\hat{I} \odot M_t, \; I^* \odot M_t)}{2} |
| \end{equation} |
| D-SSIM captures structural and perceptual differences that L1 misses, particularly for specular highlights and fine tissue texture. |
|
|
| \subsection{Depth Loss (Scale-Invariant)} |
|
|
| Following Eigen et al.~\cite{eigen}, we use a scale-invariant depth loss that handles the unknown global scale/shift of endoscopic depth: |
| \begin{equation} |
| \calL_{\text{depth}} = \frac{1}{|\calV|} \sum_{\bp \in \calV} d_\bp^2 - \frac{0.5}{|\calV|^2} \left( \sum_{\bp \in \calV} d_\bp \right)^2 |
| \label{eq:depth} |
| \end{equation} |
| where $d_\bp = \log \hat{D}(\bp) - \log D^*(\bp)$ is the log-space depth difference at pixel $\bp$, and $\calV$ is the set of pixels with valid ground truth depth. |
|
|
| \paragraph{Motivation.} Endoscopic depth from structured-light systems (C3VD, SCARED) has a known scale, but monocular depth estimates (Depth-Anything) are only accurate up to scale and shift. The scale-invariant formulation is robust to both cases. |
|
|
| \subsection{Tool Occlusion Handling} |
| \label{sec:tools} |
|
|
| Surgical instruments are rigid, fast-moving objects that occupy significant portions of the field of view. They must be explicitly excluded from both reconstruction and loss computation: |
|
|
| \paragraph{Initialization exclusion.} |
| During Holistic Gaussian Initialization (Sec.~\ref{sec:hgi}), tool pixels are masked: |
| \begin{equation} |
| P = \bigcup_{t} K^{-1} T_t D_t (I_t \odot M_t) |
| \end{equation} |
| where $M_t$ is the binary tissue mask ($M_t(\bp) = 1$ for tissue, $0$ for tool). |
|
|
| \paragraph{Loss masking.} |
| All loss terms are computed only on tissue pixels: |
| \begin{equation} |
| \calL_1^{\text{masked}} = \frac{1}{|\calV_t|} \sum_{\bp \in \calV_t} \left| \hat{I}(\bp) - I^*(\bp) \right| |
| \end{equation} |
|
|
| \paragraph{Densification exclusion.} |
| Gradient signals from tool boundaries are excluded from the adaptive density control, preventing the creation of Gaussians that model instrument surfaces. |
|
|
| |
| \section{Holistic Gaussian Initialization (HGI)} |
| \label{sec:hgi} |
| |
|
|
| Standard 3DGS initializes from COLMAP sparse points, which are extremely sparse on texture-less endoscopic tissue. HGI provides dense initialization: |
|
|
| \begin{equation} |
| \boxed{ |
| P = \text{Subsample}_{0.1\%}\!\left( \bigcup_{t=1}^{T} \Pi_t^{-1}(D_t, M_t) \right) |
| } |
| \end{equation} |
|
|
| where the backprojection operator for frame $t$ is: |
| \begin{equation} |
| \Pi_t^{-1}(D_t, M_t) = \left\{ T_t \cdot K^{-1} \begin{pmatrix} u \\ v \\ 1 \end{pmatrix} D_t(u,v) \;\middle|\; M_t(u,v) = 1, \; D_t(u,v) > 0 \right\} |
| \end{equation} |
|
|
| The union across all $T$ frames ensures coverage of regions visible only from certain viewpoints, while the 0.1\% subsampling controls memory. |
|
|
| \paragraph{Scale initialization.} Initial Gaussian scales are set from the $k$-nearest-neighbor distances in the point cloud: |
| \begin{equation} |
| s_i^{(0)} = \log\!\left( \frac{1}{k} \sum_{j \in \text{kNN}(i)} \|\bmu_i - \bmu_j\| \right), \quad k=3 |
| \end{equation} |
|
|
|
|
| |
| \section{Training Schedule} |
| \label{sec:schedule} |
| |
|
|
| Training proceeds in two phases: |
|
|
| \paragraph{Phase 1: Static Warmup (iterations 0--1000).} |
| Only canonical Gaussians $G_i^{(0)}$ are optimized. The deformation network exists but outputs $\bDelta_\theta = \mathbf{0}$ (guaranteed by zero initialization). This phase lets the static Gaussians converge to a reasonable canonical frame. |
|
|
| \paragraph{Phase 2: Joint Optimization (iterations 1000--3000).} |
| Both canonical Gaussians and the deformation field $\theta$ are optimized jointly. Adaptive density control (split/clone/prune via absgrad statistics) runs every 100 iterations during iterations 500--2500. |
|
|
| \paragraph{Learning rates.} |
| \begin{equation} |
| \eta(t) = \eta_0 \cdot \gamma^{t/T}, \quad \gamma = 0.01^{1/T} |
| \end{equation} |
| where $\eta_0$ varies per parameter group: |
|
|
| \begin{table}[h] |
| \centering |
| \begin{tabular}{lc} |
| \toprule |
| Parameter & $\eta_0$ \\ |
| \midrule |
| Means $\bmu$ & $1.6 \times 10^{-4}$ \\ |
| Scales $\bs$ & $5 \times 10^{-3}$ \\ |
| Quaternions $\bq$ & $1 \times 10^{-3}$ \\ |
| Opacities $\alpha$ & $5 \times 10^{-2}$ \\ |
| SH coefficients & $2.5 \times 10^{-3}$ \\ |
| Deformation $\theta$ & $1.6 \times 10^{-3}$ \\ |
| \bottomrule |
| \end{tabular} |
| \end{table} |
|
|
| |
| \section{Adaptive Density Control} |
| \label{sec:densify} |
| |
|
|
| We use \textbf{absgrad}-based densification (absolute gradient values rather than gradient norms), which provides more precise split/clone decisions: |
|
|
| \paragraph{Gradient tracking.} |
| For each Gaussian $i$, we accumulate: |
| \begin{equation} |
| \bar{g}_i = \frac{1}{|\{t : i \text{ visible at } t\}|} \sum_{t : i \text{ visible}} \max_d \left| \frac{\partial \calL}{\partial \bmu'^{(d)}_i} \right| |
| \end{equation} |
| where $\bmu'_i$ is the 2D projected mean and $d$ indexes spatial dimensions. |
|
|
| \paragraph{Split rule.} If $\bar{g}_i > \tau_g$ and $\max_d e^{s_{i,d}} > \tau_s$: |
| \begin{equation} |
| G_i \rightarrow G_i^{(a)}, G_i^{(b)} \quad \text{with} \quad \bmu^{(a,b)} = \bmu_i \pm \epsilon, \; \bs^{(a,b)} = \bs_i - \log(1.6) |
| \end{equation} |
|
|
| \paragraph{Clone rule.} If $\bar{g}_i > \tau_g$ and $\max_d e^{s_{i,d}} \leq \tau_s$: |
| \begin{equation} |
| G_i \rightarrow G_i, G_i' \quad \text{with} \quad G_i' = G_i \; \text{(exact copy)} |
| \end{equation} |
|
|
| \paragraph{Prune rule.} |
| \begin{equation} |
| \text{Remove } G_i \text{ if } \sigma(\alpha_i) < \tau_\alpha \text{ or } \max_d e^{s_{i,d}} > \tau_{\text{max}} |
| \end{equation} |
|
|
| Thresholds: $\tau_g = 0.0002$, $\tau_s = 0.01$, $\tau_\alpha = 0.005$, $\tau_{\text{max}} = 0.1$. |
|
|
|
|
| |
| \section{Expected Performance} |
| \label{sec:expected} |
| |
|
|
| Based on published results from EndoGaussian~\cite{endogaussian} and Endo-4DGS~\cite{endo4dgs}: |
|
|
| \begin{table}[h] |
| \centering |
| \begin{tabular}{lccc} |
| \toprule |
| Method & PSNR $\uparrow$ & SSIM $\uparrow$ & FPS \\ |
| \midrule |
| Static 3DGS & 30--32 & 0.90 & 200+ \\ |
| EndoNeRF & 35.8 & 0.96 & 0.2 \\ |
| Endo-4DGS & 36.6 & 0.96 & 100 \\ |
| \textbf{EndoGaussian (target)} & \textbf{37.9} & \textbf{0.97} & \textbf{195} \\ |
| \bottomrule |
| \end{tabular} |
| \caption{Expected performance on EndoNeRF cutting/pulling sequences.} |
| \end{table} |
|
|
|
|
| |
| \section*{References} |
| |
|
|
| \begin{thebibliography}{9} |
|
|
| \bibitem{3dgs} |
| Kerbl, B., Kopanas, G., Leimk\"uhler, T., \& Drettakis, G. (2023). |
| 3D Gaussian Splatting for Real-Time Radiance Field Rendering. \textit{SIGGRAPH}. |
|
|
| \bibitem{endogaussian} |
| Liu, Y., et al. (2024). |
| EndoGaussian: Real-time Gaussian Splatting for Dynamic Endoscopic Scene Reconstruction. \textit{arXiv:2401.12561}. |
|
|
| \bibitem{endo4dgs} |
| Huang, Y., et al. (2024). |
| Endo-4DGS: Endoscopic Monocular Scene Reconstruction with 4D Gaussian Splatting. \textit{MICCAI}. |
|
|
| \bibitem{hexplane} |
| Cao, A., \& Johnson, J. (2023). |
| HexPlane: A Fast Representation for Dynamic Scenes. \textit{CVPR}. |
|
|
| \bibitem{eigen} |
| Eigen, D., Puhrsch, C., \& Fergus, R. (2014). |
| Depth Map Prediction from a Single Image using a Multi-Scale Deep Network. \textit{NeurIPS}. |
|
|
| \bibitem{endonerf} |
| Wang, Y., et al. (2022). |
| Neural Rendering for Stereo 3D Reconstruction of Deformable Tissues in Robotic Surgery. \textit{MICCAI}. |
|
|
| \bibitem{gsplat} |
| Ye, V., et al. (2024). |
| gsplat: An Open-Source Library for Gaussian Splatting. \textit{arXiv:2409.06765}. |
|
|
| \bibitem{depthanything} |
| Yang, L., et al. (2024). |
| Depth Anything: Unleashing the Power of Large-Scale Unlabeled Data. \textit{CVPR}. |
|
|
| \end{thebibliography} |
|
|
| \end{document} |
|
|
