docs/formalization.tex

% EndoGaussian-4D: Mathematical Formalization
% Physics-Informed Deformation Field for Real-Time 4D Gaussian Splatting
% in Endoscopic Surgery
%
% Compile: pdflatex formalization.tex
%
\documentclass[10pt,twocolumn]{article}

\usepackage[utf8]{inputenc}
\usepackage{amsmath,amssymb,amsfonts}
\usepackage{algorithm}
\usepackage{algorithmic}
\usepackage{booktabs}
\usepackage{hyperref}
\usepackage{geometry}
\geometry{margin=1in}

\newcommand{\R}{\mathbb{R}}
\newcommand{\calL}{\mathcal{L}}
\newcommand{\calP}{\mathcal{P}}
\newcommand{\calG}{\mathcal{G}}
\newcommand{\calF}{\mathcal{F}}
\newcommand{\calV}{\mathcal{V}}
\newcommand{\bmu}{\boldsymbol{\mu}}
\newcommand{\bq}{\mathbf{q}}
\newcommand{\bs}{\mathbf{s}}
\newcommand{\bx}{\mathbf{x}}
\newcommand{\bp}{\mathbf{p}}
\newcommand{\bDelta}{\boldsymbol{\Delta}}

\title{\textbf{EndoGaussian-4D: Mathematical Formalization} \\
\large Physics-Informed Deformation Field for Real-Time \\
4D Gaussian Splatting in Endoscopic Surgery}
\author{EndoGaussian-4D Research Team}
\date{Week 1 Sprint --- Phase 3 Deliverable}

\begin{document}
\maketitle

% ===========================================================================
\section{Preliminary: 3D Gaussian Splatting}
% ===========================================================================

A scene is represented as a set of $N$ anisotropic 3D Gaussians:
\begin{equation}
    \calG = \{ G_i \}_{i=1}^{N}, \quad G_i = (\bmu_i, \bq_i, \bs_i, \alpha_i, \mathbf{c}_i)
\end{equation}
where each Gaussian $G_i$ is parameterized by:
\begin{itemize}
    \item $\bmu_i \in \R^3$: Mean (center position)
    \item $\bq_i \in \R^4$: Unit quaternion encoding rotation
    \item $\bs_i \in \R^3$: Log-scale vector (anisotropic scaling)
    \item $\alpha_i \in \R$: Logit-opacity
    \item $\mathbf{c}_i \in \R^{K \times 3}$: Spherical harmonics coefficients ($K = (\ell+1)^2$ for degree $\ell$)
\end{itemize}

The covariance matrix of each Gaussian in world space is:
\begin{equation}
    \Sigma_i = R(\bq_i) \, S(\bs_i) \, S(\bs_i)^\top \, R(\bq_i)^\top
\end{equation}
where $R(\bq_i) \in SO(3)$ is the rotation matrix from quaternion $\bq_i$, and $S(\bs_i) = \text{diag}(e^{s_{i,1}}, e^{s_{i,2}}, e^{s_{i,3}})$ is the diagonal scaling matrix.

The influence of Gaussian $G_i$ at a 3D point $\bx$ is:
\begin{equation}
    g_i(\bx) = \exp\!\left( -\tfrac{1}{2} (\bx - \bmu_i)^\top \Sigma_i^{-1} (\bx - \bmu_i) \right)
\end{equation}

\paragraph{Differentiable Rasterization.}
For rendering, Gaussians are projected onto the image plane via the EWA splatting approximation. Given camera extrinsic $[R_c | \mathbf{t}_c]$ and intrinsic $K$, the 2D projected covariance is:
\begin{equation}
    \Sigma'_i = J \, W \, \Sigma_i \, W^\top \, J^\top
\end{equation}
where $W = R_c$ is the world-to-camera rotation and $J$ is the Jacobian of the perspective projection. Pixel color is computed via $\alpha$-compositing in front-to-back depth order:
\begin{equation}
    C(\bp) = \sum_{i \in \mathcal{N}} c_i \, \sigma_i \prod_{j=1}^{i-1} (1 - \sigma_j), \quad \sigma_i = \alpha_i \, g'_i(\bp)
\end{equation}
where $g'_i(\bp)$ is the 2D projected Gaussian evaluated at pixel $\bp$.

Depth is rendered analogously:
\begin{equation}
    D(\bp) = \sum_{i \in \mathcal{N}} d_i \, \sigma_i \prod_{j=1}^{i-1} (1 - \sigma_j)
\end{equation}
where $d_i$ is the depth of Gaussian $i$ along the camera ray.


% ===========================================================================
\section{Physics-Informed Deformation Field}
\label{sec:deformation}
% ===========================================================================

\subsection{Core Formulation}

The fundamental limitation of static 3DGS in endoscopic settings is that tissue deforms over time. We introduce a \textbf{temporal deformation field} that transforms canonical Gaussians to their time-specific configurations:
\begin{equation}
    \boxed{G_i(t) = G_i^{(0)} + \bDelta_\theta(\bmu_i, t)}
    \label{eq:deformation}
\end{equation}
where $G_i^{(0)}$ is the canonical (rest-state) Gaussian, and $\bDelta_\theta$ is a learned deformation field parameterized by $\theta$.

Concretely, the deformation produces per-Gaussian displacements:
\begin{equation}
    \bDelta_\theta(\bmu_i, t) = \begin{pmatrix}
        \Delta\bmu_i(t) \in \R^3 \\
        \Delta\bq_i(t) \in \R^4 \\
        \Delta\bs_i(t) \in \R^3 \\
        \Delta\alpha_i(t) \in \R
    \end{pmatrix}
\end{equation}

The deformed parameters at time $t$ are:
\begin{align}
    \bmu_i(t) &= \bmu_i^{(0)} + \Delta\bmu_i(t) \label{eq:deform_mean} \\
    \bq_i(t) &= \text{normalize}(\bq_i^{(0)} + \Delta\bq_i(t)) \label{eq:deform_quat} \\
    \bs_i(t) &= \bs_i^{(0)} + \Delta\bs_i(t) \label{eq:deform_scale} \\
    \alpha_i(t) &= \alpha_i^{(0)} + \Delta\alpha_i(t) \label{eq:deform_opacity}
\end{align}

\subsection{HexPlane Spatio-Temporal Encoding}

We encode the 4D input $(\bmu_i, t) \in \R^4$ via a HexPlane factorization \cite{hexplane} that decomposes the 4D space into 6 learnable 2D feature planes:
\begin{equation}
    \calP = \{(d_1, d_2) : d_1, d_2 \in \{x, y, z, t\}, \; d_1 < d_2 \}
\end{equation}
yielding planes: $F^{(xy)}, F^{(xz)}, F^{(yz)}, F^{(xt)}, F^{(yt)}, F^{(zt)}$.

Each plane $F_l^{(d_1, d_2)} \in \R^{C \times R_{d_1} \times R_{d_2}}$ stores $C$-dimensional features at resolution $R_{d_1} \times R_{d_2}$, with $l$ indexing the resolution level.

The encoded feature for Gaussian $i$ at time $t$ is:
\begin{equation}
    E(\bmu_i, t) = \bigoplus_{l=1}^{L} \bigoplus_{(d_1, d_2) \in \calP} \text{BilinearSample}\!\left(F_l^{(d_1, d_2)}, [\bmu_i^{d_1}, \bmu_i^{d_2}]\right)
    \label{eq:hexplane}
\end{equation}
where $\bigoplus$ denotes concatenation, $\bmu_i^{d}$ is the $d$-th coordinate of the normalized position, and $L$ is the number of resolution levels.

\paragraph{Memory complexity.} The HexPlane factorization reduces memory from $O(R^4)$ (dense 4D grid) to $O(6 L R^2)$, enabling real-time deformation of $>$100K Gaussians. With $R=64$ spatial and $R_t=75$ temporal resolution at $L=2$ levels:
\begin{equation}
    \text{Memory} = 2 \times (3 \times 64^2 + 3 \times 64 \times 75) \times C \times 4 \text{ bytes}
\end{equation}
which is approximately 12 MB for $C=32$, vastly smaller than the $64^3 \times 75 \approx 20$M entries of a dense grid.

\subsection{Deformation Decoder}

The concatenated feature $E(\bmu_i, t) \in \R^{6LC}$ is decoded by an MLP:
\begin{align}
    \mathbf{h}_0 &= E(\bmu_i, t) \\
    \mathbf{h}_{k} &= \text{ReLU}(W_k \mathbf{h}_{k-1} + \mathbf{b}_k), \quad k = 1, \ldots, K \\
    \bDelta_\theta(\bmu_i, t) &= \begin{pmatrix}
        W_\mu \mathbf{h}_K \\
        W_q \mathbf{h}_K \\
        W_s \mathbf{h}_K \\
        W_\alpha \mathbf{h}_K
    \end{pmatrix}
\end{align}

\paragraph{Zero initialization.} All output heads $(W_\mu, W_q, W_s, W_\alpha)$ are initialized with zeros:
\begin{equation}
    W_\mu = W_q = W_s = W_\alpha = \mathbf{0}, \quad \mathbf{b}_\mu = \mathbf{b}_q = \mathbf{b}_s = \mathbf{b}_\alpha = \mathbf{0}
\end{equation}
This ensures $\bDelta_\theta(\cdot, \cdot) = \mathbf{0}$ at initialization, making the deformation an identity map. The model starts from the canonical Gaussians and \textit{gradually} learns displacements, preventing early-training instability.


% ===========================================================================
\section{Physics-Informed Priors}
\label{sec:physics}
% ===========================================================================

Soft tissue deformation in endoscopic surgery obeys biomechanical constraints that inform our regularization design.

\subsection{Tissue Biomechanics Motivation}

Soft tissue (e.g., liver, colon, uterus) is approximately:
\begin{enumerate}
    \item \textbf{Locally smooth}: Tissue displacement fields are continuous and slowly varying in space ($\nabla_x \Delta\bmu$ small).
    \item \textbf{Temporally coherent}: Tissue velocity changes smoothly over time ($\partial_t \Delta\bmu$ is Lipschitz).
    \item \textbf{Volume-preserving}: Biological tissue is nearly incompressible ($\det(\nabla \Delta\bmu + I) \approx 1$).
    \item \textbf{Bounded strain}: Tissue can only stretch/compress within physiological limits.
\end{enumerate}

These physical constraints motivate the following regularization terms.

\subsection{Temporal Smoothness Prior}

We penalize rapid changes in deformation between adjacent timesteps:
\begin{equation}
    \calL_{\text{smooth}} = \frac{1}{N} \sum_{i=1}^{N} \left\| \bDelta_\theta(\bmu_i, t) - \bDelta_\theta(\bmu_i, t + \delta) \right\|_2^2
    \label{eq:smooth}
\end{equation}
where $\delta$ is a small time step. This enforces temporal coherence of the deformation field, preventing the model from producing discontinuous tissue jumps between frames.

\paragraph{Physical interpretation.} Eq.~\eqref{eq:smooth} approximates a penalty on tissue acceleration:
\begin{equation}
    \calL_{\text{smooth}} \approx \delta^2 \cdot \frac{1}{N} \sum_{i=1}^{N} \left\| \frac{\partial \bDelta_\theta}{\partial t}(\bmu_i, t) \right\|_2^2
\end{equation}
This is analogous to Tikhonov regularization on the velocity field, which is standard in biomechanical tissue simulation.

\subsection{Total Variation on HexPlane Features}

We regularize the learned feature planes to be spatially smooth:
\begin{equation}
    \calL_{\text{TV}} = \frac{1}{6L} \sum_{l=1}^{L} \sum_{(d_1,d_2) \in \calP} \left( \left\| \nabla_{h} F_l^{(d_1,d_2)} \right\|_1 + \left\| \nabla_{v} F_l^{(d_1,d_2)} \right\|_1 \right)
    \label{eq:tv}
\end{equation}
where $\nabla_h, \nabla_v$ are horizontal and vertical finite difference operators on the feature planes.

\paragraph{Physical interpretation.} TV on the \textit{temporal} planes (XT, YT, ZT) enforces that nearby Gaussians undergo similar deformations over time, encoding the physical constraint that tissue is a continuum (connected medium), not a collection of independent particles.


% ===========================================================================
\section{Loss Functions}
\label{sec:loss}
% ===========================================================================

\subsection{Complete Objective}

The full training objective combines appearance reconstruction, geometric accuracy, and physics-informed regularization:
\begin{equation}
    \boxed{
    \calL = \underbrace{(1-\lambda_1)\calL_1 + \lambda_1 \calL_{\text{D-SSIM}}}_{\text{appearance}} + \underbrace{\lambda_2 \calL_{\text{depth}}}_{\text{geometry}} + \underbrace{\lambda_3 \calL_{\text{smooth}} + \lambda_4 \calL_{\text{TV}}}_{\text{regularization}}
    }
    \label{eq:total_loss}
\end{equation}

with hyperparameters $\lambda_1 = 0.2$, $\lambda_2 = 0.1$, $\lambda_3 = 0.01$, $\lambda_4 = 0.001$ (from EndoGaussian~\cite{endogaussian}).

\subsection{Appearance Loss}

\paragraph{L1 Photometric Loss.}
\begin{equation}
    \calL_1 = \frac{1}{|\calV_t|} \sum_{\bp \in \calV_t} \left| \hat{I}(\bp) - I^*(\bp) \right|
\end{equation}
where $\calV_t = \{\bp : M_t(\bp) = 1\}$ is the set of valid pixels (tissue region, excluding tools), $\hat{I}$ is the rendered image, and $I^*$ is the ground truth.

\paragraph{D-SSIM Loss.}
\begin{equation}
    \calL_{\text{D-SSIM}} = \frac{1 - \text{SSIM}(\hat{I} \odot M_t, \; I^* \odot M_t)}{2}
\end{equation}
D-SSIM captures structural and perceptual differences that L1 misses, particularly for specular highlights and fine tissue texture.

\subsection{Depth Loss (Scale-Invariant)}

Following Eigen et al.~\cite{eigen}, we use a scale-invariant depth loss that handles the unknown global scale/shift of endoscopic depth:
\begin{equation}
    \calL_{\text{depth}} = \frac{1}{|\calV|} \sum_{\bp \in \calV} d_\bp^2 - \frac{0.5}{|\calV|^2} \left( \sum_{\bp \in \calV} d_\bp \right)^2
    \label{eq:depth}
\end{equation}
where $d_\bp = \log \hat{D}(\bp) - \log D^*(\bp)$ is the log-space depth difference at pixel $\bp$, and $\calV$ is the set of pixels with valid ground truth depth.

\paragraph{Motivation.} Endoscopic depth from structured-light systems (C3VD, SCARED) has a known scale, but monocular depth estimates (Depth-Anything) are only accurate up to scale and shift. The scale-invariant formulation is robust to both cases.

\subsection{Tool Occlusion Handling}
\label{sec:tools}

Surgical instruments are rigid, fast-moving objects that occupy significant portions of the field of view. They must be explicitly excluded from both reconstruction and loss computation:

\paragraph{Initialization exclusion.}
During Holistic Gaussian Initialization (Sec.~\ref{sec:hgi}), tool pixels are masked:
\begin{equation}
    P = \bigcup_{t} K^{-1} T_t D_t (I_t \odot M_t)
\end{equation}
where $M_t$ is the binary tissue mask ($M_t(\bp) = 1$ for tissue, $0$ for tool).

\paragraph{Loss masking.}
All loss terms are computed only on tissue pixels:
\begin{equation}
    \calL_1^{\text{masked}} = \frac{1}{|\calV_t|} \sum_{\bp \in \calV_t} \left| \hat{I}(\bp) - I^*(\bp) \right|
\end{equation}

\paragraph{Densification exclusion.}
Gradient signals from tool boundaries are excluded from the adaptive density control, preventing the creation of Gaussians that model instrument surfaces.

% ===========================================================================
\section{Holistic Gaussian Initialization (HGI)}
\label{sec:hgi}
% ===========================================================================

Standard 3DGS initializes from COLMAP sparse points, which are extremely sparse on texture-less endoscopic tissue. HGI provides dense initialization:

\begin{equation}
    \boxed{
    P = \text{Subsample}_{0.1\%}\!\left( \bigcup_{t=1}^{T} \Pi_t^{-1}(D_t, M_t) \right)
    }
\end{equation}

where the backprojection operator for frame $t$ is:
\begin{equation}
    \Pi_t^{-1}(D_t, M_t) = \left\{ T_t \cdot K^{-1} \begin{pmatrix} u \\ v \\ 1 \end{pmatrix} D_t(u,v) \;\middle|\; M_t(u,v) = 1, \; D_t(u,v) > 0 \right\}
\end{equation}

The union across all $T$ frames ensures coverage of regions visible only from certain viewpoints, while the 0.1\% subsampling controls memory.

\paragraph{Scale initialization.} Initial Gaussian scales are set from the $k$-nearest-neighbor distances in the point cloud:
\begin{equation}
    s_i^{(0)} = \log\!\left( \frac{1}{k} \sum_{j \in \text{kNN}(i)} \|\bmu_i - \bmu_j\| \right), \quad k=3
\end{equation}


% ===========================================================================
\section{Training Schedule}
\label{sec:schedule}
% ===========================================================================

Training proceeds in two phases:

\paragraph{Phase 1: Static Warmup (iterations 0--1000).}
Only canonical Gaussians $G_i^{(0)}$ are optimized. The deformation network exists but outputs $\bDelta_\theta = \mathbf{0}$ (guaranteed by zero initialization). This phase lets the static Gaussians converge to a reasonable canonical frame.

\paragraph{Phase 2: Joint Optimization (iterations 1000--3000).}
Both canonical Gaussians and the deformation field $\theta$ are optimized jointly. Adaptive density control (split/clone/prune via absgrad statistics) runs every 100 iterations during iterations 500--2500.

\paragraph{Learning rates.}
\begin{equation}
    \eta(t) = \eta_0 \cdot \gamma^{t/T}, \quad \gamma = 0.01^{1/T}
\end{equation}
where $\eta_0$ varies per parameter group:

\begin{table}[h]
\centering
\begin{tabular}{lc}
\toprule
Parameter & $\eta_0$ \\
\midrule
Means $\bmu$ & $1.6 \times 10^{-4}$ \\
Scales $\bs$ & $5 \times 10^{-3}$ \\
Quaternions $\bq$ & $1 \times 10^{-3}$ \\
Opacities $\alpha$ & $5 \times 10^{-2}$ \\
SH coefficients & $2.5 \times 10^{-3}$ \\
Deformation $\theta$ & $1.6 \times 10^{-3}$ \\
\bottomrule
\end{tabular}
\end{table}

% ===========================================================================
\section{Adaptive Density Control}
\label{sec:densify}
% ===========================================================================

We use \textbf{absgrad}-based densification (absolute gradient values rather than gradient norms), which provides more precise split/clone decisions:

\paragraph{Gradient tracking.}
For each Gaussian $i$, we accumulate:
\begin{equation}
    \bar{g}_i = \frac{1}{|\{t : i \text{ visible at } t\}|} \sum_{t : i \text{ visible}} \max_d \left| \frac{\partial \calL}{\partial \bmu'^{(d)}_i} \right|
\end{equation}
where $\bmu'_i$ is the 2D projected mean and $d$ indexes spatial dimensions.

\paragraph{Split rule.} If $\bar{g}_i > \tau_g$ and $\max_d e^{s_{i,d}} > \tau_s$:
\begin{equation}
    G_i \rightarrow G_i^{(a)}, G_i^{(b)} \quad \text{with} \quad \bmu^{(a,b)} = \bmu_i \pm \epsilon, \; \bs^{(a,b)} = \bs_i - \log(1.6)
\end{equation}

\paragraph{Clone rule.} If $\bar{g}_i > \tau_g$ and $\max_d e^{s_{i,d}} \leq \tau_s$:
\begin{equation}
    G_i \rightarrow G_i, G_i' \quad \text{with} \quad G_i' = G_i \; \text{(exact copy)}
\end{equation}

\paragraph{Prune rule.}
\begin{equation}
    \text{Remove } G_i \text{ if } \sigma(\alpha_i) < \tau_\alpha \text{ or } \max_d e^{s_{i,d}} > \tau_{\text{max}}
\end{equation}

Thresholds: $\tau_g = 0.0002$, $\tau_s = 0.01$, $\tau_\alpha = 0.005$, $\tau_{\text{max}} = 0.1$.


% ===========================================================================
\section{Expected Performance}
\label{sec:expected}
% ===========================================================================

Based on published results from EndoGaussian~\cite{endogaussian} and Endo-4DGS~\cite{endo4dgs}:

\begin{table}[h]
\centering
\begin{tabular}{lccc}
\toprule
Method & PSNR $\uparrow$ & SSIM $\uparrow$ & FPS \\
\midrule
Static 3DGS & 30--32 & 0.90 & 200+ \\
EndoNeRF & 35.8 & 0.96 & 0.2 \\
Endo-4DGS & 36.6 & 0.96 & 100 \\
\textbf{EndoGaussian (target)} & \textbf{37.9} & \textbf{0.97} & \textbf{195} \\
\bottomrule
\end{tabular}
\caption{Expected performance on EndoNeRF cutting/pulling sequences.}
\end{table}


% ===========================================================================
\section*{References}
% ===========================================================================

\begin{thebibliography}{9}

\bibitem{3dgs}
Kerbl, B., Kopanas, G., Leimk\"uhler, T., \& Drettakis, G. (2023).
3D Gaussian Splatting for Real-Time Radiance Field Rendering. \textit{SIGGRAPH}.

\bibitem{endogaussian}
Liu, Y., et al. (2024).
EndoGaussian: Real-time Gaussian Splatting for Dynamic Endoscopic Scene Reconstruction. \textit{arXiv:2401.12561}.

\bibitem{endo4dgs}
Huang, Y., et al. (2024).
Endo-4DGS: Endoscopic Monocular Scene Reconstruction with 4D Gaussian Splatting. \textit{MICCAI}.

\bibitem{hexplane}
Cao, A., \& Johnson, J. (2023).
HexPlane: A Fast Representation for Dynamic Scenes. \textit{CVPR}.

\bibitem{eigen}
Eigen, D., Puhrsch, C., \& Fergus, R. (2014).
Depth Map Prediction from a Single Image using a Multi-Scale Deep Network. \textit{NeurIPS}.

\bibitem{endonerf}
Wang, Y., et al. (2022).
Neural Rendering for Stereo 3D Reconstruction of Deformable Tissues in Robotic Surgery. \textit{MICCAI}.

\bibitem{gsplat}
Ye, V., et al. (2024).
gsplat: An Open-Source Library for Gaussian Splatting. \textit{arXiv:2409.06765}.

\bibitem{depthanything}
Yang, L., et al. (2024).
Depth Anything: Unleashing the Power of Large-Scale Unlabeled Data. \textit{CVPR}.

\end{thebibliography}

\end{document}