Add Docker-based Learn2Splat demo (viser GUI)

.dockerignore

@@ -0,0 +1,32 @@
+# Build-context / image trimming for the Learn2Splat demo Space.
+# The Dockerfile needs: demo.py, optgs/, submodules/, requirements.txt,
+# pyproject.toml, LICENSE — keep those; drop everything below.
+
+# Secrets — never copy into the image.
+.env
+.env.*
+/wandb/
+
+# Git + Python build droppings.
+.git/
+.gitignore
+**/__pycache__/
+**/*.pyc
+**/*.egg-info/
+submodules/*/build/
+
+# Large runtime artefacts — fetched into the container on first run.
+/data/
+/checkpoints/
+/results/
+
+# Repo material the demo doesn't use.
+/assets/
+/docs/
+/figures/
+/tests/
+/scripts/
+/mlcloud_scripts/
+/visualization/
+/todo/
+huggingface_space/

.gitattributes

@@ -33,3 +33,9 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+submodules/fused-ssim/images/albert.jpg filter=lfs diff=lfs merge=lfs -text
+submodules/fused-ssim/images/inference_time.png filter=lfs diff=lfs merge=lfs -text
+submodules/fused-ssim/images/inference_time_4090.png filter=lfs diff=lfs merge=lfs -text
+submodules/fused-ssim/images/predicted.jpg filter=lfs diff=lfs merge=lfs -text
+submodules/fused-ssim/images/training_time.png filter=lfs diff=lfs merge=lfs -text
+submodules/fused-ssim/images/training_time_4090.png filter=lfs diff=lfs merge=lfs -text

Dockerfile

@@ -0,0 +1,76 @@
+# Learn2Splat — interactive demo for a Hugging Face Space (Docker SDK, GPU).
+#
+# Builds the optgs package + its CUDA extensions and runs demo.py's viser GUI:
+# SfM-initialize a COLMAP scene, then refine the Gaussians with the learned
+# optimizer live in the browser. Mirrors setup.sh, minus conda — the CUDA
+# toolkit ships in the base image.
+#
+# Build context = the optgs repo root (see huggingface_space/DEPLOY.md).
+# Hardware: pick a GPU in the Space settings — A10G (24 GB) recommended; the
+# GUI holds the dense and sparse checkpoints in VRAM at once.
+
+# CUDA 12.8 devel (nvcc + headers); Ubuntu 22.04 — the OS setup.sh is tested on.
+# A devel base is required: gsplat / nerfacc JIT-compile CUDA on first use, so
+# nvcc must also be present at runtime.
+FROM nvidia/cuda:12.8.0-devel-ubuntu22.04
+
+ENV DEBIAN_FRONTEND=noninteractive \
+    PYTHONUNBUFFERED=1 \
+    PIP_NO_CACHE_DIR=1 \
+    # Compile the CUDA extensions for every GPU a Space may run on
+    # (T4 7.5 · A100 8.0 · A10G 8.6 · L4/L40S 8.9 · H100 9.0). Trim this to
+    # your chosen GPU to shorten the build.
+    TORCH_CUDA_ARCH_LIST="7.5 8.0 8.6 8.9 9.0+PTX"
+
+# Build tools + extension headers (libglm-dev) and the OpenCV runtime libs
+# (libgl1, libglib2.0-0 — optgs's COLMAP loader imports cv2).
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        python3 python3-dev python3-venv \
+        git build-essential ninja-build libglm-dev \
+        libgl1 libglib2.0-0 ca-certificates \
+ && rm -rf /var/lib/apt/lists/*
+
+# HF Spaces convention: run as a non-root user (UID 1000).
+RUN useradd -m -u 1000 user
+USER user
+ENV HOME=/home/user \
+    HF_HOME=/home/user/.cache/huggingface \
+    TORCH_HOME=/home/user/.cache/torch
+WORKDIR /home/user/app
+
+# All Python work happens in a venv on PATH (no system-Python writes).
+RUN python3 -m venv /home/user/venv
+ENV PATH=/home/user/venv/bin:$PATH
+RUN pip install --upgrade pip setuptools wheel
+
+# PyTorch (CUDA 12.8) — pinned to setup.sh.
+RUN pip install torch==2.7.1 torchvision==0.22.1 torchaudio==2.7.1 \
+        --index-url https://download.pytorch.org/whl/cu128
+
+# Python requirements (copied first so this layer caches across code edits).
+COPY --chown=user:user requirements.txt .
+RUN pip install -r requirements.txt
+
+# gsplat + nerfacc — built from git against the torch installed above.
+RUN pip install --no-build-isolation \
+        git+https://github.com/nerfstudio-project/nerfacc \
+        git+https://github.com/nerfstudio-project/gsplat.git
+
+# The optgs repo.
+COPY --chown=user:user . .
+
+# CUDA-extension submodules, then optgs itself. pycolmap is the pure-Python
+# COLMAP reader (no C++ build); the other four compile CUDA kernels.
+RUN pip install submodules/pycolmap \
+ && pip install --no-build-isolation submodules/fused-ssim \
+ && pip install --no-build-isolation submodules/simple-knn \
+ && pip install --no-build-isolation submodules/pointops \
+ && pip install --no-build-isolation submodules/fused_knn_attn \
+ && pip install --no-build-isolation --no-deps -e .
+
+# viser serves the GUI here — must equal app_port in README.md.
+EXPOSE 7860
+
+# client mode: viser ships the splats to the browser's WebGL renderer, so the
+# GPU is used only for optimization. viser binds 0.0.0.0 by default.
+CMD ["python", "demo.py", "--with-gui", "client", "--gui-port", "7860"]

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Naama Pearl and Stefano Esposito
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,13 +1,24 @@
 ---
 title: Learn2Splat
-emoji: 😻
-colorFrom: indigo
-colorTo: pink
-sdk: gradio
-sdk_version: 6.14.0
-python_version: '3.13'
-app_file: app.py
+emoji: 🪴
+colorFrom: green
+colorTo: indigo
+sdk: docker
+app_port: 7860
 pinned: false
+short_description: Interactive demo of the Learn2Splat learned 3DGS optimizer
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# Learn2Splat — interactive demo
+
+A learned optimizer for 3D Gaussian Splatting. This Space SfM-initializes a
+COLMAP scene and refines the Gaussians live in your browser: pick the
+Learn2Splat optimizer (dense or sparse checkpoint) or a 3DGS Adam baseline,
+press **Start**, and watch the splats converge.
+
+Runs `demo.py --with-gui client` from the
+[Learn2Splat repository](https://github.com/autonomousvision/learn2splat);
+the splats are drawn by viser's in-browser WebGL renderer.
+
+> Requires GPU hardware. The demo holds two checkpoints in VRAM at once —
+> an A10G (24 GB) is recommended.

demo.py

@@ -0,0 +1,766 @@
+"""End-to-end OptGS demo on a COLMAP scene.
+
+Main-codebase port of ``baselines/gsplat/examples/simple_trainer_optgs.py``:
+same flow — SfM-initialize Gaussians, refine them with the learned optimizer
+via the ``OptGS`` API, evaluate on held-out views — but using only the
+``optgs`` package (no gsplat / gsplat-examples dependency):
+
+    from optgs.experimental.api import OptGS
+
+    optgs = OptGS(checkpoint="hf://org/repo/model.ckpt", device="cuda")
+    optgs.initialize_from_tensors(gaussians, batched_views)
+    refined = optgs.optimize()          # learned optimization
+
+COLMAP loading uses ``optgs.dataset.colmap``; the SfM init builds an optgs
+``Gaussians`` directly via ``points_to_gaussians``; evaluation renders with
+the optimizer's own decoder.
+
+The scene is refined three ways and compared on held-out views: the learned
+optimizer (Learn2Splat) with the *dense* and the *sparse* checkpoint, and a
+3DGS Adam baseline (gsplat hyperparameters). All run through the same
+``optimize()`` path with identical SfM init, view minibatches and step budget.
+Each uses its checkpoint's gsplat renderer; ``--rasterize-mode`` / ``--eps2d``
+pin one renderer across all runs.
+
+Usage (run from the repo root, with ``optgs`` importable):
+
+    python demo.py                    # headless: dense + sparse checkpoints + an Adam baseline
+    python demo.py --with-gui server  # interactive viser GUI (frames rendered by the decoder)
+    python demo.py --with-gui client  # interactive viser GUI (viser's WebGL splat renderer)
+
+The demo scene and the checkpoints are fetched from the Hugging Face Hub on
+first run (cached under ./data and ./checkpoints). A CUDA device is required.
+"""
+
+import warnings
+
+# Demo: silence third-party UserWarnings (xFormers/flash-attn not installed,
+# Hydra's _self_ notice, pointops' deprecated tensor constructors) for clean output.
+warnings.filterwarnings("ignore")
+
+import json
+import os
+import time
+from dataclasses import dataclass
+from typing import Dict, List, Literal, Optional, Tuple
+
+import imageio.v2 as imageio
+import numpy as np
+import torch
+import torch.nn.functional as F
+import tyro
+from rich.console import Console
+from rich.table import Table
+from torch import Tensor
+
+console = Console()
+
+from optgs.dataset.colmap.utils import Dataset, Parser
+from optgs.experimental.initializers_utils import knn, points_to_gaussians
+from optgs.model.types import Gaussians
+from optgs.scene_trainer.common.gaussian_adapter import build_covariance
+
+# Camera near/far planes — inria's znear/zfar (also the optgs colmap-dataset
+# constants). Fixed; not a user knob.
+NEAR_PLANE = 0.01
+FAR_PLANE = 100.0
+
+# Spherical-harmonics DC -> RGB (3DGS convention: rgb = 0.5 + C0 * dc). Colours
+# the splats for viser's client-side renderer.
+SH_C0 = 0.28209479177387814
+
+# The demo scene is fetched from this Hugging Face repo on first run. The repo
+# mirrors the local layout, so e.g. ``data/mip360/garden`` in the repo lands at
+# ``./data/mip360/garden``.
+DEMO_DATA_REPO = "autonomousvision/learn2splat"
+
+# Learned-optimizer checkpoints on the Hugging Face Hub. hf:// refs are fetched
+# and cached under ./checkpoints on first use (see optgs.misc.hf_ckpt).
+CHECKPOINTS = {
+    "dense": "hf://autonomousvision/learn2splat/dense/checkpoints/epoch_5-step_50000.ckpt",
+    "sparse": "hf://autonomousvision/learn2splat/sparse/checkpoints/epoch_9-step_90000.ckpt",
+}
+
+
+def ensure_data(data_dir: str) -> None:
+    """Download the demo scene from the Hugging Face Hub if it is not present."""
+    if os.path.isdir(data_dir) and os.listdir(data_dir):
+        return
+    from huggingface_hub import snapshot_download
+
+    console.print(
+        f"[yellow]{data_dir}[/] not found — downloading from "
+        f"[cyan]hf://{DEMO_DATA_REPO}[/] …"
+    )
+    snapshot_download(
+        repo_id=DEMO_DATA_REPO,
+        allow_patterns=[f"{data_dir.rstrip('/')}/**"],
+        local_dir=".",
+    )
+    console.print(f"[green]✓[/] scene ready at [yellow]{data_dir}[/]")
+
+
+@dataclass
+class Config:
+    # Path to the COLMAP dataset (expects images/ + sparse/0/).
+    data_dir: str = "data/mip360/garden"
+    # Downsample factor for the dataset.
+    data_factor: int = 4
+    # Global multiplier on scene-size-related parameters.
+    global_scale: float = 1.0
+    # Normalize the world space.
+    normalize_world_space: bool = True
+    # Every N images is a test image, held out for evaluation.
+    test_every: int = 8
+    # Directory to save renders / stats / the refined PLY.
+    result_dir: str = "results/demo"
+    # Random seed.
+    seed: int = 42
+
+    # --- Interactive GUI ---
+    # Launch a viser GUI instead of the headless comparison. "server" renders
+    # frames with the optgs decoder; "client" uses viser's built-in WebGL
+    # Gaussian-splat renderer. Unset = headless run.
+    with_gui: Optional[Literal["client", "server"]] = None
+    # Port for the viser GUI web server (--with-gui only).
+    gui_port: int = 8080
+
+    # --- OptGS learned optimizer ---
+    # Compute device (OptGS requires CUDA).
+    device: str = "cuda"
+    # Number of learned refinement steps.
+    max_steps: int = 100
+    # Views the optimizer sees per refinement step (the view minibatch).
+    opt_batch_size: int = 8
+    # View-minibatch sampling strategy: "random", "sequential", or "fps"
+    # (farthest-point sampling over camera positions).
+    opt_batch_strategy: Literal["random", "sequential", "fps"] = "fps"
+
+    # --- gsplat renderer ---
+    # rasterize_mode / eps2d: when set, applied to every run (dense, sparse,
+    # Adam), overriding each checkpoint's decoder config so the comparison uses
+    # one renderer. Left unset, each run uses its own checkpoint's value.
+    rasterize_mode: Optional[Literal["classic", "antialiased"]] = None
+    eps2d: Optional[float] = None
+
+    # --- Initialization ---
+    # Initialization strategy: "sfm" or "random".
+    init_type: str = "sfm"
+    # Initial number of GSs. Ignored when init_type="sfm".
+    init_num_pts: int = 100_000
+    # Initial extent of GSs as a multiple of the scene extent (random init).
+    init_extent: float = 3.0
+    # Initial opacity / scale of each GS.
+    init_opa: float = 0.1
+    init_scale: float = 1.0
+
+
+def scene_extent(parser: Parser, global_scale: float) -> float:
+    """Scene-size scalar: parser extent x 1.1 x global_scale."""
+    return parser.scene_scale * 1.1 * global_scale
+
+
+def sfm_initialization(
+    parser: Parser, cfg: Config, sh_degree: int, device: torch.device, dtype: torch.dtype
+) -> Gaussians:
+    """SfM (or random) Gaussian init -> an optgs ``Gaussians`` (batch=1).
+
+    Builds the parameter tensors with the same heuristics as 3DGS / the optgs
+    COLMAP initializer, then assembles them through ``points_to_gaussians``.
+    """
+    if cfg.init_type == "sfm":
+        points = torch.from_numpy(parser.points).float()
+        rgbs = torch.from_numpy(parser.points_rgb / 255.0).float()
+    elif cfg.init_type == "random":
+        extent = scene_extent(parser, cfg.global_scale)
+        points = cfg.init_extent * extent * (
+            torch.rand((cfg.init_num_pts, 3)) * 2 - 1
+        )
+        rgbs = torch.rand((cfg.init_num_pts, 3))
+    else:
+        raise ValueError(f"unknown init_type: {cfg.init_type!r} (sfm | random)")
+
+    # GS size = average distance to the 3 nearest neighbours ([:, 1:] drops self).
+    dist2_avg = (knn(points, 4)[:, 1:] ** 2).mean(dim=-1)
+    scales = (torch.sqrt(dist2_avg) * cfg.init_scale).unsqueeze(-1).repeat(1, 3)
+    opacities = torch.full((points.shape[0],), cfg.init_opa)
+
+    # points_to_gaussians returns pre-activation params (log scales, logit
+    # opacity, sh0/shN, random quats).
+    g = points_to_gaussians(
+        {"xyz": points, "rgb": rgbs, "scales": scales, "opacities": opacities},
+        sh_degree=sh_degree,
+        device=device,
+    )
+    sh0, shN = g["sh0"], g["shN"]
+    harmonics = torch.cat([sh0, shN], dim=1) if shN is not None else sh0  # [N, K, 3]
+    harmonics = harmonics.permute(0, 2, 1)  # -> [N, 3, K]
+
+    scales_act = torch.exp(g["scales_raw"])
+    opacities_act = torch.sigmoid(g["opacities_raw"])
+    rotations = F.normalize(g["rotations_unnorm"], dim=-1)
+    covariances = build_covariance(scale=scales_act, rotation_xyzw=rotations)
+
+    def _b(t: Tensor) -> Tensor:  # add the batch dimension and cast
+        return t.unsqueeze(0).to(dtype)
+
+    return Gaussians(
+        means=_b(g["xyz"]),
+        covariances=_b(covariances),
+        harmonics=_b(harmonics),
+        opacities=_b(opacities_act),
+        scales=_b(scales_act),
+        rotations=_b(rotations),
+        rotations_unnorm=_b(g["rotations_unnorm"]),
+    )
+
+
+def collect_cameras(
+    dataset: Dataset, indices: List[int]
+) -> Tuple[Tensor, Tensor, Tensor]:
+    """Stack the selected views into ``(camtoworlds, Ks, images)``.
+
+    ``images`` is returned in [0, 1]. All views must share one (H, W) — the
+    optgs renderer takes a single image shape.
+    """
+    c2ws, ks, imgs = [], [], []
+    hw = None
+    for i in indices:
+        data = dataset[i]
+        img = data["image"] / 255.0  # [H, W, 3], float
+        if hw is None:
+            hw = img.shape[:2]
+        elif img.shape[:2] != hw:
+            raise ValueError(
+                f"all views must share one (H, W); got {tuple(img.shape[:2])} "
+                f"vs {tuple(hw)}. Render the dataset at a single resolution."
+            )
+        c2ws.append(data["camtoworld"])
+        ks.append(data["K"])
+        imgs.append(img)
+    return torch.stack(c2ws), torch.stack(ks), torch.stack(imgs)
+
+
+def build_batched_views(
+    camtoworlds: Tensor,
+    Ks: Tensor,
+    images: Tensor,
+    scene_scale: float,
+    device: torch.device,
+    dtype: torch.dtype,
+) -> dict:
+    """COLMAP cameras -> an optgs ``BatchedViews`` dict (batch=1).
+
+    COLMAP ``camtoworld`` is already optgs's extrinsics convention (OpenCV
+    camera->world). ``K`` is pixel-space; optgs wants it normalized by image
+    width/height.
+    """
+    v, h, w = images.shape[0], images.shape[1], images.shape[2]
+
+    Ks_norm = Ks.clone()
+    Ks_norm[:, 0, :] /= w  # normalized focal / principal point
+    Ks_norm[:, 1, :] /= h
+
+    image = images.permute(0, 3, 1, 2)  # [V, 3, H, W]
+
+    def _b(t: Tensor) -> Tensor:  # add the batch dimension and move to device
+        return t.unsqueeze(0).to(device=device, dtype=dtype)
+
+    return {
+        "extrinsics": _b(camtoworlds),
+        "intrinsics": _b(Ks_norm),
+        "image": _b(image),
+        "near": torch.full((1, v), NEAR_PLANE, device=device, dtype=dtype),
+        "far": torch.full((1, v), FAR_PLANE, device=device, dtype=dtype),
+        "index": torch.arange(v, device=device).unsqueeze(0),
+        "scene_scale": torch.tensor([scene_scale], device=device, dtype=dtype),
+    }
+
+
+@torch.no_grad()
+def render_and_score(
+    optgs,
+    refined: Gaussians,
+    val_bv: dict,
+    val_images: Tensor,
+    out_dir: str,
+    device: torch.device,
+) -> dict:
+    """Render one optimizer's result on the held-out views; report mean PSNR.
+
+    Saves a ``gt | pred`` strip per view under ``out_dir/renders``.
+    """
+    render_dir = os.path.join(out_dir, "renders")
+    os.makedirs(render_dir, exist_ok=True)
+    h, w = val_images.shape[1], val_images.shape[2]
+
+    out = optgs.decoder.forward(
+        refined, val_bv["extrinsics"], val_bv["intrinsics"],
+        val_bv["near"], val_bv["far"], image_shape=(h, w),
+    )
+    colors = out.color[0].clamp(0.0, 1.0)  # [V, 3, H, W]
+
+    psnrs = []
+    for i in range(colors.shape[0]):
+        gt = val_images[i].to(device)  # [H, W, 3]
+        pred = colors[i].permute(1, 2, 0)
+        psnrs.append(-10.0 * torch.log10(torch.mean((pred - gt) ** 2)).item())
+
+        canvas = torch.cat([gt, pred], dim=1).cpu().numpy()  # gt | pred
+        imageio.imwrite(
+            os.path.join(render_dir, f"val_{i:04d}.png"),
+            (canvas * 255).astype(np.uint8),
+        )
+
+    return {"psnr": float(np.mean(psnrs)), "num_views": int(colors.shape[0])}
+
+
+@torch.no_grad()
+def render_view(
+    optgs, gaussians: Gaussians, camera, height: int,
+    device: torch.device, dtype: torch.dtype,
+) -> np.ndarray:
+    """Render ``gaussians`` from a viser camera into an ``[H, W, 3]`` uint8 image.
+
+    viser cameras follow OpenCV conventions, so ``(wxyz, position)`` is directly
+    the camera-to-world transform the optgs decoder expects — no axis flip.
+    """
+    import viser.transforms as vtf
+
+    from optgs.misc.image_io import prep_image
+
+    h = int(height)
+    w = max(1, round(h * camera.aspect))  # camera.aspect = width / height
+
+    c2w = torch.eye(4, device=device, dtype=dtype)
+    c2w[:3, :3] = torch.tensor(
+        vtf.SO3(camera.wxyz).as_matrix(), device=device, dtype=dtype
+    )
+    c2w[:3, 3] = torch.tensor(camera.position, device=device, dtype=dtype)
+
+    # Normalized intrinsics from the vertical fov; the decoder un-normalizes by
+    # the image width/height.
+    fy = (h / 2.0) / float(np.tan(camera.fov / 2.0))
+    K = torch.eye(3, device=device, dtype=dtype)
+    K[0, 0] = fy / w
+    K[1, 1] = fy / h
+    K[0, 2] = 0.5
+    K[1, 2] = 0.5
+
+    near = torch.full((1, 1), NEAR_PLANE, device=device, dtype=dtype)
+    far = torch.full((1, 1), FAR_PLANE, device=device, dtype=dtype)
+    out = optgs.decoder.forward(
+        gaussians, c2w[None, None], K[None, None], near, far, image_shape=(h, w),
+    )
+    return prep_image(out.color[0, 0])  # [H, W, 3] uint8
+
+
+def gaussians_to_splat_data(gaussians: Gaussians) -> dict:
+    """An optgs ``Gaussians`` (batch=1) -> numpy arrays for viser's splat viewer.
+
+    Covariances are recomputed from scale/rotation (the optimizer updates those
+    but may leave the optional ``Gaussians.covariances`` field stale); colours
+    come from the SH DC term (degree 0 — viser's renderer is not view-dependent).
+    """
+    scales = gaussians.scales[0]
+    opacities = gaussians.opacities[0]
+    if not gaussians.stores_activated:
+        scales = torch.exp(scales)
+        opacities = torch.sigmoid(opacities)
+    rotations = F.normalize(gaussians.rotations_unnorm[0], dim=-1)
+    covariances = build_covariance(scale=scales, rotation_xyzw=rotations)
+    rgbs = (0.5 + SH_C0 * gaussians.harmonics[0, :, :, 0]).clamp(0.0, 1.0)
+
+    def _np(t: Tensor) -> np.ndarray:
+        return t.detach().cpu().numpy().astype(np.float32)
+
+    return {
+        "centers": _np(gaussians.means[0]),          # (N, 3)
+        "covariances": _np(covariances),             # (N, 3, 3)
+        "rgbs": _np(rgbs),                           # (N, 3)
+        "opacities": _np(opacities.reshape(-1, 1)),  # (N, 1)
+    }
+
+
+def run_gui(
+    instances: dict,
+    gaussians: Gaussians,
+    train_bv: dict,
+    cfg: Config,
+    device: torch.device,
+    dtype: torch.dtype,
+) -> None:
+    """Interactive viser GUI: watch the optimization, pick an optimizer, reset.
+
+    The initialization is shown first; the user picks an optimizer — the
+    Learn2Splat learned optimizer (dense or sparse checkpoint) or a 3DGS Adam
+    baseline — and clicks Start; every optimizer step is rendered and displayed;
+    Reset restores the initialization. ``cfg.with_gui`` chooses the renderer —
+    "server" (optgs decoder, frames streamed as images) or "client" (viser's
+    WebGL splats).
+
+    ``instances`` maps "dense"/"sparse" to their initialized ``OptGS``.
+    """
+    import threading
+
+    import viser
+    import viser.transforms as vtf
+
+    from optgs.experimental.api.integration.config_bridge import build_adam_baseline
+
+    mode = cfg.with_gui  # "server" | "client"
+    server = viser.ViserServer(port=cfg.gui_port)
+
+    # Optimizer dropdown label -> (instances key, whether to swap in Adam).
+    # "dense"/"sparse" run that checkpoint's own learned optimizer; "Adam" runs
+    # a 3DGS Adam baseline on the dense checkpoint's pipeline.
+    OPTIONS: Dict[str, Tuple[str, bool]] = {
+        "Learn2Splat (dense)": ("dense", False),
+        "Learn2Splat (sparse)": ("sparse", False),
+        "Adam (3DGS)": ("dense", True),
+    }
+
+    optimizer_dd = server.gui.add_dropdown("Optimizer", tuple(OPTIONS))
+
+    # Optimization controls — applied to the picked OptGS at Start; frozen
+    # while optimizing, unfrozen by Reset. opt_batch_size is capped at the
+    # number of training views (the per-step view minibatch can't exceed them).
+    n_train_views = int(train_bv["image"].shape[1])
+    max_steps_input = server.gui.add_number(
+        "Max steps", min=1, max=1000, step=1, initial_value=cfg.max_steps
+    )
+    batch_size_input = server.gui.add_number(
+        "Opt batch size", min=1, max=n_train_views, step=1,
+        initial_value=min(cfg.opt_batch_size, n_train_views),
+    )
+    strategy_dd = server.gui.add_dropdown(
+        "Opt batch strategy", ("random", "sequential", "fps"),
+        initial_value=cfg.opt_batch_strategy,
+    )
+    opt_controls = (max_steps_input, batch_size_input, strategy_dd)
+
+    start_btn = server.gui.add_button("Start optimization")
+    reset_btn = server.gui.add_button("Reset to initialization")
+    status = server.gui.add_markdown("**initialized** — pick an optimizer, then Start")
+    res_slider = (
+        server.gui.add_slider(
+            "Render height", min=240, max=1080, step=60, initial_value=540
+        )
+        if mode == "server"
+        else None
+    )
+
+    init_gaussians = gaussians.clone()  # pristine copy, for Reset
+    current = init_gaussians            # Gaussians currently displayed
+    active = instances["dense"]         # OptGS used to render + to optimize next
+    gen = None                          # optimize_iter generator while running
+    last_cam_ts: dict = {}              # client id -> last-rendered camera stamp
+    lock = threading.Lock()
+    state = {
+        "mode": "init",                 # "init" | "optimizing" | "done"
+        "step": 0,
+        "start": False,
+        "reset": False,
+        "rerender": False,              # a GUI control changed -> re-render once
+        "selected": next(iter(OPTIONS)),
+    }
+
+    @start_btn.on_click
+    def _(_) -> None:
+        with lock:
+            if state["mode"] in ("init", "done"):
+                state["selected"] = optimizer_dd.value
+                state["start"] = True
+
+    @reset_btn.on_click
+    def _(_) -> None:
+        with lock:
+            state["reset"] = True
+
+    # The render-height slider only affects server-rendered frames; re-render
+    # on change so the new resolution takes effect without a camera move.
+    if res_slider is not None:
+
+        @res_slider.on_update
+        def _(_) -> None:
+            with lock:
+                state["rerender"] = True
+
+    # Frame newly-connected clients on the first training camera (viser and
+    # optgs share the OpenCV camera-to-world convention).
+    cam_extr = train_bv["extrinsics"][0, 0].detach().cpu().numpy()
+
+    @server.on_client_connect
+    def _(client) -> None:
+        try:
+            client.camera.position = cam_extr[:3, 3]
+            client.camera.wxyz = vtf.SO3.from_matrix(cam_extr[:3, :3]).wxyz
+        except Exception:
+            pass
+
+    if mode == "client":  # show the initialization immediately
+        # Black backdrop for the WebGL splat renderer (viser's canvas is not
+        # black by default); on server.scene so late-joining clients get it.
+        server.scene.set_background_image(np.zeros((8, 8, 3), dtype=np.uint8))
+        server.scene.add_gaussian_splats(
+            "/optgs/splats", **gaussians_to_splat_data(current)
+        )
+
+    console.print(
+        f"[green]✓[/] viser GUI ([cyan]{mode}[/]) on port [cyan]{cfg.gui_port}[/]"
+        f" — forward the port over SSH and open the printed URL"
+    )
+
+    try:
+        while True:
+            changed = False
+
+            with lock:
+                do_reset, do_start = state["reset"], state["start"]
+                do_rerender = state["rerender"]
+                state["reset"] = state["start"] = state["rerender"] = False
+                selected = state["selected"]
+
+            if do_rerender:
+                changed = True  # server mode re-renders every connected client
+
+            if do_reset:
+                if gen is not None:
+                    gen.close()  # runs optimize_iter's finally -> on_scene_end()
+                    gen = None
+                current = init_gaussians
+                with lock:
+                    state["mode"], state["step"] = "init", 0
+                optimizer_dd.disabled = start_btn.disabled = False
+                for c in opt_controls:
+                    c.disabled = False
+                changed = True
+
+            if do_start and gen is None:
+                name, use_adam = OPTIONS[selected]
+                active = instances[name]
+                # Apply the GUI optimization controls before the run starts.
+                active.num_refine = int(max_steps_input.value)
+                active.opt_batch_size = int(batch_size_input.value)
+                active.opt_batch_strategy = strategy_dd.value
+                opt = (
+                    build_adam_baseline(active.num_refine).to(device)
+                    if use_adam
+                    else None
+                )
+                gen = active.optimize_iter(optimizer=opt)
+                with lock:
+                    state["mode"], state["step"] = "optimizing", 0
+                optimizer_dd.disabled = start_btn.disabled = True
+                for c in opt_controls:
+                    c.disabled = True
+
+            if gen is not None:
+                try:
+                    step, current = next(gen)
+                    changed = True
+                    with lock:
+                        state["step"] = step + 1
+                except StopIteration:
+                    gen = None
+                    with lock:
+                        state["mode"] = "done"
+                    optimizer_dd.disabled = start_btn.disabled = False
+
+            if mode == "server":
+                for cid, client in server.get_clients().items():
+                    try:
+                        cam_ts = client.camera.update_timestamp
+                        if last_cam_ts.get(cid) != cam_ts or changed:
+                            last_cam_ts[cid] = cam_ts
+                            image = render_view(
+                                active, current, client.camera,
+                                res_slider.value, device, dtype,
+                            )
+                            client.scene.set_background_image(image, format="jpeg")
+                    except Exception:
+                        continue  # no camera message from this client yet
+            elif changed:  # client mode — re-push splats when the Gaussians change
+                server.scene.add_gaussian_splats(
+                    "/optgs/splats", **gaussians_to_splat_data(current)
+                )
+
+            with lock:
+                status.content = (
+                    f"**{state['mode']}** — step "
+                    f"{state['step']}/{active.num_refine} — "
+                    f"{current.means.shape[1]} Gaussians"
+                )
+
+            if gen is None:
+                time.sleep(1 / 30)  # idle: poll cameras at ~30 Hz
+    except KeyboardInterrupt:
+        if gen is not None:
+            gen.close()
+        console.print("\n[yellow]GUI stopped.[/]")
+
+
+def main(cfg: Config) -> None:
+    # Fetch the demo scene on first run, before anything else touches it.
+    ensure_data(cfg.data_dir)
+
+    from optgs.experimental.api import OptGS, OptGSError
+    from optgs.experimental.api.integration.config_bridge import build_adam_baseline
+
+    os.makedirs(cfg.result_dir, exist_ok=True)
+    device = torch.device(cfg.device)
+    dtype = torch.float32
+
+    console.rule("[bold cyan]OptGS demo[/]  ·  Learn2Splat vs Adam")
+
+    # --- COLMAP scene, train/val split ---
+    parser = Parser(
+        data_dir=cfg.data_dir,
+        factor=cfg.data_factor,
+        normalize=cfg.normalize_world_space,
+        verbose=False,
+    )
+    dataset = Dataset(parser)
+    val_idx = [i for i in range(len(dataset)) if i % cfg.test_every == 0]
+    train_idx = [i for i in range(len(dataset)) if i % cfg.test_every != 0]
+    scene_scale = scene_extent(parser, cfg.global_scale)
+    console.print(
+        f"scene scale [cyan]{scene_scale:.4f}[/]  ·  "
+        f"train [cyan]{len(train_idx)}[/]  ·  val [cyan]{len(val_idx)}[/]"
+    )
+    train_bv = build_batched_views(
+        *collect_cameras(dataset, train_idx), scene_scale, device, dtype
+    )
+
+    # --- Interactive GUI: build both learned-optimizer checkpoints (dense and
+    # sparse), initialize each, and hand off to the viser GUI instead of the
+    # headless comparison. The GUI's Optimizer dropdown picks between them. ---
+    if cfg.with_gui is not None:
+        instances = {}
+        for name in ("dense", "sparse"):
+            try:
+                instances[name] = OptGS(
+                    checkpoint=CHECKPOINTS[name],
+                    device=cfg.device,
+                    num_refine=cfg.max_steps,
+                    opt_batch_size=cfg.opt_batch_size,
+                    opt_batch_strategy=cfg.opt_batch_strategy,
+                    rasterize_mode=cfg.rasterize_mode,
+                    eps2d=cfg.eps2d,
+                )
+            except OptGSError as e:
+                console.print(f"[bold red]OptGS error ({name}):[/] {e}")
+                raise SystemExit(1)
+
+        # One SfM init shared by both checkpoints: dense and sparse get an
+        # identical starting point, and the GUI shows a single initialization
+        # regardless of which optimizer is picked.
+        torch.manual_seed(cfg.seed)
+        np.random.seed(cfg.seed)
+        gaussians = sfm_initialization(
+            parser, cfg, instances["dense"].sh_degree, device, dtype
+        )
+        for inst in instances.values():
+            inst.initialize_from_tensors(gaussians, train_bv)
+
+        run_gui(instances, gaussians, train_bv, cfg, device, dtype)
+        return
+
+    val_c2w, val_Ks, val_images = collect_cameras(dataset, val_idx)
+    val_bv = build_batched_views(val_c2w, val_Ks, val_images, scene_scale, device, dtype)
+
+    results: dict = {}
+
+    def finish(optgs, refined, name: str, elapsed: float) -> None:
+        """Persist + evaluate one run's result under results/demo/<name>/."""
+        out_dir = os.path.join(cfg.result_dir, name)
+        os.makedirs(out_dir, exist_ok=True)
+        optgs.export_ply(os.path.join(out_dir, "point_cloud.ply"))
+        ev = render_and_score(optgs, refined, val_bv, val_images, out_dir, device)
+        results[name] = {
+            "psnr": ev["psnr"], "time": elapsed,
+            "num_views": ev["num_views"], "num_GS": int(refined.means.shape[1]),
+        }
+        console.print(
+            f"[green]✓[/] [bold]{name}[/] — PSNR [cyan]{ev['psnr']:.3f}[/]  ·  "
+            f"[cyan]{elapsed:.1f}s[/]  → [yellow]{out_dir}[/]"
+        )
+
+    # --- Learned optimizer (Learn2Splat): dense, then sparse ---
+    optgs = None
+    for name in ("dense", "sparse"):
+        optgs = None  # free the previous instance before building the next
+        torch.cuda.empty_cache()
+        try:
+            optgs = OptGS(
+                checkpoint=CHECKPOINTS[name],
+                device=cfg.device,
+                num_refine=cfg.max_steps,
+                opt_batch_size=cfg.opt_batch_size,
+                opt_batch_strategy=cfg.opt_batch_strategy,
+                rasterize_mode=cfg.rasterize_mode,
+                eps2d=cfg.eps2d,
+            )
+        except OptGSError as e:
+            console.print(f"[bold red]OptGS error ({name}):[/] {e}")
+            raise SystemExit(1)
+        # Seed *after* construction so dense and sparse get an identical SfM init.
+        torch.manual_seed(cfg.seed)
+        np.random.seed(cfg.seed)
+        gaussians = sfm_initialization(parser, cfg, optgs.sh_degree, device, dtype)
+        optgs.initialize_from_tensors(gaussians, train_bv)
+
+        torch.cuda.synchronize()  # drain setup GPU work so it isn't timed
+        tic = time.time()
+        refined = optgs.optimize()
+        torch.cuda.synchronize()
+        finish(optgs, refined, name, time.time() - tic)
+
+    # --- Fair Adam baseline: same SfM init / views / step budget / gsplat
+    # renderer, run through the same optimize() path on the last OptGS
+    # instance — only the update rule differs. ---
+    adam = build_adam_baseline(optgs.num_refine).to(device)
+    torch.cuda.synchronize()  # drain setup GPU work so it isn't timed
+    tic = time.time()
+    refined_adam = optgs.optimize(optimizer=adam)
+    torch.cuda.synchronize()
+    finish(optgs, refined_adam, "adam", time.time() - tic)
+
+    # --- Comparison table ---
+    table = Table(
+        title=(
+            f"Novel-view PSNR  ·  {results['dense']['num_views']} held-out "
+            f"views  ·  {cfg.max_steps} steps  ·  "
+            f"{results['dense']['num_GS']} Gaussians"
+        ),
+        title_style="bold",
+        caption=(
+            f"gsplat renderer  ·  "
+            f"rasterize_mode={cfg.rasterize_mode or 'per-checkpoint'}  ·  "
+            f"eps2d={cfg.eps2d if cfg.eps2d is not None else 'per-checkpoint'}"
+        ),
+    )
+    table.add_column("Optimizer")
+    table.add_column("PSNR (dB)", justify="right")
+    table.add_column("Time (s)", justify="right")
+    best = max(results, key=lambda k: results[k]["psnr"])
+    for key, label in (
+        ("dense", "Learn2Splat (dense)"),
+        ("sparse", "Learn2Splat (sparse)"),
+        ("adam", "Adam"),
+    ):
+        table.add_row(
+            label,
+            f"{results[key]['psnr']:.3f}",
+            f"{results[key]['time']:.1f}",
+            style="bold green" if key == best else None,
+        )
+    console.print(table)
+
+    with open(os.path.join(cfg.result_dir, "stats.json"), "w") as f:
+        json.dump(results, f, indent=2)
+    console.print(f"[green]✓[/] results written to [yellow]{cfg.result_dir}[/]")
+
+
+if __name__ == "__main__":
+    main(tyro.cli(Config))

optgs/__init__.py

@@ -0,0 +1 @@
+"""optgs — learned optimization for 3D Gaussian Splatting."""

optgs/config.py

@@ -0,0 +1,770 @@
+import importlib
+from copy import deepcopy
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Literal, Optional, Type, TypeVar, Any, Callable
+
+import hydra
+import torch
+from dacite import Config, from_dict, UnionMatchError
+from hydra.core.global_hydra import GlobalHydra
+from hydra.core.hydra_config import HydraConfig
+from hydra.types import RunMode
+from omegaconf import DictConfig
+from omegaconf import OmegaConf
+from pytorch_lightning.strategies import DDPStrategy, FSDPStrategy
+
+from .config_migrate import migrate, CURRENT_CFG_VERSION
+from .dataset.data_module import DataLoaderCfg, DatasetCfg
+from .global_cfg import set_cfg
+from .loss import LossCfgWrapper
+from .misc.io import CustomPath
+from .misc.io import cyan, read_omega_cfg
+from .misc.checkpointing import find_latest_ckpt
+from .misc.hf_ckpt import maybe_resolve_hf_ref
+from .paths import CKPT_DIR, RESULTS_DIR
+from .scene_trainer.scene_trainer_cfg import SceneTrainerCfg, MetaOptimizerCfg, TestCfg, TrainCfg
+
+
+# In order to extract filename or dirname from a path in the config
+def checkpoint_rel_dir(path):
+    rel_dir = CustomPath(path) - CKPT_DIR  # dir_path / checkpoints / epoch_x-step_xxxxx.ckpt
+    dir_path = rel_dir.parent.parent
+    return str(dir_path)
+
+
+OmegaConf.register_new_resolver("checkpoint_rel_dir", checkpoint_rel_dir)
+OmegaConf.register_new_resolver("parent_dir", lambda path: str(CustomPath(path).parent))
+
+
+@dataclass
+class CheckpointingCfg:
+    load: Optional[str]  # Not a path, since it could be something like wandb://...
+    every_n_train_steps: int
+    save_top_k: int
+    pretrained_model: Optional[str]
+    pretrained_monodepth: Optional[str]
+    pretrained_mvdepth: Optional[str]
+    pretrained_depth: Optional[str]
+    pretrained_scale_predictor: Optional[str]
+    pretrained_depth_teacher: Optional[str]
+    no_strict_load: bool
+    resume: bool
+    no_resume_upsampler: bool
+    partial_load: bool
+    freeze_mono_vit: bool
+    pretrained_initializer: Optional[str]
+    pretrained_optimizer: Optional[str]
+    resume_update_module: str | None
+    load_existing_cfg: bool
+
+    def __post_init__(self):
+        # Resolve any Hugging Face Hub references (hf://org/repo/file[@rev]) to
+        # local cached paths so all downstream torch.load calls work unchanged.
+        for attr in ("pretrained_model", "pretrained_optimizer", "pretrained_initializer",
+                     "pretrained_monodepth", "pretrained_mvdepth", "pretrained_depth",
+                     "pretrained_scale_predictor", "pretrained_depth_teacher",
+                     "resume_update_module"):
+            resolved = maybe_resolve_hf_ref(getattr(self, attr))
+            if resolved != getattr(self, attr):
+                setattr(self, attr, resolved)
+
+        for attr in ("pretrained_model", "pretrained_optimizer", "pretrained_initializer"):
+            path = getattr(self, attr)
+            if path is not None and Path(path).name == "last":
+                try:
+                    resolved = find_latest_ckpt(Path(path).parent)
+                    setattr(self, attr, resolved)
+                    print(f"Replacing {attr} to last checkpoint: {resolved}")
+                except Exception as e:
+                    print(cyan(f"Warning: {e}. Continuing with 'last' as {attr}."))
+
+
+@dataclass
+class MetaTrainerCfg:
+    max_steps: int
+    val_check_interval: int | float | None
+    gradient_clip_val: int | float | None
+    num_sanity_val_steps: int
+    num_nodes: int
+    eval_index: str | None
+    limit_test_batches: int | float
+    limit_train_batches: int | float
+    test: TestCfg
+    train: TrainCfg
+
+    def get_dist_strategy(self, scene_trainer_cfg: SceneTrainerCfg):
+        from .scene_trainer.initializer.initializer_resplat import ResplatInitializerCfg
+        dist_strategy = "auto"
+        if torch.cuda.device_count() > 1:
+            dist_strategy = 'ddp'
+            if isinstance(scene_trainer_cfg.scene_optimizer, ResplatInitializerCfg):
+                if scene_trainer_cfg.scene_initializer.use_gt_depth:
+                    dist_strategy = 'ddp_find_unused_parameters_true'
+                if scene_trainer_cfg.scene_initializer.use_checkpointing or scene_trainer_cfg.scene_initializer.init_use_checkpointing:
+                    dist_strategy = DDPStrategy(static_graph=True)
+                if scene_trainer_cfg.use_fsdp:
+                    def only_wrap_trainable(module, recurse, nonwrapped_numel):
+                        has_trainable = any(p.requires_grad for p in module.parameters())
+                        return has_trainable
+
+                    dist_strategy = FSDPStrategy(auto_wrap_policy=only_wrap_trainable)
+        if self.train.use_replay_buffer:
+            # When resampling from the replay buffer,
+            # we don't project the condition_features to state, so the update_proj is not used
+            dist_strategy = "ddp_find_unused_parameters_true"
+        return dist_strategy
+
+
+@dataclass
+class RootCfg:
+    wandb: dict
+    mode: Literal["train", "test"]
+    dataset: DatasetCfg
+    data_loader: DataLoaderCfg
+    scene_trainer: SceneTrainerCfg
+    meta_optimizer: MetaOptimizerCfg  ## TODO Naama: should we move under meta trainer config?
+    checkpointing: CheckpointingCfg
+    meta_trainer: MetaTrainerCfg
+    loss: list[LossCfgWrapper]
+    seed: int
+    use_plugins: bool
+    output_dir: str
+    version: int | None
+    debug_cfg: bool
+
+    def __post_init__(self):
+        if self.mode == "test":
+            self._setup_test_output_dir()
+
+    def _setup_test_output_dir(self):
+        base_res_dir = RESULTS_DIR
+        if self.meta_trainer.limit_test_batches != 1.0:
+            base_res_dir = RESULTS_DIR + f"_{self.meta_trainer.limit_test_batches}_scenes"
+        if self.output_dir == "placeholder":
+            if self.meta_trainer.test.postprocessing is not None and self.meta_trainer.test.postprocessing.is_active:
+                self.output_dir = (base_res_dir /
+                                   "nonlearned" /
+                                   "vanilla_3dgs" /
+                                   self.meta_trainer.test.postprocessing.name /
+                                   self.meta_trainer.test.postprocessing.get_dir_name(with_name=False))
+            else:
+                ckpt_path = self.checkpointing.pretrained_model or self.checkpointing.pretrained_optimizer
+                pretrained_model_rel_dir = checkpoint_rel_dir(ckpt_path)
+                self.output_dir = (base_res_dir /
+                                   "optgs" /
+                                   pretrained_model_rel_dir)
+        elif 'experimental' in str(self.output_dir):  # TODO (release): remove
+            self._setup_experimental_output_dir()
+
+    def _setup_experimental_output_dir(self):
+        resplat_str = []
+        grad_str = []
+        normgrad_str = []
+        assert self.scene_trainer.scene_optimizer.experimental_run
+        for p in self.scene_trainer.scene_optimizer.experimental_update.param_names:
+            update = getattr(self.scene_trainer.scene_optimizer.experimental_update, p)
+            use_norm_grad = getattr(self.scene_trainer.scene_optimizer.experimental_use_norm_grads, p)
+            use_grad = self.scene_trainer.scene_optimizer.experimental_use_grads and not use_norm_grad
+            use_resplat = update and not use_grad and not use_norm_grad
+            if update:
+                assert use_grad ^ use_norm_grad ^ use_resplat, f"Invalid combination for {p}: use_resplat={use_resplat}, use_grad={use_grad}, use_norm_grad={use_norm_grad}"
+                if use_resplat:
+                    resplat_str.append(p)
+                if use_grad:
+                    grad_str.append(p)
+                if use_norm_grad:
+                    normgrad_str.append(p)
+
+        if len(resplat_str) == len(self.scene_trainer.scene_optimizer.experimental_update.param_names):
+            resplat_str = ["all"]
+        if len(grad_str) == len(self.scene_trainer.scene_optimizer.experimental_update.param_names):
+            grad_str = ["all"]
+        if len(normgrad_str) == len(self.scene_trainer.scene_optimizer.experimental_update.param_names):
+            normgrad_str = ["all"]
+
+        exp_name = "_".join([
+            ("resplat_" + "_".join(resplat_str) if len(resplat_str) > 0 else ""),
+            ("grad_" + "_".join(grad_str) if len(grad_str) > 0 else ""),
+            ("normgrad_" + "_".join(normgrad_str) if len(normgrad_str) > 0 else ""),
+        ])
+
+        output_dir_str = str(self.output_dir)
+        output_dir_str = output_dir_str.replace("experimental", f"experimental_{exp_name}")
+        self.output_dir = Path(output_dir_str)
+        print(cyan(f"Experimental run, setting output_dir to {CustomPath(self.output_dir)}"))
+
+
+TYPE_HOOKS = {
+    Path: Path,
+}
+
+T = TypeVar("T")
+
+
+def get_class_by_path(path: str):
+    module_path, class_name = path.rsplit('.', 1)
+    module = importlib.import_module(module_path)
+    return getattr(module, class_name)
+
+
+def _diagnose_union_error(e: UnionMatchError, data: dict, dacite_config: Config) -> str:
+    """Try each union member individually and report per-member errors."""
+    import dataclasses
+    import typing
+    union_type = e.field_type
+    # Extract the member types from the union
+    args = typing.get_args(union_type)
+    if not args:
+        return str(e)
+    lines = [str(e), "", "Per-member diagnostics:"]
+    for member_type in args:
+        try:
+            from_dict(member_type, data, config=dacite_config)
+            lines.append(f"  {member_type.__name__}: matched OK (unexpected)")
+        except Exception as member_err:
+            lines.append(f"  {member_type.__name__}: {member_err}")
+            # For dataclasses, also check for extra/missing fields
+            if dataclasses.is_dataclass(member_type):
+                expected = {f.name for f in dataclasses.fields(member_type)}
+                provided = set(data.keys()) if isinstance(data, dict) else set()
+                missing = expected - provided
+                extra = provided - expected
+                if missing:
+                    lines.append(f"    missing fields: {missing}")
+                if extra:
+                    lines.append(f"    extra fields (ignored with strict=False): {extra}")
+    return "\n".join(lines)
+
+
+def load_typed_config(
+        cfg: DictConfig,
+        data_class: Type[T],
+        extra_type_hooks: dict = {},
+) -> T:
+    dacite_config = Config(type_hooks={**TYPE_HOOKS, **extra_type_hooks})
+    try:
+        return from_dict(
+            data_class,
+            OmegaConf.to_container(cfg),
+            config=dacite_config,
+        )
+    except UnionMatchError as e:
+        diagnostic = _diagnose_union_error(e, e.value, dacite_config)
+        print(f"\n{'='*60}\n"
+              f"Current config: {e.value}\n"
+              "\n"
+              "\n"
+              f"UnionMatchError diagnostic:\n{diagnostic}\n{'='*60}"
+              f"\n",
+              flush=True)
+        raise
+
+
+def separate_loss_cfg_wrappers(joined: dict) -> list[LossCfgWrapper]:
+    # The dummy allows the union to be converted.
+    @dataclass
+    class Dummy:
+        dummy: LossCfgWrapper
+
+    return [
+        load_typed_config(DictConfig({"dummy": {k: v}}), Dummy).dummy
+        for k, v in joined.items()
+    ]
+
+
+def universal_target_hook(cfg: dict, _: Type) -> Any:
+    """Generic hook to construct config objects from `__target__`."""
+    if not isinstance(cfg, dict):
+        return None
+    if "__target__" not in cfg:
+        return None  # Let decite handle it
+
+    cfg_copy = deepcopy(cfg)  # avoid mutating original
+    target = cfg_copy.pop("__target__")
+
+    if isinstance(target, str):
+        target_type = get_class_by_path(target)
+    else:
+        target_type = target
+
+    # Use recursive loading with known additional hooks
+    return load_typed_config(
+        DictConfig(cfg_copy),
+        target_type,
+    )
+
+
+def make_target_hook_for_type(t: Type) -> Callable:
+    return lambda cfg: universal_target_hook(cfg, t)
+
+
+def load_typed_root_config(cfg: DictConfig) -> RootCfg:
+    # scene_trainer/scene_optimizer=none loads a full dict from none.yaml;
+    # dacite can't match that dict to the None arm of SceneOptimizerCfg | None.
+    # Convert it to Python None here so dacite matches correctly.
+    scene_opt = OmegaConf.select(cfg, "scene_trainer.scene_optimizer")
+    if isinstance(scene_opt, DictConfig) and OmegaConf.select(scene_opt, "name") == "none":
+        OmegaConf.set_struct(cfg, False)
+        OmegaConf.update(cfg, "scene_trainer.scene_optimizer", None, merge=False)
+        OmegaConf.set_struct(cfg, True)
+
+    return load_typed_config(
+        cfg,
+        RootCfg,
+        {list[LossCfgWrapper]: separate_loss_cfg_wrappers}
+    )
+
+
+def should_run(cfg_dict):
+    if cfg_dict.mode == "test":
+        if cfg_dict.meta_trainer.test.skip_if_outputs_exist:
+            output_dir = cfg_dict.output_dir
+            if not output_dir.exists():
+                return True
+            metrics_path_pattern = output_dir / "metrics" / "target_*_psnr.json"
+            metric_paths = list(metrics_path_pattern.parent.glob(metrics_path_pattern.name))
+            if len(metric_paths) > 0:
+                print(cyan(f"Test metrics already exist at {metric_paths}."))
+                return False
+    return True
+
+
+def setup_cfg(cfg_dict):
+    # Get the original config from the output directory, when testing or resuming.
+    cfg_dict = merge_config_from_file(cfg_dict)
+    eval_cfg = get_eval_cfg(cfg_dict)
+    cfg = load_typed_root_config(cfg_dict)
+    # Set global cfg object.
+    set_cfg(cfg_dict)
+    # Set up the output directory.
+    setup_output_dir(cfg, cfg_dict)
+    return cfg, cfg_dict, eval_cfg  # TODO Naama: why do we need both cfg and cfg_dict?
+
+
+def flatten_wandb(cfg):
+    """Recursively replace {'desc': ..., 'value': v} with v."""
+    if isinstance(cfg, dict):
+        if "value" in cfg and len(cfg) == 2 and "desc" in cfg:
+            return flatten_wandb(cfg["value"])
+        return {k: flatten_wandb(v) for k, v in cfg.items()}
+    elif isinstance(cfg, list):
+        return [flatten_wandb(v) for v in cfg]
+    else:
+        return cfg
+
+
+def _apply_cli_overrides(merged_cfg: DictConfig, orig_cli_cfg: DictConfig, raw_overrides: list[str]) -> DictConfig:
+    """
+    Re-apply CLI overrides onto merged_cfg after the checkpoint config has been merged in.
+
+    Takes already-composed values from orig_cli_cfg rather than re-parsing the raw override
+    strings. This correctly handles:
+    - Group overrides (e.g. dataset/view_sampler=evaluation) → replace subtree from cli
+    - Complex values (e.g. loss=[mse,ssim]) → replace subtree from cli
+    - Interpolated values (e.g. output_dir=${...}) → take resolved value from cli
+    - Defaults-list overrides (+experiment=re10k) → skip (already baked into orig_cli_cfg)
+    """
+    if not raw_overrides:
+        return merged_cfg
+
+    from hydra.core.override_parser.overrides_parser import OverridesParser
+    parser = OverridesParser.create()
+    parsed = parser.parse_overrides(raw_overrides)
+
+    print(cyan(f"Re-applying {len(raw_overrides)} CLI overrides onto merged config."))
+    OmegaConf.set_struct(merged_cfg, False)
+
+    # Architecture subtrees: CLI group default fills in *new* fields only;
+    # checkpoint values win for fields that already exist.
+    ARCH_KEYS = {"scene_optimizer", "scene_initializer"}
+    # Sub-keys within ARCH_KEYS where CLI should always win over checkpoint values.
+    CLI_WINS_SUBKEYS = {"refiner"}
+
+    for override in parsed:
+        key = override.key_or_group
+        dotkey = key.replace("/", ".")
+
+        cli_val = OmegaConf.select(orig_cli_cfg, dotkey, default=None, throw_on_resolution_failure=False)
+
+        if cli_val is None:
+            # No direct config path — e.g. +experiment=re10k is a defaults-list override
+            # whose effect is already baked into orig_cli_cfg; nothing to apply.
+            print(cyan(f"  Skipping '{key}' (no direct config path in cli)"))
+            continue
+
+        # For architecture group overrides: fill in missing fields from CLI defaults
+        # without overriding checkpoint values for fields that already exist.
+        is_group_override = "/" in key or isinstance(cli_val, (DictConfig, dict, list))
+        if is_group_override and any(arch_key in dotkey for arch_key in ARCH_KEYS):
+            # If the override targets a CLI-wins sub-key directly, CLI wins entirely.
+            dotkey_parts = set(dotkey.split("."))
+            if dotkey_parts & CLI_WINS_SUBKEYS:
+                OmegaConf.update(merged_cfg, dotkey, cli_val, merge=False)
+                print(cyan(f"  '{dotkey}': replace from cli (CLI wins)"))
+                continue
+
+            existing_val = OmegaConf.select(merged_cfg, dotkey, default=None)
+            if existing_val is not None:
+                # cli_val provides new defaults; existing_val (checkpoint) wins for shared fields
+                new_val = OmegaConf.merge(cli_val, existing_val)
+                # Re-apply CLI-wins sub-keys so they override checkpoint values.
+                for subkey in CLI_WINS_SUBKEYS:
+                    cli_subval = OmegaConf.select(cli_val, subkey, default=None)
+                    if cli_subval is not None:
+                        OmegaConf.set_struct(new_val, False)
+                        OmegaConf.update(new_val, subkey, cli_subval, merge=False)
+                        print(cyan(f"  '{dotkey}.{subkey}': CLI override applied (CLI wins)"))
+                OmegaConf.update(merged_cfg, dotkey, new_val, merge=False)
+                print(cyan(f"  '{dotkey}': fill-missing from cli (checkpoint values preserved)"))
+                continue
+
+        # Group overrides and complex values replace the whole subtree;
+        # scalars are merged so sibling keys are preserved.
+        replace = is_group_override
+        print(cyan(f"  '{dotkey}': {'replace' if replace else 'update'} from cli"))
+        OmegaConf.update(merged_cfg, dotkey, cli_val, merge=not replace)
+
+    OmegaConf.set_struct(merged_cfg, True)
+    return merged_cfg
+
+
+def _print_cfg_diff(before: dict, after: dict, prefix: str = "") -> None:
+    """Recursively print keys that differ between two plain-dict config snapshots."""
+    all_keys = set(before) | set(after)
+    diffs = []
+    for k in sorted(all_keys):
+        full_key = f"{prefix}.{k}" if prefix else k
+        b_val = before.get(k, "<missing>")
+        a_val = after.get(k, "<missing>")
+        if isinstance(b_val, dict) and isinstance(a_val, dict):
+            _print_cfg_diff(b_val, a_val, prefix=full_key)
+        elif b_val != a_val:
+            diffs.append((full_key, b_val, a_val))
+    for full_key, b_val, a_val in diffs:
+        print(cyan(f"  [cfg diff]  {full_key}:  {b_val!r}  →  {a_val!r}"))
+
+
+def _find_config_for_checkpoint(ckpt_path) -> Path | None:
+    """Return the config.yaml path for a given checkpoint, or None."""
+    p = Path(ckpt_path).parent.parent / "config.yaml"
+    if p.exists():
+        return p
+    # Fall back to wandb latest-run
+    p = Path(ckpt_path).parent.parent / "wandb" / "latest-run" / "files" / "config.yaml"
+    if p.exists():
+        return p
+    return None
+
+
+def _load_checkpoint_cfg(config_path: Path) -> DictConfig:
+    """Load, migrate, and (if from wandb) flatten a checkpoint config file."""
+    cfg = read_omega_cfg(config_path)
+    cfg = migrate(cfg)
+    if "wandb" in str(config_path):
+        cfg = OmegaConf.create(flatten_wandb(OmegaConf.to_container(cfg, resolve=True)))
+    return cfg
+
+
+def _patch_scene_initializer(target_cfg: DictConfig, init_config_path: Path, context: str) -> None:
+    """
+    Load scene_trainer.scene_initializer from init_config_path and patch it into target_cfg in-place.
+    target_cfg must not be struct-protected when this is called.
+    """
+    init_cfg = _load_checkpoint_cfg(init_config_path)
+    initializer_subcfg = OmegaConf.select(init_cfg, "scene_trainer.scene_initializer", default=None)
+    if initializer_subcfg is not None:
+        print(cyan(f"{context}: patching scene_trainer.scene_initializer from pretrained_initializer config."))
+        OmegaConf.update(target_cfg, "scene_trainer.scene_initializer", initializer_subcfg, merge=True)
+    else:
+        print(cyan("pretrained_initializer config has no scene_trainer.scene_initializer key; skipping patch."))
+
+
+def _resolve_config_paths(cli_cfg) -> tuple[Path | None, Path | None]:
+    """
+    Determine which config files to load based on CLI checkpointing settings.
+
+    Returns:
+        config_path:             main checkpoint config (optimizer + initializer architecture), or None
+        initializer_config_path: separate initializer checkpoint config (overrides main for initializer), or None
+
+    Priority for config_path:
+      resume > pretrained_model > pretrained_optimizer (> pretrained_initializer sets initializer_config_path only)
+    """
+    pretrained_model = cli_cfg.checkpointing.pretrained_model
+    pretrained_optimizer = cli_cfg.checkpointing.pretrained_optimizer
+    pretrained_initializer = cli_cfg.checkpointing.pretrained_initializer
+    should_load = cli_cfg.mode == "test" or cli_cfg.checkpointing.load_existing_cfg
+
+    config_path = None
+    initializer_config_path = None
+
+    if pretrained_model is not None:
+        if should_load:
+            config_path = _find_config_for_checkpoint(pretrained_model)
+            print(cyan(f"Loading config from pretrained_model checkpoint {config_path}"
+                       if config_path else f"No config found for pretrained_model {pretrained_model}."))
+
+    elif pretrained_optimizer is not None:
+        if should_load:
+            config_path = _find_config_for_checkpoint(pretrained_optimizer)
+            print(cyan(f"Loading config from pretrained_optimizer checkpoint {config_path}"
+                       if config_path else f"No config found for pretrained_optimizer {pretrained_optimizer}."))
+            if pretrained_initializer is not None:
+                initializer_config_path = _find_config_for_checkpoint(pretrained_initializer)
+                print(cyan(f"Loading initializer config from pretrained_initializer checkpoint {initializer_config_path}"
+                           if initializer_config_path else f"No config found for pretrained_initializer {pretrained_initializer}."))
+
+    elif pretrained_initializer is not None:
+        if should_load:
+            initializer_config_path = _find_config_for_checkpoint(pretrained_initializer)
+            print(cyan(f"Loading initializer-only config from pretrained_initializer checkpoint {initializer_config_path}"
+                       if initializer_config_path else f"No config found for pretrained_initializer {pretrained_initializer}."))
+
+    else:
+        print(cyan("No pretrained_model, pretrained_optimizer, or pretrained_initializer specified, using cli config only."))
+
+    # Resume overrides config_path to point at the output directory's saved config.
+    if cli_cfg.checkpointing.resume and cli_cfg.checkpointing.load_existing_cfg:
+        config_path = Path(cli_cfg.output_dir) / "config.yaml"
+        print(cyan(f"Resuming: loading config from cfg.output_dir {config_path}"))
+    else:
+        print(cyan("Not resuming.."))
+
+    if config_path is not None and not config_path.exists():
+        print(cyan(f"Config file {config_path} does not exist. Continuing with cli config only."))
+        config_path = None
+    elif config_path is not None:
+        print(cyan(f"Found config file {config_path}."))
+
+    return config_path, initializer_config_path
+
+
+def _merge_test_mode(
+        cli_cfg: DictConfig,
+        loaded_cfg: DictConfig,
+        initializer_config_path: Path | None,
+        pretrained_initializer: str | None,
+) -> tuple[DictConfig, DictConfig]:
+    """
+    Test mode: CLI config is the base for all settings (dataset, test flags, etc.).
+    Only optimizer and initializer *architecture* are patched in from checkpoint configs.
+
+    Initializer source priority:
+      1. separate initializer_config_path  (pretrained_initializer ckpt with a config file)
+      2. main loaded_cfg                   (optimizer checkpoint's bundled initializer)
+      3. CLI config as-is                  (pretrained_initializer set but has no config file)
+
+    Returns (merged_cfg, orig_cli_cfg); orig_cli_cfg is the snapshot taken before any
+    checkpoint patches so that _apply_cli_overrides can restore explicit CLI values.
+    """
+    OmegaConf.set_struct(cli_cfg, False)
+    # Snapshot BEFORE patching: merged_cfg aliases cli_cfg, so patches below also mutate
+    # cli_cfg. _apply_cli_overrides must see the original CLI values, not the patched ones.
+    orig_cli_cfg = OmegaConf.create(
+        OmegaConf.to_container(cli_cfg, resolve=False, throw_on_missing=False)
+    )
+    merged_cfg = cli_cfg  # patched in-place
+
+    # Patch optimizer architecture from checkpoint
+    optimizer_subcfg = OmegaConf.select(loaded_cfg, "scene_trainer.scene_optimizer", default=None)
+    if optimizer_subcfg is not None:
+        print(cyan("Test mode: patching scene_trainer.scene_optimizer from checkpoint config."))
+        OmegaConf.update(merged_cfg, "scene_trainer.scene_optimizer", optimizer_subcfg, merge=True)
+
+    # Patch initializer architecture (priority order above)
+    if initializer_config_path is not None and initializer_config_path.exists():
+        _patch_scene_initializer(merged_cfg, initializer_config_path, context="Test mode")
+    elif pretrained_initializer is None:
+        pass
+        # TODO Naama
+        # No explicit initializer checkpoint — fall back to the optimizer checkpoint's initializer
+        # initializer_subcfg = OmegaConf.select(loaded_cfg, "scene_trainer.scene_initializer", default=None)
+        # if initializer_subcfg is not None:
+            # print(cyan("Test mode: patching scene_trainer.scene_initializer from checkpoint config."))
+            # OmegaConf.update(merged_cfg, "scene_trainer.scene_initializer", initializer_subcfg, merge=True)
+    else:
+        print(cyan("pretrained_initializer set but has no config file; using CLI scene_initializer config."))
+
+    OmegaConf.set_struct(merged_cfg, True)
+    return merged_cfg, orig_cli_cfg
+
+
+def _merge_train_mode(
+        cli_cfg: DictConfig,
+        loaded_cfg: DictConfig,
+        initializer_config_path: Path | None,
+) -> tuple[DictConfig, DictConfig]:
+    """
+    Train mode: checkpoint config takes priority over CLI for all existing fields
+    (preserves the trained architecture). CLI fills in any new fields added since training.
+
+    If a separate initializer checkpoint is given, its scene_initializer replaces the one
+    inside loaded_cfg before the full merge, so the right initializer architecture is used.
+
+    Returns (merged_cfg, orig_cli_cfg); orig_cli_cfg is the pre-merge snapshot used
+    by _apply_cli_overrides to restore explicit CLI values.
+    """
+    if initializer_config_path is not None and initializer_config_path.exists():
+        init_cfg = _load_checkpoint_cfg(initializer_config_path)
+        initializer_subcfg = OmegaConf.select(init_cfg, "scene_trainer.scene_initializer", default=None)
+        if initializer_subcfg is not None:
+            print(cyan("Replacing scene_trainer.scene_initializer in loaded config with initializer config."))
+            OmegaConf.update(loaded_cfg, "scene_trainer.scene_initializer", initializer_subcfg, merge=False)
+        else:
+            print(cyan("pretrained_initializer config has no scene_trainer.scene_initializer key; skipping patch."))
+
+    orig_cli_cfg = OmegaConf.create(
+        OmegaConf.to_container(cli_cfg, resolve=False, throw_on_missing=False)
+    )
+    OmegaConf.set_struct(cli_cfg, False)
+    merged_cfg = OmegaConf.merge(cli_cfg, loaded_cfg)  # loaded_cfg wins for existing fields
+    OmegaConf.set_struct(merged_cfg, True)
+    return merged_cfg, orig_cli_cfg
+
+
+def merge_config_from_file(cli_cfg):
+    # 1. Determine which config files to load.
+    config_path, initializer_config_path = _resolve_config_paths(cli_cfg)
+
+    # 2. No checkpoint config: use CLI as-is, optionally patching in initializer architecture.
+    if config_path is None:
+        print(cyan(f"No config file found, using cli config only. \n"
+                   f"Setting config version to {CURRENT_CFG_VERSION}."))
+        cli_cfg["version"] = CURRENT_CFG_VERSION
+        if initializer_config_path is not None and initializer_config_path.exists():
+            OmegaConf.set_struct(cli_cfg, False)
+            _patch_scene_initializer(cli_cfg, initializer_config_path, context="No-checkpoint")
+            OmegaConf.set_struct(cli_cfg, True)
+        return cli_cfg
+
+    # 3. Load and migrate the checkpoint config.
+    print(cyan(f"Loading config from {config_path}."))
+    loaded_cfg = _load_checkpoint_cfg(config_path)
+
+    # 4. Merge checkpoint config with CLI config (strategy differs by mode).
+    #    Test:  CLI is the base; only optimizer/initializer architecture patched from checkpoint.
+    #    Train: checkpoint takes priority; CLI fills in new fields added since training.
+    pretrained_initializer = cli_cfg.checkpointing.pretrained_initializer
+    if cli_cfg.mode == "test":
+        merged_cfg, orig_cli_cfg = _merge_test_mode(
+            cli_cfg, loaded_cfg, initializer_config_path, pretrained_initializer
+        )
+    else:
+        merged_cfg, orig_cli_cfg = _merge_train_mode(cli_cfg, loaded_cfg, initializer_config_path)
+
+    # 5. Re-apply CLI overrides so user-specified values win over loaded checkpoint config.
+    merged_cfg = _apply_cli_overrides(merged_cfg, orig_cli_cfg, list(HydraConfig.get().overrides.task))
+
+    return merged_cfg
+
+
+class SkipRun(Exception):
+    pass
+
+
+def setup_output_dir(cfg, cfg_dict):
+    if cfg.output_dir != cfg_dict.output_dir:
+        if "$" in str(cfg.output_dir):
+            # interpolated value, not sure how to make it work.
+            cfg.output_dir = CustomPath(cfg_dict.output_dir)
+    output_dir = cfg.output_dir
+    if output_dir is None:
+        output_dir = CustomPath(
+            HydraConfig.get()["runtime"]["output_dir"]
+        )
+    else:  # for resuming
+        output_dir = CustomPath(output_dir)
+        output_dir.mkdir(exist_ok=True, parents=True)
+
+    if HydraConfig.get().mode == RunMode.MULTIRUN and output_dir == "placeholder":
+        # Hack to overcome multirun issues
+        # TODO Naama, need to move to post_init of cfg
+        output_dir = CustomPath(hydra.core.hydra_config.HydraConfig.get()["run"]["dir"])
+        print(cyan(f"Multirun detected, setting output_dir to {CustomPath(output_dir):link}"))
+        # save checkoint path to a file for debugging
+        ckpt_path = cfg.checkpointing.pretrained_model or cfg.checkpointing.pretrained_optimizer
+        (output_dir / "ckpt_dir.txt").write_text(str(ckpt_path))
+    cfg_dict.output_dir = output_dir
+    cfg.output_dir = output_dir
+    output_dir.mkdir(exist_ok=True, parents=True)
+
+    if cfg.mode == 'test':
+        if cfg.meta_trainer.test.output_path is None or str(cfg.meta_trainer.test.output_path) in ['placeholder', 'outputs/test']:
+            cfg.meta_trainer.test.output_path = output_dir
+        if cfg.meta_trainer.test.compute_scores:
+            (cfg.meta_trainer.test.output_path / "metrics").mkdir(exist_ok=True, parents=True)
+    print(cyan(f"Saving outputs to {CustomPath(output_dir):link}."))
+
+    # Save the config to the output directory.
+    cfg_dict_path = output_dir / "config.yaml"
+
+    with open(cfg_dict_path, "w") as f:
+        OmegaConf.save(cfg_dict, f)
+
+
+def get_eval_cfg(cfg_dict):
+    if "meta_trainer" in cfg_dict:
+        meta_trainer_dict = cfg_dict["meta_trainer"]
+    else:
+        raise ValueError("No trainer or meta_trainer in cfg_dict")
+
+    if cfg_dict["mode"] == "train" and meta_trainer_dict["train"]["eval_model_every_n_val"] > 0:
+        eval_cfg_dict = deepcopy(cfg_dict)
+        dataset_dir = str(cfg_dict["dataset"]["roots"]).lower()
+        if "re10k" in dataset_dir:
+            if cfg_dict["dataset"]["view_sampler"]["num_context_views"] == 2:
+                eval_path = "assets/evaluation_index_re10k.json"
+            elif cfg_dict["dataset"]["view_sampler"]["num_context_views"] == 4:
+                eval_path = "assets/re10k_start_0_distance_150_ctx_4v_tgt_6v.json"
+            elif cfg_dict["dataset"]["view_sampler"]["num_context_views"] == 6:
+                eval_path = "assets/re10k_start_0_distance_200_ctx_6v_tgt_6v.json"
+            else:
+                if meta_trainer_dict["eval_index"] is not None:
+                    eval_path = None  # placeholder
+                else:
+                    raise ValueError("unsupported number of views for re10k")
+        elif "dl3dv" in dataset_dir:
+            if cfg_dict["dataset"]["view_sampler"]["num_context_views"] == 6:
+                eval_path = "assets/dl3dv_start_0_distance_50_ctx_6v_tgt_8v.json"
+            elif cfg_dict["dataset"]["view_sampler"]["num_context_views"] == 2:
+                eval_path = "assets/dl3dv_start_0_distance_20_ctx_2v_tgt_4v.json"
+            elif cfg_dict["dataset"]["view_sampler"]["num_context_views"] == 8:
+                eval_path = "assets/dl3dv_evaluation/dl3dv_start_0_distance_40_ctx_8v_tgt_8v.json"
+            elif cfg_dict["dataset"]["view_sampler"]["num_context_views"] == 16:
+                eval_path = "assets/dl3dv_evaluation/dl3dv_start_0_distance_80_ctx_16v_tgt_16v.json"
+            elif cfg_dict["dataset"]["view_sampler"]["num_context_views"] == 32:
+                eval_path = "assets/dl3dv_evaluation/dl3dv_start_0_distance_160_ctx_32v_tgt_24v.json"
+            elif cfg_dict["dataset"]["view_sampler"]["num_context_views"] == 64:
+                eval_path = "assets/dl3dv_benchmark/dl3dv_ctx_64v_tgt_every8th.json"
+            elif cfg_dict["dataset"]["view_sampler"]["num_context_views"] == -1:
+                print("Setting manually eval_path, num_context_views remains -1 for dl3dv eval")
+                eval_path = "assets/dl3dv_evaluation/dl3dv_start_0_distance_40_ctx_8v_tgt_8v.json"
+            else:
+                raise ValueError("unsupported number of views for dl3dv")
+        elif "scannet" in dataset_dir:
+            if cfg_dict["dataset"]["view_sampler"]["num_context_views"] == 2:
+                eval_path = "assets/evaluation_index_scannet_view2.json"
+            else:
+                raise ValueError("unsupported number of views for scannet")
+        elif "tartanair" in dataset_dir:
+            if cfg_dict["dataset"]["view_sampler"]["num_context_views"] == 2:
+                eval_path = 'assets/evaluation_index_tartanair_view2.json'
+            else:
+                raise ValueError("unsupported number of views for tartanair")
+        else:
+            raise Exception("Fail to load eval index path")
+        eval_cfg_dict["dataset"]["view_sampler"] = {
+            "name": "evaluation",
+            "index_path": eval_path,
+            "num_context_views": cfg_dict["dataset"]["view_sampler"]["num_context_views"],
+        }
+
+        # specify eval index
+        if meta_trainer_dict["eval_index"] is not None:
+            eval_cfg_dict["dataset"]["view_sampler"]["index_path"] = meta_trainer_dict["eval_index"]
+
+        eval_cfg = load_typed_root_config(eval_cfg_dict)
+    else:
+        eval_cfg = None
+    return eval_cfg

optgs/config/dataset/base.yaml

@@ -0,0 +1,8 @@
+image_shape: [0, 0]
+background_color: [0.0, 0.0, 0.0]
+cameras_are_circular: false
+overfit_to_scene: null
+opencv_pose_format: false
+pose_align_middle_view: false
+
+test_start_idx: 0

optgs/config/dataset/colmap.yaml

@@ -0,0 +1,12 @@
+defaults:
+  - base
+  - view_sampler: dense
+
+name: colmap
+roots: null
+scene_name: null
+normalize_world_space: false
+subsample_factor: 8
+symmetric_principal_point: false
+
+crop_size: null

optgs/config/dataset/dl3dv.yaml

@@ -0,0 +1,61 @@
+defaults:
+  - base
+  - view_sampler: boundedv2_360
+
+name: dl3dv
+roots: [datasets/dl3dv]
+make_baseline_1: false
+augment: true
+
+
+image_shape: [270, 480]
+
+baseline_epsilon: 1e-3
+max_fov: 100.0
+
+skip_bad_shape: true
+near: -1.
+far: -1.
+baseline_scale_bounds: false
+shuffle_val: true
+test_len: -1
+test_chunk_interval: 1
+sort_target_index: true
+sort_context_index: true
+
+train_times_per_scene: 1
+test_times_per_scene: 1
+ori_image_shape: [270, 480]
+overfit_max_views: 148
+use_index_to_load_chunk: false
+
+mix_tartanair: false
+no_mix_test_set: true
+load_depth: false
+center_pose: false
+
+pose_align_first_view: false
+
+scale_extrinsics: 1.
+metric_scale_align_dl3dv: false
+
+# view filtering
+min_views: 0
+max_views: 0
+highres: false
+
+# mix re10k & dl3dv
+mix_re10k: false
+re10k_min_view_dist: 40
+re10k_max_view_dist: 300
+
+# load remaining context views
+load_remain_context: false
+num_remain_context: 8
+
+# random crop in training
+random_crop: false
+min_size: null
+max_size: null
+
+index_name: index.json

optgs/config/dataset/re10k.yaml

@@ -0,0 +1,27 @@
+defaults:
+  - base
+  - view_sampler: bounded
+
+name: re10k
+roots: [datasets/re10k]
+make_baseline_1: false
+augment: true
+
+image_shape: [180, 320]
+highres: false
+
+baseline_epsilon: 1e-3
+max_fov: 100.0
+
+skip_bad_shape: true
+near: -1.
+far: -1.
+baseline_scale_bounds: true
+shuffle_val: true
+test_len: -1
+test_chunk_interval: 1
+
+use_index_to_load_chunk: false
+
+average_pose: false
+center_pose: false

optgs/config/dataset/scannet.yaml

@@ -0,0 +1,13 @@
+defaults:
+  - base
+  - view_sampler: ids
+
+name: scannet
+roots: datasets/quicksplat_spp_data_processed
+scene_name: null
+split: test
+subsample_factor: 1
+num_context_views: 100
+filter_bad_frames: true
+
+crop_size: null

optgs/config/dataset/view_sampler/all.yaml

@@ -0,0 +1 @@
+name: all

optgs/config/dataset/view_sampler/arbitrary.yaml

@@ -0,0 +1,7 @@
+name: arbitrary
+
+num_target_views: 1
+num_context_views: 2
+
+# If you want to hard-code context views, do so here.
+context_views: null

optgs/config/dataset/view_sampler/bounded.yaml

@@ -0,0 +1,12 @@
+name: bounded
+
+num_target_views: 1
+num_context_views: 2
+
+min_distance_between_context_views: 2
+max_distance_between_context_views: 6
+min_distance_to_context_views: 0
+
+warm_up_steps: 0
+initial_min_distance_between_context_views: 2
+initial_max_distance_between_context_views: 6

optgs/config/dataset/view_sampler/boundedv2.yaml

@@ -0,0 +1,15 @@
+name: boundedv2
+
+num_target_views: 1
+num_context_views: 2
+
+min_distance_between_context_views: 2
+max_distance_between_context_views: 6
+max_distance_to_context_views: 0
+
+context_gap_warm_up_steps: 0
+target_gap_warm_up_steps: 0
+
+initial_min_distance_between_context_views: 2
+initial_max_distance_between_context_views: 6
+initial_max_distance_to_context_views: 0

optgs/config/dataset/view_sampler/boundedv2_360.yaml

@@ -0,0 +1,17 @@
+name: boundedv2
+
+num_target_views: 4
+num_context_views: 4
+
+min_distance_between_context_views: 20
+max_distance_between_context_views: 50
+max_distance_to_context_views: 0
+
+context_gap_warm_up_steps: 10000
+target_gap_warm_up_steps: 0
+
+initial_min_distance_between_context_views: 15
+initial_max_distance_between_context_views: 30
+initial_max_distance_to_context_views: 0
+extra_views_sampling_strategy: farthest_point
+target_views_replace_sample: false

optgs/config/dataset/view_sampler/dense.yaml

@@ -0,0 +1,6 @@
+name: dense
+
+target_every: 8
+context_every: -1
+num_target_views: -1
+num_context_views: -1

optgs/config/dataset/view_sampler/evaluation.yaml

@@ -0,0 +1,4 @@
+name: evaluation
+
+index_path: assets/evaluation_index_re10k_video.json
+num_context_views: 2

optgs/config/dataset/view_sampler/ids.yaml

@@ -0,0 +1,4 @@
+name: ids
+
+context_views_ids: []
+target_views_ids: []

optgs/config/dataset/view_sampler_dataset_specific_config/bounded_re10k.yaml

@@ -0,0 +1,11 @@
+# @package _global_
+
+dataset:
+  view_sampler:
+    min_distance_between_context_views: 45
+    max_distance_between_context_views: 135
+    min_distance_to_context_views: 0
+    warm_up_steps: 30000
+    initial_min_distance_between_context_views: 25
+    initial_max_distance_between_context_views: 45
+    num_target_views: 4

optgs/config/dataset/view_sampler_dataset_specific_config/boundedv2_dl3dv.yaml

@@ -0,0 +1,14 @@
+# @package _global_
+
+dataset:
+  view_sampler:
+    min_distance_between_context_views: 20
+    max_distance_between_context_views: 50
+    max_distance_to_context_views: 0
+    context_gap_warm_up_steps: 10000 
+    target_gap_warm_up_steps: 0
+    initial_min_distance_between_context_views: 15
+    initial_max_distance_between_context_views: 30
+    initial_max_distance_to_context_views: 0
+    extra_views_sampling_strategy: farthest_point
+    num_target_views: 4

optgs/config/dataset/view_sampler_dataset_specific_config/evaluation_dl3dv.yaml

@@ -0,0 +1,5 @@
+# @package _global_
+
+dataset:
+  view_sampler:
+    index_path: assets/dl3dv_360_v5.json

optgs/config/dataset/view_sampler_dataset_specific_config/evaluation_re10k.yaml

@@ -0,0 +1,5 @@
+# @package _global_
+
+dataset:
+  view_sampler:
+    index_path: assets/evaluation_index_re10k.json

optgs/config/experiment/re10k_unified.yaml

@@ -0,0 +1,78 @@
+# @package _global_
+
+defaults:
+  - override /dataset: re10k
+  - override /scene_trainer/scene_initializer: resplat_v1
+  - override /scene_trainer/scene_optimizer: learn2splat
+  - override /loss: [mse, lpips]
+
+wandb:
+  name: re10k
+  tags: [re10k, 256x256]
+
+data_loader:
+  train:
+    batch_size: 14
+
+meta_trainer:
+  max_steps: 300_001
+  num_nodes: 1
+  test:
+    eval_time_skip_steps: 5
+    compute_scores: true
+    compute_scores_metrics: [psnr,ssim,lpips]
+    metrics_batch_size: 32
+
+scene_trainer:
+  initializer:
+    num_depth_candidates: 128
+    costvolume_unet_feat_dim: 128
+    costvolume_unet_channel_mult: [1,1,1]
+    costvolume_unet_attn_res: [4]
+    gaussians_per_pixel: 1
+    depth_unet_feat_dim: 32
+    depth_unet_attn_res: [16]
+    depth_unet_channel_mult: [1,1,1,1,1]
+    shim_patch_size: 16
+  use_fsdp: false
+  train_scene_init: false
+  train_scene_opt: false
+  num_update_steps: 0
+  iter_batch_size: -1
+  opt_batch_size: -1
+  train_min_refine: 0
+  train_max_refine: 0
+
+
+# lpips loss
+loss:
+  lpips:
+    apply_after_step: 0
+    weight: 0.5
+    perceptual_loss: true
+  deltas:
+    weight: 0.0
+    exclude_by_norm_grad: false
+    exclude_by_norm_grad_opposite: true
+    eps: 1e-8
+    apply_after_step: 10000000
+
+
+dataset:
+  image_shape: [256, 256]
+  roots: [datasets/re10k]
+  near: 0.01
+  far: 100.
+  baseline_scale_bounds: false
+  make_baseline_1: false
+  train_times_per_scene: 1
+  highres: false
+  scannet: false
+  tartanair: false
+  load_depth: false
+  pose_align_first_view: false
+  scale_extrinsics: 1.
+  load_remain_context: false
+  pose_align_middle_view: false
+  overfit_to_scene: null
+  opencv_pose_format: false

optgs/config/experiment/test_colmap.yaml

@@ -0,0 +1,32 @@
+# @package _global_
+
+defaults:
+  - override /dataset: colmap
+  - override /scene_trainer/scene_initializer: null    # overridden by init_opts.sh
+  - override /scene_trainer/scene_optimizer: null   # overridden by checkpoint (ours) or CLI (baselines)
+  - override /scene_trainer/decoder: gsplat
+  - override /loss: [mse]
+  - override /meta_trainer/test/postprocessing: none
+
+mode: test
+
+scene_trainer:
+  train_scene_init: false
+  train_scene_opt: false
+  opt_batch_strategy: fps
+
+checkpointing:
+  pretrained_model: null
+  pretrained_depth: null
+
+meta_trainer:
+  test:
+    compute_scores: true
+    skip_if_outputs_exist: true
+    save_cameras_json: false
+    save_render_image: false
+    save_gaussian: false
+    eval_initialization: false
+
+output_dir: placeholder
+log_slurm_id: true

optgs/config/experiment/test_dl3dv.yaml

@@ -0,0 +1,38 @@
+# @package _global_
+
+defaults:
+  - override /dataset: dl3dv
+  - override /scene_trainer/scene_initializer: null   # overridden by checkpoint (ours) or init_opts.sh
+  - override /scene_trainer/scene_optimizer: null   # overridden by checkpoint (ours) or CLI (baselines)
+  - override /scene_trainer/decoder: gsplat
+  - override /meta_trainer/test/postprocessing: none
+
+mode: test
+
+dataset:
+  roots: [datasets/dl3dv-480p-chunks]
+  near: 0.01
+  far: 200.
+  opencv_pose_format: false
+  image_shape: [256, 448]
+
+scene_trainer:
+  train_scene_init: false
+  train_scene_opt: false
+  opt_batch_strategy: fps
+
+checkpointing:
+  pretrained_model: null
+  pretrained_depth: null
+
+meta_trainer:
+  test:
+    compute_scores: true
+    skip_if_outputs_exist: false
+    save_cameras_json: false
+    save_render_image: false
+    save_gaussian: false
+    eval_initialization: false
+
+output_dir: placeholder
+log_slurm_id: true

optgs/config/experiment/test_re10k.yaml

@@ -0,0 +1,36 @@
+# @package _global_
+
+defaults:
+  - override /dataset: re10k
+  - override /scene_trainer/scene_initializer: resplat_v1
+  - override /scene_trainer/scene_optimizer: knn_based
+  - override /scene_trainer/decoder: gsplat
+  - override /loss: [mse]
+  - override /meta_trainer/test/postprocessing: none
+
+mode: test
+
+dataset:
+  image_shape: [512, 960]
+  ori_image_shape: [512, 960]
+
+scene_trainer:
+  train_scene_init: false
+  train_scene_opt: false
+  opt_batch_strategy: fps
+
+checkpointing:
+  pretrained_model: null
+  pretrained_depth: null
+
+meta_trainer:
+  test:
+    compute_scores: true
+    skip_if_outputs_exist: true
+    save_cameras_json: false
+    save_render_image: false
+    save_gaussian: false
+    eval_initialization: false
+
+output_dir: placeholder
+log_slurm_id: true

optgs/config/experiment/train_dl3dv.yaml

@@ -0,0 +1,55 @@
+# @package _global_
+
+# A shared config for training on dl3dv, used by both resplat initializer, resplat optimizer, and learn2splat optimizer.
+
+defaults:
+  - override /dataset: dl3dv
+  - override /scene_trainer/scene_initializer: resplat_v1
+  - override /scene_trainer/scene_optimizer: learn2splat
+  - override /loss: [ mse, lpips ]
+  - override /dataset/view_sampler: boundedv2_360
+
+wandb:
+  name: dl3dv
+  tags: [ dl3dv, 270x480 ]
+
+data_loader:
+  train:
+    batch_size: 1
+
+meta_trainer:
+  max_steps: 50_000
+  val_check_interval: 0.25
+  train:
+    l1_loss: true
+    depth_smooth_loss_weight: 0.0
+  test:
+    eval_time_skip_steps: 0
+    dec_chunk_size: 30
+    save_every_freq: [ 1, 10, 100, 500 ]
+    save_every_steps: [ 0, 10, 100, 1000 ]
+
+# lpips loss
+loss:
+  lpips:
+    apply_after_step: 0
+    weight: 0.5
+    perceptual_loss: true
+
+dataset:
+  roots: [ datasets/dl3dv-480p-chunks ]
+  near: 0.01
+  far: 200.
+  min_size: [ 384,512 ]
+  max_size: [ 512,960 ]
+  image_shape: [ 256, 448 ]
+  view_sampler:
+    num_context_views: 8
+    num_target_views: 6
+    min_distance_between_context_views: 24
+    max_distance_between_context_views: 45
+    initial_min_distance_between_context_views: 20
+    initial_max_distance_between_context_views: 30
+
+output_dir: placeholder
+log_slurm_id: true

optgs/config/experiment/train_l2s_sparse_dl3dv.yaml

@@ -0,0 +1,41 @@
+# @package _global_
+
+defaults:
+  - train_dl3dv
+  - override /meta_trainer/train/replay_buffer_cfg: default
+  - override /loss: [ mse, lpips, deltas ]
+
+loss:
+  mse:
+    weight: 1.0
+  lpips:
+    apply_after_step: 0
+    weight: 0.5
+    perceptual_loss: true
+  deltas:
+    weight: 1
+    exclude_by_norm_grad: true
+    exclude_by_norm_grad_opposite: true
+    eps: 1e-8
+    apply_after_step: 100
+
+meta_trainer:
+  train:
+    loss_on_input_views: true
+    loss_on_input_views_num: 4
+    use_replay_buffer: true
+
+scene_trainer:
+  train_scene_opt: true
+  num_update_steps: 4
+  train_max_refine: 6
+  train_min_refine: 1
+
+meta_optimizer:
+  lr: 1e-4
+  lr_monodepth: 0.0
+
+
+checkpointing:
+  pretrained_initializer: checkpoints/optgs/unified-dl3dv-8views/init/checkpoints/epoch_20-step_100000.ckpt  # resplat inititalizer
+  no_strict_load: false

optgs/config/experiment/train_l2s_sparse_dl3dv_no_delta.yaml

@@ -0,0 +1,35 @@
+# @package _global_
+
+defaults:
+  - train_dl3dv
+  - override /meta_trainer/train/replay_buffer_cfg: default
+  - override /loss: [ mse, lpips ]
+
+loss:
+  mse:
+    weight: 1.0
+  lpips:
+    apply_after_step: 0
+    weight: 0.5
+    perceptual_loss: true
+
+meta_trainer:
+  train:
+    loss_on_input_views: true
+    loss_on_input_views_num: 4
+    use_replay_buffer: true
+
+scene_trainer:
+  train_scene_opt: true
+  num_update_steps: 4
+  train_max_refine: 6
+  train_min_refine: 1
+
+meta_optimizer:
+  lr: 1e-4
+  lr_monodepth: 0.0
+
+
+checkpointing:
+  pretrained_initializer: checkpoints/optgs/unified-dl3dv-8views/init/checkpoints/epoch_20-step_100000.ckpt  # resplat inititalizer
+  no_strict_load: false

optgs/config/experiment/train_l2s_sparse_dl3dv_no_loss.yaml

@@ -0,0 +1,35 @@
+# @package _global_
+
+defaults:
+  - train_dl3dv
+  - override /meta_trainer/train/replay_buffer_cfg: default
+  - override /loss: [ mse, lpips ]
+
+loss:
+  mse:
+    weight: 1.0
+  lpips:
+    apply_after_step: 0
+    weight: 0.5
+    perceptual_loss: true
+
+meta_trainer:
+  train:
+    loss_on_input_views: true
+    loss_on_input_views_num: 4
+    use_replay_buffer: true
+
+scene_trainer:
+  train_scene_opt: true
+  num_update_steps: 4
+  train_max_refine: 6
+  train_min_refine: 1
+
+meta_optimizer:
+  lr: 1e-4
+  lr_monodepth: 0.0
+
+
+checkpointing:
+  pretrained_initializer: checkpoints/optgs/unified-dl3dv-8views/init/checkpoints/epoch_20-step_100000.ckpt  # resplat inititalizer
+  no_strict_load: false

optgs/config/loss/deltas.yaml

@@ -0,0 +1,6 @@
+deltas:
+  weight: 1.0
+  exclude_by_norm_grad: false
+  exclude_by_norm_grad_opposite: true
+  eps: 0.1
+  apply_after_step: 100

optgs/config/loss/gaussians.yaml

@@ -0,0 +1,6 @@
+gaussians:
+  weight: 1.0
+  weight_scales: 0.01
+  weight_opacities: 0.0
+  weight_sh: 0.005
+  sh_alpha: 1.0  # 1.0 = uniform; >1.0 = penalize higher SH degrees more

optgs/config/loss/iso_scales.yaml

@@ -0,0 +1,2 @@
+iso_scales:
+  weight: 1.0

optgs/config/loss/lpips.yaml

@@ -0,0 +1,4 @@
+lpips:
+  weight: 0.05
+  apply_after_step: 150_000
+  perceptual_loss: false

optgs/config/loss/mse.yaml

@@ -0,0 +1,2 @@
+mse:
+  weight: 1.0

optgs/config/loss/sgd.yaml

@@ -0,0 +1,2 @@
+sgd:
+  weight: 1.0

optgs/config/loss/sh0.yaml

@@ -0,0 +1,2 @@
+mse:
+  weight: 1.0

optgs/config/loss/ssim.yaml

@@ -0,0 +1,2 @@
+ssim:
+  weight: 0.2  # default in 3dgs

optgs/config/loss/stability.yaml

@@ -0,0 +1,2 @@
+stability:
+  weight: 1.0

optgs/config/main.yaml

@@ -0,0 +1,195 @@
+defaults:
+  - loss: [ mse ]
+  - dataset: null
+  - scene_trainer/scene_initializer: null
+  - scene_trainer/scene_optimizer: null
+  - scene_trainer/decoder: gsplat
+  - meta_trainer/test/postprocessing: none
+  - meta_trainer/train/replay_buffer_cfg: none
+
+wandb:
+  project: placeholder
+  entity: placeholder
+  name: placeholder
+  mode: online
+  id: null
+  notes: null
+
+mode: train
+
+data_loader:
+  train:
+    num_workers: 10
+    persistent_workers: true
+    batch_size: 4
+    seed: 1234
+  test:
+    num_workers: 4
+    persistent_workers: false
+    batch_size: 1
+    seed: 2345
+  val:
+    num_workers: 1
+    persistent_workers: true
+    batch_size: 1
+    seed: 3456
+
+meta_optimizer:
+  lr: 2.e-4
+  lr_monodepth: 2.e-6
+  lr_depth: 0.
+  warm_up_steps: 2000
+  weight_decay: 0.01
+  warm_up_ratio: 0.01
+  adamw_8bit: false
+
+checkpointing:
+  load: null
+  every_n_train_steps: 1000
+  save_top_k: 5
+  pretrained_model: null
+  pretrained_model_rel_dir: ${checkpoint_rel_dir:${checkpointing.pretrained_model}}
+  pretrained_monodepth: null
+  pretrained_mvdepth: null
+  pretrained_depth: null
+  pretrained_scale_predictor: null
+  pretrained_depth_teacher: null
+  no_strict_load: false
+  resume: false
+  no_resume_upsampler: false
+  partial_load: false
+  freeze_mono_vit: false
+  resume_update_module: null
+  pretrained_initializer: null
+  pretrained_optimizer: null
+  load_existing_cfg: false
+
+seed: 111123
+
+meta_trainer:
+  max_steps: -1
+  val_check_interval: 0.5
+  gradient_clip_val: 0.5
+  num_sanity_val_steps: 2
+  eval_index: null
+  limit_test_batches: 1.0
+  limit_train_batches: 1.0
+  num_nodes: 1
+  train:
+    depth_mode: null
+    extended_visualization: false
+    print_log_every_n_steps: 100
+    eval_model_every_n_val: 2  # quantitative evaluation every n val
+    eval_data_length: 999999
+    eval_deterministic: false
+    eval_time_skip_steps: 3
+    eval_save_model: true
+    l1_loss: false
+    intermediate_loss_weight: 0.9
+    no_viz_video: false
+    eval_depth: false
+    train_ignore_large_loss: 0.
+    no_log_projections: true
+    no_log_video: true
+    depth_loss_weight: 0.
+    log_depth_loss: true
+    depth_smooth_loss_weight: 0.01
+    depth_smooth_loss_nonorm: false
+    depth_smooth_loss_weight_nvs: 0. # for novel views
+    monodepth_loss_weight: 0. # for monocular depth loss
+    depth_teacher_loss_weight: 0.
+    viz_depth_teacher: false
+    eval_render_depth: false
+    render_depth_loss_weight: 0.
+    viz_render_depth: false
+    use_gt_depth_range: false
+    depth_range_from_disparity: false
+    max_disparity: 128.
+    min_disparity: 4.
+    loss_on_input_views: false
+    loss_on_target_views: true
+    loss_on_input_views_num: 1
+    loss_on_target_views_num: -1
+    train_window_size: null
+    half_res_lpips_loss: false
+    viz_depth_separate: false
+    # L2 weight decay on Gaussian properties (meta-loss)
+    scale_l2_loss_weight: 0.
+    sh_l2_loss_weight: 0.
+    opacity_l2_loss_weight: 0.
+    use_replay_buffer: false
+  test:
+    output_path: null
+    compute_scores: true
+    compute_scores_metrics: [psnr,ssim,lpips]
+    metrics_batch_size: 32
+    eval_time_skip_steps: 0
+    eval_initialization: true
+    save_render_image: false
+    save_render_image_last_only: false
+    save_gt_image: false
+    save_render_depth: false
+    save_gt_depth: false
+    save_error_image: false
+    save_video: false
+    save_video_fixed_view: false
+    save_video_fixed_view_index: 0
+    save_video_fixed_view_duplicate: 0
+    save_video_fixed_iteration: false
+    save_video_fixed_iteration_indices: null
+    save_video_fixed_iteration_render_fixed_view: false
+    save_video_combined: false
+    save_video_combined_iterations: null
+    save_video_combined_fixed_iteration_length: 50
+    save_gaussian: false
+    save_poses: false
+    save_cameras_json: true
+    save_cameras_npz: true
+    save_point_cloud: false
+    render_chunk_size: null
+    dec_chunk_size: null
+    stablize_camera: false
+    stab_camera_kernel: 50
+    eval_context_views: false
+    inference_window_size: null
+    profile_model: false
+    save_colmap_train_test_views: false
+    ori_colmap_data_path: null
+    adam_optimizer_step: 0
+    save_at_iters: null
+    save_every_freq: null
+    save_every_steps: null
+    skip_if_outputs_exist: false
+    scenes_filter: null
+
+    experimental_add_noise_to_images: false
+    experimental_add_noise_to_images_std: null
+
+scene_trainer:
+  use_fsdp: false
+  train_scene_init: false
+  train_scene_opt: false
+  train_min_refine: 0
+  train_max_refine: 0
+  num_update_steps: 0
+  iter_batch_size: -1
+  opt_batch_size: -1
+  opt_batch_size_min: 0
+  opt_batch_size_max: 0
+  opt_batch_strategy: random
+  sh_degree_interval: 0
+
+output_dir: null
+
+use_plugins: false
+
+log_slurm_id: false
+
+version: null
+
+profiling:
+  # one of: none, basic, advanced, pytorch
+  # advanced profiling requires pytorch-lightning-2.5.3 (default: 2.4.0)
+  mode: none
+
+debug_cfg: false

optgs/config/meta_trainer/test/postprocessing/adam.yaml

@@ -0,0 +1,10 @@
+defaults:
+  - base
+
+name: adam
+lr_data:
+  _base: 0.001
+betas: [0.9, 0.999]
+weight_decay: 0.0
+amsgrad: false
+eps: 1e-08

optgs/config/meta_trainer/test/postprocessing/base.yaml

@@ -0,0 +1,24 @@
+steps: 2000
+compute_metrics_every: 100
+lr_data:
+  _base: 1
+  _means: 1
+  _scales: 1
+  _opacities: 1
+  _quats: 1
+  _sh0: 1
+  _shN: 1
+scheduler: null
+scheduler_warm_up_ratio: 0.01
+prior_steps: 0
+
+# Means LR scheduling (defaults match vanilla optimizer)
+means_lr_final_ratio: 0.0625    # ratio of final/initial means LR (vanilla: 1e-5 / 1.6e-4)
+means_lr_delay_mult: 0.01       # ramp-up delay multiplier (vanilla default)
+means_lr_scale_by_scene_extent: true
+
+# View chunking for gradient accumulation
+chunk_size: -1  # -1 = all views at once
+
+# ADC (Adaptive Density Control) - null = disabled
+adc: null

optgs/config/meta_trainer/test/postprocessing/none.yaml

@@ -0,0 +1,5 @@
+defaults:
+  - base
+
+name: none
+steps: 0

optgs/config/meta_trainer/test/postprocessing/sgd.yaml

@@ -0,0 +1,7 @@
+defaults:
+  - base
+
+name: sgd
+momentum: 0.0
+weight_decay: 0.0
+nesterov: false

optgs/config/meta_trainer/test/postprocessing/vanilla_3dgs.yaml

@@ -0,0 +1,12 @@
+defaults:
+  - base
+  - adam
+
+lr_data:
+  _base: 1
+  _means: 1.6e-4
+  _scales: 5e-3
+  _opacities: 5e-2
+  _quats: 1e-3
+  _sh0: 2.5e-3
+  _shN: 1.25e-4 #  2.5e-3 / 20

optgs/config/meta_trainer/test/postprocessing/vanilla_3dgs_sgd.yaml

@@ -0,0 +1,12 @@
+defaults:
+  - base
+  - sgd
+
+lr_data:
+  _base: 1
+  _means: 1.6e-4
+  _scales: 5e-3
+  _opacities: 5e-2
+  _quats: 1e-3
+  _sh0: 2.5e-3
+  _shN: 1.25e-4 #  2.5e-3 / 20

optgs/config/meta_trainer/train/replay_buffer_cfg/default.yaml

@@ -0,0 +1,12 @@
+capacity: 20
+sample_batch_size: 1
+sample_prob: 0.7
+insert_prob: 0.7
+return_prob: 0.99
+simulate_ahead: true
+simulate_ahead_min_steps: 1
+simulate_ahead_max_steps: 50
+simulate_ahead_grow: 10000
+max_t: null
+push_only_if_not_full: false
+remove_strategy_when_full: oldest