src/schema.py

"""Canonical UI vocabulary for the Video Benchmark.

Single source of truth for everything the Gradio app needs to *describe*
a column, option, parameter group, leaderboard ranking, or About-page
narration. ``app.py`` imports these constants directly to wire up its
filter chips, dropdowns, table headers, and prose blocks; adding or
renaming a knob means editing exactly one file.

Each ``COLUMNS`` entry carries display metadata plus, for numeric
columns, a printf-style ``fmt_spec`` (e.g. ``"%.2f"``) so values render
consistently across the table and tooltips.
"""
from __future__ import annotations

from typing import Any

from dataclasses import dataclass


@dataclass(frozen=True)
class BenchmarkConfig:
    """Everything the UI needs to render one benchmark's tabs.

    A benchmark bundles its Hub datasets, column metadata, filter-chip
    vocabularies, leaderboard configuration, Submit-form options, and
    About/Parameters prose. ``app.py`` builds one tab set per benchmark
    from this config, so adding a benchmark means adding one instance.
    """
    key: str
    label: str
    title: str
    subtitle: str
    results_dataset: str
    submissions_dataset: str
    columns: list[dict[str, Any]]
    repos: list[str]
    filter_options: dict[str, list[str]]   # chip key -> options (UI vocabulary)
    filter_order: list[str]                 # chip render order (keys of filter_options)
    leaderboard_axes: list[str]
    leaderboard_cats: dict[str, dict[str, Any]]
    leaderboard_group_keys: tuple[str, ...]
    submit_options: dict[str, list[str]]
    submit_defaults: dict[str, Any]
    full_sweep: dict[str, Any]
    param_groups: list[dict[str, Any]]
    param_notes: dict[str, dict[str, str]]
    about_html: str
    keep_keys: tuple[str, ...]
    scatter_x: str                          # Compare scatter x metric key
    scatter_y: str                          # Compare scatter y metric key
    codec_count: int                        # for the hero stats row
    backend_count: int


# --------------------------------------------------------------------------- #
# Datasets the UI talks to. Mirrored as constants so the frontend can render
# friendly links without duplicating strings.
# --------------------------------------------------------------------------- #
RESULTS_DATASET = "lerobot/video-benchmark-results"
SUBMISSIONS_DATASET = "lerobot/video-benchmark-submissions"
RESULTS_DATASET_DEPTH = "lerobot/depth-benchmark-results"


# --------------------------------------------------------------------------- #
# Column metadata
#
# Order matters: it drives table column order, the Column-picker grouping,
# Parameters-page ordering, and Compare-tab dropdowns. ``metric=True`` flags
# numeric columns that participate in composite ranking and color ramps;
# ``lower=True`` / ``higher=True`` set the polarity.
# --------------------------------------------------------------------------- #
COLUMNS: list[dict[str, Any]] = [
    # Config
    {"key": "repo_id", "label": "Dataset", "short": "Dataset", "group": "Config",
     "desc": "Hugging Face Hub dataset repo ID. One representative episode per dataset."},
    {"key": "vcodec", "label": "Codec", "short": "Codec", "group": "Config",
     "desc": "Video codec. Generic codecs (h264, hevc, av1) are supported, or you can pin a specific encoding library (e.g. libsvtav1)."},
    {"key": "encoder", "label": "Encoder", "short": "Encoder", "group": "Config", "default_hidden": True,
     "desc": "Encoding library used to produce the video."},
    {"key": "decoder", "label": "Decoder", "short": "Decoder", "group": "Config", "default_hidden": True,
     "desc": "Decoding library used to read the video."},
    {"key": "pix_fmt", "label": "Pixel format", "short": "Pixel format", "group": "Config",
     "desc": "Pixel format. RGB uses three-channel 8-bit (uint8) formats (e.g. yuv420p, yuv444p)."},
    {"key": "g", "label": "GOP", "short": "GOP", "group": "Config",
     "desc": "Group Of Pictures — the keyframe interval. A small value keeps seeks cheap but makes files bigger, while a large value compresses better at the cost of expensive random access."},
    {"key": "crf", "label": "CRF", "short": "CRF", "group": "Config",
     "desc": "Constant Rate Factor — the quality knob: lower gives bigger files and higher fidelity, higher gives smaller files and more loss."},
    {"key": "timestamps_mode", "label": "Access pattern", "short": "Access", "group": "Config",
     "desc": "How frames are requested. 1_frame: seek, decode one frame, done. 2_frames: two adjacent frames. 6_frames: contiguous window. 2_frames_4_space: two samples four frames apart — worst-case for GOP-heavy settings."},
    {"key": "backend", "label": "Backend", "short": "Backend", "group": "Config",
     "desc": "Video decoding library (pyav, torchcodec)."},

    # Compression
    {"key": "video_images_size_ratio", "label": "Video/Image\nsize ratio ↓", "short": "Video/Image size ratio",
     "group": "Compression", "metric": True, "lower": True, "fmt_spec": "%.4f",
     "desc": "Encoded video size ÷ sum of original PNG image sizes. Lower means better compression — the video takes less disk than the raw frames."},
    {"key": "video_images_load_time_ratio", "label": "Video/Image\nload ratio ↓", "short": "Video/Image load ratio",
     "group": "Compression", "metric": True, "lower": True, "fmt_spec": "%.4f",
     "desc": "Video decoding time ÷ PNG image load time for the same frames. Below 1 means the video decodes faster than reading the raw frames; above 1 means you pay for the compression at read time."},

    # Speed
    {"key": "median_load_time_video_ms", "label": "Decoding (ms) ↓", "short": "Decoding",
     "group": "Speed", "metric": True, "lower": True, "fmt_spec": "%.2f", "std_key": "std_load_time_video_ms",
     "desc": "Median wall-clock time to decode N frames from the compressed video."},
    {"key": "encoding_time_ms", "label": "Encoding (ms) ↓", "short": "Encoding",
     "group": "Speed", "metric": True, "lower": True, "fmt_spec": "%.2f",
     "desc": "Wall time to encode the whole episode once."},
    {"key": "encoding_fps", "label": "Encoding (fps) ↑", "short": "Encoding fps",
     "group": "Speed", "metric": True, "higher": True, "fmt_spec": "%.1f",
     "desc": "Encoding throughput — frames per second across the whole episode."},

    # Quality
    {"key": "median_psnr", "label": "PSNR (dB) ↑", "short": "PSNR",
     "group": "Quality", "metric": True, "higher": True, "fmt_spec": "%.2f", "std_key": "std_psnr",
     "desc": "Peak signal-to-noise ratio in dB. Higher means the decoded frame is closer to the source, i.e. less reconstruction error."},
    {"key": "median_ssim", "label": "SSIM (0–1) ↑", "short": "SSIM",
     "group": "Quality", "metric": True, "higher": True, "fmt_spec": "%.4f", "std_key": "std_ssim",
     "desc": "Structural similarity index in [0,1]. Higher means the decoded frame better preserves the source's structure (1 = identical)."},
    {"key": "median_mse", "label": "MSE (px²) ↓", "short": "MSE",
     "group": "Quality", "metric": True, "lower": True, "fmt_spec": "%.2f", "std_key": "std_mse",
     "desc": "Mean-squared error between decoded and original frames (px²). Lower means less pixel error (0 = identical)."},
]

# Qualitative palette for leaderboard rank badges + radar polygons.
# Muted on purpose — the cards overlay four to twelve colors on the
# same card surface (and the radar stacks the same count on one axis
# system), so a saturated HF-brand palette reads as visual noise.
# These are chromatic neighbours of the brand hues, pulled 20–30%
# toward gray, so they stay recognizable next to the brand palette
# without fighting it for attention.
PODIUM_COLORS: list[str] = [
    "#6B8EBF",  # dusty blue
    "#D99B5A",  # warm sand
    "#7FA97E",  # sage
    "#A98FBF",  # muted lavender
    "#C98595",  # dusty rose
    "#6FA3A3",  # dusty teal
    "#B8A773",  # soft olive
    "#8A95A8",  # slate
    "#D1A663",  # honey
    "#7FA3B8",  # powder blue
    "#B88A73",  # terracotta
    "#8F9F8A",  # moss
]

# Codec → brand-aligned but desaturated accent color. Shared by the
# leaderboard cards (codec pill), the leaderboard dataframe (codec
# column chip), and anywhere else codec identity is rendered. Keys
# are lowercased for case-insensitive lookup.
CODEC_COLORS: dict[str, str] = {
    "h264": "#6B8EBF",       # dusty blue (mirrors HF blue)
    "hevc": "#D99B5A",       # warm sand (mirrors HF orange)
    "libsvtav1": "#7FA97E",  # sage (mirrors HF green)
}
CODEC_FALLBACK_COLOR: str = "#8A95A8"

STACK_COLORS: list[str] = [
    "#6B8EBF", "#D99B5A", "#7FA97E", "#A98FBF", "#C98595", "#6FA3A3",
]


# --------------------------------------------------------------------------- #
# Filter-bar option lists (used by the Results tab's chip filters).
# These define the *UI vocabulary*; actual row values come from the Hub
# dataset and may be a subset.
# --------------------------------------------------------------------------- #
REPOS: list[str] = [
    "lerobot/pusht_image",
    "lerobot/aloha_mobile_shrimp_image",
    "lerobot/paris_street",
    "lerobot/kitchen",
]
VCODECS: list[str] = ["h264", "hevc", "libsvtav1"]
PIX_FMTS: list[str] = ["yuv444p", "yuv420p"]
G_VALUES: list[int] = [ 2, 3, 4, 5, 6, 10, 15, 20, 40]
CRF_VALUES: list[int] = [0, 5, 10, 15, 20, 25, 30, 40, 50]
TS_MODES: list[str] = ["1_frame", "2_frames", "2_frames_4_space", "6_frames"]
BACKENDS: list[str] = ["pyav", "torchcodec"]


# --------------------------------------------------------------------------- #
# Submit-form option lists. Wider than the filter-bar lists because the
# Submit form lets users *queue* sweeps over knobs that aren't represented
# in current results.
# --------------------------------------------------------------------------- #
SUBMIT_OPTIONS: dict[str, list[str]] = {
    "repos": REPOS,
    "vcodecs": VCODECS,
    "pix_fmts": PIX_FMTS,
    "g": ["1", "2", "3", "4", "5", "6", "10", "15", "20", "40", "100"],
    "crf": ["0", "5", "10", "15", "20", "25", "30", "40", "50"],
    "timestamps_modes": TS_MODES,
    "backends": BACKENDS,
}

SUBMIT_DEFAULTS: dict[str, Any] = {
    "repos": REPOS[:2],
    "vcodecs": ["h264"],
    "pix_fmts": ["yuv420p"],
    "g": ["2", "10"],
    "crf": ["10", "30"],
    "timestamps_modes": ["1_frame", "2_frames"],
    "backends": ["pyav"],
    "samples_per_config": 50,
}


# --------------------------------------------------------------------------- #
# Maintainer "full sweep" — every curated knob, full Cartesian product. The
# Submit tab exposes this behind a collapsed accordion (no public link). The
# resulting submission size (~tens of thousands of configs) intentionally
# blows past ``MAX_CONFIGS_PER_SUBMISSION``; the corresponding handler in
# ``app.py`` bypasses that cap because this path is gated behind a confirm
# checkbox and is meant to be triggered only by maintainers re-baselining
# the leaderboard.
# --------------------------------------------------------------------------- #
FULL_SWEEP: dict[str, Any] = {
    "repos": list(REPOS),
    "vcodecs": list(VCODECS),
    "pix_fmts": list(PIX_FMTS),
    "g": [str(v) for v in G_VALUES],
    "crf": [str(v) for v in CRF_VALUES],
    "timestamps_modes": list(TS_MODES),
    "backends": list(BACKENDS),
    "samples_per_config": 50,
}


# --------------------------------------------------------------------------- #
# Leaderboards
# --------------------------------------------------------------------------- #
# All categories share the same six axes — what changes between tabs is
# the *weighting* used to rank configurations.
LEADERBOARD_AXES: list[str] = [
    "encoding_time_ms",
    "median_load_time_video_ms",
    "video_images_size_ratio",
    "median_mse",
    "median_psnr",
    "median_ssim",
]

LEADERBOARD_CATS: dict[str, dict[str, Any]] = {
    "Overall": {
        "desc": "Balanced across encoding, decoding, size, and quality.",
        "weights": {
            "encoding_time_ms": 1, "median_load_time_video_ms": 1,
            "video_images_size_ratio": 1, "median_mse": 1,
            "median_psnr": 1, "median_ssim": 1,
        },
    },
    "Quality": {
        "desc": "Pure reconstruction fidelity: MSE, PSNR, SSIM only.",
        "weights": {
            "encoding_time_ms": 0, "median_load_time_video_ms": 0,
            "video_images_size_ratio": 0, "median_mse": 1,
            "median_psnr": 1, "median_ssim": 1,
        },
    },
    "Encoding": {
        "desc": "Encoding throughput and output size, fidelity ignored.",
        "weights": {
            "encoding_time_ms": 1, "median_load_time_video_ms": 0,
            "video_images_size_ratio": 1, "median_mse": 0,
            "median_psnr": 0, "median_ssim": 0,
        },
    },
    "Decoding": {
        "desc": "Pure decoding latency at the selected access pattern.",
        "weights": {
            "encoding_time_ms": 0, "median_load_time_video_ms": 1,
            "video_images_size_ratio": 0, "median_mse": 0,
            "median_psnr": 0, "median_ssim": 0,
        },
    },
}


# --------------------------------------------------------------------------- #
# Parameters reference (drives the About tab's "Parameters reference" pane)
# --------------------------------------------------------------------------- #
PARAM_GROUPS: list[dict[str, Any]] = [
    {"t": "Inputs — what you benchmark",
     "desc": "The corpus. Each dataset is a LeRobot episode recording with a few minutes of RGB observations.",
     "keys": ["repo_id"]},
    {"t": "Encoding — how the video is compressed",
     "desc": "Passed to the FFmpeg/PyAV encoder. These are the knobs operators actually tune.",
     "keys": ["vcodec", "pix_fmt", "g", "crf", "encoder_threads"]},
    {"t": "Decoding — how you read it back",
     "desc": "The other half of the equation. The same MP4 can decode very differently depending on library and access pattern.",
     "keys": ["backend", "timestamps_mode"]},
    {"t": "Fidelity metrics — how faithful is decoded frame vs. source",
     "desc": "We decode the compressed video, re-read the original PNG frames, and compare pixel-for-pixel.",
     "keys": ["median_mse", "median_psnr", "median_ssim"]},
    {"t": "Performance metrics — how fast, how small",
     "desc": "The cost side. Run on a single CPU thread unless noted.",
     "keys": ["encoding_time_ms", "encoding_fps", "video_images_size_ratio", "median_load_time_video_ms", "video_images_load_time_ratio"]},
]

# Reference-only parameters: documented on the About page's parameter
# reference but not part of the results data, so they have no column,
# filter, or Compare entry.
PARAM_NOTES: dict[str, dict[str, str]] = {
    "encoder_threads": {
        "label": "Encoder threads",
        "desc": "CPU threads used to encode a clip. Local-only — the hosted benchmark pins encoding to one thread for comparable timings, so it is not submittable here.",
    },
}


# --------------------------------------------------------------------------- #
# About-page prose. Authored here so the React layer just renders it. The
# code snippet uses literal newlines; React renders it inside a <pre>.
# --------------------------------------------------------------------------- #
ABOUT_HTML: str = """\
<h3 id="what">What this is</h3>
<p>
A benchmark for video encoding and decoding in the context of robotics datasets. LeRobot stores episode observations as MP4 rather than PNG sequences — this page quantifies <i>how much</i> we gain in size and what we pay in decoding latency and pixel fidelity.
</p>

<p>
<h3 id="metrics">Metrics</h3>
<p>The size and speed metrics measure the cost of storing and reading frames as video; the fidelity metrics measure what you lose to lossy compression.</p>
<p><b>Size &amp; speed</b></p>
<ul>
<li><b>Video/image size ratio</b> — encoded video ÷ sum of PNG frames. Lower means better compression — the video takes less disk than the raw frames.</li>
<li><b>Video/image load ratio</b> — video decoding time ÷ PNG image load time for the same frames. <1 means the video is faster to read than the PNGs; >1 means you pay for the compression at read time.</li>
<li><b>Decoding time</b> — median wall-clock to decode N frames at a given timestamp.</li>
<li><b>Encoding time</b> — wall-clock to encode the whole clip, end-to-end.</li>
<li><b>Encoding fps</b> — encode throughput in frames per second, derived from the encoding time and the episode length. Higher means faster encoding; the handy complement to the raw time column.</li>
</ul>
<p><b>Fidelity</b> — every decoded frame is compared against the uncompressed source, pixel-for-pixel.</p>
<ul>
<li><b>PSNR (dB)</b> — peak signal-to-noise ratio in dB. Logarithmic, so +3 dB ≈ half the error. 40+ is excellent, 30 is acceptable, 20 is visible artefacts. Higher is closer to the source.</li>
<li><b>SSIM (0–1)</b> — structural similarity index, dimensionless. Perceptual — weights luminance, contrast, structure. 0.95+ is good, 0.80 is degraded. Higher is structure better preserved.</li>
<li><b>MSE (px²)</b> — mean-squared error in squared 8-bit pixel intensities (0–65025). Lower is less pixel error; 0 = identical.</li>
</ul>
</p>

<p>
<h3 id="access">Access patterns</h3>
<p>
Decode cost depends heavily on <i>how</i> frames are requested. <code>1_frame</code> pays the full seek+IDR cost per sample; <code>6_frames</code> amortizes it across contiguous frames; <code>2_frames_4_space</code> probes the worst case where the decoder must step across two distant windows.
</p>

<h3 id="repro">Reproduce locally</h3>
<p>The full sweep is open source. First, set <code>HF_RESULTS_REPO_ID</code> to the Hugging Face Hub dataset where you want results pushed:</p>
<pre style="background:var(--hf-gray-900);color:var(--hf-gray-100);padding:var(--space-4);border-radius:var(--radius-md);font-size:var(--fs-xs);overflow:auto">export HF_RESULTS_REPO_ID=lerobot/video-benchmark-results</pre>
<p>Then run the benchmark:</p>
<pre style="background:var(--hf-gray-900);color:var(--hf-gray-100);padding:var(--space-4);border-radius:var(--radius-md);font-size:var(--fs-xs);overflow:auto">python benchmark/video/run_video_benchmark.py \\
  --output-dir outputs/video_benchmark \\
  --repo-ids lerobot/pusht_image lerobot/kitchen \\
  --vcodec h264 hevc libsvtav1 \\
  --pix-fmt yuv420p yuv444p \\
  --g 2 10 40 \\
  --crf 10 20 30 \\
  --timestamps-modes 1_frame 2_frames 6_frames \\
  --backends pyav torchcodec \\
  --num-samples 50</pre>

<h3 id="contrib">Contribute</h3>
<p>
Submit your own configurations through the <b>Submit</b> tab. A background worker picks them up and pushes results to the Hub so the whole community benefits from the same measurements.
</p>
"""


RGB_KEEP_KEYS: tuple[str, ...] = (
    "repo_id", "vcodec", "encoder", "decoder", "pix_fmt", "g", "crf", "timestamps_mode", "backend",
    "video_images_size_ratio", "video_images_load_time_ratio",
    "median_load_time_video_ms", "std_load_time_video_ms",
    "median_load_time_images_ms", "std_load_time_images_ms",
    "median_psnr", "std_psnr", "median_ssim", "std_ssim",
    "median_mse", "std_mse", "encoding_fps", "encoding_time_ms", "created_at",
    "lerobot_version", "num_samples",
)

RGB = BenchmarkConfig(
    key="rgb",
    label="RGB",
    title="Video Encoding & Decoding Benchmark",
    subtitle=(
        "A live leaderboard for trade-offs between compression ratio, "
        "decoding speed, and image fidelity across video codecs, pixel "
        "formats, GOP sizes and CRF settings — measured on real robotics "
        "datasets."
    ),
    results_dataset=RESULTS_DATASET,
    submissions_dataset=SUBMISSIONS_DATASET,
    columns=COLUMNS,
    repos=REPOS,
    filter_options={
        "vcodec": VCODECS,
        "pix_fmt": PIX_FMTS,
        "backend": BACKENDS,
        "g": [str(v) for v in G_VALUES],
        "crf": [str(v) for v in CRF_VALUES],
    },
    filter_order=["vcodec", "pix_fmt", "backend", "g", "crf"],
    leaderboard_axes=LEADERBOARD_AXES,
    leaderboard_cats=LEADERBOARD_CATS,
    leaderboard_group_keys=("vcodec", "pix_fmt", "g", "crf", "backend"),
    submit_options=SUBMIT_OPTIONS,
    submit_defaults=SUBMIT_DEFAULTS,
    full_sweep=FULL_SWEEP,
    param_groups=PARAM_GROUPS,
    param_notes=PARAM_NOTES,
    about_html=ABOUT_HTML,
    keep_keys=RGB_KEEP_KEYS,
    scatter_x="video_images_size_ratio",
    scatter_y="median_psnr",
    codec_count=len(VCODECS),
    backend_count=len(BACKENDS),
)


# --------------------------------------------------------------------------- #
# Depth benchmark
#
# Mirrors the RGB structure for the depth Hub dataset
# ``lerobot/depth-benchmark-results``. Reuses the RGB ``TS_MODES`` /
# ``BACKENDS`` vocabularies defined above.
# --------------------------------------------------------------------------- #
DEPTH_COLUMNS: list[dict[str, Any]] = [
    # Config
    {"key": "repo_id", "label": "Dataset", "short": "Dataset", "group": "Config",
     "desc": "Hugging Face Hub dataset repo ID for the depth episode."},
    {"key": "vcodec", "label": "Codec", "short": "Codec", "group": "Config",
     "desc": "Video codec. Generic codecs (hevc, av1) are supported, or you can pin a specific encoding library (e.g. libaom-av1)."},
    {"key": "encoder", "label": "Encoder", "short": "Encoder", "group": "Config", "default_hidden": True,
     "desc": "Encoding library used to produce the video."},
    {"key": "decoder", "label": "Decoder", "short": "Decoder", "group": "Config", "default_hidden": True,
     "desc": "Decoding library used to read the video."},
    {"key": "pix_fmt", "label": "Pixel format", "short": "Pixel format", "group": "Config",
     "desc": "Pixel format. Depth uses high-bit-depth gray formats (e.g. gray12le, gray16le)."},
    {"key": "g", "label": "GOP", "short": "GOP", "group": "Config",
     "desc": "Group Of Pictures — the keyframe interval. A small value keeps seeks cheap but makes files bigger, while a large value compresses better at the cost of expensive random access."},
    {"key": "crf", "label": "CRF", "short": "CRF", "group": "Config",
     "desc": "Constant Rate Factor — the quality knob: lower gives bigger files and higher fidelity, higher gives smaller files and more loss (ignored when lossless)."},
    {"key": "lossless", "label": "Lossless", "short": "Lossless", "group": "Config",
     "desc": "Whether the codec runs in a mathematically lossless mode."},
    {"key": "use_log", "label": "Log encode", "short": "Log encode", "group": "Config",
     "desc": "Whether depth is log-transformed before quantization, allocating more precision to near depths."},
    {"key": "depth_min", "label": "Depth min (m)", "short": "Depth min", "group": "Config",
     "fmt_spec": "%.3f",
     "desc": "Lower bound of the depth range mapped into the encoded value range, in meters."},
    {"key": "depth_max", "label": "Depth max (m)", "short": "Depth max", "group": "Config",
     "fmt_spec": "%.3f",
     "desc": "Upper bound of the depth range mapped into the encoded value range, in meters."},
    {"key": "shift", "label": "Shift (m)", "short": "Shift", "group": "Config",
     "fmt_spec": "%.3f",
     "desc": "Offset applied to depth before encoding, in meters."},
    {"key": "timestamps_mode", "label": "Access pattern", "short": "Access", "group": "Config",
     "desc": "How frames are requested. 1_frame: seek, decode one frame, done. 2_frames: two adjacent frames. 6_frames: contiguous window. 2_frames_4_space: two samples four frames apart — worst-case for GOP-heavy settings."},
    {"key": "backend", "label": "Backend", "short": "Backend", "group": "Config",
     "desc": "Video decoding library (pyav)."},

    # Compression
    {"key": "video_images_size_ratio", "label": "Video/Image\nsize ratio ↓", "short": "Video/Image size ratio",
     "group": "Compression", "metric": True, "lower": True, "fmt_spec": "%.4f",
     "desc": "Encoded video size ÷ sum of original depth image sizes. Lower means better compression — the video takes less disk than the raw frames."},
    {"key": "video_images_load_time_ratio", "label": "Video/Image\nload ratio ↓", "short": "Video/Image load ratio",
     "group": "Compression", "metric": True, "lower": True, "fmt_spec": "%.4f",
     "desc": "Video decoding time ÷ image load time for the same frames. Below 1 means the video decodes faster than reading the raw frames; above 1 means you pay for the compression at read time."},

    # Speed
    {"key": "median_load_time_video_ms", "label": "Decoding (ms) ↓", "short": "Decoding",
     "group": "Speed", "metric": True, "lower": True, "fmt_spec": "%.2f", "std_key": "std_load_time_video_ms",
     "desc": "Median wall-clock time to decode N frames from the compressed video."},
    {"key": "encoding_time_ms", "label": "Encoding (ms) ↓", "short": "Encoding",
     "group": "Speed", "metric": True, "lower": True, "fmt_spec": "%.2f",
     "desc": "Wall time to encode the whole episode once."},
    {"key": "encoding_fps", "label": "Encoding (fps) ↑", "short": "Encoding fps",
     "group": "Speed", "metric": True, "higher": True, "fmt_spec": "%.1f",
     "desc": "Encoding throughput — frames per second across the whole episode."},

    # Quality
    {"key": "median_rmse_m", "label": "RMSE (m) ↓", "short": "RMSE",
     "group": "Quality", "metric": True, "lower": True, "fmt_spec": "%.4f", "std_key": "std_rmse_m",
     "desc": "Root-mean-squared error between decoded and source depth, in meters. Lower means the decoded depth is closer to the source."},
    {"key": "median_mae_m", "label": "MAE (m) ↓", "short": "MAE",
     "group": "Quality", "metric": True, "lower": True, "fmt_spec": "%.4f", "std_key": "std_mae_m",
     "desc": "Mean absolute error between decoded and source depth, in meters. Lower means a smaller average depth error."},
    {"key": "median_absrel", "label": "AbsRel ↓", "short": "AbsRel",
     "group": "Quality", "metric": True, "lower": True, "fmt_spec": "%.4f", "std_key": "std_absrel",
     "desc": "Absolute relative error: mean(|decoded - source| / source). Lower means less error relative to the true depth."},
    {"key": "median_delta1", "label": "δ<1.25 ↑", "short": "δ1",
     "group": "Quality", "metric": True, "higher": True, "fmt_spec": "%.4f", "std_key": "std_delta1",
     "desc": "Fraction of pixels with max(d/d*, d*/d) < 1.25. Higher means more pixels land within the accuracy threshold (1.0 = all correct)."},
    {"key": "median_delta2", "label": "δ<1.25² ↑", "short": "δ2",
     "group": "Quality", "metric": True, "higher": True, "fmt_spec": "%.4f", "std_key": "std_delta2",
     "desc": "Fraction of pixels within the 1.25² accuracy threshold. Higher means more pixels within tolerance."},
    {"key": "median_delta3", "label": "δ<1.25³ ↑", "short": "δ3",
     "group": "Quality", "metric": True, "higher": True, "fmt_spec": "%.4f", "std_key": "std_delta3",
     "desc": "Fraction of pixels within the 1.25³ accuracy threshold. Higher means more pixels within tolerance."},

    # Quantized quality (mirror set; hidden by default)
    {"key": "quant_median_rmse_m", "label": "Quant RMSE (m) ↓", "short": "Quant RMSE",
     "group": "Quantized quality", "metric": True, "lower": True, "fmt_spec": "%.4f", "std_key": "quant_std_rmse_m",
     "desc": "RMSE from quantization alone (no codec), in meters. Lower means quantization preserves depth better."},
    {"key": "quant_median_mae_m", "label": "Quant MAE (m) ↓", "short": "Quant MAE",
     "group": "Quantized quality", "metric": True, "lower": True, "fmt_spec": "%.4f", "std_key": "quant_std_mae_m",
     "desc": "MAE from quantization alone, in meters. Lower means quantization preserves depth better."},
    {"key": "quant_median_absrel", "label": "Quant AbsRel ↓", "short": "Quant AbsRel",
     "group": "Quantized quality", "metric": True, "lower": True, "fmt_spec": "%.4f", "std_key": "quant_std_absrel",
     "desc": "Absolute relative error from quantization alone. Lower means quantization adds less relative error."},
    {"key": "quant_median_delta1", "label": "Quant δ<1.25 ↑", "short": "Quant δ1",
     "group": "Quantized quality", "metric": True, "higher": True, "fmt_spec": "%.4f", "std_key": "quant_std_delta1",
     "desc": "δ<1.25 accuracy from quantization alone. Higher means quantization keeps more pixels within tolerance."},
    {"key": "quant_median_delta2", "label": "Quant δ<1.25² ↑", "short": "Quant δ2",
     "group": "Quantized quality", "metric": True, "higher": True, "fmt_spec": "%.4f", "std_key": "quant_std_delta2",
     "desc": "δ<1.25² accuracy from quantization alone. Higher means quantization keeps more pixels within tolerance."},
    {"key": "quant_median_delta3", "label": "Quant δ<1.25³ ↑", "short": "Quant δ3",
     "group": "Quantized quality", "metric": True, "higher": True, "fmt_spec": "%.4f", "std_key": "quant_std_delta3",
     "desc": "δ<1.25³ accuracy from quantization alone. Higher means quantization keeps more pixels within tolerance."},
]

DEPTH_REPOS: list[str] = ["lerobot/outdoor-depth"]
DEPTH_VCODECS: list[str] = ["hevc", "libaom-av1"]
DEPTH_PIX_FMTS: list[str] = ["gray12le"]
DEPTH_BACKENDS: list[str] = ["pyav"]
DEPTH_LOSSLESS: list[str] = ["True", "False"]
DEPTH_USE_LOG: list[str] = ["True", "False"]
DEPTH_G_VALUES: list[int] = [1, 2, 10, 40]
DEPTH_CRF_VALUES: list[int] = [0, 10, 20, 30]

DEPTH_LEADERBOARD_AXES: list[str] = [
    "encoding_time_ms",
    "median_load_time_video_ms",
    "video_images_size_ratio",
    "median_rmse_m",
    "median_absrel",
    "median_delta1",
]

DEPTH_LEADERBOARD_CATS: dict[str, dict[str, Any]] = {
    "Overall": {
        "desc": "Balanced across encoding, decoding, size, and depth fidelity.",
        "weights": {
            "encoding_time_ms": 1, "median_load_time_video_ms": 1,
            "video_images_size_ratio": 1, "median_rmse_m": 1,
            "median_absrel": 1, "median_delta1": 1,
        },
    },
    "Quality": {
        "desc": "Pure depth fidelity: RMSE, AbsRel, δ<1.25 only.",
        "weights": {
            "encoding_time_ms": 0, "median_load_time_video_ms": 0,
            "video_images_size_ratio": 0, "median_rmse_m": 1,
            "median_absrel": 1, "median_delta1": 1,
        },
    },
    "Encoding": {
        "desc": "Encoding throughput and output size, fidelity ignored.",
        "weights": {
            "encoding_time_ms": 1, "median_load_time_video_ms": 0,
            "video_images_size_ratio": 1, "median_rmse_m": 0,
            "median_absrel": 0, "median_delta1": 0,
        },
    },
    "Decoding": {
        "desc": "Pure decoding latency at the selected access pattern.",
        "weights": {
            "encoding_time_ms": 0, "median_load_time_video_ms": 1,
            "video_images_size_ratio": 0, "median_rmse_m": 0,
            "median_absrel": 0, "median_delta1": 0,
        },
    },
}

DEPTH_SUBMIT_OPTIONS: dict[str, list[str]] = {
    "repos": DEPTH_REPOS,
    "vcodecs": DEPTH_VCODECS,
    "pix_fmts": DEPTH_PIX_FMTS,
    "lossless": DEPTH_LOSSLESS,
    "use_log": DEPTH_USE_LOG,
    "g": SUBMIT_OPTIONS["g"],
    "crf": SUBMIT_OPTIONS["crf"],
    "timestamps_modes": TS_MODES,
    "backends": DEPTH_BACKENDS,
}

DEPTH_SUBMIT_DEFAULTS: dict[str, Any] = {
    "repos": DEPTH_REPOS[:1],
    "vcodecs": ["hevc"],
    "pix_fmts": ["gray12le"],
    "lossless": ["True"],
    "use_log": ["True"],
    "g": ["2"],
    "crf": ["0"],
    "timestamps_modes": ["1_frame"],
    "backends": ["pyav"],
    "depth_min": None,
    "depth_max": None,
    "shift": None,
    "samples_per_config": 50,
}

DEPTH_FULL_SWEEP: dict[str, Any] = {
    "repos": list(DEPTH_REPOS),
    "vcodecs": list(DEPTH_VCODECS),
    "pix_fmts": list(DEPTH_PIX_FMTS),
    "lossless": list(DEPTH_LOSSLESS),
    "use_log": list(DEPTH_USE_LOG),
    "g": [str(v) for v in DEPTH_G_VALUES],
    "crf": [str(v) for v in DEPTH_CRF_VALUES],
    "timestamps_modes": list(TS_MODES),
    "backends": list(DEPTH_BACKENDS),
    "depth_min": None,
    "depth_max": None,
    "shift": None,
    "samples_per_config": 50,
}

DEPTH_PARAM_GROUPS: list[dict[str, Any]] = [
    {"t": "Inputs — what you benchmark",
     "desc": "The corpus. Each dataset is a LeRobot episode recording with a few minutes of depth observations.",
     "keys": ["repo_id"]},
    {"t": "Encoding — how the depth video is compressed",
     "desc": "Passed to the FFmpeg/PyAV encoder. These are the knobs operators actually tune.",
     "keys": ["vcodec", "pix_fmt", "g", "crf", "lossless"]},
    {"t": "Quantization — how depth maps to pixels",
     "desc": "How continuous depth (meters) is mapped into the encoder's integer pixel range before compression.",
     "keys": ["use_log", "depth_min", "depth_max", "shift"]},
    {"t": "Decoding — how you read it back",
     "desc": "The other half of the equation. The same MP4 can decode very differently depending on library and access pattern.",
     "keys": ["backend", "timestamps_mode"]},
    {"t": "Fidelity metrics — decoded depth vs. source",
     "desc": "We decode the compressed video, re-read the source depth frames, and compare pixel-for-pixel in meters.",
     "keys": ["median_rmse_m", "median_mae_m", "median_absrel", "median_delta1", "median_delta2", "median_delta3"]},
    {"t": "Performance metrics — how fast, how small",
     "desc": "The cost side. Run on a single CPU thread unless noted.",
     "keys": ["encoding_time_ms", "encoding_fps", "video_images_size_ratio", "median_load_time_video_ms", "video_images_load_time_ratio"]},
]

DEPTH_ABOUT_HTML: str = """\
<h3 id="what">What this is</h3>
<p>A benchmark for encoding and decoding <i>depth</i> frames from robotics
datasets as video. Depth maps are single-channel, high-bit-depth images;
this page quantifies the size/speed wins of video encoding against the depth
error introduced by quantization and lossy codecs.</p>

<p>
<h3 id="metrics">Metrics</h3>
<p>The size and speed metrics are shared with the RGB benchmark; the fidelity metrics are depth-specific because reconstruction error is measured in meters rather than on pixel intensities.</p>
<p><b>Size &amp; speed</b></p>
<ul>
<li><b>Video/image size ratio</b> — encoded video ÷ sum of source depth frames. Lower means better compression — the video takes less disk than the raw frames.</li>
<li><b>Video/image load ratio</b> — video decoding time ÷ depth image load time for the same frames. <1 means the video is faster to read than the raw frames; >1 means you pay for the compression at read time.</li>
<li><b>Decoding time</b> — median wall-clock to decode N frames at a given timestamp.</li>
<li><b>Encoding time</b> — wall-clock to encode the whole clip, end-to-end.</li>
<li><b>Encoding fps</b> — encode throughput in frames per second, derived from the encoding time and the episode length. Higher means faster encoding; the handy complement to the raw time column.</li>
</ul>
<p><b>Depth fidelity</b> — every decoded frame is de-quantized back to metric depth and compared against the source depth map, pixel-for-pixel; errors are reported in meters. Invalid pixels (zero / no return) are excluded so they don't skew the error.</p>
<ul>
<li><b>RMSE (m)</b> — root-mean-squared error between decoded and source depth, in meters. Squares the residuals, so it punishes large per-pixel misses harder than MAE. Lower is decoded depth closer to the source.</li>
<li><b>MAE (m)</b> — mean absolute error in meters. The plain average miss per pixel. Lower is smaller average miss.</li>
<li><b>AbsRel</b> — mean of <code>|decoded − source| / source</code>. A relative error, so a 5&nbsp;cm miss at 1&nbsp;m counts far more than the same miss at 10&nbsp;m; the standard depth-estimation metric. Lower is less error relative to the true depth.</li>
<li><b>δ&lt;1.25 / 1.25² / 1.25³</b> — accuracy thresholds: the fraction of pixels whose ratio <code>max(d/d*, d*/d)</code> stays under 1.25, 1.25² and 1.25³. Higher is more pixels within tolerance (1.0 = every pixel within threshold); the cubed threshold is the most forgiving.</li>
<li><b>Quantized variants</b> (<code>Quant …</code> columns, hidden by default) — the very same metrics computed after quantization but <i>before</i> the codec. They isolate the error you lose just by squeezing continuous depth into an integer pixel range, so the gap between a metric and its <code>Quant</code> twin is the codec's own contribution.</li>
</ul>
</p>

<p>
<h3 id="access">Access patterns</h3>
<p>
Decode cost depends heavily on <i>how</i> frames are requested. <code>1_frame</code> pays the full seek+IDR cost per sample; <code>6_frames</code> amortizes it across contiguous frames; <code>2_frames_4_space</code> probes the worst case where the decoder must step across two distant windows.
</p>
</p>

<h3 id="repro">Reproduce locally</h3>
<p>The full sweep is open source. First, set <code>HF_RESULTS_REPO_ID</code> to the Hugging Face Hub dataset where you want results pushed:</p>
<pre style="background:var(--hf-gray-900);color:var(--hf-gray-100);padding:var(--space-4);border-radius:var(--radius-md);font-size:var(--fs-xs);overflow:auto">export HF_RESULTS_REPO_ID=lerobot/depth-benchmark-results</pre>
<p>Then run the benchmark:</p>
<pre style="background:var(--hf-gray-900);color:var(--hf-gray-100);padding:var(--space-4);border-radius:var(--radius-md);font-size:var(--fs-xs);overflow:auto">python benchmark/video/run_video_benchmark.py \\
  --output-dir outputs/depth_benchmark \\
  --repo-ids lerobot/outdoor-depth \\
  --vcodec hevc libaom-av1 \\
  --pix-fmt gray12le \\
  --lossless true false \\
  --use-log true false \\
  --g 2 10 40 \\
  --crf 0 10 20 30 \\
  --timestamps-modes 1_frame 2_frames 6_frames \\
  --backends pyav \\
  --num-samples 50</pre>

<h3 id="contrib">Contribute</h3>
<p>
Submit your own configurations through the <b>Submit</b> tab. A background worker picks them up and pushes results to the Hub so the whole community benefits from the same measurements.
</p>
"""

DEPTH_KEEP_KEYS: tuple[str, ...] = (
    "repo_id", "vcodec", "encoder", "decoder", "pix_fmt", "g", "crf", "lossless", "use_log",
    "depth_min", "depth_max", "shift", "timestamps_mode", "backend",
    "resolution", "num_pixels", "video_size_bytes", "images_size_bytes",
    "video_images_size_ratio", "video_images_load_time_ratio",
    "median_load_time_video_ms", "std_load_time_video_ms",
    "median_load_time_images_ms", "std_load_time_images_ms",
    "encoding_fps", "encoding_time_ms",
    "median_rmse_m", "std_rmse_m", "median_mae_m", "std_mae_m",
    "median_absrel", "std_absrel",
    "median_delta1", "std_delta1", "median_delta2", "std_delta2",
    "median_delta3", "std_delta3",
    "quant_median_rmse_m", "quant_std_rmse_m", "quant_median_mae_m", "quant_std_mae_m",
    "quant_median_absrel", "quant_std_absrel",
    "quant_median_delta1", "quant_std_delta1", "quant_median_delta2", "quant_std_delta2",
    "quant_median_delta3", "quant_std_delta3",
    "created_at", "lerobot_version", "num_samples",
)

DEPTH = BenchmarkConfig(
    key="depth",
    label="Depth",
    title="Depth Encoding & Decoding Benchmark",
    subtitle=(
        "Trade-offs between compression, decoding speed, and depth fidelity "
        "(RMSE, AbsRel, δ accuracy) across codecs, high-bit-depth pixel "
        "formats, and depth range mappings."
    ),
    results_dataset=RESULTS_DATASET_DEPTH,
    submissions_dataset=SUBMISSIONS_DATASET,
    columns=DEPTH_COLUMNS,
    repos=DEPTH_REPOS,
    filter_options={
        "vcodec": DEPTH_VCODECS,
        "pix_fmt": DEPTH_PIX_FMTS,
        "lossless": DEPTH_LOSSLESS,
        "use_log": DEPTH_USE_LOG,
        "backend": DEPTH_BACKENDS,
        "g": [str(v) for v in DEPTH_G_VALUES],
        "crf": [str(v) for v in DEPTH_CRF_VALUES],
    },
    filter_order=["vcodec", "pix_fmt", "lossless", "use_log", "backend", "g", "crf"],
    leaderboard_axes=DEPTH_LEADERBOARD_AXES,
    leaderboard_cats=DEPTH_LEADERBOARD_CATS,
    leaderboard_group_keys=("vcodec", "pix_fmt", "g", "crf", "lossless", "use_log", "backend"),
    submit_options=DEPTH_SUBMIT_OPTIONS,
    submit_defaults=DEPTH_SUBMIT_DEFAULTS,
    full_sweep=DEPTH_FULL_SWEEP,
    param_groups=DEPTH_PARAM_GROUPS,
    param_notes={},
    about_html=DEPTH_ABOUT_HTML,
    keep_keys=DEPTH_KEEP_KEYS,
    scatter_x="video_images_size_ratio",
    scatter_y="median_delta1",
    codec_count=len(DEPTH_VCODECS),
    backend_count=len(DEPTH_BACKENDS),
)

BENCHMARKS: list[BenchmarkConfig] = [RGB, DEPTH]