#!/usr/bin/env python3 """ Randomly pick N revisit candidates (gen-only MP4) and compute average PSNR(+LPIPS if available). Intended for eval_v2/static_consistency outputs: - /in_domain/loop_closure/**/*_gen_only.mp4 - /in_domain/combo_revisit_fixed_first/**/*_gen_only.mp4 Outputs: - per_video metrics JSON - aggregate summary JSON - optional side-by-side visualization PNGs (first vs last frame) """ from __future__ import annotations import argparse import json import os import random import sys from typing import Any, Dict, List, Optional, Tuple def _discover_candidates(evals_root: str) -> List[str]: cands: List[str] = [] for root, _dirs, files in os.walk(evals_root): for f in files: if f.endswith("_gen_only.mp4"): cands.append(os.path.join(root, f)) return sorted(cands) def _safe_mean(values: List[float]) -> Optional[float]: if not values: return None return float(sum(values) / len(values)) def main() -> None: p = argparse.ArgumentParser(description="Aggregate revisit metrics by random sampling") p.add_argument("--evals_root", type=str, required=True, help="Path to static_consistency/in_domain (or similar)") p.add_argument("--num_samples", type=int, default=5, help="How many videos to sample (min(len, num_samples))") p.add_argument("--seed", type=int, default=42) p.add_argument("--device", type=str, default="cuda") p.add_argument("--output_dir", type=str, required=True) p.add_argument("--write_viz", action="store_true", help="Also export side-by-side PNG for sampled videos") p.add_argument("--viz_dir", type=str, default=None, help="Override visualization output directory") args = p.parse_args() evals_root = os.path.abspath(args.evals_root) output_dir = os.path.abspath(args.output_dir) os.makedirs(output_dir, exist_ok=True) candidates = _discover_candidates(evals_root) if not candidates: summary = { "num_candidates": 0, "num_samples": 0, "aggregate": {}, "per_video": [], "note": "No *_gen_only.mp4 found under evals_root", } with open(os.path.join(output_dir, "revisit_metrics_summary.json"), "w", encoding="utf-8") as f: json.dump(summary, f, indent=2) print(f"[aggregate_revisit_metrics] No candidates under: {evals_root}", file=sys.stderr) return rng = random.Random(args.seed) num_samples = min(len(candidates), max(1, int(args.num_samples))) selected = rng.sample(candidates, k=num_samples) if num_samples < len(candidates) else candidates # Import metrics with local sys.path insertion to avoid needing package __init__.py _script_dir = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, _script_dir) import psnr_lpips # type: ignore # Optional visualization if args.write_viz: viz_dir = args.viz_dir or os.path.join(output_dir, "viz_first_vs_last") os.makedirs(viz_dir, exist_ok=True) else: viz_dir = None per_video: List[Dict[str, Any]] = [] psnr_list: List[float] = [] lpips_list: List[float] = [] for vp in selected: rel = os.path.relpath(vp, evals_root) try: res = psnr_lpips.compute_revisit_metrics(vp, device=args.device) per_video.append({"rel": rel, **res}) if res.get("psnr") is not None: psnr_list.append(float(res["psnr"])) if res.get("lpips") is not None: lpips_list.append(float(res["lpips"])) except Exception as e: per_video.append({"rel": rel, "error": str(e)}) if args.write_viz and viz_dir is not None: # Export side-by-side png try: import subprocess _viz_script = os.path.join(os.path.dirname(_script_dir), "visualize", "revisit_pairs_viz.py") out_png = os.path.join(viz_dir, rel.replace(os.sep, "__") + ".png") os.makedirs(os.path.dirname(out_png), exist_ok=True) subprocess.run( ["python3", _viz_script, "--video", vp, "--output", out_png], check=False, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, ) except Exception: pass aggregate = { "num_candidates": len(candidates), "num_samples": len(selected), "mean_psnr": _safe_mean(psnr_list), "mean_lpips": _safe_mean(lpips_list), "note": "LPIPS is None for each video when lpips dependency is missing.", } summary = { "evals_root": evals_root, "seed": args.seed, "aggregate": aggregate, "per_video": per_video, } with open(os.path.join(output_dir, "revisit_metrics_summary.json"), "w", encoding="utf-8") as f: json.dump(summary, f, indent=2) with open(os.path.join(output_dir, "revisit_metrics_per_video.json"), "w", encoding="utf-8") as f: json.dump(per_video, f, indent=2) print(f"[aggregate_revisit_metrics] wrote: {output_dir}/revisit_metrics_summary.json") if __name__ == "__main__": main()