compare_mid_frames.py

#!/usr/bin/env python3
"""
提取每个候选轨迹的中间帧，与GT（从数据集加载）的中间帧拼接对比
样式参考 WM_Planning_Evaluator.save_single_sample_panel：
- 左一：GT 中间帧
- 左二：3D 轨迹 (GT + candidates)
- 后面：P1 / P2 / ... 的中间帧
  * label 显示：P1, LPIPS, DreamSim, APE
  * P1 用绿色粗框高亮
"""
import os
import re
import json
import yaml
import pickle
import torch
from pathlib import Path
from PIL import Image
import numpy as np
import matplotlib.pyplot as plt
from glob import glob
from matplotlib import patches  # P1 绿色框
from mpl_toolkits.mplot3d import Axes3D  # noqa: F401

from datasets_v2 import TrajectoryEvalDataset
from misc import transform


def load_image(path):
    """加载图片"""
    try:
        return Image.open(path).convert("RGB")
    except Exception as e:
        print(f"Warning: Cannot load {path}: {e}")
        return None


def get_mid_frame_path(metadata_path, frames_dir):
    """从 metadata 获取中间 step 的帧路径（不含 init/goal）"""
    with open(metadata_path, 'r', encoding='utf-8') as f:
        meta = json.load(f)

    num_frames = meta.get("num_frames", 0)
    if num_frames == 0:
        return None

    # num_frames = 1(init) + step_count + 1(goal)
    step_count = num_frames - 2
    if step_count <= 0:
        return None

    mid_step_idx = step_count // 2  # 中间 step 的索引（0-based）
    mid_frame_name = f"step_{mid_step_idx:03d}.png"
    mid_frame_path = os.path.join(frames_dir, mid_frame_name)
    if os.path.exists(mid_frame_path):
        return mid_frame_path
    return None


def load_gt_mid_frame_from_dataset(run_index, dataset, config):
    """从数据集加载 GT 轨迹的中间帧（时间上居中，对应轨迹的一半）"""
    # 获取该 run 对应的数据
    idxs, obs_image, goal_image, gt_actions, goal_pos, aug_image = dataset[run_index]
    traj_id = int(idxs.item())

    # index_to_data: (f_curr, curr_time, min_goal_dist, max_goal_dist)
    f_curr, curr_time, min_goal_dist, max_goal_dist = dataset.index_to_data[run_index]
    f_goal, goal_time, _ = dataset._sample_goal(f_curr, curr_time, min_goal_dist, max_goal_dist)

    T = gt_actions.shape[0]
    if T == 0:
        return None

    traj_stride = config.get("traj_stride", 1)
    mid_step = T // 2
    mid_time = curr_time + mid_step * traj_stride

    if mid_time > goal_time:
        mid_time = goal_time
    if mid_time < curr_time:
        mid_time = curr_time

    try:
        from misc import get_data_path
        img_path = get_data_path(dataset.data_folder, f_curr, mid_time)
        if os.path.exists(img_path):
            img = Image.open(img_path).convert("RGB")
            # resize 到 config 里的 image_size
            img_size = config.get("image_size", None)
            if img_size is not None:
                if isinstance(img_size, (list, tuple)):
                    img = img.resize((img_size[1], img_size[0]), Image.BICUBIC)
                else:
                    img = img.resize((img_size, img_size), Image.BICUBIC)
            return img
    except Exception as e:
        print(f"  Warning: Cannot load GT image from dataset: {e}")
        return None

    return None


def get_gt_traj_xyz_meters(run_index, dataset, config, dataset_name):
    """
    从 dataset 的 gt_actions 里取出 GT 轨迹的 3D 点，并转换为米单位。
    与 WM_Planning_Evaluator.generate_actions 里的逻辑一致：
    - gt_actions[:, :3] 是 waypoint 单位的绝对坐标
    - metric_waypoint_spacing 决定 waypoint -> meter 的映射
    """
    idxs, obs_image, goal_image, gt_actions, goal_pos, aug_image = dataset[run_index]
    data_cfg = config["eval_datasets"][dataset_name]
    spacing = float(data_cfg["metric_waypoint_spacing"])

    gt_xyz_waypoint = gt_actions[:, :3].to("cpu").numpy()  # [T, 3] waypoint 单位
    # 加上起点 (0,0,0)，与 generate_actions 一致：[T+1, 3]
    gt_xyz_waypoint = np.concatenate(
        [np.zeros((1, 3), dtype=np.float32), gt_xyz_waypoint.astype(np.float32)], axis=0
    )
    gt_xyz_meters = gt_xyz_waypoint * spacing  # [T+1, 3]
    return gt_xyz_meters  # (T+1, 3)


def process_run(run_dir, output_dir, dataset, config, candidate_trajs, dataset_name):
    """处理一个 run_xxx 目录，生成 GT + 3D 轨迹 + candidates 的中间帧对比图"""
    run_name = os.path.basename(run_dir)
    print(f"Processing {run_name}...")

    # 解析 run_index / traj_id （与 WM evaluator 中 sid=traj_id 对应）
    try:
        run_index = int(run_name.split('_')[1])
    except Exception:
        print(f"  Warning: Cannot parse run_index from {run_name}")
        return None

    traj_id = run_index  # 与 editor/run_{sid:03d} 中 sid 对齐

    # 从数据集加载 GT 中间帧（对应轨迹的一半）
    gt_mid_img = load_gt_mid_frame_from_dataset(run_index, dataset, config)
    if gt_mid_img is None:
        print(f"  Warning: Cannot load GT mid frame for {run_name}")
        return None

    # GT 轨迹 3D 点（米）
    gt_xyz_meters = get_gt_traj_xyz_meters(run_index, dataset, config, dataset_name)

    # 读取所有 candidates 的中间帧
    candidates_dir = os.path.join(run_dir, "candidates")
    if not os.path.exists(candidates_dir):
        print(f"  Warning: {candidates_dir} not found, skipping")
        return None

    cand_dirs = sorted(glob(os.path.join(candidates_dir, "cand_*")))
    if len(cand_dirs) == 0:
        print(f"  Warning: No candidates found in {run_name}")
        return None

    cand_mid_imgs = []
    cand_infos = []  # 保存 rank / id / lpips / ape / dreamsim，用于绘制标签和 3D 轨迹

    # 该 traj 对应的所有候选轨迹（米单位），形状 [N, T+1, 4]
    if traj_id not in candidate_trajs:
        print(f"  Warning: traj_id {traj_id} not in candidate_trajs, skip 3D plot")
        cand_traj_points_list = None
    else:
        cand_traj_full = np.asarray(candidate_trajs[traj_id], dtype=np.float32)  # [N,T+1,4]
        cand_traj_points_list = cand_traj_full[:, :, :3]  # [N,T+1,3]

    for cand_dir in cand_dirs:
        cand_name = os.path.basename(cand_dir)
        cand_meta_path = os.path.join(cand_dir, "metadata.json")
        cand_frames_dir = os.path.join(cand_dir, "frames")

        if not os.path.exists(cand_meta_path):
            continue

        cand_mid_path = get_mid_frame_path(cand_meta_path, cand_frames_dir)
        if cand_mid_path is None or not os.path.exists(cand_mid_path):
            continue

        cand_mid_img = load_image(cand_mid_path)
        if cand_mid_img is None:
            continue

        # 读取 candidate 的 meta 信息，用于 label 与 dreamsim
        try:
            with open(cand_meta_path, 'r', encoding='utf-8') as f:
                cand_meta = json.load(f)
            cand_rank = cand_meta.get("candidate_rank", -1)
            cand_id = cand_meta.get("candidate_id", -1)
            lpips_loss = float(cand_meta.get("final_lpips", 0.0))
            ape = float(cand_meta.get("cand_ape", 0.0))

            # DreamSim：兼容两种可能字段名，没有就置 0.0
            ds_val = cand_meta.get("cand_dreamsim", cand_meta.get("final_dreamsim", 0.0))
            dreamsim = float(ds_val)
        except Exception:
            cand_rank = -1
            cand_id = -1
            lpips_loss = 0.0
            ape = 0.0
            dreamsim = 0.0

        cand_mid_imgs.append(cand_mid_img)
        cand_infos.append({
            "rank": cand_rank,
            "id": cand_id,
            "lpips": lpips_loss,
            "ape": ape,
            "dreamsim": dreamsim,
        })

    if len(cand_mid_imgs) == 0:
        print(f"  Warning: No valid candidate mid frames for {run_name}")
        return None

    print(f"  Found: GT + {len(cand_mid_imgs)} candidates = {1 + len(cand_mid_imgs)} total images")

    # ===== 可视化：样式参考 save_single_sample_panel =====
    # 布局：GT | Traj(3D) | P1 | P2 | ...
    ncols = 2 + len(cand_mid_imgs)
    fig = plt.figure(figsize=(4 * ncols, 4))

    # 左一：GT 中间帧
    ax_gt = fig.add_subplot(1, ncols, 1)
    ax_gt.imshow(gt_mid_img)
    ax_gt.set_title("GT (mid-frame)", fontsize=14, fontweight='bold')
    ax_gt.axis('off')

    # 左二：3D 轨迹子图
    ax_traj = fig.add_subplot(1, ncols, 2, projection='3d')

    # 颜色与参考代码保持一致
    selected_color = "#2FBF71"  # P1 绿色高亮
    other_colors = ["#F4A259", "#E4572E", "#4C78A8", "#B279A2"]

    # 找到“最佳”candidate：优先用 rank==0，否则按最小 LPIPS
    best_idx = None
    for i, info in enumerate(cand_infos):
        if info["rank"] == 0:
            best_idx = i
            break
    if best_idx is None:
        lpips_list = [info["lpips"] for info in cand_infos]
        best_idx = int(np.argmin(lpips_list))

    # 先画 GT 轨迹（米）
    if gt_xyz_meters is not None and gt_xyz_meters.shape[0] > 0:
        gx, gy, gz = gt_xyz_meters[:, 0], gt_xyz_meters[:, 1], gt_xyz_meters[:, 2]
        ax_traj.plot3D(gx, gy, gz, color="#2066E0", linewidth=3, label="GT")
        # 标记终点为 Goal
        gx_f, gy_f, gz_f = gx[-1], gy[-1], gz[-1]
        ax_traj.scatter(gx_f, gy_f, gz_f, c="#2066E0", s=40, depthshade=True)
        ax_traj.text(gx_f, gy_f, gz_f, "Goal", color="#2066E0", fontsize=9)

    # 再画 candidates 的 3D 轨迹
    if cand_traj_points_list is not None:
        for i, info in enumerate(cand_infos):
            cid = info["id"]
            # 防御：cid 可能为 -1 或超界
            if cid is None or cid < 0 or cid >= cand_traj_points_list.shape[0]:
                continue

            traj_points = cand_traj_points_list[cid]  # (T+1, 3) 米
            xs, ys, zs = traj_points[:, 0], traj_points[:, 1], traj_points[:, 2]

            if i == best_idx:
                traj_color = selected_color
            else:
                traj_color = other_colors[(i - 1) % len(other_colors)]

            ax_traj.plot3D(xs, ys, zs, color=traj_color, linewidth=3)

            # 标签放在 1/3 处，附加一点偏移，避免重合
            if len(xs) > 0:
                mid_idx = max(1, len(xs) // 3)
                x_label, y_label, z_label = xs[mid_idx], ys[mid_idx], zs[mid_idx]

                if len(xs) > 1:
                    traj_dir = np.array([xs[-1] - xs[0], ys[-1] - ys[0], zs[-1] - zs[0]])
                    norm = np.linalg.norm(traj_dir)
                    if norm > 1e-6:
                        traj_dir = traj_dir / norm
                    else:
                        traj_dir = np.array([1, 0, 0], dtype=np.float32)
                else:
                    traj_dir = np.array([1, 0, 0], dtype=np.float32)

                perp_offset = 0.08 * (i + 1)
                perp_vec = np.cross(traj_dir, np.array([0, 0, 1], dtype=np.float32))
                if np.linalg.norm(perp_vec) < 1e-6:
                    perp_vec = np.cross(traj_dir, np.array([0, 1, 0], dtype=np.float32))
                perp_vec = perp_vec / (np.linalg.norm(perp_vec) + 1e-6)
                x_label += perp_vec[0] * perp_offset
                y_label += perp_vec[1] * perp_offset
                z_label += perp_vec[2] * perp_offset

                rank = info["rank"]
                if rank is not None and rank >= 0:
                    p_label = f"P{rank + 1}"
                else:
                    p_label = f"P{i + 1}"

                # 先绘制白色描边，再绘制前景文字
                for dx, dy, dz in [(-0.002, -0.002, -0.002), (-0.002, 0.002, -0.002),
                                   (0.002, -0.002, -0.002), (0.002, 0.002, -0.002),
                                   (-0.002, -0.002, 0.002), (-0.002, 0.002, 0.002),
                                   (0.002, -0.002, 0.002), (0.002, 0.002, 0.002)]:
                    ax_traj.text(
                        x_label + dx, y_label + dy, z_label + dz,
                        p_label, fontsize=12, color="white", weight="bold", alpha=0.8
                    )
                ax_traj.text(
                    x_label, y_label, z_label,
                    p_label, fontsize=12, color=traj_color, weight="bold"
                )

    ax_traj.set_title("Trajectories (3D)")
    ax_traj.set_xlabel("X (m)")
    ax_traj.set_ylabel("Y (m)")
    ax_traj.set_zlabel("Z (m)")
    ax_traj.view_init(elev=22, azim=-60)
    ax_traj.grid(True, alpha=0.2)

    # 右侧：Candidates 中间帧 + label (P*, LPIPS, DreamSim, APE)
    axes_cands = [
        fig.add_subplot(1, ncols, 3 + i) for i in range(len(cand_mid_imgs))
    ]

    for i, (img, info, ax) in enumerate(zip(cand_mid_imgs, cand_infos, axes_cands)):
        ax.imshow(img)
        ax.axis('off')

        rank = info["rank"]
        lpips_loss = info["lpips"]
        ape = info["ape"]
        dreamsim = info["dreamsim"]

        if rank is not None and rank >= 0:
            p_label = f"P{rank + 1}"
        else:
            p_label = f"P{i + 1}"

        # 文本框，增加 DreamSim 指标
        text_str = (
            f"{p_label}\n"
            f"LPIPS: {lpips_loss:.3f}\n"
            f"DS: {dreamsim:.3f}\n"
            f"APE: {ape:.2f}"
        )
        ax.text(
            0.5, 0.02, text_str,
            transform=ax.transAxes,
            ha="center",
            va="bottom",
            fontsize=11,
            color="black",
            bbox=dict(facecolor="white", alpha=0.9, boxstyle="round,pad=0.25")
        )

        # P1 (best) 画绿色粗框
        if i == best_idx:
            rect = patches.Rectangle(
                (0, 0), 1, 1,
                transform=ax.transAxes,
                fill=False,
                linewidth=4,
                edgecolor=selected_color
            )
            ax.add_patch(rect)

    plt.tight_layout()

    # 保存
    output_path = os.path.join(output_dir, f"{run_name}_mid_compare.png")
    plt.savefig(output_path, dpi=150, bbox_inches='tight')
    plt.close()

    print(f"  Saved: {output_path}")
    return output_path


def get_dataset_eval(config, dataset_name, predefined_index=True):
    """从 sample_trajectories.py 复制的函数"""
    data_config = config["eval_datasets"][dataset_name]
    if predefined_index:
        predefined_index = f"data_splits/{dataset_name}/test/navigation_eval_16_long.pkl"
    else:
        predefined_index = None

    dataset = TrajectoryEvalDataset(
        data_folder=data_config["data_folder"],
        data_split_folder=data_config["test"],
        dataset_name=dataset_name,
        image_size=config["image_size"],
        min_dist_cat=config["trajectory_eval_distance"]["min_dist_cat"],
        max_dist_cat=config["trajectory_eval_distance"]["max_dist_cat"],
        len_traj_pred=config["trajectory_eval_len_traj_pred"],
        traj_stride=config["traj_stride"],
        context_size=config["trajectory_eval_context_size"],
        normalize=config["normalize"],
        transform=transform,
        predefined_index=predefined_index,
        traj_names="traj_names.txt"
    )
    return dataset


def load_candidate_trajs(dataset_name, input_dir):
    """
    从 RULE_N*_K*_... 目录名中解析 N，并加载对应的 candidate 轨迹：
    data_splits/{dataset_name}/test/{dataset_name}_{N}_trajectories_long.pkl
    """
    eval_dir = os.path.basename(os.path.dirname(input_dir))
    m = re.search(r"N(\d+)", eval_dir)
    if not m:
        raise RuntimeError(f"Cannot parse N from eval dir name: {eval_dir}")
    num_samples = int(m.group(1))
    pkl_path = f"data_splits/{dataset_name}/test/{dataset_name}_{num_samples}_trajectories_long.pkl"
    if not os.path.exists(pkl_path):
        raise FileNotFoundError(f"candidate traj pkl not found: {pkl_path}")
    with open(pkl_path, "rb") as f:
        candidate_trajs = pickle.load(f)
    print(f"Loaded candidate trajectories from {pkl_path} (num_samples={num_samples})")
    return candidate_trajs


def main():
    # 加载配置和数据集
    with open("config/eval_config.yaml", "r") as f:
        default_config = yaml.safe_load(f)
    config = default_config

    with open("config/nwm_cdit_airvln_16.yaml", "r") as f:
        user_config = yaml.safe_load(f)
    config.update(user_config)

    dataset_name = "airvln_16"
    dataset = get_dataset_eval(config, dataset_name, predefined_index=True)
    print(f"Loaded dataset with {len(dataset)} samples")

    # 输入目录：editor/run_xxx
    input_dir = "/data1/tpz/nwm-main/results/nwm_cdit_airvln_16/airvln_16/RULE_N3_K3_RS1_rep1_OPT11141/editor"

    # 加载 candidate 轨迹 (米单位的 x,y,z,yaw)
    candidate_trajs = load_candidate_trajs(dataset_name, input_dir)

    # 输出目录
    output_dir = os.path.join(os.path.dirname(input_dir), "mid_frame_comparison")
    os.makedirs(output_dir, exist_ok=True)

    print(f"Input directory: {input_dir}")
    print(f"Output directory: {output_dir}")
    print("-" * 60)

    # 处理所有 run 目录
    run_dirs = sorted(glob(os.path.join(input_dir, "run_*")))
    print(f"Found {len(run_dirs)} run directories")

    success_count = 0
    for run_dir in run_dirs:
        result = process_run(run_dir, output_dir, dataset, config, candidate_trajs, dataset_name)
        if result:
            success_count += 1

    print("-" * 60)
    print(f"Completed: {success_count}/{len(run_dirs)} runs processed successfully")
    print(f"Results saved to: {output_dir}")


if __name__ == "__main__":
    main()