predict.py

"""Inference for 2-frame simultaneous predictor with TTA."""
import json
import numpy as np
import torch
import sys
sys.path.insert(0, "/home/coder/code")
from flow_warp_attn_model import FlowWarpAttnUNet


def load_model(model_dir: str):
    with open(f"{model_dir}/config.json") as f:
        config = json.load(f)
    model = FlowWarpAttnUNet(
        in_channels=config["in_channels"],
        channels=config["channels"],
        out_channels=config.get("out_channels", 6)
    )
    sd = torch.load(f"{model_dir}/model.pt", map_location="cpu", weights_only=True)
    sd = {k: v.float() for k, v in sd.items()}
    model.load_state_dict(sd)
    model.eval()
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    model = model.to(device)
    return {
        "model": model,
        "device": device,
        "context_len": config["context_len"],
        "cached_pred2": None,
    }


def _prepare_input(context_frames, context_len):
    N = len(context_frames)
    if N >= context_len:
        frames = context_frames[-context_len:]
    else:
        pad = np.repeat(context_frames[:1], context_len - N, axis=0)
        frames = np.concatenate([pad, context_frames], axis=0)
    frames_f = frames.astype(np.float32) / 255.0
    frames_f = np.transpose(frames_f, (0, 3, 1, 2))
    context = frames_f.reshape(1, -1, 64, 64)
    last_frame = frames_f[-1:]
    return context, last_frame


def _run_model_tta(model, device, ctx, last):
    """Run model with TTA (horizontal flip) and return both predictions."""
    with torch.no_grad():
        ctx_t = torch.from_numpy(ctx).to(device)
        last_t = torch.from_numpy(last).to(device)
        preds1, _ = model(ctx_t, last_t)

        # Flipped
        ctx_f = torch.from_numpy(ctx[:, :, :, ::-1].copy()).to(device)
        last_f = torch.from_numpy(last[:, :, :, ::-1].copy()).to(device)
        preds2, _ = model(ctx_f, last_f)

        # Average normal and flipped-back predictions
        pred1 = (preds1[0] + preds2[0].flip(-1)) / 2.0
        pred2 = (preds1[1] + preds2[1].flip(-1)) / 2.0

    return pred1, pred2


def predict_next_frame(model_dict, context_frames: np.ndarray) -> np.ndarray:
    model = model_dict["model"]
    device = model_dict["device"]
    context_len = model_dict["context_len"]

    # Determine if this is an even or odd call
    # Even-length context (8, 10, 12, 14): run model, return pred1, cache pred2
    # Odd-length context (9, 11, 13, 15): return cached pred2
    n = len(context_frames)

    if n % 2 == 0:
        # Run model
        ctx, last = _prepare_input(context_frames, context_len)
        pred1, pred2 = _run_model_tta(model, device, ctx, last)

        # Cache pred2
        pred2_np = pred2[0].cpu().numpy()
        pred2_np = np.transpose(pred2_np, (1, 2, 0))
        model_dict["cached_pred2"] = (pred2_np * 255.0).clip(0, 255).astype(np.uint8)

        # Return pred1
        pred1_np = pred1[0].cpu().numpy()
        pred1_np = np.transpose(pred1_np, (1, 2, 0))
        return (pred1_np * 255.0).clip(0, 255).astype(np.uint8)
    else:
        # Return cached pred2
        if model_dict["cached_pred2"] is not None:
            return model_dict["cached_pred2"]
        else:
            # Fallback: run model on context minus last frame
            ctx, last = _prepare_input(context_frames[:-1], context_len)
            pred1, pred2 = _run_model_tta(model, device, ctx, last)
            pred2_np = pred2[0].cpu().numpy()
            pred2_np = np.transpose(pred2_np, (1, 2, 0))
            return (pred2_np * 255.0).clip(0, 255).astype(np.uint8)