2026-04-12-183000-multi-frame-predictor/predict.py

"""Inference for multi-frame predictor with caching + TTA."""
import json
import numpy as np
import torch
import sys
sys.path.insert(0, "/home/coder/code")
from multi_frame_model import MultiFrameFlowWarpUNet


def load_model(model_dir: str):
    with open(f"{model_dir}/config.json") as f:
        config = json.load(f)
    model = MultiFrameFlowWarpUNet(
        in_channels=config["in_channels"],
        channels=config["channels"],
        num_future=config.get("num_future", 8),
    )
    sd = torch.load(f"{model_dir}/model.pt", map_location="cpu", weights_only=True)
    sd = {k: v.float() for k, v in sd.items()}
    model.load_state_dict(sd)
    model.eval()
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    model = model.to(device)
    return {
        "model": model,
        "device": device,
        "context_len": config["context_len"],
        "num_future": config.get("num_future", 8),
        "cache": None,  # Will store (context_hash, predictions_list)
        "call_count": 0,
    }


def _context_hash(context_frames):
    """Hash first frame to identify a rollout."""
    return context_frames[0].tobytes()[:1024]


def _prepare_input(context_frames, context_len):
    N = len(context_frames)
    if N >= context_len:
        frames = context_frames[-context_len:]
    else:
        pad = np.repeat(context_frames[:1], context_len - N, axis=0)
        frames = np.concatenate([pad, context_frames], axis=0)
    frames_f = frames.astype(np.float32) / 255.0
    frames_f = np.transpose(frames_f, (0, 3, 1, 2))
    context = frames_f.reshape(1, -1, 64, 64)
    last_frame = frames_f[-1:]
    return context, last_frame


def _run_model_with_tta(model, device, context_frames, context_len):
    """Run model with TTA (horizontal flip) and return all 8 predictions."""
    ctx, last = _prepare_input(context_frames, context_len)
    with torch.no_grad():
        ctx_t = torch.from_numpy(ctx).to(device)
        last_t = torch.from_numpy(last).to(device)
        preds1, _ = model(ctx_t, last_t)

    flipped_frames = context_frames[:, :, ::-1, :].copy()
    ctx_f, last_f = _prepare_input(flipped_frames, context_len)
    with torch.no_grad():
        ctx_ft = torch.from_numpy(ctx_f).to(device)
        last_ft = torch.from_numpy(last_f).to(device)
        preds2, _ = model(ctx_ft, last_ft)

    result = []
    for p1, p2 in zip(preds1, preds2):
        p2_flipped = p2.flip(-1)
        avg = (p1 + p2_flipped) / 2.0
        pred_np = avg[0].cpu().numpy()
        pred_np = np.transpose(pred_np, (1, 2, 0))
        result.append((pred_np * 255.0).clip(0, 255).astype(np.uint8))
    return result


def predict_next_frame(model_dict, context_frames: np.ndarray) -> np.ndarray:
    model = model_dict["model"]
    device = model_dict["device"]
    context_len = model_dict["context_len"]

    # Check if we have a cached prediction for this rollout
    ctx_hash = _context_hash(context_frames)

    if model_dict["cache"] is not None:
        cached_hash, cached_preds, cached_step = model_dict["cache"]
        if cached_hash == ctx_hash and cached_step < len(cached_preds):
            pred = cached_preds[cached_step]
            model_dict["cache"] = (cached_hash, cached_preds, cached_step + 1)
            return pred

    # No cache hit - run full model prediction
    # Use only the original context (first context_len frames)
    base_context = context_frames[:context_len] if len(context_frames) >= context_len else context_frames
    all_preds = _run_model_with_tta(model, device, base_context, context_len)

    # Return first prediction and cache the rest
    model_dict["cache"] = (ctx_hash, all_preds, 1)
    return all_preds[0]