"""Inference for 2-frame simultaneous predictor with TTA.""" import json import numpy as np import torch import sys sys.path.insert(0, "/home/coder/code") from flow_warp_attn_model import FlowWarpAttnUNet def load_model(model_dir: str): with open(f"{model_dir}/config.json") as f: config = json.load(f) model = FlowWarpAttnUNet( in_channels=config["in_channels"], channels=config["channels"], out_channels=config.get("out_channels", 6) ) sd = torch.load(f"{model_dir}/model.pt", map_location="cpu", weights_only=True) sd = {k: v.float() for k, v in sd.items()} model.load_state_dict(sd) model.eval() device = torch.device("cuda" if torch.cuda.is_available() else "cpu") model = model.to(device) return { "model": model, "device": device, "context_len": config["context_len"], "cached_pred2": None, } def _prepare_input(context_frames, context_len): N = len(context_frames) if N >= context_len: frames = context_frames[-context_len:] else: pad = np.repeat(context_frames[:1], context_len - N, axis=0) frames = np.concatenate([pad, context_frames], axis=0) frames_f = frames.astype(np.float32) / 255.0 frames_f = np.transpose(frames_f, (0, 3, 1, 2)) context = frames_f.reshape(1, -1, 64, 64) last_frame = frames_f[-1:] return context, last_frame def _run_model_tta(model, device, ctx, last): """Run model with TTA (horizontal flip) and return both predictions.""" with torch.no_grad(): ctx_t = torch.from_numpy(ctx).to(device) last_t = torch.from_numpy(last).to(device) preds1, _ = model(ctx_t, last_t) # Flipped ctx_f = torch.from_numpy(ctx[:, :, :, ::-1].copy()).to(device) last_f = torch.from_numpy(last[:, :, :, ::-1].copy()).to(device) preds2, _ = model(ctx_f, last_f) # Average normal and flipped-back predictions pred1 = (preds1[0] + preds2[0].flip(-1)) / 2.0 pred2 = (preds1[1] + preds2[1].flip(-1)) / 2.0 return pred1, pred2 def predict_next_frame(model_dict, context_frames: np.ndarray) -> np.ndarray: model = model_dict["model"] device = model_dict["device"] context_len = model_dict["context_len"] # Determine if this is an even or odd call # Even-length context (8, 10, 12, 14): run model, return pred1, cache pred2 # Odd-length context (9, 11, 13, 15): return cached pred2 n = len(context_frames) if n % 2 == 0: # Run model ctx, last = _prepare_input(context_frames, context_len) pred1, pred2 = _run_model_tta(model, device, ctx, last) # Cache pred2 pred2_np = pred2[0].cpu().numpy() pred2_np = np.transpose(pred2_np, (1, 2, 0)) model_dict["cached_pred2"] = (pred2_np * 255.0).clip(0, 255).astype(np.uint8) # Return pred1 pred1_np = pred1[0].cpu().numpy() pred1_np = np.transpose(pred1_np, (1, 2, 0)) return (pred1_np * 255.0).clip(0, 255).astype(np.uint8) else: # Return cached pred2 if model_dict["cached_pred2"] is not None: return model_dict["cached_pred2"] else: # Fallback: run model on context minus last frame ctx, last = _prepare_input(context_frames[:-1], context_len) pred1, pred2 = _run_model_tta(model, device, ctx, last) pred2_np = pred2[0].cpu().numpy() pred2_np = np.transpose(pred2_np, (1, 2, 0)) return (pred2_np * 255.0).clip(0, 255).astype(np.uint8)