"""Inference for multi-frame predictor with caching + TTA.""" import json import numpy as np import torch import sys sys.path.insert(0, "/home/coder/code") from multi_frame_model import MultiFrameFlowWarpUNet def load_model(model_dir: str): with open(f"{model_dir}/config.json") as f: config = json.load(f) model = MultiFrameFlowWarpUNet( in_channels=config["in_channels"], channels=config["channels"], num_future=config.get("num_future", 8), ) sd = torch.load(f"{model_dir}/model.pt", map_location="cpu", weights_only=True) sd = {k: v.float() for k, v in sd.items()} model.load_state_dict(sd) model.eval() device = torch.device("cuda" if torch.cuda.is_available() else "cpu") model = model.to(device) return { "model": model, "device": device, "context_len": config["context_len"], "num_future": config.get("num_future", 8), "cache": None, # Will store (context_hash, predictions_list) "call_count": 0, } def _context_hash(context_frames): """Hash first frame to identify a rollout.""" return context_frames[0].tobytes()[:1024] def _prepare_input(context_frames, context_len): N = len(context_frames) if N >= context_len: frames = context_frames[-context_len:] else: pad = np.repeat(context_frames[:1], context_len - N, axis=0) frames = np.concatenate([pad, context_frames], axis=0) frames_f = frames.astype(np.float32) / 255.0 frames_f = np.transpose(frames_f, (0, 3, 1, 2)) context = frames_f.reshape(1, -1, 64, 64) last_frame = frames_f[-1:] return context, last_frame def _run_model_with_tta(model, device, context_frames, context_len): """Run model with TTA (horizontal flip) and return all 8 predictions.""" ctx, last = _prepare_input(context_frames, context_len) with torch.no_grad(): ctx_t = torch.from_numpy(ctx).to(device) last_t = torch.from_numpy(last).to(device) preds1, _ = model(ctx_t, last_t) flipped_frames = context_frames[:, :, ::-1, :].copy() ctx_f, last_f = _prepare_input(flipped_frames, context_len) with torch.no_grad(): ctx_ft = torch.from_numpy(ctx_f).to(device) last_ft = torch.from_numpy(last_f).to(device) preds2, _ = model(ctx_ft, last_ft) result = [] for p1, p2 in zip(preds1, preds2): p2_flipped = p2.flip(-1) avg = (p1 + p2_flipped) / 2.0 pred_np = avg[0].cpu().numpy() pred_np = np.transpose(pred_np, (1, 2, 0)) result.append((pred_np * 255.0).clip(0, 255).astype(np.uint8)) return result def predict_next_frame(model_dict, context_frames: np.ndarray) -> np.ndarray: model = model_dict["model"] device = model_dict["device"] context_len = model_dict["context_len"] # Check if we have a cached prediction for this rollout ctx_hash = _context_hash(context_frames) if model_dict["cache"] is not None: cached_hash, cached_preds, cached_step = model_dict["cache"] if cached_hash == ctx_hash and cached_step < len(cached_preds): pred = cached_preds[cached_step] model_dict["cache"] = (cached_hash, cached_preds, cached_step + 1) return pred # No cache hit - run full model prediction # Use only the original context (first context_len frames) base_context = context_frames[:context_len] if len(context_frames) >= context_len else context_frames all_preds = _run_model_with_tta(model, device, base_context, context_len) # Return first prediction and cache the rest model_dict["cache"] = (ctx_hash, all_preds, 1) return all_preds[0]