""" frame-compositor — GPU video compositor (ZeroGPU) Bilinear quad warp matching JS editor preview exactly. Optimizations: static layer composite cache, threaded audio+encode, precomputed frame ranges, pinned memory transfers, parallel layer pre-render. """ import json import os import queue import tempfile import threading import urllib.request import urllib.parse import re from concurrent.futures import ThreadPoolExecutor from io import BytesIO from typing import List, Dict, Tuple, Optional import subprocess import gradio as gr import spaces import numpy as np import torch import torch.nn.functional as F from PIL import Image, ImageDraw, ImageFont import av from torchcodec.decoders import VideoDecoder VIDEO_WIDTH = 540 VIDEO_HEIGHT = 960 FPS = 30 BATCH = 32 DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu") # ── Font cache ──────────────────────────────────────────────────────────────── _FONT_CACHE: Dict[str, str] = {} def _download_font(family: str, weight: int = 400, cache_dir: str = "/tmp/fonts") -> Optional[str]: os.makedirs(cache_dir, exist_ok=True) key = f"{family}_{weight}" if key in _FONT_CACHE: return _FONT_CACHE[key] slug = family.replace(" ", "_") path = os.path.join(cache_dir, f"{slug}_{weight}.ttf") if os.path.exists(path): _FONT_CACHE[key] = path return path try: url = f"https://fonts.googleapis.com/css2?family={family.replace(' ', '+')}:ital,wght@0,{weight};1,{weight}&display=swap" req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"}) with urllib.request.urlopen(req, timeout=10) as r: css = r.read().decode("utf-8") ttf_url = None for block in re.split(r"@font-face\s*\{", css)[1:]: if re.search(rf"font-weight:\s*{weight}", block): m = re.search(r"url\((https://[^)]+\.(?:ttf|otf))\)", block) if m: ttf_url = m.group(1) break if not ttf_url: m = re.search(r"url\((https://[^)]+\.(?:ttf|otf))\)", css) if m: ttf_url = m.group(1) if not ttf_url: print(f"[font] No TTF/OTF URL found for {family} w{weight}", flush=True) return None with urllib.request.urlopen(ttf_url, timeout=10) as r: with open(path, "wb") as f: f.write(r.read()) _FONT_CACHE[key] = path print(f"[font] Downloaded {family} w{weight} → {path}", flush=True) return path except Exception as e: print(f"[font] Failed for {family}: {e}", flush=True) return None # ── Layer pre-rendering ─────────────────────────────────────────────────────── def _wrap_text(text: str, font: ImageFont.FreeTypeFont, max_w: int) -> List[str]: lines = [] for paragraph in text.split("\n"): words = paragraph.split(" ") line = "" for word in words: test = f"{line} {word}".strip() bbox = font.getbbox(test) w = bbox[2] - bbox[0] if bbox else 0 if w > max_w and line: lines.append(line) line = word else: line = test if line: lines.append(line) return lines or [""] def _render_text_layer(layer: Dict) -> np.ndarray: W, H = 800, 200 img = Image.new("RGBA", (W, H), (0, 0, 0, 0)) draw = ImageDraw.Draw(img) bg_hex = layer["bgColor"] bg_r, bg_g, bg_b = int(bg_hex[1:3], 16), int(bg_hex[3:5], 16), int(bg_hex[5:7], 16) bg_a = int(layer["bgOpacity"] * 255) draw.rectangle([(0, 0), (W, H)], fill=(bg_r, bg_g, bg_b, bg_a)) font_size = int(layer["fontSize"] * 2) weight = layer.get("fontWeight", 400) font_path = _download_font(layer["font"], weight) padding = 16 avail_w = W - padding * 2 avail_h = H - padding * 2 fit_size = font_size fit_lines = [layer["text"]] for size in range(font_size, 7, -1): try: f = ImageFont.truetype(font_path, size) if font_path else ImageFont.load_default() except Exception: f = ImageFont.load_default() lines = _wrap_text(layer["text"], f, avail_w) if len(lines) * size * 1.2 <= avail_h: fit_size = size fit_lines = lines break try: font = ImageFont.truetype(font_path, fit_size) if font_path else ImageFont.load_default() except Exception: font = ImageFont.load_default() fg_hex = layer["color"] fg_r, fg_g, fg_b = int(fg_hex[1:3], 16), int(fg_hex[3:5], 16), int(fg_hex[5:7], 16) line_h = fit_size * 1.2 total_h = len(fit_lines) * line_h y = (H - total_h) / 2 + line_h / 2 for i, line in enumerate(fit_lines): draw.text((W // 2, y + i * line_h), line, font=font, fill=(fg_r, fg_g, fg_b, 255), anchor="mm") return np.array(img) # (H, W, 4) RGBA uint8 def _render_image_layer(layer: Dict) -> Optional[np.ndarray]: src = layer.get("src", "") if not src: return None try: req = urllib.request.Request(src, headers={"User-Agent": "Mozilla/5.0"}) with urllib.request.urlopen(req, timeout=15) as r: data = r.read() img = Image.open(BytesIO(data)).convert("RGBA").resize((800, 800), Image.LANCZOS) opacity = layer.get("opacity", 1.0) if opacity < 1.0: r2, g2, b2, a2 = img.split() a2 = a2.point(lambda x: int(x * opacity)) img = Image.merge("RGBA", (r2, g2, b2, a2)) print(f"[layer] Loaded image {src}", flush=True) return np.array(img) except Exception as e: print(f"[layer] Failed to load image {src}: {e}", flush=True) return None def _render_layer(layer: Dict) -> Tuple[str, Optional[np.ndarray]]: arr = _render_image_layer(layer) if layer.get("type") == "image" else _render_text_layer(layer) return layer["id"], arr def _rgba_to_gpu_tensor(arr: np.ndarray) -> torch.Tensor: """(H, W, 4) RGBA uint8 → (1, 4, H, W) float32 [0,1] on DEVICE.""" t = torch.from_numpy(arr).float() / 255.0 return t.permute(2, 0, 1).unsqueeze(0).to(DEVICE) # ── Keyframe interpolation ──────────────────────────────────────────────────── def _get_corners_at_frame(layer: Dict, f: int) -> Optional[List[Tuple[float, float]]]: kfs = sorted(layer["keyframes"], key=lambda k: k["frame"]) if not kfs: return None if f <= kfs[0]["frame"]: return [(c["x"], c["y"]) for c in kfs[0]["corners"]] if f >= kfs[-1]["frame"]: return [(c["x"], c["y"]) for c in kfs[-1]["corners"]] for i in range(len(kfs) - 1): k0, k1 = kfs[i], kfs[i + 1] if k0["frame"] <= f <= k1["frame"]: ratio = (f - k0["frame"]) / (k1["frame"] - k0["frame"]) return [ (k0["corners"][j]["x"] + (k1["corners"][j]["x"] - k0["corners"][j]["x"]) * ratio, k0["corners"][j]["y"] + (k1["corners"][j]["y"] - k0["corners"][j]["y"]) * ratio) for j in range(4) ] return [(c["x"], c["y"]) for c in kfs[0]["corners"]] def _is_static_layer(layer: Dict) -> bool: kfs = layer.get("keyframes", []) if len(kfs) <= 1: return True ref = kfs[0]["corners"] for kf in kfs[1:]: for j in range(4): if abs(kf["corners"][j]["x"] - ref[j]["x"]) > 0.01: return False if abs(kf["corners"][j]["y"] - ref[j]["y"]) > 0.01: return False return True # ── GPU bilinear quad warp ──────────────────────────────────────────────────── def _build_uv_map( corners_t: torch.Tensor, gx: torch.Tensor, gy: torch.Tensor, newton_iters: int = 8, u_init: Optional[torch.Tensor] = None, v_init: Optional[torch.Tensor] = None, ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: B = corners_t.shape[0] H, W = gx.shape tl_x = corners_t[:, 0, 0].view(B, 1, 1) tl_y = corners_t[:, 0, 1].view(B, 1, 1) tr_x = corners_t[:, 1, 0].view(B, 1, 1) tr_y = corners_t[:, 1, 1].view(B, 1, 1) bl_x = corners_t[:, 3, 0].view(B, 1, 1) bl_y = corners_t[:, 3, 1].view(B, 1, 1) br_x = corners_t[:, 2, 0].view(B, 1, 1) br_y = corners_t[:, 2, 1].view(B, 1, 1) px = gx.unsqueeze(0).expand(B, -1, -1) py = gy.unsqueeze(0).expand(B, -1, -1) u = u_init.clone() if u_init is not None else torch.full((B, H, W), 0.5, dtype=torch.float32, device=DEVICE) v = v_init.clone() if v_init is not None else torch.full((B, H, W), 0.5, dtype=torch.float32, device=DEVICE) for _ in range(newton_iters): top_x = tl_x + (tr_x - tl_x) * u top_y = tl_y + (tr_y - tl_y) * u bot_x = bl_x + (br_x - bl_x) * u bot_y = bl_y + (br_y - bl_y) * u fx = top_x + (bot_x - top_x) * v fy = top_y + (bot_y - top_y) * v dfx_du = (tr_x - tl_x) + ((br_x - bl_x) - (tr_x - tl_x)) * v dfy_du = (tr_y - tl_y) + ((br_y - bl_y) - (tr_y - tl_y)) * v dfx_dv = bot_x - top_x dfy_dv = bot_y - top_y det = dfx_du * dfy_dv - dfx_dv * dfy_du det = torch.where(det.abs() < 1e-8, torch.full_like(det, 1e-8), det) rx, ry = px - fx, py - fy u = u + (dfy_dv * rx - dfx_dv * ry) / det v = v + (dfx_du * ry - dfy_du * rx) / det u = u.clamp(0.0, 1.0) v = v.clamp(0.0, 1.0) top_x = tl_x + (tr_x - tl_x) * u top_y = tl_y + (tr_y - tl_y) * u bot_x = bl_x + (br_x - bl_x) * u bot_y = bl_y + (br_y - bl_y) * u fx = top_x + (bot_x - top_x) * v fy = top_y + (bot_y - top_y) * v inside = (((px - fx) ** 2 + (py - fy) ** 2).sqrt() < 1.5).float().unsqueeze(1) return u, v, inside def _sample_layer( layer_t: torch.Tensor, u: torch.Tensor, v: torch.Tensor, inside: torch.Tensor, B: int, ) -> Tuple[torch.Tensor, torch.Tensor]: grid = torch.stack([u * 2 - 1, v * 2 - 1], dim=-1) sampled = F.grid_sample( layer_t.expand(B, -1, -1, -1), grid, mode="bilinear", padding_mode="zeros", align_corners=True, ) return sampled[:, :3], sampled[:, 3:4] * inside # ── Main render ─────────────────────────────────────────────────────────────── @spaces.GPU(duration=10) def render(source_video_url: str, layers_json: str, progress=gr.Progress()) -> str: layers = json.loads(layers_json) print(f"[render] {len(layers)} layer(s), source: {source_video_url}", flush=True) progress(0.0, desc="Downloading source video...") tmp_in = tempfile.NamedTemporaryFile(delete=False, suffix=".mp4") req = urllib.request.Request(source_video_url, headers={"User-Agent": "Mozilla/5.0"}) with urllib.request.urlopen(req, timeout=120) as r: tmp_in.write(r.read()) tmp_in.close() # ── Parallel layer pre-render ────────────────────────────────────────── progress(0.05, desc="Pre-rendering layers...") # Parallel layer rendering layer_arrays: Dict[str, np.ndarray] = {} with ThreadPoolExecutor(max_workers=min(len(layers), 4)) as ex: for lid, arr in ex.map(_render_layer, layers): if arr is not None: layer_arrays[lid] = arr print(f"[render] Pre-rendered {len(layer_arrays)} layer(s)", flush=True) # ── GPU setup ───────────────────────────────────────────────────────── progress(0.1, desc="Decoding video...") layer_tensors: Dict[str, torch.Tensor] = { lid: _rgba_to_gpu_tensor(arr) for lid, arr in layer_arrays.items() } decoder = VideoDecoder(tmp_in.name, device=str(DEVICE)) metadata = decoder.metadata total_frames = metadata.num_frames actual_fps = float(metadata.average_fps) if metadata.average_fps else FPS print(f"[render] {total_frames} frames @ {actual_fps:.2f} fps", flush=True) # ── Precompute pixel grid ────────────────────────────────────────────── gy_grid, gx_grid = torch.meshgrid( torch.arange(VIDEO_HEIGHT, dtype=torch.float32, device=DEVICE), torch.arange(VIDEO_WIDTH, dtype=torch.float32, device=DEVICE), indexing="ij", ) # ── Precompute per-layer frame ranges ───────────────────────────────── layer_frame_ranges: Dict[str, Tuple[int, int]] = {} for layer in layers: start_f = layer["startFrame"] end_f = layer["endFrame"] + 1 layer_frame_ranges[layer["id"]] = (start_f, end_f) # ── Static layer: precompute warped composite (rgb*alpha, alpha) once ── # For static layers the result is the same for every frame — skip grid_sample entirely. static_composite: Dict[str, Tuple[torch.Tensor, torch.Tensor]] = {} static_uv: Dict[str, Tuple[torch.Tensor, torch.Tensor, torch.Tensor]] = {} for layer in layers: lid = layer["id"] lt = layer_tensors.get(lid) if lt is None: continue if not _is_static_layer(layer): continue corners = _get_corners_at_frame(layer, layer["keyframes"][0]["frame"]) corners_t = torch.tensor([corners], dtype=torch.float32, device=DEVICE) u, v, inside = _build_uv_map(corners_t, gx_grid, gy_grid) rgb, alpha = _sample_layer(lt, u, v, inside, 1) # store pre-multiplied: rgb_pre = rgb*alpha, alpha — both (1, C, H, W) static_composite[lid] = (rgb * alpha, alpha) print(f"[render] Static composite cached for layer {lid}", flush=True) # ── Animated layer warm-start ────────────────────────────────────────── prev_uv: Dict[str, Tuple[torch.Tensor, torch.Tensor]] = {} # ── Pinned output buffer ─────────────────────────────────────────────── pinned_buf = torch.empty( (BATCH, VIDEO_HEIGHT, VIDEO_WIDTH, 3), dtype=torch.uint8, pin_memory=True ) # ── Encode thread ────────────────────────────────────────────────────── progress(0.15, desc="Rendering frames...") tmp_out = tempfile.NamedTemporaryFile(delete=False, suffix=".mp4") tmp_out.close() encode_q: queue.Queue = queue.Queue(maxsize=4) encode_error: list = [] def _encode_worker(): try: with av.open(tmp_out.name, "w") as dst: v_stream = dst.add_stream("libx264", rate=int(round(actual_fps))) v_stream.width = VIDEO_WIDTH v_stream.height = VIDEO_HEIGHT v_stream.pix_fmt = "yuv420p" v_stream.options = {"preset": "ultrafast", "crf": "23"} while True: item = encode_q.get() if item is None: break for frame_rgb in item: av_frame = av.VideoFrame.from_ndarray(frame_rgb, format="rgb24") for pkt in v_stream.encode(av_frame): dst.mux(pkt) for pkt in v_stream.encode(None): dst.mux(pkt) except Exception as e: encode_error.append(e) enc_thread = threading.Thread(target=_encode_worker, daemon=True) enc_thread.start() # ── GPU composite loop ───────────────────────────────────────────────── for batch_start in range(0, total_frames, BATCH): batch_end = min(batch_start + BATCH, total_frames) B = batch_end - batch_start clip = decoder[batch_start:batch_end] frames_f = clip.data.to(DEVICE).float() / 255.0 if frames_f.shape[2] != VIDEO_HEIGHT or frames_f.shape[3] != VIDEO_WIDTH: frames_f = F.interpolate(frames_f, size=(VIDEO_HEIGHT, VIDEO_WIDTH), mode="bilinear", align_corners=False) for layer in layers: lid = layer["id"] lt = layer_tensors.get(lid) if lt is None: continue start_f, end_f = layer_frame_ranges[lid] # which indices in this batch are active active_idx = [i for i in range(B) if start_f <= batch_start + i < end_f] if not active_idx: continue if lid in static_composite: rgb_pre, alpha = static_composite[lid] # (1, 3, H, W), (1, 1, H, W) if len(active_idx) == B: frames_f = rgb_pre + frames_f * (1.0 - alpha) else: sub = rgb_pre + frames_f[active_idx] * (1.0 - alpha) frames_f = frames_f.clone() for out_i, src_i in enumerate(active_idx): frames_f[src_i] = sub[out_i] else: # animated — build UV per-frame with warm-start corners_list = [] for i in active_idx: corners = _get_corners_at_frame(layer, batch_start + i) corners_list.append(corners) Ab = len(active_idx) corners_t = torch.tensor(corners_list, dtype=torch.float32, device=DEVICE) u_init, v_init = prev_uv.get(lid, (None, None)) if u_init is not None and u_init.shape[0] != Ab: u_init = u_init[:Ab] if u_init.shape[0] > Ab else None v_init = v_init[:Ab] if v_init is not None and v_init.shape[0] > Ab else None u, v, inside = _build_uv_map(corners_t, gx_grid, gy_grid, u_init=u_init, v_init=v_init) prev_uv[lid] = (u.detach(), v.detach()) rgb, alpha = _sample_layer(lt, u, v, inside, Ab) if Ab == B: frames_f = rgb * alpha + frames_f * (1.0 - alpha) else: sub = rgb * alpha + frames_f[active_idx] * (1.0 - alpha) frames_f = frames_f.clone() for out_i, src_i in enumerate(active_idx): frames_f[src_i] = sub[out_i] # (B, 3, H, W) float → pinned uint8 → CPU numpy via async DMA out_gpu = (frames_f.permute(0, 2, 3, 1).clamp(0, 1) * 255).byte() pinned_buf[:B].copy_(out_gpu, non_blocking=True) torch.cuda.synchronize() encode_q.put(pinned_buf[:B].numpy().copy()) done = batch_end / total_frames progress(0.15 + done * 0.82, desc=f"Rendering {batch_end}/{total_frames} frames") print(f"[render] Processed {batch_end}/{total_frames} frames", flush=True) encode_q.put(None) enc_thread.join() if encode_error: raise encode_error[0] # ── Merge audio from source via ffmpeg stream copy ───────────────────── tmp_final = tempfile.NamedTemporaryFile(delete=False, suffix=".mp4") tmp_final.close() result = subprocess.run([ "ffmpeg", "-y", "-i", tmp_out.name, "-i", tmp_in.name, "-c:v", "copy", "-c:a", "copy", "-map", "0:v:0", "-map", "1:a:0?", "-shortest", tmp_final.name ], capture_output=True) if result.returncode != 0: print(f"[render] ffmpeg audio merge failed: {result.stderr.decode()}", flush=True) os.rename(tmp_out.name, tmp_final.name) else: os.unlink(tmp_out.name) os.unlink(tmp_in.name) progress(1.0, desc="Done") print(f"[render] Output: {tmp_final.name}", flush=True) return tmp_final.name # ── Gradio app ──────────────────────────────────────────────────────────────── with gr.Blocks() as demo: with gr.Row(visible=False): source_video_url = gr.Textbox(label="source_video_url") layers_json = gr.Textbox(label="layers_json") output_video = gr.File(label="output_video") gr.Button("render", visible=False).click( fn=render, inputs=[source_video_url, layers_json], outputs=[output_video], api_name="render", ) if __name__ == "__main__": demo.launch()