from __future__ import annotations import gc import inspect import random import threading import uuid from pathlib import Path import cv2 import torch from diffusers import DiffusionPipeline from diffusers.utils import export_to_video from PIL import Image from storage import OUTPUTS_ROOT, artifact_path _pipe = None _pipe_source: str | None = None _lock = threading.RLock() def _load_pipeline(model_source: str): global _pipe, _pipe_source with _lock: if _pipe is not None and _pipe_source == model_source: return _pipe if _pipe is not None: del _pipe _pipe = None gc.collect() if torch.cuda.is_available(): torch.cuda.empty_cache() model_path = artifact_path(model_source, "model") _pipe = DiffusionPipeline.from_pretrained( str(model_path), dtype=torch.bfloat16, local_files_only=True, ) if hasattr(_pipe, "enable_model_cpu_offload"): _pipe.enable_model_cpu_offload() else: _pipe.to("cuda") _pipe_source = model_source return _pipe def _set_loras(pipe, lora_paths: list[str], lora_scale: float) -> None: if hasattr(pipe, "unload_lora_weights"): try: pipe.unload_lora_weights() except Exception: pass adapter_names = [] for index, raw_path in enumerate(lora_paths): path = Path(raw_path) name = f"factory_{index}" if path.is_file(): pipe.load_lora_weights(str(path.parent), weight_name=path.name, adapter_name=name) else: pipe.load_lora_weights(str(path), adapter_name=name) adapter_names.append(name) if adapter_names and hasattr(pipe, "set_adapters"): pipe.set_adapters(adapter_names, adapter_weights=[float(lora_scale)] * len(adapter_names)) def _accepted(pipe) -> set[str]: signature = inspect.signature(pipe.__call__) return set(signature.parameters) def render_video( model_source: str, lora_paths: list[str], prompt: str, negative_prompt: str, first_frame: Image.Image | None, last_frame: Image.Image | None, width: int, height: int, frames: int, fps: int, steps: int, guidance: float, lora_scale: float, seed: int, ) -> tuple[str, str, int]: if not model_source: raise ValueError("Choose a model from factory.txt") if first_frame is None: raise ValueError("A first frame is required for the current video pipeline") seed = random.randint(0, 2**31 - 1) if int(seed) < 0 else int(seed) pipe = _load_pipeline(model_source) _set_loras(pipe, lora_paths or [], lora_scale) accepted = _accepted(pipe) values = { "prompt": prompt, "negative_prompt": negative_prompt, "image": first_frame.convert("RGB"), "last_image": last_frame.convert("RGB") if last_frame is not None else None, "width": int(width), "height": int(height), "num_frames": int(frames), "num_inference_steps": int(steps), "guidance_scale": float(guidance), "generator": torch.Generator(device="cpu").manual_seed(seed), } kwargs = {key: value for key, value in values.items() if key in accepted and value is not None} result = pipe(**kwargs) generated = result.frames if isinstance(generated, list) and generated and isinstance(generated[0], list): generated = generated[0] elif getattr(generated, "ndim", None) == 5: generated = generated[0] OUTPUTS_ROOT.mkdir(parents=True, exist_ok=True) video_path = OUTPUTS_ROOT / f"render-{uuid.uuid4().hex}.mp4" export_to_video(generated, str(video_path), fps=int(fps)) last_path = extract_last_frame(str(video_path)) return str(video_path), last_path, seed def extract_last_frame(video_path: str) -> str: capture = cv2.VideoCapture(video_path) count = int(capture.get(cv2.CAP_PROP_FRAME_COUNT)) capture.set(cv2.CAP_PROP_POS_FRAMES, max(0, count - 1)) ok, frame = capture.read() capture.release() if not ok: raise RuntimeError("Could not extract the rendered video's last frame") frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) target = OUTPUTS_ROOT / f"last-frame-{uuid.uuid4().hex}.png" Image.fromarray(frame).save(target) return str(target)