video-model-factory / runtime.py
Quantumbraid's picture
Upload 8 files
748008a verified
Raw History Blame Contribute Delete
4.41 kB
from __future__ import annotations
import gc
import inspect
import random
import threading
import uuid
from pathlib import Path
import cv2
import torch
from diffusers import DiffusionPipeline
from diffusers.utils import export_to_video
from PIL import Image
from storage import OUTPUTS_ROOT, artifact_path
_pipe = None
_pipe_source: str | None = None
_lock = threading.RLock()
def _load_pipeline(model_source: str):
global _pipe, _pipe_source
with _lock:
if _pipe is not None and _pipe_source == model_source:
return _pipe
if _pipe is not None:
del _pipe
_pipe = None
gc.collect()
if torch.cuda.is_available():
torch.cuda.empty_cache()
model_path = artifact_path(model_source, "model")
_pipe = DiffusionPipeline.from_pretrained(
str(model_path),
dtype=torch.bfloat16,
local_files_only=True,
)
if hasattr(_pipe, "enable_model_cpu_offload"):
_pipe.enable_model_cpu_offload()
else:
_pipe.to("cuda")
_pipe_source = model_source
return _pipe
def _set_loras(pipe, lora_paths: list[str], lora_scale: float) -> None:
if hasattr(pipe, "unload_lora_weights"):
try:
pipe.unload_lora_weights()
except Exception:
pass
adapter_names = []
for index, raw_path in enumerate(lora_paths):
path = Path(raw_path)
name = f"factory_{index}"
if path.is_file():
pipe.load_lora_weights(str(path.parent), weight_name=path.name, adapter_name=name)
else:
pipe.load_lora_weights(str(path), adapter_name=name)
adapter_names.append(name)
if adapter_names and hasattr(pipe, "set_adapters"):
pipe.set_adapters(adapter_names, adapter_weights=[float(lora_scale)] * len(adapter_names))
def _accepted(pipe) -> set[str]:
signature = inspect.signature(pipe.__call__)
return set(signature.parameters)
def render_video(
model_source: str,
lora_paths: list[str],
prompt: str,
negative_prompt: str,
first_frame: Image.Image | None,
last_frame: Image.Image | None,
width: int,
height: int,
frames: int,
fps: int,
steps: int,
guidance: float,
lora_scale: float,
seed: int,
) -> tuple[str, str, int]:
if not model_source:
raise ValueError("Choose a model from factory.txt")
if first_frame is None:
raise ValueError("A first frame is required for the current video pipeline")
seed = random.randint(0, 2**31 - 1) if int(seed) < 0 else int(seed)
pipe = _load_pipeline(model_source)
_set_loras(pipe, lora_paths or [], lora_scale)
accepted = _accepted(pipe)
values = {
"prompt": prompt,
"negative_prompt": negative_prompt,
"image": first_frame.convert("RGB"),
"last_image": last_frame.convert("RGB") if last_frame is not None else None,
"width": int(width),
"height": int(height),
"num_frames": int(frames),
"num_inference_steps": int(steps),
"guidance_scale": float(guidance),
"generator": torch.Generator(device="cpu").manual_seed(seed),
}
kwargs = {key: value for key, value in values.items() if key in accepted and value is not None}
result = pipe(**kwargs)
generated = result.frames
if isinstance(generated, list) and generated and isinstance(generated[0], list):
generated = generated[0]
elif getattr(generated, "ndim", None) == 5:
generated = generated[0]
OUTPUTS_ROOT.mkdir(parents=True, exist_ok=True)
video_path = OUTPUTS_ROOT / f"render-{uuid.uuid4().hex}.mp4"
export_to_video(generated, str(video_path), fps=int(fps))
last_path = extract_last_frame(str(video_path))
return str(video_path), last_path, seed
def extract_last_frame(video_path: str) -> str:
capture = cv2.VideoCapture(video_path)
count = int(capture.get(cv2.CAP_PROP_FRAME_COUNT))
capture.set(cv2.CAP_PROP_POS_FRAMES, max(0, count - 1))
ok, frame = capture.read()
capture.release()
if not ok:
raise RuntimeError("Could not extract the rendered video's last frame")
frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
target = OUTPUTS_ROOT / f"last-frame-{uuid.uuid4().hex}.png"
Image.fromarray(frame).save(target)
return str(target)