File size: 4,406 Bytes
a0d2a22 748008a a0d2a22 bb9b12e a0d2a22 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 | from __future__ import annotations
import gc
import inspect
import random
import threading
import uuid
from pathlib import Path
import cv2
import torch
from diffusers import DiffusionPipeline
from diffusers.utils import export_to_video
from PIL import Image
from storage import OUTPUTS_ROOT, artifact_path
_pipe = None
_pipe_source: str | None = None
_lock = threading.RLock()
def _load_pipeline(model_source: str):
global _pipe, _pipe_source
with _lock:
if _pipe is not None and _pipe_source == model_source:
return _pipe
if _pipe is not None:
del _pipe
_pipe = None
gc.collect()
if torch.cuda.is_available():
torch.cuda.empty_cache()
model_path = artifact_path(model_source, "model")
_pipe = DiffusionPipeline.from_pretrained(
str(model_path),
dtype=torch.bfloat16,
local_files_only=True,
)
if hasattr(_pipe, "enable_model_cpu_offload"):
_pipe.enable_model_cpu_offload()
else:
_pipe.to("cuda")
_pipe_source = model_source
return _pipe
def _set_loras(pipe, lora_paths: list[str], lora_scale: float) -> None:
if hasattr(pipe, "unload_lora_weights"):
try:
pipe.unload_lora_weights()
except Exception:
pass
adapter_names = []
for index, raw_path in enumerate(lora_paths):
path = Path(raw_path)
name = f"factory_{index}"
if path.is_file():
pipe.load_lora_weights(str(path.parent), weight_name=path.name, adapter_name=name)
else:
pipe.load_lora_weights(str(path), adapter_name=name)
adapter_names.append(name)
if adapter_names and hasattr(pipe, "set_adapters"):
pipe.set_adapters(adapter_names, adapter_weights=[float(lora_scale)] * len(adapter_names))
def _accepted(pipe) -> set[str]:
signature = inspect.signature(pipe.__call__)
return set(signature.parameters)
def render_video(
model_source: str,
lora_paths: list[str],
prompt: str,
negative_prompt: str,
first_frame: Image.Image | None,
last_frame: Image.Image | None,
width: int,
height: int,
frames: int,
fps: int,
steps: int,
guidance: float,
lora_scale: float,
seed: int,
) -> tuple[str, str, int]:
if not model_source:
raise ValueError("Choose a model from factory.txt")
if first_frame is None:
raise ValueError("A first frame is required for the current video pipeline")
seed = random.randint(0, 2**31 - 1) if int(seed) < 0 else int(seed)
pipe = _load_pipeline(model_source)
_set_loras(pipe, lora_paths or [], lora_scale)
accepted = _accepted(pipe)
values = {
"prompt": prompt,
"negative_prompt": negative_prompt,
"image": first_frame.convert("RGB"),
"last_image": last_frame.convert("RGB") if last_frame is not None else None,
"width": int(width),
"height": int(height),
"num_frames": int(frames),
"num_inference_steps": int(steps),
"guidance_scale": float(guidance),
"generator": torch.Generator(device="cpu").manual_seed(seed),
}
kwargs = {key: value for key, value in values.items() if key in accepted and value is not None}
result = pipe(**kwargs)
generated = result.frames
if isinstance(generated, list) and generated and isinstance(generated[0], list):
generated = generated[0]
elif getattr(generated, "ndim", None) == 5:
generated = generated[0]
OUTPUTS_ROOT.mkdir(parents=True, exist_ok=True)
video_path = OUTPUTS_ROOT / f"render-{uuid.uuid4().hex}.mp4"
export_to_video(generated, str(video_path), fps=int(fps))
last_path = extract_last_frame(str(video_path))
return str(video_path), last_path, seed
def extract_last_frame(video_path: str) -> str:
capture = cv2.VideoCapture(video_path)
count = int(capture.get(cv2.CAP_PROP_FRAME_COUNT))
capture.set(cv2.CAP_PROP_POS_FRAMES, max(0, count - 1))
ok, frame = capture.read()
capture.release()
if not ok:
raise RuntimeError("Could not extract the rendered video's last frame")
frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
target = OUTPUTS_ROOT / f"last-frame-{uuid.uuid4().hex}.png"
Image.fromarray(frame).save(target)
return str(target)
|