File size: 4,406 Bytes
a0d2a22
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
748008a
a0d2a22
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
bb9b12e
 
 
 
 
a0d2a22
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
from __future__ import annotations

import gc
import inspect
import random
import threading
import uuid
from pathlib import Path

import cv2
import torch
from diffusers import DiffusionPipeline
from diffusers.utils import export_to_video
from PIL import Image

from storage import OUTPUTS_ROOT, artifact_path


_pipe = None
_pipe_source: str | None = None
_lock = threading.RLock()


def _load_pipeline(model_source: str):
    global _pipe, _pipe_source
    with _lock:
        if _pipe is not None and _pipe_source == model_source:
            return _pipe
        if _pipe is not None:
            del _pipe
            _pipe = None
            gc.collect()
            if torch.cuda.is_available():
                torch.cuda.empty_cache()
        model_path = artifact_path(model_source, "model")
        _pipe = DiffusionPipeline.from_pretrained(
            str(model_path),
            dtype=torch.bfloat16,
            local_files_only=True,
        )
        if hasattr(_pipe, "enable_model_cpu_offload"):
            _pipe.enable_model_cpu_offload()
        else:
            _pipe.to("cuda")
        _pipe_source = model_source
        return _pipe


def _set_loras(pipe, lora_paths: list[str], lora_scale: float) -> None:
    if hasattr(pipe, "unload_lora_weights"):
        try:
            pipe.unload_lora_weights()
        except Exception:
            pass
    adapter_names = []
    for index, raw_path in enumerate(lora_paths):
        path = Path(raw_path)
        name = f"factory_{index}"
        if path.is_file():
            pipe.load_lora_weights(str(path.parent), weight_name=path.name, adapter_name=name)
        else:
            pipe.load_lora_weights(str(path), adapter_name=name)
        adapter_names.append(name)
    if adapter_names and hasattr(pipe, "set_adapters"):
        pipe.set_adapters(adapter_names, adapter_weights=[float(lora_scale)] * len(adapter_names))


def _accepted(pipe) -> set[str]:
    signature = inspect.signature(pipe.__call__)
    return set(signature.parameters)


def render_video(
    model_source: str,
    lora_paths: list[str],
    prompt: str,
    negative_prompt: str,
    first_frame: Image.Image | None,
    last_frame: Image.Image | None,
    width: int,
    height: int,
    frames: int,
    fps: int,
    steps: int,
    guidance: float,
    lora_scale: float,
    seed: int,
) -> tuple[str, str, int]:
    if not model_source:
        raise ValueError("Choose a model from factory.txt")
    if first_frame is None:
        raise ValueError("A first frame is required for the current video pipeline")
    seed = random.randint(0, 2**31 - 1) if int(seed) < 0 else int(seed)
    pipe = _load_pipeline(model_source)
    _set_loras(pipe, lora_paths or [], lora_scale)
    accepted = _accepted(pipe)
    values = {
        "prompt": prompt,
        "negative_prompt": negative_prompt,
        "image": first_frame.convert("RGB"),
        "last_image": last_frame.convert("RGB") if last_frame is not None else None,
        "width": int(width),
        "height": int(height),
        "num_frames": int(frames),
        "num_inference_steps": int(steps),
        "guidance_scale": float(guidance),
        "generator": torch.Generator(device="cpu").manual_seed(seed),
    }
    kwargs = {key: value for key, value in values.items() if key in accepted and value is not None}
    result = pipe(**kwargs)
    generated = result.frames
    if isinstance(generated, list) and generated and isinstance(generated[0], list):
        generated = generated[0]
    elif getattr(generated, "ndim", None) == 5:
        generated = generated[0]
    OUTPUTS_ROOT.mkdir(parents=True, exist_ok=True)
    video_path = OUTPUTS_ROOT / f"render-{uuid.uuid4().hex}.mp4"
    export_to_video(generated, str(video_path), fps=int(fps))
    last_path = extract_last_frame(str(video_path))
    return str(video_path), last_path, seed


def extract_last_frame(video_path: str) -> str:
    capture = cv2.VideoCapture(video_path)
    count = int(capture.get(cv2.CAP_PROP_FRAME_COUNT))
    capture.set(cv2.CAP_PROP_POS_FRAMES, max(0, count - 1))
    ok, frame = capture.read()
    capture.release()
    if not ok:
        raise RuntimeError("Could not extract the rendered video's last frame")
    frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
    target = OUTPUTS_ROOT / f"last-frame-{uuid.uuid4().hex}.png"
    Image.fromarray(frame).save(target)
    return str(target)