Download runtime.py from Quantumbraid/video-model-factory: direct link, hf CLI and curl.
- Browser
- Download file 4.41 kB
-
https://huggingface.co/spaces/Quantumbraid/video-model-factory/resolve/main/runtime.py
- Command line
-
hf download hf://spaces/Quantumbraid/video-model-factory/runtime.py
-
curl -L -o runtime.py https://huggingface.co/spaces/Quantumbraid/video-model-factory/resolve/main/runtime.py
4.41 kB
| from __future__ import annotations | |
| import gc | |
| import inspect | |
| import random | |
| import threading | |
| import uuid | |
| from pathlib import Path | |
| import cv2 | |
| import torch | |
| from diffusers import DiffusionPipeline | |
| from diffusers.utils import export_to_video | |
| from PIL import Image | |
| from storage import OUTPUTS_ROOT, artifact_path | |
| _pipe = None | |
| _pipe_source: str | None = None | |
| _lock = threading.RLock() | |
| def _load_pipeline(model_source: str): | |
| global _pipe, _pipe_source | |
| with _lock: | |
| if _pipe is not None and _pipe_source == model_source: | |
| return _pipe | |
| if _pipe is not None: | |
| del _pipe | |
| _pipe = None | |
| gc.collect() | |
| if torch.cuda.is_available(): | |
| torch.cuda.empty_cache() | |
| model_path = artifact_path(model_source, "model") | |
| _pipe = DiffusionPipeline.from_pretrained( | |
| str(model_path), | |
| dtype=torch.bfloat16, | |
| local_files_only=True, | |
| ) | |
| if hasattr(_pipe, "enable_model_cpu_offload"): | |
| _pipe.enable_model_cpu_offload() | |
| else: | |
| _pipe.to("cuda") | |
| _pipe_source = model_source | |
| return _pipe | |
| def _set_loras(pipe, lora_paths: list[str], lora_scale: float) -> None: | |
| if hasattr(pipe, "unload_lora_weights"): | |
| try: | |
| pipe.unload_lora_weights() | |
| except Exception: | |
| pass | |
| adapter_names = [] | |
| for index, raw_path in enumerate(lora_paths): | |
| path = Path(raw_path) | |
| name = f"factory_{index}" | |
| if path.is_file(): | |
| pipe.load_lora_weights(str(path.parent), weight_name=path.name, adapter_name=name) | |
| else: | |
| pipe.load_lora_weights(str(path), adapter_name=name) | |
| adapter_names.append(name) | |
| if adapter_names and hasattr(pipe, "set_adapters"): | |
| pipe.set_adapters(adapter_names, adapter_weights=[float(lora_scale)] * len(adapter_names)) | |
| def _accepted(pipe) -> set[str]: | |
| signature = inspect.signature(pipe.__call__) | |
| return set(signature.parameters) | |
| def render_video( | |
| model_source: str, | |
| lora_paths: list[str], | |
| prompt: str, | |
| negative_prompt: str, | |
| first_frame: Image.Image | None, | |
| last_frame: Image.Image | None, | |
| width: int, | |
| height: int, | |
| frames: int, | |
| fps: int, | |
| steps: int, | |
| guidance: float, | |
| lora_scale: float, | |
| seed: int, | |
| ) -> tuple[str, str, int]: | |
| if not model_source: | |
| raise ValueError("Choose a model from factory.txt") | |
| if first_frame is None: | |
| raise ValueError("A first frame is required for the current video pipeline") | |
| seed = random.randint(0, 2**31 - 1) if int(seed) < 0 else int(seed) | |
| pipe = _load_pipeline(model_source) | |
| _set_loras(pipe, lora_paths or [], lora_scale) | |
| accepted = _accepted(pipe) | |
| values = { | |
| "prompt": prompt, | |
| "negative_prompt": negative_prompt, | |
| "image": first_frame.convert("RGB"), | |
| "last_image": last_frame.convert("RGB") if last_frame is not None else None, | |
| "width": int(width), | |
| "height": int(height), | |
| "num_frames": int(frames), | |
| "num_inference_steps": int(steps), | |
| "guidance_scale": float(guidance), | |
| "generator": torch.Generator(device="cpu").manual_seed(seed), | |
| } | |
| kwargs = {key: value for key, value in values.items() if key in accepted and value is not None} | |
| result = pipe(**kwargs) | |
| generated = result.frames | |
| if isinstance(generated, list) and generated and isinstance(generated[0], list): | |
| generated = generated[0] | |
| elif getattr(generated, "ndim", None) == 5: | |
| generated = generated[0] | |
| OUTPUTS_ROOT.mkdir(parents=True, exist_ok=True) | |
| video_path = OUTPUTS_ROOT / f"render-{uuid.uuid4().hex}.mp4" | |
| export_to_video(generated, str(video_path), fps=int(fps)) | |
| last_path = extract_last_frame(str(video_path)) | |
| return str(video_path), last_path, seed | |
| def extract_last_frame(video_path: str) -> str: | |
| capture = cv2.VideoCapture(video_path) | |
| count = int(capture.get(cv2.CAP_PROP_FRAME_COUNT)) | |
| capture.set(cv2.CAP_PROP_POS_FRAMES, max(0, count - 1)) | |
| ok, frame = capture.read() | |
| capture.release() | |
| if not ok: | |
| raise RuntimeError("Could not extract the rendered video's last frame") | |
| frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) | |
| target = OUTPUTS_ROOT / f"last-frame-{uuid.uuid4().hex}.png" | |
| Image.fromarray(frame).save(target) | |
| return str(target) | |