Spaces:
Runtime error
Runtime error
Download queue_manager.py from Quantumbraid/wan2.2productionbuild: direct link, hf CLI and curl.
- Browser
- Download file 18.6 kB
-
https://huggingface.co/spaces/Quantumbraid/wan2.2productionbuild/resolve/main/queue_manager.py
- Command line
-
hf download hf://spaces/Quantumbraid/wan2.2productionbuild/queue_manager.py
-
curl -L -o queue_manager.py https://huggingface.co/spaces/Quantumbraid/wan2.2productionbuild/resolve/main/queue_manager.py
18.6 kB
| import os | |
| import json | |
| import time | |
| import shutil | |
| import uuid | |
| import threading | |
| import html | |
| from pathlib import Path | |
| from PIL import Image | |
| LEGACY_OUTPUT_DIR = Path("outputs") | |
| OUTPUT_DIR = Path(os.environ.get("WAN_OUTPUT_DIR", "/data/outputs" if Path("/data").is_dir() else "outputs")) | |
| UPLOADS_DIR = OUTPUT_DIR / "uploads" | |
| VIDEOS_DIR = OUTPUT_DIR / "videos" | |
| JOBS_FILE = OUTPUT_DIR / "jobs.json" | |
| def unpack_image_files(input_image): | |
| if not input_image: | |
| return [] | |
| raw_items = [] | |
| if isinstance(input_image, (list, tuple)): | |
| raw_items = list(input_image) | |
| else: | |
| raw_items = [input_image] | |
| extracted_paths = [] | |
| for item in raw_items: | |
| if item is None: | |
| continue | |
| path = None | |
| if isinstance(item, str): | |
| path = item | |
| elif hasattr(item, "path") and item.path: | |
| path = item.path | |
| elif hasattr(item, "name") and item.name: | |
| path = item.name | |
| elif isinstance(item, dict): | |
| path = item.get("path") or item.get("name") | |
| elif isinstance(item, Image.Image) or isinstance(item, np.ndarray): | |
| extracted_paths.append(item) | |
| continue | |
| if path: | |
| if os.path.isdir(path): | |
| for root, _, files in os.walk(path): | |
| for f in sorted(files): | |
| if f.lower().endswith((".png", ".jpg", ".jpeg", ".webp", ".bmp")): | |
| extracted_paths.append(os.path.join(root, f)) | |
| else: | |
| extracted_paths.append(path) | |
| return extracted_paths | |
| class QueueManager: | |
| def __init__(self): | |
| # Some UI actions (for example clear_queue) persist while already | |
| # holding the state lock, so this must be re-entrant. | |
| self.lock = threading.RLock() | |
| self.gpu_lock = threading.Lock() | |
| self.is_running = False | |
| self.worker_thread = None | |
| self.inference_fn = None | |
| self._ensure_dirs() | |
| self.jobs = self._load_jobs() | |
| self._recover_interrupted_jobs() | |
| def _ensure_dirs(self): | |
| # Preserve any queued/completed work created before outputs moved into | |
| # the mounted Space bucket. This is a copy, never a destructive move. | |
| if OUTPUT_DIR != LEGACY_OUTPUT_DIR and not JOBS_FILE.exists() and LEGACY_OUTPUT_DIR.exists(): | |
| shutil.copytree(LEGACY_OUTPUT_DIR, OUTPUT_DIR, dirs_exist_ok=True) | |
| OUTPUT_DIR.mkdir(parents=True, exist_ok=True) | |
| UPLOADS_DIR.mkdir(parents=True, exist_ok=True) | |
| VIDEOS_DIR.mkdir(parents=True, exist_ok=True) | |
| if not JOBS_FILE.exists(): | |
| with open(JOBS_FILE, "w", encoding="utf-8") as f: | |
| json.dump([], f) | |
| def _load_jobs(self): | |
| with self.lock: | |
| if JOBS_FILE.exists(): | |
| try: | |
| with open(JOBS_FILE, "r", encoding="utf-8") as f: | |
| return json.load(f) | |
| except Exception as e: | |
| print(f"[QueueManager] Error loading jobs.json: {e}") | |
| return [] | |
| return [] | |
| def _save_jobs(self): | |
| with self.lock: | |
| with open(JOBS_FILE, "w", encoding="utf-8") as f: | |
| json.dump(self.jobs, f, indent=2, ensure_ascii=False) | |
| def _recover_interrupted_jobs(self): | |
| updated = False | |
| for job in self.jobs: | |
| if job.get("status") == "running": | |
| job["status"] = "queued" | |
| updated = True | |
| if updated: | |
| self._save_jobs() | |
| def add_job(self, input_image, last_image, prompt, steps, negative_prompt, | |
| duration_seconds, sequential_renders, guidance_scale, guidance_scale_2, seed, | |
| randomize_seed, quality, scheduler, flow_shift, frame_multiplier, | |
| lora_groups): | |
| image_paths = unpack_image_files(input_image) | |
| if not image_paths: | |
| return "β Error: Please select at least one valid input image before adding to queue.", self.render_queue_html() | |
| import random | |
| added_ids = [] | |
| for idx, img_src in enumerate(image_paths): | |
| try: | |
| if isinstance(img_src, str) and os.path.exists(img_src): | |
| img_obj = Image.open(img_src) | |
| elif isinstance(img_src, Image.Image): | |
| img_obj = img_src | |
| else: | |
| img_obj = Image.fromarray(img_src) | |
| except Exception as e: | |
| print(f"[QueueManager] Error opening image index {idx}: {e}") | |
| continue | |
| timestamp_str = time.strftime("%Y%m%d_%H%M%S") | |
| short_id = uuid.uuid4().hex[:6] | |
| job_id = f"job_{timestamp_str}_{short_id}_{idx+1}" | |
| input_image_path = UPLOADS_DIR / f"{job_id}_input.png" | |
| img_obj.save(input_image_path) | |
| last_image_path = None | |
| if last_image is not None: | |
| last_image_path = str(UPLOADS_DIR / f"{job_id}_last.png") | |
| try: | |
| last_src = last_image.name if hasattr(last_image, "name") else (last_image.path if hasattr(last_image, "path") else last_image) | |
| if isinstance(last_src, Image.Image): | |
| last_src.save(last_image_path) | |
| elif isinstance(last_src, str) and os.path.exists(last_src): | |
| shutil.copy(last_src, last_image_path) | |
| else: | |
| Image.fromarray(last_src).save(last_image_path) | |
| except Exception as e: | |
| print(f"[QueueManager] Warning: failed to save last image: {e}") | |
| last_image_path = None | |
| current_seed = random.randint(0, 2147483647) if randomize_seed else int(seed if seed is not None else 42) | |
| job = { | |
| "id": job_id, | |
| "created_at": time.strftime("%Y-%m-%d %H:%M:%S"), | |
| "status": "queued", | |
| "prompt": prompt or "", | |
| "input_image_path": str(input_image_path).replace("\\", "/"), | |
| "last_image_path": str(last_image_path).replace("\\", "/") if last_image_path else None, | |
| "video_path": None, | |
| "params": { | |
| "steps": int(steps) if steps is not None else 4, | |
| "negative_prompt": negative_prompt or "", | |
| "duration_seconds": float(duration_seconds) if duration_seconds is not None else 10.0, | |
| "sequential_renders": max(1, min(12, int(sequential_renders or 1))), | |
| "guidance_scale": float(guidance_scale) if guidance_scale is not None else 1.0, | |
| "guidance_scale_2": float(guidance_scale_2) if guidance_scale_2 is not None else 1.0, | |
| "seed": current_seed, | |
| "randomize_seed": bool(randomize_seed), | |
| "quality": int(quality) if quality is not None else 6, | |
| "scheduler": str(scheduler) if scheduler else "UniPCMultistep", | |
| "flow_shift": float(flow_shift) if flow_shift is not None else 3.0, | |
| "frame_multiplier": int(frame_multiplier) if frame_multiplier is not None else 16, | |
| "lora_groups": lora_groups if isinstance(lora_groups, list) else [] | |
| }, | |
| "error": None, | |
| "completed_at": None | |
| } | |
| self.jobs.insert(0, job) | |
| added_ids.append(job_id) | |
| self._save_jobs() | |
| if len(added_ids) == 1: | |
| msg = f"β 1 job ({added_ids[0]}) added to queue." | |
| elif len(added_ids) > 1: | |
| msg = f"β {len(added_ids)} separate jobs added to queue using prompt." | |
| else: | |
| msg = "β No valid images found to queue." | |
| return msg, self.render_queue_html() | |
| def start_queue(self, inference_fn): | |
| self.inference_fn = inference_fn | |
| self.is_running = True | |
| if self.worker_thread is None or not self.worker_thread.is_alive(): | |
| self.worker_thread = threading.Thread(target=self._worker_loop, daemon=True) | |
| self.worker_thread.start() | |
| return "βΆοΈ Queue started.", self.render_queue_html() | |
| def stop_queue(self): | |
| self.is_running = False | |
| return "βΈοΈ Queue stopped.", self.render_queue_html() | |
| def clear_queue(self): | |
| with self.lock: | |
| # Keep currently running job if active, clear all other jobs | |
| self.jobs = [j for j in self.jobs if j.get("status") == "running"] | |
| self._save_jobs() | |
| return "ποΈ Queue cleared.", self.render_queue_html() | |
| def get_error_log(self): | |
| failed_jobs = [j for j in self.jobs if j.get("status") == "failed"] | |
| logs = [] | |
| if os.path.exists("app.log"): | |
| try: | |
| with open("app.log", "r", encoding="utf-8", errors="ignore") as f: | |
| lines = f.readlines() | |
| err_lines = [l.strip() for l in lines if any(k in l.lower() for k in ["error", "exception", "traceback", "failed"])] | |
| if err_lines: | |
| logs.append("=== RECENT APP LOG ERRORS ===") | |
| logs.extend(err_lines[-25:]) | |
| except Exception as e: | |
| logs.append(f"Error reading app.log: {e}") | |
| if failed_jobs: | |
| logs.append("\n=== FAILED QUEUE JOBS ===") | |
| for j in failed_jobs: | |
| logs.append(f"[{j.get('created_at')}] Job {j.get('id')}: {j.get('error', 'Unknown error')}") | |
| if not logs: | |
| return "β No errors found in logs or queue jobs! Everything is running smoothly." | |
| return "\n".join(logs) | |
| def check_queue(self): | |
| running_count = sum(1 for j in self.jobs if j["status"] == "running") | |
| queued_count = sum(1 for j in self.jobs if j["status"] == "queued") | |
| completed_count = sum(1 for j in self.jobs if j["status"] == "completed") | |
| msg = f"π Queue checked. Total: {len(self.jobs)} | Queued: {queued_count} | Running: {running_count} | Finished: {completed_count}" | |
| return msg, self.render_queue_html() | |
| def _worker_loop(self): | |
| print("[QueueManager] Background worker thread started.") | |
| while self.is_running: | |
| # Find next queued job (oldest created job that is still queued) | |
| next_job = None | |
| for job in reversed(self.jobs): | |
| if job.get("status") == "queued": | |
| next_job = job | |
| break | |
| if next_job is None: | |
| time.sleep(2) | |
| continue | |
| job_id = next_job["id"] | |
| print(f"[QueueManager] Processing job: {job_id}") | |
| # Enforce single job GPU execution | |
| with self.gpu_lock: | |
| next_job["status"] = "running" | |
| self._save_jobs() | |
| try: | |
| input_img = Image.open(next_job["input_image_path"]) | |
| last_img = Image.open(next_job["last_image_path"]) if next_job.get("last_image_path") else None | |
| params = next_job.get("params", {}) | |
| if self.inference_fn is None: | |
| raise ValueError("Inference function not bound to QueueManager.") | |
| res = self.inference_fn( | |
| input_image=input_img, | |
| last_image=last_img, | |
| prompt=next_job.get("prompt", ""), | |
| steps=params.get("steps", 4), | |
| negative_prompt=params.get("negative_prompt", ""), | |
| duration_seconds=params.get("duration_seconds", 10.0), | |
| sequential_renders=params.get("sequential_renders", 1), | |
| guidance_scale=params.get("guidance_scale", 1.0), | |
| guidance_scale_2=params.get("guidance_scale_2", 1.0), | |
| seed=params.get("seed", 42), | |
| randomize_seed=params.get("randomize_seed", False), | |
| quality=params.get("quality", 6), | |
| scheduler=params.get("scheduler", "UniPCMultistep"), | |
| flow_shift=params.get("flow_shift", 3.0), | |
| frame_multiplier=params.get("frame_multiplier", 16), | |
| lora_groups=params.get("lora_groups"), | |
| video_component=True, | |
| artifact_dir=str(VIDEOS_DIR / job_id), | |
| ) | |
| temp_video = None | |
| if isinstance(res, tuple) and len(res) >= 2: | |
| temp_video = res[1] or res[0] | |
| elif isinstance(res, str): | |
| temp_video = res | |
| dest_video = VIDEOS_DIR / f"{job_id}.mp4" | |
| if temp_video and os.path.exists(temp_video): | |
| shutil.copy(temp_video, dest_video) | |
| next_job["status"] = "completed" | |
| next_job["video_path"] = str(dest_video).replace("\\", "/") | |
| next_job["completed_at"] = time.strftime("%Y-%m-%d %H:%M:%S") | |
| print(f"[QueueManager] Job {job_id} completed successfully.") | |
| else: | |
| next_job["status"] = "failed" | |
| next_job["error"] = "Output video was not generated or file not found." | |
| print(f"[QueueManager] Job {job_id} failed: Video file not created.") | |
| except Exception as e: | |
| print(f"[QueueManager] Exception in job {job_id}: {e}") | |
| next_job["status"] = "failed" | |
| next_job["error"] = str(e) | |
| self._save_jobs() | |
| time.sleep(1) | |
| print("[QueueManager] Background worker thread stopped.") | |
| def render_queue_html(self): | |
| if not self.jobs: | |
| return """ | |
| <div class="queue-container"> | |
| <div class="queue-empty">No jobs in queue yet.<br>Upload an image and click <b>Add to Queue</b> to begin.</div> | |
| </div> | |
| """ | |
| status_class_map = { | |
| "queued": "badge-queued", | |
| "running": "badge-running", | |
| "completed": "badge-completed", | |
| "failed": "badge-failed" | |
| } | |
| status_label_map = { | |
| "queued": "Queued", | |
| "running": "Running β³", | |
| "completed": "Finished β ", | |
| "failed": "Failed β" | |
| } | |
| queue_status = "ACTIVE βΆοΈ" if self.is_running else "STOPPED βΈοΈ" | |
| status_color = "#00b894" if self.is_running else "#ea868f" | |
| cards = [] | |
| for job in self.jobs: | |
| status = job.get("status", "queued") | |
| badge_cls = status_class_map.get(status, "badge-queued") | |
| badge_lbl = status_label_map.get(status, status) | |
| job_id = job.get("id", "") | |
| created_at = job.get("created_at", "") | |
| prompt = job.get("prompt", "") | |
| video_path = job.get("video_path") | |
| params = job.get("params", {}) | |
| lora_groups = params.get("lora_groups") or [] | |
| lora_text = ", ".join(html.escape(str(name)) for name in lora_groups) if lora_groups else "None" | |
| # Gradio 6 serves allowed local files through /gradio_api/file=. | |
| # The legacy /file= route returns 404 even when the file exists. | |
| file_url = f"/gradio_api/file={video_path}" if video_path else "" | |
| # Class attribute for auto-download triggering via JS | |
| auto_dl_attr = "" | |
| if status == "completed" and file_url: | |
| auto_dl_attr = f'class="pending-auto-download" data-download-url="{file_url}" data-job-id="{job_id}" data-filename="{job_id}.mp4"' | |
| video_html = "" | |
| if status == "completed" and video_path and os.path.exists(video_path): | |
| video_html = f""" | |
| <div class="queue-card-video" {auto_dl_attr}> | |
| <video src="{file_url}" controls width="100%" style="border-radius: 8px; border: 1px solid rgba(255,255,255,0.12); margin-top: 8px; max-height: 240px; background: #000;"></video> | |
| <div style="margin-top: 8px; display: flex; gap: 8px; align-items: center; flex-wrap: wrap;"> | |
| <button onclick="loadPreview('{file_url}')" class="btn-preview-link" type="button">ποΈ Preview in Output Box</button> | |
| <a href="{file_url}" download="{job_id}.mp4" class="btn-dl-link">π₯ Download Video</a> | |
| <a href="{file_url}" target="_blank" class="btn-view-link">π Direct Link</a> | |
| </div> | |
| </div> | |
| """ | |
| elif status == "failed": | |
| err_msg = job.get("error", "Unknown error") | |
| video_html = f'<div class="queue-error">β οΈ Error: {err_msg}</div>' | |
| elif status == "running": | |
| video_html = '<div class="queue-processing">βοΈ Generating video on GPU...</div>' | |
| elif status == "queued": | |
| video_html = '<div class="queue-pending">β³ Waiting in queue...</div>' | |
| cards.append(f""" | |
| <div class="queue-card {status}"> | |
| <div class="queue-card-header"> | |
| <span class="queue-job-id">{job_id}</span> | |
| <span class="queue-badge {badge_cls}">{badge_lbl}</span> | |
| </div> | |
| <div class="queue-card-body"> | |
| <div class="queue-prompt"><b>Prompt:</b> {prompt}</div> | |
| <div class="queue-loras"><b>LoRAs:</b> {lora_text}</div> | |
| <div class="queue-meta">Created: {created_at} | Dur: {params.get('duration_seconds', 10.0)}s x {params.get('sequential_renders', 1)} segments | Steps: {params.get('steps', 4)} | FPS: {params.get('frame_multiplier', 16)}</div> | |
| {video_html} | |
| </div> | |
| </div> | |
| """) | |
| cards_html = "\n".join(cards) | |
| return f""" | |
| <div class="queue-container"> | |
| <div class="queue-header-bar"> | |
| <span>Worker Status: <span style="color: {status_color};">{queue_status}</span></span> | |
| <span>Total Jobs: {len(self.jobs)}</span> | |
| </div> | |
| <div class="queue-cards-list"> | |
| {cards_html} | |
| </div> | |
| </div> | |
| """ | |