import os import json import time import shutil import uuid import threading import html from pathlib import Path from PIL import Image LEGACY_OUTPUT_DIR = Path("outputs") OUTPUT_DIR = Path(os.environ.get("WAN_OUTPUT_DIR", "/data/outputs" if Path("/data").is_dir() else "outputs")) UPLOADS_DIR = OUTPUT_DIR / "uploads" VIDEOS_DIR = OUTPUT_DIR / "videos" JOBS_FILE = OUTPUT_DIR / "jobs.json" def unpack_image_files(input_image): if not input_image: return [] raw_items = [] if isinstance(input_image, (list, tuple)): raw_items = list(input_image) else: raw_items = [input_image] extracted_paths = [] for item in raw_items: if item is None: continue path = None if isinstance(item, str): path = item elif hasattr(item, "path") and item.path: path = item.path elif hasattr(item, "name") and item.name: path = item.name elif isinstance(item, dict): path = item.get("path") or item.get("name") elif isinstance(item, Image.Image) or isinstance(item, np.ndarray): extracted_paths.append(item) continue if path: if os.path.isdir(path): for root, _, files in os.walk(path): for f in sorted(files): if f.lower().endswith((".png", ".jpg", ".jpeg", ".webp", ".bmp")): extracted_paths.append(os.path.join(root, f)) else: extracted_paths.append(path) return extracted_paths class QueueManager: def __init__(self): # Some UI actions (for example clear_queue) persist while already # holding the state lock, so this must be re-entrant. self.lock = threading.RLock() self.gpu_lock = threading.Lock() self.is_running = False self.worker_thread = None self.inference_fn = None self._ensure_dirs() self.jobs = self._load_jobs() self._recover_interrupted_jobs() def _ensure_dirs(self): # Preserve any queued/completed work created before outputs moved into # the mounted Space bucket. This is a copy, never a destructive move. if OUTPUT_DIR != LEGACY_OUTPUT_DIR and not JOBS_FILE.exists() and LEGACY_OUTPUT_DIR.exists(): shutil.copytree(LEGACY_OUTPUT_DIR, OUTPUT_DIR, dirs_exist_ok=True) OUTPUT_DIR.mkdir(parents=True, exist_ok=True) UPLOADS_DIR.mkdir(parents=True, exist_ok=True) VIDEOS_DIR.mkdir(parents=True, exist_ok=True) if not JOBS_FILE.exists(): with open(JOBS_FILE, "w", encoding="utf-8") as f: json.dump([], f) def _load_jobs(self): with self.lock: if JOBS_FILE.exists(): try: with open(JOBS_FILE, "r", encoding="utf-8") as f: return json.load(f) except Exception as e: print(f"[QueueManager] Error loading jobs.json: {e}") return [] return [] def _save_jobs(self): with self.lock: with open(JOBS_FILE, "w", encoding="utf-8") as f: json.dump(self.jobs, f, indent=2, ensure_ascii=False) def _recover_interrupted_jobs(self): updated = False for job in self.jobs: if job.get("status") == "running": job["status"] = "queued" updated = True if updated: self._save_jobs() def add_job(self, input_image, last_image, prompt, steps, negative_prompt, duration_seconds, sequential_renders, guidance_scale, guidance_scale_2, seed, randomize_seed, quality, scheduler, flow_shift, frame_multiplier, lora_groups): image_paths = unpack_image_files(input_image) if not image_paths: return "❌ Error: Please select at least one valid input image before adding to queue.", self.render_queue_html() import random added_ids = [] for idx, img_src in enumerate(image_paths): try: if isinstance(img_src, str) and os.path.exists(img_src): img_obj = Image.open(img_src) elif isinstance(img_src, Image.Image): img_obj = img_src else: img_obj = Image.fromarray(img_src) except Exception as e: print(f"[QueueManager] Error opening image index {idx}: {e}") continue timestamp_str = time.strftime("%Y%m%d_%H%M%S") short_id = uuid.uuid4().hex[:6] job_id = f"job_{timestamp_str}_{short_id}_{idx+1}" input_image_path = UPLOADS_DIR / f"{job_id}_input.png" img_obj.save(input_image_path) last_image_path = None if last_image is not None: last_image_path = str(UPLOADS_DIR / f"{job_id}_last.png") try: last_src = last_image.name if hasattr(last_image, "name") else (last_image.path if hasattr(last_image, "path") else last_image) if isinstance(last_src, Image.Image): last_src.save(last_image_path) elif isinstance(last_src, str) and os.path.exists(last_src): shutil.copy(last_src, last_image_path) else: Image.fromarray(last_src).save(last_image_path) except Exception as e: print(f"[QueueManager] Warning: failed to save last image: {e}") last_image_path = None current_seed = random.randint(0, 2147483647) if randomize_seed else int(seed if seed is not None else 42) job = { "id": job_id, "created_at": time.strftime("%Y-%m-%d %H:%M:%S"), "status": "queued", "prompt": prompt or "", "input_image_path": str(input_image_path).replace("\\", "/"), "last_image_path": str(last_image_path).replace("\\", "/") if last_image_path else None, "video_path": None, "params": { "steps": int(steps) if steps is not None else 4, "negative_prompt": negative_prompt or "", "duration_seconds": float(duration_seconds) if duration_seconds is not None else 10.0, "sequential_renders": max(1, min(12, int(sequential_renders or 1))), "guidance_scale": float(guidance_scale) if guidance_scale is not None else 1.0, "guidance_scale_2": float(guidance_scale_2) if guidance_scale_2 is not None else 1.0, "seed": current_seed, "randomize_seed": bool(randomize_seed), "quality": int(quality) if quality is not None else 6, "scheduler": str(scheduler) if scheduler else "UniPCMultistep", "flow_shift": float(flow_shift) if flow_shift is not None else 3.0, "frame_multiplier": int(frame_multiplier) if frame_multiplier is not None else 16, "lora_groups": lora_groups if isinstance(lora_groups, list) else [] }, "error": None, "completed_at": None } self.jobs.insert(0, job) added_ids.append(job_id) self._save_jobs() if len(added_ids) == 1: msg = f"✅ 1 job ({added_ids[0]}) added to queue." elif len(added_ids) > 1: msg = f"✅ {len(added_ids)} separate jobs added to queue using prompt." else: msg = "❌ No valid images found to queue." return msg, self.render_queue_html() def start_queue(self, inference_fn): self.inference_fn = inference_fn self.is_running = True if self.worker_thread is None or not self.worker_thread.is_alive(): self.worker_thread = threading.Thread(target=self._worker_loop, daemon=True) self.worker_thread.start() return "▶️ Queue started.", self.render_queue_html() def stop_queue(self): self.is_running = False return "⏸️ Queue stopped.", self.render_queue_html() def clear_queue(self): with self.lock: # Keep currently running job if active, clear all other jobs self.jobs = [j for j in self.jobs if j.get("status") == "running"] self._save_jobs() return "🗑️ Queue cleared.", self.render_queue_html() def get_error_log(self): failed_jobs = [j for j in self.jobs if j.get("status") == "failed"] logs = [] if os.path.exists("app.log"): try: with open("app.log", "r", encoding="utf-8", errors="ignore") as f: lines = f.readlines() err_lines = [l.strip() for l in lines if any(k in l.lower() for k in ["error", "exception", "traceback", "failed"])] if err_lines: logs.append("=== RECENT APP LOG ERRORS ===") logs.extend(err_lines[-25:]) except Exception as e: logs.append(f"Error reading app.log: {e}") if failed_jobs: logs.append("\n=== FAILED QUEUE JOBS ===") for j in failed_jobs: logs.append(f"[{j.get('created_at')}] Job {j.get('id')}: {j.get('error', 'Unknown error')}") if not logs: return "✅ No errors found in logs or queue jobs! Everything is running smoothly." return "\n".join(logs) def check_queue(self): running_count = sum(1 for j in self.jobs if j["status"] == "running") queued_count = sum(1 for j in self.jobs if j["status"] == "queued") completed_count = sum(1 for j in self.jobs if j["status"] == "completed") msg = f"🔍 Queue checked. Total: {len(self.jobs)} | Queued: {queued_count} | Running: {running_count} | Finished: {completed_count}" return msg, self.render_queue_html() def _worker_loop(self): print("[QueueManager] Background worker thread started.") while self.is_running: # Find next queued job (oldest created job that is still queued) next_job = None for job in reversed(self.jobs): if job.get("status") == "queued": next_job = job break if next_job is None: time.sleep(2) continue job_id = next_job["id"] print(f"[QueueManager] Processing job: {job_id}") # Enforce single job GPU execution with self.gpu_lock: next_job["status"] = "running" self._save_jobs() try: input_img = Image.open(next_job["input_image_path"]) last_img = Image.open(next_job["last_image_path"]) if next_job.get("last_image_path") else None params = next_job.get("params", {}) if self.inference_fn is None: raise ValueError("Inference function not bound to QueueManager.") res = self.inference_fn( input_image=input_img, last_image=last_img, prompt=next_job.get("prompt", ""), steps=params.get("steps", 4), negative_prompt=params.get("negative_prompt", ""), duration_seconds=params.get("duration_seconds", 10.0), sequential_renders=params.get("sequential_renders", 1), guidance_scale=params.get("guidance_scale", 1.0), guidance_scale_2=params.get("guidance_scale_2", 1.0), seed=params.get("seed", 42), randomize_seed=params.get("randomize_seed", False), quality=params.get("quality", 6), scheduler=params.get("scheduler", "UniPCMultistep"), flow_shift=params.get("flow_shift", 3.0), frame_multiplier=params.get("frame_multiplier", 16), lora_groups=params.get("lora_groups"), video_component=True, artifact_dir=str(VIDEOS_DIR / job_id), ) temp_video = None if isinstance(res, tuple) and len(res) >= 2: temp_video = res[1] or res[0] elif isinstance(res, str): temp_video = res dest_video = VIDEOS_DIR / f"{job_id}.mp4" if temp_video and os.path.exists(temp_video): shutil.copy(temp_video, dest_video) next_job["status"] = "completed" next_job["video_path"] = str(dest_video).replace("\\", "/") next_job["completed_at"] = time.strftime("%Y-%m-%d %H:%M:%S") print(f"[QueueManager] Job {job_id} completed successfully.") else: next_job["status"] = "failed" next_job["error"] = "Output video was not generated or file not found." print(f"[QueueManager] Job {job_id} failed: Video file not created.") except Exception as e: print(f"[QueueManager] Exception in job {job_id}: {e}") next_job["status"] = "failed" next_job["error"] = str(e) self._save_jobs() time.sleep(1) print("[QueueManager] Background worker thread stopped.") def render_queue_html(self): if not self.jobs: return """