""" worker.py — Single background thread that processes generation jobs one at a time. Architecture: ┌────────────────────┐ queue.put(job_id) ┌──────────────────────┐ │ FastAPI endpoint │ ────────────────────────────► │ _job_queue (Queue) │ │ POST /generate │ └──────────┬───────────┘ └────────────────────┘ │ .get() ▼ ┌──────────────────────┐ │ worker_loop() │ │ (daemon thread) │ │ runs _process_job() │ └──────────────────────┘ Only one job runs at a time — CPU inference is single-threaded anyway. """ from __future__ import annotations import logging import os import queue import threading from datetime import datetime, timezone from typing import List logger = logging.getLogger("worker") # Internal FIFO queue — contains job_ids as strings _job_queue: queue.Queue[str] = queue.Queue() # Lock used to inspect queue contents safely _queue_lock = threading.Lock() # Ordered list of pending job IDs (mirrors the queue for position lookups) _pending: List[str] = [] _pending_lock = threading.Lock() # ── Public helpers (called from FastAPI routes) ──────────────────────────────── def enqueue(job_id: str) -> None: """Add a job to the processing queue.""" with _pending_lock: _pending.append(job_id) _job_queue.put(job_id) logger.info("Enqueued job %s (queue size: %d)", job_id, _job_queue.qsize()) def queue_position(job_id: str) -> int: """Return the 1-based position of job_id in the pending queue (0 if not found).""" with _pending_lock: try: return _pending.index(job_id) + 1 except ValueError: return 0 def queue_size() -> int: """Number of jobs still waiting (does not include the currently running job).""" return _job_queue.qsize() # ── Core processing ──────────────────────────────────────────────────────────── def _process_job(job_id: str) -> None: """Generate an image for one job. All state mutations go through job_store.""" from config import settings from job_store import job_store from pipeline_manager import pipeline_mgr from schemas import JobStatus job = job_store.get(job_id) if not job: logger.warning("Job %s not found in store — skipping.", job_id) return # Mark as processing job_store.update( job_id, status=JobStatus.PROCESSING, started_at=datetime.now(timezone.utc), progress=0, ) logger.info("Processing job %s prompt=%r", job_id, job.prompt[:60]) try: # ── Ensure pipeline is loaded ────────────────────────────────────────── if not pipeline_mgr.is_loaded: logger.info("Pipeline not loaded — loading now (job %s will wait).", job_id) pipeline_mgr.load() p = job.params # ── Step-progress callback ───────────────────────────────────────────── def on_step(step: int, total: int) -> None: pct = int(step / total * 95) # reserve last 5 % for save job_store.update(job_id, progress=pct, current_step=step, total_steps=total) # ── Inference ───────────────────────────────────────────────────────── image, used_seed = pipeline_mgr.generate( prompt=job.prompt, negative_prompt=job.negative_prompt, width=p.get("width", settings.default_width), height=p.get("height", settings.default_height), num_inference_steps=p.get("num_inference_steps", settings.default_steps), guidance_scale=p.get("guidance_scale", settings.default_guidance_scale), seed=p.get("seed", -1), scheduler_name=p.get("scheduler", "dpm++"), step_callback=on_step, ) # ── Save image ──────────────────────────────────────────────────────── job_store.update(job_id, progress=97) timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S") filename = f"{timestamp}_{job_id[:8]}_{used_seed}.png" abs_path = os.path.join(settings.outputs_dir, filename) image.save(abs_path, format="PNG", optimize=False) # ── Compute duration ─────────────────────────────────────────────────── now = datetime.now(timezone.utc) started = job_store.get(job_id).started_at duration = (now - started).total_seconds() if started else None job_store.update( job_id, status=JobStatus.COMPLETED, completed_at=now, progress=100, image_filename=filename, download_url=f"/jobs/{job_id}/download", duration_seconds=round(duration, 2) if duration else None, ) logger.info( "Job %s completed in %.1f s → %s", job_id, duration or 0, filename, ) except Exception as exc: logger.exception("Job %s failed: %s", job_id, exc) job_store.update( job_id, status=JobStatus.FAILED, completed_at=datetime.now(timezone.utc), error=str(exc), progress=0, ) # ── Worker loop (runs in its own daemon thread) ──────────────────────────────── def worker_loop() -> None: """Infinite loop: pull job IDs from the queue and process them.""" logger.info("Worker thread started.") while True: job_id = _job_queue.get() # blocks until a job arrives with _pending_lock: try: _pending.remove(job_id) except ValueError: pass try: _process_job(job_id) except Exception as exc: # Catch-all safety net — the loop must never crash logger.exception("Unhandled error in worker for job %s: %s", job_id, exc) finally: _job_queue.task_done() def start_worker() -> threading.Thread: """Spawn and return the daemon worker thread. Call once at application startup.""" t = threading.Thread(target=worker_loop, name="image-worker", daemon=True) t.start() return t