Spaces:
Sleeping
Sleeping
File size: 7,641 Bytes
5738ae5 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 | """
worker.py β Single background thread that processes generation jobs one at a time.
Architecture:
ββββββββββββββββββββββ queue.put(job_id) ββββββββββββββββββββββββ
β FastAPI endpoint β βββββββββββββββββββββββββββββΊ β _job_queue (Queue) β
β POST /generate β ββββββββββββ¬ββββββββββββ
ββββββββββββββββββββββ β .get()
βΌ
ββββββββββββββββββββββββ
β worker_loop() β
β (daemon thread) β
β runs _process_job() β
ββββββββββββββββββββββββ
Only one job runs at a time β CPU inference is single-threaded anyway.
"""
from __future__ import annotations
import logging
import os
import queue
import threading
from datetime import datetime, timezone
from typing import List
logger = logging.getLogger("worker")
# Internal FIFO queue β contains job_ids as strings
_job_queue: queue.Queue[str] = queue.Queue()
# Lock used to inspect queue contents safely
_queue_lock = threading.Lock()
# Ordered list of pending job IDs (mirrors the queue for position lookups)
_pending: List[str] = []
_pending_lock = threading.Lock()
# ββ Public helpers (called from FastAPI routes) ββββββββββββββββββββββββββββββββ
def enqueue(job_id: str) -> None:
"""Add a job to the processing queue."""
with _pending_lock:
_pending.append(job_id)
_job_queue.put(job_id)
logger.info("Enqueued job %s (queue size: %d)", job_id, _job_queue.qsize())
def queue_position(job_id: str) -> int:
"""Return the 1-based position of job_id in the pending queue (0 if not found)."""
with _pending_lock:
try:
return _pending.index(job_id) + 1
except ValueError:
return 0
def queue_size() -> int:
"""Number of jobs still waiting (does not include the currently running job)."""
return _job_queue.qsize()
# ββ Core processing ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
def _process_job(job_id: str) -> None:
"""Generate an image for one job. All state mutations go through job_store."""
from config import settings
from job_store import job_store
from pipeline_manager import pipeline_mgr
from schemas import JobStatus
job = job_store.get(job_id)
if not job:
logger.warning("Job %s not found in store β skipping.", job_id)
return
# Mark as processing
job_store.update(
job_id,
status=JobStatus.PROCESSING,
started_at=datetime.now(timezone.utc),
progress=0,
)
logger.info("Processing job %s prompt=%r", job_id, job.prompt[:60])
try:
# ββ Ensure pipeline is loaded ββββββββββββββββββββββββββββββββββββββββββ
if not pipeline_mgr.is_loaded:
logger.info("Pipeline not loaded β loading now (job %s will wait).", job_id)
pipeline_mgr.load()
p = job.params
# ββ Step-progress callback βββββββββββββββββββββββββββββββββββββββββββββ
def on_step(step: int, total: int) -> None:
pct = int(step / total * 95) # reserve last 5 % for save
job_store.update(job_id, progress=pct, current_step=step, total_steps=total)
# ββ Inference βββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
image, used_seed = pipeline_mgr.generate(
prompt=job.prompt,
negative_prompt=job.negative_prompt,
width=p.get("width", settings.default_width),
height=p.get("height", settings.default_height),
num_inference_steps=p.get("num_inference_steps", settings.default_steps),
guidance_scale=p.get("guidance_scale", settings.default_guidance_scale),
seed=p.get("seed", -1),
scheduler_name=p.get("scheduler", "dpm++"),
step_callback=on_step,
)
# ββ Save image ββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
job_store.update(job_id, progress=97)
timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S")
filename = f"{timestamp}_{job_id[:8]}_{used_seed}.png"
abs_path = os.path.join(settings.outputs_dir, filename)
image.save(abs_path, format="PNG", optimize=False)
# ββ Compute duration βββββββββββββββββββββββββββββββββββββββββββββββββββ
now = datetime.now(timezone.utc)
started = job_store.get(job_id).started_at
duration = (now - started).total_seconds() if started else None
job_store.update(
job_id,
status=JobStatus.COMPLETED,
completed_at=now,
progress=100,
image_filename=filename,
download_url=f"/jobs/{job_id}/download",
duration_seconds=round(duration, 2) if duration else None,
)
logger.info(
"Job %s completed in %.1f s β %s",
job_id,
duration or 0,
filename,
)
except Exception as exc:
logger.exception("Job %s failed: %s", job_id, exc)
job_store.update(
job_id,
status=JobStatus.FAILED,
completed_at=datetime.now(timezone.utc),
error=str(exc),
progress=0,
)
# ββ Worker loop (runs in its own daemon thread) ββββββββββββββββββββββββββββββββ
def worker_loop() -> None:
"""Infinite loop: pull job IDs from the queue and process them."""
logger.info("Worker thread started.")
while True:
job_id = _job_queue.get() # blocks until a job arrives
with _pending_lock:
try:
_pending.remove(job_id)
except ValueError:
pass
try:
_process_job(job_id)
except Exception as exc:
# Catch-all safety net β the loop must never crash
logger.exception("Unhandled error in worker for job %s: %s", job_id, exc)
finally:
_job_queue.task_done()
def start_worker() -> threading.Thread:
"""Spawn and return the daemon worker thread. Call once at application startup."""
t = threading.Thread(target=worker_loop, name="image-worker", daemon=True)
t.start()
return t
|