File size: 7,641 Bytes
5738ae5
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
"""
worker.py β€” Single background thread that processes generation jobs one at a time.

Architecture:
  β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”      queue.put(job_id)       β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”
  β”‚  FastAPI endpoint  β”‚ ────────────────────────────► β”‚  _job_queue (Queue)  β”‚
  β”‚  POST /generate    β”‚                               β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜
  β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜                                          β”‚ .get()
                                                                  β–Ό
                                                      β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”
                                                      β”‚  worker_loop()       β”‚
                                                      β”‚  (daemon thread)     β”‚
                                                      β”‚  runs _process_job() β”‚
                                                      β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜

Only one job runs at a time β€” CPU inference is single-threaded anyway.
"""
from __future__ import annotations

import logging
import os
import queue
import threading
from datetime import datetime, timezone
from typing import List

logger = logging.getLogger("worker")

# Internal FIFO queue β€” contains job_ids as strings
_job_queue: queue.Queue[str] = queue.Queue()

# Lock used to inspect queue contents safely
_queue_lock = threading.Lock()

# Ordered list of pending job IDs (mirrors the queue for position lookups)
_pending: List[str] = []
_pending_lock = threading.Lock()


# ── Public helpers (called from FastAPI routes) ────────────────────────────────

def enqueue(job_id: str) -> None:
    """Add a job to the processing queue."""
    with _pending_lock:
        _pending.append(job_id)
    _job_queue.put(job_id)
    logger.info("Enqueued job %s  (queue size: %d)", job_id, _job_queue.qsize())


def queue_position(job_id: str) -> int:
    """Return the 1-based position of job_id in the pending queue (0 if not found)."""
    with _pending_lock:
        try:
            return _pending.index(job_id) + 1
        except ValueError:
            return 0


def queue_size() -> int:
    """Number of jobs still waiting (does not include the currently running job)."""
    return _job_queue.qsize()


# ── Core processing ────────────────────────────────────────────────────────────

def _process_job(job_id: str) -> None:
    """Generate an image for one job. All state mutations go through job_store."""
    from config import settings
    from job_store import job_store
    from pipeline_manager import pipeline_mgr
    from schemas import JobStatus

    job = job_store.get(job_id)
    if not job:
        logger.warning("Job %s not found in store β€” skipping.", job_id)
        return

    # Mark as processing
    job_store.update(
        job_id,
        status=JobStatus.PROCESSING,
        started_at=datetime.now(timezone.utc),
        progress=0,
    )
    logger.info("Processing job %s  prompt=%r", job_id, job.prompt[:60])

    try:
        # ── Ensure pipeline is loaded ──────────────────────────────────────────
        if not pipeline_mgr.is_loaded:
            logger.info("Pipeline not loaded β€” loading now (job %s will wait).", job_id)
            pipeline_mgr.load()

        p = job.params

        # ── Step-progress callback ─────────────────────────────────────────────
        def on_step(step: int, total: int) -> None:
            pct = int(step / total * 95)   # reserve last 5 % for save
            job_store.update(job_id, progress=pct, current_step=step, total_steps=total)

        # ── Inference ─────────────────────────────────────────────────────────
        image, used_seed = pipeline_mgr.generate(
            prompt=job.prompt,
            negative_prompt=job.negative_prompt,
            width=p.get("width", settings.default_width),
            height=p.get("height", settings.default_height),
            num_inference_steps=p.get("num_inference_steps", settings.default_steps),
            guidance_scale=p.get("guidance_scale", settings.default_guidance_scale),
            seed=p.get("seed", -1),
            scheduler_name=p.get("scheduler", "dpm++"),
            step_callback=on_step,
        )

        # ── Save image ────────────────────────────────────────────────────────
        job_store.update(job_id, progress=97)

        timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S")
        filename = f"{timestamp}_{job_id[:8]}_{used_seed}.png"
        abs_path = os.path.join(settings.outputs_dir, filename)
        image.save(abs_path, format="PNG", optimize=False)

        # ── Compute duration ───────────────────────────────────────────────────
        now = datetime.now(timezone.utc)
        started = job_store.get(job_id).started_at
        duration = (now - started).total_seconds() if started else None

        job_store.update(
            job_id,
            status=JobStatus.COMPLETED,
            completed_at=now,
            progress=100,
            image_filename=filename,
            download_url=f"/jobs/{job_id}/download",
            duration_seconds=round(duration, 2) if duration else None,
        )

        logger.info(
            "Job %s completed in %.1f s  β†’  %s",
            job_id,
            duration or 0,
            filename,
        )

    except Exception as exc:
        logger.exception("Job %s failed: %s", job_id, exc)
        job_store.update(
            job_id,
            status=JobStatus.FAILED,
            completed_at=datetime.now(timezone.utc),
            error=str(exc),
            progress=0,
        )


# ── Worker loop (runs in its own daemon thread) ────────────────────────────────

def worker_loop() -> None:
    """Infinite loop: pull job IDs from the queue and process them."""
    logger.info("Worker thread started.")
    while True:
        job_id = _job_queue.get()   # blocks until a job arrives
        with _pending_lock:
            try:
                _pending.remove(job_id)
            except ValueError:
                pass
        try:
            _process_job(job_id)
        except Exception as exc:
            # Catch-all safety net β€” the loop must never crash
            logger.exception("Unhandled error in worker for job %s: %s", job_id, exc)
        finally:
            _job_queue.task_done()


def start_worker() -> threading.Thread:
    """Spawn and return the daemon worker thread. Call once at application startup."""
    t = threading.Thread(target=worker_loop, name="image-worker", daemon=True)
    t.start()
    return t