Spaces:
Running
Running
add qqueue
Browse files- __pycache__/main.cpython-312.pyc +0 -0
- main.py +10 -2
- services/__pycache__/workers.cpython-312.pyc +0 -0
- services/workers.py +10 -1
__pycache__/main.cpython-312.pyc
CHANGED
|
Binary files a/__pycache__/main.cpython-312.pyc and b/__pycache__/main.cpython-312.pyc differ
|
|
|
main.py
CHANGED
|
@@ -212,12 +212,20 @@ async def process_cv_async(
|
|
| 212 |
callback_url=callback_url,
|
| 213 |
callback_secret=callback_secret
|
| 214 |
)
|
| 215 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 216 |
|
| 217 |
return APIResponse(
|
| 218 |
message="CV processing started",
|
| 219 |
statusCode=202,
|
| 220 |
-
payload={"job_id": job_id, "status": "
|
| 221 |
)
|
| 222 |
|
| 223 |
|
|
|
|
| 212 |
callback_url=callback_url,
|
| 213 |
callback_secret=callback_secret
|
| 214 |
)
|
| 215 |
+
|
| 216 |
+
try:
|
| 217 |
+
await queue_manager.enqueue(task)
|
| 218 |
+
except ValueError as exc:
|
| 219 |
+
return APIResponse(
|
| 220 |
+
message=str(exc),
|
| 221 |
+
statusCode=409,
|
| 222 |
+
payload={"job_id": job_id, "status": job_store.get_job(job_id)["status"]},
|
| 223 |
+
)
|
| 224 |
|
| 225 |
return APIResponse(
|
| 226 |
message="CV processing started",
|
| 227 |
statusCode=202,
|
| 228 |
+
payload={"job_id": job_id, "status": "QUEUED"},
|
| 229 |
)
|
| 230 |
|
| 231 |
|
services/__pycache__/workers.cpython-312.pyc
CHANGED
|
Binary files a/services/__pycache__/workers.cpython-312.pyc and b/services/__pycache__/workers.cpython-312.pyc differ
|
|
|
services/workers.py
CHANGED
|
@@ -166,6 +166,9 @@ class QueueManager:
|
|
| 166 |
self.concurrency = concurrency
|
| 167 |
self.queue: asyncio.Queue = asyncio.Queue()
|
| 168 |
self.tasks: list = []
|
|
|
|
|
|
|
|
|
|
| 169 |
|
| 170 |
async def start(self) -> None:
|
| 171 |
"""Spawn background worker coroutines."""
|
|
@@ -188,6 +191,7 @@ class QueueManager:
|
|
| 188 |
try:
|
| 189 |
task: CvTask = await self.queue.get() # suspends here — event loop stays free
|
| 190 |
job_store.set_processing(task.job_id)
|
|
|
|
| 191 |
print(f"[QUEUE] Worker picked up job {task.job_id}. "
|
| 192 |
f"Remaining in queue: {self.queue.qsize()}")
|
| 193 |
await self.worker.run_cv_processing(
|
|
@@ -204,7 +208,12 @@ class QueueManager:
|
|
| 204 |
print(f"[QUEUE] Unhandled error in worker loop: {exc}")
|
| 205 |
|
| 206 |
async def enqueue(self, task: CvTask) -> None:
|
| 207 |
-
"""Add a task to the queue
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 208 |
job_store.set_queued(task.job_id)
|
| 209 |
await self.queue.put(task) # asyncio.Queue with no maxsize — never blocks
|
| 210 |
print(f"[QUEUE] Job {task.job_id} enqueued. Queue depth: {self.queue.qsize()}")
|
|
|
|
| 166 |
self.concurrency = concurrency
|
| 167 |
self.queue: asyncio.Queue = asyncio.Queue()
|
| 168 |
self.tasks: list = []
|
| 169 |
+
# Tracks job IDs currently sitting in the queue or being processed.
|
| 170 |
+
# Used to reject duplicate submissions before they reach the ML worker.
|
| 171 |
+
self._active_ids: set[str] = set()
|
| 172 |
|
| 173 |
async def start(self) -> None:
|
| 174 |
"""Spawn background worker coroutines."""
|
|
|
|
| 191 |
try:
|
| 192 |
task: CvTask = await self.queue.get() # suspends here — event loop stays free
|
| 193 |
job_store.set_processing(task.job_id)
|
| 194 |
+
self._active_ids.discard(task.job_id) # no longer queued, now processing
|
| 195 |
print(f"[QUEUE] Worker picked up job {task.job_id}. "
|
| 196 |
f"Remaining in queue: {self.queue.qsize()}")
|
| 197 |
await self.worker.run_cv_processing(
|
|
|
|
| 208 |
print(f"[QUEUE] Unhandled error in worker loop: {exc}")
|
| 209 |
|
| 210 |
async def enqueue(self, task: CvTask) -> None:
|
| 211 |
+
"""Add a task to the queue — rejects duplicate job IDs already in-flight."""
|
| 212 |
+
if task.job_id in self._active_ids:
|
| 213 |
+
print(f"[QUEUE] Duplicate submission rejected: job {task.job_id} is already queued.")
|
| 214 |
+
raise ValueError(f"Job {task.job_id} is already in the processing queue.")
|
| 215 |
+
|
| 216 |
+
self._active_ids.add(task.job_id)
|
| 217 |
job_store.set_queued(task.job_id)
|
| 218 |
await self.queue.put(task) # asyncio.Queue with no maxsize — never blocks
|
| 219 |
print(f"[QUEUE] Job {task.job_id} enqueued. Queue depth: {self.queue.qsize()}")
|