import asyncio import httpx import redis import torch from celery import Celery from celery.signals import worker_init from app.cnnPipeline.pipeline import CNNPipeline, load_cnn_model from app.core.config import settings from app.core.s3 import presigned_download_url cfg = { "broker": settings.REDIS_URL, "task": "tasks.process_scan", "dedupe_ttl": 86400, "http_timeout": 60, } celery = Celery("neuroassist", broker=cfg["broker"]) celery.conf.update( task_acks_late=True, task_ignore_result=True, worker_prefetch_multiplier=1, broker_connection_retry_on_startup=True, ) r = redis.Redis.from_url(cfg["broker"]) _pipeline: CNNPipeline | None = None @worker_init.connect def load_model(**_): """Runs once in the main worker process before the thread pool starts.""" global _pipeline torch.set_num_threads(1) _pipeline = CNNPipeline(load_cnn_model()) async def fetch_image(user_id: str, file_key: str) -> bytes: url = (await presigned_download_url(user_id, file_key))["download_url"] async with httpx.AsyncClient(timeout=cfg["http_timeout"]) as http: resp = await http.get(url) resp.raise_for_status() return resp.content @celery.task(name=cfg["task"]) def process_scan(file_key: str, user_id: str): # skip duplicate trigger events for the same scan if not r.set(f"scan:{file_key}", 1, nx=True, ex=cfg["dedupe_ttl"]): print(f"skip duplicate {file_key}") return try: image_bytes = asyncio.run(fetch_image(user_id, file_key)) print(len(image_bytes), image_bytes[:16]) # debug: remove once verified result = _pipeline.classify(image_bytes) print(f"[user {user_id}] {file_key} -> {result}") except Exception: r.delete(f"scan:{file_key}") # allow a retry raise