backend/src/workers/celery_app.py

import os
from celery import Celery
from ..config import get_settings

settings = get_settings()

# Connection pool options
_REDIS_TRANSPORT_OPTS = {
    "max_connections": 10,
    "socket_keepalive": True,
    "socket_connect_timeout": 10,
    "retry_on_timeout": True,
}

celery_app = Celery(
    "talentpulse",
    broker=settings.redis_url,
    backend=settings.redis_url,
    include=["src.workers.ingest", "src.workers.explain"],
)

celery_app.conf.update(
    task_serializer="json",
    accept_content=["json"],
    result_serializer="json",
    timezone="UTC",
    enable_utc=True,
    
    # ----------------------------------------------------
    # FREE TIER REDIS OPTIMIZATIONS (Max 30 Connections)
    # ----------------------------------------------------
    worker_send_task_events=False,        # Disable task events (saves connections)
    worker_enable_remote_control=False,   # Disable mingle/broadcast (saves 2 conns per worker)
    task_track_started=False,             # Disable state tracking noise
    result_expires=1800,
    
    # Limit broker & backend pool
    broker_transport_options=_REDIS_TRANSPORT_OPTS,
    redis_max_connections=10,
    broker_pool_limit=10,
)