backend/celery_config.py

"""
Unified Celery configuration for the Lin application.
This centralizes all Celery configuration to avoid conflicts.
"""

import os
from celery import Celery
from celery.schedules import crontab
from backend.config import Config

# Create Celery instance
celery_app = Celery('lin_app')

# Configure Celery with broker and result backend
celery_app.conf.broker_url = os.environ.get('CELERY_BROKER_URL', 'redis://localhost:6379/0')
celery_app.conf.result_backend = os.environ.get('CELERY_RESULT_BACKEND', 'redis://localhost:6379/0')

# Additional Celery configuration
celery_app.conf.update(
    # Task serialization
    task_serializer='json',
    accept_content=['json'],
    result_serializer='json',
    timezone='UTC',
    enable_utc=True,
    
    # Task routing
    task_routes={
        'backend.celery_tasks.content_tasks.generate_content_task': {'queue': 'content'},
        'backend.celery_tasks.content_tasks.publish_post_task': {'queue': 'publish'},
        'backend.celery_tasks.schedule_loader.load_schedules_task': {'queue': 'scheduler'},
    },
    
    # Worker configuration
    worker_prefetch_multiplier=1,
    task_acks_late=True,
    worker_max_tasks_per_child=100,
    
    # Beat scheduler configuration
    beat_scheduler='django_celery_beat.schedulers:DatabaseScheduler',
    beat_schedule={
        # This task will run every 5 minutes to load schedules from the database
        'load-schedules': {
            'task': 'backend.celery_tasks.schedule_loader.load_schedules_task',
            'schedule': crontab(minute='*/5'),
        },
    },
    
    # Task result expiration
    result_expires=3600,  # 1 hour
    
    # Task time limits
    task_soft_time_limit=300,  # 5 minutes
    task_time_limit=600,  # 10 minutes
    
    # Rate limiting
    task_annotations=(
        ('backend.celery_tasks.content_tasks.generate_content_task', {'rate_limit': '10/h'}),
        ('backend.celery_tasks.content_tasks.publish_post_task', {'rate_limit': '30/h'}),
    ),
    
    # Error handling
    task_reject_on_worker_lost=True,
    worker_disable_rate_limits=False,
    
    # Security
    result_backend_transport_options={'visibility': 'hidden'},
    broker_connection_max_retries=3,
    broker_connection_retry_delay=5,
)

# Import tasks to ensure they're registered
from backend import celery_tasks

__all__ = ['celery_app']