celery_config.py

import os
from dotenv import load_dotenv

# Load environment variables
load_dotenv()

# Celery configuration with hosted Redis server
redis_password = os.environ.get('REDIS_PASSWORD')
# Make sure to set these environment variables with the correct values
redis_host = os.environ.get('REDIS_HOST')
redis_port = os.environ.get('REDIS_PORT', '6379')

# Validate that we have the host information
if not redis_host:
    raise ValueError("REDIS_HOST environment variable must be set")

redis_url = f"redis://:{redis_password}@{redis_host}:{redis_port}/0"

broker_url = os.environ.get('REDIS_URL', redis_url)
result_backend = os.environ.get('REDIS_URL', redis_url)

# Task serialization format
task_serializer = 'json'
accept_content = ['json']
result_serializer = 'json'

# Enable UTC timezone
enable_utc = True

# Task execution settings
task_acks_late = True
task_reject_on_worker_lost = True
worker_max_tasks_per_child = 1000

# Task timeouts
task_time_limit = 600  # 10 minutes
task_soft_time_limit = 300  # 5 minutes

# Task routes
task_routes = {
    'utils.celery_tasks.process_log_document': {'queue': 'logs'},
    'utils.celery_tasks.process_incident_forms': {'queue': 'incidents'},
}

# Rate limits
task_annotations = {
    'utils.celery_tasks.process_log_document': {'rate_limit': '10/h'},
    'utils.celery_tasks.process_incident_forms': {'rate_limit': '20/h'},
}