app/scraper/docker-compose.yml

version: '3.8'

services:
  # Scrapy Crawler
  scraper:
    build: .
    container_name: oag_scraper
    volumes:
      - ./data:/app/data
      - ./logs:/app/logs
    environment:
      - POSTGRES_HOST=postgres
      - POSTGRES_PORT=5432
      - POSTGRES_DB=oag_knowledge
      - POSTGRES_USER=oag_scraper
      - POSTGRES_PASSWORD=changeme
      - REDIS_HOST=redis
      - REDIS_PORT=6379
      - MINIO_ENDPOINT=minio:9000
      - MINIO_ACCESS_KEY=minioadmin
      - MINIO_SECRET_KEY=minioadmin
      - MINIO_BUCKET=oag-raw-pdfs
      - QDRANT_HOST=qdrant
      - QDRANT_PORT=6333
      - NEO4J_URI=bolt://neo4j:7687
      - NEO4J_USER=neo4j
      - NEO4J_PASSWORD=password
    depends_on:
      - postgres
      - redis
      - minio
      - qdrant
      - neo4j
    networks:
      - oag-network

  # Celery Worker
  worker:
    build: .
    container_name: oag_worker
    command: celery -A workers.pdf_processor worker --loglevel=info --concurrency=2
    volumes:
      - ./data:/app/data
      - ./logs:/app/logs
    environment:
      - POSTGRES_HOST=postgres
      - REDIS_HOST=redis
      - MINIO_ENDPOINT=minio:9000
      - QDRANT_HOST=qdrant
      - NEO4J_URI=bolt://neo4j:7687
    depends_on:
      - redis
      - postgres
      - minio
      - qdrant
      - neo4j
    networks:
      - oag-network

  # PostgreSQL
  postgres:
    image: postgres:16-alpine
    container_name: oag_postgres
    environment:
      - POSTGRES_DB=oag_knowledge
      - POSTGRES_USER=oag_scraper
      - POSTGRES_PASSWORD=changeme
    volumes:
      - postgres_data:/var/lib/postgresql/data
    ports:
      - "5432:5432"
    networks:
      - oag-network

  # Redis
  redis:
    image: redis:7-alpine
    container_name: oag_redis
    ports:
      - "6379:6379"
    volumes:
      - redis_data:/data
    networks:
      - oag-network

  # MinIO
  minio:
    image: minio/minio:latest
    container_name: oag_minio
    command: server /data --console-address ":9001"
    environment:
      - MINIO_ROOT_USER=minioadmin
      - MINIO_ROOT_PASSWORD=minioadmin
    volumes:
      - minio_data:/data
    ports:
      - "9000:9000"
      - "9001:9001"
    networks:
      - oag-network

  # Qdrant
  qdrant:
    image: qdrant/qdrant:latest
    container_name: oag_qdrant
    ports:
      - "6333:6333"
    volumes:
      - qdrant_data:/qdrant/storage
    networks:
      - oag-network

  # Neo4j
  neo4j:
    image: neo4j:5-community
    container_name: oag_neo4j
    environment:
      - NEO4J_AUTH=neo4j/password
      - NEO4J_PLUGINS=["apoc"]
    volumes:
      - neo4j_data:/data
    ports:
      - "7474:7474"
      - "7687:7687"
    networks:
      - oag-network

volumes:
  postgres_data:
  redis_data:
  minio_data:
  qdrant_data:
  neo4j_data:

networks:
  oag-network:
    driver: bridge