docker-compose.yaml

x-airflow-common:
  &airflow-common
  build:
    context: ./airflow
    dockerfile: Dockerfile.airflow
  environment:
    &airflow-common-env
    AIRFLOW__CORE__EXECUTOR: LocalExecutor
    AIRFLOW__CORE__AUTH_MANAGER: airflow.providers.fab.auth_manager.fab_auth_manager.FabAuthManager 
    AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres/airflow 
    AIRFLOW__CORE__FERNET_KEY: ${AIRFLOW__CORE__FERNET_KEY} # needed for multiuser or production
    AIRFLOW__API_AUTH__JWT_SECRET: ${JWT_SECRET} 
    AIRFLOW__WEBSERVER__SECRET_KEY: ${AIRFLOW__WEBSERVER__SECRET_KEY}
    AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: 'true'
    AIRFLOW__CORE__LOAD_EXAMPLES: 'false'
    AIRFLOW__WEBSERVER__BASE_URL: http://localhost:8080
    AIRFLOW__CORE__EXECUTION_API_SERVER_URL: 'http://airflow-apiserver:8080/execution/'
    AIRFLOW__SCHEDULER__ENABLE_HEALTH_CHECK: 'true'
    # _PIP_ADDITIONAL_REQUIREMENTS: ${_PIP_ADDITIONAL_REQUIREMENTS:-}
    AIRFLOW_CONFIG: '/opt/airflow/config/airflow.cfg'
    # MLFLOW_TRACKING_URI: 'http://mlflow:5000'

  volumes:
    - ${AIRFLOW_PROJ_DIR:-.}/airflow/dags:/opt/airflow/dags
    - ${AIRFLOW_PROJ_DIR:-.}/airflow/logs:/opt/airflow/logs
    - ${AIRFLOW_PROJ_DIR:-.}/airflow/config:/opt/airflow/config
    - ${AIRFLOW_PROJ_DIR:-.}/airflow/plugins:/opt/airflow/plugins
    - ${AIRFLOW_PROJ_DIR:-.}/data:/opt/airflow/data
    - ${AIRFLOW_PROJ_DIR:-.}/scripts:/opt/airflow/scripts
    - ${AIRFLOW_PROJ_DIR:-.}/model:/opt/airflow/model
    - ${AIRFLOW_PROJ_DIR:-.}/app:/opt/airflow/app
  user: "${AIRFLOW_UID:-50000}:0"
  depends_on:
    &airflow-common-depends-on
    postgres:
      condition: service_healthy

services:
  postgres:
    image: postgres:13
    environment:
      POSTGRES_USER: ${POSTGRES_USER} # using nginx would provide an added measure of security wnen in production
      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
      POSTGRES_DB: airflow
    volumes:
      - postgres-db-volume:/var/lib/postgresql/data
    healthcheck:
      test: ["CMD", "pg_isready", "-U", "airflow"]
      interval: 10s
      retries: 5
      start_period: 5s
    restart: always

  airflow-apiserver:
    <<: *airflow-common
    command: api-server
    ports:
      - "8080:8080"
    healthcheck:
      test: ["CMD", "curl", "--fail", "http://localhost:8080/api/v2/version"]
      interval: 30s
      timeout: 10s
      retries: 5
      start_period: 30s
    restart: always
    depends_on:
      <<: *airflow-common-depends-on
      airflow-init:
        condition: service_completed_successfully

  airflow-scheduler:
    <<: *airflow-common
    command: scheduler
    healthcheck:
      test: ["CMD", "curl", "--fail", "http://localhost:8974/health"]
      interval: 30s
      timeout: 60s
      retries: 5
      start_period: 30s
    restart: always
    depends_on:
      airflow-init:
        condition: service_completed_successfully

  airflow-dag-processor:
    <<: *airflow-common
    command: dag-processor
    healthcheck:
      test: ["CMD-SHELL", 'airflow jobs check --job-type DagProcessorJob --hostname "$${HOSTNAME}"']
      interval: 30s
      timeout: 10s
      retries: 5
      start_period: 30s
    restart: always
    depends_on:
      airflow-init:
        condition: service_completed_successfully

  airflow-triggerer:
    <<: *airflow-common
    command: triggerer
    healthcheck:
      test: ["CMD-SHELL", 'airflow jobs check --job-type TriggererJob --hostname "$${HOSTNAME}"']
      interval: 40s
      timeout: 10s
      retries: 5
      start_period: 40s
    restart: always
    depends_on:
      airflow-init:
        condition: service_completed_successfully

  airflow-init:
    <<: *airflow-common
    entrypoint: /bin/bash
    command:
      - -c
      - |
        [[ -z "${AIRFLOW_UID}" ]] && export AIRFLOW_UID=$(id -u)
        mkdir -p /opt/airflow/{logs,dags,plugins,config}
        /entrypoint airflow config list >/dev/null
        chown -R "${AIRFLOW_UID}:0" /opt/airflow
        ls -la /opt/airflow/{logs,dags,plugins,config}
    environment:
      <<: *airflow-common-env
      _AIRFLOW_DB_MIGRATE: 'true'
      _AIRFLOW_WWW_USER_CREATE: 'true'
      _AIRFLOW_WWW_USER_USERNAME: ${AIRFLOW_WWW_USER_USERNAME}
      _AIRFLOW_WWW_USER_PASSWORD: ${AIRFLOW_WWW_USER_PASSWORD}
      # _PIP_ADDITIONAL_REQUIREMENTS: ${_PIP_ADDITIONAL_REQUIREMENTS}
    user: "0:0"

  airflow-cli:
    <<: *airflow-common
    profiles: [debug]
    command:
      - bash
      - -c
      - airflow
    environment:
      <<: *airflow-common-env
      CONNECTION_CHECK_MAX_COUNT: "0"
    depends_on:
      postgres:
        condition: service_healthy

  rag-api:
    build: ./app
    ports:
      - "8000:8000"
    volumes:
      - ./vectorstore:/app/vectorstore

  # streamlit:
  #   container_name: streamlit_app
  #   build:
  #     context: ./app
  #     dockerfile: Dockerfile.streamlit
  #   volumes:
  #     - ./app:/app
  #     - ./data:/data
  #     - ./script:/script
  #   working_dir: /app
  #   ports:
  #     - "127.0.0.1:8501:8501"

  # mlflow:
  #   build:
  #     context: ./mlflow
  #     dockerfile: Dockerfile.mlflow
  #   ports:
  #     - "127.0.0.1:5000:5000"

  # Optional nginx reverse proxy for shared/internal environments
  # nginx:
  #   image: nginx:alpine
  #   ports:
  #     - "80:80"
  #   volumes:
  #     - ./nginx/default.conf:/etc/nginx/conf.d/default.conf:ro
  #     - ./nginx/htpasswd:/etc/nginx/.htpasswd:ro
  #   depends_on:
  #     - airflow-apiserver
  #     - mlflow
  #     - streamlit

volumes:
  postgres-db-volume: