docker-compose.yml

# Proof tier — the real streaming stack, run locally.
# Phase 2 brings up the broker (Redpanda, Kafka-API compatible) + a console UI.
# Phases 3 add cassandra, qdrant, and the spark job to this same file.
name: streamsearch

services:
  redpanda:
    image: redpandadata/redpanda:v24.2.7
    container_name: streamsearch-redpanda
    command:
      - redpanda
      - start
      - --smp=1
      - --overprovisioned
      - --node-id=0
      - --kafka-addr=internal://0.0.0.0:9092,external://0.0.0.0:19092
      - --advertise-kafka-addr=internal://redpanda:9092,external://localhost:19092
      - --pandaproxy-addr=internal://0.0.0.0:8082,external://0.0.0.0:18082
      - --advertise-pandaproxy-addr=internal://redpanda:8082,external://localhost:18082
      - --schema-registry-addr=internal://0.0.0.0:8081,external://0.0.0.0:18081
    ports:
      - "19092:19092"   # Kafka API (host access)
      - "18081:18081"   # schema registry
      - "18082:18082"   # pandaproxy (REST)
      - "9644:9644"     # admin API
    volumes:
      - redpanda_data:/var/lib/redpanda/data
    healthcheck:
      test: ["CMD-SHELL", "rpk cluster health | grep -E 'Healthy:.+true'"]
      interval: 10s
      timeout: 5s
      retries: 12

  # Optional web UI to eyeball topics/messages (the "console consumer").
  console:
    image: redpandadata/console:v2.7.2
    container_name: streamsearch-console
    depends_on:
      redpanda:
        condition: service_healthy
    ports:
      - "8080:8080"
    environment:
      KAFKA_BROKERS: redpanda:9092

  # Vector store — pinned to match qdrant-client 1.9.2.
  qdrant:
    image: qdrant/qdrant:v1.9.7
    container_name: streamsearch-qdrant
    ports:
      - "6533:6333"   # REST/HTTP  (host 6533 to avoid clashing with other local Qdrants)
      - "6534:6334"   # gRPC
    volumes:
      - qdrant_data:/qdrant/storage

  # Document metadata store (single node).
  cassandra:
    image: cassandra:5.0
    container_name: streamsearch-cassandra
    ports:
      - "9042:9042"
    environment:
      CASSANDRA_CLUSTER_NAME: streamsearch
      HEAP_NEWSIZE: 128M
      MAX_HEAP_SIZE: 1024M
    volumes:
      - cassandra_data:/var/lib/cassandra
    healthcheck:
      test: ["CMD-SHELL", "cqlsh -e 'SELECT now() FROM system.local' || exit 1"]
      interval: 15s
      timeout: 10s
      retries: 20
      start_period: 60s

  # Spark Structured Streaming job: Kafka -> embed -> Cassandra + Qdrant.
  # Under the "stream" profile so `docker compose up -d` starts only the infra;
  # run the job explicitly with `make stream` (docker compose --profile stream up).
  spark-job:
    build:
      context: .
      dockerfile: Dockerfile.spark
    container_name: streamsearch-spark-job
    profiles: ["stream"]
    depends_on:
      redpanda:
        condition: service_healthy
      cassandra:
        condition: service_healthy
      qdrant:
        condition: service_started
    environment:
      KAFKA_BOOTSTRAP: redpanda:9092
      CASSANDRA_HOST: cassandra
      QDRANT_URL: http://qdrant:6333
    volumes:
      - spark_checkpoints:/tmp/checkpoints

volumes:
  redpanda_data:
  qdrant_data:
  cassandra_data:
  spark_checkpoints: