Spaces:
Sleeping
Sleeping
File size: 3,171 Bytes
0a602d6 4519cf7 0a602d6 4519cf7 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 | # Proof tier — the real streaming stack, run locally.
# Phase 2 brings up the broker (Redpanda, Kafka-API compatible) + a console UI.
# Phases 3 add cassandra, qdrant, and the spark job to this same file.
name: streamsearch
services:
redpanda:
image: redpandadata/redpanda:v24.2.7
container_name: streamsearch-redpanda
command:
- redpanda
- start
- --smp=1
- --overprovisioned
- --node-id=0
- --kafka-addr=internal://0.0.0.0:9092,external://0.0.0.0:19092
- --advertise-kafka-addr=internal://redpanda:9092,external://localhost:19092
- --pandaproxy-addr=internal://0.0.0.0:8082,external://0.0.0.0:18082
- --advertise-pandaproxy-addr=internal://redpanda:8082,external://localhost:18082
- --schema-registry-addr=internal://0.0.0.0:8081,external://0.0.0.0:18081
ports:
- "19092:19092" # Kafka API (host access)
- "18081:18081" # schema registry
- "18082:18082" # pandaproxy (REST)
- "9644:9644" # admin API
volumes:
- redpanda_data:/var/lib/redpanda/data
healthcheck:
test: ["CMD-SHELL", "rpk cluster health | grep -E 'Healthy:.+true'"]
interval: 10s
timeout: 5s
retries: 12
# Optional web UI to eyeball topics/messages (the "console consumer").
console:
image: redpandadata/console:v2.7.2
container_name: streamsearch-console
depends_on:
redpanda:
condition: service_healthy
ports:
- "8080:8080"
environment:
KAFKA_BROKERS: redpanda:9092
# Vector store — pinned to match qdrant-client 1.9.2.
qdrant:
image: qdrant/qdrant:v1.9.7
container_name: streamsearch-qdrant
ports:
- "6533:6333" # REST/HTTP (host 6533 to avoid clashing with other local Qdrants)
- "6534:6334" # gRPC
volumes:
- qdrant_data:/qdrant/storage
# Document metadata store (single node).
cassandra:
image: cassandra:5.0
container_name: streamsearch-cassandra
ports:
- "9042:9042"
environment:
CASSANDRA_CLUSTER_NAME: streamsearch
HEAP_NEWSIZE: 128M
MAX_HEAP_SIZE: 1024M
volumes:
- cassandra_data:/var/lib/cassandra
healthcheck:
test: ["CMD-SHELL", "cqlsh -e 'SELECT now() FROM system.local' || exit 1"]
interval: 15s
timeout: 10s
retries: 20
start_period: 60s
# Spark Structured Streaming job: Kafka -> embed -> Cassandra + Qdrant.
# Under the "stream" profile so `docker compose up -d` starts only the infra;
# run the job explicitly with `make stream` (docker compose --profile stream up).
spark-job:
build:
context: .
dockerfile: Dockerfile.spark
container_name: streamsearch-spark-job
profiles: ["stream"]
depends_on:
redpanda:
condition: service_healthy
cassandra:
condition: service_healthy
qdrant:
condition: service_started
environment:
KAFKA_BOOTSTRAP: redpanda:9092
CASSANDRA_HOST: cassandra
QDRANT_URL: http://qdrant:6333
volumes:
- spark_checkpoints:/tmp/checkpoints
volumes:
redpanda_data:
qdrant_data:
cassandra_data:
spark_checkpoints:
|