init_clickhouse.sh

#!/bin/bash
# ─────────────────────────────────────────────────────────────────────────────
#  Market-Data Observatory — Init Script
#
#  Boot sequence:
#    1. Start ClickHouse server (port 8123, internal)
#    2. Start background refresh loop (git pull every N seconds)
#    3. Start Python gateway (port 7860, public-facing)
#       → gateway waits for ClickHouse, then does initial clone + view creation
#
#  The gateway handles:
#    /api/refresh  →  on-demand git pull + view recreation
#    /api/sources  →  current config as JSON
#    /*            →  proxy to ClickHouse
# ─────────────────────────────────────────────────────────────────────────────
set -e

CONFIG="/app/sources.yaml"

echo "╔══════════════════════════════════════════════════════════╗"
echo "║   Market-Data Observatory — Boot Sequence               ║"
echo "╚══════════════════════════════════════════════════════════╝"
echo ""

# ── Validate config ──────────────────────────────────────────────────────────
if [ ! -f "$CONFIG" ]; then
  echo "[FATAL] Config file $CONFIG not found!"
  exit 1
fi

REFRESH_INTERVAL=$(yq '.refresh_interval_seconds // 300' "$CONFIG")
SOURCE_COUNT=$(yq '.sources | length' "$CONFIG")
echo "[config] $SOURCE_COUNT source(s), background refresh every ${REFRESH_INTERVAL}s"
echo ""

# ── 1. Start ClickHouse (background, port 8123) ─────────────────────────────
echo "[1/3] Starting ClickHouse on port 8123 (internal)..."
clickhouse server --config-file=/app/clickhouse-config.xml &
CH_PID=$!
echo "       PID: $CH_PID"
echo ""

# ── 2. Background auto-refresh loop ─────────────────────────────────────────
if [ "$REFRESH_INTERVAL" -gt 0 ] 2>/dev/null; then
  echo "[2/3] Starting background refresh loop (every ${REFRESH_INTERVAL}s)..."
  (
    # Wait for initial setup to complete before starting loop
    sleep "$REFRESH_INTERVAL"
    while true; do
      echo "[refresh-loop] Syncing all sources..."
      python3 /app/refresh_sources.py sync_only > /dev/null 2>&1 || true
      sleep "$REFRESH_INTERVAL"
    done
  ) &
else
  echo "[2/3] Background refresh loop DISABLED (refresh_interval_seconds is 0)."
  echo "      You must manually call /api/refresh to update data."
fi
echo ""

# ── 3. Start gateway (foreground, port 7860) ────────────────────────────────
echo "[3/3] Starting gateway on port 7860 (public)..."
echo ""
echo "╔══════════════════════════════════════════════════════════╗"
echo "║   Endpoints:                                             ║"
echo "║     /api/refresh  → instant git pull + view recreation  ║"
echo "║     /api/sources  → current config as JSON              ║"
echo "║     /?query=...   → ClickHouse SQL (proxied)            ║"
echo "╚══════════════════════════════════════════════════════════╝"
echo ""

# Gateway runs in foreground — keeps the container alive
# If it dies, the container restarts
exec python3 /app/gateway.py