web/app.py

"""
=============================================================
 BAL Chatbot — Flask Web API
 Usage: python web/app.py
=============================================================
This script:
  1. Uses Groq as the only LLM provider
  2. Loads FAISS index and chunk metadata
  3. For each /api/chat request:
       a. Retrieves the most relevant chunks (ONCE per query)
       b. Builds an augmented prompt (context + question)
       c. Sends the request through the LLM gateway
       d. Streams the response from Groq
  4. Exposes /api/health, /api/chat, /api/clear endpoints
=============================================================
Prerequisites:
  - A valid Groq API key in the GROQ_API_KEY environment variable
  - 01_build_vectorstore.py must have been run
=============================================================
"""

import os
import sys
import json
import time
import logging
import re
from pathlib import Path
from datetime import datetime, timezone
from typing import List, Dict, Generator, Optional, Tuple

import numpy as np
import faiss
from sentence_transformers import SentenceTransformer
from sqlalchemy import func
from sqlalchemy.exc import IntegrityError
from flask import request, jsonify, Response, stream_with_context, send_from_directory, session
from curl_cffi import requests as curl_requests

try:
    from config import CONFIG, SYSTEM_PROMPT, PROJECT_ROOT, WEB_DIR, LOG_DIR
except ImportError:
    from web.config import CONFIG, SYSTEM_PROMPT, PROJECT_ROOT, WEB_DIR, LOG_DIR

try:
    import extensions
    from extensions import app
    from models import User, UsageCounter, ChatLog, init_db, database_ready
except ImportError:
    import web.extensions as extensions
    from web.extensions import app
    from web.models import User, UsageCounter, ChatLog, init_db, database_ready


# ── Logging ───────────────────────────────────────────────────────────────────
logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s [%(levelname)s] %(message)s",
    handlers=[
        logging.FileHandler(LOG_DIR / "web.log", encoding="utf-8"),
        logging.StreamHandler(),
    ],
)
log = logging.getLogger(__name__)


# ═══════════════════════════════════════════════════════════════════════════════
# Auth, Persistence and Quotas
# ═══════════════════════════════════════════════════════════════════════════════

def utc_now() -> datetime:
    return datetime.now(timezone.utc)


def today_key() -> str:
    return utc_now().strftime("%Y-%m-%d")


def minute_key() -> str:
    return utc_now().strftime("%Y-%m-%dT%H:%M")


def normalize_email(email: str) -> str:
    return email.strip().lower()


def role_for_email(email: str) -> str:
    return "admin" if normalize_email(email) in CONFIG["admin_emails"] else "user"


def user_to_public(user: User) -> Dict:
    role = user.role
    is_visitor = role == "visitor" or user.provider == "fingerprint"
    return {
        "id": user.id,
        "email": None if is_visitor else user.email,
        "role": "visitor" if is_visitor else role,
        "mode": "visitor" if is_visitor else "account",
    }


def get_client_fingerprint() -> Optional[str]:
    fingerprint = (request.headers.get("X-Client-Fingerprint") or "").strip()
    if not fingerprint:
        return None
    if not re.fullmatch(r"[A-Za-z0-9_-]{8,255}", fingerprint):
        log.warning("Rejected malformed client fingerprint: %r", fingerprint[:80])
        return None
    return fingerprint


def get_current_identity() -> Optional[Dict]:
    user_id = session.get("user_id")
    if user_id:
        with extensions.SessionLocal() as db:
            user = db.get(User, int(user_id))
            if user:
                return {
                    "subject_type": "user",
                    "subject_id": str(user.id),
                    "role": user.role,
                    "public": user_to_public(user),
                }
        session.pop("user_id", None)

    fingerprint = get_client_fingerprint() or session.get("fingerprint")

    if fingerprint:
        try:
            with extensions.SessionLocal() as db:
                user = db.query(User).filter(User.fingerprint == fingerprint).first()
                if user is None:
                    log.info("No existing user with fingerprint found: %s", fingerprint)
                else:
                    log.info("Found existing user id=%s for fingerprint", user.id)
                if user is None:
                    user = User(
                        email=None,
                        fingerprint=fingerprint,
                        password_hash=None,
                        provider="fingerprint",
                        role="visitor",
                        created_at=utc_now().isoformat(),
                    )
                    db.add(user)
                    try:
                        db.commit()
                        db.refresh(user)
                        log.info("Created visitor user id=%s fingerprint=%s", user.id, fingerprint)
                    except IntegrityError:
                        db.rollback()
                        user = db.query(User).filter(User.fingerprint == fingerprint).first()
                        if user:
                            log.info("Detected concurrent creation; using existing user id=%s", user.id)

                if user:
                    session["fingerprint"] = fingerprint
                    return {
                        "subject_type": "user",
                        "subject_id": str(user.id),
                        "role": user.role,
                        "public": user_to_public(user),
                    }
        except Exception as e:
            log.exception("Failed to establish fingerprint identity (%s). Headers: %s", e, {
                "X-Forwarded-For": request.headers.get("X-Forwarded-For"),
                "X-Client-Fingerprint": request.headers.get("X-Client-Fingerprint"),
                "User-Agent": request.headers.get("User-Agent"),
            })
            return None

    return None


def get_usage(subject_type: str, subject_id: str, period_type: str, period_key: str) -> int:
    with extensions.SessionLocal() as db:
        row = db.get(UsageCounter, (subject_type, subject_id, period_type, period_key))
        return int(row.count) if row else 0


def quota_snapshot(identity: Dict) -> Dict:
    limits = CONFIG["limits"][identity["role"]]
    daily_used = get_usage(identity["subject_type"], identity["subject_id"], "day", today_key())
    minute_used = get_usage(identity["subject_type"], identity["subject_id"], "minute", minute_key())
    return {
        "daily_limit": limits["daily"],
        "daily_used": daily_used,
        "daily_remaining": max(limits["daily"] - daily_used, 0),
        "minute_limit": limits["minute"],
        "minute_used": minute_used,
        "minute_remaining": max(limits["minute"] - minute_used, 0),
    }


def check_quota(identity: Dict) -> Tuple[bool, Dict, str]:
    usage = quota_snapshot(identity)
    if usage["daily_remaining"] <= 0:
        return False, usage, "Günlük soru limitin doldu."
    if usage["minute_remaining"] <= 0:
        return False, usage, "Dakikalık soru limitine ulaştın. Biraz bekleyip tekrar dene."
    return True, usage, ""


def increment_usage(identity: Dict) -> Dict:
    now = utc_now().isoformat()
    rows = [("day", today_key()), ("minute", minute_key())]
    with extensions.SessionLocal() as db:
        for period_type, period_key in rows:
            key = (
                identity["subject_type"],
                identity["subject_id"],
                period_type,
                period_key,
            )
            counter = db.get(UsageCounter, key)
            if counter is None:
                counter = UsageCounter(
                    subject_type=identity["subject_type"],
                    subject_id=identity["subject_id"],
                    period_type=period_type,
                    period_key=period_key,
                    count=1,
                    updated_at=now,
                )
                db.add(counter)
            else:
                counter.count += 1
                counter.updated_at = now
        db.commit()
    return quota_snapshot(identity)


@app.before_request
def enforce_https():
    if request.path.startswith("/api/health"):
        return None
    if CONFIG["force_https"] and not request.is_secure:
        host = request.headers.get("Host", "")
        is_local = host.startswith("127.0.0.1") or host.startswith("localhost")
        if not is_local:
            return "", 308, {"Location": request.url.replace("http://", "https://", 1)}
    return None


# ═══════════════════════════════════════════════════════════════════════════════
# 1. Vector Store (LOCAL EMBEDDING — no API calls)
# ═══════════════════════════════════════════════════════════════════════════════

class VectorStore:
    """Manages the FAISS vector database and local embedding model."""

    def __init__(self, index_path: str, chunks_path: str, model_name: str):
        if not Path(index_path).exists():
            raise FileNotFoundError(
                f"FAISS index not found: {index_path}\n"
                "Run '01_build_vectorstore.py' first."
            )

        log.info("Loading FAISS index...")
        self.index = faiss.read_index(index_path)
        log.info(f"FAISS index loaded: {self.index.ntotal} vectors")

        log.info("Loading chunk metadata...")
        with open(chunks_path, "r", encoding="utf-8") as f:
            self.chunks: List[Dict] = json.load(f)
        log.info(f"Chunk metadata loaded: {len(self.chunks)} chunks")

        self.embedding_model_name = model_name
        self._local_model = None  # Lazy-load: shared instance from global scope

        log.info(f"✓ Vector store ready — {self.index.ntotal} chunks, model={model_name}")

    def _get_model(self) -> SentenceTransformer:
        """Returns the shared SentenceTransformer instance (lazy-loaded at startup)."""
        if extensions.embedding_model is None:
            log.info(f"Loading local embedding model: {self.embedding_model_name}")
            t0 = time.time()
            extensions.embedding_model = SentenceTransformer(self.embedding_model_name)
            log.info(
                f"✓ Model loaded in {time.time() - t0:.1f}s — "
                f"dim={extensions.embedding_model.get_sentence_embedding_dimension()}"
            )
        return extensions.embedding_model

    def _embed_text_sync(self, text: str) -> Optional[np.ndarray]:
        """
        Synchronously embeds a single text string using local SentenceTransformer.
        Returns a (1, dim) float32 numpy array normalized for cosine similarity,
        or None on failure.
        """
        try:
            model = self._get_model()
            embedding = model.encode(
                [text],
                normalize_embeddings=True,
                convert_to_numpy=True,
            ).astype("float32")
            return embedding
        except Exception as e:
            log.error(f"Local embedding failed: {e}")
            return None

    def retrieve(self, query: str, top_k: int = 5) -> List[Dict]:
        """
        Returns the top-k most relevant chunks for the given query.
        E5 model requires the 'query:' prefix for retrieval queries.
        """
        query_text = f"query: {query}"

        # Embed using local model — fast, no API latency
        embedding = self._embed_text_sync(query_text)
        if embedding is None:
            log.error(f"Could not embed query: {query[:100]}")
            raise RuntimeError(
                "Sorgu embedding'i başarısız. Lütfen daha sonra tekrar deneyin."
            )

        scores, indices = self.index.search(embedding, top_k)

        results = []
        for score, idx in zip(scores[0], indices[0]):
            if idx == -1:    # FAISS returns -1 for empty slots
                continue
            chunk = self.chunks[idx].copy()
            chunk["relevance_score"] = float(score)
            results.append(chunk)

        return results


# ═══════════════════════════════════════════════════════════════════════════════
# 2. Context Formatting
# ═══════════════════════════════════════════════════════════════════════════════

def format_context(retrieved_chunks: List[Dict], score_threshold: float = 0.35) -> str:
    """
    Builds the context string injected into the LLM prompt.
    Chunks below score_threshold are discarded to reduce noise.
    """
    if not retrieved_chunks:
        return "Bağlamda ilgili bilgi bulunamadı."

    parts = []
    for chunk in retrieved_chunks:
        if chunk.get("relevance_score", 0) < score_threshold:
            continue
        breadcrumb = chunk.get("breadcrumb", "")
        text = chunk.get("text", "")
        parts.append(f"[Kaynak: {breadcrumb}]\n{text}")

    return "\n\n---\n\n".join(parts) if parts else "Bağlamda yeterince ilgili bilgi bulunamadı."


def build_augmented_user_message(user_input: str, context: str) -> str:
    """Wraps the user question with the retrieved RAG context."""
    return (
        f"## İlgili Bağlam (Okul Bilgi Kaynağı)\n\n"
        f"{context}\n\n"
        f"---\n\n"
        f"## Kullanıcı Sorusu\n\n{user_input}"
    )


def build_sources_payload(retrieved: List[Dict], score_threshold: float = 0.35) -> List[Dict]:
    """Builds the sources list sent to the frontend after streaming ends."""
    return [
        {
            "breadcrumb": r.get("breadcrumb", ""),
            "score": round(r.get("relevance_score", 0), 3),
        }
        for r in retrieved[:3]
        if r.get("relevance_score", 0) >= score_threshold
    ]


def strip_reasoning_blocks(text: str) -> str:
    """Removes reasoning traces emitted by models that expose <think> blocks."""
    if not text:
        return text

    cleaned = re.sub(r"<think\b[^>]*>.*?</think>", "", text, flags=re.IGNORECASE | re.DOTALL)
    cleaned = re.sub(r"<thinking\b[^>]*>.*?</thinking>", "", cleaned, flags=re.IGNORECASE | re.DOTALL)
    cleaned = re.sub(r"<think\b[^>]*>.*\Z", "", cleaned, flags=re.IGNORECASE | re.DOTALL)
    cleaned = re.sub(r"<thinking\b[^>]*>.*\Z", "", cleaned, flags=re.IGNORECASE | re.DOTALL)
    return cleaned.strip()


# ═══════════════════════════════════════════════════════════════════════════════
# 3. Groq Backend (streaming)
# ═══════════════════════════════════════════════════════════════════════════════

def stream_groq_model(messages: List[Dict], model: str, api_key: str, key_index: int) -> Tuple[str, Optional[Dict]]:
    """
    Streams one Groq model attempt using curl_cffi to bypass Cloudflare 403 blocks on Render.
    Returns (full_response, failure_info).
    """
    full_response = ""
    headers = {
        "Authorization": f"Bearer {api_key}",
        "Content-Type": "application/json",
        "Accept": "text/event-stream"
    }
    payload = {
        "model": model,
        "messages": messages,
        "stream": True,
        "temperature": CONFIG["llm_temperature"],
        "max_tokens": CONFIG["llm_max_tokens"],
        "top_p": CONFIG["llm_top_p"],
    }

    try:
        log.info(
            "GROQ REQUEST -> model=%s key_index=%s url=%s",
            model,
            key_index,
            CONFIG["groq_url"]
        )
        resp = curl_requests.post(
            CONFIG["groq_url"],
            headers=headers,
            json=payload,
            stream=True,
            timeout=CONFIG["groq_timeout"],
            impersonate="chrome"
        )

        resp.raise_for_status()

        for line in resp.iter_lines():
            if not line:
                continue

            if isinstance(line, bytes):
                line_str = line.decode("utf-8")
            else:
                line_str = str(line)

            if not line_str.startswith("data: "):
                continue

            data_text = line_str[6:].strip()
            if data_text == "[DONE]":
                break

            try:
                data = json.loads(data_text)
            except json.JSONDecodeError:
                continue

            delta = data.get("choices", [{}])[0].get("delta", {})
            token = delta.get("content", "")
            if token:
                full_response += token
                yield f"data: {json.dumps({'token': token})}\n\n"

    except curl_requests.errors.RequestsError as e:
        error_msg = str(e).lower()
        reason = "exception"

        if "timeout" in error_msg:
            reason = "timeout"
            return "Groq API zaman aşımına uğradı. Lütfen tekrar deneyin.", {
                "retryable": True, "model": model, "key_index": key_index, "reason": reason
            }
        elif "connect" in error_msg or "resolve" in error_msg:
            reason = "connection"
            return "Groq API bağlantısı kurulamadı. Lütfen daha sonra tekrar deneyin.", {
                "retryable": True, "model": model, "key_index": key_index, "reason": reason
            }

        status_code = 0
        response_text = ""
        rate_headers = {}

        if hasattr(e, "response") and e.response is not None:
            status_code = e.response.status_code
            response_text = e.response.text
            log.error("FULL GROQ BODY:\n%s", response_text)
            rate_headers = {
                key: value
                for key, value in e.response.headers.items()
                if key.lower().startswith(("x-ratelimit", "retry-after", "x-request-id"))
            }
            reason = f"http_{status_code}"

        log.error(
            "Groq API HTTP error status=%s model=%s rate_headers=%s body=%s",
            status_code,
            model,
            rate_headers,
            response_text,
            exc_info=True,
        )

        if status_code > 0:
            retryable = status_code in {404, 429} or 500 <= status_code <= 599
            return f"Groq API hatası: HTTP {status_code}", {
                "retryable": retryable,
                "model": model,
                "key_index": key_index,
                "reason": reason,
                "status_code": status_code,
                "rate_headers": rate_headers,
            }
        else:
            return f"Groq API hatası: {str(e)}", {
                "retryable": True, "model": model, "key_index": key_index, "reason": reason
            }

    except Exception as e:
        log.exception("Groq streaming error model=%s", model)
        return f"Groq API hatası: {str(e)}", {
            "retryable": True,
            "model": model,
            "key_index": key_index,
            "reason": "exception",
        }

    return full_response, None


def stream_groq(messages: List[Dict]) -> Generator[str, None, None]:
    """
    Streams tokens from Groq's OpenAI-compatible Chat Completions API.
    Tries the full model chain for one API key, then rotates to the next key
    and starts from the strongest model again.
    """
    api_keys = CONFIG["groq_api_keys"]
    model_chain = CONFIG["groq_model_chain"]
    last_error = "Groq API hatası."

    if not api_keys:
        yield f"data: {json.dumps({'error': 'GROQ_API_KEY ayarlı değil.'})}\n\n"
        return

    for key_index, api_key in enumerate(api_keys, 1):
        for model_index, model in enumerate(model_chain):
            response_text = ""
            failure_info = None

            attempt = stream_groq_model(messages, model, api_key, key_index)
            while True:
                try:
                    yield next(attempt)
                except StopIteration as stop:
                    if stop.value:
                        response_text, failure_info = stop.value
                    break

            if failure_info is None:
                if model_index > 0:
                    notice = {
                        "from_model": model_chain[0],
                        "to_model": model,
                        "message": "Yoğunluk nedeniyle model düşürüldü.",
                    }
                    yield f"data: {json.dumps({'model_fallback': notice})}\n\n"
                    log.warning(
                        "Groq fallback succeeded key_index=%s original_model=%s active_model=%s",
                        key_index,
                        model_chain[0],
                        model,
                    )
                yield f"data: {json.dumps({'__full_response__': strip_reasoning_blocks(response_text)})}\n\n"
                return

            last_error = response_text
            if not failure_info.get("retryable"):
                yield f"data: {json.dumps({'error': last_error})}\n\n"
                return

            if model_index < len(model_chain) - 1:
                next_model = model_chain[model_index + 1]
                log.warning(
                    "Groq fallback switching key_index=%s from_model=%s to_model=%s reason=%s",
                    key_index,
                    model,
                    next_model,
                    failure_info.get("reason"),
                )
                continue

            if key_index < len(api_keys):
                log.warning(
                    "Groq API key exhausted key_index=%s next_key_index=%s last_model=%s reason=%s",
                    key_index,
                    key_index + 1,
                    model,
                    failure_info.get("reason"),
                )
                break

    yield f"data: {json.dumps({'error': last_error})}\n\n"
    return


# ═══════════════════════════════════════════════════════════════════════════════
# 4. LLM Gateway
# ═══════════════════════════════════════════════════════════════════════════════

class LLMGateway:
    """
    Single backend-facing entry point for all model calls.
    """

    def __init__(self, config: Dict):
        self.config = config

    @property
    def active_provider(self) -> str:
        return self.config["provider"]

    def status(self) -> Dict:
        """Returns provider readiness for /api/health."""
        return {
            "provider": self.active_provider,
            "groq": bool(self.config["groq_api_keys"]),
            "groq_key_count": len(self.config["groq_api_keys"]),
            "model_name": self.config["groq_model_chain"][0],
            "model_chain": self.config["groq_model_chain"],
            "status": "ok" if self.config["groq_api_keys"] else "degraded",
        }

    def stream_chat(
        self,
        recent_history: List[Dict],
        augmented_message: str,
    ) -> Generator[str, None, None]:
        """
        Routes one chat turn to Groq.
        """
        messages = (
            [{"role": "system", "content": SYSTEM_PROMPT}]
            + recent_history
            + [{"role": "user", "content": augmented_message}]
        )
        yield from stream_groq(messages)


# ═══════════════════════════════════════════════════════════════════════════════
# 5. Flask Routes
# ═══════════════════════════════════════════════════════════════════════════════

@app.route("/")
def index():
    """Serves the frontend HTML file."""
    return send_from_directory(WEB_DIR, "index.html")


@app.route("/<path:filename>")
def serve_files(filename):
    return send_from_directory(WEB_DIR, filename)


@app.route("/api/auth/status", methods=["GET"])
def auth_status():
    identity = get_current_identity()
    if not identity:
        return jsonify({
            "authenticated": False,
            "google_configured": bool(CONFIG["google_client_id"]),
            "google_client_id": CONFIG["google_client_id"],
            "https_required": CONFIG["force_https"],
        })

    usage = quota_snapshot(identity)
    limits = CONFIG["limits"].get(identity["role"], CONFIG["limits"]["user"])
    return jsonify({
        "authenticated": True,
        "user": identity["public"],
        "role": identity["role"],
        "daily_used": usage["daily_used"],
        "minute_used": usage["minute_used"],
        "daily_limit": limits["daily"],
        "minute_limit": limits["minute"],
        "near_limit": usage["daily_used"] >= 30,
        "google_configured": bool(CONFIG["google_client_id"]),
        "google_client_id": CONFIG["google_client_id"],
        "https_required": CONFIG["force_https"],
    })


def unsupported_auth():
    return jsonify({"error": "Authentication flow is not supported for this app."}), 404


@app.route("/api/auth/guest", methods=["POST"])
def auth_guest():
    return unsupported_auth()


@app.route("/api/auth/register", methods=["POST"])
def auth_register():
    return unsupported_auth()


@app.route("/api/auth/login", methods=["POST"])
def auth_login():
    return unsupported_auth()


@app.route("/api/auth/google", methods=["POST"])
def auth_google():
    return unsupported_auth()


@app.route("/api/auth/logout", methods=["POST"])
def auth_logout():
    return unsupported_auth()


@app.route("/api/health", methods=["GET"])
def health():
    """
    Returns a JSON status object.
    """
    status = {
        "provider": CONFIG["provider"],
        "vectorstore": extensions.vector_store is not None,
        "embedding_model": CONFIG["embedding_model"],
        "database": database_ready(),
        "chunks": extensions.vector_store.index.ntotal if extensions.vector_store else 0,
    }

    if extensions.llm_gateway is None:
        status.update({"status": "degraded", "provider": None})
        return jsonify(status)

    provider_status = extensions.llm_gateway.status()
    status.update(provider_status)

    if not extensions.vector_store or not status["database"]:
        status["status"] = "degraded"

    return jsonify(status)


@app.route("/api/chat/feedback", methods=["POST"])
def chat_feedback():
    body = request.get_json()
    if not body or "question_index" not in body:
        return jsonify({"error": "question_index gerekli"}), 400

    identity = get_current_identity()
    if not identity:
        return jsonify({"error": "Kimlik alınamadı"}), 401

    question_index = body["question_index"]
    feedback = body.get("feedback")
    feedback_text = body.get("feedback_text", "").strip()

    if feedback is not None and feedback not in ("like", "dislike"):
        return jsonify({"error": "feedback sadece 'like' veya 'dislike' olabilir"}), 400

    user_id = int(identity["subject_id"])
    try:
        with extensions.SessionLocal() as db:
            log_entry = db.query(ChatLog).filter(
                ChatLog.user_id == user_id,
                ChatLog.question_index == question_index,
            ).first()
            if not log_entry:
                return jsonify({"error": "Soru bulunamadı"}), 404

            if feedback is not None:
                log_entry.feedback = feedback
            if feedback_text:
                log_entry.feedback = "feedback"
                log_entry.feedback_text = feedback_text
            db.commit()
        return jsonify({"ok": True})
    except Exception as e:
        log.exception("Feedback save failed for user_id=%s question_index=%s", user_id, question_index)
        return jsonify({"error": "Geri bildirim kaydedilemedi"}), 500


@app.route("/api/chat", methods=["POST"])
def chat():
    """
    Main chat endpoint — Server-Sent Events (SSE) streaming.
    """
    body = request.get_json()
    if not body or not body.get("message"):
        return jsonify({"error": "message alanı gerekli", "error_type": "technical"}), 400

    user_message = body["message"].strip()
    session_id = body.get("session_id", "default")

    if not user_message:
        return jsonify({"error": "Boş mesaj", "error_type": "technical"}), 400

    identity = get_current_identity()

    if not identity:
        fallback_fingerprint = (request.headers.get("X-Client-Fingerprint") or "").strip()
        if fallback_fingerprint and re.fullmatch(r"[A-Za-z0-9_-]{8,255}", fallback_fingerprint):
            identity = {
                "subject_type": "fingerprint_fallback",
                "subject_id": fallback_fingerprint,
                "role": "visitor",
                "public": {
                    "id": 0,
                    "email": None,
                    "role": "visitor",
                    "mode": "visitor_fallback",
                },
            }
            log.warning("Using fallback identity for fingerprint: %s", fallback_fingerprint[:20])
        else:
            try:
                headers_snapshot = {
                    "X-Client-Fingerprint": request.headers.get("X-Client-Fingerprint"),
                    "User-Agent": request.headers.get("User-Agent"),
                    "Accept-Language": request.headers.get("Accept-Language"),
                    "X-Forwarded-For": request.headers.get("X-Forwarded-For"),
                }
                log.warning("Visitor identity missing for /api/chat. Request cookies: %s, headers: %s, remote_addr: %s",
                            dict(request.cookies), headers_snapshot, request.remote_addr)
            except Exception:
                log.exception("Failed to log missing identity details")
            return jsonify({"error": "Ziyaretçi kimliği alınamadı; lütfen sayfayı yenileyin.", "error_type": "technical"}), 401

    log.info(
        "CHAT REQUEST user=%s session=%s msg=%s",
        identity["subject_id"] if identity else "unknown",
        session_id,
        user_message[:200]
    )

    quota_ok, quota, quota_error = check_quota(identity)
    if not quota_ok:
        return jsonify({"error": quota_error, "error_type": "quota"}), 429

    quota = increment_usage(identity)

    if session_id not in extensions.conversation_sessions:
        extensions.conversation_sessions[session_id] = []

    history = extensions.conversation_sessions[session_id]

    # ── RAG: retrieve ONCE with local embedding ──────────────────────────────
    try:
        retrieved = extensions.vector_store.retrieve(user_message, top_k=CONFIG["retrieval_top_k"])
    except RuntimeError as e:
        error_msg = str(e)
        log.error("Embedding/retrieval failed: %s", error_msg)
        return jsonify({"error": "Şu anda çok yoğunuz. Lütfen biraz sonra tekrar dene.", "error_type": "retry"}), 503
    except Exception as e:
        log.exception("Unexpected retrieval error")
        return jsonify({"error": "Şu anda çok yoğunuz. Lütfen biraz sonra tekrar dene.", "error_type": "retry"}), 503

    context = format_context(retrieved, CONFIG["retrieval_score_threshold"])
    augmented_message = build_augmented_user_message(user_message, context)

    recent_history = history[-(CONFIG["max_history_turns"] * 2):]

    CONGESTION_THRESHOLD = CONFIG["congestion_threshold"]

    def generate():
        """
        Inner generator that drives the SSE stream.
        Intercepts the __full_response__ marker to persist history,
        then emits the final 'done' event with source metadata.
        """
        full_response = ""
        had_error = False

        with extensions.active_requests_lock:
            extensions.active_requests += 1
            current_active = extensions.active_requests
            log.info("CONGESTION active_requests=%s threshold=%s", current_active, CONGESTION_THRESHOLD)

        try:
            # Send congestion warning if threshold met or exceeded
            if current_active >= CONGESTION_THRESHOLD:
                yield f"data: {json.dumps({'congestion': True, 'active_requests': current_active})}\n\n"

            token_stream = extensions.llm_gateway.stream_chat(recent_history, augmented_message)

            for event in token_stream:
                if "__full_response__" in event:
                    try:
                        payload = json.loads(event.replace("data: ", "").strip())
                        full_response = payload.get("__full_response__", "")
                    except Exception:
                        pass
                    continue

                if '"error"' in event:
                    had_error = True

                yield event

        except Exception:
            log.exception("Unexpected error during stream generation")
        finally:
            with extensions.active_requests_lock:
                extensions.active_requests -= 1
                log.info("CONGESTION active_requests decremented to %s", extensions.active_requests)

        # ── Persist history (only on success) ────────────────────────────────
        if full_response and not had_error:
            history.append({"role": "user", "content": user_message})
            history.append({"role": "assistant", "content": full_response})
            if len(history) > CONFIG["max_history_turns"] * 2:
                extensions.conversation_sessions[session_id] = history[-(CONFIG["max_history_turns"] * 2):]

            saved_question_index = None
            try:
                with extensions.SessionLocal() as db:
                    last_index = db.query(func.max(ChatLog.question_index)).filter(
                        ChatLog.user_id == int(identity["subject_id"])
                    ).scalar()
                    saved_question_index = (last_index or 0) + 1
                    log_entry = ChatLog(
                        user_id=int(identity["subject_id"]),
                        question_index=saved_question_index,
                        question=user_message,
                        answer=full_response,
                        created_at=utc_now().isoformat(),
                    )
                    db.add(log_entry)
                    db.commit()
            except Exception:
                log.exception("Failed to save chat log for user_id=%s", identity["subject_id"])

        # ── Final event: sources ──────────────────────────────────────────────
        sources = build_sources_payload(retrieved, CONFIG["retrieval_score_threshold"])
        done_payload = {
            'done': True,
            'sources': sources,
            'near_limit': quota['daily_used'] >= 30,
        }
        if saved_question_index:
            done_payload['question_index'] = saved_question_index
        yield f"data: {json.dumps(done_payload)}\n\n"

    return Response(
        stream_with_context(generate()),
        mimetype="text/event-stream",
        headers={
            "Cache-Control": "no-cache",
            "X-Accel-Buffering": "no",
        },
    )


# ═══════════════════════════════════════════════════════════════════════════════
# 6. Startup
# ═══════════════════════════════════════════════════════════════════════════════

def startup():
    """
    Runs once before the Flask server accepts requests.
    Loads the vector store and validates Groq configuration.
    Falls back to SQLite if PostgreSQL is unreachable.
    """
    LOG_DIR.mkdir(parents=True, exist_ok=True)

    # ── Try database connection; fall back to SQLite on failure ──────────────
    db_ok = False
    try:
        init_db()
        db_ok = True
        log.info("Database connection established: %s", CONFIG["database_url"][:50])
    except Exception as e:
        log.warning("Database connection failed (%s). Falling back to SQLite.", str(e)[:80])

    if not db_ok:
        sqlite_path = str(PROJECT_ROOT / "data" / "app.db")
        CONFIG["database_url"] = f"sqlite:///{sqlite_path}"
        log.info("Switching to SQLite: %s", CONFIG["database_url"])
        extensions.reinit_engine(CONFIG["database_url"])

        try:
            init_db()
            log.info("SQLite fallback successful.")
        except Exception as e2:
            log.error("SQLite fallback also failed: %s", e2)
            sys.exit(1)

    log.info("BAL Chatbot Web API starting...")
    log.info(f"Runtime pid={os.getpid()} cwd={Path.cwd()} log_file={LOG_DIR / 'web.log'}")
    log.info(f"Provider: {CONFIG['provider']}")
    log.info(f"Embedding model: {CONFIG['embedding_model']} (local, no API)")
    log.info(f"HTTPS enforcement: {CONFIG['force_https']}")
    if not CONFIG["secret_key"]:
        log.warning("FLASK_SECRET_KEY is not set. Sessions will reset after server restart.")

    # ── Load vector store (embedding model loads lazily on first request) ─────
    try:
        extensions.vector_store = VectorStore(
            CONFIG["faiss_index_file"],
            CONFIG["chunks_meta_file"],
            CONFIG["embedding_model"],
        )
    except FileNotFoundError as e:
        log.error(str(e))
        sys.exit(1)

    # ── Pre-load embedding model at startup on HF Space (2 vCPU) ──────────────
    # Loading ~500MB model takes ~5-10s on CPU; do it here so first request is fast
    log.info("Pre-loading local embedding model (this may take a moment)...")
    try:
        t0 = time.time()
        extensions.embedding_model = SentenceTransformer(CONFIG["embedding_model"])
        log.info(
            f"✓ Embedding model loaded in {time.time() - t0:.1f}s — "
            f"dim={extensions.embedding_model.get_sentence_embedding_dimension()}"
        )
    except Exception as e:
        log.error(f"Failed to load embedding model: {e}")
        sys.exit(1)

    # ── Groq configuration check ─────────────────────────────────────────────
    if not CONFIG["groq_api_keys"]:
        log.error(
            "No Groq API key is set. "
            "Set GROQ_API_KEY, GROQ_API_KEYS or GROQ_API_KEY_1..5 before starting the server."
        )
        sys.exit(1)
    log.info(
        "Groq configured — key_count=%s primary_model=%s",
        len(CONFIG["groq_api_keys"]),
        CONFIG["groq_model_chain"][0],
    )

    extensions.llm_gateway = LLMGateway(CONFIG)
    log.info(f"LLM gateway ready — active provider: {extensions.llm_gateway.active_provider}")
    log.info(f"HF Space tuning — congestion_threshold={CONFIG['congestion_threshold']}, "
             f"max_workers={CONFIG['embedding_max_workers']}")
    port = int(os.getenv("PORT", "5000"))
    scheme = "https" if CONFIG["local_https"] and not os.getenv("PORT") else "http"
    log.info(f"Server starting on {scheme}://0.0.0.0:{port}")


# ═══════════════════════════════════════════════════════════════════════════════
# Entry Point
# ═══════════════════════════════════════════════════════════════════════════════

if __name__ == "__main__":
    startup()
    port = int(os.getenv("PORT", "7860"))
    ssl_context = "adhoc" if CONFIG["local_https"] and not os.getenv("PORT") else None
    app.run(
        host="0.0.0.0",
        port=port,
        debug=False,
        threaded=True,
        ssl_context=ssl_context,
    )