main.py

"""
main.py — Iris AI Service (v1.1 - April 2026)

AI layer for the Iris Support Portal (IrisPlus / Unified Spark Desk).
Deployed as a HuggingFace Space monofile (Flask + Gemini + AssemblyAI + Firebase).

CHANGELOG v1.1:
  - Model: gemini-3.1-flash-lite-preview (multimodal reasoning)
  - /api/kb/whatsapp-import: now accepts multipart ZIP upload
    * Extracts _chat.txt + maps image files to <Media omitted> pointers
    * Sliding-window chunking (~10k tokens / ~40k chars with overlap)
    * Multimodal: sends images inline with their surrounding text chunk
    * Strict JSON enforcement + pre-save validation
    * JSON parse error recovery (regex extraction fallback)
  - All other endpoints unchanged from v1.0

FEATURES:
  1. WhatsApp Export → Knowledge Base (ZIP multimodal, chunked, additive)
  2. Bulk KB Upload (CSV / Excel / PDF)
  3. Natural Language + Voice Ticket Submission
  4. System Tutorial Ingestion (timestamped transcripts)
  5. Agent NL/Voice Solution Writing
  6. Iris Chatbot (KB RAG)

ENV VARS:
  GOOGLE_API_KEY       — Gemini API key
  ASSEMBLYAI_API_KEY   — AssemblyAI API key
  FIREBASE             — JSON string of Firebase service account
  GEMINI_MODEL         — Override model (default: gemini-3.1-flash-lite-preview)
  PORT                 — Server port (default 7860)
"""

import os
import io
import re
import json
import time
import uuid
import logging
import base64
import hashlib
import zipfile
import tempfile
import subprocess
import threading
from functools import wraps
from collections import OrderedDict
from datetime import datetime, timezone
from typing import Any, Dict, List, Optional, Tuple

import requests
from flask import Flask, request, jsonify
from flask_cors import CORS

# ─── Logging ──────────────────────────────────────────────────────────────────

logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s | %(levelname)s | %(message)s"
)
logger = logging.getLogger("iris-ai-service")

# ─── Gemini SDK ───────────────────────────────────────────────────────────────

try:
    from google import genai
    from google.genai import types as genai_types
except Exception as e:
    genai = None
    logger.error("google-genai not installed: %s", e)

GOOGLE_API_KEY = os.environ.get("GOOGLE_API_KEY", "")
# v1.1: upgraded to gemini-3.1-flash-lite-preview for multimodal reasoning
GEMINI_MODEL   = os.environ.get("GEMINI_MODEL", "gemini-3.1-flash-lite-preview")
# Hard timeout (ms) for the long video-extraction call so it can't hang forever.
GEMINI_VIDEO_TIMEOUT_MS = int(os.environ.get("GEMINI_VIDEO_TIMEOUT_MS", "300000"))

_gemini_client = None
if genai and GOOGLE_API_KEY:
    try:
        _gemini_client = genai.Client(api_key=GOOGLE_API_KEY)
        logger.info("Gemini client ready (model=%s).", GEMINI_MODEL)
    except Exception as e:
        logger.error("Failed to init Gemini client: %s", e)

# ─── AssemblyAI ───────────────────────────────────────────────────────────────

ASSEMBLYAI_API_KEY = os.environ.get("ASSEMBLYAI_API_KEY", "")
ASSEMBLYAI_BASE    = "https://api.assemblyai.com/v2"

# ─── Firebase ─────────────────────────────────────────────────────────────────

try:
    import firebase_admin
    from firebase_admin import credentials, firestore, auth as fb_auth, storage as fb_storage
    FIREBASE_AVAILABLE = True
except ImportError:
    FIREBASE_AVAILABLE = False
    logger.warning("firebase-admin not installed. Persistence disabled.")

FIREBASE_ENV            = os.environ.get("FIREBASE", "")
FIREBASE_STORAGE_BUCKET = os.environ.get("FIREBASE_STORAGE_BUCKET", "")

def init_firestore() -> Optional[Any]:
    if not FIREBASE_AVAILABLE:
        return None
    if not firebase_admin._apps:
        if not FIREBASE_ENV:
            logger.warning("FIREBASE env var missing. Persistence disabled.")
            return None
        try:
            sa_info = json.loads(FIREBASE_ENV)
            cred    = credentials.Certificate(sa_info)
            opts    = {"storageBucket": FIREBASE_STORAGE_BUCKET} if FIREBASE_STORAGE_BUCKET else None
            firebase_admin.initialize_app(cred, opts)
            logger.info("Firebase initialized (storage bucket=%s).", FIREBASE_STORAGE_BUCKET or "none")
        except Exception as e:
            logger.critical("Firebase init failed: %s", e)
            return None
    return firestore.client()

db = init_firestore()


def get_bucket() -> Optional[Any]:
    """Return the Firebase Storage bucket, or None if not configured."""
    if not (FIREBASE_AVAILABLE and firebase_admin._apps and FIREBASE_STORAGE_BUCKET):
        return None
    try:
        return fb_storage.bucket()
    except Exception as e:
        logger.error("Storage bucket unavailable: %s", e)
        return None


def _upload_to_storage(data: bytes, dest_path: str, content_type: str) -> str:
    """
    Upload bytes to Firebase Storage and return a publicly fetchable URL.

    Twilio fetches media_url over the public internet, so the URL must be reachable
    without auth. We try make_public() (works on buckets with fine-grained ACLs); if
    the bucket uses uniform bucket-level access (the modern default — object ACLs are
    disabled and make_public raises), we fall back to a long-lived v4 signed URL.
    Returns "" on failure.
    """
    bucket = get_bucket()
    if not bucket:
        logger.warning("Storage upload skipped — bucket not configured (dest=%s)", dest_path)
        return ""
    try:
        t0   = time.time()
        blob = bucket.blob(dest_path)
        blob.upload_from_string(data, content_type=content_type)
        logger.info("[storage] uploaded %s (%d bytes) in %.1fs", dest_path, len(data), time.time() - t0)
        try:
            blob.make_public()
            logger.info("[storage] %s public URL ready", dest_path)
            return blob.public_url
        except Exception as e:
            logger.info("[storage] make_public unavailable (uniform access?): %s — signing URL", e)
            signed = _storage_signed_url(dest_path, minutes=60 * 24 * 7)
            return signed or ""
    except Exception as e:
        logger.error("[storage] upload failed for %s: %s", dest_path, e)
        return ""


def _storage_signed_url(path: str, minutes: int = 120) -> str:
    """
    Generate a v4 signed URL for a stored object. Works regardless of uniform
    bucket-level access. Used at send-time for tutorial clips so the URL Twilio
    fetches is always valid even if the object is not publicly readable.
    """
    bucket = get_bucket()
    if not (bucket and path):
        return ""
    try:
        from datetime import timedelta
        blob = bucket.blob(path)
        return blob.generate_signed_url(expiration=timedelta(minutes=minutes), version="v4")
    except Exception as e:
        logger.error("Signed URL generation failed for %s: %s", path, e)
        return ""


# Maximum length (seconds) of a tutorial clip we will cut. Generous so full steps
# aren't truncated; stream-copied clips stay well under Twilio's ~16MB media limit.
MAX_CLIP_SECONDS = 300
# Padding added to the end of a clip so the demonstrated step is not cut off abruptly.
CLIP_TAIL_PAD    = 2


def _video_duration(path: str) -> float:
    """Return the video's duration in seconds via ffprobe, or 0.0 if unknown."""
    try:
        out = subprocess.run(
            ["ffprobe", "-v", "error", "-show_entries", "format=duration",
             "-of", "default=noprint_wrappers=1:nokey=1", path],
            capture_output=True, timeout=60,
        )
        return float(out.stdout.decode().strip())
    except Exception as e:
        logger.warning("[ffprobe] duration probe failed: %s", e)
        return 0.0


def _video_codec(path: str) -> str:
    """Return the video stream's codec name (e.g. h264, hevc) for diagnostics."""
    try:
        out = subprocess.run(
            ["ffprobe", "-v", "error", "-select_streams", "v:0",
             "-show_entries", "stream=codec_name",
             "-of", "default=noprint_wrappers=1:nokey=1", path],
            capture_output=True, timeout=60,
        )
        return out.stdout.decode().strip() or "unknown"
    except Exception as e:
        logger.warning("[ffprobe] codec probe failed: %s", e)
        return "unknown"


def _crop_video_segment(src_path: str, start: int, end: int, out_path: str,
                        video_duration: float = 0.0) -> bool:
    """
    Cut [start, end] seconds out of src_path into out_path using ffmpeg.

    We re-encode (not -c copy) so the cut is frame-accurate and the resulting clip
    is a self-contained, WhatsApp-playable MP4 (H.264/AAC + faststart). When the
    real duration is known, start/end are clamped to it so Gemini timestamps that
    overshoot the video don't make ffmpeg seek past EOF (which yields no frames).
    Returns True on success.
    """
    try:
        start = max(0, int(start))
        end   = int(end) if end and int(end) > start else start + 30
        end  += CLIP_TAIL_PAD
        if end - start > MAX_CLIP_SECONDS:
            end = start + MAX_CLIP_SECONDS
        if video_duration and video_duration > 0:
            if start >= video_duration:
                logger.warning("[ffmpeg] start %ds is beyond video duration %.1fs — skipping",
                               start, video_duration)
                return False
            end = min(end, int(video_duration))
        duration = end - start
        if duration <= 0:
            logger.warning("[ffmpeg] non-positive clip duration after clamping — skipping")
            return False

        logger.info("[ffmpeg] cropping %ds segment (%ds–%ds)...", duration, start, end)

        def _run(cmd: list, label: str) -> bool:
            t0   = time.time()
            proc = subprocess.run(cmd, capture_output=True, timeout=300)
            ok   = proc.returncode == 0 and os.path.exists(out_path) and os.path.getsize(out_path) > 0
            if ok:
                logger.info("[ffmpeg] %s done in %.1fs → ok (%d bytes)",
                            label, time.time() - t0, os.path.getsize(out_path))
            else:
                logger.warning("[ffmpeg] %s failed (%d): %s", label, proc.returncode,
                               proc.stderr.decode("utf-8", "replace")[-300:])
            return ok

        # Primary: stream copy — no decode, no libx264. Fast and avoids encoder/codec
        # dependency failures ("Could not open encoder before EOF"). Fast input seek
        # lands on the nearest keyframe, which is fine for tutorial clips.
        copy_cmd = [
            "ffmpeg", "-y", "-ss", str(start), "-i", src_path, "-t", str(duration),
            "-c", "copy", "-avoid_negative_ts", "make_zero",
            "-movflags", "+faststart", out_path,
        ]
        if _run(copy_cmd, "copy"):
            return True

        # Fallback: re-encode (covers cases where copy can't produce a clean MP4).
        reencode_cmd = [
            "ffmpeg", "-y", "-ss", str(start), "-i", src_path, "-t", str(duration),
            "-c:v", "libx264", "-preset", "ultrafast", "-pix_fmt", "yuv420p",
            "-c:a", "aac", "-movflags", "+faststart", out_path,
        ]
        return _run(reencode_cmd, "re-encode")
    except subprocess.TimeoutExpired:
        logger.error("[ffmpeg] crop timed out after 300s for segment %ds–%ds", start, end)
        return False
    except FileNotFoundError:
        logger.error("[ffmpeg] not installed — cannot crop video segments.")
        return False
    except Exception as e:
        logger.error("[ffmpeg] crop error: %s", e)
        return False


# ─── Twilio / WhatsApp client (optional — only when env configured) ─────────────

try:
    import whatsapp_client as wa
    WHATSAPP_AVAILABLE = bool(os.environ.get("TWILIO_ACCOUNT_SID"))
    if WHATSAPP_AVAILABLE:
        logger.info("WhatsApp (Twilio) client loaded.")
except Exception as e:
    wa = None
    WHATSAPP_AVAILABLE = False
    logger.warning("WhatsApp client unavailable: %s", e)


# ─── Optional libs ────────────────────────────────────────────────────────────

try:
    import pandas as pd
    PANDAS_AVAILABLE = True
except ImportError:
    PANDAS_AVAILABLE = False

try:
    import pypdf
    PYPDF_AVAILABLE = True
except ImportError:
    PYPDF_AVAILABLE = False

# ─── Flask App ────────────────────────────────────────────────────────────────

app = Flask(__name__)
CORS(app)


# ══════════════════════════════════════════════════════════════════════════════
# AUTH — Firebase ID token verification + role enforcement
# ══════════════════════════════════════════════════════════════════════════════
#
# The Lovable dashboard uses Firebase Auth ONLY to sign in and obtain an ID token.
# Every API call carries `Authorization: Bearer <idToken>`. Flask verifies it with
# firebase_admin and resolves the caller's role from custom claims (fallback:
# the iris_staff/{uid} profile doc). All data work happens server-side.

def _resolve_role(decoded: Dict, uid: str) -> str:
    """Resolve a user's role from token claims, falling back to the staff profile."""
    role = decoded.get("role")
    if role:
        return role
    if db:
        try:
            snap = db.collection("iris_staff").document(uid).get()
            if snap.exists:
                return snap.to_dict().get("role", "")
        except Exception as e:
            logger.error("Role lookup error: %s", e)
    return ""


def require_auth(roles: Optional[List[str]] = None):
    """
    Decorator: require a valid Firebase ID token. If `roles` is given, the caller's
    role must be one of them. On success, attaches `request.user` = {uid, email, role}.
    """
    def decorator(fn):
        @wraps(fn)
        def wrapper(*args, **kwargs):
            if not (FIREBASE_AVAILABLE and firebase_admin._apps):
                return jsonify({"ok": False, "error": "Auth backend unavailable"}), 503
            header = request.headers.get("Authorization", "")
            if not header.startswith("Bearer "):
                return jsonify({"ok": False, "error": "Missing bearer token"}), 401
            token = header.split(" ", 1)[1].strip()
            try:
                decoded = fb_auth.verify_id_token(token)
            except Exception as e:
                logger.warning("Token verification failed: %s", e)
                return jsonify({"ok": False, "error": "Invalid or expired token"}), 401
            uid  = decoded.get("uid")
            role = _resolve_role(decoded, uid)
            if roles and role not in roles:
                return jsonify({"ok": False, "error": "Insufficient permissions"}), 403
            request.user = {"uid": uid, "email": decoded.get("email", ""), "role": role}
            return fn(*args, **kwargs)
        return wrapper
    return decorator

# ══════════════════════════════════════════════════════════════════════════════
# SHARED HELPERS
# ══════════════════════════════════════════════════════════════════════════════

# Supported image extensions for multimodal WhatsApp ingestion
SUPPORTED_IMAGE_EXTS = {".jpg", ".jpeg", ".png", ".webp", ".gif"}

# Approx chars per token (conservative for mixed Shona/English/emoji content)
CHARS_PER_TOKEN = 4
# Target ~10k tokens per chunk with ~1k token overlap
CHUNK_CHARS    = 40_000
OVERLAP_CHARS  =  4_000


def _safe_json(text: str, fallback: Any) -> Any:
    """
    Multi-strategy JSON parser.
    1. Direct parse after stripping markdown fences.
    2. Regex extraction of first [...] or {...} block.
    3. Return fallback.
    """
    if not text:
        return fallback

    # Strategy 1: strip fences
    clean = text.strip()
    for fence in ("```json", "```JSON", "```"):
        if fence in clean:
            parts = clean.split(fence)
            # take the content between the first pair of fences
            if len(parts) >= 3:
                clean = parts[1].strip()
            elif len(parts) == 2:
                clean = parts[1].split("```")[0].strip()
            break

    try:
        return json.loads(clean)
    except json.JSONDecodeError:
        pass

    # Strategy 2: regex — find outermost [...] array
    arr_match = re.search(r'\[[\s\S]*\]', clean)
    if arr_match:
        try:
            return json.loads(arr_match.group())
        except json.JSONDecodeError:
            pass

    # Strategy 3: regex — find outermost {...} object
    obj_match = re.search(r'\{[\s\S]*\}', clean)
    if obj_match:
        try:
            return json.loads(obj_match.group())
        except json.JSONDecodeError:
            pass

    logger.error("JSON parse exhausted all strategies. First 300 chars: %s", text[:300])
    return fallback


def _validate_articles(data: Any) -> List[Dict]:
    """
    Validate that extracted articles are a list of dicts with required fields.
    Filters out malformed items rather than failing the whole batch.
    """
    if not isinstance(data, list):
        logger.warning("Expected list from Gemini, got %s", type(data))
        return []
    valid = []
    for item in data:
        if not isinstance(item, dict):
            continue
        title   = str(item.get("title", "")).strip()
        content = str(item.get("content", "")).strip()
        if len(title) < 3 or len(content) < 10:
            continue
        entry = {
            "title":    title,
            "content":  content,
            "category": str(item.get("category", "General")).strip() or "General",
            "tags":     item.get("tags", []) if isinstance(item.get("tags"), list) else [],
        }
        # Preserve tutorial timestamps when present so clip cropping can use them.
        # (WhatsApp/PDF sources simply don't set these.)
        for ts_key in ("timestamp_start", "timestamp_end"):
            if item.get(ts_key) is not None:
                entry[ts_key] = item[ts_key]
        valid.append(entry)
    return valid


def _gemini_text(prompt: str, json_mode: bool = False) -> str:
    """Call Gemini with text-only content."""
    if not _gemini_client:
        return ""
    cfg = genai_types.GenerateContentConfig(
        response_mime_type="application/json"
    ) if json_mode else None
    try:
        resp = _gemini_client.models.generate_content(
            model=GEMINI_MODEL,
            contents=prompt,
            config=cfg
        )
        return resp.text or ""
    except Exception as e:
        logger.error("Gemini text call error: %s", e)
        return ""


def _gemini_multimodal(parts: list, json_mode: bool = False) -> str:
    """Call Gemini with a mixed list of text strings and image Parts."""
    if not _gemini_client:
        return ""
    cfg = genai_types.GenerateContentConfig(
        response_mime_type="application/json"
    ) if json_mode else None
    try:
        resp = _gemini_client.models.generate_content(
            model=GEMINI_MODEL,
            contents=parts,
            config=cfg
        )
        return resp.text or ""
    except Exception as e:
        logger.error("Gemini multimodal call error: %s", e)
        return ""


def _article_fingerprint(title: str, content: str) -> str:
    raw = f"{title.strip().lower()}::{content.strip().lower()[:300]}"
    return hashlib.sha256(raw.encode()).hexdigest()[:16]


def _get_existing_fingerprints() -> set:
    if not db:
        return set()
    try:
        docs = db.collection("iris_kb_articles").select(["fingerprint"]).stream()
        return {d.to_dict().get("fingerprint") for d in docs if d.to_dict().get("fingerprint")}
    except Exception as e:
        logger.error("Fingerprint fetch error: %s", e)
        return set()


def _save_kb_articles(articles: List[Dict], source_label: str) -> Dict:
    if not db:
        return {"saved": 0, "skipped": 0, "error": "Firebase unavailable"}
    existing = _get_existing_fingerprints()
    saved, skipped = 0, 0
    for article in articles:
        title   = article.get("title", "Untitled")
        content = article.get("content", "")
        fp      = _article_fingerprint(title, content)
        if fp in existing:
            skipped += 1
            continue
        doc = {
            "title":       title,
            "content":     content,
            "category":    article.get("category", "General"),
            "tags":        article.get("tags", []),
            "source":      source_label,
            "fingerprint": fp,
            "created_at":  datetime.now(timezone.utc).isoformat(),
        }
        if article.get("timestamp_start") is not None:
            doc["timestamp_start"] = article["timestamp_start"]
            doc["timestamp_end"]   = article.get("timestamp_end")
            doc["video_url"]       = article.get("video_url", "")
        # Multimodal pass-through: cropped tutorial clip + source image
        for extra_key in ("clip_url", "clip_path", "clip_start", "clip_end", "video_title", "image_url"):
            if article.get(extra_key) not in (None, ""):
                doc[extra_key] = article[extra_key]
        db.collection("iris_kb_articles").add(doc)
        existing.add(fp)
        saved += 1
    return {"saved": saved, "skipped": skipped}


# ══════════════════════════════════════════════════════════════════════════════
# WHATSAPP ZIP PROCESSOR
# ══════════════════════════════════════════════════════════════════════════════

# Regex to match WhatsApp timestamp lines
# Handles both: DD/MM/YYYY, HH:MM - Sender: message
# and:          DD/MM/YYYY, HH:MM am/pm - Sender: message
WA_LINE_RE = re.compile(
    r'^\d{1,2}/\d{1,2}/\d{4},\s+\d{1,2}:\d{2}(?:\s*[ap]m)?\s+-\s+',
    re.IGNORECASE
)

# Matches <Media omitted> or [filename.jpg] style media pointers
MEDIA_POINTER_RE = re.compile(
    r'<Media omitted>|\[?([^\]]+\.(?:jpg|jpeg|png|webp|gif|mp4|opus|aac|m4a))\]?',
    re.IGNORECASE
)


class WhatsAppZipProcessor:
    """
    Handles extraction and multimodal chunking of a WhatsApp .zip export.

    A WhatsApp export zip typically contains:
      _chat.txt          — the full conversation
      IMG-YYYYMMDD-*.jpg — attached images
      VID-*.mp4          — videos (we skip these, too large)
      PTT-*.opus         — voice notes (skipped)
    """

    def __init__(self, zip_bytes: bytes):
        self.zip_bytes   = zip_bytes
        self.chat_text   = ""
        self.media_map: Dict[str, bytes] = {}  # filename -> raw bytes

    def extract(self) -> bool:
        """Extract chat text and image files from ZIP. Returns True on success."""
        try:
            with zipfile.ZipFile(io.BytesIO(self.zip_bytes)) as zf:
                names = zf.namelist()
                logger.info("ZIP contains %d files: %s", len(names), names[:20])

                # Find chat file — WhatsApp names it _chat.txt or WhatsApp Chat with *.txt
                chat_file = None
                for name in names:
                    base = os.path.basename(name).lower()
                    if base == "_chat.txt" or (base.endswith(".txt") and "chat" in base):
                        chat_file = name
                        break
                if not chat_file:
                    # Fallback: any .txt file
                    txts = [n for n in names if n.lower().endswith(".txt")]
                    if txts:
                        chat_file = txts[0]

                if not chat_file:
                    logger.error("No chat .txt found in ZIP")
                    return False

                raw = zf.read(chat_file)
                self.chat_text = raw.decode("utf-8", errors="replace")
                logger.info("Chat text extracted: %d chars from %s", len(self.chat_text), chat_file)

                # Extract images (skip videos and audio — too large / not useful for KB)
                for name in names:
                    ext = os.path.splitext(name.lower())[1]
                    if ext in SUPPORTED_IMAGE_EXTS:
                        try:
                            self.media_map[os.path.basename(name)] = zf.read(name)
                        except Exception as e:
                            logger.warning("Could not read media file %s: %s", name, e)

                logger.info("Media files extracted: %d images", len(self.media_map))
                return True

        except zipfile.BadZipFile as e:
            logger.error("Bad ZIP file: %s", e)
            return False
        except Exception as e:
            logger.error("ZIP extraction error: %s", e)
            return False

    def _resolve_media_in_line(self, line: str) -> Optional[bytes]:
        """
        Given a chat line, check if it references a media file we have.
        Returns the image bytes if found, else None.
        """
        match = MEDIA_POINTER_RE.search(line)
        if not match:
            return None
        filename = match.group(1)  # group 1 = explicit filename, None for <Media omitted>
        if filename:
            fname = os.path.basename(filename)
            if fname in self.media_map:
                return self.media_map[fname]
        # <Media omitted> — we can't recover the file since it wasn't exported
        return None

    def build_chunks(self) -> List[Dict]:
        """
        Split chat text into overlapping chunks, each annotated with
        the image bytes found within that chunk.

        Returns list of:
          { "text": str, "images": [bytes, ...], "line_range": (start, end) }
        """
        lines  = self.chat_text.splitlines()
        chunks = []

        i        = 0
        total    = len(lines)
        char_count = 0
        chunk_lines: List[str] = []
        chunk_images: List[bytes] = []

        while i < total:
            line = lines[i]
            chunk_lines.append(line)
            char_count += len(line) + 1  # +1 for newline

            # Check if this line has an image we can include
            img_bytes = self._resolve_media_in_line(line)
            if img_bytes and len(chunk_images) < 5:  # cap images per chunk
                chunk_images.append(img_bytes)

            if char_count >= CHUNK_CHARS or i == total - 1:
                chunks.append({
                    "text":   "\n".join(chunk_lines),
                    "images": chunk_images[:],
                    "line_range": (i - len(chunk_lines) + 1, i)
                })
                logger.info(
                    "Chunk %d: %d lines, %d chars, %d images",
                    len(chunks), len(chunk_lines), char_count, len(chunk_images)
                )
                # Overlap: keep last OVERLAP_CHARS worth of lines for next chunk
                overlap_text = 0
                overlap_start = len(chunk_lines) - 1
                while overlap_start > 0 and overlap_text < OVERLAP_CHARS:
                    overlap_text += len(chunk_lines[overlap_start]) + 1
                    overlap_start -= 1
                chunk_lines  = chunk_lines[overlap_start:]
                chunk_images = []
                char_count   = sum(len(l) + 1 for l in chunk_lines)
            i += 1

        logger.info("Total chunks: %d", len(chunks))
        return chunks


# ══════════════════════════════════════════════════════════════════════════════
# WHATSAPP EXTRACTION PROMPT
# ══════════════════════════════════════════════════════════════════════════════

WHATSAPP_EXTRACTION_PROMPT = """You are a support knowledge base curator for the Iris platform, deployed across Zimbabwe.

Your task: analyse this WhatsApp support group chat segment and extract ONLY clear problem→solution pairs.

CONTEXT ABOUT THIS PLATFORM:
- "Iris" is an integrated POS (Point of Sale) and fiscalisation system with a mobile attendance and
  location-tracking module used by field sales reps and in-store tellers at retail stores.
- The POS and fiscalisation layer handles sales transactions, receipt generation, and ZIMRA fiscal
  compliance. The mobile module handles teller clock-in/out, GPS location verification, and hours tracking.
- Common POS/fiscal issues: fiscalisation failures, receipt errors, device not syncing to ZIMRA servers,
  Elixir (fiscal device software) login/password problems.
- Common mobile attendance issues: GPS location not detected, clock-in failures, app killed by Android
  battery optimiser, teller passkey problems, hours recording incorrectly, store radius too small,
  wrong teller name shown after login, app not running in the background.
- Messages mix English, Shona, and Ndebele. Understand regional vernacular (e.g. "irikudzima" = switching
  off, "ndakashanda" = I worked, "short yemahours" = hours shortage, "gadzirisayi" = fix it, "hupfu" = flour,
  "yakuda kulogwa patsva" = needs to be logged in fresh).
- If screenshots show Android error dialogs (e.g. "Service killed by system", "App stopped", "Abrupt stop"),
  reason through what that means for Android background restriction and background service killing, and include
  that diagnosis and fix in the solution content.
- If screenshots show fiscal device or POS screens, extract the error code or state shown and reason through
  the likely cause from the Elixir/ZIMRA integration context.

STRICT RULES:
1. Extract ONLY exchanges where a user described a problem AND a named support person (Tendayi, Tony, Violet,
   Rufaro, Albrighton, Ishmael, or any named responder) provided a working solution or clear instruction.
2. Ignore: greetings, media-only messages, deleted messages, clock-in screenshots with no text context,
   messages from unknown numbers with no solution attached.
3. Each article must be self-contained and usable by a support agent in future.
4. Translate all Shona/Ndebele problem descriptions to English in the article content.
5. If a screenshot appears to show an Android error or GPS issue, reason through the likely cause and
   include that reasoning in the solution content.

OUTPUT FORMAT: Return ONLY a valid JSON array. No preamble, no explanation, no markdown fences.
Every string value MUST be properly JSON-escaped. Do not use unescaped newlines, tabs, or quotes inside strings.
Use \\n for line breaks within content strings.

Schema per item:
{"title": "string (max 80 chars)", "content": "string (escaped, solution steps)", "category": "one of: Account|Technical|Location|Attendance|Device|Other", "tags": ["array", "of", "strings"]}

If no valid problem→solution pairs exist in this segment, return an empty array: []

Chat segment:
"""


def _process_chunk_with_gemini(chunk: Dict) -> List[Dict]:
    """
    Send a single chunk (text + optional images) to Gemini.
    Returns validated list of article dicts.
    """
    text_part  = WHATSAPP_EXTRACTION_PROMPT + chunk["text"]
    images     = chunk.get("images", [])

    if images and _gemini_client:
        # Build multimodal content list
        parts = [text_part]
        for img_bytes in images:
            # Detect mime type from magic bytes
            mime = "image/jpeg"
            if img_bytes[:4] == b'\x89PNG':
                mime = "image/png"
            elif img_bytes[:4] == b'RIFF':
                mime = "image/webp"
            parts.append(
                genai_types.Part.from_bytes(data=img_bytes, mime_type=mime)
            )
        raw = _gemini_multimodal(parts, json_mode=True)
    else:
        raw = _gemini_text(text_part, json_mode=True)

    if not raw:
        logger.warning("Empty Gemini response for chunk")
        return []

    parsed = _safe_json(raw, [])
    return _validate_articles(parsed)


# ══════════════════════════════════════════════════════════════════════════════
# FEATURE 1 — WhatsApp Export → Knowledge Base (v1.1: ZIP multimodal + chunked)
# ══════════════════════════════════════════════════════════════════════════════

@app.post("/api/kb/whatsapp-import")
@require_auth(roles=["admin", "support"])
def whatsapp_import():
    """
    Accepts EITHER:
      (a) multipart file upload with field "file" containing a .zip WhatsApp export, OR
      (b) JSON body { "chat_text": "..." } for plain text (legacy support)

    Processes in sliding-window chunks, sends images to Gemini multimodally.
    Saves new articles only (additive, dedup by fingerprint).
    """
    all_articles: List[Dict] = []
    source_label = "whatsapp_export"

    # ── Branch A: ZIP upload ──────────────────────────────────────────────────
    if "file" in request.files:
        f        = request.files["file"]
        filename = f.filename or ""
        if not filename.lower().endswith(".zip"):
            return jsonify({"ok": False, "error": "Expected a .zip WhatsApp export file"}), 400

        zip_bytes = f.read()
        logger.info("WhatsApp ZIP upload: %d bytes, filename=%s", len(zip_bytes), filename)

        processor = WhatsAppZipProcessor(zip_bytes)
        if not processor.extract():
            return jsonify({"ok": False, "error": "Could not extract chat from ZIP. Ensure it is a valid WhatsApp export."}), 400

        if len(processor.chat_text) < 100:
            return jsonify({"ok": False, "error": "Extracted chat text too short to process"}), 400

        chunks = processor.build_chunks()
        source_label = f"whatsapp_zip:{filename}"

        for idx, chunk in enumerate(chunks):
            logger.info("Processing chunk %d/%d", idx + 1, len(chunks))
            articles = _process_chunk_with_gemini(chunk)
            all_articles.extend(articles)
            logger.info("Chunk %d yielded %d articles (running total: %d)", idx + 1, len(articles), len(all_articles))

    # ── Branch B: Legacy plain text JSON body ─────────────────────────────────
    else:
        body     = request.get_json(silent=True) or {}
        raw_chat = body.get("chat_text", "").strip()
        if not raw_chat:
            return jsonify({"ok": False, "error": "Provide a .zip file upload or chat_text in JSON body"}), 400
        if len(raw_chat) < 100:
            return jsonify({"ok": False, "error": "Chat text too short to process"}), 400

        logger.info("WhatsApp plain text import: %d chars", len(raw_chat))

        # Chunk the plain text too (handles large exports)
        lines  = raw_chat.splitlines()
        pseudo_zip = type("PseudoZip", (), {
            "chat_text": raw_chat,
            "media_map": {}
        })()
        processor = WhatsAppZipProcessor(b"")
        processor.chat_text = raw_chat
        processor.media_map = {}
        chunks = processor.build_chunks()

        for idx, chunk in enumerate(chunks):
            logger.info("Processing text chunk %d/%d", idx + 1, len(chunks))
            articles = _process_chunk_with_gemini(chunk)
            all_articles.extend(articles)

    if not all_articles:
        logger.info("No articles extracted from this export")
        return jsonify({
            "ok": True,
            "articles_found": 0,
            "saved": 0,
            "skipped_dupes": 0,
            "note": "No clear problem→solution pairs found in this chat segment"
        })

    stats = _save_kb_articles(all_articles, source_label=source_label)
    logger.info("WhatsApp import complete: found=%d, %s", len(all_articles), stats)

    return jsonify({
        "ok":             True,
        "articles_found": len(all_articles),
        "articles":       all_articles,   # full list — frontend INSERTs to Supabase kb_articles
        "saved":          stats["saved"],
        "skipped_dupes":  stats["skipped"],
    })


# ══════════════════════════════════════════════════════════════════════════════
# FEATURE 2 — Bulk KB Upload (CSV / Excel / PDF)
# ══════════════════════════════════════════════════════════════════════════════

def _extract_text_from_pdf_bytes(pdf_bytes: bytes) -> str:
    if PYPDF_AVAILABLE:
        try:
            reader = pypdf.PdfReader(io.BytesIO(pdf_bytes))
            pages  = [p.extract_text() or "" for p in reader.pages]
            text   = "\n\n".join(pages).strip()
            if text:
                return text
        except Exception as e:
            logger.warning("pypdf extraction failed: %s", e)
    if _gemini_client:
        try:
            resp = _gemini_client.models.generate_content(
                model=GEMINI_MODEL,
                contents=[
                    "Extract all text from this PDF document. Return plain text only.",
                    genai_types.Part.from_bytes(data=pdf_bytes, mime_type="application/pdf")
                ]
            )
            return resp.text or ""
        except Exception as e:
            logger.error("Gemini PDF extraction failed: %s", e)
    return ""


PDF_KB_PROMPT = """You are a support knowledge base curator.
Convert the following document content into structured KB articles.
Each article covers one distinct topic, issue, or procedure.

Return ONLY a valid JSON array — no preamble, no markdown fences.
All string values must be properly JSON-escaped (no raw newlines inside strings, use \\n).

Schema per item:
{"title": "string", "content": "string", "category": "one of: Account|Billing|Technical|Feature|Other", "tags": ["string"]}

Document content:
"""

@app.post("/api/kb/bulk-upload")
@require_auth(roles=["admin", "support"])
def bulk_upload():
    if "file" not in request.files:
        return jsonify({"ok": False, "error": "No file uploaded"}), 400
    f         = request.files["file"]
    filename  = f.filename or ""
    ext       = filename.rsplit(".", 1)[-1].lower()
    file_data = f.read()
    articles  = []

    if ext in ("csv", "xlsx", "xls"):
        if not PANDAS_AVAILABLE:
            return jsonify({"ok": False, "error": "pandas not installed on server"}), 500
        try:
            df = pd.read_csv(io.BytesIO(file_data)) if ext == "csv" else pd.read_excel(io.BytesIO(file_data))
            df.columns = [c.strip().lower() for c in df.columns]
            if "title" not in df.columns or "content" not in df.columns:
                return jsonify({"ok": False, "error": "CSV/Excel must have 'title' and 'content' columns"}), 400
            for _, row in df.iterrows():
                tags = []
                if "tags" in df.columns and pd.notna(row.get("tags")):
                    tags = [t.strip() for t in re.split(r"[,;|]", str(row["tags"])) if t.strip()]
                articles.append({
                    "title":    str(row["title"]).strip(),
                    "content":  str(row["content"]).strip(),
                    "category": str(row.get("category", "General")).strip() if pd.notna(row.get("category")) else "General",
                    "tags":     tags,
                })
        except Exception as e:
            return jsonify({"ok": False, "error": f"Could not parse file: {e}"}), 400

    elif ext == "pdf":
        text = _extract_text_from_pdf_bytes(file_data)
        if not text:
            return jsonify({"ok": False, "error": "Could not extract text from PDF"}), 400
        raw     = _gemini_text(PDF_KB_PROMPT + text[:50000], json_mode=True)
        parsed  = _safe_json(raw, [])
        articles = _validate_articles(parsed)
        if not articles:
            return jsonify({"ok": False, "error": "Gemini PDF structuring returned no valid articles"}), 500
    else:
        return jsonify({"ok": False, "error": f"Unsupported file type .{ext}. Use csv, xlsx, or pdf"}), 400

    if not articles:
        return jsonify({"ok": False, "error": "No articles extracted from file"}), 400

    stats = _save_kb_articles(articles, source_label=f"bulk_upload:{filename}")
    return jsonify({"ok": True, "articles_found": len(articles), "articles": articles,  # full list — frontend INSERTs to Supabase kb_articles
                    "saved": stats["saved"], "skipped_dupes": stats["skipped"]})


# ══════════════════════════════════════════════════════════════════════════════
# FEATURE 3 — Ticket Submission via NL Text or Voice
# ══════════════════════════════════════════════════════════════════════════════

TICKET_EXTRACTION_PROMPT = """You are a support ticket intake system for a software support portal.

A user has described their issue in natural language. Extract structured ticket fields.

Return ONLY a valid JSON object — no preamble, no markdown fences.
All string values must be properly JSON-escaped.

Schema:
{"title": "string (max 80 chars)", "description": "string (full clear description)", "category_hint": "one of: Account|Billing|Technical|Feature|Other", "priority_hint": "one of: low|medium|high|critical", "keywords": ["string"]}

User message:
"""

def _transcribe_audio_assemblyai(audio_b64: str, audio_format: str = "wav") -> str:
    if not ASSEMBLYAI_API_KEY:
        return ""
    audio_bytes = base64.b64decode(audio_b64)
    headers     = {"authorization": ASSEMBLYAI_API_KEY}
    try:
        upload_resp = requests.post(
            f"{ASSEMBLYAI_BASE}/upload",
            headers={**headers, "Content-Type": "application/octet-stream"},
            data=audio_bytes, timeout=30
        )
        upload_resp.raise_for_status()
        upload_url = upload_resp.json().get("upload_url")
    except Exception as e:
        logger.error("AssemblyAI upload error: %s", e)
        return ""
    try:
        tx_resp = requests.post(
            f"{ASSEMBLYAI_BASE}/transcript",
            headers={**headers, "Content-Type": "application/json"},
            json={"audio_url": upload_url, "language_detection": True}, timeout=15
        )
        tx_resp.raise_for_status()
        tx_id = tx_resp.json().get("id")
    except Exception as e:
        logger.error("AssemblyAI transcript request error: %s", e)
        return ""
    for _ in range(30):
        time.sleep(3)
        try:
            poll   = requests.get(f"{ASSEMBLYAI_BASE}/transcript/{tx_id}", headers=headers, timeout=15)
            poll.raise_for_status()
            result = poll.json()
            status = result.get("status")
            if status == "completed":
                return result.get("text", "")
            elif status == "error":
                logger.error("AssemblyAI error: %s", result.get("error"))
                return ""
        except Exception as e:
            logger.error("AssemblyAI poll error: %s", e)
    return ""


def _transcribe_audio_bytes(audio_bytes: bytes) -> str:
    """Transcribe raw audio bytes (e.g. a downloaded WhatsApp voice note)."""
    if not audio_bytes:
        return ""
    return _transcribe_audio_assemblyai(base64.b64encode(audio_bytes).decode(), "ogg")


@app.post("/api/tickets/submit-nl")
def submit_ticket_nl():
    body    = request.get_json(silent=True) or {}
    message = body.get("message", "").strip()
    user_id = body.get("user_id", "anonymous")
    if not message:
        return jsonify({"ok": False, "error": "message is required"}), 400
    raw    = _gemini_text(TICKET_EXTRACTION_PROMPT + message, json_mode=True)
    ticket = _safe_json(raw, {})
    if not isinstance(ticket, dict) or not ticket.get("title"):
        return jsonify({"ok": False, "error": "Could not extract ticket info from message"}), 500
    if db:
        db.collection("iris_ai_ticket_drafts").add({
            "user_id": user_id, "raw_input": message,
            "extracted": ticket, "channel": "nl_text",
            "created_at": datetime.now(timezone.utc).isoformat(),
        })
    return jsonify({"ok": True, "ticket": ticket})


@app.post("/api/tickets/submit-voice")
def submit_ticket_voice():
    body         = request.get_json(silent=True) or {}
    audio_b64    = body.get("audio_b64", "")
    audio_format = body.get("audio_format", "wav")
    user_id      = body.get("user_id", "anonymous")
    if not audio_b64:
        return jsonify({"ok": False, "error": "audio_b64 is required"}), 400
    if not ASSEMBLYAI_API_KEY:
        return jsonify({"ok": False, "error": "AssemblyAI not configured on server"}), 500
    transcript = _transcribe_audio_assemblyai(audio_b64, audio_format)
    if not transcript:
        return jsonify({"ok": False, "error": "Transcription failed or returned empty result"}), 500
    raw    = _gemini_text(TICKET_EXTRACTION_PROMPT + transcript, json_mode=True)
    ticket = _safe_json(raw, {})
    if not isinstance(ticket, dict) or not ticket.get("title"):
        return jsonify({"ok": False, "error": "Could not extract ticket info from transcript"}), 500
    if db:
        db.collection("iris_ai_ticket_drafts").add({
            "user_id": user_id, "raw_input": transcript,
            "extracted": ticket, "channel": "voice",
            "created_at": datetime.now(timezone.utc).isoformat(),
        })
    return jsonify({"ok": True, "transcript": transcript, "ticket": ticket})


# ══════════════════════════════════════════════════════════════════════════════
# FEATURE 4 — System Tutorial Ingestion
# ══════════════════════════════════════════════════════════════════════════════

TUTORIAL_VIDEO_PROMPT = """You are a knowledge base curator watching a tutorial video about the Iris platform.

CONTEXT ABOUT IRIS:
- Iris is an integrated POS (Point of Sale) and fiscalisation system with a mobile attendance and
  location-tracking module used by tellers and field reps at retail stores in Zimbabwe.
- The POS/fiscal layer handles sales, receipts, and ZIMRA fiscal compliance (Elixir device).
- The mobile module handles teller clock-in/out, GPS location, store radius, and hours tracking.
- The Iris Support Portal is a customer support desk used by admin staff, agents, and support tiers
  to manage tickets, agents, customers, and the knowledge base.

YOUR TASK:
Watch this tutorial video in full. For every distinct feature, workflow, or task you observe being
demonstrated, extract one self-contained KB article. Identify the exact timestamp range in the video
where each demonstration occurs so users can jump directly to the relevant moment.

Be precise about timestamps — state the second at which the demonstration starts and ends.
Write step-by-step instructions based on what you see happening on screen, not generic descriptions.
If the presenter speaks, incorporate their narration into the steps.

Return ONLY a valid JSON array. No preamble, no markdown fences. All strings properly JSON-escaped.
Use \n for line breaks within content strings.

Schema per item:
{
  "title": "string — concise how-to title, max 80 chars",
  "content": "string — numbered step-by-step instructions based on what is shown",
  "category": "one of: Account|Tickets|Agents|Reports|Admin|POS|Attendance|Other",
  "tags": ["string"],
  "timestamp_start": <integer — seconds from video start where this demo begins>,
  "timestamp_end": <integer — seconds from video start where this demo ends>
}

If the video contains no discernible how-to demonstrations, return an empty array: []
"""


def _upload_video_to_gemini(video_bytes: bytes, mime_type: str, display_name: str) -> Optional[Any]:
    """
    Upload a video to the Gemini Files API and poll until processing is ACTIVE.
    Returns the uploaded file object (with .uri and .name) or None on failure.

    Gemini Files API processes video at 1 FPS, adding timestamps every second.
    Files are retained for 48 hours. We delete after use to be tidy.
    """
    if not _gemini_client:
        return None

    try:
        # Write bytes to a named temp file — Files API needs a file path or IO object
        with tempfile.NamedTemporaryFile(suffix=f".{mime_type.split('/')[-1]}", delete=False) as tmp:
            tmp.write(video_bytes)
            tmp_path = tmp.name

        logger.info("[gemini-files] uploading %s (%.2f MB)...", display_name, len(video_bytes) / 1024 / 1024)
        t_up     = time.time()
        uploaded = _gemini_client.files.upload(
            file=tmp_path,
            config={"mime_type": mime_type, "display_name": display_name}
        )
        os.unlink(tmp_path)
        logger.info("[gemini-files] upload complete in %.1fs (%s) — polling for ACTIVE...",
                    time.time() - t_up, uploaded.name)

    except Exception as e:
        logger.error("[gemini-files] upload error: %s", e)
        return None

    # Poll until state is ACTIVE (video processing complete) — max ~3 minutes
    for attempt in range(36):
        time.sleep(5)
        try:
            file_info = _gemini_client.files.get(name=uploaded.name)
            state = getattr(file_info, "state", None)
            state_str = str(state).upper() if state else ""
            logger.info("Poll %d: file state = %s", attempt + 1, state_str)
            if "ACTIVE" in state_str:
                logger.info("Video ACTIVE after %d polls (~%ds)", attempt + 1, (attempt + 1) * 5)
                return file_info
            elif "FAILED" in state_str:
                logger.error("Gemini Files API processing failed for %s", uploaded.name)
                return None
        except Exception as e:
            logger.warning("Poll error: %s", e)

    logger.error("Video did not reach ACTIVE state within timeout")
    return None


def _delete_gemini_file(file_obj: Any) -> None:
    """Best-effort cleanup of a file from the Gemini Files API."""
    try:
        _gemini_client.files.delete(name=file_obj.name)
        logger.info("Deleted Gemini file: %s", file_obj.name)
    except Exception as e:
        logger.warning("Could not delete Gemini file %s: %s", file_obj.name, e)


# Supported video MIME types for tutorial upload
SUPPORTED_VIDEO_MIMES = {
    ".mp4":  "video/mp4",
    ".mov":  "video/quicktime",
    ".avi":  "video/x-msvideo",
    ".webm": "video/webm",
    ".mkv":  "video/x-matroska",
    ".3gp":  "video/3gpp",
    ".flv":  "video/x-flv",
}


@app.post("/api/kb/tutorial-ingest")
@require_auth(roles=["admin", "support"])
def tutorial_ingest():
    """
    Accepts a tutorial video file upload (multipart, field name "file").
    Gemini watches the full video, self-generates timestamps, and extracts
    one KB article per distinct feature or task demonstrated.

    No transcript required — Gemini reasons directly from video + audio.

    Supported: mp4, mov, avi, webm, mkv, 3gp, flv
    Max practical size: ~500MB (Files API limit is 2GB, but HF Space upload limit applies)

    Returns articles with timestamp_start/end in seconds so the frontend
    can generate deep-links into the video.
    """
    if "file" not in request.files:
        return jsonify({"ok": False, "error": "No file uploaded. Use multipart field name 'file'."}), 400

    f           = request.files["file"]
    filename    = f.filename or "tutorial"
    ext         = os.path.splitext(filename.lower())[1]
    video_title = request.form.get("video_title", filename)
    video_url   = request.form.get("video_url", "")

    mime_type = SUPPORTED_VIDEO_MIMES.get(ext)
    if not mime_type:
        return jsonify({
            "ok": False,
            "error": f"Unsupported video format '{ext}'. Supported: {', '.join(SUPPORTED_VIDEO_MIMES)}"
        }), 400

    if not _gemini_client:
        return jsonify({"ok": False, "error": "Gemini client not initialised — check GOOGLE_API_KEY"}), 500

    t_ingest    = time.time()
    video_bytes = f.read()
    logger.info("[tutorial] STEP 1/5: received '%s' (%.2f MB, mime=%s)",
                video_title, len(video_bytes) / 1024 / 1024, mime_type)

    # Upload to Gemini Files API and wait for processing
    logger.info("[tutorial] STEP 2/5: uploading to Gemini Files API + waiting for ACTIVE...")
    gemini_file = _upload_video_to_gemini(video_bytes, mime_type, display_name=video_title)
    if not gemini_file:
        logger.error("[tutorial] STEP 2/5 FAILED: upload/processing did not reach ACTIVE")
        return jsonify({"ok": False, "error": "Video upload or processing by Gemini failed. Try a smaller file or check the format."}), 500

    # Ask Gemini to watch and extract articles with self-generated timestamps.
    # A hard timeout prevents the request from hanging the whole ingest indefinitely.
    try:
        logger.info("[tutorial] STEP 3/5: sending video to Gemini for extraction "
                    "(model=%s, timeout=%ds)...", GEMINI_MODEL, GEMINI_VIDEO_TIMEOUT_MS // 1000)
        t_extract = time.time()
        resp = _gemini_client.models.generate_content(
            model=GEMINI_MODEL,
            contents=[gemini_file, TUTORIAL_VIDEO_PROMPT],
            config=genai_types.GenerateContentConfig(
                response_mime_type="application/json",
                http_options=genai_types.HttpOptions(timeout=GEMINI_VIDEO_TIMEOUT_MS),
            )
        )
        raw = resp.text or ""
        logger.info("[tutorial] STEP 3/5 done: Gemini responded in %.1fs (%d chars)",
                    time.time() - t_extract, len(raw))
    except Exception as e:
        logger.error("[tutorial] Gemini video analysis FAILED after %.1fs: %s",
                     time.time() - t_extract, e)
        _delete_gemini_file(gemini_file)
        return jsonify({"ok": False, "error": f"Gemini analysis failed: {e}"}), 500
    finally:
        # Always attempt cleanup — files expire in 48h anyway but clean up early
        _delete_gemini_file(gemini_file)

    parsed   = _safe_json(raw, [])
    articles = _validate_articles(parsed) if isinstance(parsed, list) else []
    logger.info("[tutorial] STEP 4/5: parsed %d valid article(s) from Gemini response", len(articles))

    if not articles:
        logger.warning("[tutorial] No articles extracted — aborting.")
        return jsonify({
            "ok":    False,
            "error": "Gemini could not extract any how-to articles from this video. "
                     "Ensure the video contains on-screen demonstrations of Iris features."
        }), 500

    # Attach video metadata and normalise timestamp types
    for a in articles:
        a["video_url"]   = video_url
        a["video_title"] = video_title
        for ts_key in ("timestamp_start", "timestamp_end"):
            val = a.get(ts_key)
            if not isinstance(val, int):
                try:
                    a[ts_key] = int(val) if val is not None else 0
                except (TypeError, ValueError):
                    a[ts_key] = 0
    logger.info("[tutorial] article timestamps (start–end s): %s",
                [(a.get("timestamp_start"), a.get("timestamp_end")) for a in articles])

    # ── THE INNOVATION: pre-crop each demonstrated segment into a standalone clip ──
    # We write the original once to a temp file, then ffmpeg-cut each article's
    # [timestamp_start, timestamp_end] window, upload the clip to Firebase Storage,
    # and stamp clip_url on the article. The WhatsApp bot later sends this exact clip.
    bucket_ready = bool(get_bucket())
    clips_made   = 0
    if bucket_ready:
        logger.info("[tutorial] STEP 5/5: generating clips for %d article(s)...", len(articles))
        t_clips  = time.time()
        src_path = None
        try:
            with tempfile.NamedTemporaryFile(suffix=ext, delete=False) as src_tmp:
                src_tmp.write(video_bytes)
                src_path = src_tmp.name
            # Probe is logged for diagnostics only — container metadata is sometimes
            # wrong (esp. WhatsApp/edited MP4s), so we do NOT use it to skip segments.
            # We trust Gemini's timestamps (it watched the real file) and let ffmpeg
            # copy fail gracefully if a segment is genuinely past EOF.
            logger.info("[tutorial] wrote source video to temp: %s (reported duration %.1fs, codec %s)",
                        src_path, _video_duration(src_path), _video_codec(src_path))
            for idx, a in enumerate(articles, 1):
                start = a.get("timestamp_start", 0)
                end   = a.get("timestamp_end", 0)
                if not end or end <= start:
                    logger.info("[tutorial] clip %d/%d skipped — no valid timestamps (%s-%s)",
                                idx, len(articles), start, end)
                    continue
                out_path  = os.path.join(tempfile.gettempdir(), f"clip_{uuid.uuid4().hex}.mp4")
                clip_path = f"kb_clips/{uuid.uuid4().hex}.mp4"
                logger.info("[tutorial] clip %d/%d: cropping %ss–%ss...", idx, len(articles), start, end)
                if _crop_video_segment(src_path, start, end, out_path):
                    try:
                        clip_bytes = os.path.getsize(out_path)
                        logger.info("[tutorial] clip %d/%d: uploading %d bytes to %s...",
                                    idx, len(articles), clip_bytes, clip_path)
                        t_up = time.time()
                        with open(out_path, "rb") as cf:
                            data = cf.read()
                        clip_url = _upload_to_storage(data, clip_path, "video/mp4")
                        if clip_url:
                            # Store both the best-effort URL and the object path. The path
                            # lets us mint a fresh signed URL at send-time (uniform-access safe).
                            a["clip_url"]   = clip_url
                            a["clip_path"]  = clip_path
                            a["clip_start"] = start
                            a["clip_end"]   = end
                            clips_made += 1
                            logger.info("[tutorial] clip %d/%d: uploaded in %.1fs",
                                        idx, len(articles), time.time() - t_up)
                        else:
                            logger.warning("[tutorial] clip %d/%d: upload returned no URL", idx, len(articles))
                    finally:
                        try:
                            os.remove(out_path)
                        except Exception:
                            pass
                else:
                    logger.warning("[tutorial] clip %d/%d: ffmpeg crop failed", idx, len(articles))
            logger.info("[tutorial] STEP 5/5 done: %d/%d clips made in %.1fs",
                        clips_made, len(articles), time.time() - t_clips)
        except Exception as e:
            logger.error("[tutorial] Clip generation error: %s", e)
        finally:
            if src_path:
                try:
                    os.remove(src_path)
                except Exception:
                    pass
    else:
        logger.warning("[tutorial] Storage bucket not configured — skipping clip generation.")

    logger.info("[tutorial] saving %d article(s) to Firestore...", len(articles))
    stats = _save_kb_articles(articles, source_label=f"tutorial:{video_title}")
    logger.info("[tutorial] COMPLETE: %d articles, %d clips, saved=%d, skipped=%d (total %.1fs)",
                len(articles), clips_made, stats["saved"], stats["skipped"], time.time() - t_ingest)

    return jsonify({
        "ok":             True,
        "video_title":    video_title,
        "articles_found": len(articles),
        "articles":       articles,   # full list — frontend INSERTs to Supabase kb_articles
        "saved":          stats["saved"],
        "skipped_dupes":  stats["skipped"],
    })


# ══════════════════════════════════════════════════════════════════════════════
# FEATURE 5 — Agent Solution Writing (NL Text + Voice)
# ══════════════════════════════════════════════════════════════════════════════

SOLUTION_EXTRACTION_PROMPT = """You are a support knowledge base curator.
An agent has described a solution they used to resolve a ticket.
Structure this into a reusable KB article.

Return ONLY a valid JSON object — no preamble, no markdown fences.
All strings must be properly JSON-escaped.

Schema:
{"title": "string", "content": "string (clear step-by-step solution)", "category": "one of: Account|Billing|Technical|Feature|Other", "tags": ["string"]}

Agent description:
"""

@app.post("/api/kb/agent-solution-nl")
@require_auth(roles=["admin", "support"])
def agent_solution_nl():
    body      = request.get_json(silent=True) or {}
    message   = body.get("message", "").strip()
    agent_id  = body.get("agent_id", "unknown")
    ticket_id = body.get("ticket_id", "")
    if not message:
        return jsonify({"ok": False, "error": "message is required"}), 400
    raw     = _gemini_text(SOLUTION_EXTRACTION_PROMPT + message, json_mode=True)
    article = _safe_json(raw, {})
    if not isinstance(article, dict) or not article.get("title"):
        return jsonify({"ok": False, "error": "Could not structure solution"}), 500
    if ticket_id:
        article.setdefault("tags", []).append(f"ticket:{ticket_id}")
    stats = _save_kb_articles([article], source_label=f"agent:{agent_id}")
    return jsonify({"ok": True, "saved": stats["saved"],
                    "article": article,    # single article — frontend INSERTs to Supabase kb_articles
                    "articles": [article]})


@app.post("/api/kb/agent-solution-voice")
@require_auth(roles=["admin", "support"])
def agent_solution_voice():
    body         = request.get_json(silent=True) or {}
    audio_b64    = body.get("audio_b64", "")
    audio_format = body.get("audio_format", "wav")
    agent_id     = body.get("agent_id", "unknown")
    ticket_id    = body.get("ticket_id", "")
    if not audio_b64:
        return jsonify({"ok": False, "error": "audio_b64 is required"}), 400
    transcript = _transcribe_audio_assemblyai(audio_b64, audio_format)
    if not transcript:
        return jsonify({"ok": False, "error": "Transcription failed"}), 500
    raw     = _gemini_text(SOLUTION_EXTRACTION_PROMPT + transcript, json_mode=True)
    article = _safe_json(raw, {})
    if not isinstance(article, dict) or not article.get("title"):
        return jsonify({"ok": False, "error": "Could not structure solution from transcript"}), 500
    if ticket_id:
        article.setdefault("tags", []).append(f"ticket:{ticket_id}")
    stats = _save_kb_articles([article], source_label=f"agent:{agent_id}")
    return jsonify({"ok": True, "transcript": transcript, "saved": stats["saved"],
                    "article": article,    # single article — frontend INSERTs to Supabase kb_articles
                    "articles": [article]})


# ══════════════════════════════════════════════════════════════════════════════
# FEATURE 6 — Iris Chatbot (RAG over KB + Tutorials)
# ══════════════════════════════════════════════════════════════════════════════

def _search_kb(query: str, limit: int = 5) -> List[Dict]:
    if not db:
        return []
    query_terms = [t.lower() for t in query.split() if len(t) > 2]
    try:
        docs = db.collection("iris_kb_articles").order_by(
            "created_at", direction=firestore.Query.DESCENDING
        ).limit(200).stream()
        results = []
        for doc in docs:
            d     = doc.to_dict()
            text  = f"{d.get('title','')} {d.get('content','')} {' '.join(d.get('tags',[]))}".lower()
            score = sum(1 for term in query_terms if term in text)
            if score > 0:
                results.append({"score": score, **d})
        results.sort(key=lambda x: x["score"], reverse=True)
        return results[:limit]
    except Exception as e:
        logger.error("KB search error: %s", e)
        return []


CHATBOT_SYSTEM_PROMPT = """You are Iris, an intelligent support assistant for the Iris Support Portal.

Answer ONLY from the provided knowledge base context.
If the answer is in a tutorial with a timestamp, mention the video and timestamp.
Be concise, clear, and friendly. Format step-by-step answers as numbered lists.
If you cannot find the answer, say so honestly and suggest submitting a ticket.
"""

@app.post("/api/chatbot/query")
def chatbot_query():
    body       = request.get_json(silent=True) or {}
    message    = body.get("message", "").strip()
    session_id = body.get("session_id", "default")
    user_id    = body.get("user_id", "anonymous")
    if not message:
        return jsonify({"ok": False, "error": "message is required"}), 400
    kb_results = _search_kb(message, limit=5)
    context_blocks = []
    sources        = []
    for r in kb_results:
        block = f"[Article: {r.get('title')}]\n{r.get('content', '')}"
        if r.get("timestamp_start") is not None:
            ts = r["timestamp_start"]
            block += f"\n(Tutorial: {r.get('video_title','Video')} at {ts//60:02d}:{ts%60:02d}"
            if r.get("video_url"):
                block += f" — {r['video_url']}"
            block += ")"
        context_blocks.append(block)
        sources.append({
            "title":     r.get("title"),
            "category":  r.get("category"),
            "source":    r.get("source"),
            "ts_start":  r.get("timestamp_start"),
            "video_url": r.get("video_url"),
        })
    context_str  = "\n\n---\n\n".join(context_blocks) if context_blocks else "No relevant articles found."
    full_prompt  = f"{CHATBOT_SYSTEM_PROMPT}\n\nKNOWLEDGE BASE CONTEXT:\n{context_str}\n\nUSER QUESTION: {message}\n\nAnswer:"
    answer       = _gemini_text(full_prompt)
    if not answer:
        answer = "Sorry, I could not process your question right now. Please try again or submit a support ticket."
    if db:
        db.collection("iris_chatbot_logs").add({
            "user_id": user_id, "session_id": session_id,
            "message": message, "answer": answer, "sources": sources,
            "created_at": datetime.now(timezone.utc).isoformat(),
        })
    return jsonify({"ok": True, "answer": answer, "sources": sources})


# ══════════════════════════════════════════════════════════════════════════════
# KB CRUD ENDPOINTS — list / get / create / update / delete
# ══════════════════════════════════════════════════════════════════════════════

@app.post("/api/kb/articles")
@require_auth(roles=["admin", "support"])
def create_kb_article():
    """Manually create a KB article (no AI) — full-control authoring from the dashboard."""
    if not db:
        return jsonify({"ok": False, "error": "Firebase unavailable"}), 500
    body    = request.get_json(silent=True) or {}
    title   = (body.get("title") or "").strip()
    content = (body.get("content") or "").strip()
    if len(title) < 3 or len(content) < 3:
        return jsonify({"ok": False, "error": "title and content are required"}), 400
    tags = body.get("tags", [])
    if isinstance(tags, str):
        tags = [t.strip() for t in re.split(r"[,;|]", tags) if t.strip()]
    doc = {
        "title":       title,
        "content":     content,
        "category":    (body.get("category") or "General").strip(),
        "tags":        tags if isinstance(tags, list) else [],
        "source":      f"manual:{request.user.get('email','')}",
        "fingerprint": _article_fingerprint(title, content),
        "created_at":  _now_iso(),
    }
    ref = db.collection("iris_kb_articles").add(doc)
    return jsonify({"ok": True, "id": ref[1].id, "article": {"id": ref[1].id, **doc}})


@app.get("/api/kb/articles/<article_id>")
@require_auth(roles=["admin", "support"])
def get_kb_article(article_id: str):
    if not db:
        return jsonify({"ok": False, "error": "Firebase unavailable"}), 500
    snap = db.collection("iris_kb_articles").document(article_id).get()
    if not snap.exists:
        return jsonify({"ok": False, "error": "Article not found"}), 404
    return jsonify({"ok": True, "article": {"id": snap.id, **snap.to_dict()}})


@app.patch("/api/kb/articles/<article_id>")
@require_auth(roles=["admin", "support"])
def update_kb_article(article_id: str):
    """Edit an existing article — title, content, category, tags (for revisions/updates)."""
    if not db:
        return jsonify({"ok": False, "error": "Firebase unavailable"}), 500
    body    = request.get_json(silent=True) or {}
    updates = {}
    for field in ("title", "content", "category"):
        if field in body and isinstance(body[field], str) and body[field].strip():
            updates[field] = body[field].strip()
    if "tags" in body:
        tags = body["tags"]
        if isinstance(tags, str):
            tags = [t.strip() for t in re.split(r"[,;|]", tags) if t.strip()]
        updates["tags"] = tags if isinstance(tags, list) else []
    if not updates:
        return jsonify({"ok": False, "error": "No valid fields to update"}), 400
    # Keep the dedup fingerprint consistent when title/content change.
    if "title" in updates or "content" in updates:
        snap = db.collection("iris_kb_articles").document(article_id).get()
        cur  = snap.to_dict() if snap.exists else {}
        updates["fingerprint"] = _article_fingerprint(
            updates.get("title", cur.get("title", "")),
            updates.get("content", cur.get("content", "")),
        )
    updates["updated_at"] = _now_iso()
    db.collection("iris_kb_articles").document(article_id).update(updates)
    return jsonify({"ok": True, "updated": list(updates.keys())})


@app.get("/api/kb/articles")
@require_auth(roles=["admin", "support"])
def list_kb_articles():
    category = request.args.get("category", "")
    limit    = int(request.args.get("limit", 50))
    if not db:
        return jsonify({"ok": False, "error": "Firebase unavailable"}), 500
    try:
        query = db.collection("iris_kb_articles").order_by("created_at", direction=firestore.Query.DESCENDING)
        if category:
            query = query.where("category", "==", category)
        docs     = query.limit(limit).stream()
        articles = [{"id": d.id, **d.to_dict()} for d in docs]
        return jsonify({"ok": True, "articles": articles, "count": len(articles)})
    except Exception as e:
        return jsonify({"ok": False, "error": str(e)}), 500


@app.delete("/api/kb/articles/<article_id>")
@require_auth(roles=["admin", "support"])
def delete_kb_article(article_id: str):
    if not db:
        return jsonify({"ok": False, "error": "Firebase unavailable"}), 500
    try:
        db.collection("iris_kb_articles").document(article_id).delete()
        return jsonify({"ok": True})
    except Exception as e:
        return jsonify({"ok": False, "error": str(e)}), 500


# ══════════════════════════════════════════════════════════════════════════════
# ADMIN — Staff account management (admin provisions accounts, no public signup)
# ══════════════════════════════════════════════════════════════════════════════

@app.post("/api/admin/staff")
@require_auth(roles=["admin"])
def create_staff():
    body     = request.get_json(silent=True) or {}
    email    = body.get("email", "").strip().lower()
    password = body.get("password", "")
    name     = body.get("name", "").strip()
    role     = body.get("role", "support").strip()
    if not email or not password:
        return jsonify({"ok": False, "error": "email and password are required"}), 400
    if role not in ("admin", "support"):
        return jsonify({"ok": False, "error": "role must be 'admin' or 'support'"}), 400
    try:
        user = fb_auth.create_user(email=email, password=password, display_name=name or None)
        fb_auth.set_custom_user_claims(user.uid, {"role": role})
        if db:
            db.collection("iris_staff").document(user.uid).set({
                "uid": user.uid, "email": email, "name": name, "role": role,
                "disabled": False, "created_at": datetime.now(timezone.utc).isoformat(),
                "created_by": request.user.get("email", ""),
            })
        return jsonify({"ok": True, "uid": user.uid, "email": email, "role": role})
    except Exception as e:
        logger.error("create_staff error: %s", e)
        return jsonify({"ok": False, "error": str(e)}), 400


@app.get("/api/admin/staff")
@require_auth(roles=["admin"])
def list_staff():
    if not db:
        return jsonify({"ok": False, "error": "Firebase unavailable"}), 500
    try:
        docs  = db.collection("iris_staff").stream()
        staff = [{"id": d.id, **d.to_dict()} for d in docs]
        return jsonify({"ok": True, "staff": staff, "count": len(staff)})
    except Exception as e:
        return jsonify({"ok": False, "error": str(e)}), 500


@app.delete("/api/admin/staff/<uid>")
@require_auth(roles=["admin"])
def disable_staff(uid: str):
    try:
        fb_auth.update_user(uid, disabled=True)
        if db:
            db.collection("iris_staff").document(uid).set({"disabled": True}, merge=True)
        return jsonify({"ok": True})
    except Exception as e:
        logger.error("disable_staff error: %s", e)
        return jsonify({"ok": False, "error": str(e)}), 400


# ══════════════════════════════════════════════════════════════════════════════
# MULTIMODAL KB INGEST — Image + Audio (dashboard, auth-protected)
# ══════════════════════════════════════════════════════════════════════════════

IMAGE_KB_PROMPT = """You are a knowledge base curator for the Iris support platform.
You are given a screenshot or photo (often an error screen, POS display, or app state) plus
an optional caption. Extract ONE self-contained how-to / troubleshooting KB article that explains
the situation shown and how a user resolves it.

Return ONLY a valid JSON object — no preamble, no markdown fences. All strings JSON-escaped.
Schema:
{"title": "string (max 80 chars)", "content": "string (clear step-by-step explanation/solution)", "category": "one of: Account|Technical|Location|Attendance|Device|POS|Other", "tags": ["string"]}

Caption: """


@app.post("/api/kb/image-ingest")
@require_auth(roles=["admin", "support"])
def image_ingest():
    if "file" not in request.files:
        return jsonify({"ok": False, "error": "No file uploaded. Use multipart field 'file'."}), 400
    if not _gemini_client:
        return jsonify({"ok": False, "error": "Gemini client not initialised"}), 500
    f       = request.files["file"]
    caption = request.form.get("caption", "")
    img_bytes = f.read()
    mime = f.mimetype or "image/jpeg"
    try:
        part = genai_types.Part.from_bytes(data=img_bytes, mime_type=mime)
        raw  = _gemini_multimodal([IMAGE_KB_PROMPT + caption, part], json_mode=True)
    except Exception as e:
        logger.error("image_ingest vision error: %s", e)
        return jsonify({"ok": False, "error": f"Vision analysis failed: {e}"}), 500
    article = _safe_json(raw, {})
    if not isinstance(article, dict) or not article.get("title"):
        return jsonify({"ok": False, "error": "Could not extract an article from this image"}), 500
    image_url = _upload_to_storage(img_bytes, f"kb_images/{uuid.uuid4().hex}", mime)
    if image_url:
        article["image_url"] = image_url
    stats = _save_kb_articles([article], source_label="image_upload")
    return jsonify({"ok": True, "saved": stats["saved"], "article": article, "articles": [article]})


@app.post("/api/kb/audio-ingest")
@require_auth(roles=["admin", "support"])
def audio_ingest():
    if "file" not in request.files:
        return jsonify({"ok": False, "error": "No file uploaded. Use multipart field 'file'."}), 400
    if not ASSEMBLYAI_API_KEY:
        return jsonify({"ok": False, "error": "AssemblyAI not configured on server"}), 500
    f          = request.files["file"]
    audio_bytes = f.read()
    transcript = _transcribe_audio_bytes(audio_bytes)
    if not transcript:
        return jsonify({"ok": False, "error": "Transcription failed or empty"}), 500
    raw     = _gemini_text(SOLUTION_EXTRACTION_PROMPT + transcript, json_mode=True)
    article = _safe_json(raw, {})
    if not isinstance(article, dict) or not article.get("title"):
        return jsonify({"ok": False, "error": "Could not structure an article from the audio"}), 500
    stats = _save_kb_articles([article], source_label="audio_upload")
    return jsonify({"ok": True, "transcript": transcript, "saved": stats["saved"],
                    "article": article, "articles": [article]})


# ══════════════════════════════════════════════════════════════════════════════
# TICKETS — customer issues raised on WhatsApp, resolved by staff (two-way)
# ══════════════════════════════════════════════════════════════════════════════

def _now_iso() -> str:
    return datetime.now(timezone.utc).isoformat()


def _get_open_ticket_for_phone(phone: str) -> Optional[Tuple[str, Dict]]:
    """Return (doc_id, ticket) for the most recent non-resolved ticket for a phone."""
    if not db:
        return None
    try:
        # Single-field filter + small fetch, status filtered in Python — avoids
        # needing a Firestore composite index on (customer_phone, status, created_at).
        docs = (db.collection("iris_tickets")
                  .where("customer_phone", "==", phone)
                  .limit(10).stream())
        candidates = [(d.id, d.to_dict()) for d in docs]
        candidates = [c for c in candidates if c[1].get("status") in ("open", "with_agent")]
        if candidates:
            candidates.sort(key=lambda c: c[1].get("created_at", ""), reverse=True)
            return candidates[0]
    except Exception as e:
        logger.error("open-ticket lookup error: %s", e)
    return None


def _create_ticket(phone: str, first_message: str) -> Optional[str]:
    """Create a ticket from a customer's WhatsApp message, AI-pre-filling metadata."""
    if not db:
        return None
    meta = {}
    try:
        raw  = _gemini_text(TICKET_EXTRACTION_PROMPT + first_message, json_mode=True)
        meta = _safe_json(raw, {}) or {}
    except Exception as e:
        logger.error("ticket metadata extraction error: %s", e)
    doc = {
        "customer_phone": phone,
        "title":          meta.get("title") or first_message[:80],
        "description":    meta.get("description", first_message),
        "category":       meta.get("category_hint", "Other"),
        "priority":       meta.get("priority_hint", "medium"),
        "status":         "with_agent",
        "assigned_to":    "",
        "thread":         [{"sender": "customer", "text": first_message, "ts": _now_iso()}],
        "created_at":     _now_iso(),
        "updated_at":     _now_iso(),
    }
    ref = db.collection("iris_tickets").add(doc)
    ticket_id = ref[1].id
    logger.info("Created ticket %s for %s", ticket_id, phone)
    return ticket_id


def _append_ticket_message(ticket_id: str, sender: str, text: str, extra: Optional[Dict] = None) -> None:
    if not db:
        return
    entry = {"sender": sender, "text": text, "ts": _now_iso()}
    if extra:
        entry.update(extra)
    db.collection("iris_tickets").document(ticket_id).update({
        "thread": firestore.ArrayUnion([entry]),
        "updated_at": _now_iso(),
    })


@app.get("/api/tickets")
@require_auth(roles=["admin", "support"])
def list_tickets():
    if not db:
        return jsonify({"ok": False, "error": "Firebase unavailable"}), 500
    status   = request.args.get("status", "")
    assignee = request.args.get("assignee", "")
    limit    = int(request.args.get("limit", 100))
    try:
        q = db.collection("iris_tickets").order_by("updated_at", direction=firestore.Query.DESCENDING)
        if status:
            q = q.where("status", "==", status)
        if assignee:
            q = q.where("assigned_to", "==", assignee)
        docs    = q.limit(limit).stream()
        tickets = [{"id": d.id, **d.to_dict()} for d in docs]
        return jsonify({"ok": True, "tickets": tickets, "count": len(tickets)})
    except Exception as e:
        return jsonify({"ok": False, "error": str(e)}), 500


@app.get("/api/tickets/<ticket_id>")
@require_auth(roles=["admin", "support"])
def get_ticket(ticket_id: str):
    if not db:
        return jsonify({"ok": False, "error": "Firebase unavailable"}), 500
    snap = db.collection("iris_tickets").document(ticket_id).get()
    if not snap.exists:
        return jsonify({"ok": False, "error": "Ticket not found"}), 404
    return jsonify({"ok": True, "ticket": {"id": snap.id, **snap.to_dict()}})


@app.post("/api/tickets/<ticket_id>/reply")
@require_auth(roles=["admin", "support"])
def reply_ticket(ticket_id: str):
    """Staff reply → appended to thread AND delivered to the customer on WhatsApp."""
    if not db:
        return jsonify({"ok": False, "error": "Firebase unavailable"}), 500
    body = request.get_json(silent=True) or {}
    text = body.get("text", "").strip()
    if not text:
        return jsonify({"ok": False, "error": "text is required"}), 400
    snap = db.collection("iris_tickets").document(ticket_id).get()
    if not snap.exists:
        return jsonify({"ok": False, "error": "Ticket not found"}), 404
    ticket = snap.to_dict()
    agent  = request.user.get("email", "")
    _append_ticket_message(ticket_id, "agent", text, {"agent": agent})
    updates = {"status": "with_agent", "updated_at": _now_iso()}
    if not ticket.get("assigned_to"):
        updates["assigned_to"] = agent
    db.collection("iris_tickets").document(ticket_id).update(updates)
    # Deliver to the customer over WhatsApp (must be within the 24h service window)
    sent = False
    if WHATSAPP_AVAILABLE and wa:
        sent = wa.send_text_message(ticket.get("customer_phone", ""), text)
    return jsonify({"ok": True, "delivered": sent})


@app.patch("/api/tickets/<ticket_id>")
@require_auth(roles=["admin", "support"])
def update_ticket(ticket_id: str):
    if not db:
        return jsonify({"ok": False, "error": "Firebase unavailable"}), 500
    body    = request.get_json(silent=True) or {}
    updates = {}
    if "status" in body and body["status"] in ("open", "with_agent", "resolved"):
        updates["status"] = body["status"]
    if "assigned_to" in body:
        updates["assigned_to"] = body["assigned_to"]
    if "priority" in body:
        updates["priority"] = body["priority"]
    if not updates:
        return jsonify({"ok": False, "error": "No valid fields to update"}), 400
    updates["updated_at"] = _now_iso()
    db.collection("iris_tickets").document(ticket_id).update(updates)
    return jsonify({"ok": True, "updated": list(updates.keys())})


# ══════════════════════════════════════════════════════════════════════════════
# CUSTOMERS — who is interacting, by phone number (aggregated for the dashboard)
# ══════════════════════════════════════════════════════════════════════════════

def _aggregate_customers() -> List[Dict]:
    """
    Aggregate every customer the bot has interacted with, keyed by phone number:
    ticket counts by status, total interactions, and last-seen time. Shared by the
    Customers list and the analytics endpoints.
    """
    customers: Dict[str, Dict] = {}

    def _row(phone: str) -> Dict:
        return customers.setdefault(phone, {
            "phone": phone, "open_tickets": 0, "resolved_tickets": 0,
            "total_tickets": 0, "interactions": 0, "last_interaction": "",
        })

    # Tickets → status breakdown + last activity.
    for d in db.collection("iris_tickets").limit(2000).stream():
        t = d.to_dict()
        phone = t.get("customer_phone")
        if not phone:
            continue
        row = _row(phone)
        row["total_tickets"] += 1
        if t.get("status") == "resolved":
            row["resolved_tickets"] += 1
        else:
            row["open_tickets"] += 1
        upd = t.get("updated_at", "") or t.get("created_at", "")
        if upd > row["last_interaction"]:
            row["last_interaction"] = upd

    # Chatbot logs → interaction counts + last-seen.
    for d in db.collection("iris_chatbot_logs").limit(5000).stream():
        l = d.to_dict()
        phone = l.get("user_id")
        if not phone:
            continue
        row = _row(phone)
        row["interactions"] += 1
        ts = l.get("created_at", "")
        if ts > row["last_interaction"]:
            row["last_interaction"] = ts

    return list(customers.values())


@app.get("/api/customers")
@require_auth(roles=["admin", "support"])
def list_customers():
    """Who's active, who has open tickets, who's resolved — all by phone number."""
    if not db:
        return jsonify({"ok": False, "error": "Firebase unavailable"}), 500
    try:
        result = sorted(_aggregate_customers(),
                        key=lambda c: c["last_interaction"], reverse=True)
        return jsonify({"ok": True, "customers": result, "count": len(result)})
    except Exception as e:
        logger.error("list_customers error: %s", e)
        return jsonify({"ok": False, "error": str(e)}), 500


@app.get("/api/customers/<path:phone>")
@require_auth(roles=["admin", "support"])
def get_customer(phone: str):
    """One customer's full history: their tickets, recent interactions, and feedback."""
    if not db:
        return jsonify({"ok": False, "error": "Firebase unavailable"}), 500
    try:
        tickets = [{"id": d.id, **d.to_dict()} for d in
                   db.collection("iris_tickets").where("customer_phone", "==", phone).limit(200).stream()]
        tickets.sort(key=lambda t: t.get("updated_at", ""), reverse=True)
        logs = [d.to_dict() for d in
                db.collection("iris_chatbot_logs").where("user_id", "==", phone).limit(200).stream()]
        logs.sort(key=lambda l: l.get("created_at", ""), reverse=True)
        feedback = [{"id": d.id, **d.to_dict()} for d in
                    db.collection("iris_feedback").where("customer_phone", "==", phone).limit(100).stream()]
        return jsonify({"ok": True, "phone": phone, "tickets": tickets,
                        "interactions": logs[:50], "feedback": feedback})
    except Exception as e:
        logger.error("get_customer error: %s", e)
        return jsonify({"ok": False, "error": str(e)}), 500


# ══════════════════════════════════════════════════════════════════════════════
# FEEDBACK — 👎 signals from customers, flagged for the dashboard
# ══════════════════════════════════════════════════════════════════════════════

@app.get("/api/feedback")
@require_auth(roles=["admin", "support"])
def list_feedback():
    if not db:
        return jsonify({"ok": False, "error": "Firebase unavailable"}), 500
    status = request.args.get("status", "open")   # default: show what needs attention
    limit  = int(request.args.get("limit", 100))
    try:
        q    = db.collection("iris_feedback")
        if status:
            q = q.where("status", "==", status)
        items = [{"id": d.id, **d.to_dict()} for d in q.limit(limit).stream()]
        items.sort(key=lambda f: f.get("created_at", ""), reverse=True)
        return jsonify({"ok": True, "feedback": items, "count": len(items)})
    except Exception as e:
        return jsonify({"ok": False, "error": str(e)}), 500


@app.patch("/api/feedback/<feedback_id>")
@require_auth(roles=["admin", "support"])
def update_feedback(feedback_id: str):
    if not db:
        return jsonify({"ok": False, "error": "Firebase unavailable"}), 500
    body   = request.get_json(silent=True) or {}
    status = body.get("status")
    if status not in ("open", "closed"):
        return jsonify({"ok": False, "error": "status must be 'open' or 'closed'"}), 400
    db.collection("iris_feedback").document(feedback_id).update({
        "status": status, "updated_at": _now_iso(),
        "handled_by": request.user.get("email", ""),
    })
    return jsonify({"ok": True})


# ══════════════════════════════════════════════════════════════════════════════
# ANALYTICS — top users + most recurring issues (dashboard insights)
# ══════════════════════════════════════════════════════════════════════════════

# Common words to ignore when surfacing recurring-issue keywords.
_STOPWORDS = {
    "the", "a", "an", "and", "or", "but", "is", "are", "was", "were", "be", "been",
    "to", "of", "in", "on", "for", "with", "my", "me", "i", "you", "it", "this",
    "that", "how", "do", "i'm", "im", "can", "cant", "can't", "not", "no", "yes",
    "please", "help", "issue", "problem", "iris", "have", "has", "get", "got",
    "when", "what", "why", "where", "there", "your", "they", "from", "at", "as",
}


@app.get("/api/analytics/top-users")
@require_auth(roles=["admin", "support"])
def top_users():
    """Customers ranked by how much they interact — busiest phone numbers first."""
    if not db:
        return jsonify({"ok": False, "error": "Firebase unavailable"}), 500
    limit = int(request.args.get("limit", 10))
    try:
        ranked = sorted(
            _aggregate_customers(),
            key=lambda c: (c["interactions"] + c["total_tickets"], c["last_interaction"]),
            reverse=True,
        )
        return jsonify({"ok": True, "users": ranked[:limit]})
    except Exception as e:
        logger.error("top_users error: %s", e)
        return jsonify({"ok": False, "error": str(e)}), 500


@app.get("/api/analytics/recurring-issues")
@require_auth(roles=["admin", "support"])
def recurring_issues():
    """
    What customers keep asking about: ticket counts by category, plus the most
    frequent keywords across ticket titles and bot questions.
    """
    if not db:
        return jsonify({"ok": False, "error": "Firebase unavailable"}), 500
    limit = int(request.args.get("limit", 15))
    try:
        from collections import Counter
        by_category: "Counter[str]" = Counter()
        keywords:    "Counter[str]" = Counter()

        def _harvest(text: str) -> None:
            for term in re.findall(r"[a-zA-Z']{3,}", (text or "").lower()):
                if term not in _STOPWORDS:
                    keywords[term] += 1

        for d in db.collection("iris_tickets").limit(2000).stream():
            t = d.to_dict()
            by_category[t.get("category", "Other") or "Other"] += 1
            _harvest(t.get("title", ""))

        for d in db.collection("iris_chatbot_logs").limit(5000).stream():
            _harvest(d.to_dict().get("message", ""))

        return jsonify({
            "ok": True,
            "by_category":  [{"category": c, "count": n} for c, n in by_category.most_common()],
            "top_keywords": [{"term": t, "count": n} for t, n in keywords.most_common(limit)],
        })
    except Exception as e:
        logger.error("recurring_issues error: %s", e)
        return jsonify({"ok": False, "error": str(e)}), 500


# ══════════════════════════════════════════════════════════════════════════════
# WHATSAPP BOT — customer-facing channel (Twilio webhook)
# ══════════════════════════════════════════════════════════════════════════════

# Phrases that signal the customer wants a human agent rather than the bot.
_HUMAN_REQUEST_RE = re.compile(
    r"\b(human|agent|person|someone|representative|support team|talk to|speak to|call me|operator)\b",
    re.IGNORECASE,
)

# In-memory dedup of Twilio MessageSid (Twilio retries webhooks). Backed by Firestore.
_processed_msgs = OrderedDict()
_dedup_lock     = threading.Lock()


def _is_duplicate(msg_id: str) -> bool:
    if not msg_id:
        return False
    with _dedup_lock:
        if msg_id in _processed_msgs:
            return True
        _processed_msgs[msg_id] = time.time()
        # Trim memory cache
        while len(_processed_msgs) > 5000:
            _processed_msgs.popitem(last=False)
    if db:
        try:
            ref = db.collection("processed_messages").document(msg_id)
            if ref.get().exists:
                return True
            ref.set({"processed_at": _now_iso()})
        except Exception as e:
            logger.error("Dedup persistence error: %s", e)
    return False


def _build_kb_answer(query: str) -> Tuple[str, Optional[str]]:
    """
    RAG over the KB. Returns (answer_text, clip_url_or_None) where clip_url is a
    freshly-signed, publicly-fetchable URL for the best matching tutorial clip.
    answer_text is "" when nothing relevant was found (caller escalates).
    """
    kb_results = _search_kb(query, limit=5)
    if not kb_results:
        return "", None
    context_blocks = []
    clip_send_url  = None
    for r in kb_results:
        block = f"[Article: {r.get('title')}]\n{r.get('content', '')}"
        if r.get("timestamp_start") is not None:
            ts = r["timestamp_start"]
            block += f"\n(Tutorial: {r.get('video_title','Video')} at {ts//60:02d}:{ts%60:02d})"
        context_blocks.append(block)
        if clip_send_url is None:
            # Prefer a fresh signed URL from the stored path (uniform-access safe);
            # fall back to a stored public URL if that's all we have.
            if r.get("clip_path"):
                clip_send_url = _storage_signed_url(r["clip_path"], minutes=120) or r.get("clip_url")
            elif r.get("clip_url"):
                clip_send_url = r["clip_url"]
    context_str = "\n\n---\n\n".join(context_blocks)
    full_prompt = (f"{CHATBOT_SYSTEM_PROMPT}\n\nKNOWLEDGE BASE CONTEXT:\n{context_str}\n\n"
                   f"USER QUESTION: {query}\n\nAnswer:")
    answer = _gemini_text(full_prompt) or ""
    return answer, clip_send_url


def _log_whatsapp_qa(phone: str, message: str, answer: str) -> None:
    if not db:
        return
    try:
        db.collection("iris_chatbot_logs").add({
            "user_id": phone, "session_id": "whatsapp", "channel": "whatsapp",
            "message": message, "answer": answer, "created_at": _now_iso(),
        })
    except Exception as e:
        logger.error("WhatsApp QA log error: %s", e)


# ── Feedback (👍 / 👎) on bot answers ──────────────────────────────────────────
FEEDBACK_BTN_UP   = "👍 Helpful"
FEEDBACK_BTN_DOWN = "👎 Not helpful"

_POSITIVE_FB_RE = re.compile(r"(👍|\bhelpful\b|\bthis helped\b|\bthat helped\b|\bit helped\b)", re.IGNORECASE)
_NEGATIVE_FB_RE = re.compile(r"(👎|\bnot helpful\b|\bdidn'?t help\b|\bdid not help\b|\bunhelpful\b|\bno help\b)", re.IGNORECASE)

# Lightweight per-phone conversation state (last Q&A + whether we're awaiting a
# "what was missing" explanation after a 👎). In-memory: resets on restart, which
# is fine for an ephemeral feedback follow-up.
_wa_state: Dict[str, Dict] = {}
_wa_state_lock = threading.Lock()


def _set_wa_state(phone: str, **kwargs) -> None:
    with _wa_state_lock:
        _wa_state.setdefault(phone, {}).update(kwargs)


def _get_wa_state(phone: str) -> Dict:
    with _wa_state_lock:
        return dict(_wa_state.get(phone, {}))


def _clear_wa_pending(phone: str) -> None:
    with _wa_state_lock:
        if phone in _wa_state:
            _wa_state[phone].pop("pending", None)


def _store_feedback(phone: str, rating: str, question: str, answer: str, explanation: str = "") -> None:
    """Persist a feedback signal. Negative feedback is flagged open for the dashboard."""
    if not db:
        return
    try:
        db.collection("iris_feedback").add({
            "customer_phone": phone,
            "rating":         rating,            # "up" | "down"
            "question":       question,
            "answer":         answer,
            "explanation":    explanation,
            "status":         "open" if rating == "down" else "closed",
            "created_at":     _now_iso(),
        })
    except Exception as e:
        logger.error("Feedback store error: %s", e)


def _send_answer_with_feedback(phone: str, question: str, answer: str, clip_url: Optional[str]) -> None:
    """Send the answer text, then the clip as its OWN message, then a feedback prompt."""
    wa.send_text_message(phone, answer)
    if clip_url:
        # Send media as a separate message from any text/caption — WhatsApp delivers
        # the video on its own and the caption text does not get swallowed.
        wa.send_text_message(phone, "🎥 Here's a short clip showing the exact steps:")
        wa.send_video_message(phone, clip_url)
    _log_whatsapp_qa(phone, question, answer)
    _set_wa_state(phone, last_q=question, last_a=answer)
    wa.send_buttons_as_text(phone, "Did this answer your question?",
                            [FEEDBACK_BTN_UP, FEEDBACK_BTN_DOWN])


def process_text_message(text: str, phone: str) -> None:
    """Core customer conversation logic for an inbound WhatsApp text."""
    if not (WHATSAPP_AVAILABLE and wa):
        return
    try:
        text = (text or "").strip()
        if not text:
            return

        state = _get_wa_state(phone)

        # 0a. We asked "what was missing?" after a 👎 — capture this as the explanation.
        if state.get("pending") == "feedback_explanation":
            _clear_wa_pending(phone)
            _store_feedback(phone, "down", state.get("last_q", ""), state.get("last_a", ""), text)
            wa.send_text_message(phone, "🙏 Thank you — I've flagged this for our support team to review "
                                        "and improve. If you'd like a person to follow up, reply *agent*.")
            return

        # 0b. Feedback reactions on the previous answer.
        if _NEGATIVE_FB_RE.search(text):
            _set_wa_state(phone, pending="feedback_explanation")
            wa.send_text_message(phone, "Sorry that didn't help. 🙇 What were you trying to do, or what's "
                                        "missing from the answer? Tell me and I'll pass it to our team.")
            return
        if _POSITIVE_FB_RE.search(text):
            _store_feedback(phone, "up", state.get("last_q", ""), state.get("last_a", ""))
            wa.send_text_message(phone, "Great — glad that helped! 😊 Send another question any time.")
            return

        # 1. If a live ticket is already with an agent, route the message into the thread.
        open_ticket = _get_open_ticket_for_phone(phone)
        if open_ticket and open_ticket[1].get("status") == "with_agent":
            _append_ticket_message(open_ticket[0], "customer", text)
            wa.send_text_message(phone, "✅ Got it — your message was added to your support ticket. "
                                        "An agent will reply here shortly.")
            return

        # 2. Explicit request for a human → escalate.
        if _HUMAN_REQUEST_RE.search(text):
            tid = _create_ticket(phone, text)
            ref = (tid or "")[-6:].upper()
            wa.send_text_message(phone, f"🙋 I've created support ticket *#{ref}* and our team will "
                                        f"reply to you right here on WhatsApp. You can keep typing to add details.")
            return

        # 3. Try to auto-resolve from the knowledge base (answer + clip + feedback buttons).
        answer, clip_url = _build_kb_answer(text)
        if answer:
            _send_answer_with_feedback(phone, text, answer, clip_url)
            return

        # 4. Nothing relevant found → escalate to a human ticket.
        tid = _create_ticket(phone, text)
        ref = (tid or "")[-6:].upper()
        wa.send_text_message(phone, f"I couldn't find an answer for that in our help library, so I've "
                                    f"raised support ticket *#{ref}*. Our team will reply to you here shortly. 🙏")
        _log_whatsapp_qa(phone, text, "[escalated to ticket]")
    except Exception as e:
        logger.error("process_text_message error: %s", e)
        try:
            wa.send_text_message(phone, "Sorry, something went wrong on our side. Please try again shortly.")
        except Exception:
            pass


def process_audio_message(audio_url: str, phone: str) -> None:
    """Download a WhatsApp voice note, transcribe it, then treat it as a text query."""
    if not (WHATSAPP_AVAILABLE and wa):
        return
    if not ASSEMBLYAI_API_KEY:
        wa.send_text_message(phone, "Voice messages aren't supported yet — please type your question. 🙏")
        return
    tmp_path = os.path.join(tempfile.gettempdir(), f"wa_{uuid.uuid4().hex}.ogg")
    try:
        dl = wa.download_media(audio_url, tmp_path)
        if not dl:
            wa.send_text_message(phone, "Sorry, I couldn't download your voice message.")
            return
        with open(dl, "rb") as fp:
            transcript = _transcribe_audio_bytes(fp.read())
        if not transcript:
            wa.send_text_message(phone, "Sorry, I couldn't understand that voice message. Please try typing it.")
            return
        process_text_message(transcript, phone)
    except Exception as e:
        logger.error("process_audio_message error: %s", e)
    finally:
        try:
            os.remove(tmp_path)
        except Exception:
            pass


IMAGE_ISSUE_PROMPT = """A customer sent this image to support, often a screenshot of an error or app state.
Caption: "{caption}"
Briefly describe, in one or two sentences, what problem the image shows so we can search our help library.
Reply with the description only — no preamble."""


def process_image_message(image_url: str, caption: str, phone: str) -> None:
    """Understand a customer's screenshot via Gemini vision, then answer from the KB."""
    if not (WHATSAPP_AVAILABLE and wa):
        return
    tmp_path = os.path.join(tempfile.gettempdir(), f"wa_{uuid.uuid4().hex}.jpg")
    try:
        dl = wa.download_media(image_url, tmp_path)
        if not dl or not _gemini_client:
            process_text_message(caption or "I sent an image", phone)
            return
        with open(dl, "rb") as fp:
            img_bytes = fp.read()
        try:
            part = genai_types.Part.from_bytes(data=img_bytes, mime_type="image/jpeg")
            desc = _gemini_multimodal([IMAGE_ISSUE_PROMPT.format(caption=caption or ""), part]) or ""
        except Exception as e:
            logger.error("image issue vision error: %s", e)
            desc = ""
        query = (caption + " " + desc).strip() or desc or caption or "image issue"
        process_text_message(query, phone)
    except Exception as e:
        logger.error("process_image_message error: %s", e)
    finally:
        try:
            os.remove(tmp_path)
        except Exception:
            pass


@app.post("/webhook")
def whatsapp_webhook():
    """
    Twilio inbound WhatsApp webhook. Returns empty TwiML immediately and does all
    work in a background thread (Twilio expects a response within ~15s).
    """
    if not (WHATSAPP_AVAILABLE and wa):
        return ("WhatsApp not configured", 503)

    # Optional signature validation (enable in production with TWILIO_VALIDATE_SIGNATURE=true)
    if os.environ.get("TWILIO_VALIDATE_SIGNATURE", "false").lower() == "true":
        sig = request.headers.get("X-Twilio-Signature", "")
        if not wa.validate_signature(request.url, request.form.to_dict(), sig):
            logger.warning("Rejected webhook with invalid Twilio signature")
            return ("Invalid signature", 403)

    details = wa.get_message_details(request.form)
    xml     = wa.twiml_empty()
    if not details:
        return app.response_class(xml, mimetype="text/xml")

    if _is_duplicate(details.get("id")):
        logger.info("Duplicate webhook ignored: %s", details.get("id"))
        return app.response_class(xml, mimetype="text/xml")

    phone = details.get("from")
    mtype = details.get("type")

    if mtype == "text":
        threading.Thread(target=process_text_message,
                         args=(details.get("text", ""), phone), daemon=True).start()
    elif mtype == "audio":
        threading.Thread(target=process_audio_message,
                         args=(details.get("audio_url"), phone), daemon=True).start()
    elif mtype in ("image", "document"):
        url = details.get("image_url") or details.get("document_url")
        threading.Thread(target=process_image_message,
                         args=(url, details.get("caption", ""), phone), daemon=True).start()

    return app.response_class(xml, mimetype="text/xml")


# ══════════════════════════════════════════════════════════════════════════════
# HEALTH
# ══════════════════════════════════════════════════════════════════════════════

@app.get("/health")
def health():
    article_count = 0
    if db:
        try:
            docs          = db.collection("iris_kb_articles").count().get()
            article_count = docs[0][0].value
        except Exception:
            pass
    return jsonify({
        "ok":          True,
        "service":     "Iris AI Service v2.0 (WhatsApp + Helpdesk)",
        "model":       GEMINI_MODEL,
        "gemini":      bool(_gemini_client),
        "assemblyai":  bool(ASSEMBLYAI_API_KEY),
        "firebase":    bool(db),
        "storage":     bool(get_bucket()),
        "whatsapp":    bool(WHATSAPP_AVAILABLE and wa),
        "kb_articles": article_count,
    })


# ══════════════════════════════════════════════════════════════════════════════
# ENTRYPOINT
# ══════════════════════════════════════════════════════════════════════════════

if __name__ == "__main__":
    port = int(os.environ.get("PORT", 7860))
    logger.info("Iris AI Service v1.1 starting on port %d (model=%s)", port, GEMINI_MODEL)
    app.run(host="0.0.0.0", port=port)