board/engine.py

# COMPLETE REPLACEMENT

"""
Board AI Engine - FRONTEND-OFFLOADED VERSION.
Backend only handles: GPT routing, GPT XML generation, Claude JSON processing.
Frontend handles: TTS, Icons, Page URLs.
"""

import re
import json
import time
import gevent
from gevent.lock import RLock
from config import GPT_URL, MAX_CHAT_HISTORY, GPT_TIMEOUT
from http_pool import gpt_session
from subjects.loader import subject_loader
from json_processor import BoardProcessor


class BoardEngine:
    """
    Board AI Engine - Lightweight version.
    TTS, Icons, Page URLs are handled by frontend.
    Backend only does: routing + XML generation + Claude JSON conversion.
    """

    def __init__(self):
        self.gpt_url = GPT_URL
        self.board_processor = BoardProcessor()

        # Per-user state
        self._user_sessions = {}
        self._user_locks = {}
        self._master_lock = RLock()

        gevent.spawn(self._periodic_lock_cleanup)

        print("✅ BoardEngine initialized (frontend-offloaded, per-user locks)")

    # ─── Per-user lock management ───

    def _get_user_lock(self, username):
        # Fast path: no master lock needed if already exists
        existing = self._user_locks.get(username)
        if existing:
            existing['last_access'] = time.time()
            return existing['lock']

        # Slow path: create new lock
        with self._master_lock:
            if username not in self._user_locks:
                self._user_locks[username] = {
                    'lock': RLock(),
                    'last_access': time.time()
                }
            return self._user_locks[username]['lock']

    def _periodic_lock_cleanup(self):
        while True:
            gevent.sleep(1800)
            try:
                now = time.time()
                cutoff = now - 1800
                with self._master_lock:
                    stale = [u for u, info in self._user_locks.items()
                             if info['last_access'] < cutoff
                             and u not in self._user_sessions]
                    for u in stale:
                        del self._user_locks[u]
                    if stale:
                        print(f"  🗑️ Cleaned {len(stale)} stale user locks")
            except Exception:
                pass

    # ─── User session management ───

    def _get_user_session(self, username):
        lock = self._get_user_lock(username)
        with lock:
            if username not in self._user_sessions:
                self._user_sessions[username] = {
                    "subject_id": None,
                    "conversation_history": [],
                    "last_sequence": [],
                }
            return self._user_sessions[username]

    def set_subject(self, username, subject_id):
        lock = self._get_user_lock(username)
        with lock:
            if username not in self._user_sessions:
                self._user_sessions[username] = {
                    "subject_id": None,
                    "conversation_history": [],
                    "last_sequence": [],
                }
            us = self._user_sessions[username]
            if us["subject_id"] and us["subject_id"] != subject_id:
                us["conversation_history"] = []
                us["last_sequence"] = []
                print(f"  🔄 Board subject switched: {us['subject_id']} → {subject_id} for {username}")
            us["subject_id"] = subject_id
        print(f"  📋 Board subject set: {subject_id} for {username}")

    def get_subject(self, username):
        lock = self._get_user_lock(username)
        with lock:
            us = self._user_sessions.get(username, {})
            return us.get("subject_id")

    # ─── GPT call ───

    def _call_gpt5(self, user_message, system_prompt, temperature=0.7, max_tokens=4000):
        payload = {
            "user_input": user_message,
            "chat_history": [
                {"role": "system", "content": system_prompt}
            ],
            "temperature": temperature,
            "top_p": 0.95,
            "max_completion_tokens": max_tokens
        }
        try:
            response = gpt_session.post(self.gpt_url, json=payload, timeout=GPT_TIMEOUT)
            response.raise_for_status()
            return response.json().get("assistant_response", "")
        except Exception as e:
            print(f"    ❌ GPT error: {e}")
            return None

    # ─── Chat history formatting ───

    def _format_chat_history(self, username):
        us = self._get_user_session(username)
        history = us.get("conversation_history", [])
        if not history:
            return ""

        recent = history[-10:]
        parts = []
        for msg in recent:
            if msg["role"] == "user":
                parts.append(f"الطالب: {msg['content']}")
            elif msg["role"] == "assistant":
                parts.append(f"المدرس: {msg['content']}")

        return (
            "\n\n═══ سجل المحادثة السابقة ═══\n"
            + "\n".join(parts)
            + "\n═══ نهاية السجل ═══"
        )

    # ─── Routing cache for continuation messages ───

    _CONTINUATION_WORDS = [
        'أكمل', 'استمر', 'وضح', 'اشرح أكثر', 'مثال', 'أعطني مثال',
        'كمل', 'زيد', 'وضح أكثر', 'فصل أكثر', 'بالتفصيل',
        'continue', 'more', 'explain', 'go on', 'example',
        'طيب', 'تمام', 'اها', 'ثم', 'وبعدين', 'ايش بعد',
        'ok', 'yes', 'نعم', 'اي', 'صح',
    ]

    def _is_continuation_message(self, user_message, username):
        """Check if message is a simple continuation that doesn't need routing."""
        msg_lower = user_message.strip().lower()

        # Short messages that are continuations
        if len(msg_lower) < 25:
            for word in self._CONTINUATION_WORDS:
                if word in msg_lower:
                    return True

        # Check if user has recent history with a file
        us = self._get_user_session(username)
        history = us.get("conversation_history", [])
        if len(history) >= 2:
            # Had recent conversation, short follow-up
            if len(msg_lower) < 15:
                return True

        return False

    # ─── Step 1: Route message ───

    def _route_message(self, user_message, username):
        us = self._get_user_session(username)
        subject_id = us["subject_id"]
        if not subject_id:
            return "main.txt"

        # Check continuation cache first
        last_file = us.get("last_routed_file")
        if last_file and self._is_continuation_message(user_message, username):
            print(f"  ⚡ Routing cache hit: continuation → {last_file}")
            return last_file

        subject_data = subject_loader.load(subject_id)
        if not subject_data:
            return "main.txt"

        structure = subject_data.get("structure.txt", "")
        p_files = subject_data.get("_p_files", [])
        chat_history_text = self._format_chat_history(username)

        p_files_desc = "\n".join([f"- {f}: الفصل {i+1}" for i, f in enumerate(p_files)])

        routing_prompt = f"""أنت نظام توجيه ذكي لمساعد تعليمي.

مهمتك: تحليل رسالة الطالب واختيار الملف المناسب للرد.

الملفات المتاحة:
- main.txt: للتحيات، الأسئلة العامة، أي شيء لا يتعلق بفصل محدد
{p_files_desc}

فهرس الكتاب (للمساعدة في التوجيه):
{structure}
{chat_history_text}

تعليمات:
1. إذا كانت الرسالة تحية أو سؤال عام → main.txt
2. إذا كانت تسأل عن موضوع في فصل محدد → الملف المناسب
3. استخدم الفهرس لتحديد الفصل الصحيح
4. مهم جداً: إذا قال الطالب "اشرح أكثر" أو "وضح" أو أي طلب متابعة، ارجع لسجل المحادثة لتعرف الموضوع الحالي واختر نفس الملف
5. أجب فقط باسم الملف (مثال: p1.txt أو main.txt) بدون أي كلام إضافي

رسالة الطالب: {user_message}

الملف المناسب:"""

        chosen = self._call_gpt5(
            user_message, routing_prompt,
            temperature=0.2, max_tokens=50
        )

        if not chosen:
            return last_file if last_file else "main.txt"

        chosen = chosen.strip().lower()
        valid_files = ["main.txt"] + p_files

        for v in valid_files:
            if v in chosen:
                # Save for continuation cache
                lock = self._get_user_lock(username)
                with lock:
                    if username in self._user_sessions:
                        self._user_sessions[username]["last_routed_file"] = v
                return v

        return last_file if last_file else "main.txt"

    # ─── Step 2: Generate XML response ───

    def _generate_xml_response(self, user_message, chosen_file, username):
        us = self._get_user_session(username)
        subject_id = us["subject_id"]
        if not subject_id:
            return None

        subject_data = subject_loader.load(subject_id)
        if not subject_data:
            return None

        file_content = subject_data.get(chosen_file, "")
        chat_history_text = self._format_chat_history(username)

        # Get pages_base_url for the subject so GPT knows page references are available
        pages_base_url = subject_data.get("pages_base_url", "")
        page_instruction = ""
        if pages_base_url:
            page_instruction = """
• <page>رقم_الصفحة</page>
  لعرض صفحة محددة من الكتاب كصورة على السبورة
  مثال: <page>12</page> لعرض الصفحة 12 من الكتاب
  استخدمها عندما تحتاج تعرض للطالب صفحة معينة من الكتاب"""

        system_prompt = f"""انتي مدرسة خبيرة ومحترفة في هذه المادة. تشرح على سبورة تفاعلية رقمية.

═══ صيغة الرد ═══

يجب أن تردي بصيغة XML خاصة تحتوي على:

1. <board>عناصر السبورة</board> - ما سيُرسم/يُضاف على السبورة أولاً
2. <voice>نص الكلام</voice> - النص الذي سيُقرأ بصوت عالٍ للطالب بعد رسم العناصر (عربي طبيعي)

═══ عناصر السبورة المتاحة (داخل <board>) ═══

• <note>محتوى الملاحظة</note>

• <text>نص مباشر على السبورة بدون خلفية</text>

• <shape type="نوع_الشكل"/>
  الأنواع: circle, triangle, star, arrow-right, arrow-left, arrow-up, arrow-down,
  rectangle, diamond, hexagon, square, oval, arrow-double-h, checkmark, cross,
  heart, cloud, lightning, speech, process, decision

• <svg>كلمة_بحث_بالإنجليزية</svg>
  سيتم البحث عن أيقونة مرسومة يدوياً (مثل: ball, car, force, spring, weight, rope, pulley)
{page_instruction}

═══ قواعد مهمة جداً ═══

1. اشرح خطوة بخطوة: ابدأ بـ <board> ثم <voice> ثم <board> ثم <voice> وهكذا
2. اجعل الشرح متدرجاً كأنك تشرح على سبورة حقيقية أمام الطلاب
3. <board> = ما يظهر على السبورة أولاً (ملاحظات، نصوص، أشكال، صور، صفحات الكتاب)
4. <voice> = الكلام المسموع بعد رسم العناصر (طبيعي، ودود، واضح، يشرح ما تم رسمه)
5. لا تضع كل شيء دفعة واحدة - اجعله تسلسلياً
7. <svg> فقط بكلمات إنجليزية بسيطة ومعبرة
8. السبورة تعمل بنظام الإضافة - العناصر السابقة تبقى
9. استخدم <text> للعناوين والمعادلات المهمة (بدون خلفية)
10. استخدم <note> للتوضيحات والملاحظات (مع خلفية ملونة)
11. لا تستخدم أكثر من 3-4 عناصر في كل <board>
12. اجعل النص في <voice> طبيعياً كأنك تتحدث مع طالب ويشرح ما تم رسمه على السبورة
13. ارسم أولاً ثم تكلم - هذا مهم جداً!
14. راجع سجل المحادثة السابقة لتعرف ما تم شرحه وتكمل من حيث توقفت - لا تكرر ما قلته سابقاً
15. استخدم <page> عندما تريد عرض صفحة من الكتاب - مثلاً إذا الطالب سأل عن تمرين أو شكل في صفحة معينة

when user talk about something not about the subject or something funny etc... you can actually answer without the board just VOICE and be funny smart perfect girl also:
when you explain something dont make all your explain on the NOTE make the note for important point use the TEXT direct on the board and the ICONS/SHAPES

═══ محتوى المادة ═══
{file_content}
{chat_history_text}

═══ الآن أجب على سؤال الطالب ═══

رسالة الطالب: {user_message}"""

        response = self._call_gpt5(
            user_message, system_prompt,
            temperature=0.8, max_tokens=4000
        )
        return response

    # ─── Parse XML into raw segments ───

    def _parse_xml_to_raw_segments(self, xml_response):
        if not xml_response:
            return []

        pattern = r'<(voice|board)>(.*?)</\1>'
        matches = list(re.finditer(pattern, xml_response, re.DOTALL))

        if not matches:
            cleaned = re.sub(r'<[^>]+>', '', xml_response).strip()
            if cleaned:
                return [{"boards": [], "voice": cleaned}]
            return []

        groups = []
        current_boards = []

        for match in matches:
            tag_type = match.group(1)
            content = match.group(2).strip()

            if tag_type == "board":
                current_boards.append(content)
            elif tag_type == "voice":
                cleaned_voice = re.sub(r'<[^>]+>', '', content).strip()
                groups.append({
                    "boards": list(current_boards),
                    "voice": cleaned_voice
                })
                current_boards = []

        if current_boards:
            groups.append({
                "boards": list(current_boards),
                "voice": ""
            })

        return groups

    # ─── Extract frontend tasks from board content ───

    def _extract_frontend_tasks(self, board_content):
        """
        Pull out <svg>, <page> tags from board content.
        Return the tasks for frontend and cleaned board content.
        """
        tasks = {
            "icons": [],
            "pages": []
        }

        # Extract <svg> tags
        svg_pattern = r'<svg>(.*?)</svg>'
        svg_matches = re.finditer(svg_pattern, board_content, re.DOTALL)
        for match in svg_matches:
            keyword = match.group(1).strip()
            if keyword:
                tasks["icons"].append({
                    "keyword": keyword,
                    "placeholder_id": f"icon_{keyword}_{id(match)}"
                })

        # Extract <page> tags
        page_pattern = r'<page>\s*(.*?)\s*</page>'
        page_matches = re.finditer(page_pattern, board_content, re.DOTALL)
        for match in page_matches:
            page_num = match.group(1).strip()
            if page_num:
                tasks["pages"].append({
                    "page_number": page_num,
                    "placeholder_id": f"page_{page_num}_{id(match)}"
                })

        return tasks

    # ─── Build board state summary for Claude (optimization) ───

    def _build_board_summary(self, current_board_state):
        """
        Instead of sending ALL items to Claude, send summary + recent items.
        """
        total_items = len(current_board_state)

        if total_items <= 15:
            # Small board, send everything
            return json.dumps(current_board_state, ensure_ascii=False, indent=2)

        # Large board: summary + last 15 items
        recent_items = current_board_state[-15:]

        # Calculate bounding box of older items
        older_items = current_board_state[:-15]
        max_x = 0
        max_y = 0
        for item in older_items:
            x = item.get('x', 0) + item.get('w', 0)
            y = item.get('y', 0) + item.get('h', 0)
            if x > max_x:
                max_x = x
            if y > max_y:
                max_y = y

        summary = {
            "total_items_on_board": total_items,
            "older_items_count": len(older_items),
            "older_items_occupied_area": {
                "max_x": max_x,
                "max_y": max_y
            },
            "recent_items": recent_items
        }

        return json.dumps(summary, ensure_ascii=False, indent=2)

    # ─── Process a single board content into items ───

    def _process_single_board(self, board_content, current_board_state):
        board_summary = self._build_board_summary(current_board_state)

        processor_input = (
            f"BOARD NOW (make sure no X Y error):\n"
            f"{board_summary}\n\n"
            f"new board need to add :\n"
            f"<board>{board_content}</board>"
        )

        print(f"    🔧 Sending to json_processor...")
        print(f"       Current board items: {len(current_board_state)}")

        try:
            json_text = self.board_processor.convert_xml_to_json(processor_input)

            if json_text:
                new_items = json.loads(json_text)

                if isinstance(new_items, list) and new_items:
                    added_items = []
                    existing_ids = set()
                    for item in current_board_state:
                        item_key = json.dumps(item, sort_keys=True, ensure_ascii=False)
                        existing_ids.add(item_key)

                    for item in new_items:
                        item_key = json.dumps(item, sort_keys=True, ensure_ascii=False)
                        if item_key not in existing_ids:
                            added_items.append(item)

                    current_board_state.extend(added_items)
                    print(f"    ✅ json_processor: {len(new_items)} total, {len(added_items)} new")
                    return added_items, current_board_state

                elif isinstance(new_items, dict):
                    current_board_state.append(new_items)
                    print(f"    ✅ json_processor: 1 item")
                    return [new_items], current_board_state

                else:
                    print(f"    ⚠️ json_processor: unexpected format")
                    return [], current_board_state

        except json.JSONDecodeError as e:
            print(f"    ❌ json_processor invalid JSON: {e}")
            return [], current_board_state
        except Exception as e:
            print(f"    ❌ json_processor error: {e}")
            return [], current_board_state

        return [], current_board_state

    # ─── STREAMING PIPELINE ───

    def process_message_stream(self, user_message, username, frontend_board_state=None):
        """
        Stream board response.
        Backend: GPT routing + XML generation + Claude JSON.
        Frontend tasks (TTS, icons, pages) sent as metadata for browser to handle.
        """
        us = self._get_user_session(username)
        subject_id = us.get("subject_id")

        print(f"\n{'═' * 60}")
        print(f"  👤 Student ({username}): {user_message}")
        print(f"  📚 Subject: {subject_id}")
        print(f"{'═' * 60}")

        if not subject_id:
            yield json.dumps({
                "type": "error",
                "message": "لم يتم تحديد المادة للسبورة"
            }, ensure_ascii=False)
            return

        if frontend_board_state is None:
            frontend_board_state = []

        current_board_state = list(frontend_board_state)

        print(f"  📋 Board state from frontend: {len(current_board_state)} items")

        # Get pages_base_url for this subject (frontend needs it)
        subject_data = subject_loader.load(subject_id)
        pages_base_url = ""
        if subject_data:
            pages_base_url = subject_data.get("pages_base_url", "")

        # Step 1: Route
        print("\n  📍 Step 1: Routing message...")
        chosen_file = self._route_message(user_message, username)
        print(f"  📂 Chosen file: {chosen_file}")

        # Step 2: Generate XML
        print(f"  🤖 Step 2: Generating XML response...")
        xml_response = self._generate_xml_response(user_message, chosen_file, username)

        if not xml_response:
            print("  ❌ Failed to generate response")
            yield json.dumps({
                "type": "error",
                "message": "عذراً، حدث خطأ في النظام. حاول مرة أخرى."
            }, ensure_ascii=False)
            return

        print(f"  📝 XML response: {len(xml_response)} chars")

        # Step 3: Parse XML into segment groups
        print("  🔧 Step 3: Parsing XML into segment groups...")
        raw_groups = self._parse_xml_to_raw_segments(xml_response)
        total_groups = len(raw_groups)
        print(f"  📊 Found {total_groups} segment groups to stream")

        if total_groups == 0:
            yield json.dumps({
                "type": "error",
                "message": "لم يتم توليد محتوى للسبورة."
            }, ensure_ascii=False)
            return

        # Step 4: Send PREVIEW immediately (frontend tasks + raw content)
        # This lets frontend start TTS/icons BEFORE board JSON is ready

        all_voice_texts = []
        all_frontend_tasks = {
            "icons": [],
            "pages": [],
            "voices": [],
            "pages_base_url": pages_base_url
        }

        # Collect ALL frontend tasks from ALL segments upfront
        for idx, group in enumerate(raw_groups):
            voice_text = group.get("voice", "")
            if voice_text:
                all_voice_texts.append(voice_text)
                all_frontend_tasks["voices"].append({
                    "segment_index": idx,
                    "text": voice_text
                })

            for board_content in group["boards"]:
                tasks = self._extract_frontend_tasks(board_content)
                for icon_task in tasks["icons"]:
                    icon_task["segment_index"] = idx
                    all_frontend_tasks["icons"].append(icon_task)
                for page_task in tasks["pages"]:
                    page_task["segment_index"] = idx
                    all_frontend_tasks["pages"].append(page_task)

        # Send preview event: frontend starts working immediately
        preview_data = {
            "type": "preview",
            "total_segments": total_groups,
            "frontend_tasks": all_frontend_tasks,
            "chosen_file": chosen_file
        }

        print(f"  📤 Sending preview: {len(all_frontend_tasks['voices'])} voices, "
              f"{len(all_frontend_tasks['icons'])} icons, "
              f"{len(all_frontend_tasks['pages'])} pages")

        yield json.dumps(preview_data, ensure_ascii=False)

        # Step 5: Process each segment through Claude (board JSON only)
        all_segments_for_replay = []

        for idx, group in enumerate(raw_groups):
            print(f"\n  ── Segment {idx + 1}/{total_groups} ──")

            segment_board_items = []

            for board_content in group["boards"]:
                new_items, current_board_state = self._process_single_board(
                    board_content, current_board_state
                )
                segment_board_items.extend(new_items)

            voice_text = group.get("voice", "")

            segment_data = {
                "type": "segment",
                "index": idx,
                "total_segments": total_groups,
                "board_items": segment_board_items,
                "voice_text": voice_text,
                # NO audio_url - frontend handles TTS
                # Icons and pages already sent in preview
            }

            all_segments_for_replay.append(segment_data)

            print(f"    ✅ Segment {idx + 1} ready: {len(segment_board_items)} board items")

            yield json.dumps(segment_data, ensure_ascii=False)

        # Step 6: Save chat history
        lock = self._get_user_lock(username)
        with lock:
            us = self._user_sessions.get(username, {})
            if "conversation_history" not in us:
                us["conversation_history"] = []

            us["conversation_history"].append({
                "role": "user",
                "content": user_message
            })

            assistant_text = " ".join(all_voice_texts)
            if assistant_text:
                us["conversation_history"].append({
                    "role": "assistant",
                    "content": assistant_text
                })

            if len(us["conversation_history"]) > MAX_CHAT_HISTORY:
                us["conversation_history"] = us["conversation_history"][-MAX_CHAT_HISTORY:]

            us["last_sequence"] = all_segments_for_replay

        # Done signal
        yield json.dumps({
            "type": "done",
            "board_state": current_board_state,
            "chosen_file": chosen_file,
            "total_segments": total_groups
        }, ensure_ascii=False)

        print(f"\n  ✅ All {total_groups} segments streamed!")
        print(f"     Board items: {len(current_board_state)}")
        print(f"{'═' * 60}\n")

    # ─── NON-STREAMING (compatibility) ───

    def process_message(self, user_message, username, frontend_board_state=None):
        all_segments = []
        final_board_state = frontend_board_state or []
        chosen_file = None
        frontend_tasks = None

        for chunk_str in self.process_message_stream(user_message, username, frontend_board_state):
            chunk = json.loads(chunk_str)

            if chunk["type"] == "preview":
                frontend_tasks = chunk.get("frontend_tasks")

            elif chunk["type"] == "segment":
                if chunk.get("board_items"):
                    all_segments.append({
                        "type": "board_update",
                        "action": "add",
                        "items": chunk["board_items"]
                    })
                if chunk.get("voice_text"):
                    all_segments.append({
                        "type": "voice",
                        "text": chunk["voice_text"],
                        "audio_url": None  # Frontend handles TTS
                    })

            elif chunk["type"] == "done":
                final_board_state = chunk.get("board_state", [])
                chosen_file = chunk.get("chosen_file")

            elif chunk["type"] == "error":
                return {
                    "success": False,
                    "error": chunk["message"],
                    "sequence": [{
                        "type": "voice",
                        "text": chunk["message"],
                        "audio_url": None
                    }],
                    "board_state": frontend_board_state or []
                }

        return {
            "success": True,
            "chosen_file": chosen_file,
            "sequence": all_segments,
            "board_state": final_board_state,
            "frontend_tasks": frontend_tasks
        }

    # ─── Replay ───

    def get_replay_sequence(self, username):
        lock = self._get_user_lock(username)
        with lock:
            us = self._user_sessions.get(username, {})
            last_seq = us.get("last_sequence", [])

        if not last_seq:
            return {
                "success": False,
                "error": "No previous response to replay",
                "sequence": []
            }

        voice_only = []
        for item in last_seq:
            if item.get("voice_text"):
                voice_only.append({
                    "type": "voice",
                    "text": item.get("voice_text", ""),
                    # No audio_url - frontend regenerates TTS
                })

        print(f"  🔄 Replay: {len(voice_only)} voice segments")

        return {
            "success": True,
            "sequence": voice_only
        }

    # ─── Clear ───

    def clear_board(self, username):
        lock = self._get_user_lock(username)
        with lock:
            if username in self._user_sessions:
                self._user_sessions[username]["last_sequence"] = []
        print(f"  🗑️ Board cleared for {username}")
        return {"success": True, "board_state": []}

    def clear_chat_history(self, username):
        lock = self._get_user_lock(username)
        with lock:
            if username in self._user_sessions:
                self._user_sessions[username]["conversation_history"] = []
                self._user_sessions[username]["last_sequence"] = []
                self._user_sessions[username]["last_routed_file"] = None
        print(f"  🗑️ Board chat history cleared for {username}")
        return {"success": True}

    def clear_user_session(self, username):
        lock = self._get_user_lock(username)
        with lock:
            if username in self._user_sessions:
                del self._user_sessions[username]
                print(f"  🗑️ Full board session cleared for {username}")
        return {"success": True}