Update main.py

main.py

@@ -1,1191 +1,550 @@
+"""
+Marka Data / Infra API Server
+Serves syllabus structure trees and semantic search over syllabus content.
+This is NOT the app server. Do not add auth, quiz, or user endpoints here.
+"""
+
 import os
 import json
 import logging
-import re
-import time
-import threading
-import base64
-from typing import Dict, List, Tuple, Optional
-
-import numpy as np
-import fitz  # PyMuPDF
+import uuid
+from datetime import datetime
+
 from flask import Flask, request, jsonify
 from flask_cors import CORS
-from google import genai
-from sklearn.metrics.pairwise import cosine_similarity
 
 import firebase_admin
-from firebase_admin import credentials, db as firebase_db
+from firebase_admin import credentials, db
 
-# ---------------------------------------------------------------------------
-# CONFIGURATION
-# ---------------------------------------------------------------------------
-logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
-logger = logging.getLogger(__name__)
+import requests
 
-SYLLABI_DIR    = os.environ.get("SYLLABI_DIR", "syllabi")
-PAST_EXAMS_DIR = os.environ.get("PAST_EXAMS_DIR", "past_exams")
-
-# Support both naming conventions used on HuggingFace / local envs.
-GEMINI_API_KEY  = os.environ.get("GEMINI_API_KEY") or os.environ.get("Gemini")
-EMBEDDING_MODEL = os.environ.get("EMBEDDING_MODEL", "models/text-embedding-004")
-
-# Requested model bump. Keep env override so you can hotfix without code edits.
-VISION_MODEL    = os.environ.get("GEMINI_MODEL", "gemini-3.1-flash-lite")
-
-# Parser controls
-MAX_VISION_PAGES      = int(os.environ.get("MAX_VISION_PAGES", "80"))
-PAGE_CLASSIFY_DPI     = int(os.environ.get("PAGE_CLASSIFY_DPI", "72"))
-MIN_VECTOR_TEXT_CHARS = int(os.environ.get("MIN_VECTOR_TEXT_CHARS", "20"))
-
-# ---------------------------------------------------------------------------
-# COMPLETE SUBJECT REGISTRY  (all 24 PDFs on HuggingFace)
-# ---------------------------------------------------------------------------
-A_LEVEL_SUBJECTS = {
-    "A_9706": "Accounting",
-    "A_9700": "Biology",
-    "A_9609": "Business",
-    "A_9701": "Chemistry",
-    "A_9618": "Computer Science",
-    "A_9708": "Economics",
-    "A_9231": "Further Mathematics",
-    "A_9489": "History",
-    "A_9695": "Literature in English",
-    "A_9709": "Mathematics",
-    "A_9702": "Physics",
-    "A_9699": "Sociology",
-    "A_9395": "Travel and Tourism",
-}
-O_LEVEL_SUBJECTS = {
-    "O_0452": "Accounting",
-    "O_0610": "Biology",
-    "O_0450": "Business Studies",
-    "O_0620": "Chemistry",
-    "O_0478": "Computer Science",
-    "O_0500": "English Language",
-    "O_0475": "English Literature",
-    "O_0680": "Environmental Management",
-    "O_0460": "Geography",
-    "O_0470": "History",
-    "O_0625": "Physics",
-}
-ALL_SUBJECTS = {**A_LEVEL_SUBJECTS, **O_LEVEL_SUBJECTS}
-CODE_TO_LEVEL_SUBJECT: Dict[str, List[Tuple[str, str, str]]] = {}
-for sid, subject in ALL_SUBJECTS.items():
-    level, code = sid.split("_", 1)
-    CODE_TO_LEVEL_SUBJECT.setdefault(code, []).append((level, sid, subject))
-
-# ---------------------------------------------------------------------------
-# GLOBAL STATE
-# ---------------------------------------------------------------------------
-SYLLABUS_MAP: Dict = {}
-VECTOR_DB: List[Dict] = []
-VECTOR_MATRIX = None
-EXAM_MAP: Dict = {}
+from google import genai
+
+# -----------------------------------------------------------------------------
+# 1. CONFIGURATION & INITIALIZATION
+# -----------------------------------------------------------------------------
 
 app = Flask(__name__)
 CORS(app)
 
-# ---------------------------------------------------------------------------
-# FIREBASE
-# ---------------------------------------------------------------------------
-firebase_db_ref    = None
-FIREBASE_AVAILABLE = False
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
 
-def init_firebase():
-    global firebase_db_ref, FIREBASE_AVAILABLE
-    try:
-        creds_str = os.environ.get("FIREBASE")
-        db_url    = os.environ.get("Firebase_DB") or os.environ.get("FIREBASE_DB")
-        if not creds_str or not db_url:
-            logger.warning("Firebase env vars missing. Running without Firebase persistence.")
-            return False
-        if not firebase_admin._apps:
-            cred = credentials.Certificate(json.loads(creds_str))
-            firebase_admin.initialize_app(cred, {"databaseURL": db_url})
-        firebase_db_ref    = firebase_db.reference()
-        FIREBASE_AVAILABLE = True
-        logger.info("Firebase initialised (Data API).")
-        return True
-    except Exception as e:
-        logger.error(f"Firebase init failed: {e}")
-        return False
+# --- Firebase ---
+try:
+    credentials_json_string = os.environ.get("FIREBASE")
+    if not credentials_json_string:
+        raise ValueError("FIREBASE env var not set.")
+    credentials_json = json.loads(credentials_json_string)
+    firebase_db_url  = os.environ.get("Firebase_DB")
+    if not firebase_db_url:
+        raise ValueError("Firebase_DB must be set.")
+    cred = credentials.Certificate(credentials_json)
+    firebase_admin.initialize_app(cred, {"databaseURL": firebase_db_url})
+    logger.info("Firebase Admin SDK initialised.")
+except Exception as e:
+    logger.error(f"FATAL Firebase init: {e}")
+    raise SystemExit(1)
+
+db_ref = db.reference()
+
+# --- Gemini (for embedding-based search) ---
+try:
+    api_key = os.environ.get("Gemini")
+    if not api_key:
+        raise ValueError("Gemini env var not set.")
+    client = genai.Client(api_key=api_key)
+    logger.info("GenAI client initialised.")
+except Exception as e:
+    logger.error(f"FATAL GenAI init: {e}")
+    raise SystemExit(1)
+
+EMBED_MODEL = os.environ.get("GEMINI_EMBED_MODEL", "text-embedding-004")
+TEXT_MODEL  = os.environ.get("GEMINI_TEXT_MODEL", "gemini-3.1-flash-lite")
+
+# -----------------------------------------------------------------------------
+# 2. SYLLABUS SUBJECT REGISTRY
+# Maps remote_id (used in RTDB paths) to human-readable metadata
+# -----------------------------------------------------------------------------
+
+SUBJECT_REGISTRY = {
+    # A-Level
+    "A_9706": {"name": "Accounting",              "level": "A", "code": "9706"},
+    "A_9700": {"name": "Biology",                 "level": "A", "code": "9700"},
+    "A_9609": {"name": "Business",                "level": "A", "code": "9609"},
+    "A_9701": {"name": "Chemistry",               "level": "A", "code": "9701"},
+    "A_9618": {"name": "Computer Science",        "level": "A", "code": "9618"},
+    "A_9708": {"name": "Economics",               "level": "A", "code": "9708"},
+    "A_9231": {"name": "Further Mathematics",     "level": "A", "code": "9231"},
+    "A_9489": {"name": "History",                 "level": "A", "code": "9489"},
+    "A_9695": {"name": "Literature in English",   "level": "A", "code": "9695"},
+    "A_9709": {"name": "Mathematics",             "level": "A", "code": "9709"},
+    "A_9702": {"name": "Physics",                 "level": "A", "code": "9702"},
+    "A_9699": {"name": "Sociology",               "level": "A", "code": "9699"},
+    "A_9395": {"name": "Travel & Tourism",        "level": "A", "code": "9395"},
+    # O-Level / IGCSE
+    "O_0452": {"name": "Accounting",              "level": "O", "code": "0452"},
+    "O_0610": {"name": "Biology",                 "level": "O", "code": "0610"},
+    "O_0450": {"name": "Business Studies",        "level": "O", "code": "0450"},
+    "O_0620": {"name": "Chemistry",               "level": "O", "code": "0620"},
+    "O_0478": {"name": "Computer Science",        "level": "O", "code": "0478"},
+    "O_0500": {"name": "English Language",        "level": "O", "code": "0500"},
+    "O_0475": {"name": "English Literature",      "level": "O", "code": "0475"},
+    "O_0680": {"name": "Environmental Management","level": "O", "code": "0680"},
+    "O_0460": {"name": "Geography",               "level": "O", "code": "0460"},
+    "O_0470": {"name": "History",                 "level": "O", "code": "0470"},
+    "O_0625": {"name": "Physics",                 "level": "O", "code": "0625"},
+}
 
-FIREBASE_AVAILABLE = init_firebase()
+# -----------------------------------------------------------------------------
+# 3. HELPERS
+# -----------------------------------------------------------------------------
+
+def _get_syllabus_ref(remote_id: str):
+    """Return Firebase RTDB reference for a syllabus tree."""
+    return db_ref.child(f"syllabi/{remote_id}")
 
-def fb_set(path, data):
-    if not FIREBASE_AVAILABLE:
-        return
-    try:
-        firebase_db_ref.child(path).set(data)
-    except Exception as e:
-        logger.error(f"FB write [{path}]: {e}")
 
-def fb_get(path):
-    if not FIREBASE_AVAILABLE:
-        return None
+def _get_chunks_ref(remote_id: str):
+    """Return Firebase RTDB reference for syllabus content chunks."""
+    return db_ref.child(f"syllabus_chunks/{remote_id}")
+
+
+def _embed_text(text: str) -> list:
+    """Generate a text embedding using Gemini."""
     try:
-        return firebase_db_ref.child(path).get()
+        result = client.models.embed_content(
+            model=EMBED_MODEL,
+            contents=text,
+        )
+        return result.embeddings[0].values
     except Exception as e:
-        logger.error(f"FB read [{path}]: {e}")
-        return None
+        logger.error(f"Embed error: {e}")
+        return []
 
-# ---------------------------------------------------------------------------
-# GEMINI CLIENT
-# ---------------------------------------------------------------------------
-_gemini_client = None
 
-def get_gemini():
-    global _gemini_client
-    if _gemini_client is None and GEMINI_API_KEY:
-        _gemini_client = genai.Client(api_key=GEMINI_API_KEY)
-    return _gemini_client
+def _cosine_similarity(a: list, b: list) -> float:
+    """Compute cosine similarity between two vectors."""
+    if not a or not b or len(a) != len(b):
+        return 0.0
+    dot   = sum(x * y for x, y in zip(a, b))
+    mag_a = sum(x * x for x in a) ** 0.5
+    mag_b = sum(x * x for x in b) ** 0.5
+    if mag_a == 0 or mag_b == 0:
+        return 0.0
+    return dot / (mag_a * mag_b)
 
-# ---------------------------------------------------------------------------
-# PDF / SYLLABUS DETECTION HELPERS
-# ---------------------------------------------------------------------------
 
-def clean_space(text: str) -> str:
-    return re.sub(r"\s+", " ", text or "").strip()
+def _keyword_score(query: str, text: str) -> float:
+    """Simple keyword overlap fallback score."""
+    query_words = set(query.lower().split())
+    text_words  = set(text.lower().split())
+    if not query_words:
+        return 0.0
+    return len(query_words & text_words) / len(query_words)
 
-def normalise_line(line: str) -> str:
-    line = clean_space(line)
-    line = re.sub(r"^[\d\.\s]+", "", line).strip()
-    return line
 
-def safe_firebase_key(value: str) -> str:
-    return re.sub(r'[.\[\]#$/]', '_', value)
+# -----------------------------------------------------------------------------
+# 4. HEALTH
+# -----------------------------------------------------------------------------
 
-def infer_level_code_subject(filepath: str, filename: str) -> Tuple[str, str, str, str, bool]:
-    """
-    Returns: level, code, unique_id, subject_name, registry_match.
-    Fixes the old weak behaviour where parent folder was trusted blindly.
-    """
-    parts = filepath.replace("\\", "/").split("/")
-    parent = parts[-2].upper() if len(parts) >= 2 else ""
-
-    code_m = re.search(r'(?<!\d)(\d{4})(?!\d)', filename)
-    code = code_m.group(1) if code_m else "0000"
-
-    candidates = CODE_TO_LEVEL_SUBJECT.get(code, [])
-
-    if parent in {"A", "O"}:
-        preferred = [c for c in candidates if c[0] == parent]
-        if preferred:
-            level, uid, subject = preferred[0]
-            return level, code, uid, subject, True
-        # Parent says A/O but code is not in registry. Still preserve parent.
-        level = parent
-    elif len(candidates) == 1:
-        level, uid, subject = candidates[0]
-        return level, code, uid, subject, True
-    elif len(candidates) > 1:
-        # There are no overlaps in the current registry, but this keeps future safety.
-        level, uid, subject = candidates[0]
-        return level, code, uid, subject, True
-    else:
-        level = parent if parent in {"A", "O"} else "General"
-
-    uid = f"{level}_{code}"
-    fallback_subject = re.sub(r'[_\-]?\d{4}.*', '', filename, flags=re.IGNORECASE)
-    fallback_subject = fallback_subject.replace('_', ' ').replace('-', ' ').strip() or "Unknown Subject"
-    return level, code, uid, fallback_subject, False
-
-# Hard boilerplate headings. These are pages/blocks we do NOT want in the output tree.
-BOILERPLATE_HEADING_RE = re.compile(
-    r"^(about\s+this\s+syllabus|foreword|introduction|acknowledgements?|"
-    r"why\s+choose\s+(cambridge|zimsec|this\s+syllabus)|cambridge\s+learner|"
-    r"key\s+benefits?|how\s+to\s+use\s+this\s+syllabus|"
-    r"support\s+for\s+(cambridge|teachers)|resource\s+list|"
-    r"further\s+information|copyright|legal\s+notice|"
-    r"changes\s+to\s+this\s+syllabus|university\s+of\s+cambridge|"
-    r"cambridge\s+assessment\s+international|published\s+by|"
-    r"contents?|table\s+of\s+contents|"
-    r"assessment\s+at\s+a\s+glance|syllabus\s+at\s+a\s+glance|"
-    r"assessment\s+overview|scheme\s+of\s+assessment|"
-    r"assessment\s+objectives?|grade\s+descriptions?|command\s+words|"
-    r"glossary(\s+of\s+command\s+words)?|mathematical\s+notation|"
-    r"other\s+cambridge\s+qualifications|how\s+to\s+offer|progression|"
-    r"post[-\s]?qualification|school\s+supported\s+candidate|"
-    r"cambridge\s+primary|cambridge\s+lower\s+secondary|"
-    r"what\s+else\s+you\s+need\s+to\s+know|before\s+you\s+start|"
-    r"making\s+entries|after\s+the\s+exam|appendix|appendices)\b",
-    re.IGNORECASE,
-)
-
-# Strong signals for the actual learning/course content section.
-CONTENT_START_RE = re.compile(
-    r"(^|\n)\s*((\d+\.?\s*)?(subject\s+content|syllabus\s+content|curriculum\s+content|"
-    r"content\s+overview|learning\s+content|topics?\s+and\s+content|"
-    r"knowledge\s+and\s+understanding|set\s+texts|subject\s+specific\s+skills))\b",
-    re.IGNORECASE,
-)
-
-# Some syllabi have content pages made of topic numbers rather than a clean "Subject content" heading.
-STRONG_TOPIC_RE = re.compile(
-    r"(^|\n)\s*((\d+(\.\d+)*\s+)[A-Z][A-Za-z0-9,()/:\- ]{3,}|"
-    r"(Unit|Topic|Section|Module)\s+\d+\b|"
-    r"Candidates\s+should\s+be\s+able\s+to|Learning\s+objectives?)",
-    re.IGNORECASE,
-)
-
-CONTENT_END_RE = re.compile(
-    r"(^|\n)\s*((\d+\.?\s*)?(details\s+of\s+the\s+assessment|assessment\s+details|"
-    r"assessment\s+objectives?|assessment\s+criteria|scheme\s+of\s+assessment|"
-    r"command\s+words|glossary|grade\s+descriptions?|mathematical\s+notation|"
-    r"appendix|appendices|what\s+else\s+you\s+need\s+to\s+know|"
-    r"administrative\s+information|additional\s+information))\b",
-    re.IGNORECASE,
-)
-
-RESIDUAL_SKIP_RE = re.compile(
-    r"\b(Cambridge International|UCLES|Version\s+\d|Back to contents page|"
-    r"www\.cambridgeinternational\.org|For examination in|Syllabus for examination|"
-    r"Copyright|©|ISBN|Assessment at a glance|Why choose|About this syllabus)\b",
-    re.IGNORECASE,
-)
-
-PAGE_NUMBER_RE = re.compile(r"^(page\s*)?\d{1,3}\s*$", re.IGNORECASE)
-
-
-def get_page_lines(page, max_lines: int = 80) -> List[str]:
-    text = page.get_text("text") or ""
-    lines = [clean_space(x) for x in text.splitlines()]
-    return [x for x in lines if x][:max_lines]
-
-
-def extract_heading_candidates(doc, max_pages: int = 12) -> List[str]:
-    headings: List[str] = []
-    for i, page in enumerate(doc):
-        if i >= max_pages:
-            break
-        try:
-            page_dict = page.get_text("dict")
-            spans = []
-            for b in page_dict.get("blocks", []):
-                for l in b.get("lines", []):
-                    for s in l.get("spans", []):
-                        text = clean_space(s.get("text", ""))
-                        if text:
-                            spans.append((round(float(s.get("size", 0)), 1), text))
-            if not spans:
-                continue
-            sizes = sorted([s for s, _ in spans], reverse=True)
-            threshold = sizes[min(5, len(sizes) - 1)] if sizes else 0
-            for sz, text in spans:
-                if sz >= threshold and 3 <= len(text) <= 90 and not PAGE_NUMBER_RE.match(text):
-                    headings.append(text)
-        except Exception:
+@app.route("/", methods=["GET"])
+def root():
+    return jsonify({"service": "Marka Data API", "status": "ok",
+                    "version": "2.0", "subjects": len(SUBJECT_REGISTRY)})
+
+
+@app.route("/health", methods=["GET"])
+def health():
+    return jsonify({"status": "healthy", "timestamp": datetime.utcnow().isoformat()})
+
+
+# -----------------------------------------------------------------------------
+# 5. SUBJECTS LIST
+# -----------------------------------------------------------------------------
+
+@app.route("/v1/subjects", methods=["GET"])
+def list_subjects():
+    """List all subjects available in the data API."""
+    level = request.args.get("level")
+    result = []
+    for remote_id, meta in SUBJECT_REGISTRY.items():
+        if level and meta["level"] != level:
             continue
-    seen = set()
-    out = []
-    for h in headings:
-        key = h.lower()
-        if key not in seen:
-            seen.add(key)
-            out.append(h)
-    return out[:25]
+        result.append({"remoteId": remote_id, **meta})
+    result.sort(key=lambda x: (x["level"], x["name"]))
+    return jsonify(result)
+
 
+# -----------------------------------------------------------------------------
+# 6. SYLLABUS STRUCTURE
+# -----------------------------------------------------------------------------
 
-def find_course_content_window(doc) -> Tuple[int, int, str]:
+@app.route("/v1/structure/<remote_id>", methods=["GET"])
+def get_structure(remote_id):
     """
-    Finds the page range that most likely contains actual course content.
-    Old parser started too early and let Foreword/About/Contents leak into the tree.
+    Return the syllabus tree for a subject.
+    Tree shape: { tree: [ { id, title, type, children: [...] } ] }
+    Data is read from Firebase RTDB syllabi/<remote_id>.
+    If not yet seeded, returns a minimal placeholder so the app doesn't crash.
     """
-    n = len(doc)
-    start_idx: Optional[int] = None
-    start_reason = "fallback_topic_signal"
-
-    for i, page in enumerate(doc):
-        text = page.get_text("text") or ""
-        first_blob = "\n".join(get_page_lines(page, 40))
-        # Avoid TOC false positives by requiring more than just a contents list.
-        looks_like_toc = bool(re.search(r"\bcontents\b|table\s+of\s+contents", first_blob, re.I)) and len(first_blob.splitlines()) > 8
-        if CONTENT_START_RE.search(first_blob) and not looks_like_toc:
-            start_idx = i
-            start_reason = "explicit_subject_content_heading"
-            break
-        if i > 3 and STRONG_TOPIC_RE.search(text) and not BOILERPLATE_HEADING_RE.search(first_blob[:200]):
-            start_idx = i
-            start_reason = "strong_topic_signal"
-            break
-
-    if start_idx is None:
-        # Last resort: skip the typical front matter zone.
-        start_idx = min(8, max(0, n - 1))
-        start_reason = "last_resort_skip_front_matter"
-
-    end_idx = n
-    for j in range(start_idx + 1, n):
-        first_blob = "\n".join(get_page_lines(doc[j], 30))
-        if CONTENT_END_RE.search(first_blob):
-            end_idx = j
-            break
-
-    if end_idx <= start_idx:
-        end_idx = n
-
-    return start_idx, end_idx, start_reason
-
-
-def is_block_boilerplate(block_text: str) -> bool:
-    text = clean_space(block_text)
-    if not text:
-        return True
-    if PAGE_NUMBER_RE.match(text):
-        return True
-    if len(text) < 3:
-        return True
-    first_words = " ".join(text.split()[:10])
-    if BOILERPLATE_HEADING_RE.match(normalise_line(first_words)):
-        return True
-    if RESIDUAL_SKIP_RE.search(text) and len(text) < 180:
-        return True
-    return False
-
-
-def _page_to_base64_png(page, dpi=PAGE_CLASSIFY_DPI) -> str:
-    mat = fitz.Matrix(dpi / 72, dpi / 72)
-    pix = page.get_pixmap(matrix=mat, colorspace=fitz.csRGB)
-    return base64.b64encode(pix.tobytes("png")).decode("utf-8")
-
-
-def _vision_classify_page(page, page_num: int, subject_name: str, level: str) -> str:
-    """Returns 'boilerplate', 'content', or 'uncertain'."""
-    client = get_gemini()
-    if client is None:
-        return "uncertain"
+    if remote_id not in SUBJECT_REGISTRY:
+        return jsonify({"error": f"Unknown subject: {remote_id}"}), 404
+
     try:
-        b64 = _page_to_base64_png(page)
-        qualification = "Cambridge International AS & A Level" if level == "A" else "Cambridge IGCSE / O Level"
-        prompt = (
-            f"This is page {page_num + 1} of a {qualification} syllabus for {subject_name}.\n\n"
-            "Classify this page as ONE of:\n"
-            "BOILERPLATE - foreword, about this syllabus, why choose Cambridge, key benefits, contents, "
-            "assessment overview, assessment objectives, grade descriptions, command words, glossary, appendices, "
-            "administration, legal/copyright/support information.\n"
-            "CONTENT - actual course/subject matter students must learn: subject content, topic lists, learning "
-            "objectives, numbered content sections, skills, practical work content, set texts, knowledge points.\n\n"
-            "Reply with exactly one word: BOILERPLATE or CONTENT"
-        )
-        resp = client.models.generate_content(
-            model=VISION_MODEL,
-            contents=[{"role": "user", "parts": [
-                {"inline_data": {"mime_type": "image/png", "data": b64}},
-                {"text": prompt}
-            ]}]
-        )
-        answer = (resp.text or "").strip().upper()
-        if "BOILERPLATE" in answer:
-            return "boilerplate"
-        if "CONTENT" in answer:
-            return "content"
-        return "uncertain"
+        data = _get_syllabus_ref(remote_id).get()
+        if data and data.get("tree"):
+            return jsonify(data)
+        # Return empty but valid structure so app server can handle gracefully
+        logger.warning(f"No syllabus tree found for {remote_id} — returning empty.")
+        return jsonify({
+            "remoteId": remote_id,
+            "name":     SUBJECT_REGISTRY[remote_id]["name"],
+            "level":    SUBJECT_REGISTRY[remote_id]["level"],
+            "tree":     [],
+            "seeded":   False,
+        })
     except Exception as e:
-        logger.warning(f"Vision classify page {page_num + 1}: {e}")
-        return "uncertain"
+        logger.error(f"Structure fetch {remote_id}: {e}")
+        return jsonify({"error": "Failed to fetch syllabus structure"}), 500
 
 
-def classify_all_pages(doc, subject_name: str, level: str) -> Tuple[List[str], Dict]:
+@app.route("/v1/structure/<remote_id>", methods=["POST"])
+def upsert_structure(remote_id):
     """
-    Returns page labels and debug info.
-    The main fix: do not let the parser start at foreword/contents. First find the course-content window,
-    then classify only inside that window.
+    Admin endpoint to seed or update a syllabus tree.
+    Body: { tree: [ { id, title, type, children: [...] } ] }
     """
-    n = len(doc)
-    start_idx, end_idx, start_reason = find_course_content_window(doc)
-    classifications: List[str] = []
-    debug_pages = []
-
-    for i, page in enumerate(doc):
-        text = page.get_text("text") or ""
-        first_lines = get_page_lines(page, 5)
-        first_heading = first_lines[0] if first_lines else ""
-
-        if i < start_idx or i >= end_idx:
-            verdict = "boilerplate"
-            reason = "outside_content_window"
-        else:
-            first_blob = "\n".join(get_page_lines(page, 35))
-            if len(clean_space(text)) < 30:
-                verdict = "boilerplate"
-                reason = "empty_or_tiny_page"
-            elif CONTENT_END_RE.search(first_blob):
-                verdict = "boilerplate"
-                reason = "assessment_or_appendix_heading"
-            elif BOILERPLATE_HEADING_RE.match(normalise_line(first_heading)):
-                verdict = "boilerplate"
-                reason = "boilerplate_heading"
-            elif CONTENT_START_RE.search(first_blob) or STRONG_TOPIC_RE.search(text):
-                verdict = "content"
-                reason = "text_content_signal"
-            elif i < MAX_VISION_PAGES:
-                v = _vision_classify_page(page, i, subject_name, level)
-                if v == "uncertain":
-                    verdict = "content" if STRONG_TOPIC_RE.search(text) else "boilerplate"
-                    reason = "heuristic_after_uncertain_vision"
-                else:
-                    verdict = v
-                    reason = f"vision_{VISION_MODEL}"
-            else:
-                verdict = "content"
-                reason = "inside_window_after_vision_limit"
-
-        classifications.append(verdict)
-        debug_pages.append({"page": i + 1, "verdict": verdict, "reason": reason, "first": first_heading[:100]})
-        logger.info(f"  Page {i + 1}/{n}: {verdict} ({reason}) | {first_heading[:90]}")
-
-    if not any(c == "content" for c in classifications):
-        logger.warning(f"  All pages BOILERPLATE for {subject_name}; applying fallback from page {start_idx + 1}.")
-        classifications = ["content" if i >= start_idx else "boilerplate" for i in range(n)]
-
-    debug = {
-        "content_start_page": start_idx + 1,
-        "content_end_page_exclusive": end_idx + 1 if end_idx < n else None,
-        "content_start_reason": start_reason,
-        "vision_model": VISION_MODEL,
-        "page_debug": debug_pages,
+    if remote_id not in SUBJECT_REGISTRY:
+        return jsonify({"error": f"Unknown subject: {remote_id}"}), 404
+
+    data = request.get_json() or {}
+    tree = data.get("tree")
+    if not isinstance(tree, list):
+        return jsonify({"error": "tree (array) required"}), 400
+
+    doc = {
+        "remoteId":  remote_id,
+        "name":      SUBJECT_REGISTRY[remote_id]["name"],
+        "level":     SUBJECT_REGISTRY[remote_id]["level"],
+        "tree":      tree,
+        "updatedAt": datetime.utcnow().isoformat(),
+        "seeded":    True,
     }
-    return classifications, debug
-
-# ---------------------------------------------------------------------------
-# PDF PARSER
-# ---------------------------------------------------------------------------
-
-class PDFParser:
-    def __init__(self, filepath):
-        self.filepath = filepath
-        self.filename = os.path.basename(filepath)
-        self.doc = fitz.open(filepath)
-        self.level, self.subject_code, self.unique_id, self.subject_name, self.registry_match = infer_level_code_subject(filepath, self.filename)
-        self.debug_info: Dict = {}
-
-    def log_pdf_detection_summary(self):
-        first_page_lines = get_page_lines(self.doc[0], 12) if len(self.doc) else []
-        heading_candidates = extract_heading_candidates(self.doc)
-        logger.info("=" * 78)
-        logger.info(f"PDF DETECTED: {self.filepath}")
-        logger.info(f"  filename:       {self.filename}")
-        logger.info(f"  pages:          {len(self.doc)}")
-        logger.info(f"  folder level:   {self.filepath.replace('\\\\', '/').split('/')[-2] if '/' in self.filepath.replace('\\\\', '/') else 'unknown'}")
-        logger.info(f"  detected code:  {self.subject_code}")
-        logger.info(f"  resolved id:    {self.unique_id}")
-        logger.info(f"  subject:        {self.subject_name}")
-        logger.info(f"  registry match: {self.registry_match}")
-        logger.info(f"  model:          {VISION_MODEL}")
-        logger.info(f"  first-page lines seen: {first_page_lines[:8]}")
-        logger.info(f"  heading candidates seen: {heading_candidates[:15]}")
-        logger.info("=" * 78)
-
-    def get_body_font_size(self):
-        sizes: Dict[float, int] = {}
-        for page in self.doc:
-            try:
-                for b in page.get_text("dict").get("blocks", []):
-                    for l in b.get("lines", []):
-                        for s in l.get("spans", []):
-                            text = s.get("text", "")
-                            if not clean_space(text):
-                                continue
-                            sz = round(float(s.get("size", 10)), 1)
-                            sizes[sz] = sizes.get(sz, 0) + len(text)
-            except Exception:
-                continue
-        return max(sizes, key=sizes.get) if sizes else 10.0
-
-    def parse(self):
-        self.log_pdf_detection_summary()
-        body_size = self.get_body_font_size()
-        page_classes, class_debug = classify_all_pages(self.doc, self.subject_name, self.level)
-        self.debug_info = class_debug
-
-        logger.info(f"Parsing COURSE CONTENT of {self.filename} (body ~{body_size}pt)")
-        content_page_count = sum(1 for c in page_classes if c == "content")
-        logger.info(f"  {content_page_count} course-content pages out of {len(self.doc)} total")
-        logger.info(f"  content window starts page {class_debug.get('content_start_page')} via {class_debug.get('content_start_reason')}")
-
-        syllabus_tree: List[Dict] = []
-        current_topic: Optional[Dict] = None
-        current_subtopic: Optional[Dict] = None
-
-        def flush_subtopic():
-            nonlocal current_subtopic, current_topic
-            if current_subtopic and current_topic:
-                # Only keep subtopics with meaningful body content.
-                content = [c for c in current_subtopic.get("content", []) if not is_block_boilerplate(c)]
-                current_subtopic["content"] = content
-                if content or len(clean_space(current_subtopic.get("title", ""))) > 3:
-                    current_topic["children"].append(current_subtopic)
-            current_subtopic = None
-
-        def flush_topic():
-            nonlocal current_topic
-            if current_topic:
-                # Drop empty topics or headings that are actually admin labels.
-                if current_topic.get("children") and not is_block_boilerplate(current_topic.get("title", "")):
-                    syllabus_tree.append(current_topic)
-            current_topic = None
-
-        for page_num, page in enumerate(self.doc):
-            if page_classes[page_num] == "boilerplate":
-                continue
+    try:
+        _get_syllabus_ref(remote_id).set(doc)
+        logger.info(f"Syllabus tree seeded: {remote_id} ({len(tree)} top-level nodes)")
+        return jsonify({"success": True, "remoteId": remote_id,
+                        "topLevelNodes": len(tree)}), 201
+    except Exception as e:
+        logger.error(f"Structure upsert {remote_id}: {e}")
+        return jsonify({"error": "Failed to save syllabus structure"}), 500
 
-            try:
-                page_dict = page.get_text("dict")
-            except Exception as e:
-                logger.warning(f"Could not parse page {page_num + 1} of {self.filename}: {e}")
-                continue
 
-            for b in page_dict.get("blocks", []):
-                block_text_parts = []
-                max_size = 0.0
-                is_bold = False
-
-                for l in b.get("lines", []):
-                    line_parts = []
-                    for s in l.get("spans", []):
-                        t = clean_space(s.get("text", ""))
-                        if not t:
-                            continue
-                        line_parts.append(t)
-                        sz = float(s.get("size", body_size))
-                        if sz > max_size:
-                            max_size = sz
-                        if "bold" in str(s.get("font", "")).lower():
-                            is_bold = True
-                    if line_parts:
-                        block_text_parts.append(" ".join(line_parts))
-
-                block_text = clean_space(" ".join(block_text_parts))
-                if is_block_boilerplate(block_text):
-                    continue
-
-                # Prevent assessment/admin sections from leaking even if page got through.
-                if CONTENT_END_RE.match("\n" + block_text):
-                    continue
-
-                # Topic/subtopic detection tuned to syllabus PDFs.
-                looks_like_major_heading = (
-                    max_size >= body_size + 2.0
-                    and len(block_text) <= 140
-                    and not block_text.endswith(".")
-                )
-                looks_like_numbered_topic = bool(re.match(r"^(\d+(\.\d+)*\s+)[A-Z]", block_text)) and len(block_text) <= 160
-                looks_like_subheading = (
-                    (is_bold and max_size >= body_size and len(block_text) <= 180)
-                    or looks_like_numbered_topic
-                    or bool(re.match(r"^(Unit|Topic|Section|Module)\s+\d+\b", block_text, re.I))
-                )
-
-                if looks_like_major_heading:
-                    flush_subtopic()
-                    flush_topic()
-                    current_topic = {
-                        "id": f"{self.unique_id}_{len(syllabus_tree)}",
-                        "title": block_text,
-                        "type": "topic",
-                        "page": page_num + 1,
-                        "children": [],
-                    }
-                    current_subtopic = None
-                elif looks_like_subheading:
-                    if not current_topic:
-                        current_topic = {
-                            "id": f"{self.unique_id}_root",
-                            "title": "Course Content",
-                            "type": "topic",
-                            "page": page_num + 1,
-                            "children": [],
-                        }
-                    flush_subtopic()
-                    current_subtopic = {
-                        "id": f"{current_topic['id']}_{len(current_topic['children'])}",
-                        "title": block_text,
-                        "type": "subtopic",
-                        "page": page_num + 1,
-                        "content": [],
-                    }
-                else:
-                    if not current_topic:
-                        current_topic = {
-                            "id": f"{self.unique_id}_root",
-                            "title": "Course Content",
-                            "type": "topic",
-                            "page": page_num + 1,
-                            "children": [],
-                        }
-                    if not current_subtopic:
-                        current_subtopic = {
-                            "id": f"{current_topic['id']}_intro",
-                            "title": "Overview",
-                            "type": "subtopic",
-                            "page": page_num + 1,
-                            "content": [],
-                        }
-                    current_subtopic["content"].append(block_text)
-
-        flush_subtopic()
-        flush_topic()
-
-        logger.info(f"  Parsed topics for {self.unique_id}: {len(syllabus_tree)}")
-        preview = [t.get("title") for t in syllabus_tree[:8]]
-        logger.info(f"  Topic preview: {preview}")
-
-        return {
-            "meta": {
-                "id": self.unique_id,
-                "subject": self.subject_name,
-                "code": self.subject_code,
-                "level": self.level,
-                "filename": self.filename,
-                "registryMatch": self.registry_match,
-                "indexed_at": int(time.time()),
-                "parserVersion": "3.1-course-content-window",
-                "model": VISION_MODEL,
-                "contentStartPage": class_debug.get("content_start_page"),
-                "contentStartReason": class_debug.get("content_start_reason"),
-            },
-            "debug": {
-                "pageClassification": class_debug,
-            },
-            "tree": syllabus_tree,
-        }
-
-# ---------------------------------------------------------------------------
-# PAST EXAM PARSER
-# ---------------------------------------------------------------------------
-
-class ExamPaperParser:
-    def __init__(self, filepath):
-        self.filepath = filepath
-        self.filename = os.path.basename(filepath)
-        self.doc = fitz.open(filepath)
-        self.level, self.subject_code, self.unique_id, self.subject_name, self.registry_match = infer_level_code_subject(filepath, self.filename)
-        year_m = re.search(r'\b(20\d{2}|19\d{2})\b', self.filename)
-        self.year = year_m.group(1) if year_m else "Unknown"
-        sess_m = re.search(r'(may[_\-]?june|oct[_\-]?nov|feb[_\-]?mar|summer|winter|s\d|w\d|m\d)', self.filename, re.IGNORECASE)
-        self.session = sess_m.group(1).upper() if sess_m else "Unknown"
-        paper_m = re.search(r'[_\-]p(\d)|paper[\s_\-]?(\d)', self.filename, re.IGNORECASE)
-        self.paper_num = (paper_m.group(1) or paper_m.group(2)) if paper_m else "1"
-        self.paper_id = f"{self.unique_id}_{self.year}_{self.session}_P{self.paper_num}"
-
-    def extract_pages(self):
-        return [
-            {"page": i + 1, "text": p.get_text("text").strip()[:3000]}
-            for i, p in enumerate(self.doc)
-            if p.get_text("text").strip()
-        ]
+# -----------------------------------------------------------------------------
+# 7. CONTENT CHUNKS (for search)
+# -----------------------------------------------------------------------------
 
-    def extract_questions(self):
-        full = "\n".join(p["text"] for p in self.extract_pages())
-        pat = re.compile(r'(?:^|\n)\s*(\d{1,2})\s*[\.\)]\s+(.+?)(?=\n\s*\d{1,2}\s*[\.\)]|\Z)', re.DOTALL | re.MULTILINE)
-        return [
-            {"number": int(m.group(1)), "text": m.group(2).strip()[:2000]}
-            for m in pat.finditer(full)
-            if len(m.group(2).strip()) > 20
-        ]
+@app.route("/v1/chunks/<remote_id>", methods=["POST"])
+def upsert_chunks(remote_id):
+    """
+    Seed content chunks for a subject.
+    Body: { chunks: [ { id, nodeId, nodeTitle, content, keywords } ] }
+    Embeddings are computed and stored alongside.
+    """
+    if remote_id not in SUBJECT_REGISTRY:
+        return jsonify({"error": f"Unknown subject: {remote_id}"}), 404
+
+    data   = request.get_json() or {}
+    chunks = data.get("chunks") or []
+    if not chunks:
+        return jsonify({"error": "chunks array required"}), 400
+
+    saved  = 0
+    errors = 0
+    chunks_ref = _get_chunks_ref(remote_id)
 
-    def parse(self):
-        logger.info(f"Exam PDF detected: {self.filepath} -> {self.unique_id} {self.subject_name}, year={self.year}, paper={self.paper_num}")
-        return {
-            "meta": {
-                "paperId": self.paper_id,
-                "subjectId": self.unique_id,
-                "subject": self.subject_name,
-                "subjectCode": self.subject_code,
-                "level": self.level,
-                "year": self.year,
-                "session": self.session,
-                "paperNumber": self.paper_num,
-                "filename": self.filename,
-                "totalPages": len(self.doc),
-                "indexed_at": int(time.time()),
-            },
-            "pages": self.extract_pages(),
-            "questions": self.extract_questions(),
-        }
-
-# ---------------------------------------------------------------------------
-# EMBEDDINGS
-# ---------------------------------------------------------------------------
-
-def generate_embeddings(texts):
-    client = get_gemini()
-    if client is None:
-        logger.warning("Gemini API key not configured. Returning zero embeddings.")
-        return [np.zeros(768).tolist() for _ in texts]
-
-    results = []
-    for i in range(0, len(texts), 10):
-        batch = texts[i:i + 10]
+    for chunk in chunks:
+        chunk_id = chunk.get("id") or uuid.uuid4().hex
+        content  = chunk.get("content", "")
+        if not content:
+            continue
         try:
-            resp = client.models.embed_content(model=EMBEDDING_MODEL, contents=batch)
-            for emb in resp.embeddings:
-                results.append(emb.values)
+            embedding = _embed_text(content)
+            doc = {
+                "chunkId":   chunk_id,
+                "subjectId": remote_id,
+                "nodeId":    chunk.get("nodeId", ""),
+                "nodeTitle": chunk.get("nodeTitle", ""),
+                "content":   content,
+                "keywords":  chunk.get("keywords") or [],
+                "embedding": embedding,
+                "createdAt": datetime.utcnow().isoformat(),
+            }
+            chunks_ref.child(chunk_id).set(doc)
+            saved += 1
         except Exception as e:
-            logger.error(f"Embed batch {i}: {e}")
-            for _ in batch:
-                results.append(np.zeros(768).tolist())
-    return results
-
-# ---------------------------------------------------------------------------
-# FIREBASE PERSISTENCE
-# ---------------------------------------------------------------------------
-
-def load_index_from_firebase():
-    global SYLLABUS_MAP, VECTOR_DB, VECTOR_MATRIX, EXAM_MAP
-    if not FIREBASE_AVAILABLE:
-        return False
-    logger.info("Loading index from Firebase ...")
-    try:
-        fb_syllabi = fb_get("data_api/syllabi")
-        if not fb_syllabi:
-            return False
-        SYLLABUS_MAP = fb_syllabi
-
-        fb_vectors = fb_get("data_api/vectors")
-        if not fb_vectors:
-            return False
-
-        VECTOR_DB = []
-        valid = []
-        expected_dim = 768
-        iterable = sorted(fb_vectors.keys()) if isinstance(fb_vectors, dict) else range(len(fb_vectors))
-        for entry in iterable:
-            item = fb_vectors[entry] if isinstance(fb_vectors, dict) else fb_vectors[entry]
-            if not item:
-                continue
-            raw_vec = item.get("vector")
-            if not raw_vec:
-                continue
-            try:
-                vec = np.array(raw_vec, dtype=np.float32)
-                if vec.ndim != 1 or len(vec) != expected_dim:
-                    logger.warning(f"Skipping vector with wrong shape: {vec.shape}")
-                    continue
-                VECTOR_DB.append({"vector": vec, "meta": item["meta"]})
-                valid.append(vec)
-            except Exception as ve:
-                logger.warning(f"Skipping malformed vector entry: {ve}")
-                continue
+            logger.error(f"Chunk save error {chunk_id}: {e}")
+            errors += 1
 
-        VECTOR_MATRIX = np.vstack(valid).astype(np.float32) if valid else None
-        logger.info(f"Vector matrix shape: {VECTOR_MATRIX.shape if VECTOR_MATRIX is not None else None}")
+    return jsonify({"success": True, "saved": saved, "errors": errors,
+                    "remoteId": remote_id}), 201
 
-        fb_exams = fb_get("data_api/exams")
-        if fb_exams:
-            EXAM_MAP = fb_exams
 
-        logger.info(f"Loaded: {len(SYLLABUS_MAP)} syllabi, {len(VECTOR_DB)} vectors, {len(EXAM_MAP)} exam subjects.")
-        return True
+@app.route("/v1/chunks/<remote_id>", methods=["GET"])
+def list_chunks(remote_id):
+    """List all chunks for a subject (without embeddings for brevity)."""
+    if remote_id not in SUBJECT_REGISTRY:
+        return jsonify({"error": f"Unknown subject: {remote_id}"}), 404
+    try:
+        raw    = _get_chunks_ref(remote_id).get() or {}
+        result = []
+        for cid, chunk in raw.items():
+            if not chunk: continue
+            out = {k: chunk[k] for k in (
+                "chunkId", "nodeId", "nodeTitle", "content", "keywords", "createdAt"
+            ) if k in chunk}
+            result.append(out)
+        return jsonify({"remoteId": remote_id, "count": len(result), "chunks": result})
     except Exception as e:
-        logger.error(f"Firebase load: {e}")
-        return False
-
-
-def save_syllabus(sid, data):
-    fb_set(f"data_api/syllabi/{sid}", data)
-
-
-def save_all_vectors():
-    fb_data = {}
-    for i, entry in enumerate(VECTOR_DB):
-        fb_data[f"v_{i:06d}"] = {
-            "vector": entry["vector"].tolist() if isinstance(entry["vector"], np.ndarray) else entry["vector"],
-            "meta": entry["meta"],
-        }
-    fb_set("data_api/vectors", fb_data)
-
-
-def save_exam(sid, exam_data):
-    safe = safe_firebase_key(exam_data["meta"]["paperId"])
-    fb_set(f"data_api/exams/{sid}/{safe}", exam_data)
-
-# ---------------------------------------------------------------------------
-# INDEX BUILDER
-# ---------------------------------------------------------------------------
-
-def syllabus_sort_key(path: str):
-    level, code, uid, subject, registry_match = infer_level_code_subject(path, os.path.basename(path))
-    return (level, subject, code, path)
-
-
-def build_index():
-    global SYLLABUS_MAP, VECTOR_DB, VECTOR_MATRIX, EXAM_MAP
-    logger.info("Full index build starting ...")
-    logger.info(f"Using Gemini model for page classification: {VISION_MODEL}")
-    parsed_data = []
-
-    if os.path.exists(SYLLABI_DIR):
-        syllabus_paths = []
-        for root, _, files in os.walk(SYLLABI_DIR):
-            for f in files:
-                if f.lower().endswith(".pdf"):
-                    syllabus_paths.append(os.path.join(root, f))
-        logger.info(f"Found {len(syllabus_paths)} syllabus PDF(s).")
-        for path in sorted(syllabus_paths, key=syllabus_sort_key):
-            logger.info(f"Syllabus queued: {path}")
-            try:
-                parser = PDFParser(path)
-                data = parser.parse()
-                parsed_data.append(data)
-                SYLLABUS_MAP[data["meta"]["id"]] = data
-                save_syllabus(data["meta"]["id"], data)
-            except Exception as e:
-                logger.exception(f"Failed syllabus parse {path}: {e}")
-
-    if os.path.exists(PAST_EXAMS_DIR):
-        exam_paths = []
-        for root, _, files in os.walk(PAST_EXAMS_DIR):
-            for f in files:
-                if f.lower().endswith(".pdf"):
-                    exam_paths.append(os.path.join(root, f))
-        logger.info(f"Found {len(exam_paths)} past exam PDF(s).")
-        for path in sorted(exam_paths):
-            logger.info(f"Exam queued: {path}")
-            try:
-                parser = ExamPaperParser(path)
-                exam_data = parser.parse()
-                sid = exam_data["meta"]["subjectId"]
-                EXAM_MAP.setdefault(sid, {})
-                safe = safe_firebase_key(exam_data["meta"]["paperId"])
-                EXAM_MAP[sid][safe] = exam_data
-                save_exam(sid, exam_data)
-            except Exception as e:
-                logger.exception(f"Failed exam parse {path}: {e}")
-
-    if not parsed_data:
-        logger.info("Nothing to vectorize.")
-        return
-
-    chunks, metas = [], []
-    for item in parsed_data:
-        mb = item["meta"]
-        for topic in item.get("tree", []):
-            for sub in topic.get("children", []):
-                blob = "\n".join(sub.get("content", []))
-                if len(clean_space(blob)) < MIN_VECTOR_TEXT_CHARS:
-                    continue
-                chunks.append(f"{mb['subject']} {mb['level']} {mb['code']} - {topic['title']} - {sub['title']}:\n{blob}")
-                metas.append({
-                    "subject_id": mb["id"],
-                    "subject": mb["subject"],
-                    "level": mb["level"],
-                    "code": mb["code"],
-                    "topic_id": topic["id"],
-                    "topic_title": topic["title"],
-                    "subtopic_id": sub["id"],
-                    "title": sub["title"],
-                    "content": blob,
-                })
-
-    logger.info(f"Embedding {len(chunks)} course-content chunks ...")
-    vecs = generate_embeddings(chunks)
-    VECTOR_DB = []
-    valid = []
-    for i, v in enumerate(vecs):
-        nv = np.array(v, dtype=np.float32)
-        VECTOR_DB.append({"vector": nv, "meta": metas[i]})
-        valid.append(nv)
-    VECTOR_MATRIX = np.vstack(valid).astype(np.float32) if valid else None
-    save_all_vectors()
-    logger.info(f"Index done: {len(SYLLABUS_MAP)} syllabi, {len(VECTOR_DB)} vectors.")
-
-
-def _incremental_vectorize(syllabus_data):
-    global VECTOR_DB, VECTOR_MATRIX
-    mb = syllabus_data["meta"]
-    chunks, metas = [], []
-    for topic in syllabus_data.get("tree", []):
-        for sub in topic.get("children", []):
-            blob = "\n".join(sub.get("content", []))
-            if len(clean_space(blob)) < MIN_VECTOR_TEXT_CHARS:
+        logger.error(f"Chunk list {remote_id}: {e}")
+        return jsonify({"error": "Failed to list chunks"}), 500
+
+
+# -----------------------------------------------------------------------------
+# 8. SEMANTIC SEARCH
+# -----------------------------------------------------------------------------
+
+@app.route("/v1/search", methods=["POST"])
+def search_syllabus():
+    """
+    Semantic search over syllabus content chunks.
+    Body: { query: "...", filter_subject_id: "A_9702" (optional), top_k: 5 }
+    Returns: { results: [ { chunkId, nodeId, nodeTitle, content, score } ] }
+
+    Strategy:
+      1. If query embedding succeeds, use cosine similarity.
+      2. Fall back to keyword overlap if embedding fails.
+    """
+    data      = request.get_json() or {}
+    query     = (data.get("query") or "").strip()
+    filter_id = data.get("filter_subject_id")
+    top_k     = int(data.get("top_k") or 5)
+
+    if not query:
+        return jsonify({"error": "query required"}), 400
+
+    # Determine which subjects to search
+    if filter_id:
+        if filter_id not in SUBJECT_REGISTRY:
+            return jsonify({"error": f"Unknown subject: {filter_id}"}), 404
+        subject_ids = [filter_id]
+    else:
+        subject_ids = list(SUBJECT_REGISTRY.keys())
+
+    # Generate query embedding
+    query_embedding = _embed_text(query)
+    use_embedding   = bool(query_embedding)
+
+    scored_chunks = []
+
+    for remote_id in subject_ids:
+        try:
+            raw = _get_chunks_ref(remote_id).get() or {}
+        except Exception as e:
+            logger.error(f"Search chunk fetch {remote_id}: {e}")
+            continue
+
+        for cid, chunk in raw.items():
+            if not chunk or not chunk.get("content"):
                 continue
-            chunks.append(f"{mb['subject']} {mb['level']} {mb['code']} - {topic['title']} - {sub['title']}:\n{blob}")
-            metas.append({
-                "subject_id": mb["id"],
-                "subject": mb["subject"],
-                "level": mb["level"],
-                "code": mb["code"],
-                "topic_id": topic["id"],
-                "topic_title": topic["title"],
-                "subtopic_id": sub["id"],
-                "title": sub["title"],
-                "content": blob,
+
+            content = chunk["content"]
+
+            if use_embedding:
+                chunk_embedding = chunk.get("embedding") or []
+                score = _cosine_similarity(query_embedding, chunk_embedding)
+            else:
+                # Keyword fallback
+                score = _keyword_score(query, content)
+                # Also score against nodeTitle
+                title_score = _keyword_score(query, chunk.get("nodeTitle", ""))
+                score = max(score, title_score)
+
+            scored_chunks.append({
+                "chunkId":   cid,
+                "subjectId": remote_id,
+                "nodeId":    chunk.get("nodeId", ""),
+                "nodeTitle": chunk.get("nodeTitle", ""),
+                "content":   content,
+                "score":     round(score, 4),
             })
-    if not chunks:
-        return
-    for i, v in enumerate(generate_embeddings(chunks)):
-        VECTOR_DB.append({"vector": np.array(v, dtype=np.float32), "meta": metas[i]})
-    if VECTOR_DB:
-        VECTOR_MATRIX = np.vstack([e["vector"] for e in VECTOR_DB]).astype(np.float32)
-    save_all_vectors()
-
-# ---------------------------------------------------------------------------
-# WATCHER
-# ---------------------------------------------------------------------------
-_indexed_files = set()
-
-def _collect_existing():
-    for d in [SYLLABI_DIR, PAST_EXAMS_DIR]:
-        if not os.path.exists(d):
-            continue
-        for root, _, files in os.walk(d):
-            for f in files:
-                if f.lower().endswith(".pdf"):
-                    _indexed_files.add(os.path.join(root, f))
 
+    # Sort by score descending, return top_k
+    scored_chunks.sort(key=lambda x: -x["score"])
+    results = scored_chunks[:top_k]
 
-def _watch(interval=30):
-    while True:
-        time.sleep(interval)
-        for directory, is_exam in [(SYLLABI_DIR, False), (PAST_EXAMS_DIR, True)]:
-            if not os.path.exists(directory):
-                continue
-            for root, _, files in os.walk(directory):
-                for f in files:
-                    if not f.lower().endswith(".pdf"):
-                        continue
-                    path = os.path.join(root, f)
-                    if path in _indexed_files:
-                        continue
-                    _indexed_files.add(path)
-                    logger.info(f"New PDF detected by watcher: {path}")
-                    try:
-                        if is_exam:
-                            parser = ExamPaperParser(path)
-                            exam_data = parser.parse()
-                            sid = exam_data["meta"]["subjectId"]
-                            EXAM_MAP.setdefault(sid, {})
-                            safe = safe_firebase_key(exam_data["meta"]["paperId"])
-                            EXAM_MAP[sid][safe] = exam_data
-                            save_exam(sid, exam_data)
-                        else:
-                            parser = PDFParser(path)
-                            data = parser.parse()
-                            SYLLABUS_MAP[data["meta"]["id"]] = data
-                            save_syllabus(data["meta"]["id"], data)
-                            _incremental_vectorize(data)
-                    except Exception as e:
-                        logger.exception(f"Watch parse failed {path}: {e}")
-
-# ---------------------------------------------------------------------------
-# API
-# ---------------------------------------------------------------------------
-
-@app.route('/', methods=['GET'])
-def index():
     return jsonify({
-        "name": "Marka Data API",
-        "version": "3.1",
-        "status": "online",
-        "subjects_loaded": len(SYLLABUS_MAP),
-        "vector_chunks": len(VECTOR_DB),
-        "firebase": FIREBASE_AVAILABLE,
-        "model": VISION_MODEL,
-        "parser": "course-content-window-with-pdf-logging",
-        "endpoints": [
-            "GET  /health",
-            "GET  /v1/subjects",
-            "GET  /v1/structure/<subject_id>",
-            "POST /v1/search",
-            "GET  /v1/exams",
-            "GET  /v1/exams/<paper_id>",
-            "GET  /v1/exams/<paper_id>/questions",
-            "POST /v1/rebuild",
-            "GET|POST /v1/reset",
-        ],
+        "query":         query,
+        "filterSubject": filter_id,
+        "topK":          top_k,
+        "totalScanned":  len(scored_chunks),
+        "usedEmbedding": use_embedding,
+        "results":       results,
     })
 
-@app.route('/health', methods=['GET'])
-def health():
-    return jsonify({
-        "status": "online",
-        "subjects_loaded": list(SYLLABUS_MAP.keys()),
-        "subject_count": len(SYLLABUS_MAP),
-        "vector_chunks": len(VECTOR_DB),
-        "exam_subjects": list(EXAM_MAP.keys()),
-        "firebase": FIREBASE_AVAILABLE,
-        "registered_subjects": ALL_SUBJECTS,
-        "model": VISION_MODEL,
-        "embedding_model": EMBEDDING_MODEL,
-    })
 
-@app.route('/v1/subjects', methods=['GET'])
-def list_subjects():
-    result = []
-    for sid, data in SYLLABUS_MAP.items():
-        meta = data.get("meta", {"id": sid})
-        result.append({**meta, "indexed": True})
-    for uid, name in ALL_SUBJECTS.items():
-        if uid not in SYLLABUS_MAP:
-            level = "A" if uid.startswith("A_") else "O"
-            result.append({"id": uid, "subject": name, "code": uid.split("_")[1], "level": level, "indexed": False})
-    return jsonify(sorted(result, key=lambda x: (x.get("level", ""), x.get("subject", ""), x.get("code", ""))))
-
-@app.route('/v1/structure/<subject_id>', methods=['GET'])
-def get_structure(subject_id):
-    data = SYLLABUS_MAP.get(subject_id)
-    if not data:
-        return jsonify({"error": "Subject not found"}), 404
-    return jsonify(data)
-
-@app.route('/v1/search', methods=['POST'])
-def search():
-    if VECTOR_MATRIX is None or not VECTOR_DB:
-        return jsonify({"error": "Index not ready"}), 503
-    req = request.json or {}
-    q = req.get("query")
-    sf = req.get("filter_subject_id")
-    if not q:
-        return jsonify({"error": "Query required"}), 400
-    c = get_gemini()
-    if c is None:
-        return jsonify({"error": "Embedding API not configured"}), 503
+# -----------------------------------------------------------------------------
+# 9. NODE DETAIL
+# -----------------------------------------------------------------------------
+
+@app.route("/v1/node/<remote_id>/<node_id>", methods=["GET"])
+def get_node_detail(remote_id, node_id):
+    """
+    Return full content for a specific syllabus node.
+    Searches chunks for all content matching this nodeId.
+    """
+    if remote_id not in SUBJECT_REGISTRY:
+        return jsonify({"error": f"Unknown subject: {remote_id}"}), 404
+
     try:
-        resp = c.models.embed_content(model=EMBEDDING_MODEL, contents=q)
-        qv = np.array(resp.embeddings[0].values, dtype=np.float32).reshape(1, -1)
+        raw    = _get_chunks_ref(remote_id).get() or {}
+        chunks = [
+            {k: c[k] for k in ("chunkId", "nodeId", "nodeTitle", "content", "keywords") if k in c}
+            for c in raw.values()
+            if c and c.get("nodeId") == node_id
+        ]
+        if not chunks:
+            return jsonify({"error": "Node content not found"}), 404
+
+        combined_content = "\n\n".join(c["content"] for c in chunks)
+        node_title       = chunks[0].get("nodeTitle", node_id) if chunks else node_id
+
+        return jsonify({
+            "remoteId":   remote_id,
+            "nodeId":     node_id,
+            "nodeTitle":  node_title,
+            "chunks":     chunks,
+            "fullContent": combined_content,
+        })
     except Exception as e:
-        logger.error(f"Embed query failed: {e}")
-        return jsonify({"error": f"Embedding failed: {str(e)}"}), 500
+        logger.error(f"Node detail {remote_id}/{node_id}: {e}")
+        return jsonify({"error": "Failed to fetch node content"}), 500
+
+
+# -----------------------------------------------------------------------------
+# 10. PAST PAPERS INDEX
+# -----------------------------------------------------------------------------
+
+@app.route("/v1/papers/<remote_id>", methods=["GET"])
+def list_papers_for_subject(remote_id):
+    """
+    List past paper metadata for a subject from RTDB.
+    Papers are stored by the app server; this endpoint provides the index.
+    """
+    if remote_id not in SUBJECT_REGISTRY:
+        return jsonify({"error": f"Unknown subject: {remote_id}"}), 404
+
     try:
-        scores = cosine_similarity(qv, VECTOR_MATRIX)[0]
+        papers_ref = db_ref.child("papers")
+        all_papers = papers_ref.get() or {}
+        # Match by remoteId or by subjectId prefix pattern
+        meta = SUBJECT_REGISTRY[remote_id]
+        matching = [
+            p for p in all_papers.values()
+            if p and (
+                p.get("remoteId") == remote_id
+                or (p.get("level") == meta["level"] and
+                    p.get("code") == meta["code"])
+            )
+        ]
+        matching.sort(key=lambda x: x.get("year", ""), reverse=True)
+        return jsonify({"remoteId": remote_id, "papers": matching})
     except Exception as e:
-        logger.error(f"Cosine similarity failed: {e}, matrix shape: {VECTOR_MATRIX.shape if VECTOR_MATRIX is not None else None}, query shape: {qv.shape}")
-        return jsonify({"error": f"Search index error: {str(e)}"}), 500
-    results = []
-    for idx in np.argsort(scores)[::-1]:
-        if scores[idx] < 0.3:
-            break
-        meta = VECTOR_DB[idx]["meta"]
-        if sf and meta["subject_id"] != sf:
+        logger.error(f"Papers index {remote_id}: {e}")
+        return jsonify({"error": "Failed to fetch papers index"}), 500
+
+
+# -----------------------------------------------------------------------------
+# 11. BATCH STRUCTURE SEED HELPER
+# -----------------------------------------------------------------------------
+
+@app.route("/v1/admin/seed-structures", methods=["POST"])
+def batch_seed_structures():
+    """
+    Batch seed multiple syllabus trees at once.
+    Body: { structures: { "A_9702": { tree: [...] }, ... } }
+    """
+    data       = request.get_json() or {}
+    structures = data.get("structures") or {}
+    if not structures:
+        return jsonify({"error": "structures object required"}), 400
+
+    results = {}
+    for remote_id, payload in structures.items():
+        if remote_id not in SUBJECT_REGISTRY:
+            results[remote_id] = {"ok": False, "error": "Unknown subject"}
             continue
-        results.append({
-            "score": float(scores[idx]),
-            "subject_id": meta["subject_id"],
-            "subject": meta.get("subject"),
-            "level": meta.get("level"),
-            "code": meta.get("code"),
-            "topic": meta.get("topic_title"),
-            "title": meta["title"],
-            "content": meta["content"],
-            "node_id": meta["subtopic_id"],
-        })
-        if len(results) >= int(req.get("limit", 5)):
-            break
+        tree = payload.get("tree")
+        if not isinstance(tree, list):
+            results[remote_id] = {"ok": False, "error": "tree array required"}
+            continue
+        try:
+            doc = {
+                "remoteId":  remote_id,
+                "name":      SUBJECT_REGISTRY[remote_id]["name"],
+                "level":     SUBJECT_REGISTRY[remote_id]["level"],
+                "tree":      tree,
+                "updatedAt": datetime.utcnow().isoformat(),
+                "seeded":    True,
+            }
+            _get_syllabus_ref(remote_id).set(doc)
+            results[remote_id] = {"ok": True, "topLevelNodes": len(tree)}
+        except Exception as e:
+            results[remote_id] = {"ok": False, "error": str(e)}
+
     return jsonify({"results": results})
 
-@app.route('/v1/exams', methods=['GET'])
-def list_exams():
-    sid = request.args.get("subject_id")
-    out = []
-    for s, papers in EXAM_MAP.items():
-        if sid and s != sid:
-            continue
-        for p in papers.values():
-            if isinstance(p, dict) and "meta" in p:
-                out.append(p["meta"])
-    return jsonify(out)
-
-@app.route('/v1/exams/<paper_id>', methods=['GET'])
-def get_exam(paper_id):
-    safe = safe_firebase_key(paper_id)
-    for _, papers in EXAM_MAP.items():
-        for key, paper in papers.items():
-            if key == safe or (isinstance(paper, dict) and paper.get("meta", {}).get("paperId") == paper_id):
-                return jsonify(paper)
-    return jsonify({"error": "Not found"}), 404
-
-@app.route('/v1/exams/<paper_id>/questions', methods=['GET'])
-def get_exam_questions(paper_id):
-    safe = safe_firebase_key(paper_id)
-    for _, papers in EXAM_MAP.items():
-        for key, paper in papers.items():
-            if key == safe or (isinstance(paper, dict) and paper.get("meta", {}).get("paperId") == paper_id):
-                return jsonify({"paperId": paper_id, "meta": paper.get("meta"), "questions": paper.get("questions", [])})
-    return jsonify({"error": "Not found"}), 404
-
-@app.route('/v1/reset', methods=['GET', 'POST'])
-def reset_and_rebuild():
-    """
-    One-time cache reset endpoint.
-    Wipes Firebase syllabus/vector/exam cache, then triggers a full fresh rebuild from PDF files.
 
-    Browser: GET /v1/reset?token=marka-reset-2026
-    Better: set REBUILD_SECRET and call GET /v1/reset?token=<your-secret>
-    """
-    expected = os.environ.get("REBUILD_SECRET", "marka-reset-2026")
-    token = request.args.get("token") or request.headers.get("Authorization", "").replace("Bearer ", "").strip()
-    if token != expected:
-        return jsonify({"error": "Unauthorized", "hint": "Pass ?token=<REBUILD_SECRET> in the URL"}), 401
-
-    def _reset_bg():
-        global SYLLABUS_MAP, VECTOR_DB, VECTOR_MATRIX, EXAM_MAP
-        logger.info("=== CACHE RESET STARTED ===")
-        if FIREBASE_AVAILABLE:
-            for node in ["data_api/syllabi", "data_api/vectors", "data_api/exams"]:
-                try:
-                    firebase_db_ref.child(node).delete()
-                    logger.info(f"  ✓ Deleted {node} from Firebase")
-                except Exception as e:
-                    logger.error(f"  ✗ Failed to delete {node}: {e}")
-        else:
-            logger.warning("  Firebase not available — skipping Firebase wipe")
-
-        SYLLABUS_MAP = {}
-        VECTOR_DB = []
-        VECTOR_MATRIX = None
-        EXAM_MAP = {}
-        logger.info("  ✓ In-memory state cleared")
-        build_index()
-        logger.info("=== CACHE RESET COMPLETE ===")
-
-    threading.Thread(target=_reset_bg, daemon=True).start()
+# -----------------------------------------------------------------------------
+# 12. STATS
+# -----------------------------------------------------------------------------
+
+@app.route("/v1/stats", methods=["GET"])
+def data_api_stats():
+    """Return coverage stats — which subjects have trees and chunks seeded."""
+    stats = []
+    for remote_id, meta in SUBJECT_REGISTRY.items():
+        try:
+            tree_data   = _get_syllabus_ref(remote_id).get() or {}
+            chunks_data = _get_chunks_ref(remote_id).get() or {}
+            tree_count  = len(tree_data.get("tree", []))
+            chunk_count = len(chunks_data)
+        except Exception:
+            tree_count  = 0
+            chunk_count = 0
+        stats.append({
+            "remoteId":   remote_id,
+            "name":       meta["name"],
+            "level":      meta["level"],
+            "hasTree":    tree_count > 0,
+            "treeNodes":  tree_count,
+            "chunkCount": chunk_count,
+        })
+    stats.sort(key=lambda x: (x["level"], x["name"]))
+    seeded_count = sum(1 for s in stats if s["hasTree"])
     return jsonify({
-        "status": "reset started",
-        "message": "Firebase cache wiped. Full rebuild running in background. Check /health and HuggingFace logs.",
-        "model": VISION_MODEL,
-        "watch": "/health",
-    }), 202
-
-@app.route('/v1/rebuild', methods=['POST'])
-def trigger_rebuild():
-    secret = os.environ.get("REBUILD_SECRET", "")
-    if secret and request.headers.get("Authorization", "") != f"Bearer {secret}":
-        return jsonify({"error": "Unauthorized"}), 401
-
-    def _bg():
-        global SYLLABUS_MAP, VECTOR_DB, VECTOR_MATRIX, EXAM_MAP
-        SYLLABUS_MAP = {}
-        VECTOR_DB = []
-        VECTOR_MATRIX = None
-        EXAM_MAP = {}
-        build_index()
-
-    threading.Thread(target=_bg, daemon=True).start()
-    return jsonify({"status": "rebuild started", "model": VISION_MODEL}), 202
-
-# ---------------------------------------------------------------------------
-# STARTUP
-# ---------------------------------------------------------------------------
-
-def start_app():
-    for d in [SYLLABI_DIR, PAST_EXAMS_DIR]:
-        if not os.path.exists(d):
-            os.makedirs(os.path.join(d, "A"), exist_ok=True)
-            os.makedirs(os.path.join(d, "O"), exist_ok=True)
-
-    if not load_index_from_firebase():
-        build_index()
-    else:
-        logger.info("Served from Firebase cache. Use /v1/reset to force re-parse with the new parser.")
+        "totalSubjects": len(stats),
+        "seededSubjects": seeded_count,
+        "subjects": stats,
+        "generatedAt": datetime.utcnow().isoformat(),
+    })
 
-    _collect_existing()
-    threading.Thread(target=_watch, daemon=True).start()
-    logger.info("Watcher started.")
 
-with app.app_context():
-    start_app()
+# -----------------------------------------------------------------------------
+# MAIN
+# -----------------------------------------------------------------------------
 
-if __name__ == '__main__':
-    app.run(host='0.0.0.0', port=int(os.environ.get("PORT", "7860")))
+if __name__ == "__main__":
+    port = int(os.environ.get("PORT", 7861))
+    app.run(debug=True, host="0.0.0.0", port=port)