Update main.py

main.py

@@ -5,6 +5,8 @@ import re
 import time
 import threading
 import base64
+from typing import Dict, List, Tuple, Optional
+
 import numpy as np
 import fitz  # PyMuPDF
 from flask import Flask, request, jsonify
@@ -21,13 +23,20 @@ from firebase_admin import credentials, db as firebase_db
 logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
 logger = logging.getLogger(__name__)
 
-SYLLABI_DIR    = "syllabi"
-PAST_EXAMS_DIR = "past_exams"
+SYLLABI_DIR    = os.environ.get("SYLLABI_DIR", "syllabi")
+PAST_EXAMS_DIR = os.environ.get("PAST_EXAMS_DIR", "past_exams")
 
-# Support both naming conventions (HuggingFace Space may use either)
+# Support both naming conventions used on HuggingFace / local envs.
 GEMINI_API_KEY  = os.environ.get("GEMINI_API_KEY") or os.environ.get("Gemini")
-EMBEDDING_MODEL = "models/text-embedding-004"
-VISION_MODEL    = "gemini-2.0-flash"
+EMBEDDING_MODEL = os.environ.get("EMBEDDING_MODEL", "models/text-embedding-004")
+
+# Requested model bump. Keep env override so you can hotfix without code edits.
+VISION_MODEL    = os.environ.get("GEMINI_MODEL", "gemini-3.1-flash-lite")
+
+# Parser controls
+MAX_VISION_PAGES      = int(os.environ.get("MAX_VISION_PAGES", "80"))
+PAGE_CLASSIFY_DPI     = int(os.environ.get("PAGE_CLASSIFY_DPI", "72"))
+MIN_VECTOR_TEXT_CHARS = int(os.environ.get("MIN_VECTOR_TEXT_CHARS", "20"))
 
 # ---------------------------------------------------------------------------
 # COMPLETE SUBJECT REGISTRY  (all 24 PDFs on HuggingFace)
@@ -61,14 +70,18 @@ O_LEVEL_SUBJECTS = {
     "O_0625": "Physics",
 }
 ALL_SUBJECTS = {**A_LEVEL_SUBJECTS, **O_LEVEL_SUBJECTS}
+CODE_TO_LEVEL_SUBJECT: Dict[str, List[Tuple[str, str, str]]] = {}
+for sid, subject in ALL_SUBJECTS.items():
+    level, code = sid.split("_", 1)
+    CODE_TO_LEVEL_SUBJECT.setdefault(code, []).append((level, sid, subject))
 
 # ---------------------------------------------------------------------------
 # GLOBAL STATE
 # ---------------------------------------------------------------------------
-SYLLABUS_MAP  = {}
-VECTOR_DB     = []
+SYLLABUS_MAP: Dict = {}
+VECTOR_DB: List[Dict] = []
 VECTOR_MATRIX = None
-EXAM_MAP      = {}
+EXAM_MAP: Dict = {}
 
 app = Flask(__name__)
 CORS(app)
@@ -83,9 +96,9 @@ def init_firebase():
     global firebase_db_ref, FIREBASE_AVAILABLE
     try:
         creds_str = os.environ.get("FIREBASE")
-        db_url    = os.environ.get("Firebase_DB")
+        db_url    = os.environ.get("Firebase_DB") or os.environ.get("FIREBASE_DB")
         if not creds_str or not db_url:
-            logger.warning("Firebase env vars missing.")
+            logger.warning("Firebase env vars missing. Running without Firebase persistence.")
             return False
         if not firebase_admin._apps:
             cred = credentials.Certificate(json.loads(creds_str))
@@ -101,13 +114,18 @@ def init_firebase():
 FIREBASE_AVAILABLE = init_firebase()
 
 def fb_set(path, data):
-    if not FIREBASE_AVAILABLE: return
-    try: firebase_db_ref.child(path).set(data)
-    except Exception as e: logger.error(f"FB write [{path}]: {e}")
+    if not FIREBASE_AVAILABLE:
+        return
+    try:
+        firebase_db_ref.child(path).set(data)
+    except Exception as e:
+        logger.error(f"FB write [{path}]: {e}")
 
 def fb_get(path):
-    if not FIREBASE_AVAILABLE: return None
-    try: return firebase_db_ref.child(path).get()
+    if not FIREBASE_AVAILABLE:
+        return None
+    try:
+        return firebase_db_ref.child(path).get()
     except Exception as e:
         logger.error(f"FB read [{path}]: {e}")
         return None
@@ -124,62 +142,230 @@ def get_gemini():
     return _gemini_client
 
 # ---------------------------------------------------------------------------
-# VISION-BASED PAGE CLASSIFIER
-# Renders each page as an image and asks Gemini to classify it.
-# Falls back to heuristic if vision call fails or key is absent.
+# PDF / SYLLABUS DETECTION HELPERS
 # ---------------------------------------------------------------------------
 
-DEFINITE_BOILERPLATE_RE = re.compile(
-    r'^\s*(about\s+this\s+syllabus|foreword|acknowledgements?|'
-    r'why\s+choose\s+(cambridge|zimsec|this\s+syllabus)|cambridge\s+learner|'
-    r'key\s+benefits?|how\s+to\s+use\s+this\s+syllabus|'
-    r'support\s+for\s+(cambridge|teachers)|resource\s+list|'
-    r'further\s+information|copyright|legal\s+notice|'
-    r'changes\s+to\s+this\s+syllabus|university\s+of\s+cambridge|'
-    r'cambridge\s+assessment\s+international|published\s+by|'
-    r'contents?\s*$|table\s+of\s+contents?|'
-    r'assessment\s+at\s+a\s+glance|syllabus\s+at\s+a\s+glance|'
-    r'grade\s+descriptions?|command\s+words|glossary\s+of\s+command|'
-    r'mathematical\s+notation|other\s+cambridge\s+qualifications|'
-    r'how\s+to\s+offer|progression|post[-\s]?qualification|'
-    r'school\s+supported\s+candidate|cambridge\s+primary|cambridge\s+lower\s+secondary)\s*$',
-    re.IGNORECASE
+def clean_space(text: str) -> str:
+    return re.sub(r"\s+", " ", text or "").strip()
+
+def normalise_line(line: str) -> str:
+    line = clean_space(line)
+    line = re.sub(r"^[\d\.\s]+", "", line).strip()
+    return line
+
+def safe_firebase_key(value: str) -> str:
+    return re.sub(r'[.\[\]#$/]', '_', value)
+
+def infer_level_code_subject(filepath: str, filename: str) -> Tuple[str, str, str, str, bool]:
+    """
+    Returns: level, code, unique_id, subject_name, registry_match.
+    Fixes the old weak behaviour where parent folder was trusted blindly.
+    """
+    parts = filepath.replace("\\", "/").split("/")
+    parent = parts[-2].upper() if len(parts) >= 2 else ""
+
+    code_m = re.search(r'(?<!\d)(\d{4})(?!\d)', filename)
+    code = code_m.group(1) if code_m else "0000"
+
+    candidates = CODE_TO_LEVEL_SUBJECT.get(code, [])
+
+    if parent in {"A", "O"}:
+        preferred = [c for c in candidates if c[0] == parent]
+        if preferred:
+            level, uid, subject = preferred[0]
+            return level, code, uid, subject, True
+        # Parent says A/O but code is not in registry. Still preserve parent.
+        level = parent
+    elif len(candidates) == 1:
+        level, uid, subject = candidates[0]
+        return level, code, uid, subject, True
+    elif len(candidates) > 1:
+        # There are no overlaps in the current registry, but this keeps future safety.
+        level, uid, subject = candidates[0]
+        return level, code, uid, subject, True
+    else:
+        level = parent if parent in {"A", "O"} else "General"
+
+    uid = f"{level}_{code}"
+    fallback_subject = re.sub(r'[_\-]?\d{4}.*', '', filename, flags=re.IGNORECASE)
+    fallback_subject = fallback_subject.replace('_', ' ').replace('-', ' ').strip() or "Unknown Subject"
+    return level, code, uid, fallback_subject, False
+
+# Hard boilerplate headings. These are pages/blocks we do NOT want in the output tree.
+BOILERPLATE_HEADING_RE = re.compile(
+    r"^(about\s+this\s+syllabus|foreword|introduction|acknowledgements?|"
+    r"why\s+choose\s+(cambridge|zimsec|this\s+syllabus)|cambridge\s+learner|"
+    r"key\s+benefits?|how\s+to\s+use\s+this\s+syllabus|"
+    r"support\s+for\s+(cambridge|teachers)|resource\s+list|"
+    r"further\s+information|copyright|legal\s+notice|"
+    r"changes\s+to\s+this\s+syllabus|university\s+of\s+cambridge|"
+    r"cambridge\s+assessment\s+international|published\s+by|"
+    r"contents?|table\s+of\s+contents|"
+    r"assessment\s+at\s+a\s+glance|syllabus\s+at\s+a\s+glance|"
+    r"assessment\s+overview|scheme\s+of\s+assessment|"
+    r"assessment\s+objectives?|grade\s+descriptions?|command\s+words|"
+    r"glossary(\s+of\s+command\s+words)?|mathematical\s+notation|"
+    r"other\s+cambridge\s+qualifications|how\s+to\s+offer|progression|"
+    r"post[-\s]?qualification|school\s+supported\s+candidate|"
+    r"cambridge\s+primary|cambridge\s+lower\s+secondary|"
+    r"what\s+else\s+you\s+need\s+to\s+know|before\s+you\s+start|"
+    r"making\s+entries|after\s+the\s+exam|appendix|appendices)\b",
+    re.IGNORECASE,
 )
 
+# Strong signals for the actual learning/course content section.
 CONTENT_START_RE = re.compile(
-    r'(^|\n)\s*(\d+\.?\d*\s+[A-Z][a-z]|\d+\s+[A-Z][a-z]|'
-    r'subject\s+content|'
-    r'unit\s+\d|topic\s+\d|section\s+\d|module\s+\d|'
-    r'component\s+\d|paper\s+\d|'
-    r'learning\s+objectives|knowledge\s+and\s+understanding|'
-    r'candidates\s+should\s+be\s+able)',
-    re.IGNORECASE
+    r"(^|\n)\s*((\d+\.?\s*)?(subject\s+content|syllabus\s+content|curriculum\s+content|"
+    r"content\s+overview|learning\s+content|topics?\s+and\s+content|"
+    r"knowledge\s+and\s+understanding|set\s+texts|subject\s+specific\s+skills))\b",
+    re.IGNORECASE,
+)
+
+# Some syllabi have content pages made of topic numbers rather than a clean "Subject content" heading.
+STRONG_TOPIC_RE = re.compile(
+    r"(^|\n)\s*((\d+(\.\d+)*\s+)[A-Z][A-Za-z0-9,()/:\- ]{3,}|"
+    r"(Unit|Topic|Section|Module)\s+\d+\b|"
+    r"Candidates\s+should\s+be\s+able\s+to|Learning\s+objectives?)",
+    re.IGNORECASE,
+)
+
+CONTENT_END_RE = re.compile(
+    r"(^|\n)\s*((\d+\.?\s*)?(details\s+of\s+the\s+assessment|assessment\s+details|"
+    r"assessment\s+objectives?|assessment\s+criteria|scheme\s+of\s+assessment|"
+    r"command\s+words|glossary|grade\s+descriptions?|mathematical\s+notation|"
+    r"appendix|appendices|what\s+else\s+you\s+need\s+to\s+know|"
+    r"administrative\s+information|additional\s+information))\b",
+    re.IGNORECASE,
+)
+
+RESIDUAL_SKIP_RE = re.compile(
+    r"\b(Cambridge International|UCLES|Version\s+\d|Back to contents page|"
+    r"www\.cambridgeinternational\.org|For examination in|Syllabus for examination|"
+    r"Copyright|©|ISBN|Assessment at a glance|Why choose|About this syllabus)\b",
+    re.IGNORECASE,
 )
 
-def _page_to_base64_png(page, dpi=72) -> str:
-    mat  = fitz.Matrix(dpi / 72, dpi / 72)
-    pix  = page.get_pixmap(matrix=mat, colorspace=fitz.csRGB)
+PAGE_NUMBER_RE = re.compile(r"^(page\s*)?\d{1,3}\s*$", re.IGNORECASE)
+
+
+def get_page_lines(page, max_lines: int = 80) -> List[str]:
+    text = page.get_text("text") or ""
+    lines = [clean_space(x) for x in text.splitlines()]
+    return [x for x in lines if x][:max_lines]
+
+
+def extract_heading_candidates(doc, max_pages: int = 12) -> List[str]:
+    headings: List[str] = []
+    for i, page in enumerate(doc):
+        if i >= max_pages:
+            break
+        try:
+            page_dict = page.get_text("dict")
+            spans = []
+            for b in page_dict.get("blocks", []):
+                for l in b.get("lines", []):
+                    for s in l.get("spans", []):
+                        text = clean_space(s.get("text", ""))
+                        if text:
+                            spans.append((round(float(s.get("size", 0)), 1), text))
+            if not spans:
+                continue
+            sizes = sorted([s for s, _ in spans], reverse=True)
+            threshold = sizes[min(5, len(sizes) - 1)] if sizes else 0
+            for sz, text in spans:
+                if sz >= threshold and 3 <= len(text) <= 90 and not PAGE_NUMBER_RE.match(text):
+                    headings.append(text)
+        except Exception:
+            continue
+    seen = set()
+    out = []
+    for h in headings:
+        key = h.lower()
+        if key not in seen:
+            seen.add(key)
+            out.append(h)
+    return out[:25]
+
+
+def find_course_content_window(doc) -> Tuple[int, int, str]:
+    """
+    Finds the page range that most likely contains actual course content.
+    Old parser started too early and let Foreword/About/Contents leak into the tree.
+    """
+    n = len(doc)
+    start_idx: Optional[int] = None
+    start_reason = "fallback_topic_signal"
+
+    for i, page in enumerate(doc):
+        text = page.get_text("text") or ""
+        first_blob = "\n".join(get_page_lines(page, 40))
+        # Avoid TOC false positives by requiring more than just a contents list.
+        looks_like_toc = bool(re.search(r"\bcontents\b|table\s+of\s+contents", first_blob, re.I)) and len(first_blob.splitlines()) > 8
+        if CONTENT_START_RE.search(first_blob) and not looks_like_toc:
+            start_idx = i
+            start_reason = "explicit_subject_content_heading"
+            break
+        if i > 3 and STRONG_TOPIC_RE.search(text) and not BOILERPLATE_HEADING_RE.search(first_blob[:200]):
+            start_idx = i
+            start_reason = "strong_topic_signal"
+            break
+
+    if start_idx is None:
+        # Last resort: skip the typical front matter zone.
+        start_idx = min(8, max(0, n - 1))
+        start_reason = "last_resort_skip_front_matter"
+
+    end_idx = n
+    for j in range(start_idx + 1, n):
+        first_blob = "\n".join(get_page_lines(doc[j], 30))
+        if CONTENT_END_RE.search(first_blob):
+            end_idx = j
+            break
+
+    if end_idx <= start_idx:
+        end_idx = n
+
+    return start_idx, end_idx, start_reason
+
+
+def is_block_boilerplate(block_text: str) -> bool:
+    text = clean_space(block_text)
+    if not text:
+        return True
+    if PAGE_NUMBER_RE.match(text):
+        return True
+    if len(text) < 3:
+        return True
+    first_words = " ".join(text.split()[:10])
+    if BOILERPLATE_HEADING_RE.match(normalise_line(first_words)):
+        return True
+    if RESIDUAL_SKIP_RE.search(text) and len(text) < 180:
+        return True
+    return False
+
+
+def _page_to_base64_png(page, dpi=PAGE_CLASSIFY_DPI) -> str:
+    mat = fitz.Matrix(dpi / 72, dpi / 72)
+    pix = page.get_pixmap(matrix=mat, colorspace=fitz.csRGB)
     return base64.b64encode(pix.tobytes("png")).decode("utf-8")
 
-def _vision_classify_page(page, page_num: int, subject_name: str) -> str:
+
+def _vision_classify_page(page, page_num: int, subject_name: str, level: str) -> str:
     """Returns 'boilerplate', 'content', or 'uncertain'."""
     client = get_gemini()
     if client is None:
         return "uncertain"
     try:
-        b64    = _page_to_base64_png(page)
+        b64 = _page_to_base64_png(page)
+        qualification = "Cambridge International AS & A Level" if level == "A" else "Cambridge IGCSE / O Level"
         prompt = (
-            f"This is page {page_num + 1} of a Cambridge International AS & A Level / "
-            f"IGCSE syllabus for {subject_name}.\n\n"
+            f"This is page {page_num + 1} of a {qualification} syllabus for {subject_name}.\n\n"
             "Classify this page as ONE of:\n"
-            "BOILERPLATE - administrative or introductory content: foreword, about this "
-            "syllabus, why choose Cambridge, key benefits, Cambridge learner attributes, "
-            "how to use this syllabus, table of contents, copyright, assessment overview "
-            "tables, grade descriptions, command words, mathematical notation appendix, "
-            "support information, changes to syllabus, qualification overview.\n"
-            "CONTENT - actual subject matter students must learn: topic lists, learning "
-            "objectives, numbered content sections, subject-specific knowledge points, "
-            "skills, practical work descriptions, candidate assessment criteria.\n\n"
+            "BOILERPLATE - foreword, about this syllabus, why choose Cambridge, key benefits, contents, "
+            "assessment overview, assessment objectives, grade descriptions, command words, glossary, appendices, "
+            "administration, legal/copyright/support information.\n"
+            "CONTENT - actual course/subject matter students must learn: subject content, topic lists, learning "
+            "objectives, numbered content sections, skills, practical work content, set texts, knowledge points.\n\n"
             "Reply with exactly one word: BOILERPLATE or CONTENT"
         )
         resp = client.models.generate_content(
@@ -190,244 +376,335 @@ def _vision_classify_page(page, page_num: int, subject_name: str) -> str:
             ]}]
         )
         answer = (resp.text or "").strip().upper()
-        if "BOILERPLATE" in answer: return "boilerplate"
-        if "CONTENT" in answer:     return "content"
+        if "BOILERPLATE" in answer:
+            return "boilerplate"
+        if "CONTENT" in answer:
+            return "content"
         return "uncertain"
     except Exception as e:
-        logger.warning(f"Vision classify page {page_num}: {e}")
+        logger.warning(f"Vision classify page {page_num + 1}: {e}")
         return "uncertain"
 
-def classify_all_pages(doc, subject_name: str) -> list:
+
+def classify_all_pages(doc, subject_name: str, level: str) -> Tuple[List[str], Dict]:
     """
-    Returns list of 'boilerplate' or 'content' for each page.
-    Uses vision for first 40 pages, heuristic after that.
-    Caches result to avoid re-classifying on incremental runs.
+    Returns page labels and debug info.
+    The main fix: do not let the parser start at foreword/contents. First find the course-content window,
+    then classify only inside that window.
     """
-    classifications = []
     n = len(doc)
+    start_idx, end_idx, start_reason = find_course_content_window(doc)
+    classifications: List[str] = []
+    debug_pages = []
 
     for i, page in enumerate(doc):
-        text = page.get_text("text").strip()
-
-        # Pages beyond the front-matter zone are almost always content
-        if i >= 40:
-            classifications.append("content")
-            continue
-
-        # Hard-rule catch
-        first_lines = [l.strip() for l in text.splitlines() if l.strip()][:3]
-        if first_lines and DEFINITE_BOILERPLATE_RE.match(first_lines[0]):
-            classifications.append("boilerplate")
-            continue
+        text = page.get_text("text") or ""
+        first_lines = get_page_lines(page, 5)
+        first_heading = first_lines[0] if first_lines else ""
 
-        # Empty page
-        if len(text) < 30:
-            classifications.append("boilerplate")
-            continue
+        if i < start_idx or i >= end_idx:
+            verdict = "boilerplate"
+            reason = "outside_content_window"
+        else:
+            first_blob = "\n".join(get_page_lines(page, 35))
+            if len(clean_space(text)) < 30:
+                verdict = "boilerplate"
+                reason = "empty_or_tiny_page"
+            elif CONTENT_END_RE.search(first_blob):
+                verdict = "boilerplate"
+                reason = "assessment_or_appendix_heading"
+            elif BOILERPLATE_HEADING_RE.match(normalise_line(first_heading)):
+                verdict = "boilerplate"
+                reason = "boilerplate_heading"
+            elif CONTENT_START_RE.search(first_blob) or STRONG_TOPIC_RE.search(text):
+                verdict = "content"
+                reason = "text_content_signal"
+            elif i < MAX_VISION_PAGES:
+                v = _vision_classify_page(page, i, subject_name, level)
+                if v == "uncertain":
+                    verdict = "content" if STRONG_TOPIC_RE.search(text) else "boilerplate"
+                    reason = "heuristic_after_uncertain_vision"
+                else:
+                    verdict = v
+                    reason = f"vision_{VISION_MODEL}"
+            else:
+                verdict = "content"
+                reason = "inside_window_after_vision_limit"
 
-        # Vision call
-        verdict = _vision_classify_page(page, i, subject_name)
-        if verdict == "uncertain":
-            verdict = "content" if CONTENT_START_RE.search(text) else "boilerplate"
         classifications.append(verdict)
-        logger.info(f"  Page {i+1}/{n}: {verdict}")
+        debug_pages.append({"page": i + 1, "verdict": verdict, "reason": reason, "first": first_heading[:100]})
+        logger.info(f"  Page {i + 1}/{n}: {verdict} ({reason}) | {first_heading[:90]}")
 
-    # Safety: if vision misclassified everything as boilerplate, use heuristic fallback
     if not any(c == "content" for c in classifications):
-        logger.warning(f"  All pages BOILERPLATE for {subject_name} — applying heuristic fallback.")
-        classifications = []
-        found_content = False
-        for i, page in enumerate(doc):
-            text = page.get_text("text")
-            if not found_content and CONTENT_START_RE.search(text):
-                found_content = True
-            classifications.append("content" if found_content else "boilerplate")
-
-    return classifications
-
+        logger.warning(f"  All pages BOILERPLATE for {subject_name}; applying fallback from page {start_idx + 1}.")
+        classifications = ["content" if i >= start_idx else "boilerplate" for i in range(n)]
+
+    debug = {
+        "content_start_page": start_idx + 1,
+        "content_end_page_exclusive": end_idx + 1 if end_idx < n else None,
+        "content_start_reason": start_reason,
+        "vision_model": VISION_MODEL,
+        "page_debug": debug_pages,
+    }
+    return classifications, debug
 
 # ---------------------------------------------------------------------------
-# PDF PARSER  — Vision-enhanced
+# PDF PARSER
 # ---------------------------------------------------------------------------
 
 class PDFParser:
     def __init__(self, filepath):
-        self.filepath     = filepath
-        self.filename     = os.path.basename(filepath)
-        self.doc          = fitz.open(filepath)
-        parts             = filepath.replace("\\", "/").split("/")
-        self.level        = parts[-2] if len(parts) > 1 else "General"
-        code_m            = re.search(r'\d{4}', self.filename)
-        self.subject_code = code_m.group(0) if code_m else "0000"
-        self.unique_id    = f"{self.level}_{self.subject_code}"
-        self.subject_name = ALL_SUBJECTS.get(
-            self.unique_id,
-            re.sub(r'[_\-]\d{4}.*', '', self.filename.replace('_', ' ')).strip()
-        )
+        self.filepath = filepath
+        self.filename = os.path.basename(filepath)
+        self.doc = fitz.open(filepath)
+        self.level, self.subject_code, self.unique_id, self.subject_name, self.registry_match = infer_level_code_subject(filepath, self.filename)
+        self.debug_info: Dict = {}
+
+    def log_pdf_detection_summary(self):
+        first_page_lines = get_page_lines(self.doc[0], 12) if len(self.doc) else []
+        heading_candidates = extract_heading_candidates(self.doc)
+        logger.info("=" * 78)
+        logger.info(f"PDF DETECTED: {self.filepath}")
+        logger.info(f"  filename:       {self.filename}")
+        logger.info(f"  pages:          {len(self.doc)}")
+        logger.info(f"  folder level:   {self.filepath.replace('\\\\', '/').split('/')[-2] if '/' in self.filepath.replace('\\\\', '/') else 'unknown'}")
+        logger.info(f"  detected code:  {self.subject_code}")
+        logger.info(f"  resolved id:    {self.unique_id}")
+        logger.info(f"  subject:        {self.subject_name}")
+        logger.info(f"  registry match: {self.registry_match}")
+        logger.info(f"  model:          {VISION_MODEL}")
+        logger.info(f"  first-page lines seen: {first_page_lines[:8]}")
+        logger.info(f"  heading candidates seen: {heading_candidates[:15]}")
+        logger.info("=" * 78)
 
     def get_body_font_size(self):
-        sizes = {}
+        sizes: Dict[float, int] = {}
         for page in self.doc:
-            for b in page.get_text("dict")["blocks"]:
-                for l in b.get("lines", []):
-                    for s in l.get("spans", []):
-                        sz = round(s["size"], 1)
-                        sizes[sz] = sizes.get(sz, 0) + len(s["text"])
+            try:
+                for b in page.get_text("dict").get("blocks", []):
+                    for l in b.get("lines", []):
+                        for s in l.get("spans", []):
+                            text = s.get("text", "")
+                            if not clean_space(text):
+                                continue
+                            sz = round(float(s.get("size", 10)), 1)
+                            sizes[sz] = sizes.get(sz, 0) + len(text)
+            except Exception:
+                continue
         return max(sizes, key=sizes.get) if sizes else 10.0
 
     def parse(self):
-        body_size     = self.get_body_font_size()
-        page_classes  = classify_all_pages(self.doc, self.subject_name)
-        topic_pattern = re.compile(r'^(\d+\.?\s|Key\s+Question\s)', re.IGNORECASE)
+        self.log_pdf_detection_summary()
+        body_size = self.get_body_font_size()
+        page_classes, class_debug = classify_all_pages(self.doc, self.subject_name, self.level)
+        self.debug_info = class_debug
 
-        logger.info(f"Parsing content of {self.filename} (body ~{body_size}pt)")
+        logger.info(f"Parsing COURSE CONTENT of {self.filename} (body ~{body_size}pt)")
         content_page_count = sum(1 for c in page_classes if c == "content")
-        logger.info(f"  {content_page_count} content pages out of {len(self.doc)} total")
-
-        syllabus_tree    = []
-        current_topic    = None
-        current_subtopic = None
+        logger.info(f"  {content_page_count} course-content pages out of {len(self.doc)} total")
+        logger.info(f"  content window starts page {class_debug.get('content_start_page')} via {class_debug.get('content_start_reason')}")
+
+        syllabus_tree: List[Dict] = []
+        current_topic: Optional[Dict] = None
+        current_subtopic: Optional[Dict] = None
+
+        def flush_subtopic():
+            nonlocal current_subtopic, current_topic
+            if current_subtopic and current_topic:
+                # Only keep subtopics with meaningful body content.
+                content = [c for c in current_subtopic.get("content", []) if not is_block_boilerplate(c)]
+                current_subtopic["content"] = content
+                if content or len(clean_space(current_subtopic.get("title", ""))) > 3:
+                    current_topic["children"].append(current_subtopic)
+            current_subtopic = None
+
+        def flush_topic():
+            nonlocal current_topic
+            if current_topic:
+                # Drop empty topics or headings that are actually admin labels.
+                if current_topic.get("children") and not is_block_boilerplate(current_topic.get("title", "")):
+                    syllabus_tree.append(current_topic)
+            current_topic = None
 
         for page_num, page in enumerate(self.doc):
             if page_classes[page_num] == "boilerplate":
                 continue
 
-            for b in page.get_text("dict")["blocks"]:
-                block_text = ""
-                max_size   = 0
-                is_bold    = False
+            try:
+                page_dict = page.get_text("dict")
+            except Exception as e:
+                logger.warning(f"Could not parse page {page_num + 1} of {self.filename}: {e}")
+                continue
+
+            for b in page_dict.get("blocks", []):
+                block_text_parts = []
+                max_size = 0.0
+                is_bold = False
 
                 for l in b.get("lines", []):
+                    line_parts = []
                     for s in l.get("spans", []):
-                        t = s["text"].strip()
-                        if not t: continue
-                        block_text += t + " "
-                        if s["size"] > max_size: max_size = s["size"]
-                        if "bold" in s["font"].lower(): is_bold = True
-
-                block_text = block_text.strip()
-                if len(block_text) < 3: continue
-
-                # Skip residual boilerplate blocks within content pages
-                first_words = " ".join(block_text.split()[:6])
-                if DEFINITE_BOILERPLATE_RE.match(first_words):
+                        t = clean_space(s.get("text", ""))
+                        if not t:
+                            continue
+                        line_parts.append(t)
+                        sz = float(s.get("size", body_size))
+                        if sz > max_size:
+                            max_size = sz
+                        if "bold" in str(s.get("font", "")).lower():
+                            is_bold = True
+                    if line_parts:
+                        block_text_parts.append(" ".join(line_parts))
+
+                block_text = clean_space(" ".join(block_text_parts))
+                if is_block_boilerplate(block_text):
+                    continue
+
+                # Prevent assessment/admin sections from leaking even if page got through.
+                if CONTENT_END_RE.match("\n" + block_text):
                     continue
 
-                # TOPIC
-                if max_size > body_size + 2:
-                    if current_subtopic and current_topic:
-                        current_topic["children"].append(current_subtopic)
-                        current_subtopic = None
-                    if current_topic:
-                        syllabus_tree.append(current_topic)
+                # Topic/subtopic detection tuned to syllabus PDFs.
+                looks_like_major_heading = (
+                    max_size >= body_size + 2.0
+                    and len(block_text) <= 140
+                    and not block_text.endswith(".")
+                )
+                looks_like_numbered_topic = bool(re.match(r"^(\d+(\.\d+)*\s+)[A-Z]", block_text)) and len(block_text) <= 160
+                looks_like_subheading = (
+                    (is_bold and max_size >= body_size and len(block_text) <= 180)
+                    or looks_like_numbered_topic
+                    or bool(re.match(r"^(Unit|Topic|Section|Module)\s+\d+\b", block_text, re.I))
+                )
+
+                if looks_like_major_heading:
+                    flush_subtopic()
+                    flush_topic()
                     current_topic = {
-                        "id":       f"{self.unique_id}_{len(syllabus_tree)}",
-                        "title":    block_text,
-                        "type":     "topic",
-                        "children": []
+                        "id": f"{self.unique_id}_{len(syllabus_tree)}",
+                        "title": block_text,
+                        "type": "topic",
+                        "page": page_num + 1,
+                        "children": [],
                     }
                     current_subtopic = None
-
-                # SUBTOPIC
-                elif (is_bold and max_size >= body_size) or \
-                     (topic_pattern.match(block_text) and max_size >= body_size):
-                    if current_subtopic and current_topic:
-                        current_topic["children"].append(current_subtopic)
+                elif looks_like_subheading:
                     if not current_topic:
                         current_topic = {
                             "id": f"{self.unique_id}_root",
-                            "title": "Syllabus Content",
+                            "title": "Course Content",
                             "type": "topic",
-                            "children": []
+                            "page": page_num + 1,
+                            "children": [],
                         }
+                    flush_subtopic()
                     current_subtopic = {
-                        "id":      f"{current_topic['id']}_{len(current_topic['children'])}",
-                        "title":   block_text,
-                        "type":    "subtopic",
-                        "content": []
+                        "id": f"{current_topic['id']}_{len(current_topic['children'])}",
+                        "title": block_text,
+                        "type": "subtopic",
+                        "page": page_num + 1,
+                        "content": [],
                     }
-
-                # BODY
-                elif max_size <= body_size + 1:
-                    if current_subtopic:
-                        current_subtopic["content"].append(block_text)
-                    elif current_topic:
+                else:
+                    if not current_topic:
+                        current_topic = {
+                            "id": f"{self.unique_id}_root",
+                            "title": "Course Content",
+                            "type": "topic",
+                            "page": page_num + 1,
+                            "children": [],
+                        }
+                    if not current_subtopic:
                         current_subtopic = {
-                            "id":      f"{current_topic['id']}_intro",
-                            "title":   "Overview",
-                            "type":    "subtopic",
-                            "content": [block_text]
+                            "id": f"{current_topic['id']}_intro",
+                            "title": "Overview",
+                            "type": "subtopic",
+                            "page": page_num + 1,
+                            "content": [],
                         }
+                    current_subtopic["content"].append(block_text)
+
+        flush_subtopic()
+        flush_topic()
 
-        if current_subtopic and current_topic:
-            current_topic["children"].append(current_subtopic)
-        if current_topic:
-            syllabus_tree.append(current_topic)
+        logger.info(f"  Parsed topics for {self.unique_id}: {len(syllabus_tree)}")
+        preview = [t.get("title") for t in syllabus_tree[:8]]
+        logger.info(f"  Topic preview: {preview}")
 
         return {
             "meta": {
-                "id":         self.unique_id,
-                "subject":    self.subject_name,
-                "code":       self.subject_code,
-                "level":      self.level,
-                "filename":   self.filename,
-                "indexed_at": int(time.time())
+                "id": self.unique_id,
+                "subject": self.subject_name,
+                "code": self.subject_code,
+                "level": self.level,
+                "filename": self.filename,
+                "registryMatch": self.registry_match,
+                "indexed_at": int(time.time()),
+                "parserVersion": "3.1-course-content-window",
+                "model": VISION_MODEL,
+                "contentStartPage": class_debug.get("content_start_page"),
+                "contentStartReason": class_debug.get("content_start_reason"),
             },
-            "tree": syllabus_tree
+            "debug": {
+                "pageClassification": class_debug,
+            },
+            "tree": syllabus_tree,
         }
 
-
 # ---------------------------------------------------------------------------
 # PAST EXAM PARSER
 # ---------------------------------------------------------------------------
 
 class ExamPaperParser:
     def __init__(self, filepath):
-        self.filepath     = filepath
-        self.filename     = os.path.basename(filepath)
-        self.doc          = fitz.open(filepath)
-        parts             = filepath.replace("\\", "/").split("/")
-        self.level        = parts[-2] if len(parts) > 1 else "General"
-        code_m            = re.search(r'\b(\d{4})\b', self.filename)
-        self.subject_code = code_m.group(1) if code_m else "0000"
-        self.unique_id    = f"{self.level}_{self.subject_code}"
-        year_m            = re.search(r'\b(20\d{2}|19\d{2})\b', self.filename)
-        self.year         = year_m.group(1) if year_m else "Unknown"
-        sess_m            = re.search(r'(may[_\-]?june|oct[_\-]?nov|feb[_\-]?mar|summer|winter|s\d|w\d|m\d)', self.filename, re.IGNORECASE)
-        self.session      = sess_m.group(1).upper() if sess_m else "Unknown"
-        paper_m           = re.search(r'[_\-]p(\d)|paper[\s_\-]?(\d)', self.filename, re.IGNORECASE)
-        self.paper_num    = (paper_m.group(1) or paper_m.group(2)) if paper_m else "1"
-        self.paper_id     = f"{self.unique_id}_{self.year}_{self.session}_P{self.paper_num}"
+        self.filepath = filepath
+        self.filename = os.path.basename(filepath)
+        self.doc = fitz.open(filepath)
+        self.level, self.subject_code, self.unique_id, self.subject_name, self.registry_match = infer_level_code_subject(filepath, self.filename)
+        year_m = re.search(r'\b(20\d{2}|19\d{2})\b', self.filename)
+        self.year = year_m.group(1) if year_m else "Unknown"
+        sess_m = re.search(r'(may[_\-]?june|oct[_\-]?nov|feb[_\-]?mar|summer|winter|s\d|w\d|m\d)', self.filename, re.IGNORECASE)
+        self.session = sess_m.group(1).upper() if sess_m else "Unknown"
+        paper_m = re.search(r'[_\-]p(\d)|paper[\s_\-]?(\d)', self.filename, re.IGNORECASE)
+        self.paper_num = (paper_m.group(1) or paper_m.group(2)) if paper_m else "1"
+        self.paper_id = f"{self.unique_id}_{self.year}_{self.session}_P{self.paper_num}"
 
     def extract_pages(self):
-        return [{"page": i + 1, "text": p.get_text("text").strip()[:3000]}
-                for i, p in enumerate(self.doc) if p.get_text("text").strip()]
+        return [
+            {"page": i + 1, "text": p.get_text("text").strip()[:3000]}
+            for i, p in enumerate(self.doc)
+            if p.get_text("text").strip()
+        ]
 
     def extract_questions(self):
         full = "\n".join(p["text"] for p in self.extract_pages())
-        pat  = re.compile(r'(?:^|\n)\s*(\d{1,2})\s*[\.\)]\s+(.+?)(?=\n\s*\d{1,2}\s*[\.\)]|\Z)', re.DOTALL | re.MULTILINE)
-        return [{"number": int(m.group(1)), "text": m.group(2).strip()[:2000]}
-                for m in pat.finditer(full) if len(m.group(2).strip()) > 20]
+        pat = re.compile(r'(?:^|\n)\s*(\d{1,2})\s*[\.\)]\s+(.+?)(?=\n\s*\d{1,2}\s*[\.\)]|\Z)', re.DOTALL | re.MULTILINE)
+        return [
+            {"number": int(m.group(1)), "text": m.group(2).strip()[:2000]}
+            for m in pat.finditer(full)
+            if len(m.group(2).strip()) > 20
+        ]
 
     def parse(self):
+        logger.info(f"Exam PDF detected: {self.filepath} -> {self.unique_id} {self.subject_name}, year={self.year}, paper={self.paper_num}")
         return {
             "meta": {
-                "paperId":     self.paper_id,
-                "subjectId":   self.unique_id,
+                "paperId": self.paper_id,
+                "subjectId": self.unique_id,
+                "subject": self.subject_name,
                 "subjectCode": self.subject_code,
-                "level":       self.level,
-                "year":        self.year,
-                "session":     self.session,
+                "level": self.level,
+                "year": self.year,
+                "session": self.session,
                 "paperNumber": self.paper_num,
-                "filename":    self.filename,
-                "totalPages":  len(self.doc),
-                "indexed_at":  int(time.time())
+                "filename": self.filename,
+                "totalPages": len(self.doc),
+                "indexed_at": int(time.time()),
             },
-            "pages":     self.extract_pages(),
-            "questions": self.extract_questions()
+            "pages": self.extract_pages(),
+            "questions": self.extract_questions(),
         }
 
-
 # ---------------------------------------------------------------------------
 # EMBEDDINGS
 # ---------------------------------------------------------------------------
@@ -435,7 +712,9 @@ class ExamPaperParser:
 def generate_embeddings(texts):
     client = get_gemini()
     if client is None:
+        logger.warning("Gemini API key not configured. Returning zero embeddings.")
         return [np.zeros(768).tolist() for _ in texts]
+
     results = []
     for i in range(0, len(texts), 10):
         batch = texts[i:i + 10]
@@ -449,30 +728,36 @@ def generate_embeddings(texts):
                 results.append(np.zeros(768).tolist())
     return results
 
-
 # ---------------------------------------------------------------------------
 # FIREBASE PERSISTENCE
 # ---------------------------------------------------------------------------
 
 def load_index_from_firebase():
     global SYLLABUS_MAP, VECTOR_DB, VECTOR_MATRIX, EXAM_MAP
-    if not FIREBASE_AVAILABLE: return False
+    if not FIREBASE_AVAILABLE:
+        return False
     logger.info("Loading index from Firebase ...")
     try:
         fb_syllabi = fb_get("data_api/syllabi")
-        if not fb_syllabi: return False
+        if not fb_syllabi:
+            return False
         SYLLABUS_MAP = fb_syllabi
 
         fb_vectors = fb_get("data_api/vectors")
-        if not fb_vectors: return False
-        VECTOR_DB  = []
-        valid      = []
-        expected_dim = 768  # text-embedding-004 output dimension
-        for entry in sorted(fb_vectors.keys() if isinstance(fb_vectors, dict) else range(len(fb_vectors))):
+        if not fb_vectors:
+            return False
+
+        VECTOR_DB = []
+        valid = []
+        expected_dim = 768
+        iterable = sorted(fb_vectors.keys()) if isinstance(fb_vectors, dict) else range(len(fb_vectors))
+        for entry in iterable:
             item = fb_vectors[entry] if isinstance(fb_vectors, dict) else fb_vectors[entry]
-            if not item: continue
+            if not item:
+                continue
             raw_vec = item.get("vector")
-            if not raw_vec: continue
+            if not raw_vec:
+                continue
             try:
                 vec = np.array(raw_vec, dtype=np.float32)
                 if vec.ndim != 1 or len(vec) != expected_dim:
@@ -483,8 +768,8 @@ def load_index_from_firebase():
             except Exception as ve:
                 logger.warning(f"Skipping malformed vector entry: {ve}")
                 continue
-        if valid:
-            VECTOR_MATRIX = np.vstack(valid).astype(np.float32)
+
+        VECTOR_MATRIX = np.vstack(valid).astype(np.float32) if valid else None
         logger.info(f"Vector matrix shape: {VECTOR_MATRIX.shape if VECTOR_MATRIX is not None else None}")
 
         fb_exams = fb_get("data_api/exams")
@@ -497,63 +782,77 @@ def load_index_from_firebase():
         logger.error(f"Firebase load: {e}")
         return False
 
+
 def save_syllabus(sid, data):
     fb_set(f"data_api/syllabi/{sid}", data)
 
+
 def save_all_vectors():
     fb_data = {}
     for i, entry in enumerate(VECTOR_DB):
         fb_data[f"v_{i:06d}"] = {
             "vector": entry["vector"].tolist() if isinstance(entry["vector"], np.ndarray) else entry["vector"],
-            "meta":   entry["meta"]
+            "meta": entry["meta"],
         }
     fb_set("data_api/vectors", fb_data)
 
+
 def save_exam(sid, exam_data):
-    safe = re.sub(r'[.\[\]#$/]', '_', exam_data["meta"]["paperId"])
+    safe = safe_firebase_key(exam_data["meta"]["paperId"])
     fb_set(f"data_api/exams/{sid}/{safe}", exam_data)
 
-
 # ---------------------------------------------------------------------------
 # INDEX BUILDER
 # ---------------------------------------------------------------------------
 
+def syllabus_sort_key(path: str):
+    level, code, uid, subject, registry_match = infer_level_code_subject(path, os.path.basename(path))
+    return (level, subject, code, path)
+
+
 def build_index():
     global SYLLABUS_MAP, VECTOR_DB, VECTOR_MATRIX, EXAM_MAP
     logger.info("Full index build starting ...")
+    logger.info(f"Using Gemini model for page classification: {VISION_MODEL}")
     parsed_data = []
 
     if os.path.exists(SYLLABI_DIR):
+        syllabus_paths = []
         for root, _, files in os.walk(SYLLABI_DIR):
-            for f in sorted(files):
-                if not f.endswith(".pdf"): continue
-                path = os.path.join(root, f)
-                logger.info(f"Syllabus: {path}")
-                try:
-                    parser = PDFParser(path)
-                    data   = parser.parse()
-                    parsed_data.append(data)
-                    SYLLABUS_MAP[data["meta"]["id"]] = data
-                    save_syllabus(data["meta"]["id"], data)
-                except Exception as e:
-                    logger.error(f"{path}: {e}")
+            for f in files:
+                if f.lower().endswith(".pdf"):
+                    syllabus_paths.append(os.path.join(root, f))
+        logger.info(f"Found {len(syllabus_paths)} syllabus PDF(s).")
+        for path in sorted(syllabus_paths, key=syllabus_sort_key):
+            logger.info(f"Syllabus queued: {path}")
+            try:
+                parser = PDFParser(path)
+                data = parser.parse()
+                parsed_data.append(data)
+                SYLLABUS_MAP[data["meta"]["id"]] = data
+                save_syllabus(data["meta"]["id"], data)
+            except Exception as e:
+                logger.exception(f"Failed syllabus parse {path}: {e}")
 
     if os.path.exists(PAST_EXAMS_DIR):
+        exam_paths = []
         for root, _, files in os.walk(PAST_EXAMS_DIR):
-            for f in sorted(files):
-                if not f.endswith(".pdf"): continue
-                path = os.path.join(root, f)
-                logger.info(f"Exam: {path}")
-                try:
-                    parser    = ExamPaperParser(path)
-                    exam_data = parser.parse()
-                    sid       = exam_data["meta"]["subjectId"]
-                    if sid not in EXAM_MAP: EXAM_MAP[sid] = {}
-                    safe      = re.sub(r'[.\[\]#$/]', '_', exam_data["meta"]["paperId"])
-                    EXAM_MAP[sid][safe] = exam_data
-                    save_exam(sid, exam_data)
-                except Exception as e:
-                    logger.error(f"{path}: {e}")
+            for f in files:
+                if f.lower().endswith(".pdf"):
+                    exam_paths.append(os.path.join(root, f))
+        logger.info(f"Found {len(exam_paths)} past exam PDF(s).")
+        for path in sorted(exam_paths):
+            logger.info(f"Exam queued: {path}")
+            try:
+                parser = ExamPaperParser(path)
+                exam_data = parser.parse()
+                sid = exam_data["meta"]["subjectId"]
+                EXAM_MAP.setdefault(sid, {})
+                safe = safe_firebase_key(exam_data["meta"]["paperId"])
+                EXAM_MAP[sid][safe] = exam_data
+                save_exam(sid, exam_data)
+            except Exception as e:
+                logger.exception(f"Failed exam parse {path}: {e}")
 
     if not parsed_data:
         logger.info("Nothing to vectorize.")
@@ -562,57 +861,66 @@ def build_index():
     chunks, metas = [], []
     for item in parsed_data:
         mb = item["meta"]
-        for topic in item["tree"]:
+        for topic in item.get("tree", []):
             for sub in topic.get("children", []):
                 blob = "\n".join(sub.get("content", []))
-                if len(blob) < 10: continue
-                chunks.append(f"{mb['subject']} {mb['level']} - {topic['title']} - {sub['title']}:\n{blob}")
+                if len(clean_space(blob)) < MIN_VECTOR_TEXT_CHARS:
+                    continue
+                chunks.append(f"{mb['subject']} {mb['level']} {mb['code']} - {topic['title']} - {sub['title']}:\n{blob}")
                 metas.append({
-                    "subject_id":  mb["id"],
-                    "topic_id":    topic["id"],
+                    "subject_id": mb["id"],
+                    "subject": mb["subject"],
+                    "level": mb["level"],
+                    "code": mb["code"],
+                    "topic_id": topic["id"],
+                    "topic_title": topic["title"],
                     "subtopic_id": sub["id"],
-                    "title":       sub["title"],
-                    "content":     blob
+                    "title": sub["title"],
+                    "content": blob,
                 })
 
-    logger.info(f"Embedding {len(chunks)} chunks ...")
+    logger.info(f"Embedding {len(chunks)} course-content chunks ...")
     vecs = generate_embeddings(chunks)
     VECTOR_DB = []
-    valid     = []
+    valid = []
     for i, v in enumerate(vecs):
-        nv = np.array(v)
+        nv = np.array(v, dtype=np.float32)
         VECTOR_DB.append({"vector": nv, "meta": metas[i]})
         valid.append(nv)
-    if valid:
-        VECTOR_MATRIX = np.vstack(valid)
+    VECTOR_MATRIX = np.vstack(valid).astype(np.float32) if valid else None
     save_all_vectors()
     logger.info(f"Index done: {len(SYLLABUS_MAP)} syllabi, {len(VECTOR_DB)} vectors.")
 
 
 def _incremental_vectorize(syllabus_data):
     global VECTOR_DB, VECTOR_MATRIX
-    mb     = syllabus_data["meta"]
+    mb = syllabus_data["meta"]
     chunks, metas = [], []
-    for topic in syllabus_data["tree"]:
+    for topic in syllabus_data.get("tree", []):
         for sub in topic.get("children", []):
             blob = "\n".join(sub.get("content", []))
-            if len(blob) < 10: continue
-            chunks.append(f"{mb['subject']} {mb['level']} - {topic['title']} - {sub['title']}:\n{blob}")
+            if len(clean_space(blob)) < MIN_VECTOR_TEXT_CHARS:
+                continue
+            chunks.append(f"{mb['subject']} {mb['level']} {mb['code']} - {topic['title']} - {sub['title']}:\n{blob}")
             metas.append({
-                "subject_id":  mb["id"],
-                "topic_id":    topic["id"],
+                "subject_id": mb["id"],
+                "subject": mb["subject"],
+                "level": mb["level"],
+                "code": mb["code"],
+                "topic_id": topic["id"],
+                "topic_title": topic["title"],
                 "subtopic_id": sub["id"],
-                "title":       sub["title"],
-                "content":     blob
+                "title": sub["title"],
+                "content": blob,
             })
-    if not chunks: return
+    if not chunks:
+        return
     for i, v in enumerate(generate_embeddings(chunks)):
-        VECTOR_DB.append({"vector": np.array(v), "meta": metas[i]})
+        VECTOR_DB.append({"vector": np.array(v, dtype=np.float32), "meta": metas[i]})
     if VECTOR_DB:
-        VECTOR_MATRIX = np.vstack([e["vector"] for e in VECTOR_DB])
+        VECTOR_MATRIX = np.vstack([e["vector"] for e in VECTOR_DB]).astype(np.float32)
     save_all_vectors()
 
-
 # ---------------------------------------------------------------------------
 # WATCHER
 # ---------------------------------------------------------------------------
@@ -620,42 +928,46 @@ _indexed_files = set()
 
 def _collect_existing():
     for d in [SYLLABI_DIR, PAST_EXAMS_DIR]:
-        if not os.path.exists(d): continue
+        if not os.path.exists(d):
+            continue
         for root, _, files in os.walk(d):
             for f in files:
-                if f.endswith(".pdf"):
+                if f.lower().endswith(".pdf"):
                     _indexed_files.add(os.path.join(root, f))
 
+
 def _watch(interval=30):
     while True:
         time.sleep(interval)
         for directory, is_exam in [(SYLLABI_DIR, False), (PAST_EXAMS_DIR, True)]:
-            if not os.path.exists(directory): continue
+            if not os.path.exists(directory):
+                continue
             for root, _, files in os.walk(directory):
                 for f in files:
-                    if not f.endswith(".pdf"): continue
+                    if not f.lower().endswith(".pdf"):
+                        continue
                     path = os.path.join(root, f)
-                    if path in _indexed_files: continue
+                    if path in _indexed_files:
+                        continue
                     _indexed_files.add(path)
-                    logger.info(f"New PDF: {path}")
+                    logger.info(f"New PDF detected by watcher: {path}")
                     try:
                         if is_exam:
-                            parser    = ExamPaperParser(path)
+                            parser = ExamPaperParser(path)
                             exam_data = parser.parse()
-                            sid       = exam_data["meta"]["subjectId"]
-                            if sid not in EXAM_MAP: EXAM_MAP[sid] = {}
-                            safe      = re.sub(r'[.\[\]#$/]', '_', exam_data["meta"]["paperId"])
+                            sid = exam_data["meta"]["subjectId"]
+                            EXAM_MAP.setdefault(sid, {})
+                            safe = safe_firebase_key(exam_data["meta"]["paperId"])
                             EXAM_MAP[sid][safe] = exam_data
                             save_exam(sid, exam_data)
                         else:
                             parser = PDFParser(path)
-                            data   = parser.parse()
+                            data = parser.parse()
                             SYLLABUS_MAP[data["meta"]["id"]] = data
                             save_syllabus(data["meta"]["id"], data)
                             _incremental_vectorize(data)
                     except Exception as e:
-                        logger.error(f"Watch {path}: {e}")
-
+                        logger.exception(f"Watch parse failed {path}: {e}")
 
 # ---------------------------------------------------------------------------
 # API
@@ -663,14 +975,15 @@ def _watch(interval=30):
 
 @app.route('/', methods=['GET'])
 def index():
-    """Root route — required for HuggingFace Spaces health checks and iframe rendering."""
     return jsonify({
-        "name":            "Marka Data API",
-        "version":         "2.0",
-        "status":          "online",
+        "name": "Marka Data API",
+        "version": "3.1",
+        "status": "online",
         "subjects_loaded": len(SYLLABUS_MAP),
-        "vector_chunks":   len(VECTOR_DB),
-        "firebase":        FIREBASE_AVAILABLE,
+        "vector_chunks": len(VECTOR_DB),
+        "firebase": FIREBASE_AVAILABLE,
+        "model": VISION_MODEL,
+        "parser": "course-content-window-with-pdf-logging",
         "endpoints": [
             "GET  /health",
             "GET  /v1/subjects",
@@ -679,34 +992,36 @@ def index():
             "GET  /v1/exams",
             "GET  /v1/exams/<paper_id>",
             "GET  /v1/exams/<paper_id>/questions",
-            "POST /v1/rebuild"
-        ]
+            "POST /v1/rebuild",
+            "GET|POST /v1/reset",
+        ],
     })
 
-
 @app.route('/health', methods=['GET'])
 def health():
     return jsonify({
-        "status":          "online",
+        "status": "online",
         "subjects_loaded": list(SYLLABUS_MAP.keys()),
-        "subject_count":   len(SYLLABUS_MAP),
-        "vector_chunks":   len(VECTOR_DB),
-        "exam_subjects":   list(EXAM_MAP.keys()),
-        "firebase":        FIREBASE_AVAILABLE,
-        "registered_subjects": ALL_SUBJECTS
+        "subject_count": len(SYLLABUS_MAP),
+        "vector_chunks": len(VECTOR_DB),
+        "exam_subjects": list(EXAM_MAP.keys()),
+        "firebase": FIREBASE_AVAILABLE,
+        "registered_subjects": ALL_SUBJECTS,
+        "model": VISION_MODEL,
+        "embedding_model": EMBEDDING_MODEL,
     })
 
 @app.route('/v1/subjects', methods=['GET'])
 def list_subjects():
     result = []
     for sid, data in SYLLABUS_MAP.items():
-        result.append({**data.get("meta", {"id": sid}), "indexed": True})
+        meta = data.get("meta", {"id": sid})
+        result.append({**meta, "indexed": True})
     for uid, name in ALL_SUBJECTS.items():
         if uid not in SYLLABUS_MAP:
             level = "A" if uid.startswith("A_") else "O"
-            result.append({"id": uid, "subject": name, "code": uid.split("_")[1],
-                           "level": level, "indexed": False})
-    return jsonify(result)
+            result.append({"id": uid, "subject": name, "code": uid.split("_")[1], "level": level, "indexed": False})
+    return jsonify(sorted(result, key=lambda x: (x.get("level", ""), x.get("subject", ""), x.get("code", ""))))
 
 @app.route('/v1/structure/<subject_id>', methods=['GET'])
 def get_structure(subject_id):
@@ -719,9 +1034,9 @@ def get_structure(subject_id):
 def search():
     if VECTOR_MATRIX is None or not VECTOR_DB:
         return jsonify({"error": "Index not ready"}), 503
-    req  = request.json or {}
-    q    = req.get("query")
-    sf   = req.get("filter_subject_id")
+    req = request.json or {}
+    q = req.get("query")
+    sf = req.get("filter_subject_id")
     if not q:
         return jsonify({"error": "Query required"}), 400
     c = get_gemini()
@@ -729,24 +1044,35 @@ def search():
         return jsonify({"error": "Embedding API not configured"}), 503
     try:
         resp = c.models.embed_content(model=EMBEDDING_MODEL, contents=q)
-        qv   = np.array(resp.embeddings[0].values).reshape(1, -1)
+        qv = np.array(resp.embeddings[0].values, dtype=np.float32).reshape(1, -1)
     except Exception as e:
         logger.error(f"Embed query failed: {e}")
         return jsonify({"error": f"Embedding failed: {str(e)}"}), 500
     try:
-        scores  = cosine_similarity(qv, VECTOR_MATRIX)[0]
+        scores = cosine_similarity(qv, VECTOR_MATRIX)[0]
     except Exception as e:
         logger.error(f"Cosine similarity failed: {e}, matrix shape: {VECTOR_MATRIX.shape if VECTOR_MATRIX is not None else None}, query shape: {qv.shape}")
         return jsonify({"error": f"Search index error: {str(e)}"}), 500
     results = []
     for idx in np.argsort(scores)[::-1]:
-        if scores[idx] < 0.3: break
+        if scores[idx] < 0.3:
+            break
         meta = VECTOR_DB[idx]["meta"]
-        if sf and meta["subject_id"] != sf: continue
-        results.append({"score": float(scores[idx]), "subject_id": meta["subject_id"],
-                        "title": meta["title"], "content": meta["content"],
-                        "node_id": meta["subtopic_id"]})
-        if len(results) >= 5: break
+        if sf and meta["subject_id"] != sf:
+            continue
+        results.append({
+            "score": float(scores[idx]),
+            "subject_id": meta["subject_id"],
+            "subject": meta.get("subject"),
+            "level": meta.get("level"),
+            "code": meta.get("code"),
+            "topic": meta.get("topic_title"),
+            "title": meta["title"],
+            "content": meta["content"],
+            "node_id": meta["subtopic_id"],
+        })
+        if len(results) >= int(req.get("limit", 5)):
+            break
     return jsonify({"results": results})
 
 @app.route('/v1/exams', methods=['GET'])
@@ -754,7 +1080,8 @@ def list_exams():
     sid = request.args.get("subject_id")
     out = []
     for s, papers in EXAM_MAP.items():
-        if sid and s != sid: continue
+        if sid and s != sid:
+            continue
         for p in papers.values():
             if isinstance(p, dict) and "meta" in p:
                 out.append(p["meta"])
@@ -762,7 +1089,7 @@ def list_exams():
 
 @app.route('/v1/exams/<paper_id>', methods=['GET'])
 def get_exam(paper_id):
-    safe = re.sub(r'[.\[\]#$/]', '_', paper_id)
+    safe = safe_firebase_key(paper_id)
     for _, papers in EXAM_MAP.items():
         for key, paper in papers.items():
             if key == safe or (isinstance(paper, dict) and paper.get("meta", {}).get("paperId") == paper_id):
@@ -771,78 +1098,54 @@ def get_exam(paper_id):
 
 @app.route('/v1/exams/<paper_id>/questions', methods=['GET'])
 def get_exam_questions(paper_id):
-    safe = re.sub(r'[.\[\]#$/]', '_', paper_id)
+    safe = safe_firebase_key(paper_id)
     for _, papers in EXAM_MAP.items():
         for key, paper in papers.items():
             if key == safe or (isinstance(paper, dict) and paper.get("meta", {}).get("paperId") == paper_id):
                 return jsonify({"paperId": paper_id, "meta": paper.get("meta"), "questions": paper.get("questions", [])})
     return jsonify({"error": "Not found"}), 404
 
-
 @app.route('/v1/reset', methods=['GET', 'POST'])
 def reset_and_rebuild():
     """
     One-time cache reset endpoint.
-    Wipes data_api/syllabi and data_api/vectors from Firebase,
-    then triggers a full fresh rebuild from the PDF files.
+    Wipes Firebase syllabus/vector/exam cache, then triggers a full fresh rebuild from PDF files.
 
-    Protected by REBUILD_SECRET env var. If that var is not set,
-    protected by a hardcoded one-time token so it cannot be called accidentally.
-
-    Call from browser:  GET /v1/reset?token=marka-reset-2026
-    Or as POST with Authorization: Bearer marka-reset-2026
+    Browser: GET /v1/reset?token=marka-reset-2026
+    Better: set REBUILD_SECRET and call GET /v1/reset?token=<your-secret>
     """
-    # Token check — use REBUILD_SECRET if set, otherwise the hardcoded token
     expected = os.environ.get("REBUILD_SECRET", "marka-reset-2026")
-    token = (
-        request.args.get("token")
-        or request.headers.get("Authorization", "").replace("Bearer ", "").strip()
-    )
+    token = request.args.get("token") or request.headers.get("Authorization", "").replace("Bearer ", "").strip()
     if token != expected:
-        return jsonify({
-            "error": "Unauthorized",
-            "hint":  "Pass ?token=<REBUILD_SECRET> in the URL"
-        }), 401
+        return jsonify({"error": "Unauthorized", "hint": "Pass ?token=<REBUILD_SECRET> in the URL"}), 401
 
     def _reset_bg():
         global SYLLABUS_MAP, VECTOR_DB, VECTOR_MATRIX, EXAM_MAP
         logger.info("=== CACHE RESET STARTED ===")
-
-        # 1. Wipe Firebase cache nodes
         if FIREBASE_AVAILABLE:
-            try:
-                firebase_db_ref.child("data_api/syllabi").delete()
-                logger.info("  ✓ Deleted data_api/syllabi from Firebase")
-            except Exception as e:
-                logger.error(f"  ✗ Failed to delete syllabi: {e}")
-            try:
-                firebase_db_ref.child("data_api/vectors").delete()
-                logger.info("  ✓ Deleted data_api/vectors from Firebase")
-            except Exception as e:
-                logger.error(f"  ✗ Failed to delete vectors: {e}")
+            for node in ["data_api/syllabi", "data_api/vectors", "data_api/exams"]:
+                try:
+                    firebase_db_ref.child(node).delete()
+                    logger.info(f"  ✓ Deleted {node} from Firebase")
+                except Exception as e:
+                    logger.error(f"  ✗ Failed to delete {node}: {e}")
         else:
             logger.warning("  Firebase not available — skipping Firebase wipe")
 
-        # 2. Clear in-memory state
-        SYLLABUS_MAP  = {}
-        VECTOR_DB     = []
+        SYLLABUS_MAP = {}
+        VECTOR_DB = []
         VECTOR_MATRIX = None
-        EXAM_MAP      = {}
+        EXAM_MAP = {}
         logger.info("  ✓ In-memory state cleared")
-
-        # 3. Full rebuild from PDFs with vision classifier
-        logger.info("  Starting full rebuild with vision-based parser ...")
         build_index()
         logger.info("=== CACHE RESET COMPLETE ===")
 
     threading.Thread(target=_reset_bg, daemon=True).start()
-
     return jsonify({
-        "status":  "reset started",
-        "message": "Firebase cache wiped. Full rebuild running in background. "
-                   "Check /health in 3-5 minutes to see subject and vector counts update. "
-                   "All 24 PDFs will be re-parsed with the vision classifier.",
-        "watch":   "/health"
+        "status": "reset started",
+        "message": "Firebase cache wiped. Full rebuild running in background. Check /health and HuggingFace logs.",
+        "model": VISION_MODEL,
+        "watch": "/health",
     }), 202
 
 @app.route('/v1/rebuild', methods=['POST'])
@@ -850,13 +1153,17 @@ def trigger_rebuild():
     secret = os.environ.get("REBUILD_SECRET", "")
     if secret and request.headers.get("Authorization", "") != f"Bearer {secret}":
         return jsonify({"error": "Unauthorized"}), 401
+
     def _bg():
         global SYLLABUS_MAP, VECTOR_DB, VECTOR_MATRIX, EXAM_MAP
-        SYLLABUS_MAP = {}; VECTOR_DB = []; VECTOR_MATRIX = None; EXAM_MAP = {}
+        SYLLABUS_MAP = {}
+        VECTOR_DB = []
+        VECTOR_MATRIX = None
+        EXAM_MAP = {}
         build_index()
-    threading.Thread(target=_bg, daemon=True).start()
-    return jsonify({"status": "rebuild started"}), 202
 
+    threading.Thread(target=_bg, daemon=True).start()
+    return jsonify({"status": "rebuild started", "model": VISION_MODEL}), 202
 
 # ---------------------------------------------------------------------------
 # STARTUP
@@ -867,10 +1174,12 @@ def start_app():
         if not os.path.exists(d):
             os.makedirs(os.path.join(d, "A"), exist_ok=True)
             os.makedirs(os.path.join(d, "O"), exist_ok=True)
+
     if not load_index_from_firebase():
         build_index()
     else:
-        logger.info("Served from Firebase cache.")
+        logger.info("Served from Firebase cache. Use /v1/reset to force re-parse with the new parser.")
+
     _collect_existing()
     threading.Thread(target=_watch, daemon=True).start()
     logger.info("Watcher started.")
@@ -879,4 +1188,4 @@ with app.app_context():
     start_app()
 
 if __name__ == '__main__':
-    app.run(host='0.0.0.0', port=7860)
+    app.run(host='0.0.0.0', port=int(os.environ.get("PORT", "7860")))