scripts/generate_source_backed_science_blueprints.py

"""Author SSLC science lesson blueprints from the archived Samagra textbooks.

The output is curriculum data, not a rendered video. It is deliberately strict:
no generic filler, no invented PYQ claims, and every mission cites textbook pages.
"""

from __future__ import annotations

import argparse
import json
import os
import re
import time
import urllib.error
import urllib.request
from pathlib import Path


ROOT = Path(__file__).resolve().parents[2]
MARKDOWN = ROOT / "outputs" / "sslc-syllabus-2025" / "markdown"
OUTPUT = ROOT / "data" / "curriculum" / "kerala-sslc" / "science-blueprints"

CHAPTERS = [
    ("bio-p1-c1", "Biology", "Genetics of Life", "Biology_1744290909995_925880328.md", 7, 34),
    ("bio-p1-c2", "Biology", "Paths of Evolution", "Biology_1744290909995_925880328.md", 35, 70),
    ("bio-p1-c3", "Biology", "Behind Sensations", "Biology_1744290909995_925880328.md", 71, 104),
    ("bio-p1-c4", "Biology", "Chemoreception in Organisms", "Biology -2_1760767396803_208104835.md", 7, 38),
    ("bio-p1-c5", "Biology", "Immunity and Healthcare", "Biology -2_1760767396803_208104835.md", 39, 70),
    ("bio-p1-c6", "Biology", "Biology and Technology", "Biology -2_1760767396803_208104835.md", 71, 104),
    ("chem-p1-c1", "Chemistry", "Nomenclature of Organic Compounds and Isomerism", "Chemistry_1744349699750_895630016.md", 7, 32),
    ("chem-p1-c2", "Chemistry", "Chemical Reactions of Organic Compounds", "Chemistry_1744349699750_895630016.md", 33, 48),
    ("chem-p1-c3", "Chemistry", "Periodic Table and Electron Configuration", "Chemistry_1744349699750_895630016.md", 49, 72),
    ("chem-p1-c4", "Chemistry", "Gas Laws and Mole Concept", "Chemistry_1744349699750_895630016.md", 73, 104),
    ("chem-p2-c5", "Chemistry", "Electrochemistry", "Chemistry -2_1760767939403_130644948.md", 7, 24),
    ("chem-p2-c6", "Chemistry", "Metals", "Chemistry -2_1760767939403_130644948.md", 25, 44),
    ("chem-p2-c7", "Chemistry", "Some Compounds of Industrial Importance", "Chemistry -2_1760767939403_130644948.md", 45, 104),
]

PLACEHOLDERS = re.compile(r"option [abcd]|point [12] about|key principle|focus on the mechanism|final output", re.I)


def env_value(name: str) -> str:
    def clean(raw: str) -> str:
        return raw.strip().strip('"').strip("'").strip()

    value = os.getenv(name)
    if value:
        return clean(value)
    env_path = ROOT / "backend" / ".env"
    for line in env_path.read_text(encoding="utf-8").splitlines():
        if line.startswith(f"{name}="):
            return clean(line.split("=", 1)[1])
    return ""


def valid_gemini_key(value: str) -> bool:
    """Reject comments, sample values, and malformed keys before network use."""
    return value.startswith("AIza") and len(value) >= 30 and not any(char.isspace() for char in value)


def valid_openrouter_key(value: str) -> bool:
    """Reject comments, sample values, and malformed keys before network use."""
    return value.startswith("sk-or-") and len(value) >= 20 and not any(char.isspace() for char in value)


def valid_groq_key(value: str) -> bool:
    """Reject comments, sample values, and malformed keys before network use."""
    return value.startswith("gsk_") and len(value) >= 20 and not any(char.isspace() for char in value)


def page_text(path: Path, first_page: int, last_page: int) -> str:
    raw = path.read_text(encoding="utf-8", errors="replace")
    matches = list(re.finditer(r"^## Page (\d+)\s*$", raw, flags=re.MULTILINE))
    sections: list[str] = []
    for index, match in enumerate(matches):
        page = int(match.group(1))
        if first_page <= page <= last_page:
            end = matches[index + 1].start() if index + 1 < len(matches) else len(raw)
            sections.append(raw[match.start() : end].strip())
    return "\n\n".join(sections)


def skeleton_prompt_for(chapter_id: str, subject: str, title: str, first_page: int, last_page: int, source_name: str, source: str) -> str:
    return f"""You are a master Kerala SSLC science teacher and curriculum editor.
Plan a source-grounded lesson blueprint for Class 10 {subject}, chapter \"{title}\" ({chapter_id}).
This is a PLANNING pass only. Do not write full explanations yet.

NON-NEGOTIABLE RULES
- Use ONLY the textbook excerpt below for scientific claims and syllabus order.
- Missions must cover the chapter's real subtopics in the textbook's own order, not generic filler.
- Do not use filler such as \"key principle\", \"point 1\", \"Option A\", or vague \"process/result\" labels.
- Every mission must cite one or more PDF page numbers between {first_page} and {last_page}.

Return one JSON object with this exact shape:
{{
  "learning_objectives": ["5-8 concrete, testable objectives"],
  "missions": [
    {{"mission_id": "M1", "title": "specific textbook subtopic", "source_pages": [7, 8], "hook": "one curiosity question", "focus_notes": "2-3 sentences on exactly which facts/terms/diagrams from the excerpt this mission must cover"}}
  ],
  "chapter_recap": ["8-14 ordered retrieval points covering the whole chapter"]
}}

Create 5-8 missions so the complete class can teach the whole chapter in 30-40 minutes without repeating content.

TEXTBOOK EXCERPT ({source_name}, PDF pages {first_page}-{last_page})
---
{source}
---
"""


def mission_prompt_for(
    chapter_id: str,
    subject: str,
    chapter_title: str,
    first_page: int,
    last_page: int,
    source_name: str,
    excerpt: str,
    mission_skeleton: dict,
) -> str:
    return f"""You are a master Kerala SSLC science teacher writing ONE teaching mission for Class 10 {subject}, chapter \"{chapter_title}\" ({chapter_id}).

MISSION TO WRITE
- mission_id: {mission_skeleton['mission_id']}
- title: {mission_skeleton['title']}
- source_pages: {mission_skeleton['source_pages']}
- hook: {mission_skeleton['hook']}
- focus: {mission_skeleton.get('focus_notes', '')}

NON-NEGOTIABLE RULES
- Use ONLY the textbook excerpt below for scientific claims.
- Teach so a careful Class 7 student can understand, but preserve exact Class 10 scientific vocabulary and exam value.
- Do not use filler such as \"key principle\", \"point 1\", \"Option A\", \"sure-shot\", or vague \"process/result\" answers.
- Do not claim a question is a PYQ. Label questions \"textbook-aligned practice\".
- Give a reason before a rule. Use a concrete everyday observation only when it genuinely clarifies the science.
- Follow: curiosity hook -> simple explanation -> exact terms -> purposeful board/diagram -> notebook note -> exam practice -> misconception check -> recap.
- Biology diagrams must name the structure/process and say what the student should notice. Chemistry visuals may use equations, particle sketches, apparatus, tables, or reaction flow only where useful.
- Do not suggest decorative imagery.
- Cite one or more PDF page numbers between {first_page} and {last_page} in every board frame.
- Keep sentences natural for spoken Indian English TTS. No Manglish unless a single short line would genuinely rescue understanding.

Return one JSON object with this exact shape (no extra keys, no markdown fences):
{{
  "mission_id": "{mission_skeleton['mission_id']}",
  "title": "{mission_skeleton['title']}",
  "source_pages": {mission_skeleton['source_pages']},
  "hook": "one curiosity question",
  "teacher_intro": "2-3 spoken sentences",
  "explanation_blocks": [{{"heading":"...","explanation":"3-6 clear spoken sentences","key_terms":["..."]}}],
  "analogy_or_observation": "specific and scientifically accurate, or empty string",
  "malayalam_rescue": "optional single simple Manglish line, or empty string",
  "board_frames": [{{"title":"...","teacher_action":"what appears in order","board_text":["..."],"diagram_description":"purposeful scientific diagram or empty string","student_focus":"what to notice","source_page":{mission_skeleton['source_pages'][0]}}}],
  "notebook_notes": ["3-6 concise exact notes"],
  "exam_practice": [{{"marks":2,"question":"...","answer_points":["..."],"keywords":["..."],"common_mistakes":["..."]}}],
  "quick_check": {{"question":"...","answer":"...","explanation":"..."}},
  "misconception_check": "specific wrong idea and correction",
  "recap": ["2-4 retrieval points"]
}}

Produce 2-4 explanation_blocks and 1-3 board_frames. Be complete but concise; do not pad with repetition.

TEXTBOOK EXCERPT ({source_name}, PDF pages {first_page}-{last_page})
---
{excerpt}
---
"""


def prompt_for(chapter_id: str, subject: str, title: str, first_page: int, last_page: int, source_name: str, source: str) -> str:
    return f"""You are a master Kerala SSLC science teacher and curriculum editor.
Create a source-grounded lesson blueprint for Class 10 {subject}, chapter \"{title}\" ({chapter_id}).

NON-NEGOTIABLE RULES
- Use ONLY the textbook excerpt below for scientific claims and syllabus order.
- Teach so a careful Class 7 student can understand, but preserve exact Class 10 scientific vocabulary and exam value.
- Do not use filler such as \"key principle\", \"point 1\", \"Option A\", \"sure-shot\", or vague \"process/result\" answers.
- Do not claim a question is a PYQ. Label questions \"textbook-aligned practice\".
- Give a reason before a rule. Use a concrete everyday observation only when it genuinely clarifies the science.
- Each mission must follow: curiosity hook -> simple explanation -> exact terms -> purposeful board/diagram -> notebook note -> exam practice -> misconception check -> recap.
- Biology diagrams must name the structure/process and say what the student should notice. Chemistry visuals may use equations, particle sketches, apparatus, tables, or reaction flow only where useful.
- Do not suggest decorative imagery.
- Every mission must cite one or more PDF page numbers between {first_page} and {last_page}.
- Keep sentences natural for spoken Indian English TTS. No Manglish unless a single short line would genuinely rescue understanding.

Return one JSON object with this exact shape:
{{
  "schema_version": 1,
  "chapter_id": "{chapter_id}",
  "subject": "{subject}",
  "chapter_title": "{title}",
  "source_book": "Kerala SCERT Standard X {subject} 2025",
  "source_pdf_pages": [{first_page}, {last_page}],
  "learning_objectives": ["..."],
  "missions": [
    {{
      "mission_id": "M1",
      "title": "specific textbook subtopic",
      "source_pages": [7],
      "hook": "one curiosity question",
      "teacher_intro": "2-3 spoken sentences",
      "explanation_blocks": [{{"heading":"...","explanation":"3-6 clear spoken sentences","key_terms":["..."]}}],
      "analogy_or_observation": "specific and scientifically accurate, or empty string",
      "malayalam_rescue": "optional single simple Manglish line, or empty string",
      "board_frames": [{{"title":"...","teacher_action":"what appears in order","board_text":["..."],"diagram_description":"purposeful scientific diagram or empty string","student_focus":"what to notice","source_page":7}}],
      "notebook_notes": ["3-6 concise exact notes"],
      "exam_practice": [{{"marks":2,"question":"...","answer_points":["..."],"keywords":["..."],"common_mistakes":["..."]}}],
      "quick_check": {{"question":"...","answer":"...","explanation":"..."}},
      "misconception_check": "specific wrong idea and correction",
      "recap": ["2-4 retrieval points"]
    }}
  ],
  "chapter_recap": ["8-14 ordered points"],
  "render_guidance": {{"textbook_figures_first": true, "generated_images_only_when_missing": true, "avoid_continuous_svg": true}}
}}

Create 5-8 missions so the complete class can teach the whole chapter in 30-40 minutes without repeating content.

TEXTBOOK EXCERPT ({source_name}, PDF pages {first_page}-{last_page})
---
{source}
---
"""


def call_gemini(prompt: str, api_key: str, model: str) -> dict:
    url = f"https://generativelanguage.googleapis.com/v1beta/models/{model}:generateContent?key={api_key}"
    payload = {
        "contents": [{"role": "user", "parts": [{"text": prompt}]}],
        "generationConfig": {"temperature": 0.2, "responseMimeType": "application/json", "maxOutputTokens": 16384},
    }
    request = urllib.request.Request(url, data=json.dumps(payload).encode("utf-8"), headers={"Content-Type": "application/json"}, method="POST")
    with urllib.request.urlopen(request, timeout=240) as response:
        result = json.loads(response.read().decode("utf-8"))
    text = result["candidates"][0]["content"]["parts"][0]["text"]
    return json.loads(text)


def call_openrouter(prompt: str, api_key: str, model: str) -> dict:
    url = "https://openrouter.ai/api/v1/chat/completions"
    payload = {
        "model": model,
        "messages": [{"role": "user", "content": prompt}],
        "temperature": 0.2,
        "max_tokens": 14000,
        "response_format": {"type": "json_object"},
    }
    request = urllib.request.Request(
        url,
        data=json.dumps(payload).encode("utf-8"),
        headers={
            "Content-Type": "application/json",
            "Authorization": f"Bearer {api_key}",
            "HTTP-Referer": "https://docdoe.in",
            "X-Title": "DocDoe SSLC Curriculum Authoring",
        },
        method="POST",
    )
    with urllib.request.urlopen(request, timeout=240) as response:
        result = json.loads(response.read().decode("utf-8"))
    text = result["choices"][0]["message"]["content"]
    return json.loads(text)


def call_groq(prompt: str, api_key: str, model: str) -> dict:
    url = "https://api.groq.com/openai/v1/chat/completions"
    payload = {
        "model": model,
        "messages": [{"role": "user", "content": prompt}],
        "temperature": 0.2,
        "max_tokens": 8192,
        "response_format": {"type": "json_object"},
    }
    request = urllib.request.Request(
        url,
        data=json.dumps(payload).encode("utf-8"),
        headers={
            "Content-Type": "application/json",
            "Authorization": f"Bearer {api_key}",
            # Groq's Cloudflare front-end 403s urllib's default UA string.
            "User-Agent": "python-requests/2.31.0",
        },
        method="POST",
    )
    with urllib.request.urlopen(request, timeout=240) as response:
        result = json.loads(response.read().decode("utf-8"))
    text = result["choices"][0]["message"]["content"]
    return json.loads(text)


def validate(data: dict, chapter_id: str, subject: str, title: str, first_page: int, last_page: int) -> list[str]:
    issues: list[str] = []
    if data.get("chapter_id") != chapter_id or data.get("subject") != subject or data.get("chapter_title") != title:
        issues.append("identity_mismatch")
    missions = data.get("missions") or []
    if not 5 <= len(missions) <= 8:
        issues.append("mission_count_not_5_to_8")
    if PLACEHOLDERS.search(json.dumps(data)):
        issues.append("placeholder_language_found")
    for mission in missions:
        pages = mission.get("source_pages") or []
        if not pages or any(not isinstance(page, int) or page < first_page or page > last_page for page in pages):
            issues.append(f"{mission.get('mission_id', 'unknown')}_invalid_source_pages")
        if len(mission.get("explanation_blocks") or []) < 2:
            issues.append(f"{mission.get('mission_id', 'unknown')}_thin_explanation")
        if len(mission.get("notebook_notes") or []) < 3:
            issues.append(f"{mission.get('mission_id', 'unknown')}_thin_notes")
        if not mission.get("board_frames") or not mission.get("exam_practice"):
            issues.append(f"{mission.get('mission_id', 'unknown')}_missing_board_or_practice")
    return sorted(set(issues))


def generate_with_retries(generate, prompt: str, chapter_id: str, label: str, attempts: int = 5) -> dict:
    for attempt in range(1, attempts + 1):
        try:
            return generate(prompt)
        except (urllib.error.URLError, TimeoutError, KeyError, json.JSONDecodeError) as exc:
            if attempt == attempts:
                raise
            # Rate limits (429) need a real cooldown, not a quick backoff.
            wait = 25 if isinstance(exc, urllib.error.HTTPError) and exc.code == 429 else attempt * 4
            print(f"[{chapter_id}] {label} attempt {attempt} failed: {type(exc).__name__}; retrying in {wait}s", flush=True)
            time.sleep(wait)
    raise RuntimeError("unreachable")


def mission_excerpt(markdown_path: Path, mission_pages: list[int], first_page: int, last_page: int) -> str:
    pages = mission_pages or [first_page]
    window_start = max(first_page, min(pages) - 1)
    window_end = min(last_page, max(pages) + 1)
    excerpt = page_text(markdown_path, window_start, window_end)
    if len(excerpt) < 400:
        excerpt = page_text(markdown_path, first_page, last_page)
    return excerpt


def synthesize_chapter_two_phase(
    generate,
    chapter_id: str,
    subject: str,
    title: str,
    markdown_file: str,
    first_page: int,
    last_page: int,
) -> dict:
    markdown_path = MARKDOWN / markdown_file
    full_source = page_text(markdown_path, first_page, last_page)
    if len(full_source) < 1000:
        raise RuntimeError(f"Textbook extraction too short for {chapter_id}: {len(full_source)} characters")

    skeleton_prompt = skeleton_prompt_for(chapter_id, subject, title, first_page, last_page, markdown_file, full_source)
    skeleton = generate_with_retries(generate, skeleton_prompt, chapter_id, "skeleton")
    missions_skeleton = skeleton.get("missions") or []
    if not 5 <= len(missions_skeleton) <= 8:
        print(f"[{chapter_id}] skeleton returned {len(missions_skeleton)} missions (want 5-8); continuing anyway", flush=True)

    missions: list[dict] = []
    for mission_skeleton in missions_skeleton:
        excerpt = mission_excerpt(markdown_path, mission_skeleton.get("source_pages") or [], first_page, last_page)
        prompt = mission_prompt_for(chapter_id, subject, title, first_page, last_page, markdown_file, excerpt, mission_skeleton)
        mission = generate_with_retries(generate, prompt, chapter_id, f"mission {mission_skeleton.get('mission_id')}")
        missions.append(mission)
        print(f"[{chapter_id}] mission {mission_skeleton.get('mission_id')} written", flush=True)
        time.sleep(2)

    return {
        "schema_version": 1,
        "chapter_id": chapter_id,
        "subject": subject,
        "chapter_title": title,
        "source_book": f"Kerala SCERT Standard X {subject} 2025",
        "source_pdf_pages": [first_page, last_page],
        "learning_objectives": skeleton.get("learning_objectives") or [],
        "missions": missions,
        "chapter_recap": skeleton.get("chapter_recap") or [],
        "render_guidance": {"textbook_figures_first": True, "generated_images_only_when_missing": True, "avoid_continuous_svg": True},
    }


def main() -> int:
    parser = argparse.ArgumentParser()
    parser.add_argument("--chapter-id", action="append", dest="chapter_ids")
    parser.add_argument("--provider", choices=["gemini", "groq", "openrouter"], default=None)
    parser.add_argument("--model", default=None)
    parser.add_argument("--force", action="store_true")
    args = parser.parse_args()

    gemini_key = env_value("GEMINI_API_KEY")
    groq_key = env_value("GROQ_API_KEY")
    openrouter_key = env_value("OPENROUTER_API_KEY")
    provider = args.provider
    if provider is None:
        if valid_gemini_key(gemini_key):
            provider = "gemini"
        elif valid_groq_key(groq_key):
            provider = "groq"
        else:
            provider = "openrouter"

    if provider == "gemini":
        if not valid_gemini_key(gemini_key):
            raise SystemExit("GEMINI_API_KEY is missing or is still a placeholder; no curriculum was generated")
        model = args.model or "gemini-2.5-flash"
        generate = lambda prompt: call_gemini(prompt, gemini_key, model)  # noqa: E731
    elif provider == "groq":
        if not valid_groq_key(groq_key):
            raise SystemExit("GROQ_API_KEY is missing or is still a placeholder; no curriculum was generated")
        model = args.model or "meta-llama/llama-4-scout-17b-16e-instruct"
        generate = lambda prompt: call_groq(prompt, groq_key, model)  # noqa: E731
    else:
        if not valid_openrouter_key(openrouter_key):
            raise SystemExit("OPENROUTER_API_KEY is missing or is still a placeholder; no curriculum was generated")
        model = args.model or "google/gemini-2.5-flash"
        generate = lambda prompt: call_openrouter(prompt, openrouter_key, model)  # noqa: E731

    selected = [chapter for chapter in CHAPTERS if not args.chapter_ids or chapter[0] in args.chapter_ids]
    OUTPUT.mkdir(parents=True, exist_ok=True)
    reports = []
    for chapter_id, subject, title, markdown_file, first_page, last_page in selected:
        output = OUTPUT / f"{chapter_id}.json"
        try:
            if output.exists() and not args.force:
                data = json.loads(output.read_text(encoding="utf-8"))
            else:
                data = synthesize_chapter_two_phase(generate, chapter_id, subject, title, markdown_file, first_page, last_page)
                output.write_text(json.dumps(data, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
        except Exception as exc:  # noqa: BLE001 - one bad chapter must not abort the whole batch
            report = {"chapter_id": chapter_id, "passed": False, "issues": [f"generation_failed: {type(exc).__name__}: {exc}"], "missions": 0, "output": output.as_posix()}
            reports.append(report)
            print(json.dumps(report), flush=True)
            print(f"[{chapter_id}] generation failed after all retries; continuing with remaining chapters", flush=True)
            continue
        issues = validate(data, chapter_id, subject, title, first_page, last_page)
        report = {"chapter_id": chapter_id, "passed": not issues, "issues": issues, "missions": len(data.get("missions") or []), "output": output.as_posix()}
        reports.append(report)
        print(json.dumps(report), flush=True)
    (OUTPUT / "generation-report.json").write_text(json.dumps(reports, indent=2) + "\n", encoding="utf-8")
    return 0 if all(report["passed"] for report in reports) else 1


if __name__ == "__main__":
    raise SystemExit(main())