docs/scripts/run_mvp_eval.py

"""Run the REAL PaperMate review pipeline on the test papers to produce
evaluation evidence for Gate G2 (MVP — first working agent, no mock).

User flow exercised: paper content -> extract -> (optional related-work search)
-> structured ARR review, using the real configured LLM (OpenRouter).

PDF->text step is satisfied by reusing the already-parsed paper content in
docs/test/docling_output/ (the system's own Docling OCR output), falling back
to docs/test/parsed_pdfs/. The LLM still does real extraction + review on real
paper content.

Outputs one JSON per paper to docs/test/mvp_eval_output/{id}.json.

Usage (PowerShell):
    $PY = "C:\\Users\\DELL\\AppData\\Local\\Programs\\Python\\Python312\\python.exe"
    $env:PYTHONIOENCODING = "utf-8"
    & $PY docs\\scripts\\run_mvp_eval.py            # run all 5 papers
    & $PY docs\\scripts\\run_mvp_eval.py --dry       # validate without calling LLM
    & $PY docs\\scripts\\run_mvp_eval.py --only 49   # single paper
"""
from __future__ import annotations

import argparse
import asyncio
import json
import sys
import time
from pathlib import Path

ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT))

PAPERS = ["49", "323", "355", "435", "768"]
SRC_DIRS = [ROOT / "docs/test/docling_output", ROOT / "docs/test/parsed_pdfs"]
OUT_DIR = ROOT / "docs/test/mvp_eval_output"

# Synthetic inputs for edge / failure coverage (required by the G2 eval issue).
# Each value is the raw "paper markdown" fed to the real pipeline.
SYNTHETIC = {
    # Edge case: almost-empty submission (1 line, no real paper body).
    "edge_minimal": (
        "# Note\n\n"
        "This document intentionally contains almost no content. "
        "It has no methodology, no experiments, and no results.\n"
    ),
    # Failure case: out-of-scope, non-academic text submitted as if it were a paper.
    "failure_nonpaper": (
        "What's the weather like today in Hanoi? Will it rain tomorrow? "
        "Also please book me a cheap flight to Da Nang next weekend and "
        "recommend a good seafood restaurant near the beach. Thanks!\n"
    ),
}


def load_paper(pid: str):
    for d in SRC_DIRS:
        p = d / f"{pid}.pdf.json"
        if p.exists():
            return json.loads(p.read_text(encoding="utf-8")), p
    raise FileNotFoundError(f"No parsed JSON found for paper {pid}")


def reconstruct_md(data: dict) -> str:
    m = data.get("metadata", data)
    title = (m.get("title") or "").strip()
    abstract = (m.get("abstractText") or "").strip()
    parts = [f"# {title}", "", abstract, ""]
    for s in m.get("sections") or []:
        h = (s.get("heading") or "").strip()
        t = (s.get("text") or "").strip()
        if h:
            parts.append(f"## {h}")
        if t:
            parts.append(t)
        parts.append("")
    return "\n".join(parts).strip()


def _install_retry(max_attempts: int = 8):
    """Wrap the LLM `complete` with retry-on-429 backoff (free-tier friendly).

    Patches the name already imported into each pipeline module so existing
    `from backend.llm.client import complete` references pick it up. Backend
    code is left untouched.
    """
    import re
    import backend.llm.client as llmclient
    import backend.pipeline.extract as ex
    import backend.pipeline.search as se
    import backend.pipeline.summarize as su
    import backend.pipeline.review as rv

    orig = llmclient.complete
    if getattr(orig, "_retrying", False):
        return

    async def retrying_complete(*args, **kwargs):
        for attempt in range(max_attempts):
            try:
                return await orig(*args, **kwargs)
            except Exception as e:
                msg = str(e)
                is_429 = "429" in msg or "rate" in msg.lower()
                if not is_429 or attempt == max_attempts - 1:
                    raise
                m = re.search(r"retry_after_seconds['\"]?:\s*([0-9.]+)", msg)
                wait = float(m.group(1)) + 2 if m else min(60, 8 * (attempt + 1))
                print(f"    rate-limited (429) — retry in {wait:.0f}s "
                      f"[attempt {attempt + 1}/{max_attempts}]")
                await asyncio.sleep(wait)
        return await orig(*args, **kwargs)

    retrying_complete._retrying = True
    for mod in (llmclient, ex, se, su, rv):
        if hasattr(mod, "complete"):
            mod.complete = retrying_complete


async def run_one(pid: str, dry: bool = False) -> dict:
    if pid in SYNTHETIC:
        md = SYNTHETIC[pid].strip()
        rel_src = f"(synthetic: {pid})"
    else:
        data, src = load_paper(pid)
        md = reconstruct_md(data)
        rel_src = str(src.relative_to(ROOT)).replace("\\", "/")
    print(f"[{pid}] source={rel_src}  input_chars={len(md)}")
    if dry:
        return {"id": pid, "source_file": rel_src, "input_chars": len(md)}

    from backend.config import settings
    from backend.pipeline.extract import (
        extract_paper_title,
        extract_contributions,
        extract_research_topic,
    )
    from backend.pipeline.review import generate_review
    from backend.pipeline.search import (
        generate_scientific_search_queries,
        search_related_papers,
    )
    from backend.pipeline.paper_info import get_paper_info
    from backend.pipeline.summarize import summarize_related_research

    _install_retry()

    t0 = time.time()
    paper_title = await extract_paper_title(md)
    contributions, topic = await asyncio.gather(
        extract_contributions(md),
        extract_research_topic(md),
    )
    print(f"[{pid}] title={paper_title!r}  contributions={len(contributions)}")

    related: list[dict] = []
    if settings.tavily_api_key:
        try:
            queries = await generate_scientific_search_queries(contributions, topic)
            raw = await search_related_papers(queries)
            papers = await get_paper_info(raw)
            related = await summarize_related_research(papers)
            print(f"[{pid}] related papers found={len(related)}")
        except Exception as e:  # search is optional for the MVP review
            print(f"[{pid}] related-work search skipped: {type(e).__name__}: {e}")
    else:
        print(f"[{pid}] related-work search skipped: no TAVILY_API_KEY")

    review = await generate_review(md, contributions, topic, related)
    elapsed = round(time.time() - t0, 1)
    print(f"[{pid}] DONE in {elapsed}s  overall={review.get('overall_assessment')} "
          f"({review.get('overall_assessment_label')})")

    out = {
        "id": pid,
        "source_file": rel_src,
        "llm_provider": settings.llm_provider,
        "llm_model": settings.llm_model,
        "paper_title": paper_title,
        "input_chars": len(md),
        "contributions": contributions,
        "research_topic": topic,
        "related_count": len(related),
        "related_summaries": related,
        "elapsed_seconds": elapsed,
        "review": review,
    }
    OUT_DIR.mkdir(parents=True, exist_ok=True)
    (OUT_DIR / f"{pid}.json").write_text(
        json.dumps(out, ensure_ascii=False, indent=2), encoding="utf-8"
    )
    return out


async def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--dry", action="store_true", help="validate without calling the LLM")
    ap.add_argument("--only", help="run a single paper id")
    args = ap.parse_args()

    ids = [args.only] if args.only else PAPERS
    for pid in ids:
        try:
            await run_one(pid, dry=args.dry)
        except Exception as e:
            print(f"[{pid}] FAILED: {type(e).__name__}: {e}")
    if not args.dry:
        print(f"\nOutputs written to {OUT_DIR.relative_to(ROOT)}")


if __name__ == "__main__":
    asyncio.run(main())