Spaces:
Running
Running
| """Run the REAL PaperMate review pipeline on the test papers to produce | |
| evaluation evidence for Gate G2 (MVP — first working agent, no mock). | |
| User flow exercised: paper content -> extract -> (optional related-work search) | |
| -> structured ARR review, using the real configured LLM (OpenRouter). | |
| PDF->text step is satisfied by reusing the already-parsed paper content in | |
| docs/test/docling_output/ (the system's own Docling OCR output), falling back | |
| to docs/test/parsed_pdfs/. The LLM still does real extraction + review on real | |
| paper content. | |
| Outputs one JSON per paper to docs/test/mvp_eval_output/{id}.json. | |
| Usage (PowerShell): | |
| $PY = "C:\\Users\\DELL\\AppData\\Local\\Programs\\Python\\Python312\\python.exe" | |
| $env:PYTHONIOENCODING = "utf-8" | |
| & $PY docs\\scripts\\run_mvp_eval.py # run all 5 papers | |
| & $PY docs\\scripts\\run_mvp_eval.py --dry # validate without calling LLM | |
| & $PY docs\\scripts\\run_mvp_eval.py --only 49 # single paper | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import asyncio | |
| import json | |
| import sys | |
| import time | |
| from pathlib import Path | |
| ROOT = Path(__file__).resolve().parents[2] | |
| sys.path.insert(0, str(ROOT)) | |
| PAPERS = ["49", "323", "355", "435", "768"] | |
| SRC_DIRS = [ROOT / "docs/test/docling_output", ROOT / "docs/test/parsed_pdfs"] | |
| OUT_DIR = ROOT / "docs/test/mvp_eval_output" | |
| # Synthetic inputs for edge / failure coverage (required by the G2 eval issue). | |
| # Each value is the raw "paper markdown" fed to the real pipeline. | |
| SYNTHETIC = { | |
| # Edge case: almost-empty submission (1 line, no real paper body). | |
| "edge_minimal": ( | |
| "# Note\n\n" | |
| "This document intentionally contains almost no content. " | |
| "It has no methodology, no experiments, and no results.\n" | |
| ), | |
| # Failure case: out-of-scope, non-academic text submitted as if it were a paper. | |
| "failure_nonpaper": ( | |
| "What's the weather like today in Hanoi? Will it rain tomorrow? " | |
| "Also please book me a cheap flight to Da Nang next weekend and " | |
| "recommend a good seafood restaurant near the beach. Thanks!\n" | |
| ), | |
| } | |
| def load_paper(pid: str): | |
| for d in SRC_DIRS: | |
| p = d / f"{pid}.pdf.json" | |
| if p.exists(): | |
| return json.loads(p.read_text(encoding="utf-8")), p | |
| raise FileNotFoundError(f"No parsed JSON found for paper {pid}") | |
| def reconstruct_md(data: dict) -> str: | |
| m = data.get("metadata", data) | |
| title = (m.get("title") or "").strip() | |
| abstract = (m.get("abstractText") or "").strip() | |
| parts = [f"# {title}", "", abstract, ""] | |
| for s in m.get("sections") or []: | |
| h = (s.get("heading") or "").strip() | |
| t = (s.get("text") or "").strip() | |
| if h: | |
| parts.append(f"## {h}") | |
| if t: | |
| parts.append(t) | |
| parts.append("") | |
| return "\n".join(parts).strip() | |
| def _install_retry(max_attempts: int = 8): | |
| """Wrap the LLM `complete` with retry-on-429 backoff (free-tier friendly). | |
| Patches the name already imported into each pipeline module so existing | |
| `from backend.llm.client import complete` references pick it up. Backend | |
| code is left untouched. | |
| """ | |
| import re | |
| import backend.llm.client as llmclient | |
| import backend.pipeline.extract as ex | |
| import backend.pipeline.search as se | |
| import backend.pipeline.summarize as su | |
| import backend.pipeline.review as rv | |
| orig = llmclient.complete | |
| if getattr(orig, "_retrying", False): | |
| return | |
| async def retrying_complete(*args, **kwargs): | |
| for attempt in range(max_attempts): | |
| try: | |
| return await orig(*args, **kwargs) | |
| except Exception as e: | |
| msg = str(e) | |
| is_429 = "429" in msg or "rate" in msg.lower() | |
| if not is_429 or attempt == max_attempts - 1: | |
| raise | |
| m = re.search(r"retry_after_seconds['\"]?:\s*([0-9.]+)", msg) | |
| wait = float(m.group(1)) + 2 if m else min(60, 8 * (attempt + 1)) | |
| print(f" rate-limited (429) — retry in {wait:.0f}s " | |
| f"[attempt {attempt + 1}/{max_attempts}]") | |
| await asyncio.sleep(wait) | |
| return await orig(*args, **kwargs) | |
| retrying_complete._retrying = True | |
| for mod in (llmclient, ex, se, su, rv): | |
| if hasattr(mod, "complete"): | |
| mod.complete = retrying_complete | |
| async def run_one(pid: str, dry: bool = False) -> dict: | |
| if pid in SYNTHETIC: | |
| md = SYNTHETIC[pid].strip() | |
| rel_src = f"(synthetic: {pid})" | |
| else: | |
| data, src = load_paper(pid) | |
| md = reconstruct_md(data) | |
| rel_src = str(src.relative_to(ROOT)).replace("\\", "/") | |
| print(f"[{pid}] source={rel_src} input_chars={len(md)}") | |
| if dry: | |
| return {"id": pid, "source_file": rel_src, "input_chars": len(md)} | |
| from backend.config import settings | |
| from backend.pipeline.extract import ( | |
| extract_paper_title, | |
| extract_contributions, | |
| extract_research_topic, | |
| ) | |
| from backend.pipeline.review import generate_review | |
| from backend.pipeline.search import ( | |
| generate_scientific_search_queries, | |
| search_related_papers, | |
| ) | |
| from backend.pipeline.paper_info import get_paper_info | |
| from backend.pipeline.summarize import summarize_related_research | |
| _install_retry() | |
| t0 = time.time() | |
| paper_title = await extract_paper_title(md) | |
| contributions, topic = await asyncio.gather( | |
| extract_contributions(md), | |
| extract_research_topic(md), | |
| ) | |
| print(f"[{pid}] title={paper_title!r} contributions={len(contributions)}") | |
| related: list[dict] = [] | |
| if settings.tavily_api_key: | |
| try: | |
| queries = await generate_scientific_search_queries(contributions, topic) | |
| raw = await search_related_papers(queries) | |
| papers = await get_paper_info(raw) | |
| related = await summarize_related_research(papers) | |
| print(f"[{pid}] related papers found={len(related)}") | |
| except Exception as e: # search is optional for the MVP review | |
| print(f"[{pid}] related-work search skipped: {type(e).__name__}: {e}") | |
| else: | |
| print(f"[{pid}] related-work search skipped: no TAVILY_API_KEY") | |
| review = await generate_review(md, contributions, topic, related) | |
| elapsed = round(time.time() - t0, 1) | |
| print(f"[{pid}] DONE in {elapsed}s overall={review.get('overall_assessment')} " | |
| f"({review.get('overall_assessment_label')})") | |
| out = { | |
| "id": pid, | |
| "source_file": rel_src, | |
| "llm_provider": settings.llm_provider, | |
| "llm_model": settings.llm_model, | |
| "paper_title": paper_title, | |
| "input_chars": len(md), | |
| "contributions": contributions, | |
| "research_topic": topic, | |
| "related_count": len(related), | |
| "related_summaries": related, | |
| "elapsed_seconds": elapsed, | |
| "review": review, | |
| } | |
| OUT_DIR.mkdir(parents=True, exist_ok=True) | |
| (OUT_DIR / f"{pid}.json").write_text( | |
| json.dumps(out, ensure_ascii=False, indent=2), encoding="utf-8" | |
| ) | |
| return out | |
| async def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--dry", action="store_true", help="validate without calling the LLM") | |
| ap.add_argument("--only", help="run a single paper id") | |
| args = ap.parse_args() | |
| ids = [args.only] if args.only else PAPERS | |
| for pid in ids: | |
| try: | |
| await run_one(pid, dry=args.dry) | |
| except Exception as e: | |
| print(f"[{pid}] FAILED: {type(e).__name__}: {e}") | |
| if not args.dry: | |
| print(f"\nOutputs written to {OUT_DIR.relative_to(ROOT)}") | |
| if __name__ == "__main__": | |
| asyncio.run(main()) | |