scripts/run_batch_test.py

# -*- coding: utf-8 -*-
#!/usr/bin/env python3
import os, sys
os.environ["PYTHONIOENCODING"] = "utf-8"
if sys.stdout.encoding != "utf-8":
    sys.stdout.reconfigure(encoding="utf-8", errors="replace")
    sys.stderr.reconfigure(encoding="utf-8", errors="replace")
"""
Batch pipeline runner for test_5_papers.
Runs CitationEdge on each PDF, saves raw JSON results to test_5_papers_sol/,
then writes a human-interpretable analysis report.

Usage:
    python scripts/run_batch_test.py
"""
from __future__ import annotations

import asyncio
import json
import sys
import time
from pathlib import Path
from datetime import datetime

sys.path.insert(0, str(Path(__file__).parent.parent))

from orchestrators.custom_orchestrator import CitationEdgeOrchestrator
from utils.logger import get_logger

logger = get_logger("batch_test")

INPUT_DIR  = Path(__file__).parent.parent / "test_5_papers"
OUTPUT_DIR = Path(__file__).parent.parent / "test_5_papers_sol"
OUTPUT_DIR.mkdir(exist_ok=True)


# ── Pipeline runner ────────────────────────────────────────────────────────────

async def run_one(orchestrator: CitationEdgeOrchestrator, pdf: Path) -> dict:
    job_id = f"batch_{pdf.stem}"
    logger.info(f"\n{'='*65}")
    logger.info(f">> {pdf.name}")
    logger.info(f"{'='*65}")
    t0 = time.monotonic()
    try:
        result = await orchestrator.run(job_id=job_id, pdf_path=str(pdf))
        result["_pdf"] = pdf.name
        result["_wall_s"] = round(time.monotonic() - t0, 1)
    except Exception as exc:
        logger.error(f"Pipeline crashed for {pdf.name}: {exc}")
        result = {"_pdf": pdf.name, "error": str(exc), "status": "crashed",
                  "_wall_s": round(time.monotonic() - t0, 1)}
    # Persist raw result
    out = OUTPUT_DIR / f"{pdf.stem}_result.json"
    out.write_text(json.dumps(result, indent=2, default=str), encoding="utf-8")
    logger.info(f"✓  Saved → {out.name}")
    return result


async def main():
    pdfs = sorted(INPUT_DIR.glob("*.pdf"))
    if not pdfs:
        logger.error(f"No PDFs found in {INPUT_DIR}")
        return

    logger.info(f"Found {len(pdfs)} papers: {[p.name for p in pdfs]}")
    orchestrator = CitationEdgeOrchestrator()
    all_results = []

    for pdf in pdfs:
        r = await run_one(orchestrator, pdf)
        all_results.append(r)

    # Save combined results
    combined = OUTPUT_DIR / "_all_results.json"
    combined.write_text(json.dumps(all_results, indent=2, default=str), encoding="utf-8")

    # Generate human-interpretable analysis report
    report = build_analysis_report(all_results)
    report_path = OUTPUT_DIR / "_analysis_report.md"
    report_path.write_text(report, encoding="utf-8")
    logger.info(f"\n✓  Analysis report → {report_path}")
    print(report)


# ── Analysis report builder ────────────────────────────────────────────────────

SCORE_KEYS = [
    "overall_score",
    "claim_quality_score",
    "citation_health_score",
    "argumentation_score",
    "novelty_score",
    "evidence_score",
]

EXPECTED_RANGES = {
    "overall_score":        (4.0, 10.0),
    "claim_quality_score":  (4.0, 10.0),
    "citation_health_score":(3.0, 10.0),
    "argumentation_score":  (3.0, 10.0),
    "novelty_score":        (3.0, 10.0),
    "evidence_score":       (3.0, 10.0),
}

VERDICT_PASS   = {"supported", "likely_supported", "verified"}
VERDICT_REJECT = {"refuted", "contradicted", "likely_refuted"}

DOMAIN_NOTES = {
    "llm_reasoning_divide_conquer": "LLM reasoning — expect high novelty, rich claims, moderate citation gaps",
    "kg_embedding_survey":          "Survey paper — expect many citations, low gaps, lower novelty score",
    "federated_chest_radiograph":   "Applied FL/medical — expect strong evidence, moderate claims",
    "yolov10_endtoend":             "CV/detection paper — expect strong empirical claims, high citation health",
    "gnn_deeper":                   "Architecture paper — expect high novelty, methodological claims",
}


def _score_badge(val: float | None, key: str) -> str:
    if val is None:
        return "N/A"
    lo, hi = EXPECTED_RANGES.get(key, (4.0, 10.0))
    mark = "✅" if lo <= val <= hi else ("⚠️" if val < lo else "⬆️")
    return f"{val:.1f}/10 {mark}"


def build_analysis_report(results: list[dict]) -> str:
    now = datetime.now().strftime("%Y-%m-%d %H:%M")
    lines = [
        f"# CitationEdge Batch Test — Analysis Report",
        f"*Generated: {now} | Papers: {len(results)}*",
        "",
        "---",
        "",
        "## 1. Pipeline Execution Summary",
        "",
        f"| Paper | Status | Duration |",
        f"|-------|--------|----------|",
    ]
    for r in results:
        name = r.get("_pdf", "?")
        status = r.get("status", "?")
        dur = r.get("_wall_s") or r.get("duration_s", "?")
        icon = "✅" if "complet" in str(status) else "❌"
        lines.append(f"| `{name}` | {icon} {status} | {dur}s |")

    # ── Per-paper deep dive ───────────────────────────────────────────────────
    lines += ["", "---", "", "## 2. Per-Paper Analysis", ""]

    for r in results:
        stem = Path(r.get("_pdf", "unknown")).stem
        lines += [f"### 📄 {r.get('_pdf', stem)}", ""]

        domain_note = DOMAIN_NOTES.get(stem, "")
        if domain_note:
            lines += [f"> **Domain context:** {domain_note}", ""]

        if r.get("error"):
            lines += [f"**ERROR:** `{r['error']}`", ""]
            continue

        scores = r.get("scores", {})
        claims = r.get("claims", [])
        gaps   = r.get("citation_gaps", [])
        args   = r.get("argument_graph", [])
        cf     = r.get("counterfactuality", {})
        claim_cfs = r.get("claim_counterfactualities", [])
        keywords  = r.get("keywords", [])
        agents    = r.get("agents", {})

        # Scores table
        lines += ["#### Scores", ""]
        lines += ["| Metric | Value | Expected? |", "|--------|-------|-----------|"]
        for k in SCORE_KEYS:
            v = scores.get(k)
            lines.append(f"| {k.replace('_', ' ').title()} | {_score_badge(v, k)} | {EXPECTED_RANGES.get(k, '')} |")

        # Agent statuses
        failed = [a for a, d in agents.items() if d.get("status") == "failed"]
        if failed:
            lines += ["", f"⚠️ **Failed agents:** {', '.join(failed)}"]

        # Keywords
        if keywords:
            top_kw = ", ".join(f"`{k}`" for k in keywords[:10])
            lines += ["", f"**Keywords ({len(keywords)}):** {top_kw}"]
            exp_note = _interpret_keywords(keywords, stem)
            if exp_note:
                lines += [f"  → *{exp_note}*"]

        # Claims
        lines += ["", f"**Claims extracted:** {len(claims)}"]
        if claims:
            supported = sum(1 for c in claims if c.get("verdict") in VERDICT_PASS)
            refuted   = sum(1 for c in claims if c.get("verdict") in VERDICT_REJECT)
            unverif   = len(claims) - supported - refuted
            lines += [
                f"  - Supported/verified: {supported}",
                f"  - Refuted/contradicted: {refuted}",
                f"  - Unverified/unknown: {unverif}",
            ]
            top_claims = [c for c in claims if c.get("text")][:3]
            if top_claims:
                lines += ["", "  *Top claims:*"]
                for i, c in enumerate(top_claims, 1):
                    verdict = c.get("verdict", "?")
                    conf    = c.get("verify_confidence") or c.get("confidence") or "?"
                    text    = (c["text"][:120] + "…") if len(c.get("text","")) > 120 else c.get("text","")
                    lines.append(f"  {i}. [{verdict}] ({conf:.2f} conf) — {text}" if isinstance(conf, float) else
                                 f"  {i}. [{verdict}] — {text}")
            lines += _interpret_claims(claims, stem)

        # Citation gaps
        lines += ["", f"**Citation gaps:** {len(gaps)}"]
        if gaps:
            high_sev = [g for g in gaps if g.get("severity") in ("high", "critical")]
            if high_sev:
                lines += [f"  ⚠️ High-severity gaps: {len(high_sev)}"]
            for g in gaps[:3]:
                desc = (g.get("gap","")[:100] + "…") if len(g.get("gap","")) > 100 else g.get("gap","")
                lines.append(f"  - [{g.get('severity','?')}] {desc}")
            lines += _interpret_gaps(gaps, stem)

        # Counterfactuality
        if cf:
            cf_score = cf.get("overall_score", "?")
            lines += [
                "",
                f"**Counterfactuality score:** {cf_score}",
                f"  Common type: {cf.get('common_type','?')}",
            ]
            if cf.get("summary"):
                lines += [f"  Summary: {cf['summary'][:200]}"]
            lines += _interpret_cf(cf, claim_cfs, stem)

        # Argumentation
        if args:
            strengths = args[0].get("strengths", []) if args else []
            weaknesses = args[0].get("weaknesses", []) if args else []
            lines += ["", "**Argumentation:**"]
            if strengths:
                lines += [f"  Strengths: {'; '.join(str(s) for s in strengths[:2])}"]
            if weaknesses:
                lines += [f"  Weaknesses: {'; '.join(str(w) for w in weaknesses[:2])}"]

        # Human verdict
        lines += ["", "#### 🧑‍🔬 Human Interpretation"]
        lines += _human_verdict(r, stem)
        lines += ["", "---", ""]

    # ── Cross-paper comparison ────────────────────────────────────────────────
    lines += ["", "## 3. Cross-Paper Comparison", ""]
    lines += ["| Paper | Overall | Claims | Gaps | CF Score |",
              "|-------|---------|--------|------|----------|"]
    for r in results:
        if r.get("error"):
            lines.append(f"| `{r.get('_pdf','?')}` | ERROR | — | — | — |")
            continue
        sc  = r.get("scores", {})
        ov  = f"{sc.get('overall_score','?'):.1f}" if isinstance(sc.get("overall_score"), float) else "?"
        ncl = len(r.get("claims", []))
        ngp = len(r.get("citation_gaps", []))
        cf  = r.get("counterfactuality", {}).get("overall_score", "?")
        lines.append(f"| `{r.get('_pdf','?')}` | {ov}/10 | {ncl} | {ngp} | {cf} |")

    # ── Overall verdict ───────────────────────────────────────────────────────
    lines += ["", "## 4. Overall Pipeline Quality Assessment", ""]
    valid = [r for r in results if not r.get("error")]
    if valid:
        avg_overall = sum(
            r.get("scores", {}).get("overall_score", 0) for r in valid
            if isinstance(r.get("scores", {}).get("overall_score"), (int, float))
        ) / max(len(valid), 1)

        crash_count = sum(1 for r in results if r.get("error"))
        pass_count  = sum(1 for r in results if "complet" in str(r.get("status","")))

        lines += [
            f"- **Papers processed:** {len(results)}",
            f"- **Successful runs:** {pass_count}/{len(results)}",
            f"- **Average overall score:** {avg_overall:.1f}/10",
            f"- **Crash rate:** {crash_count}/{len(results)}",
            "",
        ]

        if avg_overall >= 6.0 and pass_count == len(results):
            lines += ["✅ **Pipeline is performing as expected across all test papers.**"]
        elif avg_overall >= 5.0 and pass_count >= len(results) * 0.8:
            lines += ["⚠️ **Pipeline mostly functional but some papers show degraded output. Review failed agents.**"]
        else:
            lines += ["❌ **Pipeline quality is below expected threshold. Investigate failures.**"]

    lines += [
        "",
        "---",
        f"*Report auto-generated by `scripts/run_batch_test.py`*",
    ]
    return "\n".join(lines)


# ── Domain-aware interpretation helpers ───────────────────────────────────────

def _interpret_keywords(kws: list, stem: str) -> list[str]:
    kw_set = {k.lower() for k in kws}
    notes = []
    if stem == "kg_embedding_survey" and any("embedding" in k or "knowledge" in k for k in kw_set):
        notes.append("Keywords correctly capture KG embedding themes ✅")
    elif stem == "yolov10_endtoend" and any("detection" in k or "yolo" in k for k in kw_set):
        notes.append("Keywords correctly capture detection/YOLO themes ✅")
    elif stem == "llm_reasoning_divide_conquer" and any("reasoning" in k or "llm" in k or "language model" in k for k in kw_set):
        notes.append("Keywords correctly capture LLM reasoning themes ✅")
    elif stem == "federated_chest_radiograph" and any("federated" in k or "radiograph" in k or "chest" in k for k in kw_set):
        notes.append("Keywords correctly capture federated medical imaging themes ✅")
    elif stem == "gnn_deeper" and any("graph" in k or "neural" in k or "gnn" in k for k in kw_set):
        notes.append("Keywords correctly capture GNN themes ✅")
    else:
        notes.append("Keywords may not fully reflect domain — review KBIR output")
    return notes


def _interpret_claims(claims: list, stem: str) -> list[str]:
    notes = []
    if not claims:
        return ["  ⚠️ No claims extracted — check ClaimAgent and section parsing"]
    supported = sum(1 for c in claims if c.get("verdict") in VERDICT_PASS)
    support_pct = supported / len(claims) * 100 if claims else 0
    if support_pct >= 60:
        notes.append(f"  ✅ {support_pct:.0f}% claim support rate is healthy for a research paper")
    elif support_pct >= 30:
        notes.append(f"  ⚠️ {support_pct:.0f}% support rate — moderate; some claims may be speculative")
    else:
        notes.append(f"  ❌ Low support rate ({support_pct:.0f}%) — claims may be under-evidenced or verifier struggled")

    if stem == "kg_embedding_survey":
        notes.append("  Survey papers should have high citation support — low rate would indicate verifier gap")
    return notes


def _interpret_gaps(gaps: list, stem: str) -> list[str]:
    notes = []
    high = sum(1 for g in gaps if g.get("severity") in ("high","critical"))
    if stem == "kg_embedding_survey" and len(gaps) <= 3:
        notes.append(f"  ✅ Survey paper with few gaps ({len(gaps)}) — expected, surveys are typically well-cited")
    elif high >= 3:
        notes.append(f"  ⚠️ {high} high-severity gaps — paper may have weak citation coverage in key areas")
    elif len(gaps) == 0:
        notes.append("  ✅ No citation gaps found — good coverage, or CitationGapAgent may need more context")
    return notes


def _interpret_cf(cf: dict, claim_cfs: list, stem: str) -> list[str]:
    notes = []
    score = cf.get("overall_score")
    if isinstance(score, (int, float)):
        if score >= 7:
            notes.append("  ✅ High counterfactuality score — claims are well-grounded and falsifiable")
        elif score >= 4:
            notes.append("  ⚠️ Moderate counterfactuality — some claims lack clear falsifiability")
        else:
            notes.append("  ❌ Low counterfactuality — paper may contain speculative or unfalsifiable claims")
    high_cf = [c for c in claim_cfs if isinstance(c.get("counterfactual_score"), (int,float)) and c["counterfactual_score"] >= 0.7]
    if high_cf:
        notes.append(f"  {len(high_cf)} claim(s) flagged as highly counterfactual — may warrant manual review")
    return notes


def _human_verdict(r: dict, stem: str) -> list[str]:
    scores   = r.get("scores", {})
    claims   = r.get("claims", [])
    gaps     = r.get("citation_gaps", [])
    agents   = r.get("agents", {})
    overall  = scores.get("overall_score")
    failed   = [a for a, d in agents.items() if d.get("status") == "failed"]
    notes = []

    # Overall score interpretation
    if isinstance(overall, float):
        if overall >= 7.5:
            notes.append(f"Score {overall:.1f}/10 → **Strong paper.** Claims are well-supported and argumentation is solid.")
        elif overall >= 5.5:
            notes.append(f"Score {overall:.1f}/10 → **Adequate paper.** Reasonable quality with some gaps or weak evidence.")
        else:
            notes.append(f"Score {overall:.1f}/10 → **Below average.** Significant weaknesses in claims, citations, or argumentation.")

    # Domain-specific expected behaviour
    expected = {
        "kg_embedding_survey":          "Survey: expect many citations, broad keyword coverage, moderate novelty.",
        "llm_reasoning_divide_conquer": "LLM: expect novel claims, empirical evidence, recent references.",
        "federated_chest_radiograph":   "Medical FL: expect strong methodological claims, clinical evidence.",
        "yolov10_endtoend":             "CV: expect quantitative claims (mAP, FPS), strong benchmark citations.",
        "gnn_deeper":                   "GNN architecture: expect theoretical + empirical claims, ablation studies.",
    }.get(stem, "")
    if expected:
        notes.append(f"**Expected profile:** {expected}")

    # Failure flags
    if failed:
        notes.append(f"**⚠️ Pipeline gaps:** Agents `{', '.join(failed)}` failed — results may be incomplete.")

    if not claims:
        notes.append("**⚠️ No claims extracted** — ClaimAgent may have failed or sections were not parsed.")
    if not gaps and stem != "kg_embedding_survey":
        notes.append("CitationGap found nothing — could be thorough citations or gap detection needs tuning.")

    if not notes:
        notes.append("Pipeline ran cleanly with no anomalies detected.")

    return [f"- {n}" for n in notes]


if __name__ == "__main__":
    asyncio.run(main())