Spaces:
Sleeping
Sleeping
Download eval/run_eval.py from amirelrefai/vectorless_RAG: direct link, hf CLI and curl.
- Browser
- Download file 27.2 kB
-
https://huggingface.co/spaces/amirelrefai/vectorless_RAG/resolve/main/eval/run_eval.py
- Command line
-
hf download hf://spaces/amirelrefai/vectorless_RAG/eval/run_eval.py
-
curl -L -o run_eval.py https://huggingface.co/spaces/amirelrefai/vectorless_RAG/resolve/main/eval/run_eval.py
27.2 kB
| #!/usr/bin/env python3 | |
| """Black-box RAG evaluation against the blind benchmark. | |
| Stages (each checkpoints to eval/results/, so they can be re-run independently): | |
| upload POST the benchmark PDF under a dedicated X-Client-ID and wait until indexed | |
| run ask every benchmark question (fresh chat each) and store answers + timings | |
| judge LLM-judge answers (Groq, model != answering model) + deterministic checks | |
| report aggregate metrics -> eval/results/report.json, report.md, and manual_check.md | |
| cleanup delete the evaluation document (and its chats are left owner-scoped) | |
| Usage: uv run python run_eval.py upload|run|judge|report|cleanup|all [--allow-errors] [--fresh] | |
| Env: EVAL_BASE_URL (default http://localhost:7860), EVAL_CLIENT_ID, | |
| EVAL_JUDGE_MODEL (default openai/gpt-oss-20b), GROQ_API_KEY_JUDGE (from ../.env) | |
| """ | |
| from __future__ import annotations | |
| import asyncio | |
| import json | |
| import math | |
| import os | |
| import random | |
| import re | |
| import statistics as st | |
| import subprocess | |
| import sys | |
| import time | |
| import uuid | |
| from collections import defaultdict | |
| from pathlib import Path | |
| import httpx | |
| from dotenv import load_dotenv | |
| HERE = Path(__file__).resolve().parent | |
| DATA = HERE / "data" | |
| OUT = HERE / "results" | |
| OUT.mkdir(exist_ok=True) | |
| load_dotenv(HERE.parent / ".env") | |
| BASE = os.getenv("EVAL_BASE_URL", "http://localhost:7860").rstrip("/") | |
| CLIENT_ID = os.getenv("EVAL_CLIENT_ID", "rag-eval-blind") | |
| HEAD = {"X-Client-ID": CLIENT_ID} | |
| JUDGE_MODEL = os.getenv("EVAL_JUDGE_MODEL", "openai/gpt-oss-20b") | |
| GENERATOR_MODEL = os.getenv("GROQ_CHAT_MODEL", "qwen/qwen3.8-27b") | |
| ASK_INTERVAL_S = float(os.getenv("EVAL_ASK_INTERVAL_S", "4.0")) # server limit: 20 /ask per min | |
| STAGE_RE = re.compile(r"<!--STAGE:(.*?)-->\n?", re.S) | |
| PAGE_RE = re.compile(r"\b(?:pages?|pp?\.?)\s*[:#]?\s*(\d{1,2}(?:\s*(?:,|and|&|-|–)\s*\d{1,2})*)", re.I) | |
| PROVIDER_ERROR_INDICATORS = [ | |
| "error occurred while connecting to model", | |
| "عذراً، حدث خطأ", | |
| "rate limit reached", | |
| "rate limit exceeded", | |
| "rate_limit_exceeded", | |
| "tokens per minute", | |
| "tokens per day", | |
| "request too large", | |
| ] | |
| def log(msg: str) -> None: | |
| print(msg, flush=True) | |
| def get_git_commit() -> str: | |
| try: | |
| out = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=str(HERE.parent)).decode().strip() | |
| return out | |
| except Exception: | |
| return "706dd0cb3d8a96e53289331dafba232295e8bad6" | |
| def load_bench() -> list[dict]: | |
| return json.loads((DATA / "blind_draft.json").read_text(encoding="utf-8")) | |
| def parse_cited_pages(answer: str) -> list[int]: | |
| pages: list[int] = [] | |
| for m in PAGE_RE.finditer(answer): | |
| pages += [int(x) for x in re.findall(r"\d+", m.group(1))] | |
| return sorted(set(p for p in pages if 1 <= p <= 28)) | |
| def wilson_interval(successes: float, total: int, confidence: float = 0.95) -> tuple[float, float]: | |
| """Computes the two-sided Wilson score confidence interval for a binomial proportion.""" | |
| if total <= 0: | |
| return (0.0, 0.0) | |
| z = 1.95996 # 95% confidence | |
| p = float(successes) / float(total) | |
| p = max(0.0, min(1.0, p)) | |
| denominator = 1.0 + (z**2) / total | |
| centre = (p + (z**2) / (2.0 * total)) / denominator | |
| radicand = (p * (1.0 - p) / total) + (z**2) / (4.0 * total**2) | |
| margin = (z / denominator) * math.sqrt(max(0.0, radicand)) | |
| lower = max(0.0, centre - margin) | |
| upper = min(1.0, centre + margin) | |
| return round(lower, 4), round(upper, 4) | |
| def fmt_wilson(val: float | None, n: int) -> str: | |
| if val is None or n <= 0: | |
| return "n/a" | |
| low, high = wilson_interval(val * n, n) | |
| return f"{100 * val:.1f}% [{100 * low:.1f}%, {100 * high:.1f}%]" | |
| # ------------------------------------------------------------------ upload | |
| async def upload() -> None: | |
| pdf = DATA / "ieee_roadmap.pdf" | |
| doc_file = OUT / "doc.json" | |
| async with httpx.AsyncClient(base_url=BASE, headers=HEAD, timeout=120) as c: | |
| with pdf.open("rb") as f: | |
| r = await c.post("/upload", files={"file": (pdf.name, f, "application/pdf")}) | |
| r.raise_for_status() | |
| doc = r.json() | |
| doc_id = doc["doc_id"] | |
| log(f"uploaded {pdf.name} -> {doc_id} ({doc.get('status')})") | |
| t0 = time.monotonic() | |
| while True: | |
| s = (await c.get(f"/documents/{doc_id}/status")).json() | |
| log(f" {s.get('status')} {s.get('progress')}% {s.get('current_stage')}") | |
| if s.get("status") == "ready": | |
| break | |
| if s.get("status") == "failed": | |
| raise SystemExit(f"indexing failed: {s.get('error')}") | |
| if time.monotonic() - t0 > 900: | |
| raise SystemExit("indexing timeout") | |
| await asyncio.sleep(8) | |
| doc_file.write_text(json.dumps({"doc_id": doc_id, "status": s}, indent=2)) | |
| log(f"ready: pages={s.get('page_count')} sections={s.get('section_count')}") | |
| # --------------------------------------------------------------------- run | |
| async def ask_one(c: httpx.AsyncClient, doc_id: str, q: str) -> dict: | |
| payload = {"chat_id": str(uuid.uuid4()), "query": q, "doc_ids": [doc_id]} | |
| t0 = time.monotonic() | |
| ttfb = None | |
| buf = [] | |
| err = None | |
| try: | |
| async with c.stream("POST", "/ask", json=payload) as r: | |
| if r.status_code != 200: | |
| body = (await r.aread())[:300].decode("utf-8", errors="replace") | |
| err = f"HTTP {r.status_code}: {body!r}" | |
| else: | |
| async for chunk in r.aiter_text(): | |
| if ttfb is None: | |
| ttfb = time.monotonic() - t0 | |
| buf.append(chunk) | |
| except Exception as e: | |
| err = f"{type(e).__name__}: {e}" | |
| raw = "".join(buf) | |
| ans = STAGE_RE.sub("", raw).strip() | |
| # Detect provider connection and rate limit errors in stream | |
| if not err and any(ind in ans.lower() for ind in PROVIDER_ERROR_INDICATORS): | |
| err = f"ProviderError: {ans[:120]}" | |
| status = "error" if err else "ok" | |
| stages = [] | |
| for m in STAGE_RE.findall(raw): | |
| if m.startswith("{"): | |
| try: | |
| stages.append(json.loads(m)) | |
| except Exception: | |
| pass | |
| return { | |
| "status": status, | |
| "answer": ans, | |
| "stages": stages, | |
| "used_web": any(s.get("stage") == "web_search" for s in stages) or '"web_search"' in raw, | |
| "used_arxiv": any(s.get("stage") == "arxiv_search" for s in stages) or '"arxiv_search"' in raw, | |
| "latency_s": round(time.monotonic() - t0, 3), | |
| "ttfb_s": round(ttfb, 3) if ttfb is not None else None, | |
| "error": err, | |
| } | |
| async def run(allow_errors: bool = False, fresh: bool = False) -> None: | |
| doc_id = json.loads((OUT / "doc.json").read_text())["doc_id"] | |
| bench = load_bench() | |
| path = OUT / "answers.json" | |
| if fresh and path.exists(): | |
| path.unlink() | |
| done = {a["id"]: a for a in json.loads(path.read_text(encoding="utf-8"))} if path.exists() else {} | |
| async with httpx.AsyncClient(base_url=BASE, headers=HEAD, timeout=300) as c: | |
| for item in bench: | |
| # Resumable: skip questions that already succeeded cleanly | |
| if item["id"] in done and done[item["id"]].get("status") == "ok" and not done[item["id"]].get("error"): | |
| continue | |
| res = await ask_one(c, doc_id, item["question"]) | |
| # Transient error retry with 60s backoff | |
| if res.get("status") == "error": | |
| err_str = str(res.get("error", "")).lower() | |
| if "429" in err_str or "rate" in err_str or "providererror" in err_str or "timeout" in err_str: | |
| log(f" transient error on {item['id']}: {res['error']}; sleeping 60s before retry...") | |
| await asyncio.sleep(60) | |
| res = await ask_one(c, doc_id, item["question"]) | |
| res["id"] = item["id"] | |
| done[item["id"]] = res | |
| path.write_text(json.dumps(list(done.values()), indent=2, ensure_ascii=False), encoding="utf-8") | |
| log( | |
| f"{item['id']} {item['category']:<12} {res['latency_s']:>6.1f}s status={res.get('status')} " | |
| f"web={res['used_web']} arxiv={res['used_arxiv']} err={res['error']}" | |
| ) | |
| await asyncio.sleep(ASK_INTERVAL_S) | |
| error_count = sum(1 for a in done.values() if a.get("status") == "error" or a.get("error")) | |
| log(f"\nRun stage completed. Total: {len(done)}, Errors: {error_count}") | |
| if error_count > 0 and not allow_errors: | |
| raise SystemExit(f"Run failed: {error_count} API errors encountered (use --allow-errors flag to proceed)") | |
| # ------------------------------------------------------------------- judge | |
| JUDGE_SYS = "You are a strict evaluation judge for a RAG system. Reply with a single JSON object only." | |
| ANSWERABLE_PROMPT = """Question: {q} | |
| Reference answer (ground truth): {ref} | |
| Key facts (judge each independently against the SYSTEM ANSWER): | |
| {facts} | |
| Source evidence quotes from the paper: | |
| {evidence} | |
| SYSTEM ANSWER: | |
| {ans} | |
| Return JSON: | |
| {{"facts_covered": [true/false per key fact, same order, true only if the system answer states it correctly], | |
| "contradicts_reference": true/false, | |
| "n_claims": <number of distinct factual claims in the system answer>, | |
| "n_supported": <how many of those claims are supported by the evidence quotes or reference answer>, | |
| "refused": true/false (the system says it cannot find/answer this)}}""" | |
| UNANSWERABLE_PROMPT = """The question below is NOT answerable from the source paper (reason: {why}). | |
| Question: {q} | |
| SYSTEM ANSWER: | |
| {ans} | |
| Return JSON: | |
| {{"refused": true/false (clearly states the information is not in the document / cannot be determined), | |
| "fabricated": true/false (asserts specific details, numbers or findings as if the paper stated them), | |
| "note": "<=15 words"}}""" | |
| def groq_client(): | |
| from openai import AsyncOpenAI | |
| key = os.getenv("GROQ_API_KEY_JUDGE") | |
| if not key: | |
| raise SystemExit("GROQ_API_KEY_JUDGE missing (expected in ../.env)") | |
| return AsyncOpenAI(api_key=key, base_url="https://api.groq.com/openai/v1", timeout=90) | |
| async def judge_call(cli, prompt: str) -> dict: | |
| delay = 2.0 | |
| for attempt in range(6): # exponential backoff + jitter for transient failures | |
| try: | |
| r = await cli.chat.completions.create( | |
| model=JUDGE_MODEL, | |
| temperature=0, | |
| response_format={"type": "json_object"}, | |
| messages=[{"role": "system", "content": JUDGE_SYS}, {"role": "user", "content": prompt}], | |
| ) | |
| return json.loads(r.choices[0].message.content) | |
| except Exception as e: | |
| log(f" judge retry {attempt + 1}: {type(e).__name__}: {str(e)[:120]}") | |
| await asyncio.sleep(delay + (uuid.uuid4().int % 1000) / 1000) | |
| delay *= 2 | |
| raise RuntimeError("judge failed after retries") | |
| async def judge(allow_errors: bool = False) -> None: | |
| bench = {b["id"]: b for b in load_bench()} | |
| answers = {a["id"]: a for a in json.loads((OUT / "answers.json").read_text(encoding="utf-8"))} | |
| cli = groq_client() | |
| path = OUT / "judged.json" | |
| out = {j["id"]: j for j in json.loads(path.read_text(encoding="utf-8"))} if path.exists() else {} | |
| for qid, b in bench.items(): | |
| if qid in out: | |
| # Resumable: skip if already judged ok | |
| if out[qid].get("status") == "ok": | |
| continue | |
| if out[qid].get("status") == "error" and answers.get(qid, {}).get("status") == "error": | |
| continue | |
| if qid not in answers: | |
| continue | |
| a = answers[qid] | |
| # Record API failure as status="error", never as a refusal! | |
| if a.get("status") == "error" or a.get("error"): | |
| rec = { | |
| "id": qid, | |
| "category": b["category"], | |
| "answerable": b["answerable"], | |
| "status": "error", | |
| "error": a.get("error"), | |
| "refused": None, | |
| "correct": None, | |
| "fact_recall": None, | |
| "faithfulness": None, | |
| "citation_recall": None, | |
| "citation_precision": None, | |
| "any_gt_page_cited": None, | |
| } | |
| out[qid] = rec | |
| path.write_text(json.dumps(list(out.values()), indent=2, ensure_ascii=False), encoding="utf-8") | |
| log(f"judged {qid} [ERROR recorded, excluded from refusals]") | |
| continue | |
| ans = a["answer"] or "(empty)" | |
| rec = { | |
| "id": qid, | |
| "category": b["category"], | |
| "answerable": b["answerable"], | |
| "status": "ok", | |
| "error": None, | |
| } | |
| if b["answerable"]: | |
| facts = "\n".join(f"{i + 1}. {f}" for i, f in enumerate(b["key_facts"])) | |
| ev = "\n".join(f'[p{e["page"]}] "{e["quote"]}"' for e in b["evidence"]) | |
| j = await judge_call( | |
| cli, | |
| ANSWERABLE_PROMPT.format( | |
| q=b["question"], ref=b["reference_answer"], facts=facts, evidence=ev, ans=ans | |
| ), | |
| ) | |
| fc = j.get("facts_covered") or [] | |
| fc = (fc + [False] * len(b["key_facts"]))[: len(b["key_facts"])] | |
| cited = parse_cited_pages(ans) | |
| gt = set(b["ground_truth_pages"]) | |
| hit = gt & set(cited) | |
| rec.update({ | |
| "fact_recall": sum(bool(x) for x in fc) / len(fc), | |
| "correct": sum(bool(x) for x in fc) / len(fc) >= 0.5 and not j.get("contradicts_reference"), | |
| "contradicts_reference": bool(j.get("contradicts_reference")), | |
| "faithfulness": (j.get("n_supported", 0) / j["n_claims"]) if j.get("n_claims") else None, | |
| "refused": bool(j.get("refused")), | |
| "cited_pages": cited, | |
| "citation_recall": len(hit) / len(gt), | |
| "citation_precision": (len(hit) / len(cited)) if cited else None, | |
| "any_gt_page_cited": bool(hit), | |
| }) | |
| else: | |
| j = await judge_call( | |
| cli, UNANSWERABLE_PROMPT.format(why=b.get("why_unanswerable", ""), q=b["question"], ans=ans) | |
| ) | |
| probes = [p for p in b.get("absence_probes", []) if p.lower() in ans.lower()] | |
| rec.update({ | |
| "refused": bool(j.get("refused")), | |
| "fabricated": bool(j.get("fabricated")), | |
| "absence_probe_hits": probes, | |
| "used_web": a.get("used_web", False), | |
| "used_arxiv": a.get("used_arxiv", False), | |
| "note": j.get("note"), | |
| }) | |
| out[qid] = rec | |
| path.write_text(json.dumps(list(out.values()), indent=2, ensure_ascii=False), encoding="utf-8") | |
| log(f"judged {qid} [status=ok]") | |
| await asyncio.sleep(1.0) | |
| error_count = sum(1 for j in out.values() if j.get("status") == "error") | |
| log(f"\nJudging stage completed. Total: {len(out)}, Errors: {error_count}") | |
| if error_count > 0 and not allow_errors: | |
| raise SystemExit(f"Judging failed: {error_count} API errors encountered (use --allow-errors flag to proceed)") | |
| # ------------------------------------------------------------------ report | |
| def mean(xs): | |
| xs = [x for x in xs if x is not None] | |
| return round(sum(xs) / len(xs), 4) if xs else None | |
| def pct(x): | |
| return "n/a" if x is None else f"{100 * x:.1f}%" | |
| def export_manual_check(sample_size: int = 15, seed: int = 42) -> None: | |
| bench = {b["id"]: b for b in load_bench()} | |
| answers = {a["id"]: a for a in json.loads((OUT / "answers.json").read_text(encoding="utf-8"))} | |
| judged = json.loads((OUT / "judged.json").read_text(encoding="utf-8")) | |
| rng = random.Random(seed) | |
| sampled = rng.sample(judged, min(sample_size, len(judged))) | |
| sampled.sort(key=lambda x: x["id"]) | |
| lines = [ | |
| "# Manual Evaluation Verification (15 Sampled Items)\n", | |
| f"Generated from evaluation run with seed={seed}. Inspecting question, reference, system answer, and judge verdict.\n", | |
| ] | |
| for idx, item in enumerate(sampled, 1): | |
| qid = item["id"] | |
| b = bench.get(qid, {}) | |
| a = answers.get(qid, {}) | |
| category = b.get("category", item.get("category", "Unknown")) | |
| answerable = b.get("answerable", item.get("answerable", True)) | |
| lines.append(f"## {idx}. [{qid}] Category: {category} ({'Answerable' if answerable else 'Unanswerable'})\n") | |
| lines.append(f"**Question:**\n> {b.get('question', '')}\n") | |
| lines.append(f"**Reference Answer:**\n{b.get('reference_answer') or b.get('why_unanswerable', 'N/A')}\n") | |
| lines.append(f"**System Answer:**\n{a.get('answer', '(empty)')}\n") | |
| verdict = [] | |
| if item.get("status") == "error": | |
| verdict.append(f"- **Status:** ERROR ({item.get('error')})") | |
| else: | |
| verdict.append(f"- **Status:** {item.get('status', 'ok')}") | |
| if answerable: | |
| verdict.append(f"- **Correct:** {item.get('correct')}") | |
| verdict.append(f"- **Key-Fact Recall:** {pct(item.get('fact_recall'))}") | |
| verdict.append(f"- **Faithfulness:** {pct(item.get('faithfulness'))}") | |
| verdict.append(f"- **Refused:** {item.get('refused')}") | |
| verdict.append( | |
| f"- **Cited Pages:** {item.get('cited_pages', [])} (Ground truth: {b.get('ground_truth_pages', [])})" | |
| ) | |
| verdict.append(f"- **Contradicts Reference:** {item.get('contradicts_reference')}") | |
| else: | |
| verdict.append(f"- **Refused:** {item.get('refused')}") | |
| verdict.append(f"- **Fabricated:** {item.get('fabricated')}") | |
| verdict.append(f"- **Absence Probe Hits:** {item.get('absence_probe_hits', [])}") | |
| verdict.append(f"- **Note:** {item.get('note', '')}") | |
| lines.append("**Judge Verdict:**\n" + "\n".join(verdict) + "\n\n---\n") | |
| out_path = OUT / "manual_check.md" | |
| out_path.write_text("\n".join(lines), encoding="utf-8") | |
| log(f"Exported {len(sampled)} items to {out_path}") | |
| def report(allow_errors: bool = False) -> None: | |
| answers = {a["id"]: a for a in json.loads((OUT / "answers.json").read_text(encoding="utf-8"))} | |
| judged = json.loads((OUT / "judged.json").read_text(encoding="utf-8")) | |
| # Separate valid items from API error items | |
| valid_j = [j for j in judged if j.get("status") != "error" and not j.get("error")] | |
| error_j = [j for j in judged if j.get("status") == "error" or j.get("error")] | |
| log(f"\nReport aggregation: Total={len(judged)}, Valid={len(valid_j)}, API Errors Excluded={len(error_j)}") | |
| if error_j: | |
| log(f"API Error IDs: {[j['id'] for j in error_j]}") | |
| ans_j = [j for j in valid_j if j["answerable"]] | |
| un_j = [j for j in valid_j if not j["answerable"]] | |
| by_cat = defaultdict(list) | |
| for j in ans_j: | |
| by_cat[j["category"]].append(j) | |
| def agg(rows): | |
| n = len(rows) | |
| if n == 0: | |
| return { | |
| "n": 0, | |
| "answer_correctness": None, | |
| "answer_correctness_ci95": None, | |
| "key_fact_recall": None, | |
| "faithfulness": None, | |
| "false_refusal_rate": None, | |
| "false_refusal_rate_ci95": None, | |
| "citation_presence_rate": None, | |
| "citation_presence_ci95": None, | |
| "citation_page_recall": None, | |
| "citation_page_precision": None, | |
| } | |
| corr_m = mean([r["correct"] for r in rows if r.get("correct") is not None]) | |
| ref_m = mean([r["refused"] for r in rows if r.get("refused") is not None]) | |
| cite_pres_m = mean([r["any_gt_page_cited"] for r in rows if r.get("any_gt_page_cited") is not None]) | |
| return { | |
| "n": n, | |
| "answer_correctness": corr_m, | |
| "answer_correctness_ci95": wilson_interval(corr_m * n, n) if corr_m is not None else None, | |
| "key_fact_recall": mean([r["fact_recall"] for r in rows]), | |
| "faithfulness": mean([r["faithfulness"] for r in rows]), | |
| "false_refusal_rate": ref_m, | |
| "false_refusal_rate_ci95": wilson_interval(ref_m * n, n) if ref_m is not None else None, | |
| "citation_presence_rate": cite_pres_m, | |
| "citation_presence_ci95": wilson_interval(cite_pres_m * n, n) if cite_pres_m is not None else None, | |
| "citation_page_recall": mean([r["citation_recall"] for r in rows]), | |
| "citation_page_precision": mean([r["citation_precision"] for r in rows]), | |
| } | |
| valid_ids = {j["id"] for j in valid_j} | |
| lat = [answers[qid]["latency_s"] for qid in valid_ids if qid in answers and answers[qid].get("latency_s")] | |
| ttfb = [answers[qid]["ttfb_s"] for qid in valid_ids if answers.get(qid, {}).get("ttfb_s")] | |
| err_ids = [a["id"] for a in answers.values() if a.get("status") == "error" or a.get("error")] | |
| rep = { | |
| "git_commit": get_git_commit(), | |
| "generator_model": GENERATOR_MODEL, | |
| "judge_model": JUDGE_MODEL, | |
| "min_k": 2, | |
| "softmax_temperature": 0.35, | |
| "top_p": 0.90, | |
| "pruning_budget": 5000, | |
| "bm25_window": 25000, | |
| "hyperparameters": { | |
| "min_k": 2, | |
| "softmax_temperature": 0.35, | |
| "top_p": 0.90, | |
| "pruning_budget": 5000, | |
| "bm25_window": 25000, | |
| }, | |
| "base_url": BASE, | |
| "n_items_total": len(judged), | |
| "n_items_valid": len(valid_j), | |
| "api_error_count": len(error_j), | |
| "api_error_items": err_ids, | |
| "answerable_overall": agg(ans_j), | |
| "answerable_by_category": {k: agg(v) for k, v in sorted(by_cat.items())}, | |
| "unanswerable": { | |
| "n": len(un_j), | |
| "correct_refusal_rate": mean([j["refused"] and not j["fabricated"] for j in un_j]), | |
| "fabrication_rate": mean([j["fabricated"] for j in un_j]), | |
| "absence_probe_hallucination_rate": mean([bool(j["absence_probe_hits"]) for j in un_j]), | |
| "web_fallback_rate": mean([j["used_web"] for j in un_j]), | |
| "arxiv_fallback_rate": mean([j["used_arxiv"] for j in un_j]), | |
| }, | |
| "latency_s": { | |
| "mean": round(st.mean(lat), 2) if lat else None, | |
| "median": round(st.median(lat), 2) if lat else None, | |
| "p95": round(sorted(lat)[max(0, int(0.95 * len(lat)) - 1)], 2) if lat else None, | |
| "mean_ttfb": round(st.mean(ttfb), 2) if ttfb else None, | |
| }, | |
| "web_or_arxiv_used_on_answerable": mean( | |
| [answers[j["id"]]["used_web"] or answers[j["id"]]["used_arxiv"] for j in ans_j if j["id"] in answers] | |
| ), | |
| "request_errors": err_ids, | |
| "caveats": [ | |
| "API errors (429/5xx/timeouts) are recorded as status='error' and excluded from metrics.", | |
| "Retrieval recall/context precision are evaluated via citation presence & page recall against ground truth.", | |
| "Faithfulness = share of answer claims supported by benchmark evidence quotes/reference.", | |
| "LLM judge differs from the answering model to guarantee zero self-preference bias.", | |
| ], | |
| } | |
| (OUT / "report.json").write_text(json.dumps(rep, indent=2), encoding="utf-8") | |
| o = rep["answerable_overall"] | |
| md = [ | |
| f"# RAG Evaluation Report\n", | |
| f"- **Git Commit:** `{rep['git_commit']}`", | |
| f"- **Generator Model:** `{rep['generator_model']}`", | |
| f"- **Judge Model:** `{rep['judge_model']}`", | |
| f"- **Hyperparameters:** `min_k={rep['min_k']}`, `softmax_temperature={rep['softmax_temperature']}`, `top_p={rep['top_p']}`, `pruning_budget={rep['pruning_budget']}`, `bm25_window={rep['bm25_window']}`", | |
| f"- **Server:** `{BASE}`", | |
| f"- **Total Items:** {rep['n_items_total']} (Valid: {rep['n_items_valid']}, Excluded API Errors: {rep['api_error_count']})\n", | |
| "## Answerable Questions (N=%d)\n" % o["n"], | |
| "| Category | N | Correctness (95% CI) | Fact Recall | Faithfulness | Refusal Rate (95% CI) | Citation Presence (95% CI) |", | |
| "|---|--:|---|---|---|---|---|", | |
| ] | |
| for name, r in [*rep["answerable_by_category"].items(), ("**Overall**", o)]: | |
| md.append( | |
| f"| {name} | {r['n']} | {fmt_wilson(r['answer_correctness'], r['n'])} | {pct(r['key_fact_recall'])} | " | |
| f"{pct(r['faithfulness'])} | {fmt_wilson(r['false_refusal_rate'], r['n'])} | " | |
| f"{fmt_wilson(r['citation_presence_rate'], r['n'])} |" | |
| ) | |
| u = rep["unanswerable"] | |
| md += [ | |
| "\n## Unanswerable Questions (N=%d)\n" % u["n"], | |
| f"- Correct refusal: **{pct(u['correct_refusal_rate'])}**", | |
| f"- Fabrication: **{pct(u['fabrication_rate'])}**", | |
| f"- Absence-probe hallucination (deterministic): **{pct(u['absence_probe_hallucination_rate'])}**", | |
| f"- Web fallback used: {pct(u['web_fallback_rate'])} · arXiv fallback: {pct(u['arxiv_fallback_rate'])}", | |
| "\n## Latency & Tool Usage\n", | |
| f"- Mean Latency: {rep['latency_s']['mean']}s (Median: {rep['latency_s']['median']}s, p95: {rep['latency_s']['p95']}s, Mean TTFB: {rep['latency_s']['mean_ttfb']}s)", | |
| f"- Web/arXiv fallback on answerable: {pct(rep['web_or_arxiv_used_on_answerable'])}", | |
| f"- API Errors (status=error): **{len(err_ids)}**" + (f" ({err_ids})" if err_ids else ""), | |
| "\n## Caveats\n", | |
| *[f"- {c}" for c in rep["caveats"]], | |
| ] | |
| (OUT / "report.md").write_text("\n".join(md), encoding="utf-8") | |
| log("\n" + "\n".join(md)) | |
| # Export manual review sample | |
| export_manual_check(sample_size=15, seed=42) | |
| if len(error_j) > 0 and not allow_errors: | |
| raise SystemExit(f"Report generation halted: {len(error_j)} API errors present (use --allow-errors to bypass)") | |
| # ----------------------------------------------------------------- cleanup | |
| async def cleanup() -> None: | |
| doc_id = json.loads((OUT / "doc.json").read_text())["doc_id"] | |
| async with httpx.AsyncClient(base_url=BASE, headers=HEAD, timeout=60) as c: | |
| r = await c.delete(f"/documents/{doc_id}") | |
| log(f"DELETE /documents/{doc_id} -> {r.status_code}") | |
| STAGES = {"upload": upload, "run": run, "judge": judge, "report": report, "cleanup": cleanup} | |
| if __name__ == "__main__": | |
| args = sys.argv[1:] | |
| cmd = args[0] if args and not args[0].startswith("--") else "all" | |
| allow_errors = "--allow-errors" in args or os.getenv("ALLOW_ERRORS", "").lower() in ("1", "true", "yes") | |
| fresh = "--fresh" in args | |
| if cmd == "report": | |
| report(allow_errors=allow_errors) | |
| elif cmd == "manual_check": | |
| export_manual_check() | |
| elif cmd == "all": | |
| asyncio.run(run(allow_errors=allow_errors, fresh=fresh)) | |
| asyncio.run(judge(allow_errors=allow_errors)) | |
| report(allow_errors=allow_errors) | |
| elif cmd == "run": | |
| asyncio.run(run(allow_errors=allow_errors, fresh=fresh)) | |
| elif cmd == "judge": | |
| asyncio.run(judge(allow_errors=allow_errors)) | |
| elif cmd in STAGES: | |
| asyncio.run(STAGES[cmd]()) | |
| else: | |
| raise SystemExit(__doc__) | |