Spaces:
Sleeping
Sleeping
Download eval/retrieval_eval.py from amirelrefai/vectorless_RAG: direct link, hf CLI and curl.
- Browser
- Download file 14.7 kB
-
https://huggingface.co/spaces/amirelrefai/vectorless_RAG/resolve/main/eval/retrieval_eval.py
- Command line
-
hf download hf://spaces/amirelrefai/vectorless_RAG/eval/retrieval_eval.py
-
curl -L -o retrieval_eval.py https://huggingface.co/spaces/amirelrefai/vectorless_RAG/resolve/main/eval/retrieval_eval.py
14.7 kB
| #!/usr/bin/env python3 | |
| """Judge-free retrieval evaluation + offline parameter search for the Vectorless RAG pipeline. | |
| No LLM is called. The pipeline's deterministic stages (BM25, weighted RRF + nucleus cutoff, | |
| paragraph pruning, intent routing) are imported from ../app and replayed offline against the | |
| indexed tree of the benchmark paper. The only non-deterministic stage, the LLM tree search, is | |
| *replayed* from the node ids the live server logged (`[HYBRID-RETRIEVAL] llm_nodes=...`) during | |
| `run_eval.py run`. | |
| Metrics (per answerable question, against verbatim gold evidence quotes in blind_draft.json): | |
| evidence_recall share of gold quotes that appear in the final context handed to the generator | |
| full_hit_rate share of questions where *all* gold quotes are in the context | |
| page_recall share of gold pages present in the context's page markers | |
| ctx_chars context size (token-cost proxy); density = gold chars / ctx chars | |
| index_coverage share of gold quotes present in ANY node text (ceiling set by ingestion) | |
| Usage: uv run python eval/retrieval_eval.py [--refresh-tree] [--no-grid] | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import itertools | |
| import json | |
| import logging | |
| import re | |
| import statistics as st | |
| import sys | |
| import urllib.request | |
| from collections import defaultdict | |
| from pathlib import Path | |
| HERE = Path(__file__).resolve().parent | |
| ROOT = HERE.parent | |
| OUT = HERE / "results" | |
| OUT.mkdir(exist_ok=True) | |
| # keep the app's logger from attaching a file handler (it would append to ../app.log) | |
| logging.getLogger("VectorlessRAG").addHandler(logging.NullHandler()) | |
| sys.path.insert(0, str(ROOT / "app")) | |
| from core.bm25 import BM25NodeIndex # noqa: E402 | |
| from core.intent_classifier import intent_classifier # noqa: E402 | |
| from core.rag_pipeline import VectorlessRAG # noqa: E402 | |
| logging.disable(logging.CRITICAL) | |
| BASE = "http://localhost:7860" | |
| CLIENT = "rag-eval-blind" | |
| DOC_NAME = "ieee_roadmap.pdf" | |
| MARK = re.compile(r"---\s*\[PAGE\s+(\d+)\]\s*---", re.I) | |
| # Parameters of the pipeline as shipped (rag_pipeline.py / bm25.py) | |
| BASELINE = dict(k1=1.5, b=0.75, text_chars=3000, tw=3, sw=2, bm_k=5, | |
| w_llm=0.6, top_k=3, top_p=0.75, min_p=0.18) | |
| # ----------------------------------------------------------------------------- data | |
| def load_tree(refresh: bool) -> list: | |
| p = OUT / "tree.json" | |
| if refresh or not p.exists(): | |
| doc_id = json.loads((OUT / "doc.json").read_text())["doc_id"] | |
| req = urllib.request.Request(f"{BASE}/documents/{doc_id}/tree", headers={"X-Client-ID": CLIENT}) | |
| p.write_text(urllib.request.urlopen(req, timeout=30).read().decode()) | |
| return json.loads(p.read_text())["tree"] | |
| def flatten(tree: list) -> list[dict]: | |
| out: list[dict] = [] | |
| def walk(ns): | |
| for n in ns: | |
| if isinstance(n, dict): | |
| if n.get("node_id"): | |
| out.append(n) | |
| walk(n.get("nodes") or []) | |
| walk(tree) | |
| return out | |
| def parse_llm_nodes(bench: list[dict]) -> dict[str, list[str]]: | |
| """Map benchmark id -> node ids returned by the live LLM tree search (from app.log).""" | |
| log = (ROOT / "app.log").read_text(errors="ignore").splitlines() | |
| qi = re.compile(r"\[QUERY-INTENT\] query='(.*?)' intent=") | |
| hy = re.compile(r"\[HYBRID-RETRIEVAL\] doc='%s' llm_nodes=\[(.*?)\] bm25_nodes=\[(.*?)\] fused=\[(.*?)\]" % re.escape(DOC_NAME)) | |
| pairs: dict[str, tuple[list, list, list]] = {} | |
| pending = None | |
| for line in log: | |
| m = qi.search(line) | |
| if m: | |
| pending = m.group(1) | |
| continue | |
| m = hy.search(line) | |
| if m and pending is not None: | |
| ids = lambda s: re.findall(r"'(\w+)'", s) # noqa: E731 | |
| pairs[pending] = (ids(m.group(1)), ids(m.group(2)), ids(m.group(3))) # last occurrence wins | |
| pending = None | |
| res, logged = {}, {} | |
| for b in bench: | |
| key = b["question"][:50] | |
| if key in pairs: | |
| res[b["id"]] = pairs[key][0] | |
| logged[b["id"]] = pairs[key] | |
| return res, logged | |
| # ----------------------------------------------------------------------- text match | |
| def norm(s: str) -> str: | |
| return " ".join(re.findall(r"[a-z0-9]+", s.lower())) | |
| def quote_in(quote: str, ctx_norm: str) -> bool: | |
| q = norm(quote) | |
| if q in ctx_norm: | |
| return True | |
| toks = q.split() | |
| grams = [" ".join(toks[i:i + 4]) for i in range(max(1, len(toks) - 3))] | |
| return sum(g in ctx_norm for g in grams) / len(grams) >= 0.9 | |
| # ------------------------------------------------------------------------- pipeline | |
| class TunableBM25(BM25NodeIndex): | |
| """Same Okapi BM25 as the app but with the field boosts / text window exposed.""" | |
| def __init__(self, nodes, k1, b, text_chars, tw, sw): | |
| self._cfg = (text_chars, tw, sw) | |
| super().__init__(nodes, k1=k1, b=b) | |
| def _build_index(self, nodes): | |
| from collections import Counter | |
| text_chars, tw, sw = self._cfg | |
| self.node_ids, self.tokenized_corpus, self.doc_lens, self.doc_freqs = [], [], [], Counter() | |
| for node in nodes: | |
| nid = node.get("node_id") | |
| if not nid: | |
| continue | |
| text = node.get("text") or node.get("content") or "" | |
| text = text if text_chars is None else text[:text_chars] | |
| toks = (self.tokenize(node.get("title") or "") * tw + self.tokenize(node.get("summary") or "") * sw | |
| + self.tokenize(text)) or ["document"] | |
| self.node_ids.append(nid) | |
| self.tokenized_corpus.append(toks) | |
| self.doc_lens.append(len(toks)) | |
| self.doc_freqs.update(set(toks)) | |
| self.N = len(self.node_ids) | |
| self.avgdl = sum(self.doc_lens) / self.N if self.N else 1.0 | |
| class Sim: | |
| def __init__(self, tree: list): | |
| self.tree = tree | |
| self.nodes = flatten(tree) | |
| self.map = {n["node_id"]: n for n in self.nodes} | |
| self._idx: dict[tuple, TunableBM25] = {} | |
| def bm25(self, query: str, k1, b, text_chars, tw, sw, bm_k): | |
| key = (k1, b, text_chars, tw, sw) | |
| if key not in self._idx: | |
| self._idx[key] = TunableBM25(self.nodes, k1, b, text_chars, tw, sw) | |
| return [nid for nid, sc in self._idx[key].search(query, top_k=bm_k) if sc > 0] | |
| def context(self, query: str, llm_ids: list[str], cfg: dict, prune: bool = True) -> list[dict]: | |
| intent = intent_classifier.classify(query, doc_count=1) | |
| is_global = intent["is_global_or_summary"] | |
| ql = query.lower() | |
| multi = ("multi" in ql or "between" in ql or "connect" in ql | |
| or ("and" in ql and ("relat" in ql or "compar" in ql or "synthes" in ql))) | |
| bm = self.bm25(query, cfg["k1"], cfg["b"], cfg["text_chars"], cfg["tw"], cfg["sw"], cfg["bm_k"]) | |
| w = cfg["w_llm"] | |
| fused = VectorlessRAG.reciprocal_rank_fusion( | |
| ranked_lists=[(llm_ids, w), (bm, 1.0 - w)], top_k=cfg["top_k"], | |
| top_p=cfg["top_p"], min_p=cfg["min_p"], is_multi_hop=multi or is_global) | |
| nodes = [dict(self.map[nid]) for nid, _ in fused if nid in self.map] | |
| injected: list[dict] = [] | |
| if is_global: | |
| d = VectorlessRAG._create_fallback_dossier_from_tree(self.tree, DOC_NAME) | |
| injected.append(VectorlessRAG._create_dossier_node(d, DOC_NAME, "doc")) | |
| injected += VectorlessRAG._extract_root_chapter_nodes(self.tree, DOC_NAME, "doc") | |
| ids = {n["node_id"] for n in injected} | |
| allnodes = injected + [n for n in nodes if n["node_id"] not in ids] | |
| return VectorlessRAG.prune_sub_nodes(query, allnodes, deep_analysis=False) if prune else allnodes | |
| def score_item(item: dict, ctx_nodes: list[dict]) -> dict: | |
| text = "\n".join((n.get("text") or n.get("content") or "") for n in ctx_nodes) | |
| cn = norm(text) | |
| pages = set() | |
| for n in ctx_nodes: | |
| pages |= {int(p) for p in (n.get("active_pages") or []) if str(p).isdigit()} | |
| pages |= {int(x) for x in MARK.findall(n.get("text") or "")} | |
| found = [quote_in(e["quote"], cn) for e in item["evidence"]] | |
| gt = set(item["ground_truth_pages"]) | |
| gold_chars = sum(len(e["quote"]) for e in item["evidence"]) | |
| return { | |
| "evidence_recall": sum(found) / len(found), | |
| "full_hit": all(found), | |
| "page_recall": len(gt & pages) / len(gt), | |
| "ctx_chars": len(text), | |
| "density": gold_chars / max(len(text), 1), | |
| "nodes": [n["node_id"] for n in ctx_nodes if not n.get("is_dossier")], | |
| } | |
| def evaluate(sim: Sim, items, llm, cfg, prune=True) -> list[dict]: | |
| return [dict(id=it["id"], category=it["category"], **score_item( | |
| it, sim.context(it["question"], llm.get(it["id"], []), cfg, prune))) for it in items] | |
| def summarize(rows: list[dict]) -> dict: | |
| m = lambda k: round(st.mean(r[k] for r in rows), 4) # noqa: E731 | |
| return {"n": len(rows), "evidence_recall": m("evidence_recall"), "full_hit_rate": m("full_hit"), | |
| "page_recall": m("page_recall"), "ctx_chars": round(st.mean(r["ctx_chars"] for r in rows)), | |
| "density": m("density")} | |
| # ------------------------------------------------------------------------------ main | |
| def main() -> None: | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--refresh-tree", action="store_true") | |
| ap.add_argument("--no-grid", action="store_true") | |
| args = ap.parse_args() | |
| bench = json.loads((HERE / "data" / "blind_draft.json").read_text()) | |
| items = [b for b in bench if b["answerable"]] | |
| sim = Sim(load_tree(args.refresh_tree)) | |
| llm, logged = parse_llm_nodes(bench) | |
| items = [b for b in items if b["id"] in llm] | |
| print(f"replaying {len(items)} answerable questions with logged LLM tree-search results") | |
| # ---- harness validity: do offline BM25 / fusion reproduce the live server's logged nodes? | |
| agree_bm = agree_fused = 0 | |
| for b in items: | |
| l_ids, l_bm, l_fused = logged[b["id"]] | |
| bm = sim.bm25(b["question"], 1.5, 0.75, 3000, 3, 2, 5) | |
| fused = [n for n, _ in VectorlessRAG.reciprocal_rank_fusion( | |
| ranked_lists=[(l_ids, 0.6), (bm, 0.4)], top_k=3, top_p=0.75, min_p=0.18, is_multi_hop=False)] | |
| agree_bm += bm == l_bm | |
| agree_fused += fused == l_fused or set(fused) == set(l_fused) | |
| print(f"harness check: BM25 identical to live log on {agree_bm}/{len(items)}; " | |
| f"fused set identical on {agree_fused}/{len(items)} (multi-hop flag may differ)") | |
| # ---- tree-search health & ingestion ceiling | |
| empty = sum(1 for b in items if not llm[b["id"]]) | |
| all_text = norm(" ".join((n.get("text") or "") for n in sim.nodes)) | |
| cov = [quote_in(e["quote"], all_text) for b in items for e in b["evidence"]] | |
| print(f"LLM tree search returned NO nodes for {empty}/{len(items)} questions") | |
| print(f"index coverage (gold quote present in some node text): {sum(cov)}/{len(cov)} = {sum(cov)/len(cov):.1%}") | |
| # ---- baseline + ablations | |
| report: dict = {"harness": {"bm25_agree": agree_bm, "fused_agree": agree_fused, "n": len(items)}, | |
| "llm_tree_search_empty": empty, "index_coverage": sum(cov) / len(cov)} | |
| variants = { | |
| "baseline (shipped)": BASELINE, | |
| "BM25 only": {**BASELINE, "w_llm": 0.0}, | |
| "LLM tree search only": {**BASELINE, "w_llm": 1.0}, | |
| } | |
| print("\n== baseline & ablations ==") | |
| for name, cfg in variants.items(): | |
| rows = evaluate(sim, items, llm, cfg) | |
| report[name] = summarize(rows) | |
| print(f"{name:<24}", report[name]) | |
| pre = summarize(evaluate(sim, items, llm, BASELINE, prune=False)) | |
| print(f"{'baseline, NO pruning':<24}", pre, " <- ceiling lost to paragraph pruning") | |
| report["baseline_no_pruning"] = pre | |
| base_rows = evaluate(sim, items, llm, BASELINE) | |
| by_cat = defaultdict(list) | |
| for r in base_rows: | |
| by_cat[r["category"]].append(r) | |
| print("\n== baseline by category ==") | |
| report["baseline_by_category"] = {} | |
| for c, rs in sorted(by_cat.items()): | |
| report["baseline_by_category"][c] = summarize(rs) | |
| print(f"{c:<10}", report["baseline_by_category"][c]) | |
| misses = [r for r in base_rows if r["evidence_recall"] < 1] | |
| print(f"\nquestions with incomplete evidence: {[ (r['id'], round(r['evidence_recall'],2)) for r in misses]}") | |
| report["baseline_misses"] = [(r["id"], r["evidence_recall"], r["nodes"]) for r in misses] | |
| if not args.no_grid: | |
| grid = dict( | |
| k1=[1.2, 1.5], b=[0.5, 0.75], text_chars=[3000, 6000, None], tw=[3, 1], sw=[2], | |
| bm_k=[5, 8], w_llm=[0.0, 0.4, 0.6, 0.8], top_k=[3, 4, 5, 6], top_p=[0.75, 0.9, None], min_p=[0.18]) | |
| keys = list(grid) | |
| combos = list(itertools.product(*[grid[k] for k in keys])) | |
| print(f"\n== grid search over {len(combos)} configs (train = even positions, validate = odd positions) ==") | |
| train = [b for i, b in enumerate(items) if i % 2 == 0] | |
| valid = [b for i, b in enumerate(items) if i % 2 == 1] | |
| scored = [] | |
| for vals in combos: | |
| cfg = dict(zip(keys, vals)) | |
| s = summarize(evaluate(sim, train, llm, cfg)) | |
| scored.append((s["evidence_recall"], -s["ctx_chars"], cfg, s)) | |
| scored.sort(key=lambda x: (x[0], x[1]), reverse=True) | |
| top = scored[:8] | |
| base_tr = summarize(evaluate(sim, train, llm, BASELINE)) | |
| base_va = summarize(evaluate(sim, valid, llm, BASELINE)) | |
| print("baseline train", base_tr, "\n valid", base_va) | |
| report["grid"] = [] | |
| for rank, (_, _, cfg, s) in enumerate(top, 1): | |
| v = summarize(evaluate(sim, valid, llm, cfg)) | |
| diff = {k: cfg[k] for k in keys if cfg[k] != BASELINE[k]} | |
| print(f"#{rank} train={s['evidence_recall']:.3f}/{s['ctx_chars']}ch valid={v['evidence_recall']:.3f}/{v['ctx_chars']}ch changes={diff}") | |
| report["grid"].append({"changes": diff, "train": s, "valid": v}) | |
| report["baseline_train"], report["baseline_valid"] = base_tr, base_va | |
| # one-factor-at-a-time sensitivity on ALL questions (more interpretable than the joint grid) | |
| print("\n== one-factor-at-a-time (all questions) ==") | |
| report["oat"] = {} | |
| for k in keys: | |
| line = [] | |
| for v in grid[k]: | |
| s = summarize(evaluate(sim, items, llm, {**BASELINE, k: v})) | |
| line.append(f"{k}={v}: R={s['evidence_recall']:.3f} hit={s['full_hit_rate']:.2f} ch={s['ctx_chars']}") | |
| report["oat"][f"{k}={v}"] = s | |
| print(" | ".join(line)) | |
| (OUT / "retrieval_report.json").write_text(json.dumps(report, indent=2, default=str)) | |
| print(f"\nwrote {OUT / 'retrieval_report.json'}") | |
| if __name__ == "__main__": | |
| main() | |