vectorless_RAG / eval /retrieval_eval.py
amirelrefai's picture
refactor: clean up evaluation suite and consolidate files under eval directory
29dae30
Raw History Blame Contribute Delete
14.7 kB
#!/usr/bin/env python3
"""Judge-free retrieval evaluation + offline parameter search for the Vectorless RAG pipeline.
No LLM is called. The pipeline's deterministic stages (BM25, weighted RRF + nucleus cutoff,
paragraph pruning, intent routing) are imported from ../app and replayed offline against the
indexed tree of the benchmark paper. The only non-deterministic stage, the LLM tree search, is
*replayed* from the node ids the live server logged (`[HYBRID-RETRIEVAL] llm_nodes=...`) during
`run_eval.py run`.
Metrics (per answerable question, against verbatim gold evidence quotes in blind_draft.json):
evidence_recall share of gold quotes that appear in the final context handed to the generator
full_hit_rate share of questions where *all* gold quotes are in the context
page_recall share of gold pages present in the context's page markers
ctx_chars context size (token-cost proxy); density = gold chars / ctx chars
index_coverage share of gold quotes present in ANY node text (ceiling set by ingestion)
Usage: uv run python eval/retrieval_eval.py [--refresh-tree] [--no-grid]
"""
from __future__ import annotations
import argparse
import itertools
import json
import logging
import re
import statistics as st
import sys
import urllib.request
from collections import defaultdict
from pathlib import Path
HERE = Path(__file__).resolve().parent
ROOT = HERE.parent
OUT = HERE / "results"
OUT.mkdir(exist_ok=True)
# keep the app's logger from attaching a file handler (it would append to ../app.log)
logging.getLogger("VectorlessRAG").addHandler(logging.NullHandler())
sys.path.insert(0, str(ROOT / "app"))
from core.bm25 import BM25NodeIndex # noqa: E402
from core.intent_classifier import intent_classifier # noqa: E402
from core.rag_pipeline import VectorlessRAG # noqa: E402
logging.disable(logging.CRITICAL)
BASE = "http://localhost:7860"
CLIENT = "rag-eval-blind"
DOC_NAME = "ieee_roadmap.pdf"
MARK = re.compile(r"---\s*\[PAGE\s+(\d+)\]\s*---", re.I)
# Parameters of the pipeline as shipped (rag_pipeline.py / bm25.py)
BASELINE = dict(k1=1.5, b=0.75, text_chars=3000, tw=3, sw=2, bm_k=5,
w_llm=0.6, top_k=3, top_p=0.75, min_p=0.18)
# ----------------------------------------------------------------------------- data
def load_tree(refresh: bool) -> list:
p = OUT / "tree.json"
if refresh or not p.exists():
doc_id = json.loads((OUT / "doc.json").read_text())["doc_id"]
req = urllib.request.Request(f"{BASE}/documents/{doc_id}/tree", headers={"X-Client-ID": CLIENT})
p.write_text(urllib.request.urlopen(req, timeout=30).read().decode())
return json.loads(p.read_text())["tree"]
def flatten(tree: list) -> list[dict]:
out: list[dict] = []
def walk(ns):
for n in ns:
if isinstance(n, dict):
if n.get("node_id"):
out.append(n)
walk(n.get("nodes") or [])
walk(tree)
return out
def parse_llm_nodes(bench: list[dict]) -> dict[str, list[str]]:
"""Map benchmark id -> node ids returned by the live LLM tree search (from app.log)."""
log = (ROOT / "app.log").read_text(errors="ignore").splitlines()
qi = re.compile(r"\[QUERY-INTENT\] query='(.*?)' intent=")
hy = re.compile(r"\[HYBRID-RETRIEVAL\] doc='%s' llm_nodes=\[(.*?)\] bm25_nodes=\[(.*?)\] fused=\[(.*?)\]" % re.escape(DOC_NAME))
pairs: dict[str, tuple[list, list, list]] = {}
pending = None
for line in log:
m = qi.search(line)
if m:
pending = m.group(1)
continue
m = hy.search(line)
if m and pending is not None:
ids = lambda s: re.findall(r"'(\w+)'", s) # noqa: E731
pairs[pending] = (ids(m.group(1)), ids(m.group(2)), ids(m.group(3))) # last occurrence wins
pending = None
res, logged = {}, {}
for b in bench:
key = b["question"][:50]
if key in pairs:
res[b["id"]] = pairs[key][0]
logged[b["id"]] = pairs[key]
return res, logged
# ----------------------------------------------------------------------- text match
def norm(s: str) -> str:
return " ".join(re.findall(r"[a-z0-9]+", s.lower()))
def quote_in(quote: str, ctx_norm: str) -> bool:
q = norm(quote)
if q in ctx_norm:
return True
toks = q.split()
grams = [" ".join(toks[i:i + 4]) for i in range(max(1, len(toks) - 3))]
return sum(g in ctx_norm for g in grams) / len(grams) >= 0.9
# ------------------------------------------------------------------------- pipeline
class TunableBM25(BM25NodeIndex):
"""Same Okapi BM25 as the app but with the field boosts / text window exposed."""
def __init__(self, nodes, k1, b, text_chars, tw, sw):
self._cfg = (text_chars, tw, sw)
super().__init__(nodes, k1=k1, b=b)
def _build_index(self, nodes):
from collections import Counter
text_chars, tw, sw = self._cfg
self.node_ids, self.tokenized_corpus, self.doc_lens, self.doc_freqs = [], [], [], Counter()
for node in nodes:
nid = node.get("node_id")
if not nid:
continue
text = node.get("text") or node.get("content") or ""
text = text if text_chars is None else text[:text_chars]
toks = (self.tokenize(node.get("title") or "") * tw + self.tokenize(node.get("summary") or "") * sw
+ self.tokenize(text)) or ["document"]
self.node_ids.append(nid)
self.tokenized_corpus.append(toks)
self.doc_lens.append(len(toks))
self.doc_freqs.update(set(toks))
self.N = len(self.node_ids)
self.avgdl = sum(self.doc_lens) / self.N if self.N else 1.0
class Sim:
def __init__(self, tree: list):
self.tree = tree
self.nodes = flatten(tree)
self.map = {n["node_id"]: n for n in self.nodes}
self._idx: dict[tuple, TunableBM25] = {}
def bm25(self, query: str, k1, b, text_chars, tw, sw, bm_k):
key = (k1, b, text_chars, tw, sw)
if key not in self._idx:
self._idx[key] = TunableBM25(self.nodes, k1, b, text_chars, tw, sw)
return [nid for nid, sc in self._idx[key].search(query, top_k=bm_k) if sc > 0]
def context(self, query: str, llm_ids: list[str], cfg: dict, prune: bool = True) -> list[dict]:
intent = intent_classifier.classify(query, doc_count=1)
is_global = intent["is_global_or_summary"]
ql = query.lower()
multi = ("multi" in ql or "between" in ql or "connect" in ql
or ("and" in ql and ("relat" in ql or "compar" in ql or "synthes" in ql)))
bm = self.bm25(query, cfg["k1"], cfg["b"], cfg["text_chars"], cfg["tw"], cfg["sw"], cfg["bm_k"])
w = cfg["w_llm"]
fused = VectorlessRAG.reciprocal_rank_fusion(
ranked_lists=[(llm_ids, w), (bm, 1.0 - w)], top_k=cfg["top_k"],
top_p=cfg["top_p"], min_p=cfg["min_p"], is_multi_hop=multi or is_global)
nodes = [dict(self.map[nid]) for nid, _ in fused if nid in self.map]
injected: list[dict] = []
if is_global:
d = VectorlessRAG._create_fallback_dossier_from_tree(self.tree, DOC_NAME)
injected.append(VectorlessRAG._create_dossier_node(d, DOC_NAME, "doc"))
injected += VectorlessRAG._extract_root_chapter_nodes(self.tree, DOC_NAME, "doc")
ids = {n["node_id"] for n in injected}
allnodes = injected + [n for n in nodes if n["node_id"] not in ids]
return VectorlessRAG.prune_sub_nodes(query, allnodes, deep_analysis=False) if prune else allnodes
def score_item(item: dict, ctx_nodes: list[dict]) -> dict:
text = "\n".join((n.get("text") or n.get("content") or "") for n in ctx_nodes)
cn = norm(text)
pages = set()
for n in ctx_nodes:
pages |= {int(p) for p in (n.get("active_pages") or []) if str(p).isdigit()}
pages |= {int(x) for x in MARK.findall(n.get("text") or "")}
found = [quote_in(e["quote"], cn) for e in item["evidence"]]
gt = set(item["ground_truth_pages"])
gold_chars = sum(len(e["quote"]) for e in item["evidence"])
return {
"evidence_recall": sum(found) / len(found),
"full_hit": all(found),
"page_recall": len(gt & pages) / len(gt),
"ctx_chars": len(text),
"density": gold_chars / max(len(text), 1),
"nodes": [n["node_id"] for n in ctx_nodes if not n.get("is_dossier")],
}
def evaluate(sim: Sim, items, llm, cfg, prune=True) -> list[dict]:
return [dict(id=it["id"], category=it["category"], **score_item(
it, sim.context(it["question"], llm.get(it["id"], []), cfg, prune))) for it in items]
def summarize(rows: list[dict]) -> dict:
m = lambda k: round(st.mean(r[k] for r in rows), 4) # noqa: E731
return {"n": len(rows), "evidence_recall": m("evidence_recall"), "full_hit_rate": m("full_hit"),
"page_recall": m("page_recall"), "ctx_chars": round(st.mean(r["ctx_chars"] for r in rows)),
"density": m("density")}
# ------------------------------------------------------------------------------ main
def main() -> None:
ap = argparse.ArgumentParser()
ap.add_argument("--refresh-tree", action="store_true")
ap.add_argument("--no-grid", action="store_true")
args = ap.parse_args()
bench = json.loads((HERE / "data" / "blind_draft.json").read_text())
items = [b for b in bench if b["answerable"]]
sim = Sim(load_tree(args.refresh_tree))
llm, logged = parse_llm_nodes(bench)
items = [b for b in items if b["id"] in llm]
print(f"replaying {len(items)} answerable questions with logged LLM tree-search results")
# ---- harness validity: do offline BM25 / fusion reproduce the live server's logged nodes?
agree_bm = agree_fused = 0
for b in items:
l_ids, l_bm, l_fused = logged[b["id"]]
bm = sim.bm25(b["question"], 1.5, 0.75, 3000, 3, 2, 5)
fused = [n for n, _ in VectorlessRAG.reciprocal_rank_fusion(
ranked_lists=[(l_ids, 0.6), (bm, 0.4)], top_k=3, top_p=0.75, min_p=0.18, is_multi_hop=False)]
agree_bm += bm == l_bm
agree_fused += fused == l_fused or set(fused) == set(l_fused)
print(f"harness check: BM25 identical to live log on {agree_bm}/{len(items)}; "
f"fused set identical on {agree_fused}/{len(items)} (multi-hop flag may differ)")
# ---- tree-search health & ingestion ceiling
empty = sum(1 for b in items if not llm[b["id"]])
all_text = norm(" ".join((n.get("text") or "") for n in sim.nodes))
cov = [quote_in(e["quote"], all_text) for b in items for e in b["evidence"]]
print(f"LLM tree search returned NO nodes for {empty}/{len(items)} questions")
print(f"index coverage (gold quote present in some node text): {sum(cov)}/{len(cov)} = {sum(cov)/len(cov):.1%}")
# ---- baseline + ablations
report: dict = {"harness": {"bm25_agree": agree_bm, "fused_agree": agree_fused, "n": len(items)},
"llm_tree_search_empty": empty, "index_coverage": sum(cov) / len(cov)}
variants = {
"baseline (shipped)": BASELINE,
"BM25 only": {**BASELINE, "w_llm": 0.0},
"LLM tree search only": {**BASELINE, "w_llm": 1.0},
}
print("\n== baseline & ablations ==")
for name, cfg in variants.items():
rows = evaluate(sim, items, llm, cfg)
report[name] = summarize(rows)
print(f"{name:<24}", report[name])
pre = summarize(evaluate(sim, items, llm, BASELINE, prune=False))
print(f"{'baseline, NO pruning':<24}", pre, " <- ceiling lost to paragraph pruning")
report["baseline_no_pruning"] = pre
base_rows = evaluate(sim, items, llm, BASELINE)
by_cat = defaultdict(list)
for r in base_rows:
by_cat[r["category"]].append(r)
print("\n== baseline by category ==")
report["baseline_by_category"] = {}
for c, rs in sorted(by_cat.items()):
report["baseline_by_category"][c] = summarize(rs)
print(f"{c:<10}", report["baseline_by_category"][c])
misses = [r for r in base_rows if r["evidence_recall"] < 1]
print(f"\nquestions with incomplete evidence: {[ (r['id'], round(r['evidence_recall'],2)) for r in misses]}")
report["baseline_misses"] = [(r["id"], r["evidence_recall"], r["nodes"]) for r in misses]
if not args.no_grid:
grid = dict(
k1=[1.2, 1.5], b=[0.5, 0.75], text_chars=[3000, 6000, None], tw=[3, 1], sw=[2],
bm_k=[5, 8], w_llm=[0.0, 0.4, 0.6, 0.8], top_k=[3, 4, 5, 6], top_p=[0.75, 0.9, None], min_p=[0.18])
keys = list(grid)
combos = list(itertools.product(*[grid[k] for k in keys]))
print(f"\n== grid search over {len(combos)} configs (train = even positions, validate = odd positions) ==")
train = [b for i, b in enumerate(items) if i % 2 == 0]
valid = [b for i, b in enumerate(items) if i % 2 == 1]
scored = []
for vals in combos:
cfg = dict(zip(keys, vals))
s = summarize(evaluate(sim, train, llm, cfg))
scored.append((s["evidence_recall"], -s["ctx_chars"], cfg, s))
scored.sort(key=lambda x: (x[0], x[1]), reverse=True)
top = scored[:8]
base_tr = summarize(evaluate(sim, train, llm, BASELINE))
base_va = summarize(evaluate(sim, valid, llm, BASELINE))
print("baseline train", base_tr, "\n valid", base_va)
report["grid"] = []
for rank, (_, _, cfg, s) in enumerate(top, 1):
v = summarize(evaluate(sim, valid, llm, cfg))
diff = {k: cfg[k] for k in keys if cfg[k] != BASELINE[k]}
print(f"#{rank} train={s['evidence_recall']:.3f}/{s['ctx_chars']}ch valid={v['evidence_recall']:.3f}/{v['ctx_chars']}ch changes={diff}")
report["grid"].append({"changes": diff, "train": s, "valid": v})
report["baseline_train"], report["baseline_valid"] = base_tr, base_va
# one-factor-at-a-time sensitivity on ALL questions (more interpretable than the joint grid)
print("\n== one-factor-at-a-time (all questions) ==")
report["oat"] = {}
for k in keys:
line = []
for v in grid[k]:
s = summarize(evaluate(sim, items, llm, {**BASELINE, k: v}))
line.append(f"{k}={v}: R={s['evidence_recall']:.3f} hit={s['full_hit_rate']:.2f} ch={s['ctx_chars']}")
report["oat"][f"{k}={v}"] = s
print(" | ".join(line))
(OUT / "retrieval_report.json").write_text(json.dumps(report, indent=2, default=str))
print(f"\nwrote {OUT / 'retrieval_report.json'}")
if __name__ == "__main__":
main()