Spaces:
Sleeping
Sleeping
File size: 14,709 Bytes
29dae30 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 | #!/usr/bin/env python3
"""Judge-free retrieval evaluation + offline parameter search for the Vectorless RAG pipeline.
No LLM is called. The pipeline's deterministic stages (BM25, weighted RRF + nucleus cutoff,
paragraph pruning, intent routing) are imported from ../app and replayed offline against the
indexed tree of the benchmark paper. The only non-deterministic stage, the LLM tree search, is
*replayed* from the node ids the live server logged (`[HYBRID-RETRIEVAL] llm_nodes=...`) during
`run_eval.py run`.
Metrics (per answerable question, against verbatim gold evidence quotes in blind_draft.json):
evidence_recall share of gold quotes that appear in the final context handed to the generator
full_hit_rate share of questions where *all* gold quotes are in the context
page_recall share of gold pages present in the context's page markers
ctx_chars context size (token-cost proxy); density = gold chars / ctx chars
index_coverage share of gold quotes present in ANY node text (ceiling set by ingestion)
Usage: uv run python eval/retrieval_eval.py [--refresh-tree] [--no-grid]
"""
from __future__ import annotations
import argparse
import itertools
import json
import logging
import re
import statistics as st
import sys
import urllib.request
from collections import defaultdict
from pathlib import Path
HERE = Path(__file__).resolve().parent
ROOT = HERE.parent
OUT = HERE / "results"
OUT.mkdir(exist_ok=True)
# keep the app's logger from attaching a file handler (it would append to ../app.log)
logging.getLogger("VectorlessRAG").addHandler(logging.NullHandler())
sys.path.insert(0, str(ROOT / "app"))
from core.bm25 import BM25NodeIndex # noqa: E402
from core.intent_classifier import intent_classifier # noqa: E402
from core.rag_pipeline import VectorlessRAG # noqa: E402
logging.disable(logging.CRITICAL)
BASE = "http://localhost:7860"
CLIENT = "rag-eval-blind"
DOC_NAME = "ieee_roadmap.pdf"
MARK = re.compile(r"---\s*\[PAGE\s+(\d+)\]\s*---", re.I)
# Parameters of the pipeline as shipped (rag_pipeline.py / bm25.py)
BASELINE = dict(k1=1.5, b=0.75, text_chars=3000, tw=3, sw=2, bm_k=5,
w_llm=0.6, top_k=3, top_p=0.75, min_p=0.18)
# ----------------------------------------------------------------------------- data
def load_tree(refresh: bool) -> list:
p = OUT / "tree.json"
if refresh or not p.exists():
doc_id = json.loads((OUT / "doc.json").read_text())["doc_id"]
req = urllib.request.Request(f"{BASE}/documents/{doc_id}/tree", headers={"X-Client-ID": CLIENT})
p.write_text(urllib.request.urlopen(req, timeout=30).read().decode())
return json.loads(p.read_text())["tree"]
def flatten(tree: list) -> list[dict]:
out: list[dict] = []
def walk(ns):
for n in ns:
if isinstance(n, dict):
if n.get("node_id"):
out.append(n)
walk(n.get("nodes") or [])
walk(tree)
return out
def parse_llm_nodes(bench: list[dict]) -> dict[str, list[str]]:
"""Map benchmark id -> node ids returned by the live LLM tree search (from app.log)."""
log = (ROOT / "app.log").read_text(errors="ignore").splitlines()
qi = re.compile(r"\[QUERY-INTENT\] query='(.*?)' intent=")
hy = re.compile(r"\[HYBRID-RETRIEVAL\] doc='%s' llm_nodes=\[(.*?)\] bm25_nodes=\[(.*?)\] fused=\[(.*?)\]" % re.escape(DOC_NAME))
pairs: dict[str, tuple[list, list, list]] = {}
pending = None
for line in log:
m = qi.search(line)
if m:
pending = m.group(1)
continue
m = hy.search(line)
if m and pending is not None:
ids = lambda s: re.findall(r"'(\w+)'", s) # noqa: E731
pairs[pending] = (ids(m.group(1)), ids(m.group(2)), ids(m.group(3))) # last occurrence wins
pending = None
res, logged = {}, {}
for b in bench:
key = b["question"][:50]
if key in pairs:
res[b["id"]] = pairs[key][0]
logged[b["id"]] = pairs[key]
return res, logged
# ----------------------------------------------------------------------- text match
def norm(s: str) -> str:
return " ".join(re.findall(r"[a-z0-9]+", s.lower()))
def quote_in(quote: str, ctx_norm: str) -> bool:
q = norm(quote)
if q in ctx_norm:
return True
toks = q.split()
grams = [" ".join(toks[i:i + 4]) for i in range(max(1, len(toks) - 3))]
return sum(g in ctx_norm for g in grams) / len(grams) >= 0.9
# ------------------------------------------------------------------------- pipeline
class TunableBM25(BM25NodeIndex):
"""Same Okapi BM25 as the app but with the field boosts / text window exposed."""
def __init__(self, nodes, k1, b, text_chars, tw, sw):
self._cfg = (text_chars, tw, sw)
super().__init__(nodes, k1=k1, b=b)
def _build_index(self, nodes):
from collections import Counter
text_chars, tw, sw = self._cfg
self.node_ids, self.tokenized_corpus, self.doc_lens, self.doc_freqs = [], [], [], Counter()
for node in nodes:
nid = node.get("node_id")
if not nid:
continue
text = node.get("text") or node.get("content") or ""
text = text if text_chars is None else text[:text_chars]
toks = (self.tokenize(node.get("title") or "") * tw + self.tokenize(node.get("summary") or "") * sw
+ self.tokenize(text)) or ["document"]
self.node_ids.append(nid)
self.tokenized_corpus.append(toks)
self.doc_lens.append(len(toks))
self.doc_freqs.update(set(toks))
self.N = len(self.node_ids)
self.avgdl = sum(self.doc_lens) / self.N if self.N else 1.0
class Sim:
def __init__(self, tree: list):
self.tree = tree
self.nodes = flatten(tree)
self.map = {n["node_id"]: n for n in self.nodes}
self._idx: dict[tuple, TunableBM25] = {}
def bm25(self, query: str, k1, b, text_chars, tw, sw, bm_k):
key = (k1, b, text_chars, tw, sw)
if key not in self._idx:
self._idx[key] = TunableBM25(self.nodes, k1, b, text_chars, tw, sw)
return [nid for nid, sc in self._idx[key].search(query, top_k=bm_k) if sc > 0]
def context(self, query: str, llm_ids: list[str], cfg: dict, prune: bool = True) -> list[dict]:
intent = intent_classifier.classify(query, doc_count=1)
is_global = intent["is_global_or_summary"]
ql = query.lower()
multi = ("multi" in ql or "between" in ql or "connect" in ql
or ("and" in ql and ("relat" in ql or "compar" in ql or "synthes" in ql)))
bm = self.bm25(query, cfg["k1"], cfg["b"], cfg["text_chars"], cfg["tw"], cfg["sw"], cfg["bm_k"])
w = cfg["w_llm"]
fused = VectorlessRAG.reciprocal_rank_fusion(
ranked_lists=[(llm_ids, w), (bm, 1.0 - w)], top_k=cfg["top_k"],
top_p=cfg["top_p"], min_p=cfg["min_p"], is_multi_hop=multi or is_global)
nodes = [dict(self.map[nid]) for nid, _ in fused if nid in self.map]
injected: list[dict] = []
if is_global:
d = VectorlessRAG._create_fallback_dossier_from_tree(self.tree, DOC_NAME)
injected.append(VectorlessRAG._create_dossier_node(d, DOC_NAME, "doc"))
injected += VectorlessRAG._extract_root_chapter_nodes(self.tree, DOC_NAME, "doc")
ids = {n["node_id"] for n in injected}
allnodes = injected + [n for n in nodes if n["node_id"] not in ids]
return VectorlessRAG.prune_sub_nodes(query, allnodes, deep_analysis=False) if prune else allnodes
def score_item(item: dict, ctx_nodes: list[dict]) -> dict:
text = "\n".join((n.get("text") or n.get("content") or "") for n in ctx_nodes)
cn = norm(text)
pages = set()
for n in ctx_nodes:
pages |= {int(p) for p in (n.get("active_pages") or []) if str(p).isdigit()}
pages |= {int(x) for x in MARK.findall(n.get("text") or "")}
found = [quote_in(e["quote"], cn) for e in item["evidence"]]
gt = set(item["ground_truth_pages"])
gold_chars = sum(len(e["quote"]) for e in item["evidence"])
return {
"evidence_recall": sum(found) / len(found),
"full_hit": all(found),
"page_recall": len(gt & pages) / len(gt),
"ctx_chars": len(text),
"density": gold_chars / max(len(text), 1),
"nodes": [n["node_id"] for n in ctx_nodes if not n.get("is_dossier")],
}
def evaluate(sim: Sim, items, llm, cfg, prune=True) -> list[dict]:
return [dict(id=it["id"], category=it["category"], **score_item(
it, sim.context(it["question"], llm.get(it["id"], []), cfg, prune))) for it in items]
def summarize(rows: list[dict]) -> dict:
m = lambda k: round(st.mean(r[k] for r in rows), 4) # noqa: E731
return {"n": len(rows), "evidence_recall": m("evidence_recall"), "full_hit_rate": m("full_hit"),
"page_recall": m("page_recall"), "ctx_chars": round(st.mean(r["ctx_chars"] for r in rows)),
"density": m("density")}
# ------------------------------------------------------------------------------ main
def main() -> None:
ap = argparse.ArgumentParser()
ap.add_argument("--refresh-tree", action="store_true")
ap.add_argument("--no-grid", action="store_true")
args = ap.parse_args()
bench = json.loads((HERE / "data" / "blind_draft.json").read_text())
items = [b for b in bench if b["answerable"]]
sim = Sim(load_tree(args.refresh_tree))
llm, logged = parse_llm_nodes(bench)
items = [b for b in items if b["id"] in llm]
print(f"replaying {len(items)} answerable questions with logged LLM tree-search results")
# ---- harness validity: do offline BM25 / fusion reproduce the live server's logged nodes?
agree_bm = agree_fused = 0
for b in items:
l_ids, l_bm, l_fused = logged[b["id"]]
bm = sim.bm25(b["question"], 1.5, 0.75, 3000, 3, 2, 5)
fused = [n for n, _ in VectorlessRAG.reciprocal_rank_fusion(
ranked_lists=[(l_ids, 0.6), (bm, 0.4)], top_k=3, top_p=0.75, min_p=0.18, is_multi_hop=False)]
agree_bm += bm == l_bm
agree_fused += fused == l_fused or set(fused) == set(l_fused)
print(f"harness check: BM25 identical to live log on {agree_bm}/{len(items)}; "
f"fused set identical on {agree_fused}/{len(items)} (multi-hop flag may differ)")
# ---- tree-search health & ingestion ceiling
empty = sum(1 for b in items if not llm[b["id"]])
all_text = norm(" ".join((n.get("text") or "") for n in sim.nodes))
cov = [quote_in(e["quote"], all_text) for b in items for e in b["evidence"]]
print(f"LLM tree search returned NO nodes for {empty}/{len(items)} questions")
print(f"index coverage (gold quote present in some node text): {sum(cov)}/{len(cov)} = {sum(cov)/len(cov):.1%}")
# ---- baseline + ablations
report: dict = {"harness": {"bm25_agree": agree_bm, "fused_agree": agree_fused, "n": len(items)},
"llm_tree_search_empty": empty, "index_coverage": sum(cov) / len(cov)}
variants = {
"baseline (shipped)": BASELINE,
"BM25 only": {**BASELINE, "w_llm": 0.0},
"LLM tree search only": {**BASELINE, "w_llm": 1.0},
}
print("\n== baseline & ablations ==")
for name, cfg in variants.items():
rows = evaluate(sim, items, llm, cfg)
report[name] = summarize(rows)
print(f"{name:<24}", report[name])
pre = summarize(evaluate(sim, items, llm, BASELINE, prune=False))
print(f"{'baseline, NO pruning':<24}", pre, " <- ceiling lost to paragraph pruning")
report["baseline_no_pruning"] = pre
base_rows = evaluate(sim, items, llm, BASELINE)
by_cat = defaultdict(list)
for r in base_rows:
by_cat[r["category"]].append(r)
print("\n== baseline by category ==")
report["baseline_by_category"] = {}
for c, rs in sorted(by_cat.items()):
report["baseline_by_category"][c] = summarize(rs)
print(f"{c:<10}", report["baseline_by_category"][c])
misses = [r for r in base_rows if r["evidence_recall"] < 1]
print(f"\nquestions with incomplete evidence: {[ (r['id'], round(r['evidence_recall'],2)) for r in misses]}")
report["baseline_misses"] = [(r["id"], r["evidence_recall"], r["nodes"]) for r in misses]
if not args.no_grid:
grid = dict(
k1=[1.2, 1.5], b=[0.5, 0.75], text_chars=[3000, 6000, None], tw=[3, 1], sw=[2],
bm_k=[5, 8], w_llm=[0.0, 0.4, 0.6, 0.8], top_k=[3, 4, 5, 6], top_p=[0.75, 0.9, None], min_p=[0.18])
keys = list(grid)
combos = list(itertools.product(*[grid[k] for k in keys]))
print(f"\n== grid search over {len(combos)} configs (train = even positions, validate = odd positions) ==")
train = [b for i, b in enumerate(items) if i % 2 == 0]
valid = [b for i, b in enumerate(items) if i % 2 == 1]
scored = []
for vals in combos:
cfg = dict(zip(keys, vals))
s = summarize(evaluate(sim, train, llm, cfg))
scored.append((s["evidence_recall"], -s["ctx_chars"], cfg, s))
scored.sort(key=lambda x: (x[0], x[1]), reverse=True)
top = scored[:8]
base_tr = summarize(evaluate(sim, train, llm, BASELINE))
base_va = summarize(evaluate(sim, valid, llm, BASELINE))
print("baseline train", base_tr, "\n valid", base_va)
report["grid"] = []
for rank, (_, _, cfg, s) in enumerate(top, 1):
v = summarize(evaluate(sim, valid, llm, cfg))
diff = {k: cfg[k] for k in keys if cfg[k] != BASELINE[k]}
print(f"#{rank} train={s['evidence_recall']:.3f}/{s['ctx_chars']}ch valid={v['evidence_recall']:.3f}/{v['ctx_chars']}ch changes={diff}")
report["grid"].append({"changes": diff, "train": s, "valid": v})
report["baseline_train"], report["baseline_valid"] = base_tr, base_va
# one-factor-at-a-time sensitivity on ALL questions (more interpretable than the joint grid)
print("\n== one-factor-at-a-time (all questions) ==")
report["oat"] = {}
for k in keys:
line = []
for v in grid[k]:
s = summarize(evaluate(sim, items, llm, {**BASELINE, k: v}))
line.append(f"{k}={v}: R={s['evidence_recall']:.3f} hit={s['full_hit_rate']:.2f} ch={s['ctx_chars']}")
report["oat"][f"{k}={v}"] = s
print(" | ".join(line))
(OUT / "retrieval_report.json").write_text(json.dumps(report, indent=2, default=str))
print(f"\nwrote {OUT / 'retrieval_report.json'}")
if __name__ == "__main__":
main()
|