Download cbx_rag_eval.py from CobuildX-AI/cbx-rag-context-utilization-68M: direct link, hf CLI and curl.
- Browser
- Download file 16.8 kB
-
https://huggingface.co/CobuildX-AI/cbx-rag-context-utilization-68M/resolve/main/cbx_rag_eval.py
- Command line
-
hf download hf://CobuildX-AI/cbx-rag-context-utilization-68M/cbx_rag_eval.py
-
curl -L -o cbx_rag_eval.py https://huggingface.co/CobuildX-AI/cbx-rag-context-utilization-68M/resolve/main/cbx_rag_eval.py
16.8 kB
| """CBX RAG evaluator v0.1: four Ettin-68M judges for retrieval-augmented answers. | |
| from cbx_rag_eval import Faithfulness, ResponseRelevance, ContextRelevance, ContextUtilization | |
| judge = ContextRelevance(".") # folder with model.safetensors + cbx_rag_config.json | |
| out = judge(question, passages) # passages: str or list of str (or list of sentence lists) | |
| Every judge works on sentences. Raw text is split per profile (`split_sentences`, chosen on validation): "contract" | |
| splits on line breaks and then with NLTK punkt; "financial" splits on blank lines and then with pysbd. Pass pre-split | |
| sentences (a list of strings per passage, `answer_sentences=`) to skip the splitter. | |
| Profiles (settings fixed on validation, see the model card): | |
| "contract" long single documents (CUAD-like) | |
| "financial" several short passages, numbers and tables (FinQA-like); faithfulness adds a numeric check | |
| "auto" (default) "contract" for a single passage, "financial" for two or more | |
| """ | |
| import json | |
| import math | |
| import os | |
| import re | |
| from collections import Counter | |
| import numpy as np | |
| import torch | |
| import torch.nn as nn | |
| from safetensors.torch import load_file | |
| from transformers import AutoConfig, AutoModel, AutoTokenizer | |
| try: | |
| from cbx_rag_numcheck import Checker | |
| except ImportError: # development tree | |
| from numcheck import Checker | |
| VERSION = "v0.1" | |
| # ---------------------------------------------------------------- sentences | |
| _WS = re.compile(r"\s+") | |
| _LINE = re.compile(r"\n+") | |
| def _punkt(): | |
| import nltk | |
| try: | |
| nltk.data.find("tokenizers/punkt_tab") | |
| except LookupError: | |
| nltk.download("punkt_tab", quiet=True) | |
| from nltk.tokenize import sent_tokenize | |
| return sent_tokenize | |
| _PARA = re.compile(r"\n\s*\n") | |
| _SEG = None | |
| def _pysbd(): | |
| global _SEG | |
| if _SEG is None: | |
| import warnings | |
| with warnings.catch_warnings(): | |
| warnings.simplefilter("ignore") | |
| import pysbd | |
| _SEG = pysbd.Segmenter(language="en", clean=False) | |
| return _SEG | |
| def split_sentences(text, profile="contract"): | |
| """Whitespace-collapsed sentences. "contract": line breaks, then NLTK punkt inside each line. "financial": blank | |
| lines, then pysbd inside each paragraph. A JSON table ('[[' ...) stays one sentence.""" | |
| text = str(text) | |
| if text.lstrip().startswith("[["): | |
| return [text.strip()] | |
| if profile == "financial": | |
| parts = [x for p in _PARA.split(text) for x in _pysbd().segment(p)] | |
| else: | |
| tok = _punkt() | |
| parts = [x for line in _LINE.split(text) for x in tok(line)] | |
| out = [_WS.sub(" ", x).strip() for x in parts] | |
| return [x for x in out if x] | |
| def _letters(n): | |
| s = "" | |
| n += 1 | |
| while n: | |
| n, r = divmod(n - 1, 26) | |
| s = chr(97 + r) + s | |
| return s | |
| def _passages(passages): | |
| return [passages] if isinstance(passages, str) else list(passages) | |
| def _context(passages, profile): | |
| """passages: str | list of str | list of list of str -> list of {key, doc, text}.""" | |
| ctx = [] | |
| for di, p in enumerate(_passages(passages)): | |
| sents = list(p) if isinstance(p, (list, tuple)) else split_sentences(p, profile) | |
| ctx += [{"key": f"{di}{_letters(j)}", "doc": di, "text": str(s)} for j, s in enumerate(sents)] | |
| return ctx | |
| def _answer(answer, answer_sentences, profile): | |
| sents = list(answer_sentences) if answer_sentences is not None else split_sentences(answer, profile) | |
| return [{"key": _letters(j), "text": str(s)} for j, s in enumerate(sents)] | |
| def _profile(profile, passages): | |
| """"auto": one passage -> "contract", two or more -> "financial" (fixed on validation).""" | |
| if profile in ("contract", "financial"): | |
| return profile | |
| if profile not in (None, "auto"): | |
| raise ValueError("profile must be 'auto', 'contract' or 'financial'") | |
| return "contract" if len(_passages(passages)) <= 1 else "financial" | |
| # ---------------------------------------------------------------- BM25 (as in training) | |
| _TOKEN = re.compile(r"[a-z0-9]+") | |
| _STOP = set("the a an of to and or in on for by with is are be this that any as at from it its such shall may " | |
| "will which their there other than not no does do".split()) | |
| def _toks(text): | |
| return [t for t in _TOKEN.findall(text.lower()) if t not in _STOP] | |
| class BM25: | |
| def __init__(self, docs, k1=1.2, b=0.75): | |
| self.docs = [Counter(_toks(d)) for d in docs] | |
| self.len = [sum(d.values()) for d in self.docs] | |
| self.avg = (sum(self.len) / len(self.len)) if self.len else 1 | |
| df = Counter() | |
| for d in self.docs: | |
| df.update(d.keys()) | |
| n = len(docs) | |
| self.idf = {t: math.log(1 + (n - c + 0.5) / (c + 0.5)) for t, c in df.items()} | |
| self.k1, self.b = k1, b | |
| def scores(self, query): | |
| q = Counter(_toks(query)) | |
| out = [] | |
| for d, ln in zip(self.docs, self.len): | |
| s = 0.0 | |
| for t in q: | |
| if t in d: | |
| tf = d[t] | |
| s += self.idf[t] * tf * (self.k1 + 1) / (tf + self.k1 * (1 - self.b + self.b * ln / self.avg)) | |
| out.append(s) | |
| return out | |
| # ---------------------------------------------------------------- model | |
| class _Classifier(nn.Module): | |
| def __init__(self, config, n_classes): | |
| super().__init__() | |
| self.backbone = AutoModel.from_config(config, attn_implementation="sdpa") | |
| self.head = nn.Linear(config.hidden_size, n_classes) | |
| def forward(self, input_ids, attention_mask): | |
| h = self.backbone(input_ids=input_ids, attention_mask=attention_mask).last_hidden_state | |
| m = attention_mask.unsqueeze(-1).to(h.dtype) | |
| return self.head((h * m).sum(1) / m.sum(1).clamp(min=1)) | |
| class _Judge: | |
| TASK = None | |
| def __init__(self, path=".", device=None, batch_tokens=None): | |
| with open(os.path.join(path, "cbx_rag_config.json")) as f: | |
| self.cfg = json.load(f) | |
| if self.cfg["task"] != self.TASK: | |
| raise ValueError(f"{path} holds the {self.cfg['task']} model, not {self.TASK}") | |
| self.device = device or ("cuda" if torch.cuda.is_available() else "cpu") | |
| self.tok = AutoTokenizer.from_pretrained(path) | |
| self.model = _Classifier(AutoConfig.from_pretrained(path), self.cfg["n_classes"]) | |
| self.model.load_state_dict(load_file(os.path.join(path, "model.safetensors"))) | |
| self.model.to(self.device).eval() | |
| self.batch_tokens = batch_tokens or self.cfg["batch_tokens"] | |
| self._len = {} | |
| def _ids(self, text): | |
| return self.tok(text, add_special_tokens=False)["input_ids"] | |
| def _encode(self, rows): | |
| cls, sep = self.tok.cls_token_id, self.tok.sep_token_id | |
| out = [] | |
| for q, ev, s in rows: | |
| qi = self._ids(q)[:self.cfg["q_max"]] | |
| si = self._ids(s)[:384] | |
| ei = self._ids(ev) | |
| room = self.cfg["max_len"] - len(qi) - len(si) - 4 | |
| out.append([cls] + qi + [sep] + ei[:max(0, room)] + [sep] + si + [sep]) | |
| return out | |
| def _probs(self, rows): | |
| """rows: (question, evidence, sentence) -> class probabilities [n, n_classes].""" | |
| enc = self._encode(rows) | |
| probs = np.zeros((len(enc), self.cfg["n_classes"]), dtype=np.float32) | |
| if not enc: | |
| return probs | |
| lengths = [len(x) for x in enc] | |
| order = sorted(range(len(enc)), key=lambda i: lengths[i]) | |
| pad = self.tok.pad_token_id | |
| cur = [] | |
| def flush(): | |
| n = max(lengths[i] for i in cur) | |
| ids = torch.full((len(cur), n), pad, dtype=torch.long) | |
| mask = torch.zeros((len(cur), n), dtype=torch.long) | |
| for k, i in enumerate(cur): | |
| ids[k, :lengths[i]] = torch.tensor(enc[i]) | |
| mask[k, :lengths[i]] = 1 | |
| ids, mask = ids.to(self.device), mask.to(self.device) | |
| if self.device.startswith("cuda"): | |
| with torch.autocast("cuda", dtype=torch.float16): | |
| logits = self.model(ids, mask).float() | |
| else: | |
| logits = self.model(ids, mask).float() | |
| probs[cur] = torch.softmax(logits, -1).cpu().numpy() | |
| for i in order: | |
| if cur and lengths[i] * (len(cur) + 1) > self.batch_tokens * 2: | |
| flush() | |
| cur = [] | |
| cur.append(i) | |
| flush() | |
| return probs | |
| # ---------------------------------------------------------------- faithfulness | |
| class Faithfulness(_Judge): | |
| """Is each answer sentence supported by the passages? Returns sentence scores, an answer score and the | |
| hallucination flag (an answer is flagged when any sentence is).""" | |
| TASK = "faithfulness" | |
| VERDICTS = ("all_verified", "some_unverified") | |
| def _ntok(self, text): | |
| n = self._len.get(text) | |
| if n is None: | |
| n = len(self._ids(text)) | |
| if len(self._len) < 200_000: | |
| self._len[text] = n | |
| return n | |
| def _line(self, c): | |
| text = c["text"] | |
| cap = self.cfg["sentence_cap_tokens"] | |
| if self._ntok(text) > cap: | |
| text = self.tok.decode(self._ids(text)[:cap]) + " ..." | |
| return f"[{c['key']}] {text}" | |
| def _evidence(self, question, ctx, sents, si, bm25): | |
| budget = self.cfg["evidence_budget"] | |
| overhead = self._ntok(question) + self._ntok(sents[si]["text"]) + 8 | |
| room = max(64, budget - overhead) | |
| lines = [self._line(c) for c in ctx] | |
| lens = [self._ntok(x) for x in lines] | |
| if sum(lens) <= room: | |
| return "\n".join(lines) | |
| chosen, used = set(), 0 | |
| def take(i): | |
| nonlocal used | |
| if i in chosen or i < 0 or i >= len(ctx) or used + lens[i] > room: | |
| return False | |
| chosen.add(i) | |
| used += lens[i] | |
| return True | |
| query = question + " " + sents[si]["text"] | |
| if si > 0: | |
| query += " " + sents[si - 1]["text"] | |
| sc = bm25.scores(query) | |
| for i in sorted(range(len(ctx)), key=lambda i: -sc[i]): | |
| if used >= room - 16 or sc[i] <= 0: | |
| break | |
| for j in (i, i - 1, i + 1): | |
| take(j) | |
| if used < room: | |
| for i in range(len(ctx)): | |
| if not take(i) and used + min(lens) > room: | |
| break | |
| return "\n".join(lines[i] for i in sorted(chosen)) | |
| def __call__(self, question, passages, answer, answer_sentences=None, profile="auto", operating_point="default"): | |
| """operating_point: "default" (flags about 4.5% of sentences on validation) or "high_recall".""" | |
| prof = _profile(profile, passages) | |
| ctx = _context(passages, prof) | |
| sents = _answer(answer, answer_sentences, prof) | |
| pc = self.cfg["profiles"][prof] | |
| bm25 = BM25([c["text"] for c in ctx]) | |
| rows = [(question, self._evidence(question, ctx, sents, si, bm25), s["text"]) for si, s in enumerate(sents)] | |
| p = 1 - self._probs(rows)[:, 0] # probability "unsupported" | |
| score, checks = p.astype(float), [None] * len(sents) | |
| thr = pc["thresholds"][operating_point] | |
| if pc.get("numeric_check") and operating_point == "default": | |
| ck = Checker(" ".join(c["text"] for c in ctx)) | |
| checks = ck.check_answer([s["text"] for s in sents]) | |
| c = pc["combiner"] | |
| X = [] | |
| for pi, nc in zip(p, checks): | |
| q = min(max(float(pi), 1e-6), 1 - 1e-6) | |
| X.append([math.log(q / (1 - q))] + [float(nc["verdict"] == v) for v in self.VERDICTS] | |
| + [min(nc["n_unverified"], 3), float(nc["eq_bad"] > 0), float(nc["eq_ok"] > 0)]) | |
| score = 1 / (1 + np.exp(-(np.array(X) @ np.array(c["coef"]) + c["intercept"]))) | |
| out = [{"key": s["key"], "text": s["text"], "unsupported_prob": round(float(pi), 6), | |
| "score": round(float(si), 6), "flag": bool(si >= thr), | |
| **({"numbers": checks[k]["verdict"]} if checks[k] else {})} | |
| for k, (s, pi, si) in enumerate(zip(sents, p, score))] | |
| return {"profile": prof, "operating_point": operating_point, "sentences": out, | |
| "answer_score": round(float(max(score)) if len(out) else 0.0, 6), | |
| "hallucination": any(x["flag"] for x in out)} | |
| # ---------------------------------------------------------------- response relevance | |
| class ResponseRelevance(_Judge): | |
| """Does the answer address the question? direct / partial / irrelevant (question and answer only).""" | |
| TASK = "response_relevance" | |
| def __call__(self, question, answer): | |
| return self.batch([(question, answer)])[0] | |
| def batch(self, pairs): | |
| pr = self._probs([(q, "", a) for q, a in pairs]) | |
| labels = self.cfg["labels"] | |
| return [{"label": labels[int(x.argmax())], "probs": {l: round(float(v), 6) for l, v in zip(labels, x)}} | |
| for x in pr] | |
| # ---------------------------------------------------------------- context relevance / utilization | |
| def _answer_score(p, spec): | |
| if spec["agg"] == "count": | |
| return float((p >= spec["thr"]).mean()) | |
| if spec["agg"] == "mean_iso": | |
| return float(np.interp(p, spec["iso_x"], spec["iso_y"]).mean()) | |
| return float(p.mean()) | |
| def _ap_at_k(v): | |
| v = np.asarray(v, float) | |
| if v.sum() == 0: | |
| return 0.0 | |
| return float(((np.cumsum(v) / np.arange(1, len(v) + 1)) * v).sum() / v.sum()) | |
| class _ContextJudge(_Judge): | |
| def _rows(self, q, ctx, idx): | |
| w, ch = self.cfg["window"], self.cfg["neighbour_chars"] | |
| rows = [] | |
| for i in idx: | |
| c = ctx[i] | |
| js = [j for j in range(i - w, i + w + 1) if j != i and 0 <= j < len(ctx) and ctx[j]["doc"] == c["doc"]] | |
| rows.append((q, " ".join(ctx[j]["text"][:ch] for j in js), c["text"])) | |
| return rows | |
| def _run(self, q, ctx, pc, prefilter_query=None): | |
| idx = list(range(len(ctx))) | |
| top = pc.get("prefilter_top_n") | |
| if top and prefilter_query is not None and len(ctx) > top: | |
| sc = np.asarray(BM25([c["text"] for c in ctx]).scores(prefilter_query)) | |
| m = np.zeros(len(ctx), bool) | |
| for i in np.argsort(-sc)[:top]: | |
| m[max(0, i - 1):i + 2] = True | |
| idx = [i for i in idx if m[i]] | |
| p = np.zeros(len(ctx)) | |
| if idx: | |
| p[idx] = 1 - self._probs(self._rows(q, ctx, idx))[:, 0] | |
| return p, len(idx) | |
| class ContextRelevance(_ContextJudge): | |
| """Is each context sentence relevant to the question? Returns sentence scores, the answer-level relevance | |
| fraction and, with two or more passages, context precision (AP over passages in the given order).""" | |
| TASK = "context_relevance" | |
| def __call__(self, question, passages, profile="auto"): | |
| prof = _profile(profile, passages) | |
| ctx = _context(passages, prof) | |
| pc = self.cfg["profiles"][prof] | |
| p, _ = self._run(question, ctx, pc) | |
| thr = pc["sentence_threshold"] | |
| out = {"profile": prof, | |
| "sentences": [{"key": c["key"], "passage": c["doc"], "text": c["text"], "score": round(float(x), 6), | |
| "relevant": bool(x >= thr)} for c, x in zip(ctx, p)], | |
| "relevance": round(_answer_score(p, pc["answer_score"]), 6)} | |
| docs = sorted({c["doc"] for c in ctx}) | |
| if len(docs) > 1: | |
| d = np.array([c["doc"] for c in ctx]) | |
| out["passage_relevant"] = [bool((p[d == k] >= thr).any()) for k in docs] | |
| out["context_precision"] = round(_ap_at_k(out["passage_relevant"]), 6) | |
| return out | |
| class ContextUtilization(_ContextJudge): | |
| """Did the answer use each context sentence? Returns sentence scores and the utilization fraction.""" | |
| TASK = "context_utilization" | |
| def __call__(self, question, passages, answer, profile="auto", prefilter=True): | |
| prof = _profile(profile, passages) | |
| ctx = _context(passages, prof) | |
| pc = dict(self.cfg["profiles"][prof]) | |
| if not prefilter: | |
| pc["prefilter_top_n"] = None | |
| q = question + " || " + str(answer)[:1200] | |
| p, n_scored = self._run(q, ctx, pc, prefilter_query=question + " " + str(answer)) | |
| thr = pc["sentence_threshold"] | |
| return {"profile": prof, "sentences_scored": n_scored, | |
| "sentences": [{"key": c["key"], "passage": c["doc"], "text": c["text"], "score": round(float(x), 6), | |
| "utilized": bool(x >= thr)} for c, x in zip(ctx, p)], | |
| "utilization": round(_answer_score(p, pc["answer_score"]), 6)} | |