finalign / evaluate.py
pdmdp's picture
Upload 22 files
c0f3416 verified
Raw History Blame Contribute Delete
6.65 kB
"""Run fixed component ablations and record per-query effectiveness and latency."""
import argparse
import json
import platform
import random
import time
from datetime import UTC, datetime
from importlib.metadata import version
from pathlib import Path
import numpy as np
from finalign.core import Options, metrics, ranked
from finalign.index import Index, LexicalSearch
from finalign.neural import NeuralSearch
CONFIGURATIONS = {
"bm25": None,
"bge": Options(),
"bge_keywords": Options(keywords=True),
"bge_hyde": Options(hyde=True),
"bge_expanded": Options(keywords=True, hyde=True),
"hybrid": Options(keywords=True, hyde=True, lexical=True),
"full": Options(keywords=True, hyde=True, lexical=True, rerank=True),
"full_without_keywords": Options(hyde=True, lexical=True, rerank=True),
"full_without_hyde": Options(keywords=True, lexical=True, rerank=True),
"full_without_bm25": Options(keywords=True, hyde=True, rerank=True),
}
def summarize(records):
output = {}
for config in sorted({r["configuration"] for r in records}):
rows = [r for r in records if r["configuration"] == config and r["repeat"] > 0]
# Repeats measure timing, not extra independent queries.
first = {r["qid"]: r for r in reversed(rows)}
stages = sorted({stage for row in rows for stage in row["timings_ms"]})
output[config] = {
"queries": len(first), "timed_runs": len(rows),
"effectiveness": {m: float(np.mean([r["metrics"][m] for r in first.values()]))
for m in ("nDCG@10", "AP", "Recall@10")},
"latency_ms": {stage: {"p50": float(np.median([r["timings_ms"][stage] for r in rows])),
"p95": float(np.percentile([r["timings_ms"][stage] for r in rows], 95))}
for stage in stages}}
return output
def main(args):
if args.repeats < 1 or args.limit < 0:
raise ValueError("Use at least one repeat and a nonnegative query limit.")
configurations = args.configurations.split(",")
unknown = set(configurations) - CONFIGURATIONS.keys()
if unknown:
raise ValueError(f"Unknown configurations: {sorted(unknown)}")
if args.output.exists():
raise ValueError("The output directory already exists; use a new run name.")
args.output.mkdir(parents=True)
start = time.perf_counter()
index = Index(args.index)
lexical = LexicalSearch(index)
encoder_only = all(c in ("bm25", "bge") for c in configurations)
engine = None if configurations == ["bm25"] else NeuralSearch(index, args.device, encoder_only)
startup_seconds = time.perf_counter() - start
rows = [r for r in index.queries[args.split] if any(rel > 0 for rel in r["qrels"].values())]
random.Random(2026).shuffle(rows)
if args.limit:
rows = rows[:args.limit]
metadata = {"created_utc": datetime.now(UTC).isoformat(), "split": args.split,
"query_ids": [r["qid"] for r in rows], "configurations": configurations,
"repeats": args.repeats, "startup_seconds": startup_seconds,
"device": args.device, "platform": platform.platform(), "python": platform.python_version(),
"model_revisions": index.models, "index_manifest": index.manifest,
"packages": {p: version(p) for p in ("torch", "transformers", "sentence-transformers",
"pyterrier", "numpy")},
"notes": "One warm-up per configuration is excluded. GPU synchronization is included. "
"Queueing and model loading are excluded from query timings. No query-result cache. "
"The course test split informed past design choices; dev is used for new tuning."}
(args.output / "run.json").write_text(json.dumps(metadata, indent=2) + "\n")
records = []
with (args.output / "queries.jsonl").open("w") as stream:
# Warm-up is separate from the retained sample and does not become another observation.
for repeat in range(args.repeats + 1):
batch = rows[:1] if repeat == 0 else rows
for row in batch:
order = configurations.copy()
random.Random(f"{row['qid']}:{repeat}").shuffle(order)
for config in order:
started = time.perf_counter()
scores, lexical_ms = lexical.search(row["query"])
options = CONFIGURATIONS[config]
timings = {"BM25": lexical_ms}
result = {"ranking": ranked(scores), "variants": {}, "seed": None}
if options is not None:
if not options.lexical:
started = time.perf_counter()
result = engine.search(row["query"], scores, options)
timings.update(result["timings_ms"])
# BM25 is displayed alongside every demo run, but ablations without
# lexical fusion must not count it as part of the neural pipeline.
if not options.lexical:
timings.pop("BM25")
timings["Total"] = (time.perf_counter() - started) * 1000
record = {"qid": row["qid"], "query": row["query"], "configuration": config,
"repeat": repeat, "metrics": metrics([d for d, _ in result["ranking"]], row["qrels"]),
"timings_ms": timings, "variants": result["variants"], "seed": result["seed"],
"ranking": result["ranking"]}
records.append(record)
stream.write(json.dumps(record) + "\n")
stream.flush()
print(f"repeat={repeat} qid={row['qid']}", flush=True)
(args.output / "summary.json").write_text(json.dumps(summarize(records), indent=2) + "\n")
if __name__ == "__main__":
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--index", type=Path, required=True)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--split", choices=["dev", "test"], default="dev")
parser.add_argument("--device", choices=["cpu", "mps", "cuda"], default="cpu")
parser.add_argument("--limit", type=int, default=0, help="0 evaluates every judged query")
parser.add_argument("--repeats", type=int, default=3)
parser.add_argument("--configurations", default=",".join(CONFIGURATIONS))
main(parser.parse_args())