File size: 6,646 Bytes
c0f3416
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
"""Run fixed component ablations and record per-query effectiveness and latency."""

import argparse
import json
import platform
import random
import time
from datetime import UTC, datetime
from importlib.metadata import version
from pathlib import Path

import numpy as np
from finalign.core import Options, metrics, ranked
from finalign.index import Index, LexicalSearch
from finalign.neural import NeuralSearch

CONFIGURATIONS = {
    "bm25": None,
    "bge": Options(),
    "bge_keywords": Options(keywords=True),
    "bge_hyde": Options(hyde=True),
    "bge_expanded": Options(keywords=True, hyde=True),
    "hybrid": Options(keywords=True, hyde=True, lexical=True),
    "full": Options(keywords=True, hyde=True, lexical=True, rerank=True),
    "full_without_keywords": Options(hyde=True, lexical=True, rerank=True),
    "full_without_hyde": Options(keywords=True, lexical=True, rerank=True),
    "full_without_bm25": Options(keywords=True, hyde=True, rerank=True),
}


def summarize(records):
    output = {}
    for config in sorted({r["configuration"] for r in records}):
        rows = [r for r in records if r["configuration"] == config and r["repeat"] > 0]
        # Repeats measure timing, not extra independent queries.
        first = {r["qid"]: r for r in reversed(rows)}
        stages = sorted({stage for row in rows for stage in row["timings_ms"]})
        output[config] = {
            "queries": len(first), "timed_runs": len(rows),
            "effectiveness": {m: float(np.mean([r["metrics"][m] for r in first.values()]))
                              for m in ("nDCG@10", "AP", "Recall@10")},
            "latency_ms": {stage: {"p50": float(np.median([r["timings_ms"][stage] for r in rows])),
                                    "p95": float(np.percentile([r["timings_ms"][stage] for r in rows], 95))}
                           for stage in stages}}
    return output


def main(args):
    if args.repeats < 1 or args.limit < 0:
        raise ValueError("Use at least one repeat and a nonnegative query limit.")
    configurations = args.configurations.split(",")
    unknown = set(configurations) - CONFIGURATIONS.keys()
    if unknown:
        raise ValueError(f"Unknown configurations: {sorted(unknown)}")
    if args.output.exists():
        raise ValueError("The output directory already exists; use a new run name.")
    args.output.mkdir(parents=True)
    start = time.perf_counter()
    index = Index(args.index)
    lexical = LexicalSearch(index)
    encoder_only = all(c in ("bm25", "bge") for c in configurations)
    engine = None if configurations == ["bm25"] else NeuralSearch(index, args.device, encoder_only)
    startup_seconds = time.perf_counter() - start
    rows = [r for r in index.queries[args.split] if any(rel > 0 for rel in r["qrels"].values())]
    random.Random(2026).shuffle(rows)
    if args.limit:
        rows = rows[:args.limit]
    metadata = {"created_utc": datetime.now(UTC).isoformat(), "split": args.split,
                "query_ids": [r["qid"] for r in rows], "configurations": configurations,
                "repeats": args.repeats, "startup_seconds": startup_seconds,
                "device": args.device, "platform": platform.platform(), "python": platform.python_version(),
                "model_revisions": index.models, "index_manifest": index.manifest,
                "packages": {p: version(p) for p in ("torch", "transformers", "sentence-transformers",
                                                      "pyterrier", "numpy")},
                "notes": "One warm-up per configuration is excluded. GPU synchronization is included. "
                         "Queueing and model loading are excluded from query timings. No query-result cache. "
                         "The course test split informed past design choices; dev is used for new tuning."}
    (args.output / "run.json").write_text(json.dumps(metadata, indent=2) + "\n")
    records = []
    with (args.output / "queries.jsonl").open("w") as stream:
        # Warm-up is separate from the retained sample and does not become another observation.
        for repeat in range(args.repeats + 1):
            batch = rows[:1] if repeat == 0 else rows
            for row in batch:
                order = configurations.copy()
                random.Random(f"{row['qid']}:{repeat}").shuffle(order)
                for config in order:
                    started = time.perf_counter()
                    scores, lexical_ms = lexical.search(row["query"])
                    options = CONFIGURATIONS[config]
                    timings = {"BM25": lexical_ms}
                    result = {"ranking": ranked(scores), "variants": {}, "seed": None}
                    if options is not None:
                        if not options.lexical:
                            started = time.perf_counter()
                        result = engine.search(row["query"], scores, options)
                        timings.update(result["timings_ms"])
                        # BM25 is displayed alongside every demo run, but ablations without
                        # lexical fusion must not count it as part of the neural pipeline.
                        if not options.lexical:
                            timings.pop("BM25")
                    timings["Total"] = (time.perf_counter() - started) * 1000
                    record = {"qid": row["qid"], "query": row["query"], "configuration": config,
                              "repeat": repeat, "metrics": metrics([d for d, _ in result["ranking"]], row["qrels"]),
                              "timings_ms": timings, "variants": result["variants"], "seed": result["seed"],
                              "ranking": result["ranking"]}
                    records.append(record)
                    stream.write(json.dumps(record) + "\n")
                    stream.flush()
                print(f"repeat={repeat} qid={row['qid']}", flush=True)
    (args.output / "summary.json").write_text(json.dumps(summarize(records), indent=2) + "\n")


if __name__ == "__main__":
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--index", type=Path, required=True)
    parser.add_argument("--output", type=Path, required=True)
    parser.add_argument("--split", choices=["dev", "test"], default="dev")
    parser.add_argument("--device", choices=["cpu", "mps", "cuda"], default="cpu")
    parser.add_argument("--limit", type=int, default=0, help="0 evaluates every judged query")
    parser.add_argument("--repeats", type=int, default=3)
    parser.add_argument("--configurations", default=",".join(CONFIGURATIONS))
    main(parser.parse_args())