File size: 4,597 Bytes
e2c9d42 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 | #!/usr/bin/env python3
"""Build one split per RAGTruth task type for a train/test/transfer experiment.
RAGTruth carries three genuinely different generation settings -- answering from
retrieved passages (QA), summarising news (Summary), and writing prose from
structured data (Data2txt). Training on one and evaluating on all three is the
design Lookback Lens used to show that hidden-state probes overfit their
training distribution: 100.0 train / 97.4 in-domain test / 58.3 transfer, versus
88.3 / 87.1 / 66.1 for attention ratios.
Our RAGTruth numbers so far are entirely in-domain, which is the column where a
hidden-state probe is expected to look strongest and where its weakness is
invisible. This makes the other columns measurable.
Each split restricts train/validation to one task; the eval set stays whole, so
one scoring pass yields both in-domain and transfer numbers once predictions are
sliced by `extra.task_type`. Nothing is re-extracted -- the feature cache is
keyed on row content, not on the split.
Usage:
python scripts/make_ragtruth_task_splits.py \
--input ~/Desktop/datasets/hallucinations/ragtruth_llama2_7b_chat.jsonl \
--out-dir ~/Desktop/datasets/hallucinations/task_splits
"""
from __future__ import annotations
import argparse
import collections
import json
from pathlib import Path
import sys
REPO_ROOT = Path(__file__).resolve().parents[1]
if str(REPO_ROOT) not in sys.path:
sys.path.insert(0, str(REPO_ROOT))
from scripts.make_eval_split import (
NATIVE_DOMAIN_SPLIT_SCHEME,
build_native_split,
dataset_binding,
)
TASK_FIELD = "metadata.extra.task_type"
def parse_args(argv=None):
p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
p.add_argument("--input", type=Path, required=True)
p.add_argument("--out-dir", type=Path, required=True)
p.add_argument("--validation-fraction", type=float, default=0.15)
p.add_argument("--seed", type=int, default=42)
return p.parse_args(argv)
def main(argv=None):
args = parse_args(argv)
rows = [json.loads(l) for l in args.input.open(encoding="utf-8")]
tasks = sorted({r["metadata"]["extra"]["task_type"] for r in rows})
args.out_dir.mkdir(parents=True, exist_ok=True)
test_by_task = collections.Counter(
r["metadata"]["extra"]["task_type"]
for r in rows
if r["metadata"]["split"] == "test"
)
print(f"tasks: {tasks}")
print(f"test rows per task (the transfer targets): {dict(test_by_task)}\n")
for task in tasks:
built = build_native_split(
rows,
seed=args.seed,
validation_fraction=args.validation_fraction,
native_split_field="metadata.split",
eval_value="test",
grouping_key="prompt",
train_domain_field=TASK_FIELD,
train_domain_value=task,
)["split"]
# Guards: training must be single-task, evaluation must stay whole.
train_tasks = {
rows[i]["metadata"]["extra"]["task_type"] for i in built["train_indices"]
}
assert train_tasks == {task}, f"train leaked other tasks: {train_tasks}"
eval_tasks = {
rows[i]["metadata"]["extra"]["task_type"] for i in built["eval_indices"]
}
assert eval_tasks == set(tasks), "eval must span every task for transfer"
summary = {
"scheme": NATIVE_DOMAIN_SPLIT_SCHEME,
"seed": args.seed,
"n": len(rows),
"dataset_binding": dataset_binding(rows),
"test_fraction": len(built["eval_indices"]) / len(rows),
"validation_fraction": args.validation_fraction,
"native": {
"split_field": "metadata.split",
"eval_value": "test",
"grouping_key": "prompt",
"train_domain_field": TASK_FIELD,
"train_domain_value": task,
},
"provenance": {
"design": "train on one task, evaluate on all -- slice predictions by task_type",
"train_domain": task,
},
"split": built,
}
out = args.out_dir / f"train_{task.lower()}.json"
out.write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8")
print(
f" train={task:<9} {len(built['train_indices']):>5} train "
f"{len(built['validation_indices']):>4} val "
f"{len(built['eval_indices']):>4} eval(all tasks) -> {out.name}"
)
return 0
if __name__ == "__main__":
raise SystemExit(main())
|