Spaces:
Runtime error
Runtime error
File size: 2,791 Bytes
6dfa658 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 | from __future__ import annotations
import argparse
import json
from pathlib import Path
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--input", type=Path, default=Path("outputs/qa_eval_extractive_bm25_full.json"))
parser.add_argument("--output", type=Path, default=Path("outputs/qa_error_analysis.md"))
parser.add_argument("--max-examples", type=int, default=12)
args = parser.parse_args()
data = json.loads(args.input.read_text(encoding="utf-8"))
answers = data["answers"]
retrieval_failures = [
row for row in answers if row["metrics"]["top5_source_hit"] == 0.0
]
top1_failures = [
row for row in answers
if row["metrics"]["top1_source_hit"] == 0.0 and row["metrics"]["top5_source_hit"] == 1.0
]
low_f1 = sorted(answers, key=lambda row: row["metrics"]["token_f1"])[: args.max_examples]
lines = [
"# QA Error Analysis",
"",
"## Summary",
"",
f"- Total questions: {len(answers)}",
f"- Top-5 retrieval failures: {len(retrieval_failures)}",
f"- Correct source in top-5 but not top-1: {len(top1_failures)}",
"",
"## Failure Types",
"",
"1. Retrieval failure: the gold source is not present in top-5, so the answer cannot be grounded correctly.",
"2. Ranking failure: the gold source is in top-5 but not ranked first; the extractive baseline cites the wrong top-1 passage.",
"3. Answer formatting mismatch: the retrieved source is correct but generated/extractive wording differs from the verified answer.",
"",
"## Lowest Token-F1 Examples",
"",
]
for row in low_f1:
metrics = row["metrics"]
lines.extend(
[
f"### {row['question_id']}",
"",
f"Question: {row['question']}",
"",
f"Gold source ids: {', '.join(row['gold_source_ids'])}",
"",
f"Retrieved ids: {', '.join(row['retrieved_ids'][:5])}",
"",
(
"Metrics: "
f"F1={metrics['token_f1']:.3f}, "
f"Top1Hit={metrics['top1_source_hit']:.0f}, "
f"Top5Hit={metrics['top5_source_hit']:.0f}, "
f"CitationAcc={metrics['citation_label_accuracy']:.0f}"
),
"",
]
)
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text("\n".join(lines), encoding="utf-8")
print(f"Wrote {args.output}")
print(f"Top-5 retrieval failures: {len(retrieval_failures)}")
print(f"Ranking failures: {len(top1_failures)}")
if __name__ == "__main__":
main()
|