File size: 1,472 Bytes
6e04720 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 | import argparse
import pandas as pd
parser = argparse.ArgumentParser()
parser.add_argument("--input", default="afghanistan_local_knowledge_scored.csv")
parser.add_argument("--output", default="evaluation_summary.csv")
args = parser.parse_args()
df = pd.read_csv(args.input)
required = {
"condition", "answerability", "human_score",
"unsupported_specific_claim"
}
missing = required - set(df.columns)
if missing:
raise ValueError(f"Missing required columns: {sorted(missing)}")
conditions = ["closed_book", "uncertainty_instruction", "source_supplied"]
rows = []
for condition in conditions:
part = df[df["condition"] == condition]
a = part[part["answerability"] == "answerable_from_source"]
u = part[part["answerability"] == "not_in_source"]
rows.append({
"condition": condition,
"total_responses": len(part),
"answerable_questions": len(a),
"exact_correct": int((a["human_score"] == "correct").sum()),
"partially_correct": int((a["human_score"] == "partially_correct").sum()),
"appropriate_abstentions": int(
(u["human_score"] == "appropriate_abstention").sum()
),
"unsupported_or_contradicted_flags": int(
(part["unsupported_specific_claim"].astype(str).str.lower() == "yes").sum()
)
})
result = pd.DataFrame(rows)
result.to_csv(args.output, index=False)
print(result.to_string(index=False))
print(f"\\nSaved summary to {args.output}")
|