Spaces:
Running
Running
SooryaKiranKakkarayil
Day 6: fix batch2 labels, fix LLM-judge verdict bug, add pass_k and rpd diagnostic scripts
6b6557c | """ | |
| eval/fix_batch2_labels.py | |
| One-off fix for a batch2 data bug: prepare_golden_sample_batch2.py wrote | |
| raw numeric HF dataset labels ('1'/'0') for the two new rows (row_0060, | |
| row_0061) instead of the string convention ('phishing'/'legitimate') used | |
| by the original 60 rows. score_golden_eval.py only recognizes the exact | |
| string "phishing", so the numeric labels were silently mis-binned as | |
| "legitimate" -- making a correct model prediction (row_0060, correctly | |
| flagged high-risk) look like a false positive in the confusion matrix. | |
| Confirmed mapping (verified against actual row content on Day 6): | |
| '1' -> phishing (row_0060: obfuscated software-spam) | |
| '0' -> legitimate (row_0061: energy-scheduling log) | |
| This script fixes BOTH files that hold a copy of true_label: | |
| - eval/golden_sample.csv (label column) | |
| - eval/golden_eval_results.jsonl (true_label field) | |
| Safe to re-run: only touches rows whose label is exactly '1' or '0', | |
| so already-fixed rows (now 'phishing'/'legitimate') are left untouched. | |
| """ | |
| import csv | |
| import json | |
| from pathlib import Path | |
| LABEL_MAP = {"1": "phishing", "0": "legitimate"} | |
| CSV_PATH = Path("eval/golden_sample.csv") | |
| JSONL_PATH = Path("eval/golden_eval_results.jsonl") | |
| def fix_csv(): | |
| rows = list(csv.DictReader(open(CSV_PATH, encoding="utf-8"))) | |
| fieldnames = list(rows[0].keys()) | |
| changed = 0 | |
| for r in rows: | |
| if r.get("label") in LABEL_MAP: | |
| r["label"] = LABEL_MAP[r["label"]] | |
| changed += 1 | |
| with open(CSV_PATH, "w", encoding="utf-8", newline="") as f: | |
| w = csv.DictWriter(f, fieldnames=fieldnames) | |
| w.writeheader() | |
| w.writerows(rows) | |
| print(f"[csv] fixed {changed} row(s) in {CSV_PATH}") | |
| def fix_jsonl(): | |
| rows = [] | |
| with open(JSONL_PATH, encoding="utf-8") as f: | |
| for line in f: | |
| line = line.strip() | |
| if line: | |
| rows.append(json.loads(line)) | |
| changed = 0 | |
| for r in rows: | |
| if r.get("true_label") in LABEL_MAP: | |
| r["true_label"] = LABEL_MAP[r["true_label"]] | |
| changed += 1 | |
| with open(JSONL_PATH, "w", encoding="utf-8") as f: | |
| for r in rows: | |
| f.write(json.dumps(r) + "\n") | |
| print(f"[jsonl] fixed {changed} row(s) in {JSONL_PATH}") | |
| if __name__ == "__main__": | |
| fix_csv() | |
| fix_jsonl() | |
| print("done") |