"""Offline localization breakdown for the matched full-VALID step-324 evaluation.""" from __future__ import annotations import hashlib import json from collections import defaultdict from pathlib import Path import numpy as np from src.rewards.cache import cache_path from src.rewards.checker import statement_iou RUN_DIR = Path("results/phase7/matched_vllm_step324_fullvalid") OUTPUT = Path("results/phase7/step324_localization_breakdown.json") CORRECTED_METRICS = Path("results/phase7/matched_vllm_step324_fullvalid/paired_metrics_corrected.json") def _load_jsonl(path): return [json.loads(line) for line in path.read_text().splitlines() if line.strip()] def _bootstrap(values, seed, replicates=10000): arr = np.asarray(values, dtype=float) if not len(arr): return None rng = np.random.default_rng(seed) draws = arr[rng.integers(0, len(arr), size=(replicates, len(arr)))].mean(axis=1) return [float(x) for x in np.quantile(draws, [.025, .975])] def _summarize(rows, annotations, model_name, graph_dir): vulnerable = [] safe = [] cache_trace_eligible = 0 trace_gt_count = 0 for row in rows: ann = annotations[row["sample_id"]] pred_lines = set(int(x) for x in row["pred_lines"]) trace_lines = [int(step["line"]) for step in row["parsed_trace"]] if int(row["target"]) == 1: l1 = set(int(x) for x in ann.get("l1_lines", [])) l2 = set(int(x) for x in ann.get("l2_lines", [])) gold_union = l1 | l2 vulnerable.append({ "sample_id": row["sample_id"], "pair_id": row["pair_id"], "label_correct": bool(row["format_ok"] and row["pred_label"] == "VULNERABLE"), "format_ok": bool(row["format_ok"]), "predicted_label": row["pred_label"], "pred_line_count": len(pred_lines), "pred_empty": not pred_lines, "iou_l2": statement_iou(pred_lines, l2), "iou_l1_union_l2": statement_iou(pred_lines, gold_union), "l2_recall": len(pred_lines & l2) / len(l2) if l2 else None, "l2_precision": len(pred_lines & l2) / len(pred_lines) if pred_lines else 0.0, "trace_step_count": len(trace_lines), "trace_gt_available": bool(ann.get("trace_gt")), "trace_graph_available": cache_path(graph_dir, row["sample_id"]).exists(), }) trace_gt_count += int(bool(ann.get("trace_gt"))) cache_trace_eligible += int(bool(ann.get("trace_gt")) and cache_path(graph_dir, row["sample_id"]).exists()) else: safe.append({"sample_id": row["sample_id"], "pair_id": row["pair_id"], "label_correct": bool(row["format_ok"] and row["pred_label"] == "SAFE"), "format_ok": bool(row["format_ok"]), "predicted_label": row["pred_label"], "pred_line_count": len(pred_lines), "pred_empty": not pred_lines, "trace_step_count": len(trace_lines)}) def localization_summary(group): return { "n": len(group), "mean_iou_l2": float(np.mean([x["iou_l2"] for x in group])) if group else None, "mean_iou_l1_union_l2": float(np.mean([x["iou_l1_union_l2"] for x in group])) if group else None, "mean_l2_recall": float(np.mean([x["l2_recall"] for x in group if x["l2_recall"] is not None])) if any(x["l2_recall"] is not None for x in group) else None, "mean_l2_precision": float(np.mean([x["l2_precision"] for x in group])) if group else None, "empty_prediction_fraction": float(np.mean([x["pred_empty"] for x in group])) if group else None, "mean_predicted_statement_lines": float(np.mean([x["pred_line_count"] for x in group])) if group else None, } correct = [x for x in vulnerable if x["label_correct"]] incorrect = [x for x in vulnerable if not x["label_correct"]] parsed_vulnerable = [x for x in vulnerable if x["format_ok"]] return { "model": model_name, "vulnerable_all": localization_summary(vulnerable), "vulnerable_parsed_only": localization_summary(parsed_vulnerable), "vulnerable_label_correct": localization_summary(correct), "vulnerable_label_incorrect": localization_summary(incorrect), "safe_rows": {"n": len(safe), "label_correct": sum(x["label_correct"] for x in safe), "nonempty_statement_fraction": float(np.mean([not x["pred_empty"] for x in safe])) if safe else None, "mean_statement_lines": float(np.mean([x["pred_line_count"] for x in safe])) if safe else None}, "trace_ground_truth": {"vulnerable_rows_with_trace_gt": trace_gt_count, "vulnerable_rows_with_cached_dfg": cache_trace_eligible, "vulnerable_rows_with_any_cached_graph": sum(x["trace_graph_available"] for x in vulnerable), "trace_steps_present_fraction": float(np.mean([x["trace_step_count"] > 0 for x in vulnerable])) if vulnerable else None, "mean_trace_steps": float(np.mean([x["trace_step_count"] for x in vulnerable])) if vulnerable else None}, "per_vulnerable_row": vulnerable, } def run(): output = json.loads((RUN_DIR / "paired_metrics.json").read_text()) sft = _load_jsonl(RUN_DIR / "sft_v3_step500_predictions.jsonl") grpo = _load_jsonl(RUN_DIR / "grpo_step324_predictions.jsonl") if len(sft) != 2208 or len(grpo) != 2208: raise ValueError("expected both models to cover all 2,208 VALID rows") if [(r["sample_id"], r["prompt_sha256"]) for r in sft] != [ (r["sample_id"], r["prompt_sha256"]) for r in grpo]: raise ValueError("paired evaluation rows/prompts differ") valid_pairs = set(json.loads(Path("data/splits/chronological/valid.json").read_text())) if {str(r["pair_id"]) for r in sft} != valid_pairs: raise ValueError("evaluation predictions do not exactly match chronological VALID") annotations = {} valid_ids = {r["sample_id"] for r in sft} for line in Path("data/processed/annotations.jsonl").read_text().splitlines(): if line.strip(): ann = json.loads(line) if ann.get("sample_id") in valid_ids: annotations[ann["sample_id"]] = ann if set(annotations) != valid_ids: raise ValueError("some VALID sample annotations are missing") # Parsed trace lines are extracted from the raw v2 completions. The cached R3 DFG # artifacts were built for TRAIN, so VALID DFG trace scores are reported unavailable. from src.contracts.parser import parse_output for row in (*sft, *grpo): parsed = parse_output(row["completion"], contract_version=2) row["parsed_trace"] = [{"line": step.line} for step in parsed.trace] graph_dir = Path("data/cpg_cache/reward_graphs") sft_result = _summarize(sft, annotations, "sft_v3_step500", graph_dir) grpo_result = _summarize(grpo, annotations, "grpo_step324", graph_dir) sft_by_id = {r["sample_id"]: r for r in sft_result["per_vulnerable_row"]} grpo_by_id = {r["sample_id"]: r for r in grpo_result["per_vulnerable_row"]} paired = [grpo_by_id[sid]["iou_l2"] - sft_by_id[sid]["iou_l2"] for sid in sorted(sft_by_id)] jointly_parsed_ids = sorted(sid for sid in sft_by_id if sft_by_id[sid]["format_ok"] and grpo_by_id[sid]["format_ok"]) jointly_parsed_delta = [grpo_by_id[sid]["iou_l2"] - sft_by_id[sid]["iou_l2"] for sid in jointly_parsed_ids] union_delta = [grpo_by_id[sid]["iou_l1_union_l2"] - sft_by_id[sid]["iou_l1_union_l2"] for sid in sorted(sft_by_id)] by_pair = defaultdict(list) for sid in sorted(sft_by_id): pid = grpo_by_id[sid]["pair_id"] by_pair[pid].append((grpo_by_id[sid]["iou_l2"] - sft_by_id[sid]["iou_l2"], grpo_by_id[sid]["iou_l1_union_l2"] - sft_by_id[sid]["iou_l1_union_l2"])) pair_deltas = [np.mean(v, axis=0) for v in by_pair.values()] rng = np.random.default_rng(42) pick = rng.integers(0, len(pair_deltas), size=(10000, len(pair_deltas))) boot = np.asarray(pair_deltas)[pick].mean(axis=1) parsed_pair_values = [] for pair_id in sorted({grpo_by_id[sid]["pair_id"] for sid in jointly_parsed_ids}): ids = [sid for sid in jointly_parsed_ids if grpo_by_id[sid]["pair_id"] == pair_id] parsed_pair_values.append(float(np.mean([grpo_by_id[sid]["iou_l2"] - sft_by_id[sid]["iou_l2"] for sid in ids]))) parsed_rng = np.random.default_rng(43) parsed_pick = parsed_rng.integers(0, len(parsed_pair_values), size=(10000, len(parsed_pair_values))) parsed_boot = np.asarray(parsed_pair_values)[parsed_pick].mean(axis=1) enriched = {} for name, model_rows in (("sft_v3_step500", sft), ("grpo_step324", grpo)): enriched[name] = [{**row, "gold_lines": (list(annotations[row["sample_id"]].get("l2_lines", [])) if int(row["target"]) == 1 else [])} for row in model_rows] from scripts.eval_paired_adapters import paired_summary corrected_paired_metrics = paired_summary(enriched) result = { "purpose": "offline full-VALID localization breakdown of preserved matched greedy outputs", "n_rows": 2208, "n_vulnerable_rows": 1104, "n_pairs": 1104, "models": {"sft_v3_step500": {k: v for k, v in sft_result.items() if k != "per_vulnerable_row"}, "grpo_step324": {k: v for k, v in grpo_result.items() if k != "per_vulnerable_row"}}, "paired_grpo_minus_sft_iou_delta": { "mean_iou_l2_delta": float(np.mean(paired)), "pair_cluster_bootstrap_95_ci": [float(x) for x in np.quantile(boot[:, 0], [.025, .975])], "parsed_both_models": {"n_vulnerable_rows": len(jointly_parsed_ids), "mean_iou_l2_delta": float(np.mean(jointly_parsed_delta)), "pair_cluster_bootstrap_95_ci": [float(x) for x in np.quantile(parsed_boot, [.025, .975])]}, "mean_iou_l1_union_l2_delta": float(np.mean(union_delta)), "l1_union_l2_pair_cluster_bootstrap_95_ci": [float(x) for x in np.quantile(boot[:, 1], [.025, .975])]}, "corrected_matched_validation_metrics": corrected_paired_metrics, "corrected_metric_note": "Recomputed from preserved prediction JSONL plus frozen VALID annotations; the original paired_metrics.json omitted gold_lines during generation, so its statement-IoU fields were invalid.", "r3_evaluation": "unavailable: 0/831 VALID vulnerable rows with trace_gt had a cached DFG graph; no new graph builds were run", "provenance": { "sft_predictions_sha256": hashlib.sha256((RUN_DIR / "sft_v3_step500_predictions.jsonl").read_bytes()).hexdigest(), "grpo_predictions_sha256": hashlib.sha256((RUN_DIR / "grpo_step324_predictions.jsonl").read_bytes()).hexdigest(), "annotations_sha256": hashlib.sha256(Path("data/processed/annotations.jsonl").read_bytes()).hexdigest(), "valid_split_sha256": hashlib.sha256(Path("data/splits/chronological/valid.json").read_bytes()).hexdigest(), "paired_metrics_sha256": hashlib.sha256((RUN_DIR / "paired_metrics.json").read_bytes()).hexdigest(), "test_read": False}, "limitations": ["Statement localization uses the frozen L2 and L1∪L2 line sets; no annotations were changed.", "Trace step counts are descriptive only; DFG validity cannot be scored on VALID because no cached VALID DFGs exist.", "Predictions are from one greedy pass per model and one frozen checkpoint pair."]} result["models"]["sft_v3_step500"]["per_vulnerable_row"] = sft_result["per_vulnerable_row"] result["models"]["grpo_step324"]["per_vulnerable_row"] = grpo_result["per_vulnerable_row"] OUTPUT.parent.mkdir(parents=True, exist_ok=True) OUTPUT.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n") corrected = {**corrected_paired_metrics, "created_by": "scripts/analyze_step324_localization.py", "gold_lines_source": "data/processed/annotations.jsonl, L2 for vulnerable rows and empty for SAFE rows", "original_paired_metrics_sha256": hashlib.sha256((RUN_DIR / "paired_metrics.json").read_bytes()).hexdigest(), "test_touched": False} CORRECTED_METRICS.write_text(json.dumps(corrected, indent=2, sort_keys=True) + "\n") return result if __name__ == "__main__": print(json.dumps(run(), indent=2))