| { | |
| "format_version": 2, | |
| "base_model": "answerdotai/ModernBERT-base", | |
| "labels": [ | |
| "clean", | |
| "buggy" | |
| ], | |
| "clean_label": "clean", | |
| "max_length": 512, | |
| "languages": [ | |
| "python", | |
| "java", | |
| "cpp" | |
| ], | |
| "normalization": { | |
| "python": "ast_unparse_strip_docstrings_v1", | |
| "java": "c_family_strip_comments_v1", | |
| "cpp": "c_family_strip_comments_v1" | |
| }, | |
| "objective": "binary classification (buggy vs unchanged) + pairwise ranking (buggy above its fix)", | |
| "datasets": [ | |
| "nebius/SWE-rebench", | |
| "princeton-nlp/SWE-bench (train)", | |
| "nebius/SWE-bench-extra", | |
| "SWE-Gym/SWE-Gym", | |
| "ByteDance-Seed/Multi-SWE-RL", | |
| "bigcode/commitpackft", | |
| "ASSERT-KTH/megadiff-single-function", | |
| "bstee615/bigvul" | |
| ], | |
| "gold_test": [ | |
| "princeton-nlp/SWE-bench_Verified", | |
| "ByteDance-Seed/Multi-SWE-bench" | |
| ], | |
| "status": "final (metrics: held-out test projects)", | |
| "metrics": { | |
| "real_auc": 0.4935, | |
| "pair_acc": 0.4221, | |
| "clean_auc": 0.7545, | |
| "bug_auc": 0.668, | |
| "bug_f1": 0.5768, | |
| "per_language": { | |
| "python": { | |
| "real_auc": 0.489, | |
| "pair_acc": 0.4088, | |
| "clean_auc": 0.7216, | |
| "bug_auc": 0.6276, | |
| "n_pairs": 5516, | |
| "n_clean": 8133, | |
| "bug_f1": 0.6007 | |
| }, | |
| "java": { | |
| "real_auc": 0.4721, | |
| "pair_acc": 0.3256, | |
| "clean_auc": 0.8162, | |
| "bug_auc": 0.7633, | |
| "n_pairs": 129, | |
| "n_clean": 709, | |
| "bug_f1": 0.3673 | |
| }, | |
| "cpp": { | |
| "real_auc": 0.5173, | |
| "pair_acc": 0.5717, | |
| "clean_auc": 0.7052, | |
| "bug_auc": 0.6801, | |
| "n_pairs": 572, | |
| "n_clean": 3702, | |
| "bug_f1": 0.4105 | |
| } | |
| }, | |
| "gold_real_auc": 0.4971, | |
| "gold_pair_acc": 0.4735 | |
| }, | |
| "trained_at": "2026-11-04T11:07:17Z", | |
| "validation_metrics": { | |
| "real_auc": 0.4941, | |
| "pair_acc": 0.4197, | |
| "clean_auc": 0.7916, | |
| "bug_auc": 0.6859, | |
| "bug_f1": 0.6451, | |
| "per_language": { | |
| "python": { | |
| "real_auc": 0.491, | |
| "pair_acc": 0.4167, | |
| "clean_auc": 0.7366, | |
| "bug_auc": 0.6138, | |
| "n_pairs": 1500, | |
| "n_clean": 1500, | |
| "bug_f1": 0.6886 | |
| }, | |
| "java": { | |
| "real_auc": 0.4761, | |
| "pair_acc": 0.4071, | |
| "clean_auc": 0.8034, | |
| "bug_auc": 0.7524, | |
| "n_pairs": 113, | |
| "n_clean": 612, | |
| "bug_f1": 0.3169 | |
| }, | |
| "cpp": { | |
| "real_auc": 0.5306, | |
| "pair_acc": 0.4884, | |
| "clean_auc": 0.9166, | |
| "bug_auc": 0.8852, | |
| "n_pairs": 86, | |
| "n_clean": 973, | |
| "bug_f1": 0.4345 | |
| } | |
| } | |
| }, | |
| "gold_metrics": { | |
| "all": { | |
| "real_auc": 0.4971, | |
| "pair_acc": 0.4735, | |
| "clean_auc": 0.5773, | |
| "bug_auc": 0.5467, | |
| "n_pairs": 1491, | |
| "n_clean": 2428, | |
| "bug_f1": 0.4955 | |
| }, | |
| "python": { | |
| "real_auc": 0.4822, | |
| "pair_acc": 0.4045, | |
| "clean_auc": 0.7194, | |
| "bug_auc": 0.6311, | |
| "n_pairs": 660, | |
| "n_clean": 1114, | |
| "bug_f1": 0.5937 | |
| }, | |
| "java": { | |
| "real_auc": 0.4953, | |
| "pair_acc": 0.4149, | |
| "clean_auc": 0.7049, | |
| "bug_auc": 0.6258, | |
| "n_pairs": 188, | |
| "n_clean": 310, | |
| "bug_f1": 0.5371 | |
| }, | |
| "cpp": { | |
| "real_auc": 0.5062, | |
| "pair_acc": 0.5614, | |
| "clean_auc": 0.4768, | |
| "bug_auc": 0.4883, | |
| "n_pairs": 643, | |
| "n_clean": 1004, | |
| "bug_f1": 0.3735 | |
| } | |
| } | |
| } |