{
  "format_version": 2,
  "base_model": "answerdotai/ModernBERT-base",
  "labels": [
    "clean",
    "buggy"
  ],
  "clean_label": "clean",
  "max_length": 512,
  "languages": [
    "python",
    "java",
    "cpp"
  ],
  "normalization": {
    "python": "ast_unparse_strip_docstrings_v1",
    "java": "c_family_strip_comments_v1",
    "cpp": "c_family_strip_comments_v1"
  },
  "objective": "binary classification (buggy vs unchanged) + pairwise ranking (buggy above its fix)",
  "datasets": [
    "nebius/SWE-rebench",
    "princeton-nlp/SWE-bench (train)",
    "nebius/SWE-bench-extra",
    "SWE-Gym/SWE-Gym",
    "ByteDance-Seed/Multi-SWE-RL",
    "bigcode/commitpackft",
    "ASSERT-KTH/megadiff-single-function",
    "bstee615/bigvul"
  ],
  "gold_test": [
    "princeton-nlp/SWE-bench_Verified",
    "ByteDance-Seed/Multi-SWE-bench"
  ],
  "status": "final (metrics: held-out test projects)",
  "metrics": {
    "real_auc": 0.4935,
    "pair_acc": 0.4221,
    "clean_auc": 0.7545,
    "bug_auc": 0.668,
    "bug_f1": 0.5768,
    "per_language": {
      "python": {
        "real_auc": 0.489,
        "pair_acc": 0.4088,
        "clean_auc": 0.7216,
        "bug_auc": 0.6276,
        "n_pairs": 5516,
        "n_clean": 8133,
        "bug_f1": 0.6007
      },
      "java": {
        "real_auc": 0.4721,
        "pair_acc": 0.3256,
        "clean_auc": 0.8162,
        "bug_auc": 0.7633,
        "n_pairs": 129,
        "n_clean": 709,
        "bug_f1": 0.3673
      },
      "cpp": {
        "real_auc": 0.5173,
        "pair_acc": 0.5717,
        "clean_auc": 0.7052,
        "bug_auc": 0.6801,
        "n_pairs": 572,
        "n_clean": 3702,
        "bug_f1": 0.4105
      }
    },
    "gold_real_auc": 0.4971,
    "gold_pair_acc": 0.4735
  },
  "trained_at": "2026-11-04T11:07:17Z",
  "validation_metrics": {
    "real_auc": 0.4941,
    "pair_acc": 0.4197,
    "clean_auc": 0.7916,
    "bug_auc": 0.6859,
    "bug_f1": 0.6451,
    "per_language": {
      "python": {
        "real_auc": 0.491,
        "pair_acc": 0.4167,
        "clean_auc": 0.7366,
        "bug_auc": 0.6138,
        "n_pairs": 1500,
        "n_clean": 1500,
        "bug_f1": 0.6886
      },
      "java": {
        "real_auc": 0.4761,
        "pair_acc": 0.4071,
        "clean_auc": 0.8034,
        "bug_auc": 0.7524,
        "n_pairs": 113,
        "n_clean": 612,
        "bug_f1": 0.3169
      },
      "cpp": {
        "real_auc": 0.5306,
        "pair_acc": 0.4884,
        "clean_auc": 0.9166,
        "bug_auc": 0.8852,
        "n_pairs": 86,
        "n_clean": 973,
        "bug_f1": 0.4345
      }
    }
  },
  "gold_metrics": {
    "all": {
      "real_auc": 0.4971,
      "pair_acc": 0.4735,
      "clean_auc": 0.5773,
      "bug_auc": 0.5467,
      "n_pairs": 1491,
      "n_clean": 2428,
      "bug_f1": 0.4955
    },
    "python": {
      "real_auc": 0.4822,
      "pair_acc": 0.4045,
      "clean_auc": 0.7194,
      "bug_auc": 0.6311,
      "n_pairs": 660,
      "n_clean": 1114,
      "bug_f1": 0.5937
    },
    "java": {
      "real_auc": 0.4953,
      "pair_acc": 0.4149,
      "clean_auc": 0.7049,
      "bug_auc": 0.6258,
      "n_pairs": 188,
      "n_clean": 310,
      "bug_f1": 0.5371
    },
    "cpp": {
      "real_auc": 0.5062,
      "pair_acc": 0.5614,
      "clean_auc": 0.4768,
      "bug_auc": 0.4883,
      "n_pairs": 643,
      "n_clean": 1004,
      "bug_f1": 0.3735
    }
  }
}