reliability_refresh_results.json

{
  "methodology_note": "RQ1-only refresh (ECE + reliability diagram) against whatever checkpoint currently sits at best_{model}.pth -- i.e. the promoted dropout-fix checkpoints for xception/vit, deployed 2026-07-25. Same held-out val split (seed=42, val_fraction=0.15) as every other analysis in this project. RQ2/RQ3 unaffected by the dropout-fix and intentionally not re-run here.",
  "val_frame_count": 11666,
  "val_video_count": 1049,
  "models": {
    "xception": {
      "frame_ece": 0.07310491284879563,
      "video_ece": 0.0832806306958424,
      "frame_auroc": 0.9926626311220669,
      "video_auroc": 0.9983982202447164,
      "frame_reliability": {
        "bin_accuracies": [
          0.8854988001371272,
          0.2261904761904762,
          0.14960629921259844,
          0.208,
          0.05128205128205128,
          0.9420289855072463,
          0.9622641509433962,
          1.0,
          0.9887323943661972,
          1.0
        ],
        "bin_confidences": [
          0.9913699195327876,
          0.8552316016118441,
          0.7487110670391969,
          0.649741117477417,
          0.5505873545622214,
          0.548205548438473,
          0.6523496576075284,
          0.755149797157005,
          0.855771786562154,
          0.9926880709155389
        ],
        "bin_counts": [
          2917,
          168,
          127,
          125,
          117,
          138,
          159,
          189,
          355,
          7045
        ],
        "bin_boundaries": [
          0.0,
          0.1,
          0.2,
          0.30000000000000004,
          0.4,
          0.5,
          0.6000000000000001,
          0.7000000000000001,
          0.8,
          0.9,
          1.0
        ]
      },
      "video_reliability": {
        "bin_accuracies": [
          0.96,
          0.3333333333333333,
          0.0,
          0.125,
          0.0,
          1.0,
          1.0,
          1.0,
          1.0,
          1.0
        ],
        "bin_confidences": [
          0.9878810474176987,
          0.8538837263479946,
          0.7486330546078864,
          0.6525810244027526,
          0.5475465111155062,
          0.5623856821957894,
          0.6473097034453295,
          0.7552529573418257,
          0.8575347691066373,
          0.9847406328305219
        ],
        "bin_counts": [
          150,
          15,
          11,
          8,
          10,
          23,
          34,
          47,
          98,
          650
        ],
        "bin_boundaries": [
          0.0,
          0.1,
          0.2,
          0.30000000000000004,
          0.4,
          0.5,
          0.6000000000000001,
          0.7000000000000001,
          0.8,
          0.9,
          1.0
        ]
      }
    },
    "vit": {
      "frame_ece": 0.07872088314652545,
      "video_ece": 0.10137274672790363,
      "frame_auroc": 0.9715813834827611,
      "video_auroc": 0.9871486837226549,
      "frame_reliability": {
        "bin_accuracies": [
          0.9080404429465575,
          0.5904936014625228,
          0.4224137931034483,
          0.34069400630914826,
          0.22491349480968859,
          0.8299319727891157,
          0.8939393939393939,
          0.9242424242424242,
          0.9603960396039604,
          0.9964504678928686
        ],
        "bin_confidences": [
          0.9660204226349188,
          0.8549974169411868,
          0.7543369423726509,
          0.6503367849892998,
          0.5479899870689352,
          0.5508707601602386,
          0.6460320423949849,
          0.7516925097113908,
          0.8540437360211174,
          0.988450788930371
        ],
        "bin_counts": [
          2077,
          547,
          348,
          317,
          289,
          294,
          330,
          396,
          606,
          6198
        ],
        "bin_boundaries": [
          0.0,
          0.1,
          0.2,
          0.30000000000000004,
          0.4,
          0.5,
          0.6000000000000001,
          0.7000000000000001,
          0.8,
          0.9,
          1.0
        ]
      },
      "video_reliability": {
        "bin_accuracies": [
          0.945054945054945,
          0.8604651162790697,
          0.4411764705882353,
          0.16666666666666666,
          0.08,
          0.8947368421052632,
          1.0,
          1.0,
          1.0,
          0.9983221476510067
        ],
        "bin_confidences": [
          0.9590439275298254,
          0.854219067267543,
          0.7445819370010321,
          0.6437882977661987,
          0.5484745028614998,
          0.5426871372576754,
          0.653532554063832,
          0.7467026308680382,
          0.8560403551440686,
          0.9782306852762983
        ],
        "bin_counts": [
          91,
          43,
          34,
          30,
          25,
          38,
          49,
          52,
          90,
          596
        ],
        "bin_boundaries": [
          0.0,
          0.1,
          0.2,
          0.30000000000000004,
          0.4,
          0.5,
          0.6000000000000001,
          0.7000000000000001,
          0.8,
          0.9,
          1.0
        ]
      }
    }
  }
}