{
  "checkpoint": "artifacts/partial-source-holdout-tinytcn/best.pt",
  "checkpoint_sha256": "4d5286c562fa7d2ccaf3ae3da7e0e09ad370a8b9821d9fd32ed0037972457a1e",
  "data_revision": "e564e2ac567f774d1880aa1db6ce97afb8c519b7",
  "data_scope": "preview-only source-held-out stress: 3265 rows from train-00010-of-00083; official test remains sealed",
  "dataset_revision": null,
  "development_only": true,
  "evaluation_source": {
    "bytes": 2962445,
    "dataset_id": null,
    "kind": "file",
    "path": "data/processed/partial-source-holdout.jsonl",
    "revision": "e564e2ac567f774d1880aa1db6ce97afb8c519b7",
    "sha256": "ab7e65eba9a8130fb94549a9f4ae106169a4db9c9465c3dd2dd38f7f4375e85d"
  },
  "freeze_manifest_sha256": null,
  "grouped_bootstrap_95ci": {
    "false_positive_rate": {
      "estimate": 0.018461538461538463,
      "lower": 0.0,
      "upper": 0.046249495899986545
    },
    "recall": {
      "estimate": 0.04024767801857585,
      "lower": 0.02043705778301887,
      "upper": 0.0955957896857373
    }
  },
  "metrics": {
    "accuracy": 0.5123456790123457,
    "average_precision": 0.5499581204674366,
    "balanced_accuracy": 0.5108930697785187,
    "brier_score": 0.28202905638850445,
    "count": 648,
    "expected_calibration_error": 0.16490815351084795,
    "f1": 0.07602339181286549,
    "false_negative_rate": 0.9597523219814241,
    "false_positive_rate": 0.018461538461538463,
    "fn": 310,
    "fp": 6,
    "log_loss": 0.7782068838177573,
    "macro_f1": 0.37239324726911616,
    "negative_count": 325,
    "positive_count": 323,
    "precision": 0.6842105263157895,
    "recall": 0.04024767801857585,
    "roc_auc": 0.5522743510359609,
    "specificity": 0.9815384615384616,
    "threshold": 0.7510702013969421,
    "tn": 319,
    "tp": 13
  },
  "official_test": false,
  "operating_points": {
    "0.01": {
      "accuracy": 0.5138888888888888,
      "average_precision": 0.5499581204674366,
      "balanced_accuracy": 0.5124124791617052,
      "brier_score": 0.28202905638850445,
      "count": 648,
      "expected_calibration_error": 0.16490815351084795,
      "f1": 0.06528189910979229,
      "false_negative_rate": 0.9659442724458205,
      "false_positive_rate": 0.009230769230769232,
      "fn": 312,
      "fp": 3,
      "log_loss": 0.7782068838177573,
      "macro_f1": 0.36840737291256037,
      "max_false_positive_rate": 0.01,
      "negative_count": 325,
      "positive_count": 323,
      "precision": 0.7857142857142857,
      "recall": 0.034055727554179564,
      "roc_auc": 0.5522743510359609,
      "specificity": 0.9907692307692307,
      "threshold": 0.755566418170929,
      "tn": 322,
      "tp": 11
    },
    "0.02": {
      "accuracy": 0.5123456790123457,
      "average_precision": 0.5499581204674366,
      "balanced_accuracy": 0.5108930697785187,
      "brier_score": 0.28202905638850445,
      "count": 648,
      "expected_calibration_error": 0.16490815351084795,
      "f1": 0.07602339181286549,
      "false_negative_rate": 0.9597523219814241,
      "false_positive_rate": 0.018461538461538463,
      "fn": 310,
      "fp": 6,
      "log_loss": 0.7782068838177573,
      "macro_f1": 0.37239324726911616,
      "max_false_positive_rate": 0.02,
      "negative_count": 325,
      "positive_count": 323,
      "precision": 0.6842105263157895,
      "recall": 0.04024767801857585,
      "roc_auc": 0.5522743510359609,
      "specificity": 0.9815384615384616,
      "threshold": 0.7510702013969421,
      "tn": 319,
      "tp": 13
    },
    "0.05": {
      "accuracy": 0.5185185185185185,
      "average_precision": 0.5499581204674366,
      "balanced_accuracy": 0.5171802810192904,
      "brier_score": 0.28202905638850445,
      "count": 648,
      "expected_calibration_error": 0.16490815351084795,
      "f1": 0.14754098360655737,
      "false_negative_rate": 0.9164086687306502,
      "false_positive_rate": 0.04923076923076923,
      "fn": 296,
      "fp": 16,
      "log_loss": 0.7782068838177573,
      "macro_f1": 0.4060285563194077,
      "max_false_positive_rate": 0.05,
      "negative_count": 325,
      "positive_count": 323,
      "precision": 0.627906976744186,
      "recall": 0.08359133126934984,
      "roc_auc": 0.5522743510359609,
      "specificity": 0.9507692307692308,
      "threshold": 0.727832555770874,
      "tn": 309,
      "tp": 27
    }
  },
  "operating_points_note": "Label-dependent threshold sweep for curve analysis only. The deployed result is `metrics` at the checkpoint's frozen validation-selected threshold.",
  "operational": {
    "accuracy": 0.5123456790123457,
    "audio_hours": 1.552732013888889,
    "average_precision": 0.5499581204674366,
    "balanced_accuracy": 0.5108930697785187,
    "brier_score": 0.28202905638850445,
    "count": 648,
    "expected_calibration_error": 0.16490815351084795,
    "f1": 0.07602339181286549,
    "false_interruptions": 6,
    "false_interruptions_per_hour": 3.8641568192908724,
    "false_negative_rate": 0.9597523219814241,
    "false_positive_rate": 0.018461538461538463,
    "fn": 310,
    "fp": 6,
    "log_loss": 0.7782068838177573,
    "macro_f1": 0.37239324726911616,
    "negative_count": 325,
    "positive_count": 323,
    "precision": 0.6842105263157895,
    "recall": 0.04024767801857585,
    "roc_auc": 0.5522743510359609,
    "scope": "independent labeled clips; per-hour rate is a clip-normalized proxy, not an online conversation measurement",
    "sequence_metrics_available": false,
    "sequence_metrics_unavailable_reason": "source records contain no genuine turn_id or conversation_id",
    "specificity": 0.9815384615384616,
    "threshold": 0.7510702013969421,
    "tn": 319,
    "tp": 13
  },
  "reliability": [
    {
      "count": 32,
      "lower": 0.0,
      "mean_probability": 0.08213151805102825,
      "observed_rate": 0.21875,
      "upper": 0.1
    },
    {
      "count": 116,
      "lower": 0.1,
      "mean_probability": 0.15063592823671884,
      "observed_rate": 0.5,
      "upper": 0.2
    },
    {
      "count": 84,
      "lower": 0.2,
      "mean_probability": 0.25053229023303303,
      "observed_rate": 0.5,
      "upper": 0.3
    },
    {
      "count": 73,
      "lower": 0.3,
      "mean_probability": 0.3514139423631642,
      "observed_rate": 0.4931506849315068,
      "upper": 0.4
    },
    {
      "count": 102,
      "lower": 0.4,
      "mean_probability": 0.4513314664363861,
      "observed_rate": 0.5196078431372549,
      "upper": 0.5
    },
    {
      "count": 108,
      "lower": 0.5,
      "mean_probability": 0.5456479418056982,
      "observed_rate": 0.48148148148148145,
      "upper": 0.6
    },
    {
      "count": 73,
      "lower": 0.6,
      "mean_probability": 0.6492549148324418,
      "observed_rate": 0.547945205479452,
      "upper": 0.7
    },
    {
      "count": 60,
      "lower": 0.7,
      "mean_probability": 0.7395102163155873,
      "observed_rate": 0.5833333333333334,
      "upper": 0.8
    },
    {
      "count": 0,
      "lower": 0.8,
      "mean_probability": 0.0,
      "observed_rate": 0.0,
      "upper": 0.9
    },
    {
      "count": 0,
      "lower": 0.9,
      "mean_probability": 0.0,
      "observed_rate": 0.0,
      "upper": 1.0
    }
  ],
  "slices": {
    "dataset": {
      "human_5": {
        "accuracy": 0.5319148936170213,
        "average_precision": 0.7346027845874109,
        "balanced_accuracy": 0.56,
        "brier_score": 0.2237639915125358,
        "count": 47,
        "expected_calibration_error": 0.11744399448024466,
        "f1": 0.21428571428571427,
        "false_negative_rate": 0.88,
        "false_positive_rate": 0.0,
        "fn": 22,
        "fp": 0,
        "log_loss": 0.6315959169824472,
        "macro_f1": 0.44047619047619047,
        "negative_count": 22,
        "positive_count": 25,
        "precision": 1.0,
        "recall": 0.12,
        "roc_auc": 0.6727272727272727,
        "specificity": 1.0,
        "threshold": 0.7510702013969421,
        "tn": 22,
        "tp": 3
      },
      "liva_1": {
        "accuracy": 0.511049723756906,
        "average_precision": 0.534675734736703,
        "balanced_accuracy": 0.4976173020527859,
        "brier_score": 0.28168741657223345,
        "count": 362,
        "expected_calibration_error": 0.1480350368836308,
        "f1": 0.022099447513812154,
        "false_negative_rate": 0.9886363636363636,
        "false_positive_rate": 0.016129032258064516,
        "fn": 174,
        "fp": 3,
        "log_loss": 0.7808010075770082,
        "macro_f1": 0.34806629834254144,
        "negative_count": 186,
        "positive_count": 176,
        "precision": 0.4,
        "recall": 0.011363636363636364,
        "roc_auc": 0.5610031769305963,
        "specificity": 0.9838709677419355,
        "threshold": 0.7510702013969421,
        "tn": 183,
        "tp": 2
      },
      "midcentury_1": {
        "accuracy": 0.5204081632653061,
        "average_precision": 0.49467764090184185,
        "balanced_accuracy": 0.53,
        "brier_score": 0.33650612559098597,
        "count": 98,
        "expected_calibration_error": 0.33408278599381447,
        "f1": 0.11320754716981132,
        "false_negative_rate": 0.94,
        "false_positive_rate": 0.0,
        "fn": 47,
        "fp": 0,
        "log_loss": 0.8977698354532542,
        "macro_f1": 0.3922681092492413,
        "negative_count": 48,
        "positive_count": 50,
        "precision": 1.0,
        "recall": 0.06,
        "roc_auc": 0.36125,
        "specificity": 1.0,
        "threshold": 0.7510702013969421,
        "tn": 48,
        "tp": 3
      },
      "mundo_1": {
        "accuracy": 0.5581395348837209,
        "average_precision": 0.43688015249152684,
        "balanced_accuracy": 0.5054824561403509,
        "brier_score": 0.31313111102791236,
        "count": 43,
        "expected_calibration_error": 0.2613871014395425,
        "f1": 0.09523809523809523,
        "false_negative_rate": 0.9473684210526315,
        "false_positive_rate": 0.041666666666666664,
        "fn": 18,
        "fp": 1,
        "log_loss": 0.835991683286482,
        "macro_f1": 0.4014652014652015,
        "negative_count": 24,
        "positive_count": 19,
        "precision": 0.5,
        "recall": 0.05263157894736842,
        "roc_auc": 0.43859649122807015,
        "specificity": 0.9583333333333334,
        "threshold": 0.7510702013969421,
        "tn": 23,
        "tp": 1
      },
      "rime_2": {
        "accuracy": 0.46511627906976744,
        "average_precision": 0.5809101598141188,
        "balanced_accuracy": 0.4751082251082251,
        "brier_score": 0.3177046643212644,
        "count": 43,
        "expected_calibration_error": 0.25279066001260003,
        "f1": 0.08,
        "false_negative_rate": 0.9545454545454546,
        "false_positive_rate": 0.09523809523809523,
        "fn": 21,
        "fp": 2,
        "log_loss": 0.8849156531870208,
        "macro_f1": 0.35147540983606557,
        "negative_count": 21,
        "positive_count": 22,
        "precision": 0.3333333333333333,
        "recall": 0.045454545454545456,
        "roc_auc": 0.5584415584415584,
        "specificity": 0.9047619047619048,
        "threshold": 0.7510702013969421,
        "tn": 19,
        "tp": 1
      }
    },
    "duration_bin": {
      "2-4s": {
        "accuracy": 0.5210084033613446,
        "average_precision": 0.6060288083109266,
        "balanced_accuracy": 0.5382003395585738,
        "brier_score": 0.27422464350587894,
        "count": 119,
        "expected_calibration_error": 0.17818899408859362,
        "f1": 0.2191780821917808,
        "false_negative_rate": 0.8709677419354839,
        "false_positive_rate": 0.05263157894736842,
        "fn": 54,
        "fp": 3,
        "log_loss": 0.7509756278070072,
        "macro_f1": 0.4368617683686177,
        "negative_count": 57,
        "positive_count": 62,
        "precision": 0.7272727272727273,
        "recall": 0.12903225806451613,
        "roc_auc": 0.5464063384267119,
        "specificity": 0.9473684210526315,
        "threshold": 0.7510702013969421,
        "tn": 54,
        "tp": 8
      },
      "4-8s": {
        "accuracy": 0.5058823529411764,
        "average_precision": 0.5215388633267756,
        "balanced_accuracy": 0.5058823529411764,
        "brier_score": 0.29757245936805765,
        "count": 170,
        "expected_calibration_error": 0.22203312945716525,
        "f1": 0.045454545454545456,
        "false_negative_rate": 0.9764705882352941,
        "false_positive_rate": 0.011764705882352941,
        "fn": 83,
        "fp": 1,
        "log_loss": 0.8124832159457299,
        "macro_f1": 0.356060606060606,
        "negative_count": 85,
        "positive_count": 85,
        "precision": 0.6666666666666666,
        "recall": 0.023529411764705882,
        "roc_auc": 0.5038062283737024,
        "specificity": 0.9882352941176471,
        "threshold": 0.7510702013969421,
        "tn": 84,
        "tp": 2
      },
      ">=8s": {
        "accuracy": 0.5115606936416185,
        "average_precision": 0.5595854046358658,
        "balanced_accuracy": 0.5002674422491893,
        "brier_score": 0.27696240306064007,
        "count": 346,
        "expected_calibration_error": 0.14117037789153225,
        "f1": 0.023121387283236993,
        "false_negative_rate": 0.9881656804733728,
        "false_positive_rate": 0.011299435028248588,
        "fn": 167,
        "fp": 2,
        "log_loss": 0.7709613916460462,
        "macro_f1": 0.348747591522158,
        "negative_count": 177,
        "positive_count": 169,
        "precision": 0.5,
        "recall": 0.011834319526627219,
        "roc_auc": 0.5790124694948684,
        "specificity": 0.9887005649717514,
        "threshold": 0.7510702013969421,
        "tn": 175,
        "tp": 2
      }
    },
    "filler_type": {
      "unknown": {
        "accuracy": 0.5117370892018779,
        "average_precision": 0.5459168650210282,
        "balanced_accuracy": 0.5095221301357785,
        "brier_score": 0.28353119603052307,
        "count": 639,
        "expected_calibration_error": 0.17034158953618558,
        "f1": 0.07142857142857142,
        "false_negative_rate": 0.9622641509433962,
        "false_positive_rate": 0.018691588785046728,
        "fn": 306,
        "fp": 6,
        "log_loss": 0.7816831013535286,
        "macro_f1": 0.3701091901728844,
        "negative_count": 321,
        "positive_count": 318,
        "precision": 0.6666666666666666,
        "recall": 0.03773584905660377,
        "roc_auc": 0.5477869864221477,
        "specificity": 0.9813084112149533,
        "threshold": 0.7510702013969421,
        "tn": 315,
        "tp": 12
      }
    },
    "language": {
      "eng": {
        "accuracy": 0.509090909090909,
        "average_precision": 0.5689565954839015,
        "balanced_accuracy": 0.5114311942647316,
        "brier_score": 0.27981849713314155,
        "count": 605,
        "expected_calibration_error": 0.1584163401860836,
        "f1": 0.07476635514018691,
        "false_negative_rate": 0.9605263157894737,
        "false_positive_rate": 0.016611295681063124,
        "fn": 292,
        "fp": 5,
        "log_loss": 0.774099865012542,
        "macro_f1": 0.37034155777256816,
        "negative_count": 301,
        "positive_count": 304,
        "precision": 0.7058823529411765,
        "recall": 0.039473684210526314,
        "roc_auc": 0.5654288337121874,
        "specificity": 0.9833887043189369,
        "threshold": 0.7510702013969421,
        "tn": 296,
        "tp": 12
      },
      "spa": {
        "accuracy": 0.5581395348837209,
        "average_precision": 0.43688015249152684,
        "balanced_accuracy": 0.5054824561403509,
        "brier_score": 0.31313111102791236,
        "count": 43,
        "expected_calibration_error": 0.2613871014395425,
        "f1": 0.09523809523809523,
        "false_negative_rate": 0.9473684210526315,
        "false_positive_rate": 0.041666666666666664,
        "fn": 18,
        "fp": 1,
        "log_loss": 0.835991683286482,
        "macro_f1": 0.4014652014652015,
        "negative_count": 24,
        "positive_count": 19,
        "precision": 0.5,
        "recall": 0.05263157894736842,
        "roc_auc": 0.43859649122807015,
        "specificity": 0.9583333333333334,
        "threshold": 0.7510702013969421,
        "tn": 23,
        "tp": 1
      }
    },
    "synthetic": {
      "False": {
        "accuracy": 0.5181818181818182,
        "average_precision": 0.5245109263654216,
        "balanced_accuracy": 0.5095238095238096,
        "brier_score": 0.28896361905882645,
        "count": 550,
        "expected_calibration_error": 0.18397142849185252,
        "f1": 0.0636042402826855,
        "false_negative_rate": 0.9666666666666667,
        "false_positive_rate": 0.014285714285714285,
        "fn": 261,
        "fp": 4,
        "log_loss": 0.7932073801759811,
        "macro_f1": 0.36962341757096334,
        "negative_count": 280,
        "positive_count": 270,
        "precision": 0.6923076923076923,
        "recall": 0.03333333333333333,
        "roc_auc": 0.5231084656084656,
        "specificity": 0.9857142857142858,
        "threshold": 0.7510702013969421,
        "tn": 276,
        "tp": 9
      },
      "True": {
        "accuracy": 0.47959183673469385,
        "average_precision": 0.6790413484001439,
        "balanced_accuracy": 0.5155136268343816,
        "brier_score": 0.24311059242241156,
        "count": 98,
        "expected_calibration_error": 0.15580201186999984,
        "f1": 0.13559322033898305,
        "false_negative_rate": 0.9245283018867925,
        "false_positive_rate": 0.044444444444444446,
        "fn": 49,
        "fp": 2,
        "log_loss": 0.6940204246644598,
        "macro_f1": 0.38166522330817765,
        "negative_count": 45,
        "positive_count": 53,
        "precision": 0.6666666666666666,
        "recall": 0.07547169811320754,
        "roc_auc": 0.680083857442348,
        "specificity": 0.9555555555555556,
        "threshold": 0.7510702013969421,
        "tn": 43,
        "tp": 4
      }
    }
  },
  "source_split": "validation",
  "split": "validation",
  "threshold": 0.7510702013969421,
  "training_status": "preview-only",
  "worst_groups": {
    "highest_false_positive_rate": {
      "count": 43,
      "dimension": "dataset",
      "false_positive_rate": 0.09523809523809523,
      "value": "rime_2"
    },
    "lowest_recall": {
      "count": 362,
      "dimension": "dataset",
      "recall": 0.011363636363636364,
      "value": "liva_1"
    }
  }
}