| { |
| "checkpoint": "artifacts/partial-source-holdout-tinytcn/best.pt", |
| "checkpoint_sha256": "4d5286c562fa7d2ccaf3ae3da7e0e09ad370a8b9821d9fd32ed0037972457a1e", |
| "data_revision": "e564e2ac567f774d1880aa1db6ce97afb8c519b7", |
| "data_scope": "preview-only source-held-out stress: 3265 rows from train-00010-of-00083; official test remains sealed", |
| "dataset_revision": null, |
| "development_only": true, |
| "evaluation_source": { |
| "bytes": 2962445, |
| "dataset_id": null, |
| "kind": "file", |
| "path": "data/processed/partial-source-holdout.jsonl", |
| "revision": "e564e2ac567f774d1880aa1db6ce97afb8c519b7", |
| "sha256": "ab7e65eba9a8130fb94549a9f4ae106169a4db9c9465c3dd2dd38f7f4375e85d" |
| }, |
| "freeze_manifest_sha256": null, |
| "grouped_bootstrap_95ci": { |
| "false_positive_rate": { |
| "estimate": 0.018461538461538463, |
| "lower": 0.0, |
| "upper": 0.046249495899986545 |
| }, |
| "recall": { |
| "estimate": 0.04024767801857585, |
| "lower": 0.02043705778301887, |
| "upper": 0.0955957896857373 |
| } |
| }, |
| "metrics": { |
| "accuracy": 0.5123456790123457, |
| "average_precision": 0.5499581204674366, |
| "balanced_accuracy": 0.5108930697785187, |
| "brier_score": 0.28202905638850445, |
| "count": 648, |
| "expected_calibration_error": 0.16490815351084795, |
| "f1": 0.07602339181286549, |
| "false_negative_rate": 0.9597523219814241, |
| "false_positive_rate": 0.018461538461538463, |
| "fn": 310, |
| "fp": 6, |
| "log_loss": 0.7782068838177573, |
| "macro_f1": 0.37239324726911616, |
| "negative_count": 325, |
| "positive_count": 323, |
| "precision": 0.6842105263157895, |
| "recall": 0.04024767801857585, |
| "roc_auc": 0.5522743510359609, |
| "specificity": 0.9815384615384616, |
| "threshold": 0.7510702013969421, |
| "tn": 319, |
| "tp": 13 |
| }, |
| "official_test": false, |
| "operating_points": { |
| "0.01": { |
| "accuracy": 0.5138888888888888, |
| "average_precision": 0.5499581204674366, |
| "balanced_accuracy": 0.5124124791617052, |
| "brier_score": 0.28202905638850445, |
| "count": 648, |
| "expected_calibration_error": 0.16490815351084795, |
| "f1": 0.06528189910979229, |
| "false_negative_rate": 0.9659442724458205, |
| "false_positive_rate": 0.009230769230769232, |
| "fn": 312, |
| "fp": 3, |
| "log_loss": 0.7782068838177573, |
| "macro_f1": 0.36840737291256037, |
| "max_false_positive_rate": 0.01, |
| "negative_count": 325, |
| "positive_count": 323, |
| "precision": 0.7857142857142857, |
| "recall": 0.034055727554179564, |
| "roc_auc": 0.5522743510359609, |
| "specificity": 0.9907692307692307, |
| "threshold": 0.755566418170929, |
| "tn": 322, |
| "tp": 11 |
| }, |
| "0.02": { |
| "accuracy": 0.5123456790123457, |
| "average_precision": 0.5499581204674366, |
| "balanced_accuracy": 0.5108930697785187, |
| "brier_score": 0.28202905638850445, |
| "count": 648, |
| "expected_calibration_error": 0.16490815351084795, |
| "f1": 0.07602339181286549, |
| "false_negative_rate": 0.9597523219814241, |
| "false_positive_rate": 0.018461538461538463, |
| "fn": 310, |
| "fp": 6, |
| "log_loss": 0.7782068838177573, |
| "macro_f1": 0.37239324726911616, |
| "max_false_positive_rate": 0.02, |
| "negative_count": 325, |
| "positive_count": 323, |
| "precision": 0.6842105263157895, |
| "recall": 0.04024767801857585, |
| "roc_auc": 0.5522743510359609, |
| "specificity": 0.9815384615384616, |
| "threshold": 0.7510702013969421, |
| "tn": 319, |
| "tp": 13 |
| }, |
| "0.05": { |
| "accuracy": 0.5185185185185185, |
| "average_precision": 0.5499581204674366, |
| "balanced_accuracy": 0.5171802810192904, |
| "brier_score": 0.28202905638850445, |
| "count": 648, |
| "expected_calibration_error": 0.16490815351084795, |
| "f1": 0.14754098360655737, |
| "false_negative_rate": 0.9164086687306502, |
| "false_positive_rate": 0.04923076923076923, |
| "fn": 296, |
| "fp": 16, |
| "log_loss": 0.7782068838177573, |
| "macro_f1": 0.4060285563194077, |
| "max_false_positive_rate": 0.05, |
| "negative_count": 325, |
| "positive_count": 323, |
| "precision": 0.627906976744186, |
| "recall": 0.08359133126934984, |
| "roc_auc": 0.5522743510359609, |
| "specificity": 0.9507692307692308, |
| "threshold": 0.727832555770874, |
| "tn": 309, |
| "tp": 27 |
| } |
| }, |
| "operating_points_note": "Label-dependent threshold sweep for curve analysis only. The deployed result is `metrics` at the checkpoint's frozen validation-selected threshold.", |
| "operational": { |
| "accuracy": 0.5123456790123457, |
| "audio_hours": 1.552732013888889, |
| "average_precision": 0.5499581204674366, |
| "balanced_accuracy": 0.5108930697785187, |
| "brier_score": 0.28202905638850445, |
| "count": 648, |
| "expected_calibration_error": 0.16490815351084795, |
| "f1": 0.07602339181286549, |
| "false_interruptions": 6, |
| "false_interruptions_per_hour": 3.8641568192908724, |
| "false_negative_rate": 0.9597523219814241, |
| "false_positive_rate": 0.018461538461538463, |
| "fn": 310, |
| "fp": 6, |
| "log_loss": 0.7782068838177573, |
| "macro_f1": 0.37239324726911616, |
| "negative_count": 325, |
| "positive_count": 323, |
| "precision": 0.6842105263157895, |
| "recall": 0.04024767801857585, |
| "roc_auc": 0.5522743510359609, |
| "scope": "independent labeled clips; per-hour rate is a clip-normalized proxy, not an online conversation measurement", |
| "sequence_metrics_available": false, |
| "sequence_metrics_unavailable_reason": "source records contain no genuine turn_id or conversation_id", |
| "specificity": 0.9815384615384616, |
| "threshold": 0.7510702013969421, |
| "tn": 319, |
| "tp": 13 |
| }, |
| "reliability": [ |
| { |
| "count": 32, |
| "lower": 0.0, |
| "mean_probability": 0.08213151805102825, |
| "observed_rate": 0.21875, |
| "upper": 0.1 |
| }, |
| { |
| "count": 116, |
| "lower": 0.1, |
| "mean_probability": 0.15063592823671884, |
| "observed_rate": 0.5, |
| "upper": 0.2 |
| }, |
| { |
| "count": 84, |
| "lower": 0.2, |
| "mean_probability": 0.25053229023303303, |
| "observed_rate": 0.5, |
| "upper": 0.3 |
| }, |
| { |
| "count": 73, |
| "lower": 0.3, |
| "mean_probability": 0.3514139423631642, |
| "observed_rate": 0.4931506849315068, |
| "upper": 0.4 |
| }, |
| { |
| "count": 102, |
| "lower": 0.4, |
| "mean_probability": 0.4513314664363861, |
| "observed_rate": 0.5196078431372549, |
| "upper": 0.5 |
| }, |
| { |
| "count": 108, |
| "lower": 0.5, |
| "mean_probability": 0.5456479418056982, |
| "observed_rate": 0.48148148148148145, |
| "upper": 0.6 |
| }, |
| { |
| "count": 73, |
| "lower": 0.6, |
| "mean_probability": 0.6492549148324418, |
| "observed_rate": 0.547945205479452, |
| "upper": 0.7 |
| }, |
| { |
| "count": 60, |
| "lower": 0.7, |
| "mean_probability": 0.7395102163155873, |
| "observed_rate": 0.5833333333333334, |
| "upper": 0.8 |
| }, |
| { |
| "count": 0, |
| "lower": 0.8, |
| "mean_probability": 0.0, |
| "observed_rate": 0.0, |
| "upper": 0.9 |
| }, |
| { |
| "count": 0, |
| "lower": 0.9, |
| "mean_probability": 0.0, |
| "observed_rate": 0.0, |
| "upper": 1.0 |
| } |
| ], |
| "slices": { |
| "dataset": { |
| "human_5": { |
| "accuracy": 0.5319148936170213, |
| "average_precision": 0.7346027845874109, |
| "balanced_accuracy": 0.56, |
| "brier_score": 0.2237639915125358, |
| "count": 47, |
| "expected_calibration_error": 0.11744399448024466, |
| "f1": 0.21428571428571427, |
| "false_negative_rate": 0.88, |
| "false_positive_rate": 0.0, |
| "fn": 22, |
| "fp": 0, |
| "log_loss": 0.6315959169824472, |
| "macro_f1": 0.44047619047619047, |
| "negative_count": 22, |
| "positive_count": 25, |
| "precision": 1.0, |
| "recall": 0.12, |
| "roc_auc": 0.6727272727272727, |
| "specificity": 1.0, |
| "threshold": 0.7510702013969421, |
| "tn": 22, |
| "tp": 3 |
| }, |
| "liva_1": { |
| "accuracy": 0.511049723756906, |
| "average_precision": 0.534675734736703, |
| "balanced_accuracy": 0.4976173020527859, |
| "brier_score": 0.28168741657223345, |
| "count": 362, |
| "expected_calibration_error": 0.1480350368836308, |
| "f1": 0.022099447513812154, |
| "false_negative_rate": 0.9886363636363636, |
| "false_positive_rate": 0.016129032258064516, |
| "fn": 174, |
| "fp": 3, |
| "log_loss": 0.7808010075770082, |
| "macro_f1": 0.34806629834254144, |
| "negative_count": 186, |
| "positive_count": 176, |
| "precision": 0.4, |
| "recall": 0.011363636363636364, |
| "roc_auc": 0.5610031769305963, |
| "specificity": 0.9838709677419355, |
| "threshold": 0.7510702013969421, |
| "tn": 183, |
| "tp": 2 |
| }, |
| "midcentury_1": { |
| "accuracy": 0.5204081632653061, |
| "average_precision": 0.49467764090184185, |
| "balanced_accuracy": 0.53, |
| "brier_score": 0.33650612559098597, |
| "count": 98, |
| "expected_calibration_error": 0.33408278599381447, |
| "f1": 0.11320754716981132, |
| "false_negative_rate": 0.94, |
| "false_positive_rate": 0.0, |
| "fn": 47, |
| "fp": 0, |
| "log_loss": 0.8977698354532542, |
| "macro_f1": 0.3922681092492413, |
| "negative_count": 48, |
| "positive_count": 50, |
| "precision": 1.0, |
| "recall": 0.06, |
| "roc_auc": 0.36125, |
| "specificity": 1.0, |
| "threshold": 0.7510702013969421, |
| "tn": 48, |
| "tp": 3 |
| }, |
| "mundo_1": { |
| "accuracy": 0.5581395348837209, |
| "average_precision": 0.43688015249152684, |
| "balanced_accuracy": 0.5054824561403509, |
| "brier_score": 0.31313111102791236, |
| "count": 43, |
| "expected_calibration_error": 0.2613871014395425, |
| "f1": 0.09523809523809523, |
| "false_negative_rate": 0.9473684210526315, |
| "false_positive_rate": 0.041666666666666664, |
| "fn": 18, |
| "fp": 1, |
| "log_loss": 0.835991683286482, |
| "macro_f1": 0.4014652014652015, |
| "negative_count": 24, |
| "positive_count": 19, |
| "precision": 0.5, |
| "recall": 0.05263157894736842, |
| "roc_auc": 0.43859649122807015, |
| "specificity": 0.9583333333333334, |
| "threshold": 0.7510702013969421, |
| "tn": 23, |
| "tp": 1 |
| }, |
| "rime_2": { |
| "accuracy": 0.46511627906976744, |
| "average_precision": 0.5809101598141188, |
| "balanced_accuracy": 0.4751082251082251, |
| "brier_score": 0.3177046643212644, |
| "count": 43, |
| "expected_calibration_error": 0.25279066001260003, |
| "f1": 0.08, |
| "false_negative_rate": 0.9545454545454546, |
| "false_positive_rate": 0.09523809523809523, |
| "fn": 21, |
| "fp": 2, |
| "log_loss": 0.8849156531870208, |
| "macro_f1": 0.35147540983606557, |
| "negative_count": 21, |
| "positive_count": 22, |
| "precision": 0.3333333333333333, |
| "recall": 0.045454545454545456, |
| "roc_auc": 0.5584415584415584, |
| "specificity": 0.9047619047619048, |
| "threshold": 0.7510702013969421, |
| "tn": 19, |
| "tp": 1 |
| } |
| }, |
| "duration_bin": { |
| "2-4s": { |
| "accuracy": 0.5210084033613446, |
| "average_precision": 0.6060288083109266, |
| "balanced_accuracy": 0.5382003395585738, |
| "brier_score": 0.27422464350587894, |
| "count": 119, |
| "expected_calibration_error": 0.17818899408859362, |
| "f1": 0.2191780821917808, |
| "false_negative_rate": 0.8709677419354839, |
| "false_positive_rate": 0.05263157894736842, |
| "fn": 54, |
| "fp": 3, |
| "log_loss": 0.7509756278070072, |
| "macro_f1": 0.4368617683686177, |
| "negative_count": 57, |
| "positive_count": 62, |
| "precision": 0.7272727272727273, |
| "recall": 0.12903225806451613, |
| "roc_auc": 0.5464063384267119, |
| "specificity": 0.9473684210526315, |
| "threshold": 0.7510702013969421, |
| "tn": 54, |
| "tp": 8 |
| }, |
| "4-8s": { |
| "accuracy": 0.5058823529411764, |
| "average_precision": 0.5215388633267756, |
| "balanced_accuracy": 0.5058823529411764, |
| "brier_score": 0.29757245936805765, |
| "count": 170, |
| "expected_calibration_error": 0.22203312945716525, |
| "f1": 0.045454545454545456, |
| "false_negative_rate": 0.9764705882352941, |
| "false_positive_rate": 0.011764705882352941, |
| "fn": 83, |
| "fp": 1, |
| "log_loss": 0.8124832159457299, |
| "macro_f1": 0.356060606060606, |
| "negative_count": 85, |
| "positive_count": 85, |
| "precision": 0.6666666666666666, |
| "recall": 0.023529411764705882, |
| "roc_auc": 0.5038062283737024, |
| "specificity": 0.9882352941176471, |
| "threshold": 0.7510702013969421, |
| "tn": 84, |
| "tp": 2 |
| }, |
| ">=8s": { |
| "accuracy": 0.5115606936416185, |
| "average_precision": 0.5595854046358658, |
| "balanced_accuracy": 0.5002674422491893, |
| "brier_score": 0.27696240306064007, |
| "count": 346, |
| "expected_calibration_error": 0.14117037789153225, |
| "f1": 0.023121387283236993, |
| "false_negative_rate": 0.9881656804733728, |
| "false_positive_rate": 0.011299435028248588, |
| "fn": 167, |
| "fp": 2, |
| "log_loss": 0.7709613916460462, |
| "macro_f1": 0.348747591522158, |
| "negative_count": 177, |
| "positive_count": 169, |
| "precision": 0.5, |
| "recall": 0.011834319526627219, |
| "roc_auc": 0.5790124694948684, |
| "specificity": 0.9887005649717514, |
| "threshold": 0.7510702013969421, |
| "tn": 175, |
| "tp": 2 |
| } |
| }, |
| "filler_type": { |
| "unknown": { |
| "accuracy": 0.5117370892018779, |
| "average_precision": 0.5459168650210282, |
| "balanced_accuracy": 0.5095221301357785, |
| "brier_score": 0.28353119603052307, |
| "count": 639, |
| "expected_calibration_error": 0.17034158953618558, |
| "f1": 0.07142857142857142, |
| "false_negative_rate": 0.9622641509433962, |
| "false_positive_rate": 0.018691588785046728, |
| "fn": 306, |
| "fp": 6, |
| "log_loss": 0.7816831013535286, |
| "macro_f1": 0.3701091901728844, |
| "negative_count": 321, |
| "positive_count": 318, |
| "precision": 0.6666666666666666, |
| "recall": 0.03773584905660377, |
| "roc_auc": 0.5477869864221477, |
| "specificity": 0.9813084112149533, |
| "threshold": 0.7510702013969421, |
| "tn": 315, |
| "tp": 12 |
| } |
| }, |
| "language": { |
| "eng": { |
| "accuracy": 0.509090909090909, |
| "average_precision": 0.5689565954839015, |
| "balanced_accuracy": 0.5114311942647316, |
| "brier_score": 0.27981849713314155, |
| "count": 605, |
| "expected_calibration_error": 0.1584163401860836, |
| "f1": 0.07476635514018691, |
| "false_negative_rate": 0.9605263157894737, |
| "false_positive_rate": 0.016611295681063124, |
| "fn": 292, |
| "fp": 5, |
| "log_loss": 0.774099865012542, |
| "macro_f1": 0.37034155777256816, |
| "negative_count": 301, |
| "positive_count": 304, |
| "precision": 0.7058823529411765, |
| "recall": 0.039473684210526314, |
| "roc_auc": 0.5654288337121874, |
| "specificity": 0.9833887043189369, |
| "threshold": 0.7510702013969421, |
| "tn": 296, |
| "tp": 12 |
| }, |
| "spa": { |
| "accuracy": 0.5581395348837209, |
| "average_precision": 0.43688015249152684, |
| "balanced_accuracy": 0.5054824561403509, |
| "brier_score": 0.31313111102791236, |
| "count": 43, |
| "expected_calibration_error": 0.2613871014395425, |
| "f1": 0.09523809523809523, |
| "false_negative_rate": 0.9473684210526315, |
| "false_positive_rate": 0.041666666666666664, |
| "fn": 18, |
| "fp": 1, |
| "log_loss": 0.835991683286482, |
| "macro_f1": 0.4014652014652015, |
| "negative_count": 24, |
| "positive_count": 19, |
| "precision": 0.5, |
| "recall": 0.05263157894736842, |
| "roc_auc": 0.43859649122807015, |
| "specificity": 0.9583333333333334, |
| "threshold": 0.7510702013969421, |
| "tn": 23, |
| "tp": 1 |
| } |
| }, |
| "synthetic": { |
| "False": { |
| "accuracy": 0.5181818181818182, |
| "average_precision": 0.5245109263654216, |
| "balanced_accuracy": 0.5095238095238096, |
| "brier_score": 0.28896361905882645, |
| "count": 550, |
| "expected_calibration_error": 0.18397142849185252, |
| "f1": 0.0636042402826855, |
| "false_negative_rate": 0.9666666666666667, |
| "false_positive_rate": 0.014285714285714285, |
| "fn": 261, |
| "fp": 4, |
| "log_loss": 0.7932073801759811, |
| "macro_f1": 0.36962341757096334, |
| "negative_count": 280, |
| "positive_count": 270, |
| "precision": 0.6923076923076923, |
| "recall": 0.03333333333333333, |
| "roc_auc": 0.5231084656084656, |
| "specificity": 0.9857142857142858, |
| "threshold": 0.7510702013969421, |
| "tn": 276, |
| "tp": 9 |
| }, |
| "True": { |
| "accuracy": 0.47959183673469385, |
| "average_precision": 0.6790413484001439, |
| "balanced_accuracy": 0.5155136268343816, |
| "brier_score": 0.24311059242241156, |
| "count": 98, |
| "expected_calibration_error": 0.15580201186999984, |
| "f1": 0.13559322033898305, |
| "false_negative_rate": 0.9245283018867925, |
| "false_positive_rate": 0.044444444444444446, |
| "fn": 49, |
| "fp": 2, |
| "log_loss": 0.6940204246644598, |
| "macro_f1": 0.38166522330817765, |
| "negative_count": 45, |
| "positive_count": 53, |
| "precision": 0.6666666666666666, |
| "recall": 0.07547169811320754, |
| "roc_auc": 0.680083857442348, |
| "specificity": 0.9555555555555556, |
| "threshold": 0.7510702013969421, |
| "tn": 43, |
| "tp": 4 |
| } |
| } |
| }, |
| "source_split": "validation", |
| "split": "validation", |
| "threshold": 0.7510702013969421, |
| "training_status": "preview-only", |
| "worst_groups": { |
| "highest_false_positive_rate": { |
| "count": 43, |
| "dimension": "dataset", |
| "false_positive_rate": 0.09523809523809523, |
| "value": "rime_2" |
| }, |
| "lowest_recall": { |
| "count": 362, |
| "dimension": "dataset", |
| "recall": 0.011363636363636364, |
| "value": "liva_1" |
| } |
| } |
| } |
