| { |
| "checkpoint": "artifacts/partial-shard-warmstart-lr3e4-5ep/best.pt", |
| "checkpoint_sha256": "e919cdbdc89046bd82fccf52a460406643e917b06d42a966d0e0104b2974f6c1", |
| "data_revision": "e564e2ac567f774d1880aa1db6ce97afb8c519b7", |
| "data_scope": "preview-only: 3265 rows from train-00010-of-00083; official test remains sealed", |
| "dataset_revision": null, |
| "development_only": true, |
| "evaluation_source": { |
| "bytes": 2791055, |
| "dataset_id": null, |
| "kind": "file", |
| "path": "data/processed/partial-iid-splits.jsonl", |
| "revision": "e564e2ac567f774d1880aa1db6ce97afb8c519b7", |
| "sha256": "cae0765d18467186c0e890c35fd94c0fcc70ae3483b5a1ce31814dd8ffa37b50" |
| }, |
| "freeze_manifest_sha256": null, |
| "grouped_bootstrap_95ci": { |
| "false_positive_rate": { |
| "estimate": 0.018072289156626505, |
| "lower": 0.0, |
| "upper": 0.041963307776560786 |
| }, |
| "recall": { |
| "estimate": 0.0625, |
| "lower": 0.028232028053769738, |
| "upper": 0.1 |
| } |
| }, |
| "metrics": { |
| "accuracy": 0.5306748466257669, |
| "average_precision": 0.6880689959324726, |
| "balanced_accuracy": 0.5222138554216867, |
| "brier_score": 0.2133037066638398, |
| "count": 326, |
| "expected_calibration_error": 0.1055241439925747, |
| "f1": 0.11560693641618497, |
| "false_negative_rate": 0.9375, |
| "false_positive_rate": 0.018072289156626505, |
| "fn": 150, |
| "fp": 3, |
| "log_loss": 0.6135001064370812, |
| "macro_f1": 0.39809574378220525, |
| "negative_count": 166, |
| "positive_count": 160, |
| "precision": 0.7692307692307693, |
| "recall": 0.0625, |
| "roc_auc": 0.7389683734939759, |
| "specificity": 0.9819277108433735, |
| "threshold": 0.741000771522522, |
| "tn": 163, |
| "tp": 10 |
| }, |
| "official_test": false, |
| "operating_points": { |
| "0.01": { |
| "accuracy": 0.5214723926380368, |
| "average_precision": 0.6880689959324726, |
| "balanced_accuracy": 0.5126129518072289, |
| "brier_score": 0.2133037066638398, |
| "count": 326, |
| "expected_calibration_error": 0.1055241439925747, |
| "f1": 0.060240963855421686, |
| "false_negative_rate": 0.96875, |
| "false_positive_rate": 0.006024096385542169, |
| "fn": 155, |
| "fp": 1, |
| "log_loss": 0.6135001064370812, |
| "macro_f1": 0.369626654767217, |
| "max_false_positive_rate": 0.01, |
| "negative_count": 166, |
| "positive_count": 160, |
| "precision": 0.8333333333333334, |
| "recall": 0.03125, |
| "roc_auc": 0.7389683734939759, |
| "specificity": 0.9939759036144579, |
| "threshold": 0.7609416246414185, |
| "tn": 165, |
| "tp": 5 |
| }, |
| "0.02": { |
| "accuracy": 0.5306748466257669, |
| "average_precision": 0.6880689959324726, |
| "balanced_accuracy": 0.5222138554216867, |
| "brier_score": 0.2133037066638398, |
| "count": 326, |
| "expected_calibration_error": 0.1055241439925747, |
| "f1": 0.11560693641618497, |
| "false_negative_rate": 0.9375, |
| "false_positive_rate": 0.018072289156626505, |
| "fn": 150, |
| "fp": 3, |
| "log_loss": 0.6135001064370812, |
| "macro_f1": 0.39809574378220525, |
| "max_false_positive_rate": 0.02, |
| "negative_count": 166, |
| "positive_count": 160, |
| "precision": 0.7692307692307693, |
| "recall": 0.0625, |
| "roc_auc": 0.7389683734939759, |
| "specificity": 0.9819277108433735, |
| "threshold": 0.741000771522522, |
| "tn": 163, |
| "tp": 10 |
| }, |
| "0.05": { |
| "accuracy": 0.5245398773006135, |
| "average_precision": 0.6880689959324726, |
| "balanced_accuracy": 0.5165286144578313, |
| "brier_score": 0.2133037066638398, |
| "count": 326, |
| "expected_calibration_error": 0.1055241439925747, |
| "f1": 0.143646408839779, |
| "false_negative_rate": 0.91875, |
| "false_positive_rate": 0.04819277108433735, |
| "fn": 147, |
| "fp": 8, |
| "log_loss": 0.6135001064370812, |
| "macro_f1": 0.4072796800037536, |
| "max_false_positive_rate": 0.05, |
| "negative_count": 166, |
| "positive_count": 160, |
| "precision": 0.6190476190476191, |
| "recall": 0.08125, |
| "roc_auc": 0.7389683734939759, |
| "specificity": 0.9518072289156626, |
| "threshold": 0.7245736718177795, |
| "tn": 158, |
| "tp": 13 |
| } |
| }, |
| "operating_points_note": "Label-dependent threshold sweep for curve analysis only. The deployed result is `metrics` at the checkpoint's frozen validation-selected threshold.", |
| "operational": { |
| "accuracy": 0.5306748466257669, |
| "audio_hours": 0.7063338715277777, |
| "average_precision": 0.6880689959324726, |
| "balanced_accuracy": 0.5222138554216867, |
| "brier_score": 0.2133037066638398, |
| "count": 326, |
| "expected_calibration_error": 0.1055241439925747, |
| "f1": 0.11560693641618497, |
| "false_interruptions": 3, |
| "false_interruptions_per_hour": 4.247283219635348, |
| "false_negative_rate": 0.9375, |
| "false_positive_rate": 0.018072289156626505, |
| "fn": 150, |
| "fp": 3, |
| "log_loss": 0.6135001064370812, |
| "macro_f1": 0.39809574378220525, |
| "negative_count": 166, |
| "positive_count": 160, |
| "precision": 0.7692307692307693, |
| "recall": 0.0625, |
| "roc_auc": 0.7389683734939759, |
| "scope": "independent labeled clips; per-hour rate is a clip-normalized proxy, not an online conversation measurement", |
| "sequence_metrics_available": false, |
| "sequence_metrics_unavailable_reason": "source records contain no genuine turn_id or conversation_id", |
| "specificity": 0.9819277108433735, |
| "threshold": 0.741000771522522, |
| "tn": 163, |
| "tp": 10 |
| }, |
| "reliability": [ |
| { |
| "count": 8, |
| "lower": 0.0, |
| "mean_probability": 0.08739477209746838, |
| "observed_rate": 0.125, |
| "upper": 0.1 |
| }, |
| { |
| "count": 57, |
| "lower": 0.1, |
| "mean_probability": 0.14298640361480547, |
| "observed_rate": 0.10526315789473684, |
| "upper": 0.2 |
| }, |
| { |
| "count": 56, |
| "lower": 0.2, |
| "mean_probability": 0.24947392541382993, |
| "observed_rate": 0.48214285714285715, |
| "upper": 0.3 |
| }, |
| { |
| "count": 40, |
| "lower": 0.3, |
| "mean_probability": 0.35491041019558905, |
| "observed_rate": 0.35, |
| "upper": 0.4 |
| }, |
| { |
| "count": 43, |
| "lower": 0.4, |
| "mean_probability": 0.45311658812123673, |
| "observed_rate": 0.5813953488372093, |
| "upper": 0.5 |
| }, |
| { |
| "count": 39, |
| "lower": 0.5, |
| "mean_probability": 0.5574724888190244, |
| "observed_rate": 0.6153846153846154, |
| "upper": 0.6 |
| }, |
| { |
| "count": 53, |
| "lower": 0.6, |
| "mean_probability": 0.6452814441806866, |
| "observed_rate": 0.8113207547169812, |
| "upper": 0.7 |
| }, |
| { |
| "count": 30, |
| "lower": 0.7, |
| "mean_probability": 0.7383094807465871, |
| "observed_rate": 0.6666666666666666, |
| "upper": 0.8 |
| }, |
| { |
| "count": 0, |
| "lower": 0.8, |
| "mean_probability": 0.0, |
| "observed_rate": 0.0, |
| "upper": 0.9 |
| }, |
| { |
| "count": 0, |
| "lower": 0.9, |
| "mean_probability": 0.0, |
| "observed_rate": 0.0, |
| "upper": 1.0 |
| } |
| ], |
| "slices": { |
| "dataset": { |
| "chirp3_1": { |
| "accuracy": 0.5480225988700564, |
| "average_precision": 0.6646168482728126, |
| "balanced_accuracy": 0.5249615975422427, |
| "brier_score": 0.2094916437112573, |
| "count": 177, |
| "expected_calibration_error": 0.10510466656105666, |
| "f1": 0.13043478260869565, |
| "false_negative_rate": 0.9285714285714286, |
| "false_positive_rate": 0.021505376344086023, |
| "fn": 78, |
| "fp": 2, |
| "log_loss": 0.6028296526829398, |
| "macro_f1": 0.412545635579157, |
| "negative_count": 93, |
| "positive_count": 84, |
| "precision": 0.75, |
| "recall": 0.07142857142857142, |
| "roc_auc": 0.7382232462877624, |
| "specificity": 0.978494623655914, |
| "threshold": 0.741000771522522, |
| "tn": 91, |
| "tp": 6 |
| }, |
| "chirp3_2": { |
| "accuracy": 0.5764705882352941, |
| "average_precision": 0.6926460598775176, |
| "balanced_accuracy": 0.5166103603603603, |
| "brier_score": 0.19349550820379868, |
| "count": 85, |
| "expected_calibration_error": 0.12449237152057539, |
| "f1": 0.1, |
| "false_negative_rate": 0.9459459459459459, |
| "false_positive_rate": 0.020833333333333332, |
| "fn": 35, |
| "fp": 1, |
| "log_loss": 0.5705930106009551, |
| "macro_f1": 0.4115384615384615, |
| "negative_count": 48, |
| "positive_count": 37, |
| "precision": 0.6666666666666666, |
| "recall": 0.05405405405405406, |
| "roc_auc": 0.7792792792792793, |
| "specificity": 0.9791666666666666, |
| "threshold": 0.741000771522522, |
| "tn": 47, |
| "tp": 2 |
| }, |
| "liva_1": { |
| "accuracy": 0.3611111111111111, |
| "average_precision": 0.7967089154342165, |
| "balanced_accuracy": 0.5208333333333334, |
| "brier_score": 0.28008335716505756, |
| "count": 36, |
| "expected_calibration_error": 0.26551183147562873, |
| "f1": 0.08, |
| "false_negative_rate": 0.9583333333333334, |
| "false_positive_rate": 0.0, |
| "fn": 23, |
| "fp": 0, |
| "log_loss": 0.7668865136017733, |
| "macro_f1": 0.2953191489361702, |
| "negative_count": 12, |
| "positive_count": 24, |
| "precision": 1.0, |
| "recall": 0.041666666666666664, |
| "roc_auc": 0.6354166666666666, |
| "specificity": 1.0, |
| "threshold": 0.741000771522522, |
| "tn": 12, |
| "tp": 1 |
| } |
| }, |
| "duration_bin": { |
| "2-4s": { |
| "accuracy": 0.5897435897435898, |
| "average_precision": 0.7161800176088101, |
| "balanced_accuracy": 0.5986842105263157, |
| "brier_score": 0.20126385365673957, |
| "count": 39, |
| "expected_calibration_error": 0.2071125757617828, |
| "f1": 0.38461538461538464, |
| "false_negative_rate": 0.75, |
| "false_positive_rate": 0.05263157894736842, |
| "fn": 15, |
| "fp": 1, |
| "log_loss": 0.5851101759971097, |
| "macro_f1": 0.5384615384615384, |
| "negative_count": 19, |
| "positive_count": 20, |
| "precision": 0.8333333333333334, |
| "recall": 0.25, |
| "roc_auc": 0.718421052631579, |
| "specificity": 0.9473684210526315, |
| "threshold": 0.741000771522522, |
| "tn": 18, |
| "tp": 5 |
| }, |
| "4-8s": { |
| "accuracy": 0.5337837837837838, |
| "average_precision": 0.6484258629260561, |
| "balanced_accuracy": 0.5078754578754578, |
| "brier_score": 0.22099832665123215, |
| "count": 148, |
| "expected_calibration_error": 0.11727086781851345, |
| "f1": 0.0547945205479452, |
| "false_negative_rate": 0.9714285714285714, |
| "false_positive_rate": 0.01282051282051282, |
| "fn": 68, |
| "fp": 1, |
| "log_loss": 0.6313563076194476, |
| "macro_f1": 0.3726887400946004, |
| "negative_count": 78, |
| "positive_count": 70, |
| "precision": 0.6666666666666666, |
| "recall": 0.02857142857142857, |
| "roc_auc": 0.7208791208791209, |
| "specificity": 0.9871794871794872, |
| "threshold": 0.741000771522522, |
| "tn": 77, |
| "tp": 2 |
| }, |
| ">=8s": { |
| "accuracy": 0.5147058823529411, |
| "average_precision": 0.7332799215478968, |
| "balanced_accuracy": 0.5147058823529412, |
| "brier_score": 0.21026609608377042, |
| "count": 136, |
| "expected_calibration_error": 0.10151389422004713, |
| "f1": 0.08333333333333333, |
| "false_negative_rate": 0.9558823529411765, |
| "false_positive_rate": 0.014705882352941176, |
| "fn": 65, |
| "fp": 1, |
| "log_loss": 0.6061258970823683, |
| "macro_f1": 0.3766666666666667, |
| "negative_count": 68, |
| "positive_count": 68, |
| "precision": 0.75, |
| "recall": 0.04411764705882353, |
| "roc_auc": 0.7588667820069204, |
| "specificity": 0.9852941176470589, |
| "threshold": 0.741000771522522, |
| "tn": 67, |
| "tp": 3 |
| } |
| }, |
| "filler_type": { |
| "endfiller": { |
| "accuracy": 1.0, |
| "average_precision": null, |
| "balanced_accuracy": 1.0, |
| "brier_score": 0.11988233328937586, |
| "count": 64, |
| "expected_calibration_error": 0.2932384021114558, |
| "f1": null, |
| "false_negative_rate": null, |
| "false_positive_rate": 0.0, |
| "fn": 0, |
| "fp": 0, |
| "log_loss": 0.3894655147868388, |
| "macro_f1": 1.0, |
| "negative_count": 64, |
| "positive_count": 0, |
| "precision": null, |
| "recall": null, |
| "roc_auc": null, |
| "specificity": 1.0, |
| "threshold": 0.741000771522522, |
| "tn": 64, |
| "tp": 0 |
| }, |
| "midfiller": { |
| "accuracy": 0.5522388059701493, |
| "average_precision": 0.6342371202899987, |
| "balanced_accuracy": 0.5255980326402862, |
| "brier_score": 0.2159177382893442, |
| "count": 134, |
| "expected_calibration_error": 0.12705712759895113, |
| "f1": 0.14285714285714285, |
| "false_negative_rate": 0.9206349206349206, |
| "false_positive_rate": 0.028169014084507043, |
| "fn": 58, |
| "fp": 2, |
| "log_loss": 0.6196895811384678, |
| "macro_f1": 0.4199134199134199, |
| "negative_count": 71, |
| "positive_count": 63, |
| "precision": 0.7142857142857143, |
| "recall": 0.07936507936507936, |
| "roc_auc": 0.7165213503241672, |
| "specificity": 0.971830985915493, |
| "threshold": 0.741000771522522, |
| "tn": 69, |
| "tp": 5 |
| }, |
| "no_filler": { |
| "accuracy": 0.13846153846153847, |
| "average_precision": 0.9282220984617144, |
| "balanced_accuracy": 0.4544334975369458, |
| "brier_score": 0.2636098061789016, |
| "count": 65, |
| "expected_calibration_error": 0.39574201336273784, |
| "f1": 0.0967741935483871, |
| "false_negative_rate": 0.9482758620689655, |
| "false_positive_rate": 0.14285714285714285, |
| "fn": 55, |
| "fp": 1, |
| "log_loss": 0.7362453768676601, |
| "macro_f1": 0.1366223908918406, |
| "negative_count": 7, |
| "positive_count": 58, |
| "precision": 0.75, |
| "recall": 0.05172413793103448, |
| "roc_auc": 0.6798029556650246, |
| "specificity": 0.8571428571428571, |
| "threshold": 0.741000771522522, |
| "tn": 6, |
| "tp": 3 |
| }, |
| "unknown": { |
| "accuracy": 0.4126984126984127, |
| "average_precision": 0.799471612772439, |
| "balanced_accuracy": 0.5256410256410257, |
| "brier_score": 0.25074483665858716, |
| "count": 63, |
| "expected_calibration_error": 0.19315069298895576, |
| "f1": 0.0975609756097561, |
| "false_negative_rate": 0.9487179487179487, |
| "false_positive_rate": 0.0, |
| "fn": 37, |
| "fp": 0, |
| "log_loss": 0.7012839425901298, |
| "macro_f1": 0.3311334289813486, |
| "negative_count": 24, |
| "positive_count": 39, |
| "precision": 1.0, |
| "recall": 0.05128205128205128, |
| "roc_auc": 0.6720085470085471, |
| "specificity": 1.0, |
| "threshold": 0.741000771522522, |
| "tn": 24, |
| "tp": 2 |
| } |
| }, |
| "language": { |
| "eng": { |
| "accuracy": 0.4342105263157895, |
| "average_precision": 0.838347854611721, |
| "balanced_accuracy": 0.5222222222222223, |
| "brier_score": 0.2267736908018496, |
| "count": 76, |
| "expected_calibration_error": 0.17203282682519208, |
| "f1": 0.0851063829787234, |
| "false_negative_rate": 0.9555555555555556, |
| "false_positive_rate": 0.0, |
| "fn": 43, |
| "fp": 0, |
| "log_loss": 0.6475126882660074, |
| "macro_f1": 0.33779128672745695, |
| "negative_count": 31, |
| "positive_count": 45, |
| "precision": 1.0, |
| "recall": 0.044444444444444446, |
| "roc_auc": 0.7483870967741936, |
| "specificity": 1.0, |
| "threshold": 0.741000771522522, |
| "tn": 31, |
| "tp": 2 |
| } |
| }, |
| "synthetic": { |
| "False": { |
| "accuracy": 0.4, |
| "average_precision": 0.8139063380698389, |
| "balanced_accuracy": 0.5285714285714286, |
| "brier_score": 0.2517751278920657, |
| "count": 55, |
| "expected_calibration_error": 0.20835820707407862, |
| "f1": 0.10810810810810811, |
| "false_negative_rate": 0.9428571428571428, |
| "false_positive_rate": 0.0, |
| "fn": 33, |
| "fp": 0, |
| "log_loss": 0.7044581255390688, |
| "macro_f1": 0.32802665679378007, |
| "negative_count": 20, |
| "positive_count": 35, |
| "precision": 1.0, |
| "recall": 0.05714285714285714, |
| "roc_auc": 0.6785714285714286, |
| "specificity": 1.0, |
| "threshold": 0.741000771522522, |
| "tn": 20, |
| "tp": 2 |
| }, |
| "True": { |
| "accuracy": 0.5571955719557196, |
| "average_precision": 0.6669351194253922, |
| "balanced_accuracy": 0.5217260273972603, |
| "brier_score": 0.20549585364704118, |
| "count": 271, |
| "expected_calibration_error": 0.09752494687853705, |
| "f1": 0.11764705882352941, |
| "false_negative_rate": 0.936, |
| "false_positive_rate": 0.02054794520547945, |
| "fn": 117, |
| "fp": 3, |
| "log_loss": 0.595039991859187, |
| "macro_f1": 0.41104027818023764, |
| "negative_count": 146, |
| "positive_count": 125, |
| "precision": 0.7272727272727273, |
| "recall": 0.064, |
| "roc_auc": 0.7485479452054794, |
| "specificity": 0.9794520547945206, |
| "threshold": 0.741000771522522, |
| "tn": 143, |
| "tp": 8 |
| } |
| } |
| }, |
| "source_split": "validation", |
| "split": "validation", |
| "threshold": 0.741000771522522, |
| "training_status": "preview-only", |
| "worst_groups": { |
| "highest_false_positive_rate": { |
| "count": 65, |
| "dimension": "filler_type", |
| "false_positive_rate": 0.14285714285714285, |
| "value": "no_filler" |
| }, |
| "lowest_recall": { |
| "count": 148, |
| "dimension": "duration_bin", |
| "recall": 0.02857142857142857, |
| "value": "4-8s" |
| } |
| } |
| } |
