| { |
| "epochs": [ |
| { |
| "per_head": { |
| "relation_to_previous": { |
| "accuracy": 0.6775, |
| "macro_f1": 0.2761, |
| "per_label": { |
| "new": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 102 |
| }, |
| "follow_up": { |
| "precision": 0.4885, |
| "recall": 1.0, |
| "f1": 0.6564, |
| "support": 85 |
| }, |
| "correction": { |
| "precision": 0.0, |
| "recall": 0.0, |
| "f1": 0.0, |
| "support": 29 |
| }, |
| "confirmation": { |
| "precision": 0.0, |
| "recall": 0.0, |
| "f1": 0.0, |
| "support": 20 |
| }, |
| "cancellation": { |
| "precision": 0.0, |
| "recall": 0.0, |
| "f1": 0.0, |
| "support": 21 |
| }, |
| "closure": { |
| "precision": 0.0, |
| "recall": 0.0, |
| "f1": 0.0, |
| "support": 19 |
| } |
| }, |
| "confusion_matrix": [ |
| [ |
| 102, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0 |
| ], |
| [ |
| 0, |
| 85, |
| 0, |
| 0, |
| 0, |
| 0 |
| ], |
| [ |
| 0, |
| 29, |
| 0, |
| 0, |
| 0, |
| 0 |
| ], |
| [ |
| 0, |
| 20, |
| 0, |
| 0, |
| 0, |
| 0 |
| ], |
| [ |
| 0, |
| 21, |
| 0, |
| 0, |
| 0, |
| 0 |
| ], |
| [ |
| 0, |
| 19, |
| 0, |
| 0, |
| 0, |
| 0 |
| ] |
| ] |
| }, |
| "actionability": { |
| "accuracy": 0.4891, |
| "macro_f1": 0.219, |
| "per_label": { |
| "none": { |
| "precision": 0.0, |
| "recall": 0.0, |
| "f1": 0.0, |
| "support": 61 |
| }, |
| "review": { |
| "precision": 0.0, |
| "recall": 0.0, |
| "f1": 0.0, |
| "support": 79 |
| }, |
| "act": { |
| "precision": 0.4909, |
| "recall": 0.9926, |
| "f1": 0.6569, |
| "support": 136 |
| } |
| }, |
| "confusion_matrix": [ |
| [ |
| 0, |
| 0, |
| 61 |
| ], |
| [ |
| 0, |
| 0, |
| 79 |
| ], |
| [ |
| 1, |
| 0, |
| 135 |
| ] |
| ] |
| }, |
| "retention": { |
| "accuracy": 0.5181, |
| "macro_f1": 0.2477, |
| "per_label": { |
| "ephemeral": { |
| "precision": 1.0, |
| "recall": 0.0337, |
| "f1": 0.0652, |
| "support": 89 |
| }, |
| "useful": { |
| "precision": 0.5128, |
| "recall": 1.0, |
| "f1": 0.678, |
| "support": 140 |
| }, |
| "remember": { |
| "precision": 0.0, |
| "recall": 0.0, |
| "f1": 0.0, |
| "support": 47 |
| } |
| }, |
| "confusion_matrix": [ |
| [ |
| 3, |
| 86, |
| 0 |
| ], |
| [ |
| 0, |
| 140, |
| 0 |
| ], |
| [ |
| 0, |
| 47, |
| 0 |
| ] |
| ] |
| }, |
| "urgency": { |
| "accuracy": 0.4928, |
| "macro_f1": 0.2201, |
| "per_label": { |
| "low": { |
| "precision": 0.4928, |
| "recall": 1.0, |
| "f1": 0.6602, |
| "support": 136 |
| }, |
| "medium": { |
| "precision": 0.0, |
| "recall": 0.0, |
| "f1": 0.0, |
| "support": 95 |
| }, |
| "high": { |
| "precision": 0.0, |
| "recall": 0.0, |
| "f1": 0.0, |
| "support": 45 |
| } |
| }, |
| "confusion_matrix": [ |
| [ |
| 136, |
| 0, |
| 0 |
| ], |
| [ |
| 95, |
| 0, |
| 0 |
| ], |
| [ |
| 45, |
| 0, |
| 0 |
| ] |
| ] |
| } |
| }, |
| "overall": { |
| "exact_match": 0.0616, |
| "macro_average_f1": 0.2407, |
| "automation_safe_accuracy": 0.0, |
| "automation_safe_coverage": 0.0, |
| "confidence_threshold": 0.8, |
| "confidence_calibration": { |
| "ece": 0.463661, |
| "bins": [ |
| { |
| "range": [ |
| 0.4, |
| 0.5 |
| ], |
| "count": 58, |
| "avg_confidence": 0.481, |
| "accuracy": 0.0517 |
| }, |
| { |
| "range": [ |
| 0.5, |
| 0.6 |
| ], |
| "count": 218, |
| "avg_confidence": 0.537, |
| "accuracy": 0.0642 |
| } |
| ] |
| } |
| }, |
| "training": { |
| "epoch": 1, |
| "loss": 4.6568 |
| } |
| }, |
| { |
| "per_head": { |
| "relation_to_previous": { |
| "accuracy": 0.7283, |
| "macro_f1": 0.4422, |
| "per_label": { |
| "new": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 102 |
| }, |
| "follow_up": { |
| "precision": 0.6124, |
| "recall": 0.9294, |
| "f1": 0.7383, |
| "support": 85 |
| }, |
| "correction": { |
| "precision": 0.0, |
| "recall": 0.0, |
| "f1": 0.0, |
| "support": 29 |
| }, |
| "confirmation": { |
| "precision": 1.0, |
| "recall": 0.2, |
| "f1": 0.3333, |
| "support": 20 |
| }, |
| "cancellation": { |
| "precision": 0.0, |
| "recall": 0.0, |
| "f1": 0.0, |
| "support": 21 |
| }, |
| "closure": { |
| "precision": 0.4444, |
| "recall": 0.8421, |
| "f1": 0.5818, |
| "support": 19 |
| } |
| }, |
| "confusion_matrix": [ |
| [ |
| 102, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0 |
| ], |
| [ |
| 0, |
| 79, |
| 0, |
| 0, |
| 0, |
| 6 |
| ], |
| [ |
| 0, |
| 24, |
| 0, |
| 0, |
| 0, |
| 5 |
| ], |
| [ |
| 0, |
| 8, |
| 3, |
| 4, |
| 0, |
| 5 |
| ], |
| [ |
| 0, |
| 15, |
| 2, |
| 0, |
| 0, |
| 4 |
| ], |
| [ |
| 0, |
| 3, |
| 0, |
| 0, |
| 0, |
| 16 |
| ] |
| ] |
| }, |
| "actionability": { |
| "accuracy": 0.5688, |
| "macro_f1": 0.5548, |
| "per_label": { |
| "none": { |
| "precision": 0.5303, |
| "recall": 0.5738, |
| "f1": 0.5512, |
| "support": 61 |
| }, |
| "review": { |
| "precision": 0.4124, |
| "recall": 0.5063, |
| "f1": 0.4545, |
| "support": 79 |
| }, |
| "act": { |
| "precision": 0.7257, |
| "recall": 0.6029, |
| "f1": 0.6586, |
| "support": 136 |
| } |
| }, |
| "confusion_matrix": [ |
| [ |
| 35, |
| 18, |
| 8 |
| ], |
| [ |
| 16, |
| 40, |
| 23 |
| ], |
| [ |
| 15, |
| 39, |
| 82 |
| ] |
| ] |
| }, |
| "retention": { |
| "accuracy": 0.5254, |
| "macro_f1": 0.3733, |
| "per_label": { |
| "ephemeral": { |
| "precision": 0.4845, |
| "recall": 0.5281, |
| "f1": 0.5054, |
| "support": 89 |
| }, |
| "useful": { |
| "precision": 0.5475, |
| "recall": 0.7, |
| "f1": 0.6144, |
| "support": 140 |
| }, |
| "remember": { |
| "precision": 0.0, |
| "recall": 0.0, |
| "f1": 0.0, |
| "support": 47 |
| } |
| }, |
| "confusion_matrix": [ |
| [ |
| 47, |
| 42, |
| 0 |
| ], |
| [ |
| 42, |
| 98, |
| 0 |
| ], |
| [ |
| 8, |
| 39, |
| 0 |
| ] |
| ] |
| }, |
| "urgency": { |
| "accuracy": 0.5362, |
| "macro_f1": 0.3555, |
| "per_label": { |
| "low": { |
| "precision": 0.5604, |
| "recall": 0.8529, |
| "f1": 0.6764, |
| "support": 136 |
| }, |
| "medium": { |
| "precision": 0.4638, |
| "recall": 0.3368, |
| "f1": 0.3902, |
| "support": 95 |
| }, |
| "high": { |
| "precision": 0.0, |
| "recall": 0.0, |
| "f1": 0.0, |
| "support": 45 |
| } |
| }, |
| "confusion_matrix": [ |
| [ |
| 116, |
| 20, |
| 0 |
| ], |
| [ |
| 63, |
| 32, |
| 0 |
| ], |
| [ |
| 28, |
| 17, |
| 0 |
| ] |
| ] |
| } |
| }, |
| "overall": { |
| "exact_match": 0.1123, |
| "macro_average_f1": 0.4314, |
| "automation_safe_accuracy": 0.0, |
| "automation_safe_coverage": 0.0, |
| "confidence_threshold": 0.8, |
| "confidence_calibration": { |
| "ece": 0.445582, |
| "bins": [ |
| { |
| "range": [ |
| 0.4, |
| 0.5 |
| ], |
| "count": 44, |
| "avg_confidence": 0.4826, |
| "accuracy": 0.0455 |
| }, |
| { |
| "range": [ |
| 0.5, |
| 0.6 |
| ], |
| "count": 208, |
| "avg_confidence": 0.5677, |
| "accuracy": 0.125 |
| }, |
| { |
| "range": [ |
| 0.6, |
| 0.7 |
| ], |
| "count": 24, |
| "avg_confidence": 0.6107, |
| "accuracy": 0.125 |
| } |
| ] |
| } |
| }, |
| "training": { |
| "epoch": 2, |
| "loss": 3.7776 |
| } |
| }, |
| { |
| "per_head": { |
| "relation_to_previous": { |
| "accuracy": 0.808, |
| "macro_f1": 0.6475, |
| "per_label": { |
| "new": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 102 |
| }, |
| "follow_up": { |
| "precision": 0.7009, |
| "recall": 0.9647, |
| "f1": 0.8119, |
| "support": 85 |
| }, |
| "correction": { |
| "precision": 0.5294, |
| "recall": 0.3103, |
| "f1": 0.3913, |
| "support": 29 |
| }, |
| "confirmation": { |
| "precision": 0.9231, |
| "recall": 0.6, |
| "f1": 0.7273, |
| "support": 20 |
| }, |
| "cancellation": { |
| "precision": 0.75, |
| "recall": 0.1429, |
| "f1": 0.24, |
| "support": 21 |
| }, |
| "closure": { |
| "precision": 0.6522, |
| "recall": 0.7895, |
| "f1": 0.7143, |
| "support": 19 |
| } |
| }, |
| "confusion_matrix": [ |
| [ |
| 102, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0 |
| ], |
| [ |
| 0, |
| 82, |
| 1, |
| 0, |
| 0, |
| 2 |
| ], |
| [ |
| 0, |
| 16, |
| 9, |
| 1, |
| 1, |
| 2 |
| ], |
| [ |
| 0, |
| 4, |
| 0, |
| 12, |
| 0, |
| 4 |
| ], |
| [ |
| 0, |
| 11, |
| 7, |
| 0, |
| 3, |
| 0 |
| ], |
| [ |
| 0, |
| 4, |
| 0, |
| 0, |
| 0, |
| 15 |
| ] |
| ] |
| }, |
| "actionability": { |
| "accuracy": 0.6304, |
| "macro_f1": 0.5798, |
| "per_label": { |
| "none": { |
| "precision": 0.6591, |
| "recall": 0.4754, |
| "f1": 0.5524, |
| "support": 61 |
| }, |
| "review": { |
| "precision": 0.5882, |
| "recall": 0.3797, |
| "f1": 0.4615, |
| "support": 79 |
| }, |
| "act": { |
| "precision": 0.6354, |
| "recall": 0.8456, |
| "f1": 0.7256, |
| "support": 136 |
| } |
| }, |
| "confusion_matrix": [ |
| [ |
| 29, |
| 7, |
| 25 |
| ], |
| [ |
| 8, |
| 30, |
| 41 |
| ], |
| [ |
| 7, |
| 14, |
| 115 |
| ] |
| ] |
| }, |
| "retention": { |
| "accuracy": 0.6703, |
| "macro_f1": 0.6498, |
| "per_label": { |
| "ephemeral": { |
| "precision": 0.661, |
| "recall": 0.4382, |
| "f1": 0.527, |
| "support": 89 |
| }, |
| "useful": { |
| "precision": 0.6398, |
| "recall": 0.85, |
| "f1": 0.7301, |
| "support": 140 |
| }, |
| "remember": { |
| "precision": 0.871, |
| "recall": 0.5745, |
| "f1": 0.6923, |
| "support": 47 |
| } |
| }, |
| "confusion_matrix": [ |
| [ |
| 39, |
| 49, |
| 1 |
| ], |
| [ |
| 18, |
| 119, |
| 3 |
| ], |
| [ |
| 2, |
| 18, |
| 27 |
| ] |
| ] |
| }, |
| "urgency": { |
| "accuracy": 0.5688, |
| "macro_f1": 0.4235, |
| "per_label": { |
| "low": { |
| "precision": 0.694, |
| "recall": 0.6838, |
| "f1": 0.6889, |
| "support": 136 |
| }, |
| "medium": { |
| "precision": 0.4565, |
| "recall": 0.6632, |
| "f1": 0.5408, |
| "support": 95 |
| }, |
| "high": { |
| "precision": 0.25, |
| "recall": 0.0222, |
| "f1": 0.0408, |
| "support": 45 |
| } |
| }, |
| "confusion_matrix": [ |
| [ |
| 93, |
| 42, |
| 1 |
| ], |
| [ |
| 30, |
| 63, |
| 2 |
| ], |
| [ |
| 11, |
| 33, |
| 1 |
| ] |
| ] |
| } |
| }, |
| "overall": { |
| "exact_match": 0.2101, |
| "macro_average_f1": 0.5752, |
| "automation_safe_accuracy": 0.0, |
| "automation_safe_coverage": 0.0, |
| "confidence_threshold": 0.8, |
| "confidence_calibration": { |
| "ece": 0.396414, |
| "bins": [ |
| { |
| "range": [ |
| 0.4, |
| 0.5 |
| ], |
| "count": 10, |
| "avg_confidence": 0.4772, |
| "accuracy": 0.2 |
| }, |
| { |
| "range": [ |
| 0.5, |
| 0.6 |
| ], |
| "count": 128, |
| "avg_confidence": 0.5669, |
| "accuracy": 0.1484 |
| }, |
| { |
| "range": [ |
| 0.6, |
| 0.7 |
| ], |
| "count": 129, |
| "avg_confidence": 0.64, |
| "accuracy": 0.2326 |
| }, |
| { |
| "range": [ |
| 0.7, |
| 0.8 |
| ], |
| "count": 9, |
| "avg_confidence": 0.7196, |
| "accuracy": 0.7778 |
| } |
| ] |
| } |
| }, |
| "training": { |
| "epoch": 3, |
| "loss": 3.3415 |
| } |
| }, |
| { |
| "per_head": { |
| "relation_to_previous": { |
| "accuracy": 0.8333, |
| "macro_f1": 0.694, |
| "per_label": { |
| "new": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 102 |
| }, |
| "follow_up": { |
| "precision": 0.8571, |
| "recall": 0.9176, |
| "f1": 0.8864, |
| "support": 85 |
| }, |
| "correction": { |
| "precision": 0.5652, |
| "recall": 0.4483, |
| "f1": 0.5, |
| "support": 29 |
| }, |
| "confirmation": { |
| "precision": 0.8, |
| "recall": 0.6, |
| "f1": 0.6857, |
| "support": 20 |
| }, |
| "cancellation": { |
| "precision": 0.5455, |
| "recall": 0.2857, |
| "f1": 0.375, |
| "support": 21 |
| }, |
| "closure": { |
| "precision": 0.5588, |
| "recall": 1.0, |
| "f1": 0.717, |
| "support": 19 |
| } |
| }, |
| "confusion_matrix": [ |
| [ |
| 102, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0 |
| ], |
| [ |
| 0, |
| 78, |
| 3, |
| 0, |
| 1, |
| 3 |
| ], |
| [ |
| 0, |
| 7, |
| 13, |
| 1, |
| 4, |
| 4 |
| ], |
| [ |
| 0, |
| 2, |
| 0, |
| 12, |
| 0, |
| 6 |
| ], |
| [ |
| 0, |
| 4, |
| 7, |
| 2, |
| 6, |
| 2 |
| ], |
| [ |
| 0, |
| 0, |
| 0, |
| 0, |
| 0, |
| 19 |
| ] |
| ] |
| }, |
| "actionability": { |
| "accuracy": 0.6486, |
| "macro_f1": 0.6252, |
| "per_label": { |
| "none": { |
| "precision": 0.5634, |
| "recall": 0.6557, |
| "f1": 0.6061, |
| "support": 61 |
| }, |
| "review": { |
| "precision": 0.5882, |
| "recall": 0.5063, |
| "f1": 0.5442, |
| "support": 79 |
| }, |
| "act": { |
| "precision": 0.7226, |
| "recall": 0.7279, |
| "f1": 0.7253, |
| "support": 136 |
| } |
| }, |
| "confusion_matrix": [ |
| [ |
| 40, |
| 8, |
| 13 |
| ], |
| [ |
| 14, |
| 40, |
| 25 |
| ], |
| [ |
| 17, |
| 20, |
| 99 |
| ] |
| ] |
| }, |
| "retention": { |
| "accuracy": 0.6703, |
| "macro_f1": 0.6542, |
| "per_label": { |
| "ephemeral": { |
| "precision": 0.6067, |
| "recall": 0.6067, |
| "f1": 0.6067, |
| "support": 89 |
| }, |
| "useful": { |
| "precision": 0.673, |
| "recall": 0.7643, |
| "f1": 0.7157, |
| "support": 140 |
| }, |
| "remember": { |
| "precision": 0.8571, |
| "recall": 0.5106, |
| "f1": 0.64, |
| "support": 47 |
| } |
| }, |
| "confusion_matrix": [ |
| [ |
| 54, |
| 35, |
| 0 |
| ], |
| [ |
| 29, |
| 107, |
| 4 |
| ], |
| [ |
| 6, |
| 17, |
| 24 |
| ] |
| ] |
| }, |
| "urgency": { |
| "accuracy": 0.5906, |
| "macro_f1": 0.4633, |
| "per_label": { |
| "low": { |
| "precision": 0.6477, |
| "recall": 0.8382, |
| "f1": 0.7308, |
| "support": 136 |
| }, |
| "medium": { |
| "precision": 0.4783, |
| "recall": 0.4632, |
| "f1": 0.4706, |
| "support": 95 |
| }, |
| "high": { |
| "precision": 0.625, |
| "recall": 0.1111, |
| "f1": 0.1887, |
| "support": 45 |
| } |
| }, |
| "confusion_matrix": [ |
| [ |
| 114, |
| 21, |
| 1 |
| ], |
| [ |
| 49, |
| 44, |
| 2 |
| ], |
| [ |
| 13, |
| 27, |
| 5 |
| ] |
| ] |
| } |
| }, |
| "overall": { |
| "exact_match": 0.2319, |
| "macro_average_f1": 0.6092, |
| "automation_safe_accuracy": 0.0, |
| "automation_safe_coverage": 0.0, |
| "confidence_threshold": 0.8, |
| "confidence_calibration": { |
| "ece": 0.402291, |
| "bins": [ |
| { |
| "range": [ |
| 0.4, |
| 0.5 |
| ], |
| "count": 4, |
| "avg_confidence": 0.4762, |
| "accuracy": 0.0 |
| }, |
| { |
| "range": [ |
| 0.5, |
| 0.6 |
| ], |
| "count": 74, |
| "avg_confidence": 0.563, |
| "accuracy": 0.1486 |
| }, |
| { |
| "range": [ |
| 0.6, |
| 0.7 |
| ], |
| "count": 159, |
| "avg_confidence": 0.6473, |
| "accuracy": 0.2138 |
| }, |
| { |
| "range": [ |
| 0.7, |
| 0.8 |
| ], |
| "count": 39, |
| "avg_confidence": 0.7317, |
| "accuracy": 0.4872 |
| } |
| ] |
| } |
| }, |
| "training": { |
| "epoch": 4, |
| "loss": 2.9715 |
| } |
| }, |
| { |
| "per_head": { |
| "relation_to_previous": { |
| "accuracy": 0.8442, |
| "macro_f1": 0.7157, |
| "per_label": { |
| "new": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 102 |
| }, |
| "follow_up": { |
| "precision": 0.8571, |
| "recall": 0.9176, |
| "f1": 0.8864, |
| "support": 85 |
| }, |
| "correction": { |
| "precision": 0.5926, |
| "recall": 0.5517, |
| "f1": 0.5714, |
| "support": 29 |
| }, |
| "confirmation": { |
| "precision": 0.8571, |
| "recall": 0.6, |
| "f1": 0.7059, |
| "support": 20 |
| }, |
| "cancellation": { |
| "precision": 0.6667, |
| "recall": 0.2857, |
| "f1": 0.4, |
| "support": 21 |
| }, |
| "closure": { |
| "precision": 0.5758, |
| "recall": 1.0, |
| "f1": 0.7308, |
| "support": 19 |
| } |
| }, |
| "confusion_matrix": [ |
| [ |
| 102, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0 |
| ], |
| [ |
| 0, |
| 78, |
| 3, |
| 0, |
| 1, |
| 3 |
| ], |
| [ |
| 0, |
| 7, |
| 16, |
| 1, |
| 2, |
| 3 |
| ], |
| [ |
| 0, |
| 2, |
| 0, |
| 12, |
| 0, |
| 6 |
| ], |
| [ |
| 0, |
| 4, |
| 8, |
| 1, |
| 6, |
| 2 |
| ], |
| [ |
| 0, |
| 0, |
| 0, |
| 0, |
| 0, |
| 19 |
| ] |
| ] |
| }, |
| "actionability": { |
| "accuracy": 0.6558, |
| "macro_f1": 0.6342, |
| "per_label": { |
| "none": { |
| "precision": 0.5972, |
| "recall": 0.7049, |
| "f1": 0.6466, |
| "support": 61 |
| }, |
| "review": { |
| "precision": 0.5493, |
| "recall": 0.4937, |
| "f1": 0.52, |
| "support": 79 |
| }, |
| "act": { |
| "precision": 0.7444, |
| "recall": 0.7279, |
| "f1": 0.7361, |
| "support": 136 |
| } |
| }, |
| "confusion_matrix": [ |
| [ |
| 43, |
| 12, |
| 6 |
| ], |
| [ |
| 12, |
| 39, |
| 28 |
| ], |
| [ |
| 17, |
| 20, |
| 99 |
| ] |
| ] |
| }, |
| "retention": { |
| "accuracy": 0.6703, |
| "macro_f1": 0.6666, |
| "per_label": { |
| "ephemeral": { |
| "precision": 0.5816, |
| "recall": 0.6404, |
| "f1": 0.6096, |
| "support": 89 |
| }, |
| "useful": { |
| "precision": 0.6846, |
| "recall": 0.7286, |
| "f1": 0.7059, |
| "support": 140 |
| }, |
| "remember": { |
| "precision": 0.8966, |
| "recall": 0.5532, |
| "f1": 0.6842, |
| "support": 47 |
| } |
| }, |
| "confusion_matrix": [ |
| [ |
| 57, |
| 32, |
| 0 |
| ], |
| [ |
| 35, |
| 102, |
| 3 |
| ], |
| [ |
| 6, |
| 15, |
| 26 |
| ] |
| ] |
| }, |
| "urgency": { |
| "accuracy": 0.5978, |
| "macro_f1": 0.4845, |
| "per_label": { |
| "low": { |
| "precision": 0.6948, |
| "recall": 0.7868, |
| "f1": 0.7379, |
| "support": 136 |
| }, |
| "medium": { |
| "precision": 0.4815, |
| "recall": 0.5474, |
| "f1": 0.5123, |
| "support": 95 |
| }, |
| "high": { |
| "precision": 0.4286, |
| "recall": 0.1333, |
| "f1": 0.2034, |
| "support": 45 |
| } |
| }, |
| "confusion_matrix": [ |
| [ |
| 107, |
| 27, |
| 2 |
| ], |
| [ |
| 37, |
| 52, |
| 6 |
| ], |
| [ |
| 10, |
| 29, |
| 6 |
| ] |
| ] |
| } |
| }, |
| "overall": { |
| "exact_match": 0.25, |
| "macro_average_f1": 0.6252, |
| "automation_safe_accuracy": 0.8, |
| "automation_safe_coverage": 0.0181, |
| "confidence_threshold": 0.8, |
| "confidence_calibration": { |
| "ece": 0.407114, |
| "bins": [ |
| { |
| "range": [ |
| 0.4, |
| 0.5 |
| ], |
| "count": 1, |
| "avg_confidence": 0.4972, |
| "accuracy": 0.0 |
| }, |
| { |
| "range": [ |
| 0.5, |
| 0.6 |
| ], |
| "count": 55, |
| "avg_confidence": 0.5704, |
| "accuracy": 0.1818 |
| }, |
| { |
| "range": [ |
| 0.6, |
| 0.7 |
| ], |
| "count": 143, |
| "avg_confidence": 0.6475, |
| "accuracy": 0.1538 |
| }, |
| { |
| "range": [ |
| 0.7, |
| 0.8 |
| ], |
| "count": 72, |
| "avg_confidence": 0.7343, |
| "accuracy": 0.4583 |
| }, |
| { |
| "range": [ |
| 0.8, |
| 0.9 |
| ], |
| "count": 5, |
| "avg_confidence": 0.8067, |
| "accuracy": 0.8 |
| } |
| ] |
| } |
| }, |
| "training": { |
| "epoch": 5, |
| "loss": 2.7301 |
| } |
| }, |
| { |
| "per_head": { |
| "relation_to_previous": { |
| "accuracy": 0.8587, |
| "macro_f1": 0.7646, |
| "per_label": { |
| "new": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 102 |
| }, |
| "follow_up": { |
| "precision": 0.7921, |
| "recall": 0.9412, |
| "f1": 0.8602, |
| "support": 85 |
| }, |
| "correction": { |
| "precision": 0.64, |
| "recall": 0.5517, |
| "f1": 0.5926, |
| "support": 29 |
| }, |
| "confirmation": { |
| "precision": 0.8125, |
| "recall": 0.65, |
| "f1": 0.7222, |
| "support": 20 |
| }, |
| "cancellation": { |
| "precision": 0.8182, |
| "recall": 0.4286, |
| "f1": 0.5625, |
| "support": 21 |
| }, |
| "closure": { |
| "precision": 0.8095, |
| "recall": 0.8947, |
| "f1": 0.85, |
| "support": 19 |
| } |
| }, |
| "confusion_matrix": [ |
| [ |
| 102, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0 |
| ], |
| [ |
| 0, |
| 80, |
| 4, |
| 0, |
| 0, |
| 1 |
| ], |
| [ |
| 0, |
| 8, |
| 16, |
| 2, |
| 2, |
| 1 |
| ], |
| [ |
| 0, |
| 4, |
| 1, |
| 13, |
| 0, |
| 2 |
| ], |
| [ |
| 0, |
| 7, |
| 4, |
| 1, |
| 9, |
| 0 |
| ], |
| [ |
| 0, |
| 2, |
| 0, |
| 0, |
| 0, |
| 17 |
| ] |
| ] |
| }, |
| "actionability": { |
| "accuracy": 0.6884, |
| "macro_f1": 0.6666, |
| "per_label": { |
| "none": { |
| "precision": 0.6333, |
| "recall": 0.623, |
| "f1": 0.6281, |
| "support": 61 |
| }, |
| "review": { |
| "precision": 0.5976, |
| "recall": 0.6203, |
| "f1": 0.6087, |
| "support": 79 |
| }, |
| "act": { |
| "precision": 0.7687, |
| "recall": 0.7574, |
| "f1": 0.763, |
| "support": 136 |
| } |
| }, |
| "confusion_matrix": [ |
| [ |
| 38, |
| 14, |
| 9 |
| ], |
| [ |
| 8, |
| 49, |
| 22 |
| ], |
| [ |
| 14, |
| 19, |
| 103 |
| ] |
| ] |
| }, |
| "retention": { |
| "accuracy": 0.6703, |
| "macro_f1": 0.6452, |
| "per_label": { |
| "ephemeral": { |
| "precision": 0.6, |
| "recall": 0.6067, |
| "f1": 0.6034, |
| "support": 89 |
| }, |
| "useful": { |
| "precision": 0.6707, |
| "recall": 0.7857, |
| "f1": 0.7237, |
| "support": 140 |
| }, |
| "remember": { |
| "precision": 0.9545, |
| "recall": 0.4468, |
| "f1": 0.6087, |
| "support": 47 |
| } |
| }, |
| "confusion_matrix": [ |
| [ |
| 54, |
| 35, |
| 0 |
| ], |
| [ |
| 29, |
| 110, |
| 1 |
| ], |
| [ |
| 7, |
| 19, |
| 21 |
| ] |
| ] |
| }, |
| "urgency": { |
| "accuracy": 0.6196, |
| "macro_f1": 0.5411, |
| "per_label": { |
| "low": { |
| "precision": 0.7576, |
| "recall": 0.7353, |
| "f1": 0.7463, |
| "support": 136 |
| }, |
| "medium": { |
| "precision": 0.5, |
| "recall": 0.6316, |
| "f1": 0.5581, |
| "support": 95 |
| }, |
| "high": { |
| "precision": 0.4583, |
| "recall": 0.2444, |
| "f1": 0.3188, |
| "support": 45 |
| } |
| }, |
| "confusion_matrix": [ |
| [ |
| 100, |
| 32, |
| 4 |
| ], |
| [ |
| 26, |
| 60, |
| 9 |
| ], |
| [ |
| 6, |
| 28, |
| 11 |
| ] |
| ] |
| } |
| }, |
| "overall": { |
| "exact_match": 0.308, |
| "macro_average_f1": 0.6544, |
| "automation_safe_accuracy": 1.0, |
| "automation_safe_coverage": 0.0072, |
| "confidence_threshold": 0.8, |
| "confidence_calibration": { |
| "ece": 0.360595, |
| "bins": [ |
| { |
| "range": [ |
| 0.5, |
| 0.6 |
| ], |
| "count": 43, |
| "avg_confidence": 0.5795, |
| "accuracy": 0.1628 |
| }, |
| { |
| "range": [ |
| 0.6, |
| 0.7 |
| ], |
| "count": 154, |
| "avg_confidence": 0.6525, |
| "accuracy": 0.2338 |
| }, |
| { |
| "range": [ |
| 0.7, |
| 0.8 |
| ], |
| "count": 77, |
| "avg_confidence": 0.737, |
| "accuracy": 0.5195 |
| }, |
| { |
| "range": [ |
| 0.8, |
| 0.9 |
| ], |
| "count": 2, |
| "avg_confidence": 0.8115, |
| "accuracy": 1.0 |
| } |
| ] |
| } |
| }, |
| "training": { |
| "epoch": 6, |
| "loss": 2.5877 |
| } |
| }, |
| { |
| "per_head": { |
| "relation_to_previous": { |
| "accuracy": 0.8659, |
| "macro_f1": 0.7757, |
| "per_label": { |
| "new": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 102 |
| }, |
| "follow_up": { |
| "precision": 0.8387, |
| "recall": 0.9176, |
| "f1": 0.8764, |
| "support": 85 |
| }, |
| "correction": { |
| "precision": 0.6296, |
| "recall": 0.5862, |
| "f1": 0.6071, |
| "support": 29 |
| }, |
| "confirmation": { |
| "precision": 0.875, |
| "recall": 0.7, |
| "f1": 0.7778, |
| "support": 20 |
| }, |
| "cancellation": { |
| "precision": 0.6923, |
| "recall": 0.4286, |
| "f1": 0.5294, |
| "support": 21 |
| }, |
| "closure": { |
| "precision": 0.76, |
| "recall": 1.0, |
| "f1": 0.8636, |
| "support": 19 |
| } |
| }, |
| "confusion_matrix": [ |
| [ |
| 102, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0 |
| ], |
| [ |
| 0, |
| 78, |
| 3, |
| 0, |
| 2, |
| 2 |
| ], |
| [ |
| 0, |
| 8, |
| 17, |
| 1, |
| 2, |
| 1 |
| ], |
| [ |
| 0, |
| 4, |
| 0, |
| 14, |
| 0, |
| 2 |
| ], |
| [ |
| 0, |
| 3, |
| 7, |
| 1, |
| 9, |
| 1 |
| ], |
| [ |
| 0, |
| 0, |
| 0, |
| 0, |
| 0, |
| 19 |
| ] |
| ] |
| }, |
| "actionability": { |
| "accuracy": 0.6812, |
| "macro_f1": 0.6558, |
| "per_label": { |
| "none": { |
| "precision": 0.6032, |
| "recall": 0.623, |
| "f1": 0.6129, |
| "support": 61 |
| }, |
| "review": { |
| "precision": 0.5974, |
| "recall": 0.5823, |
| "f1": 0.5897, |
| "support": 79 |
| }, |
| "act": { |
| "precision": 0.7647, |
| "recall": 0.7647, |
| "f1": 0.7647, |
| "support": 136 |
| } |
| }, |
| "confusion_matrix": [ |
| [ |
| 38, |
| 15, |
| 8 |
| ], |
| [ |
| 9, |
| 46, |
| 24 |
| ], |
| [ |
| 16, |
| 16, |
| 104 |
| ] |
| ] |
| }, |
| "retention": { |
| "accuracy": 0.6848, |
| "macro_f1": 0.6739, |
| "per_label": { |
| "ephemeral": { |
| "precision": 0.6235, |
| "recall": 0.5955, |
| "f1": 0.6092, |
| "support": 89 |
| }, |
| "useful": { |
| "precision": 0.6855, |
| "recall": 0.7786, |
| "f1": 0.7291, |
| "support": 140 |
| }, |
| "remember": { |
| "precision": 0.8438, |
| "recall": 0.5745, |
| "f1": 0.6835, |
| "support": 47 |
| } |
| }, |
| "confusion_matrix": [ |
| [ |
| 53, |
| 35, |
| 1 |
| ], |
| [ |
| 27, |
| 109, |
| 4 |
| ], |
| [ |
| 5, |
| 15, |
| 27 |
| ] |
| ] |
| }, |
| "urgency": { |
| "accuracy": 0.6449, |
| "macro_f1": 0.5761, |
| "per_label": { |
| "low": { |
| "precision": 0.75, |
| "recall": 0.7721, |
| "f1": 0.7609, |
| "support": 136 |
| }, |
| "medium": { |
| "precision": 0.5413, |
| "recall": 0.6211, |
| "f1": 0.5784, |
| "support": 95 |
| }, |
| "high": { |
| "precision": 0.5185, |
| "recall": 0.3111, |
| "f1": 0.3889, |
| "support": 45 |
| } |
| }, |
| "confusion_matrix": [ |
| [ |
| 105, |
| 27, |
| 4 |
| ], |
| [ |
| 27, |
| 59, |
| 9 |
| ], |
| [ |
| 8, |
| 23, |
| 14 |
| ] |
| ] |
| } |
| }, |
| "overall": { |
| "exact_match": 0.3116, |
| "macro_average_f1": 0.6704, |
| "automation_safe_accuracy": 0.8, |
| "automation_safe_coverage": 0.0362, |
| "confidence_threshold": 0.8, |
| "confidence_calibration": { |
| "ece": 0.370519, |
| "bins": [ |
| { |
| "range": [ |
| 0.5, |
| 0.6 |
| ], |
| "count": 24, |
| "avg_confidence": 0.5696, |
| "accuracy": 0.25 |
| }, |
| { |
| "range": [ |
| 0.6, |
| 0.7 |
| ], |
| "count": 141, |
| "avg_confidence": 0.6522, |
| "accuracy": 0.2057 |
| }, |
| { |
| "range": [ |
| 0.7, |
| 0.8 |
| ], |
| "count": 101, |
| "avg_confidence": 0.7378, |
| "accuracy": 0.4257 |
| }, |
| { |
| "range": [ |
| 0.8, |
| 0.9 |
| ], |
| "count": 10, |
| "avg_confidence": 0.8123, |
| "accuracy": 0.8 |
| } |
| ] |
| } |
| }, |
| "training": { |
| "epoch": 7, |
| "loss": 2.4515 |
| } |
| }, |
| { |
| "per_head": { |
| "relation_to_previous": { |
| "accuracy": 0.8768, |
| "macro_f1": 0.7893, |
| "per_label": { |
| "new": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 102 |
| }, |
| "follow_up": { |
| "precision": 0.8764, |
| "recall": 0.9176, |
| "f1": 0.8966, |
| "support": 85 |
| }, |
| "correction": { |
| "precision": 0.6786, |
| "recall": 0.6552, |
| "f1": 0.6667, |
| "support": 29 |
| }, |
| "confirmation": { |
| "precision": 0.8235, |
| "recall": 0.7, |
| "f1": 0.7568, |
| "support": 20 |
| }, |
| "cancellation": { |
| "precision": 0.7143, |
| "recall": 0.4762, |
| "f1": 0.5714, |
| "support": 21 |
| }, |
| "closure": { |
| "precision": 0.7308, |
| "recall": 1.0, |
| "f1": 0.8444, |
| "support": 19 |
| } |
| }, |
| "confusion_matrix": [ |
| [ |
| 102, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0 |
| ], |
| [ |
| 0, |
| 78, |
| 3, |
| 0, |
| 2, |
| 2 |
| ], |
| [ |
| 0, |
| 5, |
| 19, |
| 2, |
| 2, |
| 1 |
| ], |
| [ |
| 0, |
| 3, |
| 0, |
| 14, |
| 0, |
| 3 |
| ], |
| [ |
| 0, |
| 3, |
| 6, |
| 1, |
| 10, |
| 1 |
| ], |
| [ |
| 0, |
| 0, |
| 0, |
| 0, |
| 0, |
| 19 |
| ] |
| ] |
| }, |
| "actionability": { |
| "accuracy": 0.7101, |
| "macro_f1": 0.6834, |
| "per_label": { |
| "none": { |
| "precision": 0.6786, |
| "recall": 0.623, |
| "f1": 0.6496, |
| "support": 61 |
| }, |
| "review": { |
| "precision": 0.6571, |
| "recall": 0.5823, |
| "f1": 0.6174, |
| "support": 79 |
| }, |
| "act": { |
| "precision": 0.7467, |
| "recall": 0.8235, |
| "f1": 0.7832, |
| "support": 136 |
| } |
| }, |
| "confusion_matrix": [ |
| [ |
| 38, |
| 13, |
| 10 |
| ], |
| [ |
| 5, |
| 46, |
| 28 |
| ], |
| [ |
| 13, |
| 11, |
| 112 |
| ] |
| ] |
| }, |
| "retention": { |
| "accuracy": 0.7029, |
| "macro_f1": 0.6849, |
| "per_label": { |
| "ephemeral": { |
| "precision": 0.6628, |
| "recall": 0.6404, |
| "f1": 0.6514, |
| "support": 89 |
| }, |
| "useful": { |
| "precision": 0.7025, |
| "recall": 0.7929, |
| "f1": 0.745, |
| "support": 140 |
| }, |
| "remember": { |
| "precision": 0.8125, |
| "recall": 0.5532, |
| "f1": 0.6582, |
| "support": 47 |
| } |
| }, |
| "confusion_matrix": [ |
| [ |
| 57, |
| 31, |
| 1 |
| ], |
| [ |
| 24, |
| 111, |
| 5 |
| ], |
| [ |
| 5, |
| 16, |
| 26 |
| ] |
| ] |
| }, |
| "urgency": { |
| "accuracy": 0.6449, |
| "macro_f1": 0.5777, |
| "per_label": { |
| "low": { |
| "precision": 0.7536, |
| "recall": 0.7647, |
| "f1": 0.7591, |
| "support": 136 |
| }, |
| "medium": { |
| "precision": 0.5357, |
| "recall": 0.6316, |
| "f1": 0.5797, |
| "support": 95 |
| }, |
| "high": { |
| "precision": 0.5385, |
| "recall": 0.3111, |
| "f1": 0.3944, |
| "support": 45 |
| } |
| }, |
| "confusion_matrix": [ |
| [ |
| 104, |
| 28, |
| 4 |
| ], |
| [ |
| 27, |
| 60, |
| 8 |
| ], |
| [ |
| 7, |
| 24, |
| 14 |
| ] |
| ] |
| } |
| }, |
| "overall": { |
| "exact_match": 0.3406, |
| "macro_average_f1": 0.6838, |
| "automation_safe_accuracy": 0.8, |
| "automation_safe_coverage": 0.0543, |
| "confidence_threshold": 0.8, |
| "confidence_calibration": { |
| "ece": 0.347199, |
| "bins": [ |
| { |
| "range": [ |
| 0.5, |
| 0.6 |
| ], |
| "count": 23, |
| "avg_confidence": 0.572, |
| "accuracy": 0.2174 |
| }, |
| { |
| "range": [ |
| 0.6, |
| 0.7 |
| ], |
| "count": 134, |
| "avg_confidence": 0.6518, |
| "accuracy": 0.2239 |
| }, |
| { |
| "range": [ |
| 0.7, |
| 0.8 |
| ], |
| "count": 104, |
| "avg_confidence": 0.7415, |
| "accuracy": 0.4519 |
| }, |
| { |
| "range": [ |
| 0.8, |
| 0.9 |
| ], |
| "count": 15, |
| "avg_confidence": 0.8136, |
| "accuracy": 0.8 |
| } |
| ] |
| } |
| }, |
| "training": { |
| "epoch": 8, |
| "loss": 2.3349 |
| } |
| }, |
| { |
| "per_head": { |
| "relation_to_previous": { |
| "accuracy": 0.8841, |
| "macro_f1": 0.8031, |
| "per_label": { |
| "new": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 102 |
| }, |
| "follow_up": { |
| "precision": 0.8764, |
| "recall": 0.9176, |
| "f1": 0.8966, |
| "support": 85 |
| }, |
| "correction": { |
| "precision": 0.7, |
| "recall": 0.7241, |
| "f1": 0.7119, |
| "support": 29 |
| }, |
| "confirmation": { |
| "precision": 0.875, |
| "recall": 0.7, |
| "f1": 0.7778, |
| "support": 20 |
| }, |
| "cancellation": { |
| "precision": 0.7692, |
| "recall": 0.4762, |
| "f1": 0.5882, |
| "support": 21 |
| }, |
| "closure": { |
| "precision": 0.7308, |
| "recall": 1.0, |
| "f1": 0.8444, |
| "support": 19 |
| } |
| }, |
| "confusion_matrix": [ |
| [ |
| 102, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0 |
| ], |
| [ |
| 0, |
| 78, |
| 3, |
| 0, |
| 2, |
| 2 |
| ], |
| [ |
| 0, |
| 5, |
| 21, |
| 1, |
| 1, |
| 1 |
| ], |
| [ |
| 0, |
| 3, |
| 0, |
| 14, |
| 0, |
| 3 |
| ], |
| [ |
| 0, |
| 3, |
| 6, |
| 1, |
| 10, |
| 1 |
| ], |
| [ |
| 0, |
| 0, |
| 0, |
| 0, |
| 0, |
| 19 |
| ] |
| ] |
| }, |
| "actionability": { |
| "accuracy": 0.6993, |
| "macro_f1": 0.6742, |
| "per_label": { |
| "none": { |
| "precision": 0.629, |
| "recall": 0.6393, |
| "f1": 0.6341, |
| "support": 61 |
| }, |
| "review": { |
| "precision": 0.6267, |
| "recall": 0.5949, |
| "f1": 0.6104, |
| "support": 79 |
| }, |
| "act": { |
| "precision": 0.7698, |
| "recall": 0.7868, |
| "f1": 0.7782, |
| "support": 136 |
| } |
| }, |
| "confusion_matrix": [ |
| [ |
| 39, |
| 14, |
| 8 |
| ], |
| [ |
| 8, |
| 47, |
| 24 |
| ], |
| [ |
| 15, |
| 14, |
| 107 |
| ] |
| ] |
| }, |
| "retention": { |
| "accuracy": 0.6739, |
| "macro_f1": 0.6684, |
| "per_label": { |
| "ephemeral": { |
| "precision": 0.5842, |
| "recall": 0.6629, |
| "f1": 0.6211, |
| "support": 89 |
| }, |
| "useful": { |
| "precision": 0.7042, |
| "recall": 0.7143, |
| "f1": 0.7092, |
| "support": 140 |
| }, |
| "remember": { |
| "precision": 0.8182, |
| "recall": 0.5745, |
| "f1": 0.675, |
| "support": 47 |
| } |
| }, |
| "confusion_matrix": [ |
| [ |
| 59, |
| 29, |
| 1 |
| ], |
| [ |
| 35, |
| 100, |
| 5 |
| ], |
| [ |
| 7, |
| 13, |
| 27 |
| ] |
| ] |
| }, |
| "urgency": { |
| "accuracy": 0.6413, |
| "macro_f1": 0.5729, |
| "per_label": { |
| "low": { |
| "precision": 0.7413, |
| "recall": 0.7794, |
| "f1": 0.7599, |
| "support": 136 |
| }, |
| "medium": { |
| "precision": 0.5327, |
| "recall": 0.6, |
| "f1": 0.5644, |
| "support": 95 |
| }, |
| "high": { |
| "precision": 0.5385, |
| "recall": 0.3111, |
| "f1": 0.3944, |
| "support": 45 |
| } |
| }, |
| "confusion_matrix": [ |
| [ |
| 106, |
| 26, |
| 4 |
| ], |
| [ |
| 30, |
| 57, |
| 8 |
| ], |
| [ |
| 7, |
| 24, |
| 14 |
| ] |
| ] |
| } |
| }, |
| "overall": { |
| "exact_match": 0.3188, |
| "macro_average_f1": 0.6797, |
| "automation_safe_accuracy": 0.6818, |
| "automation_safe_coverage": 0.0797, |
| "confidence_threshold": 0.8, |
| "confidence_calibration": { |
| "ece": 0.370942, |
| "bins": [ |
| { |
| "range": [ |
| 0.5, |
| 0.6 |
| ], |
| "count": 28, |
| "avg_confidence": 0.5705, |
| "accuracy": 0.1071 |
| }, |
| { |
| "range": [ |
| 0.6, |
| 0.7 |
| ], |
| "count": 129, |
| "avg_confidence": 0.6544, |
| "accuracy": 0.2713 |
| }, |
| { |
| "range": [ |
| 0.7, |
| 0.8 |
| ], |
| "count": 97, |
| "avg_confidence": 0.7429, |
| "accuracy": 0.3608 |
| }, |
| { |
| "range": [ |
| 0.8, |
| 0.9 |
| ], |
| "count": 22, |
| "avg_confidence": 0.815, |
| "accuracy": 0.6818 |
| } |
| ] |
| } |
| }, |
| "training": { |
| "epoch": 9, |
| "loss": 2.2636 |
| } |
| }, |
| { |
| "per_head": { |
| "relation_to_previous": { |
| "accuracy": 0.8804, |
| "macro_f1": 0.7966, |
| "per_label": { |
| "new": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 102 |
| }, |
| "follow_up": { |
| "precision": 0.8764, |
| "recall": 0.9176, |
| "f1": 0.8966, |
| "support": 85 |
| }, |
| "correction": { |
| "precision": 0.6897, |
| "recall": 0.6897, |
| "f1": 0.6897, |
| "support": 29 |
| }, |
| "confirmation": { |
| "precision": 0.875, |
| "recall": 0.7, |
| "f1": 0.7778, |
| "support": 20 |
| }, |
| "cancellation": { |
| "precision": 0.7143, |
| "recall": 0.4762, |
| "f1": 0.5714, |
| "support": 21 |
| }, |
| "closure": { |
| "precision": 0.7308, |
| "recall": 1.0, |
| "f1": 0.8444, |
| "support": 19 |
| } |
| }, |
| "confusion_matrix": [ |
| [ |
| 102, |
| 0, |
| 0, |
| 0, |
| 0, |
| 0 |
| ], |
| [ |
| 0, |
| 78, |
| 3, |
| 0, |
| 2, |
| 2 |
| ], |
| [ |
| 0, |
| 5, |
| 20, |
| 1, |
| 2, |
| 1 |
| ], |
| [ |
| 0, |
| 3, |
| 0, |
| 14, |
| 0, |
| 3 |
| ], |
| [ |
| 0, |
| 3, |
| 6, |
| 1, |
| 10, |
| 1 |
| ], |
| [ |
| 0, |
| 0, |
| 0, |
| 0, |
| 0, |
| 19 |
| ] |
| ] |
| }, |
| "actionability": { |
| "accuracy": 0.7174, |
| "macro_f1": 0.697, |
| "per_label": { |
| "none": { |
| "precision": 0.6557, |
| "recall": 0.6557, |
| "f1": 0.6557, |
| "support": 61 |
| }, |
| "review": { |
| "precision": 0.642, |
| "recall": 0.6582, |
| "f1": 0.65, |
| "support": 79 |
| }, |
| "act": { |
| "precision": 0.791, |
| "recall": 0.7794, |
| "f1": 0.7852, |
| "support": 136 |
| } |
| }, |
| "confusion_matrix": [ |
| [ |
| 40, |
| 14, |
| 7 |
| ], |
| [ |
| 6, |
| 52, |
| 21 |
| ], |
| [ |
| 15, |
| 15, |
| 106 |
| ] |
| ] |
| }, |
| "retention": { |
| "accuracy": 0.6848, |
| "macro_f1": 0.6687, |
| "per_label": { |
| "ephemeral": { |
| "precision": 0.6222, |
| "recall": 0.6292, |
| "f1": 0.6257, |
| "support": 89 |
| }, |
| "useful": { |
| "precision": 0.6993, |
| "recall": 0.7643, |
| "f1": 0.7304, |
| "support": 140 |
| }, |
| "remember": { |
| "precision": 0.7879, |
| "recall": 0.5532, |
| "f1": 0.65, |
| "support": 47 |
| } |
| }, |
| "confusion_matrix": [ |
| [ |
| 56, |
| 32, |
| 1 |
| ], |
| [ |
| 27, |
| 107, |
| 6 |
| ], |
| [ |
| 7, |
| 14, |
| 26 |
| ] |
| ] |
| }, |
| "urgency": { |
| "accuracy": 0.6304, |
| "macro_f1": 0.5648, |
| "per_label": { |
| "low": { |
| "precision": 0.7324, |
| "recall": 0.7647, |
| "f1": 0.7482, |
| "support": 136 |
| }, |
| "medium": { |
| "precision": 0.5185, |
| "recall": 0.5895, |
| "f1": 0.5517, |
| "support": 95 |
| }, |
| "high": { |
| "precision": 0.5385, |
| "recall": 0.3111, |
| "f1": 0.3944, |
| "support": 45 |
| } |
| }, |
| "confusion_matrix": [ |
| [ |
| 104, |
| 28, |
| 4 |
| ], |
| [ |
| 31, |
| 56, |
| 8 |
| ], |
| [ |
| 7, |
| 24, |
| 14 |
| ] |
| ] |
| } |
| }, |
| "overall": { |
| "exact_match": 0.337, |
| "macro_average_f1": 0.6818, |
| "automation_safe_accuracy": 0.7, |
| "automation_safe_coverage": 0.0725, |
| "confidence_threshold": 0.8, |
| "confidence_calibration": { |
| "ece": 0.357574, |
| "bins": [ |
| { |
| "range": [ |
| 0.5, |
| 0.6 |
| ], |
| "count": 22, |
| "avg_confidence": 0.5729, |
| "accuracy": 0.1364 |
| }, |
| { |
| "range": [ |
| 0.6, |
| 0.7 |
| ], |
| "count": 128, |
| "avg_confidence": 0.6553, |
| "accuracy": 0.2344 |
| }, |
| { |
| "range": [ |
| 0.7, |
| 0.8 |
| ], |
| "count": 106, |
| "avg_confidence": 0.7441, |
| "accuracy": 0.434 |
| }, |
| { |
| "range": [ |
| 0.8, |
| 0.9 |
| ], |
| "count": 20, |
| "avg_confidence": 0.817, |
| "accuracy": 0.7 |
| } |
| ] |
| } |
| }, |
| "training": { |
| "epoch": 10, |
| "loss": 2.2342 |
| } |
| } |
| ], |
| "best_macro_average_f1": 0.6838 |
| } |
