| { |
| "handler": { |
| "task": "handler", |
| "best_epoch": 5, |
| "training_seconds": 304.73208945803344, |
| "split_sizes": { |
| "train": 2400, |
| "validation": 300, |
| "test": 300 |
| }, |
| "validation": { |
| "loss": 0.5876449354489645, |
| "accuracy": 0.8466666666666667, |
| "macro_f1": 0.8471922277069336 |
| }, |
| "test": { |
| "loss": 0.6294241340955099, |
| "accuracy": 0.8166666666666667, |
| "macro_f1": 0.816800011054457 |
| }, |
| "test_confusions": [ |
| { |
| "gold": "human", |
| "predicted": "llm", |
| "count": 18 |
| }, |
| { |
| "gold": "llm", |
| "predicted": "auto", |
| "count": 15 |
| }, |
| { |
| "gold": "llm", |
| "predicted": "human", |
| "count": 10 |
| }, |
| { |
| "gold": "auto", |
| "predicted": "llm", |
| "count": 5 |
| }, |
| { |
| "gold": "auto", |
| "predicted": "human", |
| "count": 5 |
| }, |
| { |
| "gold": "human", |
| "predicted": "auto", |
| "count": 2 |
| } |
| ], |
| "history": [ |
| { |
| "epoch": 1, |
| "train_loss": 0.9450590944290161, |
| "validation_loss": 0.640375197728475, |
| "validation_accuracy": 0.7833333333333333, |
| "validation_macro_f1": 0.7873895322407908 |
| }, |
| { |
| "epoch": 2, |
| "train_loss": 0.3881728690862656, |
| "validation_loss": 0.5689977049827576, |
| "validation_accuracy": 0.8366666666666667, |
| "validation_macro_f1": 0.8295788111791534 |
| }, |
| { |
| "epoch": 3, |
| "train_loss": 0.18638452569643657, |
| "validation_loss": 0.6046478613217672, |
| "validation_accuracy": 0.8033333333333333, |
| "validation_macro_f1": 0.8073069144063059 |
| }, |
| { |
| "epoch": 4, |
| "train_loss": 0.17458565334479015, |
| "validation_loss": 0.5988235028584799, |
| "validation_accuracy": 0.8333333333333334, |
| "validation_macro_f1": 0.8331806628375255 |
| }, |
| { |
| "epoch": 5, |
| "train_loss": 0.17275874733924865, |
| "validation_loss": 0.5876449354489645, |
| "validation_accuracy": 0.8466666666666667, |
| "validation_macro_f1": 0.8471922277069336 |
| }, |
| { |
| "epoch": 6, |
| "train_loss": 0.17210762143135072, |
| "validation_loss": 0.5890995128949483, |
| "validation_accuracy": 0.84, |
| "validation_macro_f1": 0.839868859643578 |
| }, |
| { |
| "epoch": 7, |
| "train_loss": 0.17215188225110373, |
| "validation_loss": 0.5889288687705994, |
| "validation_accuracy": 0.84, |
| "validation_macro_f1": 0.8401046298542054 |
| }, |
| { |
| "epoch": 8, |
| "train_loss": 0.1719669226805369, |
| "validation_loss": 0.5910564653078715, |
| "validation_accuracy": 0.8366666666666667, |
| "validation_macro_f1": 0.8366387893269615 |
| } |
| ] |
| }, |
| "human": { |
| "task": "human", |
| "best_epoch": 6, |
| "training_seconds": 111.95542904100148, |
| "split_sizes": { |
| "train": 880, |
| "validation": 110, |
| "test": 110 |
| }, |
| "validation": { |
| "loss": 1.267178336056796, |
| "accuracy": 0.8363636363636363, |
| "macro_f1": 0.7900596877869606 |
| }, |
| "test": { |
| "loss": 1.4121767195788297, |
| "accuracy": 0.7454545454545455, |
| "macro_f1": 0.6897269397269397 |
| }, |
| "test_confusions": [ |
| { |
| "gold": "multi_intent_with_financial_risk", |
| "predicted": "refund_missing_after_deadline", |
| "count": 5 |
| }, |
| { |
| "gold": "account_takeover", |
| "predicted": "personal_data_or_payment_card_exposed", |
| "count": 5 |
| }, |
| { |
| "gold": "order_marked_delivered_but_not_received", |
| "predicted": "multi_intent_with_financial_risk", |
| "count": 5 |
| }, |
| { |
| "gold": "change_delivery_address_after_dispatch", |
| "predicted": "cancel_order_after_dispatch", |
| "count": 5 |
| }, |
| { |
| "gold": "fraud_or_phishing", |
| "predicted": "account_takeover", |
| "count": 3 |
| }, |
| { |
| "gold": "missing_item_in_delivered_order", |
| "predicted": "refund_amount_dispute", |
| "count": 3 |
| }, |
| { |
| "gold": "fraud_or_phishing", |
| "predicted": "privacy_data_deletion_request", |
| "count": 2 |
| } |
| ], |
| "history": [ |
| { |
| "epoch": 1, |
| "train_loss": 3.0446458036249333, |
| "validation_loss": 2.9481882918964732, |
| "validation_accuracy": 0.3181818181818182, |
| "validation_macro_f1": 0.275695272469466 |
| }, |
| { |
| "epoch": 2, |
| "train_loss": 2.5696960145776924, |
| "validation_loss": 2.4496376210992987, |
| "validation_accuracy": 0.5818181818181818, |
| "validation_macro_f1": 0.5069444444444445 |
| }, |
| { |
| "epoch": 3, |
| "train_loss": 1.8286097873340954, |
| "validation_loss": 1.8624537467956543, |
| "validation_accuracy": 0.7909090909090909, |
| "validation_macro_f1": 0.7550946528219256 |
| }, |
| { |
| "epoch": 4, |
| "train_loss": 1.237558323686773, |
| "validation_loss": 1.5112867398695513, |
| "validation_accuracy": 0.8, |
| "validation_macro_f1": 0.7386363636363636 |
| }, |
| { |
| "epoch": 5, |
| "train_loss": 0.899948058345101, |
| "validation_loss": 1.34841853055087, |
| "validation_accuracy": 0.8272727272727273, |
| "validation_macro_f1": 0.7803030303030304 |
| }, |
| { |
| "epoch": 6, |
| "train_loss": 0.7283082962036133, |
| "validation_loss": 1.267178336056796, |
| "validation_accuracy": 0.8363636363636363, |
| "validation_macro_f1": 0.7900596877869606 |
| }, |
| { |
| "epoch": 7, |
| "train_loss": 0.6527352463115346, |
| "validation_loss": 1.2407976128838278, |
| "validation_accuracy": 0.8090909090909091, |
| "validation_macro_f1": 0.7660697887970614 |
| }, |
| { |
| "epoch": 8, |
| "train_loss": 0.6315497528422963, |
| "validation_loss": 1.2309001250700518, |
| "validation_accuracy": 0.8181818181818182, |
| "validation_macro_f1": 0.7732847960120687 |
| } |
| ] |
| }, |
| "llm": { |
| "task": "llm", |
| "best_epoch": 4, |
| "training_seconds": 87.89807533397106, |
| "split_sizes": { |
| "train": 800, |
| "validation": 100, |
| "test": 100 |
| }, |
| "validation": { |
| "loss": 1.663049850463867, |
| "accuracy": 0.71, |
| "macro_f1": 0.6540935672514621 |
| }, |
| "test": { |
| "loss": 1.4101154947280883, |
| "accuracy": 0.8, |
| "macro_f1": 0.754532967032967 |
| }, |
| "test_confusions": [ |
| { |
| "gold": "possible_mass_incident", |
| "predicted": "damaged_item_first_steps", |
| "count": 5 |
| }, |
| { |
| "gold": "payment_pending_explanation", |
| "predicted": "loyalty_points_missing", |
| "count": 5 |
| }, |
| { |
| "gold": "delivery_delay", |
| "predicted": "change_delivery_address_before_dispatch", |
| "count": 5 |
| }, |
| { |
| "gold": "otp_code_not_received", |
| "predicted": "promo_code_not_applied", |
| "count": 3 |
| }, |
| { |
| "gold": "refund_status_before_deadline", |
| "predicted": "payment_pending_explanation", |
| "count": 2 |
| } |
| ], |
| "history": [ |
| { |
| "epoch": 1, |
| "train_loss": 2.9181075382232664, |
| "validation_loss": 2.8307441329956053, |
| "validation_accuracy": 0.4, |
| "validation_macro_f1": 0.3274270868020868 |
| }, |
| { |
| "epoch": 2, |
| "train_loss": 2.467692804336548, |
| "validation_loss": 2.37255952835083, |
| "validation_accuracy": 0.63, |
| "validation_macro_f1": 0.5793831168831168 |
| }, |
| { |
| "epoch": 3, |
| "train_loss": 1.7932574939727783, |
| "validation_loss": 1.940031542778015, |
| "validation_accuracy": 0.7, |
| "validation_macro_f1": 0.6371212121212121 |
| }, |
| { |
| "epoch": 4, |
| "train_loss": 1.2383545541763306, |
| "validation_loss": 1.663049850463867, |
| "validation_accuracy": 0.71, |
| "validation_macro_f1": 0.6540935672514621 |
| }, |
| { |
| "epoch": 5, |
| "train_loss": 0.9075340437889099, |
| "validation_loss": 1.513033676147461, |
| "validation_accuracy": 0.71, |
| "validation_macro_f1": 0.6506944444444445 |
| }, |
| { |
| "epoch": 6, |
| "train_loss": 0.7291339612007142, |
| "validation_loss": 1.4201061391830445, |
| "validation_accuracy": 0.71, |
| "validation_macro_f1": 0.6527777777777778 |
| }, |
| { |
| "epoch": 7, |
| "train_loss": 0.6559483313560486, |
| "validation_loss": 1.3904729557037354, |
| "validation_accuracy": 0.71, |
| "validation_macro_f1": 0.6506944444444445 |
| } |
| ] |
| }, |
| "auto": { |
| "task": "auto", |
| "best_epoch": 3, |
| "training_seconds": 68.58642412500922, |
| "split_sizes": { |
| "train": 720, |
| "validation": 90, |
| "test": 90 |
| }, |
| "validation": { |
| "loss": 1.5290983120600383, |
| "accuracy": 0.8888888888888888, |
| "macro_f1": 0.8611111111111112 |
| }, |
| "test": { |
| "loss": 1.6726625601450602, |
| "accuracy": 0.7777777777777778, |
| "macro_f1": 0.712962962962963 |
| }, |
| "test_confusions": [ |
| { |
| "gold": "notification_settings", |
| "predicted": "product_availability_notification", |
| "count": 5 |
| }, |
| { |
| "gold": "subscription_features", |
| "predicted": "general_refund_timeline", |
| "count": 5 |
| }, |
| { |
| "gold": "how_to_contact_support", |
| "predicted": "app_update_instructions", |
| "count": 5 |
| }, |
| { |
| "gold": "password_reset_instructions", |
| "predicted": "product_availability_notification", |
| "count": 5 |
| } |
| ], |
| "history": [ |
| { |
| "epoch": 1, |
| "train_loss": 2.8345796267191568, |
| "validation_loss": 2.685704141192966, |
| "validation_accuracy": 0.4222222222222222, |
| "validation_macro_f1": 0.36838347901566293 |
| }, |
| { |
| "epoch": 2, |
| "train_loss": 2.3430909898546006, |
| "validation_loss": 2.083689069747925, |
| "validation_accuracy": 0.7333333333333333, |
| "validation_macro_f1": 0.6763407072230602 |
| }, |
| { |
| "epoch": 3, |
| "train_loss": 1.6193762381871541, |
| "validation_loss": 1.5290983120600383, |
| "validation_accuracy": 0.8888888888888888, |
| "validation_macro_f1": 0.8611111111111112 |
| }, |
| { |
| "epoch": 4, |
| "train_loss": 1.0734790404637655, |
| "validation_loss": 1.1841111924913195, |
| "validation_accuracy": 0.8888888888888888, |
| "validation_macro_f1": 0.8611111111111112 |
| }, |
| { |
| "epoch": 5, |
| "train_loss": 0.7524434738688999, |
| "validation_loss": 1.0104022343953452, |
| "validation_accuracy": 0.8888888888888888, |
| "validation_macro_f1": 0.8611111111111112 |
| }, |
| { |
| "epoch": 6, |
| "train_loss": 0.6061955703629388, |
| "validation_loss": 0.9299269636472066, |
| "validation_accuracy": 0.8888888888888888, |
| "validation_macro_f1": 0.8611111111111112 |
| } |
| ] |
| } |
| } |
|
|
