{
  "handler": {
    "task": "handler",
    "best_epoch": 5,
    "training_seconds": 304.73208945803344,
    "split_sizes": {
      "train": 2400,
      "validation": 300,
      "test": 300
    },
    "validation": {
      "loss": 0.5876449354489645,
      "accuracy": 0.8466666666666667,
      "macro_f1": 0.8471922277069336
    },
    "test": {
      "loss": 0.6294241340955099,
      "accuracy": 0.8166666666666667,
      "macro_f1": 0.816800011054457
    },
    "test_confusions": [
      {
        "gold": "human",
        "predicted": "llm",
        "count": 18
      },
      {
        "gold": "llm",
        "predicted": "auto",
        "count": 15
      },
      {
        "gold": "llm",
        "predicted": "human",
        "count": 10
      },
      {
        "gold": "auto",
        "predicted": "llm",
        "count": 5
      },
      {
        "gold": "auto",
        "predicted": "human",
        "count": 5
      },
      {
        "gold": "human",
        "predicted": "auto",
        "count": 2
      }
    ],
    "history": [
      {
        "epoch": 1,
        "train_loss": 0.9450590944290161,
        "validation_loss": 0.640375197728475,
        "validation_accuracy": 0.7833333333333333,
        "validation_macro_f1": 0.7873895322407908
      },
      {
        "epoch": 2,
        "train_loss": 0.3881728690862656,
        "validation_loss": 0.5689977049827576,
        "validation_accuracy": 0.8366666666666667,
        "validation_macro_f1": 0.8295788111791534
      },
      {
        "epoch": 3,
        "train_loss": 0.18638452569643657,
        "validation_loss": 0.6046478613217672,
        "validation_accuracy": 0.8033333333333333,
        "validation_macro_f1": 0.8073069144063059
      },
      {
        "epoch": 4,
        "train_loss": 0.17458565334479015,
        "validation_loss": 0.5988235028584799,
        "validation_accuracy": 0.8333333333333334,
        "validation_macro_f1": 0.8331806628375255
      },
      {
        "epoch": 5,
        "train_loss": 0.17275874733924865,
        "validation_loss": 0.5876449354489645,
        "validation_accuracy": 0.8466666666666667,
        "validation_macro_f1": 0.8471922277069336
      },
      {
        "epoch": 6,
        "train_loss": 0.17210762143135072,
        "validation_loss": 0.5890995128949483,
        "validation_accuracy": 0.84,
        "validation_macro_f1": 0.839868859643578
      },
      {
        "epoch": 7,
        "train_loss": 0.17215188225110373,
        "validation_loss": 0.5889288687705994,
        "validation_accuracy": 0.84,
        "validation_macro_f1": 0.8401046298542054
      },
      {
        "epoch": 8,
        "train_loss": 0.1719669226805369,
        "validation_loss": 0.5910564653078715,
        "validation_accuracy": 0.8366666666666667,
        "validation_macro_f1": 0.8366387893269615
      }
    ]
  },
  "human": {
    "task": "human",
    "best_epoch": 6,
    "training_seconds": 111.95542904100148,
    "split_sizes": {
      "train": 880,
      "validation": 110,
      "test": 110
    },
    "validation": {
      "loss": 1.267178336056796,
      "accuracy": 0.8363636363636363,
      "macro_f1": 0.7900596877869606
    },
    "test": {
      "loss": 1.4121767195788297,
      "accuracy": 0.7454545454545455,
      "macro_f1": 0.6897269397269397
    },
    "test_confusions": [
      {
        "gold": "multi_intent_with_financial_risk",
        "predicted": "refund_missing_after_deadline",
        "count": 5
      },
      {
        "gold": "account_takeover",
        "predicted": "personal_data_or_payment_card_exposed",
        "count": 5
      },
      {
        "gold": "order_marked_delivered_but_not_received",
        "predicted": "multi_intent_with_financial_risk",
        "count": 5
      },
      {
        "gold": "change_delivery_address_after_dispatch",
        "predicted": "cancel_order_after_dispatch",
        "count": 5
      },
      {
        "gold": "fraud_or_phishing",
        "predicted": "account_takeover",
        "count": 3
      },
      {
        "gold": "missing_item_in_delivered_order",
        "predicted": "refund_amount_dispute",
        "count": 3
      },
      {
        "gold": "fraud_or_phishing",
        "predicted": "privacy_data_deletion_request",
        "count": 2
      }
    ],
    "history": [
      {
        "epoch": 1,
        "train_loss": 3.0446458036249333,
        "validation_loss": 2.9481882918964732,
        "validation_accuracy": 0.3181818181818182,
        "validation_macro_f1": 0.275695272469466
      },
      {
        "epoch": 2,
        "train_loss": 2.5696960145776924,
        "validation_loss": 2.4496376210992987,
        "validation_accuracy": 0.5818181818181818,
        "validation_macro_f1": 0.5069444444444445
      },
      {
        "epoch": 3,
        "train_loss": 1.8286097873340954,
        "validation_loss": 1.8624537467956543,
        "validation_accuracy": 0.7909090909090909,
        "validation_macro_f1": 0.7550946528219256
      },
      {
        "epoch": 4,
        "train_loss": 1.237558323686773,
        "validation_loss": 1.5112867398695513,
        "validation_accuracy": 0.8,
        "validation_macro_f1": 0.7386363636363636
      },
      {
        "epoch": 5,
        "train_loss": 0.899948058345101,
        "validation_loss": 1.34841853055087,
        "validation_accuracy": 0.8272727272727273,
        "validation_macro_f1": 0.7803030303030304
      },
      {
        "epoch": 6,
        "train_loss": 0.7283082962036133,
        "validation_loss": 1.267178336056796,
        "validation_accuracy": 0.8363636363636363,
        "validation_macro_f1": 0.7900596877869606
      },
      {
        "epoch": 7,
        "train_loss": 0.6527352463115346,
        "validation_loss": 1.2407976128838278,
        "validation_accuracy": 0.8090909090909091,
        "validation_macro_f1": 0.7660697887970614
      },
      {
        "epoch": 8,
        "train_loss": 0.6315497528422963,
        "validation_loss": 1.2309001250700518,
        "validation_accuracy": 0.8181818181818182,
        "validation_macro_f1": 0.7732847960120687
      }
    ]
  },
  "llm": {
    "task": "llm",
    "best_epoch": 4,
    "training_seconds": 87.89807533397106,
    "split_sizes": {
      "train": 800,
      "validation": 100,
      "test": 100
    },
    "validation": {
      "loss": 1.663049850463867,
      "accuracy": 0.71,
      "macro_f1": 0.6540935672514621
    },
    "test": {
      "loss": 1.4101154947280883,
      "accuracy": 0.8,
      "macro_f1": 0.754532967032967
    },
    "test_confusions": [
      {
        "gold": "possible_mass_incident",
        "predicted": "damaged_item_first_steps",
        "count": 5
      },
      {
        "gold": "payment_pending_explanation",
        "predicted": "loyalty_points_missing",
        "count": 5
      },
      {
        "gold": "delivery_delay",
        "predicted": "change_delivery_address_before_dispatch",
        "count": 5
      },
      {
        "gold": "otp_code_not_received",
        "predicted": "promo_code_not_applied",
        "count": 3
      },
      {
        "gold": "refund_status_before_deadline",
        "predicted": "payment_pending_explanation",
        "count": 2
      }
    ],
    "history": [
      {
        "epoch": 1,
        "train_loss": 2.9181075382232664,
        "validation_loss": 2.8307441329956053,
        "validation_accuracy": 0.4,
        "validation_macro_f1": 0.3274270868020868
      },
      {
        "epoch": 2,
        "train_loss": 2.467692804336548,
        "validation_loss": 2.37255952835083,
        "validation_accuracy": 0.63,
        "validation_macro_f1": 0.5793831168831168
      },
      {
        "epoch": 3,
        "train_loss": 1.7932574939727783,
        "validation_loss": 1.940031542778015,
        "validation_accuracy": 0.7,
        "validation_macro_f1": 0.6371212121212121
      },
      {
        "epoch": 4,
        "train_loss": 1.2383545541763306,
        "validation_loss": 1.663049850463867,
        "validation_accuracy": 0.71,
        "validation_macro_f1": 0.6540935672514621
      },
      {
        "epoch": 5,
        "train_loss": 0.9075340437889099,
        "validation_loss": 1.513033676147461,
        "validation_accuracy": 0.71,
        "validation_macro_f1": 0.6506944444444445
      },
      {
        "epoch": 6,
        "train_loss": 0.7291339612007142,
        "validation_loss": 1.4201061391830445,
        "validation_accuracy": 0.71,
        "validation_macro_f1": 0.6527777777777778
      },
      {
        "epoch": 7,
        "train_loss": 0.6559483313560486,
        "validation_loss": 1.3904729557037354,
        "validation_accuracy": 0.71,
        "validation_macro_f1": 0.6506944444444445
      }
    ]
  },
  "auto": {
    "task": "auto",
    "best_epoch": 3,
    "training_seconds": 68.58642412500922,
    "split_sizes": {
      "train": 720,
      "validation": 90,
      "test": 90
    },
    "validation": {
      "loss": 1.5290983120600383,
      "accuracy": 0.8888888888888888,
      "macro_f1": 0.8611111111111112
    },
    "test": {
      "loss": 1.6726625601450602,
      "accuracy": 0.7777777777777778,
      "macro_f1": 0.712962962962963
    },
    "test_confusions": [
      {
        "gold": "notification_settings",
        "predicted": "product_availability_notification",
        "count": 5
      },
      {
        "gold": "subscription_features",
        "predicted": "general_refund_timeline",
        "count": 5
      },
      {
        "gold": "how_to_contact_support",
        "predicted": "app_update_instructions",
        "count": 5
      },
      {
        "gold": "password_reset_instructions",
        "predicted": "product_availability_notification",
        "count": 5
      }
    ],
    "history": [
      {
        "epoch": 1,
        "train_loss": 2.8345796267191568,
        "validation_loss": 2.685704141192966,
        "validation_accuracy": 0.4222222222222222,
        "validation_macro_f1": 0.36838347901566293
      },
      {
        "epoch": 2,
        "train_loss": 2.3430909898546006,
        "validation_loss": 2.083689069747925,
        "validation_accuracy": 0.7333333333333333,
        "validation_macro_f1": 0.6763407072230602
      },
      {
        "epoch": 3,
        "train_loss": 1.6193762381871541,
        "validation_loss": 1.5290983120600383,
        "validation_accuracy": 0.8888888888888888,
        "validation_macro_f1": 0.8611111111111112
      },
      {
        "epoch": 4,
        "train_loss": 1.0734790404637655,
        "validation_loss": 1.1841111924913195,
        "validation_accuracy": 0.8888888888888888,
        "validation_macro_f1": 0.8611111111111112
      },
      {
        "epoch": 5,
        "train_loss": 0.7524434738688999,
        "validation_loss": 1.0104022343953452,
        "validation_accuracy": 0.8888888888888888,
        "validation_macro_f1": 0.8611111111111112
      },
      {
        "epoch": 6,
        "train_loss": 0.6061955703629388,
        "validation_loss": 0.9299269636472066,
        "validation_accuracy": 0.8888888888888888,
        "validation_macro_f1": 0.8611111111111112
      }
    ]
  }
}