laya-banking77 / eval.json
VibeHash's picture
Add laya-banking77: Laya fine-tuned on Banking77, its scores and the code that made it
4e09985 verified
Raw History Blame Contribute Delete
2.9 kB
{
"name": "seed0",
"test_sha256": "ebc2da4677c6a810d0384e696a427c2177cad289248ed59fe3ef8366e187404b",
"head_max_len": 512,
"batched": {
"batch_size": 32,
"seconds": 32.058506000001216,
"ms_per_message": 10.40860584415624
},
"n": 3080,
"right": 2817,
"accuracy": 0.9146103896103897,
"ci95": [
0.9042170986779753,
0.9239707423975795
],
"mean_confidence": 0.9173325324675324,
"ece": 0.03548967532467404,
"gate": {
"threshold": 0.979,
"answered": 1523,
"coverage": 0.49448051948051946,
"accuracy": 0.9881812212738017,
"errors": 18
},
"curve": [
{
"target": 0.95,
"threshold": 0.7374,
"answered": 2829,
"coverage": 0.9185064935064935,
"accuracy": 0.9501590668080594
},
{
"target": 0.98,
"threshold": 0.9605,
"answered": 2163,
"coverage": 0.7022727272727273,
"accuracy": 0.9801202034211743
},
{
"target": 0.99,
"threshold": 0.9806,
"answered": 1427,
"coverage": 0.4633116883116883,
"accuracy": 0.9901892081289418
}
],
"weakest_intents": [
[
"fiat currency support",
0.75
],
[
"declined transfer",
0.775
],
[
"pending transfer",
0.775
],
[
"balance not updated after bank transfer",
0.775
],
[
"top up by bank transfer charge",
0.8
],
[
"top up reverted",
0.8
],
[
"direct debit payment not recognised",
0.8
],
[
"topping up by card",
0.825
]
],
"top_confusions": [
{
"gold": "fiat currency support",
"pred": "exchange via app",
"count": 6
},
{
"gold": "top up reverted",
"pred": "top up failed",
"count": 5
},
{
"gold": "card arrival",
"pred": "card delivery estimate",
"count": 4
},
{
"gold": "card delivery estimate",
"pred": "card arrival",
"count": 4
},
{
"gold": "top up by bank transfer charge",
"pred": "transfer fee charged",
"count": 4
},
{
"gold": "pending transfer",
"pred": "transfer timing",
"count": 4
},
{
"gold": "beneficiary not allowed",
"pred": "failed transfer",
"count": 4
},
{
"gold": "card payment fee charged",
"pred": "extra charge on statement",
"count": 3
},
{
"gold": "supported cards and currencies",
"pred": "getting spare card",
"count": 3
},
{
"gold": "compromised card",
"pred": "terminate account",
"count": 3
}
],
"single": {
"n": 200,
"median_ms": 22.391899983631447,
"p95_ms": 24.00200002011843
}
}