File size: 2,636 Bytes
ee6c673 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 | {
"model": "pace-intent-router-v8",
"eval_date": "2026-07-13",
"eval_split": "pace-intent-heldout-v5",
"eval_size": 14995,
"overall_accuracy": 0.9552517505835278,
"latency_ms": {
"p50": 3.1020641326904297,
"mean": 3.2842456757843435
},
"per_class": {
"chitchat": {
"accuracy": 0.9381672597864769,
"count": 2248
},
"phoneLargeModel": {
"accuracy": 0.9844444444444445,
"count": 450
},
"pureKnowledge": {
"accuracy": 0.964,
"count": 3000
},
"research": {
"accuracy": 0.9344444444444444,
"count": 900
},
"screenAction": {
"accuracy": 0.9678279713285548,
"count": 5999
},
"screenDescription": {
"accuracy": 0.9760978321289605,
"count": 1799
},
"unknown": {
"accuracy": 0.7963272120200334,
"count": 599
}
},
"baselines": {
"apple-fm-3b": {
"overall_accuracy": 0.765,
"eval_size": 200,
"latency_mean_ms": 1597,
"error_rate": 0.005,
"source": "pace/leanring-buddyTests/PaceFMIntentClassifierBenchmarkTests.swift",
"notes": "200 stratified examples, 1s delay between calls, fresh session per call. FM conflates research with pureKnowledge (33.3% vs 93.4%). One refusal."
},
"qwen3-4b-instruct-4bit": {
"overall_accuracy": 0.847542627883651,
"eval_size": 997,
"latency_p50_ms": 240,
"source": "runs/2026-07-13-pace-intent-router-v1/eval-qwen-baseline.json"
},
"pace-intent-router-v5": {
"overall_accuracy": 0.9590530176725576,
"eval_size": 14995,
"latency_p50_ms": 4.642009735107422,
"source": "runs/2026-07-13-pace-intent-router-v1/eval-candidate-v5.json"
}
},
"head_to_head": {
"vs_apple_fm": {
"accuracy_delta_pp": 19.0,
"latency_speedup": "515x",
"note": "FM is 3B in-process but beats it on every class. FM conflates research with pureKnowledge."
},
"vs_qwen3_4b": {
"accuracy_delta_pp": 10.77,
"latency_speedup": "77x",
"size_ratio": "4x smaller"
},
"vs_v5": {
"accuracy_delta_pp": -0.38,
"unknown_class_delta_pp": 9.35,
"note": "v8 trades slight overall accuracy for significantly better unknown-class recall"
}
},
"caveats": [
"All eval data is synthetic — real-world distribution will differ",
"Unknown class accuracy (79.6%) is measured on synthetic unknowns",
"Not wired into shipping Pace app yet — Apple FM is the current production classifier despite lower measured accuracy (76.5% vs 95.5%)",
"Byte-level vocab truncates queries at 128 bytes"
]
}
|