{
  "model": "pace-intent-router-v8",
  "eval_date": "2026-07-13",
  "eval_split": "pace-intent-heldout-v5",
  "eval_size": 14995,
  "overall_accuracy": 0.9552517505835278,
  "latency_ms": {
    "p50": 3.1020641326904297,
    "mean": 3.2842456757843435
  },
  "per_class": {
    "chitchat": {
      "accuracy": 0.9381672597864769,
      "count": 2248
    },
    "phoneLargeModel": {
      "accuracy": 0.9844444444444445,
      "count": 450
    },
    "pureKnowledge": {
      "accuracy": 0.964,
      "count": 3000
    },
    "research": {
      "accuracy": 0.9344444444444444,
      "count": 900
    },
    "screenAction": {
      "accuracy": 0.9678279713285548,
      "count": 5999
    },
    "screenDescription": {
      "accuracy": 0.9760978321289605,
      "count": 1799
    },
    "unknown": {
      "accuracy": 0.7963272120200334,
      "count": 599
    }
  },
  "baselines": {
    "apple-fm-3b": {
      "overall_accuracy": 0.765,
      "eval_size": 200,
      "latency_mean_ms": 1597,
      "error_rate": 0.005,
      "source": "pace/leanring-buddyTests/PaceFMIntentClassifierBenchmarkTests.swift",
      "notes": "200 stratified examples, 1s delay between calls, fresh session per call. FM conflates research with pureKnowledge (33.3% vs 93.4%). One refusal."
    },
    "qwen3-4b-instruct-4bit": {
      "overall_accuracy": 0.847542627883651,
      "eval_size": 997,
      "latency_p50_ms": 240,
      "source": "runs/2026-07-13-pace-intent-router-v1/eval-qwen-baseline.json"
    },
    "pace-intent-router-v5": {
      "overall_accuracy": 0.9590530176725576,
      "eval_size": 14995,
      "latency_p50_ms": 4.642009735107422,
      "source": "runs/2026-07-13-pace-intent-router-v1/eval-candidate-v5.json"
    }
  },
  "head_to_head": {
    "vs_apple_fm": {
      "accuracy_delta_pp": 19.0,
      "latency_speedup": "515x",
      "note": "FM is 3B in-process but beats it on every class. FM conflates research with pureKnowledge."
    },
    "vs_qwen3_4b": {
      "accuracy_delta_pp": 10.77,
      "latency_speedup": "77x",
      "size_ratio": "4x smaller"
    },
    "vs_v5": {
      "accuracy_delta_pp": -0.38,
      "unknown_class_delta_pp": 9.35,
      "note": "v8 trades slight overall accuracy for significantly better unknown-class recall"
    }
  },
  "caveats": [
    "All eval data is synthetic — real-world distribution will differ",
    "Unknown class accuracy (79.6%) is measured on synthetic unknowns",
    "Not wired into shipping Pace app yet — Apple FM is the current production classifier despite lower measured accuracy (76.5% vs 95.5%)",
    "Byte-level vocab truncates queries at 128 bytes"
  ]
}