| { |
| "model": "laya", |
| "bundle": "/Users/andaloes/mlx-work/laya/bundles/laya-CoreAI", |
| "reference": "laya package eager PyTorch forward on CPU (same build_sequence tokens)", |
| "cases": [ |
| { |
| "case": 1, |
| "kind": "choice", |
| "options": 3, |
| "state_tokens": 59, |
| "reference_choice": "billing", |
| "coreai_choice": "billing", |
| "selected_answer_matches": true, |
| "max_probability_difference": 2.149687983177273e-09, |
| "max_abs_logit_difference": 6.67572021484375e-06 |
| }, |
| { |
| "case": 2, |
| "kind": "choice", |
| "options": 3, |
| "state_tokens": 55, |
| "reference_choice": "technical", |
| "coreai_choice": "technical", |
| "selected_answer_matches": true, |
| "max_probability_difference": 3.2404249949991026e-08, |
| "max_abs_logit_difference": 4.291534423828125e-06 |
| }, |
| { |
| "case": 3, |
| "kind": "noul", |
| "options": 2, |
| "state_tokens": 48, |
| "reference_choice": "true", |
| "coreai_choice": "true", |
| "selected_answer_matches": true, |
| "max_probability_difference": 4.996124104150734e-07, |
| "max_abs_logit_difference": 6.318092346191406e-06 |
| }, |
| { |
| "case": 4, |
| "kind": "noul", |
| "options": 2, |
| "state_tokens": 47, |
| "reference_choice": "false", |
| "coreai_choice": "false", |
| "selected_answer_matches": true, |
| "max_probability_difference": 4.186273309600419e-08, |
| "max_abs_logit_difference": 5.245208740234375e-05 |
| }, |
| { |
| "case": 5, |
| "kind": "score", |
| "options": 3, |
| "state_tokens": 45, |
| "reference_choice": "blocking", |
| "coreai_choice": "blocking", |
| "selected_answer_matches": true, |
| "max_probability_difference": 3.617309453085582e-07, |
| "max_abs_logit_difference": 8.940696716308594e-06 |
| }, |
| { |
| "case": 6, |
| "kind": "score", |
| "options": 3, |
| "state_tokens": 42, |
| "reference_choice": "soon", |
| "coreai_choice": "soon", |
| "selected_answer_matches": true, |
| "max_probability_difference": 6.151039953494752e-07, |
| "max_abs_logit_difference": 5.9604644775390625e-06 |
| } |
| ], |
| "summary": { |
| "cases": 6, |
| "all_selected_answers_match": true, |
| "max_probability_difference": 6.151039953494752e-07, |
| "note": "Laya's own decision cases on one checkpoint. Not a benchmark of accuracy or calibration; it measures conversion drift only." |
| } |
| } |
|
|
