| { |
| "model": "d3-flash", |
| "bundle": "/Users/andaloes/mlx-work/coreai/bundles/d3-flash-CoreAI", |
| "reference": "upstream BF16 d3_runtime.D3 on CPU (torch)", |
| "device": "macOS 27, coreai-core 1.0.0b3, coreai-torch 0.4.3", |
| "cases": [ |
| { |
| "case": 1, |
| "kind": "choice", |
| "options": 3, |
| "reference_choice": "billing", |
| "coreai_choice": "billing", |
| "selected_answer_matches": true, |
| "max_probability_difference": 0.0011223266959178302, |
| "reference_seconds": 13.58, |
| "coreai_seconds": 1.09 |
| }, |
| { |
| "case": 2, |
| "kind": "noul", |
| "options": 2, |
| "reference_choice": "true", |
| "coreai_choice": "true", |
| "selected_answer_matches": true, |
| "max_probability_difference": 6.457854446062505e-05, |
| "reference_seconds": 13.31, |
| "coreai_seconds": 1.17 |
| }, |
| { |
| "case": 3, |
| "kind": "choice", |
| "options": 3, |
| "reference_choice": "p1", |
| "coreai_choice": "p1", |
| "selected_answer_matches": true, |
| "max_probability_difference": 0.0005070228983186098, |
| "reference_seconds": 14.24, |
| "coreai_seconds": 1.04 |
| }, |
| { |
| "case": 4, |
| "kind": "choice", |
| "options": 3, |
| "reference_choice": "privacy", |
| "coreai_choice": "privacy", |
| "selected_answer_matches": true, |
| "max_probability_difference": 0.0006648782686865529, |
| "reference_seconds": 13.2, |
| "coreai_seconds": 1.05 |
| }, |
| { |
| "case": 5, |
| "kind": "noul", |
| "options": 2, |
| "reference_choice": "true", |
| "coreai_choice": "true", |
| "selected_answer_matches": true, |
| "max_probability_difference": 0.0011427603984022205, |
| "reference_seconds": 12.59, |
| "coreai_seconds": 1.03 |
| }, |
| { |
| "case": 6, |
| "kind": "choice", |
| "options": 3, |
| "reference_choice": "patch_now", |
| "coreai_choice": "patch_now", |
| "selected_answer_matches": true, |
| "max_probability_difference": 0.0005288345735383615, |
| "reference_seconds": 13.59, |
| "coreai_seconds": 1.03 |
| } |
| ], |
| "summary": { |
| "cases": 6, |
| "all_selected_answers_match": true, |
| "max_probability_difference": 0.0011427603984022205, |
| "media_cases": 0, |
| "note": "Text-only smoke cases. Media (image/video) paths and long-prompt capacity are not measured here. Small conversion smoke tests, not a quality benchmark." |
| } |
| } |
|
|
