| { |
| "model": "d3-mini", |
| "bundle": "/Users/andaloes/mlx-work/coreai/bundles/d3-mini-CoreAI", |
| "reference": "upstream BF16 d3_runtime.D3 on CPU (torch)", |
| "device": "macOS 27, coreai-core 1.0.0b3, coreai-torch 0.4.3", |
| "cases": [ |
| { |
| "case": 1, |
| "kind": "choice", |
| "options": 3, |
| "reference_choice": "billing", |
| "coreai_choice": "billing", |
| "selected_answer_matches": true, |
| "max_probability_difference": 5.510886245080858e-05, |
| "reference_seconds": 8.34, |
| "coreai_seconds": 1.26 |
| }, |
| { |
| "case": 2, |
| "kind": "noul", |
| "options": 2, |
| "reference_choice": "true", |
| "coreai_choice": "true", |
| "selected_answer_matches": true, |
| "max_probability_difference": 0.00025170385801920325, |
| "reference_seconds": 8.25, |
| "coreai_seconds": 0.61 |
| }, |
| { |
| "case": 3, |
| "kind": "choice", |
| "options": 3, |
| "reference_choice": "p1", |
| "coreai_choice": "p1", |
| "selected_answer_matches": true, |
| "max_probability_difference": 0.00031542212962154537, |
| "reference_seconds": 8.71, |
| "coreai_seconds": 0.59 |
| }, |
| { |
| "case": 4, |
| "kind": "choice", |
| "options": 3, |
| "reference_choice": "privacy", |
| "coreai_choice": "privacy", |
| "selected_answer_matches": true, |
| "max_probability_difference": 0.000957128167021426, |
| "reference_seconds": 8.12, |
| "coreai_seconds": 0.6 |
| }, |
| { |
| "case": 5, |
| "kind": "noul", |
| "options": 2, |
| "reference_choice": "true", |
| "coreai_choice": "true", |
| "selected_answer_matches": true, |
| "max_probability_difference": 0.0013866785234378787, |
| "reference_seconds": 7.83, |
| "coreai_seconds": 0.59 |
| }, |
| { |
| "case": 6, |
| "kind": "choice", |
| "options": 3, |
| "reference_choice": "patch_now", |
| "coreai_choice": "patch_now", |
| "selected_answer_matches": true, |
| "max_probability_difference": 0.00010795998240775706, |
| "reference_seconds": 8.3, |
| "coreai_seconds": 0.6 |
| } |
| ], |
| "summary": { |
| "cases": 6, |
| "all_selected_answers_match": true, |
| "max_probability_difference": 0.0013866785234378787, |
| "media_cases": 0, |
| "note": "Text-only smoke cases. Media (image/video) paths and long-prompt capacity are not measured here. Small conversion smoke tests, not a quality benchmark." |
| } |
| } |
|
|
