| { |
| "model": "d3", |
| "bundle": "/Users/andaloes/mlx-work/coreai/bundles/d3-CoreAI", |
| "reference": "upstream BF16 d3_runtime.D3 on CPU (torch), measured in a separate process", |
| "device": "macOS 27, coreai-core 1.0.0b3, coreai-torch 0.4.3", |
| "cases": [ |
| { |
| "case": 1, |
| "kind": "choice", |
| "options": 3, |
| "reference_choice": "billing", |
| "coreai_choice": "billing", |
| "selected_answer_matches": true, |
| "max_probability_difference": 1.0777304399578469e-05, |
| "coreai_seconds": 3.4 |
| }, |
| { |
| "case": 2, |
| "kind": "noul", |
| "options": 2, |
| "reference_choice": "true", |
| "coreai_choice": "true", |
| "selected_answer_matches": true, |
| "max_probability_difference": 0.001235121800388872, |
| "coreai_seconds": 2.9 |
| }, |
| { |
| "case": 3, |
| "kind": "choice", |
| "options": 3, |
| "reference_choice": "p1", |
| "coreai_choice": "p1", |
| "selected_answer_matches": true, |
| "max_probability_difference": 0.0007533040125704549, |
| "coreai_seconds": 2.9 |
| }, |
| { |
| "case": 4, |
| "kind": "choice", |
| "options": 3, |
| "reference_choice": "privacy", |
| "coreai_choice": "privacy", |
| "selected_answer_matches": true, |
| "max_probability_difference": 4.279321103917155e-05, |
| "coreai_seconds": 2.9 |
| }, |
| { |
| "case": 5, |
| "kind": "noul", |
| "options": 2, |
| "reference_choice": "true", |
| "coreai_choice": "true", |
| "selected_answer_matches": true, |
| "max_probability_difference": 0.00031628914563774924, |
| "coreai_seconds": 2.9 |
| }, |
| { |
| "case": 6, |
| "kind": "choice", |
| "options": 3, |
| "reference_choice": "patch_now", |
| "coreai_choice": "patch_now", |
| "selected_answer_matches": true, |
| "max_probability_difference": 6.207629011623084e-05, |
| "coreai_seconds": 2.9 |
| } |
| ], |
| "summary": { |
| "cases": 6, |
| "all_selected_answers_match": true, |
| "max_probability_difference": 0.001235121800388872, |
| "media_cases": 0, |
| "note": "Text-only smoke cases. Media paths and long-prompt capacity are not measured. Small conversion smoke tests, not a quality benchmark." |
| } |
| } |
|
|
