| { |
| "source_model": "google/embeddinggemma-2", |
| "source_revision": "914f7f89142e33e77833254d9c9b90c3cef7303b", |
| "mlx_vlm_revision": "3d87e88402f307efbf68e568971aa887ee7d9ed0", |
| "variant": "8bit", |
| "hardware": "Apple M3 Ultra", |
| "mlx_version": "0.32.3", |
| "transformers_version": "5.18.0.dev0", |
| "validation_scope": "Six multilingual text inputs and synthetic image, audio, two-frame video, and text+image; numerical smoke checks, not a retrieval benchmark.", |
| "tensor_count": 1816, |
| "tensor_dtypes": [ |
| "BF16", |
| "U32" |
| ], |
| "weight_bytes": 1234238431, |
| "cases": { |
| "audio": { |
| "shape": [ |
| 1, |
| 768 |
| ], |
| "finite": true, |
| "minimum_cosine_vs_torch_fp32": 0.9998035070404524, |
| "maximum_absolute_error_vs_torch_fp32": 0.002476184321109907, |
| "minimum_cosine_vs_source_bf16": 0.9997744741131873, |
| "minimum_truncated_cosine_vs_torch_fp32": 0.9998247233031456 |
| }, |
| "image": { |
| "shape": [ |
| 1, |
| 768 |
| ], |
| "finite": true, |
| "minimum_cosine_vs_torch_fp32": 0.9998904477000761, |
| "maximum_absolute_error_vs_torch_fp32": 0.0019366481302732774, |
| "minimum_cosine_vs_source_bf16": 0.9999318244275229, |
| "minimum_truncated_cosine_vs_torch_fp32": 0.9998950619484888 |
| }, |
| "text": { |
| "shape": [ |
| 6, |
| 768 |
| ], |
| "finite": true, |
| "minimum_cosine_vs_torch_fp32": 0.9998142728897385, |
| "maximum_absolute_error_vs_torch_fp32": 0.0022990193383245483, |
| "minimum_cosine_vs_source_bf16": 0.9997514709446282, |
| "minimum_truncated_cosine_vs_torch_fp32": 0.9998170484369469, |
| "retrieval_scores_venus_mars": [ |
| 0.678564727306366, |
| 0.8342599868774414 |
| ] |
| }, |
| "text_image": { |
| "shape": [ |
| 1, |
| 768 |
| ], |
| "finite": true, |
| "minimum_cosine_vs_torch_fp32": 0.9998230728231623, |
| "maximum_absolute_error_vs_torch_fp32": 0.002104893632396264, |
| "minimum_cosine_vs_source_bf16": 0.999874734632146, |
| "minimum_truncated_cosine_vs_torch_fp32": 0.999830769497079 |
| }, |
| "video": { |
| "shape": [ |
| 1, |
| 768 |
| ], |
| "finite": true, |
| "minimum_cosine_vs_torch_fp32": 0.9998359946491495, |
| "maximum_absolute_error_vs_torch_fp32": 0.0021304845535331423, |
| "minimum_cosine_vs_source_bf16": 0.9998825464984411, |
| "minimum_truncated_cosine_vs_torch_fp32": 0.9998420387979006 |
| } |
| } |
| } |
|
|
