| { |
| "cases": [ |
| { |
| "coherence_detail": "coherent surface form", |
| "coherent": true, |
| "elapsed_seconds": 0.2902, |
| "fact_detail": "privacy/locality concepts present", |
| "facts_correct": true, |
| "fixture": null, |
| "generation_tokens": 38, |
| "id": "text_privacy", |
| "passed": true, |
| "peak_memory_gb": 2.501529802, |
| "prompt": "In one concise sentence, explain how running a vision-language model entirely on-device can improve privacy for clinical documents.", |
| "prompt_tokens": 30, |
| "response": "Running a vision-language model entirely on-device can improve privacy for clinical documents by reducing the need to transmit sensitive data over the network, thereby minimizing the risk of data breaches and unauthorized access." |
| }, |
| { |
| "coherence_detail": "coherent surface form", |
| "coherent": true, |
| "elapsed_seconds": 0.0379, |
| "fact_detail": "all expected facts present", |
| "facts_correct": true, |
| "fixture": null, |
| "generation_tokens": 2, |
| "id": "text_fact_extraction", |
| "passed": true, |
| "peak_memory_gb": 2.533350538, |
| "prompt": "A synthetic note states: \"The follow-up appointment is scheduled for Tuesday at 10:30 AM.\" What day is the follow-up? Answer with only the day.", |
| "prompt_tokens": 44, |
| "response": "Tuesday" |
| }, |
| { |
| "coherence_detail": "coherent surface form", |
| "coherent": true, |
| "elapsed_seconds": 0.9818, |
| "fact_detail": "all expected facts present", |
| "facts_correct": true, |
| "fixture": "synthetic_clinical_document.png", |
| "generation_tokens": 32, |
| "id": "image_clinical_document", |
| "passed": true, |
| "peak_memory_gb": 5.120861918, |
| "prompt": "This is synthetic test data. In one concise sentence, report the exact patient name, record ID, medication with dose and frequency, and allergy shown in the image.", |
| "prompt_tokens": 1161, |
| "response": "The synthetic test data includes patient Alex Rivera, record ID SYN-2048, medication Metformin 500 mg twice daily, and allergy to Penicillin." |
| }, |
| { |
| "coherence_detail": "coherent surface form", |
| "coherent": true, |
| "elapsed_seconds": 0.7279, |
| "fact_detail": "all expected facts present", |
| "facts_correct": true, |
| "fixture": "synthetic_clinic_chart.png", |
| "generation_tokens": 5, |
| "id": "image_chart", |
| "passed": true, |
| "peak_memory_gb": 5.120861918, |
| "prompt": "Which category has the tallest bar, and what exact value is printed above it? Answer concisely.", |
| "prompt_tokens": 1053, |
| "response": "Screening, 42" |
| } |
| ], |
| "device": { |
| "architecture": "applegpu_g15d", |
| "device_name": "Apple M3 Ultra", |
| "max_buffer_length": 373662154752, |
| "max_recommended_working_set_size": 498216206336, |
| "memory_size": 549755813888, |
| "resource_limit": 499000 |
| }, |
| "load_seconds": 0.8207, |
| "mlx_vlm_revision": "dd79a5d8caf3edafd6fa9e6326d7ce4977ddcbfc", |
| "model_path": "OpenMed/North-Micro-Vision-Instruct-5bit-mlx", |
| "passed": true, |
| "runtime_versions": { |
| "huggingface-hub": "1.27.0", |
| "mlx": "0.32.0", |
| "mlx-lm": "0.31.3", |
| "mlx-metal": "0.32.0", |
| "mlx-vlm": "0.6.10", |
| "transformers": "5.15.0" |
| }, |
| "schema_version": 1, |
| "source_model": "CohereLabs/North-Micro-Vision-Instruct", |
| "source_revision": "373bda96ac70bf89f99f7048f420cf00dc07c149", |
| "variant": "5bit", |
| "weights": { |
| "actual": { |
| "sha256": "03394bbd5f8cd1d79abda120374c69d431bd697c0677642af7c5806cded6f3f4", |
| "size": 2415483751 |
| }, |
| "embedding_health": { |
| "healthy": true, |
| "tensors": { |
| "language_model.model.embed_tokens.biases": { |
| "dtype": "mlx.core.bfloat16", |
| "healthy": true, |
| "nonzero_count": 8388608, |
| "shape": [ |
| 262144, |
| 32 |
| ], |
| "size": 8388608 |
| }, |
| "language_model.model.embed_tokens.scales": { |
| "dtype": "mlx.core.bfloat16", |
| "healthy": true, |
| "nonzero_count": 8388608, |
| "shape": [ |
| 262144, |
| 32 |
| ], |
| "size": 8388608 |
| }, |
| "language_model.model.embed_tokens.weight": { |
| "dtype": "mlx.core.uint32", |
| "healthy": true, |
| "nonzero_count": 83886080, |
| "shape": [ |
| 262144, |
| 320 |
| ], |
| "size": 83886080 |
| } |
| } |
| }, |
| "payload_valid": true, |
| "reference": { |
| "sha256": "ce3041919c41d5efdee8c86302acecdb9914d564149c6c7f8cb1dda212004f5c", |
| "size": 2415483751 |
| }, |
| "reference_hash_match": false, |
| "reference_size_match": true |
| } |
| } |
|
|
