| { |
| "cases": [ |
| { |
| "coherence_detail": "coherent surface form", |
| "coherent": true, |
| "elapsed_seconds": 0.2915, |
| "fact_detail": "privacy/locality concepts present", |
| "facts_correct": true, |
| "fixture": null, |
| "generation_tokens": 26, |
| "id": "text_privacy", |
| "passed": true, |
| "peak_memory_gb": 5.063733678, |
| "prompt": "In one concise sentence, explain how running a vision-language model entirely on-device can improve privacy for clinical documents.", |
| "prompt_tokens": 30, |
| "response": "Running a vision-language model entirely on-device reduces the need for cloud storage and transmission of sensitive clinical data, thereby enhancing privacy." |
| }, |
| { |
| "coherence_detail": "coherent surface form", |
| "coherent": true, |
| "elapsed_seconds": 0.0468, |
| "fact_detail": "all expected facts present", |
| "facts_correct": true, |
| "fixture": null, |
| "generation_tokens": 2, |
| "id": "text_fact_extraction", |
| "passed": true, |
| "peak_memory_gb": 5.063733678, |
| "prompt": "A synthetic note states: \"The follow-up appointment is scheduled for Tuesday at 10:30 AM.\" What day is the follow-up? Answer with only the day.", |
| "prompt_tokens": 44, |
| "response": "Tuesday" |
| }, |
| { |
| "coherence_detail": "coherent surface form", |
| "coherent": true, |
| "elapsed_seconds": 1.0647, |
| "fact_detail": "all expected facts present", |
| "facts_correct": true, |
| "fixture": "synthetic_clinical_document.png", |
| "generation_tokens": 32, |
| "id": "image_clinical_document", |
| "passed": true, |
| "peak_memory_gb": 7.675185886, |
| "prompt": "This is synthetic test data. In one concise sentence, report the exact patient name, record ID, medication with dose and frequency, and allergy shown in the image.", |
| "prompt_tokens": 1161, |
| "response": "The synthetic test data includes patient Alex Rivera (Record ID: SYN-2048), prescribed Metformin 500 mg twice daily, with a penicillin allergy." |
| }, |
| { |
| "coherence_detail": "coherent surface form", |
| "coherent": true, |
| "elapsed_seconds": 0.7383, |
| "fact_detail": "all expected facts present", |
| "facts_correct": true, |
| "fixture": "synthetic_clinic_chart.png", |
| "generation_tokens": 5, |
| "id": "image_chart", |
| "passed": true, |
| "peak_memory_gb": 7.675185886, |
| "prompt": "Which category has the tallest bar, and what exact value is printed above it? Answer concisely.", |
| "prompt_tokens": 1053, |
| "response": "Screening, 42" |
| } |
| ], |
| "device": { |
| "architecture": "applegpu_g15d", |
| "device_name": "Apple M3 Ultra", |
| "max_buffer_length": 373662154752, |
| "max_recommended_working_set_size": 498216206336, |
| "memory_size": 549755813888, |
| "resource_limit": 499000 |
| }, |
| "load_seconds": 0.8684, |
| "mlx_vlm_revision": "dd79a5d8caf3edafd6fa9e6326d7ce4977ddcbfc", |
| "model_path": "OpenMed/North-Micro-Vision-Instruct-bf16-mlx", |
| "passed": true, |
| "runtime_versions": { |
| "huggingface-hub": "1.27.0", |
| "mlx": "0.32.0", |
| "mlx-lm": "0.31.3", |
| "mlx-metal": "0.32.0", |
| "mlx-vlm": "0.6.10", |
| "transformers": "5.15.0" |
| }, |
| "schema_version": 1, |
| "source_model": "CohereLabs/North-Micro-Vision-Instruct", |
| "source_revision": "373bda96ac70bf89f99f7048f420cf00dc07c149", |
| "variant": "bf16", |
| "weights": { |
| "actual": { |
| "sha256": "83b1212694fe6bdb7013c81899a237933a68bfde21aacd6974f9944cb1239360", |
| "size": 4969764749 |
| }, |
| "embedding_health": null, |
| "payload_valid": true, |
| "reference": { |
| "sha256": "83b1212694fe6bdb7013c81899a237933a68bfde21aacd6974f9944cb1239360", |
| "size": 4969764749 |
| }, |
| "reference_hash_match": true, |
| "reference_size_match": true |
| } |
| } |
|
|
