File size: 3,784 Bytes
a7ec518 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 | {
"cases": [
{
"coherence_detail": "coherent surface form",
"coherent": true,
"elapsed_seconds": 0.2915,
"fact_detail": "privacy/locality concepts present",
"facts_correct": true,
"fixture": null,
"generation_tokens": 26,
"id": "text_privacy",
"passed": true,
"peak_memory_gb": 5.063733678,
"prompt": "In one concise sentence, explain how running a vision-language model entirely on-device can improve privacy for clinical documents.",
"prompt_tokens": 30,
"response": "Running a vision-language model entirely on-device reduces the need for cloud storage and transmission of sensitive clinical data, thereby enhancing privacy."
},
{
"coherence_detail": "coherent surface form",
"coherent": true,
"elapsed_seconds": 0.0468,
"fact_detail": "all expected facts present",
"facts_correct": true,
"fixture": null,
"generation_tokens": 2,
"id": "text_fact_extraction",
"passed": true,
"peak_memory_gb": 5.063733678,
"prompt": "A synthetic note states: \"The follow-up appointment is scheduled for Tuesday at 10:30 AM.\" What day is the follow-up? Answer with only the day.",
"prompt_tokens": 44,
"response": "Tuesday"
},
{
"coherence_detail": "coherent surface form",
"coherent": true,
"elapsed_seconds": 1.0647,
"fact_detail": "all expected facts present",
"facts_correct": true,
"fixture": "synthetic_clinical_document.png",
"generation_tokens": 32,
"id": "image_clinical_document",
"passed": true,
"peak_memory_gb": 7.675185886,
"prompt": "This is synthetic test data. In one concise sentence, report the exact patient name, record ID, medication with dose and frequency, and allergy shown in the image.",
"prompt_tokens": 1161,
"response": "The synthetic test data includes patient Alex Rivera (Record ID: SYN-2048), prescribed Metformin 500 mg twice daily, with a penicillin allergy."
},
{
"coherence_detail": "coherent surface form",
"coherent": true,
"elapsed_seconds": 0.7383,
"fact_detail": "all expected facts present",
"facts_correct": true,
"fixture": "synthetic_clinic_chart.png",
"generation_tokens": 5,
"id": "image_chart",
"passed": true,
"peak_memory_gb": 7.675185886,
"prompt": "Which category has the tallest bar, and what exact value is printed above it? Answer concisely.",
"prompt_tokens": 1053,
"response": "Screening, 42"
}
],
"device": {
"architecture": "applegpu_g15d",
"device_name": "Apple M3 Ultra",
"max_buffer_length": 373662154752,
"max_recommended_working_set_size": 498216206336,
"memory_size": 549755813888,
"resource_limit": 499000
},
"load_seconds": 0.8684,
"mlx_vlm_revision": "dd79a5d8caf3edafd6fa9e6326d7ce4977ddcbfc",
"model_path": "OpenMed/North-Micro-Vision-Instruct-bf16-mlx",
"passed": true,
"runtime_versions": {
"huggingface-hub": "1.27.0",
"mlx": "0.32.0",
"mlx-lm": "0.31.3",
"mlx-metal": "0.32.0",
"mlx-vlm": "0.6.10",
"transformers": "5.15.0"
},
"schema_version": 1,
"source_model": "CohereLabs/North-Micro-Vision-Instruct",
"source_revision": "373bda96ac70bf89f99f7048f420cf00dc07c149",
"variant": "bf16",
"weights": {
"actual": {
"sha256": "83b1212694fe6bdb7013c81899a237933a68bfde21aacd6974f9944cb1239360",
"size": 4969764749
},
"embedding_health": null,
"payload_valid": true,
"reference": {
"sha256": "83b1212694fe6bdb7013c81899a237933a68bfde21aacd6974f9944cb1239360",
"size": 4969764749
},
"reference_hash_match": true,
"reference_size_match": true
}
}
|