{
  "cases": [
    {
      "coherence_detail": "coherent surface form",
      "coherent": true,
      "elapsed_seconds": 0.2227,
      "fact_detail": "privacy/locality concepts present",
      "facts_correct": true,
      "fixture": null,
      "generation_tokens": 26,
      "id": "text_privacy",
      "passed": true,
      "peak_memory_gb": 2.744799434,
      "prompt": "In one concise sentence, explain how running a vision-language model entirely on-device can improve privacy for clinical documents.",
      "prompt_tokens": 30,
      "response": "Running a vision-language model entirely on-device reduces the need for cloud storage and transmission of sensitive clinical data, thereby enhancing privacy."
    },
    {
      "coherence_detail": "coherent surface form",
      "coherent": true,
      "elapsed_seconds": 0.0383,
      "fact_detail": "all expected facts present",
      "facts_correct": true,
      "fixture": null,
      "generation_tokens": 2,
      "id": "text_fact_extraction",
      "passed": true,
      "peak_memory_gb": 2.776603786,
      "prompt": "A synthetic note states: \"The follow-up appointment is scheduled for Tuesday at 10:30 AM.\" What day is the follow-up? Answer with only the day.",
      "prompt_tokens": 44,
      "response": "Tuesday"
    },
    {
      "coherence_detail": "coherent surface form",
      "coherent": true,
      "elapsed_seconds": 0.989,
      "fact_detail": "all expected facts present",
      "facts_correct": true,
      "fixture": "synthetic_clinical_document.png",
      "generation_tokens": 32,
      "id": "image_clinical_document",
      "passed": true,
      "peak_memory_gb": 5.36413155,
      "prompt": "This is synthetic test data. In one concise sentence, report the exact patient name, record ID, medication with dose and frequency, and allergy shown in the image.",
      "prompt_tokens": 1161,
      "response": "The synthetic test data includes patient Alex Rivera (Record ID: SYN-2048), prescribed Metformin 500 mg twice daily, with a penicillin allergy."
    },
    {
      "coherence_detail": "coherent surface form",
      "coherent": true,
      "elapsed_seconds": 0.73,
      "fact_detail": "all expected facts present",
      "facts_correct": true,
      "fixture": "synthetic_clinic_chart.png",
      "generation_tokens": 5,
      "id": "image_chart",
      "passed": true,
      "peak_memory_gb": 5.36413155,
      "prompt": "Which category has the tallest bar, and what exact value is printed above it? Answer concisely.",
      "prompt_tokens": 1053,
      "response": "Screening, 42"
    }
  ],
  "device": {
    "architecture": "applegpu_g15d",
    "device_name": "Apple M3 Ultra",
    "max_buffer_length": 373662154752,
    "max_recommended_working_set_size": 498216206336,
    "memory_size": 549755813888,
    "resource_limit": 499000
  },
  "load_seconds": 0.8184,
  "mlx_vlm_revision": "dd79a5d8caf3edafd6fa9e6326d7ce4977ddcbfc",
  "model_path": "OpenMed/North-Micro-Vision-Instruct-6bit-mlx",
  "passed": true,
  "runtime_versions": {
    "huggingface-hub": "1.27.0",
    "mlx": "0.32.0",
    "mlx-lm": "0.31.3",
    "mlx-metal": "0.32.0",
    "mlx-vlm": "0.6.10",
    "transformers": "5.15.0"
  },
  "schema_version": 1,
  "source_model": "CohereLabs/North-Micro-Vision-Instruct",
  "source_revision": "373bda96ac70bf89f99f7048f420cf00dc07c149",
  "variant": "6bit",
  "weights": {
    "actual": {
      "sha256": "564282d38108f0406bed6602fde32bc3f3877b3848f42cf0ad3d514aba9ced02",
      "size": 2658753473
    },
    "embedding_health": {
      "healthy": true,
      "tensors": {
        "language_model.model.embed_tokens.biases": {
          "dtype": "mlx.core.bfloat16",
          "healthy": true,
          "nonzero_count": 8388608,
          "shape": [
            262144,
            32
          ],
          "size": 8388608
        },
        "language_model.model.embed_tokens.scales": {
          "dtype": "mlx.core.bfloat16",
          "healthy": true,
          "nonzero_count": 8388608,
          "shape": [
            262144,
            32
          ],
          "size": 8388608
        },
        "language_model.model.embed_tokens.weight": {
          "dtype": "mlx.core.uint32",
          "healthy": true,
          "nonzero_count": 100663296,
          "shape": [
            262144,
            384
          ],
          "size": 100663296
        }
      }
    },
    "payload_valid": true,
    "reference": {
      "sha256": "21f16dea6c6274241538410c6c017155f2b5b223f8d31e2e05d687679fdcd2a0",
      "size": 2658753473
    },
    "reference_hash_match": false,
    "reference_size_match": true
  }
}