{
  "cases": [
    {
      "coherence_detail": "coherent surface form",
      "coherent": true,
      "elapsed_seconds": 0.2902,
      "fact_detail": "privacy/locality concepts present",
      "facts_correct": true,
      "fixture": null,
      "generation_tokens": 38,
      "id": "text_privacy",
      "passed": true,
      "peak_memory_gb": 2.501529802,
      "prompt": "In one concise sentence, explain how running a vision-language model entirely on-device can improve privacy for clinical documents.",
      "prompt_tokens": 30,
      "response": "Running a vision-language model entirely on-device can improve privacy for clinical documents by reducing the need to transmit sensitive data over the network, thereby minimizing the risk of data breaches and unauthorized access."
    },
    {
      "coherence_detail": "coherent surface form",
      "coherent": true,
      "elapsed_seconds": 0.0379,
      "fact_detail": "all expected facts present",
      "facts_correct": true,
      "fixture": null,
      "generation_tokens": 2,
      "id": "text_fact_extraction",
      "passed": true,
      "peak_memory_gb": 2.533350538,
      "prompt": "A synthetic note states: \"The follow-up appointment is scheduled for Tuesday at 10:30 AM.\" What day is the follow-up? Answer with only the day.",
      "prompt_tokens": 44,
      "response": "Tuesday"
    },
    {
      "coherence_detail": "coherent surface form",
      "coherent": true,
      "elapsed_seconds": 0.9818,
      "fact_detail": "all expected facts present",
      "facts_correct": true,
      "fixture": "synthetic_clinical_document.png",
      "generation_tokens": 32,
      "id": "image_clinical_document",
      "passed": true,
      "peak_memory_gb": 5.120861918,
      "prompt": "This is synthetic test data. In one concise sentence, report the exact patient name, record ID, medication with dose and frequency, and allergy shown in the image.",
      "prompt_tokens": 1161,
      "response": "The synthetic test data includes patient Alex Rivera, record ID SYN-2048, medication Metformin 500 mg twice daily, and allergy to Penicillin."
    },
    {
      "coherence_detail": "coherent surface form",
      "coherent": true,
      "elapsed_seconds": 0.7279,
      "fact_detail": "all expected facts present",
      "facts_correct": true,
      "fixture": "synthetic_clinic_chart.png",
      "generation_tokens": 5,
      "id": "image_chart",
      "passed": true,
      "peak_memory_gb": 5.120861918,
      "prompt": "Which category has the tallest bar, and what exact value is printed above it? Answer concisely.",
      "prompt_tokens": 1053,
      "response": "Screening, 42"
    }
  ],
  "device": {
    "architecture": "applegpu_g15d",
    "device_name": "Apple M3 Ultra",
    "max_buffer_length": 373662154752,
    "max_recommended_working_set_size": 498216206336,
    "memory_size": 549755813888,
    "resource_limit": 499000
  },
  "load_seconds": 0.8207,
  "mlx_vlm_revision": "dd79a5d8caf3edafd6fa9e6326d7ce4977ddcbfc",
  "model_path": "OpenMed/North-Micro-Vision-Instruct-5bit-mlx",
  "passed": true,
  "runtime_versions": {
    "huggingface-hub": "1.27.0",
    "mlx": "0.32.0",
    "mlx-lm": "0.31.3",
    "mlx-metal": "0.32.0",
    "mlx-vlm": "0.6.10",
    "transformers": "5.15.0"
  },
  "schema_version": 1,
  "source_model": "CohereLabs/North-Micro-Vision-Instruct",
  "source_revision": "373bda96ac70bf89f99f7048f420cf00dc07c149",
  "variant": "5bit",
  "weights": {
    "actual": {
      "sha256": "03394bbd5f8cd1d79abda120374c69d431bd697c0677642af7c5806cded6f3f4",
      "size": 2415483751
    },
    "embedding_health": {
      "healthy": true,
      "tensors": {
        "language_model.model.embed_tokens.biases": {
          "dtype": "mlx.core.bfloat16",
          "healthy": true,
          "nonzero_count": 8388608,
          "shape": [
            262144,
            32
          ],
          "size": 8388608
        },
        "language_model.model.embed_tokens.scales": {
          "dtype": "mlx.core.bfloat16",
          "healthy": true,
          "nonzero_count": 8388608,
          "shape": [
            262144,
            32
          ],
          "size": 8388608
        },
        "language_model.model.embed_tokens.weight": {
          "dtype": "mlx.core.uint32",
          "healthy": true,
          "nonzero_count": 83886080,
          "shape": [
            262144,
            320
          ],
          "size": 83886080
        }
      }
    },
    "payload_valid": true,
    "reference": {
      "sha256": "ce3041919c41d5efdee8c86302acecdb9914d564149c6c7f8cb1dda212004f5c",
      "size": 2415483751
    },
    "reference_hash_match": false,
    "reference_size_match": true
  }
}