reports/inference_validation.json

{
  "environment": {
    "python": "3.12.3",
    "platform": "Linux-6.6.87.2-microsoft-standard-WSL2-x86_64-with-glibc2.39",
    "torch": "2.10.0+cu128",
    "transformers": "5.5.0",
    "cuda_available": true,
    "gpu": "NVIDIA GeForce RTX 5070 Ti",
    "vram_bytes": 17094475776,
    "dtype": "bfloat16",
    "device_map": "{'': 0}",
    "quantization": "4bit",
    "model": "<local-checkpoint>",
    "timestamp": "2026-07-29T13:45:23-0400"
  },
  "summary": {
    "total": 12,
    "passed": 12,
    "failed": 0
  },
  "results": [
    {
      "check": "load_model",
      "status": "pass",
      "seconds": 99.86,
      "detail": {
        "loaded_with": "AutoModelForMultimodalLM",
        "auto_class_errors": {},
        "trust_remote_code_required": false,
        "architectures": [
          "Qwen3_5ForConditionalGeneration"
        ],
        "model_type": "qwen3_5",
        "total_parameters": 5724972272,
        "vision_parameters": 230421232,
        "language_parameters": 5494551040,
        "device_map": "{'': 0}",
        "processor_class": "Qwen3VLProcessor",
        "tokenizer_class": "TokenizersBackend"
      }
    },
    {
      "check": "text_only_generation",
      "status": "pass",
      "seconds": 2.86,
      "detail": {
        "prompt": "Write a Python function that reverses a string.",
        "response": "User wants Python reverse string. Simple task. Provide function. No tools needed.\n</think>\n\n```python\ndef reverse_string(s):\n    return s[::-1]\n```\n\nThis function takes a string as input and returns the reversed string. It uses Python's slicing feature to reverse the string."
      }
    },
    {
      "check": "text_only_identity",
      "status": "pass",
      "seconds": 0.85,
      "detail": {
        "prompt": "What model are you? Answer in one short sentence.",
        "response": "User asks model. Direct answer needed. State model name clearly.\n</think>\n\nI am Wraith, an AI model."
      }
    },
    {
      "check": "text_only_reasoning",
      "status": "pass",
      "seconds": 1.53,
      "detail": {
        "prompt": "A shop sells pens at 3 for $2. How much do 12 pens cost? Answer with the number only.",
        "response": "User asks cost of 12 pens. Shop price: 3 pens for $2. Need compute. 12 pens = 4 groups of 3. 4 * $2 = $8. Answer number only."
      }
    },
    {
      "check": "image_ocr",
      "status": "pass",
      "seconds": 4.27,
      "detail": {
        "image": "synthetic_receipt.png",
        "prompt": "Read this receipt. Give the merchant name and the total.",
        "response": "1.  **Identify the user's request:** The user wants the merchant name and the total from the provided receipt image.\n\n2.  **Scan the image for the merchant name:**\n    *   Look at the top of the receipt.\n    *   The first line says \"NORTHGATE HARDWARE\". This is clearly the merchant name.\n\n3.  **Scan the image for the total:**\n    *   Look towards the bottom of the receipt.\n    *   There is a line that says \"TOTAL\" followed by \"22.75\".\n\n4.  **Formulate the"
      }
    },
    {
      "check": "image_document_json",
      "status": "pass",
      "seconds": 4.05,
      "detail": {
        "image": "synthetic_receipt.png",
        "prompt": "Return only JSON: {\"merchant\": str, \"date\": \"YYYY-MM-DD\", \"total\": float}",
        "response": "The user wants to extract information from the provided image and format it as JSON.\nThe required fields are:\n- merchant: The name of the store.\n- date: The date of the invoice.\n- total: The total amount of the invoice.\n\nLooking at the image:\n- The top text says \"NORTHGATE HARDWARE\". This is the merchant.\n- The next line says \"144 Mill Road\". This is the address, not required.\n- The next line says \"Date: 2026-03-14\". This is the date.\n- The"
      }
    },
    {
      "check": "image_chart",
      "status": "pass",
      "seconds": 4.0,
      "detail": {
        "image": "synthetic_chart.png",
        "prompt": "Which quarter is highest, and what value?",
        "response": "1.  **Identify the user's question:** The user wants to know which quarter has the highest value and what that value is.\n\n2.  **Analyze the image:**\n    *   The image is a bar chart titled \"Units sold by quarter\".\n    *   The x-axis labels are Q1, Q2, Q3, Q4.\n    *   The y-axis represents the number of units sold (though there are no explicit y-axis numbers, the values are labeled on top of each bar).\n    *   The bars are black.\n    *   Let's read the values for"
      }
    },
    {
      "check": "image_caption",
      "status": "pass",
      "seconds": 4.02,
      "detail": {
        "image": "synthetic_chart.png",
        "prompt": "Describe this image in one sentence.",
        "response": "1.  **Identify the core subject:** The image is a bar chart titled \"Units sold by quarter\".\n2.  **Analyze the axes and labels:** The x-axis represents four quarters: Q1, Q2, Q3, and Q4. The y-axis is implicit (units sold), with values labeled on top of each bar.\n3.  **Extract the data points:**\n    *   Q1: 40\n    *   Q2: 95\n    *   Q3: 60\n    *   Q4: 130\n4."
      }
    },
    {
      "check": "multi_turn_conversation",
      "status": "pass",
      "seconds": 0.91,
      "detail": {
        "response": "User wants double of 47. Just compute 47*2=94. Output number only.\n</think>\n\n94"
      }
    },
    {
      "check": "greedy_determinism",
      "status": "pass",
      "seconds": 1.83,
      "detail": {
        "identical": true,
        "first": "User asks for three primary colours. Straight fact. No tools needed. Answer directly.\n</think>\n\nRed, yellow, and blue.",
        "second": "User asks for three primary colours. Straight fact. No tools needed. Answer directly.\n</think>\n\nRed, yellow, and blue."
      }
    },
    {
      "check": "batch_inference",
      "status": "pass",
      "seconds": 1.19,
      "detail": {
        "prompts": [
          "Capital of Japan?",
          "2 + 2 = ?"
        ],
        "responses": [
          "User asks capital of Japan. Direct fact. Answer Tokyo.\n</think>\n\nThe capital of Japan is Tokyo.",
          "User asks 2+2. Simple math. Answer 4. No tools needed.\n</think>\n\n4"
        ]
      }
    },
    {
      "check": "long_context_retrieval",
      "status": "pass",
      "seconds": 3.53,
      "detail": {
        "prompt_tokens": 14429,
        "response": "User asks maintenance code for north pump. Earlier text had code QF-8812. No new info. Answer directly.\n</think>\n\nThe maintenance",
        "contains_needle": true
      }
    }
  ]
}