File size: 6,755 Bytes
12cd919 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 | {
"environment": {
"python": "3.12.3",
"platform": "Linux-6.6.87.2-microsoft-standard-WSL2-x86_64-with-glibc2.39",
"torch": "2.10.0+cu128",
"transformers": "5.5.0",
"cuda_available": true,
"gpu": "NVIDIA GeForce RTX 5070 Ti",
"vram_bytes": 17094475776,
"dtype": "bfloat16",
"device_map": "{'': 0}",
"quantization": "4bit",
"model": "<local-checkpoint>",
"timestamp": "2026-07-29T13:45:23-0400"
},
"summary": {
"total": 12,
"passed": 12,
"failed": 0
},
"results": [
{
"check": "load_model",
"status": "pass",
"seconds": 99.86,
"detail": {
"loaded_with": "AutoModelForMultimodalLM",
"auto_class_errors": {},
"trust_remote_code_required": false,
"architectures": [
"Qwen3_5ForConditionalGeneration"
],
"model_type": "qwen3_5",
"total_parameters": 5724972272,
"vision_parameters": 230421232,
"language_parameters": 5494551040,
"device_map": "{'': 0}",
"processor_class": "Qwen3VLProcessor",
"tokenizer_class": "TokenizersBackend"
}
},
{
"check": "text_only_generation",
"status": "pass",
"seconds": 2.86,
"detail": {
"prompt": "Write a Python function that reverses a string.",
"response": "User wants Python reverse string. Simple task. Provide function. No tools needed.\n</think>\n\n```python\ndef reverse_string(s):\n return s[::-1]\n```\n\nThis function takes a string as input and returns the reversed string. It uses Python's slicing feature to reverse the string."
}
},
{
"check": "text_only_identity",
"status": "pass",
"seconds": 0.85,
"detail": {
"prompt": "What model are you? Answer in one short sentence.",
"response": "User asks model. Direct answer needed. State model name clearly.\n</think>\n\nI am Wraith, an AI model."
}
},
{
"check": "text_only_reasoning",
"status": "pass",
"seconds": 1.53,
"detail": {
"prompt": "A shop sells pens at 3 for $2. How much do 12 pens cost? Answer with the number only.",
"response": "User asks cost of 12 pens. Shop price: 3 pens for $2. Need compute. 12 pens = 4 groups of 3. 4 * $2 = $8. Answer number only."
}
},
{
"check": "image_ocr",
"status": "pass",
"seconds": 4.27,
"detail": {
"image": "synthetic_receipt.png",
"prompt": "Read this receipt. Give the merchant name and the total.",
"response": "1. **Identify the user's request:** The user wants the merchant name and the total from the provided receipt image.\n\n2. **Scan the image for the merchant name:**\n * Look at the top of the receipt.\n * The first line says \"NORTHGATE HARDWARE\". This is clearly the merchant name.\n\n3. **Scan the image for the total:**\n * Look towards the bottom of the receipt.\n * There is a line that says \"TOTAL\" followed by \"22.75\".\n\n4. **Formulate the"
}
},
{
"check": "image_document_json",
"status": "pass",
"seconds": 4.05,
"detail": {
"image": "synthetic_receipt.png",
"prompt": "Return only JSON: {\"merchant\": str, \"date\": \"YYYY-MM-DD\", \"total\": float}",
"response": "The user wants to extract information from the provided image and format it as JSON.\nThe required fields are:\n- merchant: The name of the store.\n- date: The date of the invoice.\n- total: The total amount of the invoice.\n\nLooking at the image:\n- The top text says \"NORTHGATE HARDWARE\". This is the merchant.\n- The next line says \"144 Mill Road\". This is the address, not required.\n- The next line says \"Date: 2026-03-14\". This is the date.\n- The"
}
},
{
"check": "image_chart",
"status": "pass",
"seconds": 4.0,
"detail": {
"image": "synthetic_chart.png",
"prompt": "Which quarter is highest, and what value?",
"response": "1. **Identify the user's question:** The user wants to know which quarter has the highest value and what that value is.\n\n2. **Analyze the image:**\n * The image is a bar chart titled \"Units sold by quarter\".\n * The x-axis labels are Q1, Q2, Q3, Q4.\n * The y-axis represents the number of units sold (though there are no explicit y-axis numbers, the values are labeled on top of each bar).\n * The bars are black.\n * Let's read the values for"
}
},
{
"check": "image_caption",
"status": "pass",
"seconds": 4.02,
"detail": {
"image": "synthetic_chart.png",
"prompt": "Describe this image in one sentence.",
"response": "1. **Identify the core subject:** The image is a bar chart titled \"Units sold by quarter\".\n2. **Analyze the axes and labels:** The x-axis represents four quarters: Q1, Q2, Q3, and Q4. The y-axis is implicit (units sold), with values labeled on top of each bar.\n3. **Extract the data points:**\n * Q1: 40\n * Q2: 95\n * Q3: 60\n * Q4: 130\n4."
}
},
{
"check": "multi_turn_conversation",
"status": "pass",
"seconds": 0.91,
"detail": {
"response": "User wants double of 47. Just compute 47*2=94. Output number only.\n</think>\n\n94"
}
},
{
"check": "greedy_determinism",
"status": "pass",
"seconds": 1.83,
"detail": {
"identical": true,
"first": "User asks for three primary colours. Straight fact. No tools needed. Answer directly.\n</think>\n\nRed, yellow, and blue.",
"second": "User asks for three primary colours. Straight fact. No tools needed. Answer directly.\n</think>\n\nRed, yellow, and blue."
}
},
{
"check": "batch_inference",
"status": "pass",
"seconds": 1.19,
"detail": {
"prompts": [
"Capital of Japan?",
"2 + 2 = ?"
],
"responses": [
"User asks capital of Japan. Direct fact. Answer Tokyo.\n</think>\n\nThe capital of Japan is Tokyo.",
"User asks 2+2. Simple math. Answer 4. No tools needed.\n</think>\n\n4"
]
}
},
{
"check": "long_context_retrieval",
"status": "pass",
"seconds": 3.53,
"detail": {
"prompt_tokens": 14429,
"response": "User asks maintenance code for north pump. Earlier text had code QF-8812. No new info. Answer directly.\n</think>\n\nThe maintenance",
"contains_needle": true
}
}
]
}
|