{
"schema_version": 1,
"target": "MLX-4bit",
"source_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0",
"validated_at": "2026-08-15T14:17:10.521605+00:00",
"structural": {
"passed": true,
"passes": [
"artifact directory exists",
"atomic build completion record",
"local SHA-256 manifest",
"all build-hashed files present (23)",
"all build payload SHA-256 hashes match",
"per-artifact quantization manifest",
"artifact manifest base model",
"artifact manifest source revision",
"artifact manifest declares vanilla quantization",
"plausible size 16.95 GB in [16, 21]",
"required sidecar config.json",
"required sidecar tokenizer_config.json",
"required sidecar generation_config.json",
"required sidecar preprocessor_config.json",
"required sidecar video_preprocessor_config.json",
"required sidecar chat_template.jinja",
"required sidecar tokenizer.json",
"required sidecar vocab.json",
"required sidecar merges.txt",
"required sidecar LICENSE",
"chat template byte-identical to source",
"generation_config.json semantically intact",
"preprocessor_config.json semantically intact",
"video_preprocessor_config.json semantically intact",
"tokenizer.json byte-identical to source",
"vocab.json byte-identical to source",
"merges.txt byte-identical to source",
"LICENSE byte-identical to source",
"tokenizer config preserves chat_template",
"tokenizer config preserves eos_token",
"tokenizer config preserves pad_token",
"tokenizer config preserves additional_special_tokens",
"official internal architecture id retained",
"MTP layer declaration retained",
"vision configuration retained",
"image special token id retained",
"video special token id retained",
"vision-start token id retained",
"vision-end token id retained",
"text architecture preserves num_hidden_layers",
"text architecture preserves hidden_size",
"text architecture preserves vocab_size",
"text architecture preserves max_position_embeddings",
"text architecture preserves full_attention_interval",
"text architecture preserves layer_types",
"text architecture preserves linear_conv_kernel_dim",
"text architecture preserves linear_key_head_dim",
"text architecture preserves linear_num_key_heads",
"text architecture preserves linear_num_value_heads",
"text architecture preserves linear_value_head_dim",
"full vision configuration retained",
"main safetensors readable",
"vision tensors retained (333)",
"main MLX bundle does not misplace MTP tensors",
"MTP drafter tensors retained (15)",
"MTP drafter model type",
"all floating tensors finite"
],
"failures": [],
"warnings": [],
"disk_size_bytes": 16946673512
},
"runtime": {
"runtime": "mlx-vlm",
"model": "/Users/oz/Documents/ChatGPT/qwen3.8-27b/qwen38-27b-quants/quantization-suite/artifacts/MLX-4bit",
"artifact_role": "quantized_candidate",
"source_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0",
"text_tests": [
{
"id": "fact",
"category": "factual",
"passed": true,
"failures": [],
"text": "The capital of France is Paris.",
"finish_reason": "stop",
"prompt_tokens": 24,
"generation_tokens": 8,
"generation_tps": 7.708863217103737,
"peak_memory_gb": 17.650076006,
"wall_seconds": 2.303429124993272,
"loop_score": 0.0
},
{
"id": "math",
"category": "mathematical_reasoning",
"passed": true,
"failures": [],
"text": "The user asks a simple multiplication problem: 6 rows of 7 blue marbles. I need to show a short calculation.\n\n6 × 7 = 42\n\n\n6 × 7 = **42** blue marbles.",
"finish_reason": "stop",
"prompt_tokens": 36,
"generation_tokens": 52,
"generation_tps": 6.755312460186613,
"peak_memory_gb": 17.750078564,
"wall_seconds": 8.222668000031263,
"loop_score": 0.0
},
{
"id": "code",
"category": "coding",
"passed": true,
"failures": [],
"text": "The user wants a Python function called `clamp(x, low, high)` that clamps a value `x` to be within the inclusive range `[low, high]`. This is a straightforward utility function. I need to return a single fenced code block.\n\nThe logic is simple:\n- If x < low, return low\n- If x > high, return high\n- Otherwise, return x\n\nI can write this with if/elif/else or use `max(low, min(x, high))`. The latter is more concise and Pythonic. Let me go with the concise version but also consider readability. I'll use the `max(min())` approach as it's clean and idiomatic.\n\n\n```python\ndef clamp(x, low, high):\n \"\"\"Return x limited to the inclusive range [low, high].\"\"\"\n return max(low, min(x, high))\n```",
"finish_reason": "stop",
"prompt_tokens": 67,
"generation_tokens": 188,
"generation_tps": 6.539796290219941,
"peak_memory_gb": 17.831113159,
"wall_seconds": 29.75335170899052,
"loop_score": 0.0
},
{
"id": "multiturn",
"category": "multi_turn",
"passed": true,
"failures": [],
"text": "cedar",
"finish_reason": "stop",
"prompt_tokens": 50,
"generation_tokens": 3,
"generation_tps": 10.821136394561984,
"peak_memory_gb": 17.831113159,
"wall_seconds": 0.8749772500013933,
"loop_score": 0.0
},
{
"id": "thinking_off",
"category": "thinking_disabled",
"passed": true,
"failures": [],
"text": "Yes, 17 is prime.\n\n**Reason:** It has no divisors other than 1 and itself (it is not divisible by any prime number less than its square root, $\\sqrt{17} \\approx 4.12$, i.e., 2, 3, or 4).",
"finish_reason": "stop",
"prompt_tokens": 25,
"generation_tokens": 66,
"generation_tps": 6.707765791331622,
"peak_memory_gb": 17.831113159,
"wall_seconds": 10.531146291992627,
"loop_score": 0.0
},
{
"id": "reasoning_low",
"category": "reasoning_effort_low",
"passed": true,
"failures": [],
"text": "Compare 3/5 and 5/9.\n\n3/5 = 0.6\n5/9 ≈ 0.555...\n\nSo 3/5 is larger.\n\n\n3/5",
"finish_reason": "stop",
"prompt_tokens": 60,
"generation_tokens": 48,
"generation_tps": 6.5793178833779375,
"peak_memory_gb": 17.831113159,
"wall_seconds": 7.950709457974881,
"loop_score": 0.0
},
{
"id": "reasoning_high",
"category": "reasoning_effort_high",
"passed": true,
"failures": [],
"text": "We need answer user: Find smallest positive integer divisible by 6,8,15. Explain briefly. Need compute LCM. Let's reason: 6=2*3, 8=2^3, 15=3*5. LCM max powers: 2^3 *3*5=8*15=120. Smallest positive integer divisible by all is 120. Need final concise.\n\n\nThe smallest positive integer divisible by 6, 8, and 15 is their least common multiple.\n\n- \\(6 = 2 \\cdot 3\\)\n- \\(8 = 2^3\\)\n- \\(15 = 3 \\cdot 5\\)\n\nTake the highest power of each prime:\n\n\\[\n\\text{LCM} = 2^3 \\cdot 3 \\cdot 5 = 8 \\cdot 15 = 120\n\\]\n\nSo the answer is **120**.",
"finish_reason": "stop",
"prompt_tokens": 73,
"generation_tokens": 211,
"generation_tps": 6.50073680608126,
"peak_memory_gb": 17.850597815,
"wall_seconds": 33.41109549999237,
"loop_score": 0.0
},
{
"id": "long_form",
"category": "long_form",
"passed": true,
"failures": [],
"text": "Checksums serve as the foundational integrity mechanism for reproducible machine learning by providing a unique digital fingerprint for every artifact in the release pipeline. By calculating cryptographic hashes for source code, data snapshots, and model binaries, developers can verify that the exact files used during training are identical to those deployed in production. This eliminates ambiguity regarding file corruption or silent modifications, ensuring that the computational inputs remain constant across different environments and time periods, which is critical for debugging and auditing model behavior.\n\nImmutable revisions extend this integrity to the versioning layer by locking specific states of the project into unchangeable records. Unlike mutable branches that can be altered, an immutable revision captures a precise snapshot of the codebase, dependency versions, and configuration parameters at a specific point in time. This ensures that any team member or automated system can reconstruct the exact experimental conditions of a past run, preventing the \"it worked on my machine\" problem and allowing for reliable comparison between different model iterations without the risk of underlying code changes skewing the results.\n\nTogether, these mechanisms create a verifiable chain of custody that bridges the gap between experimental development and stable deployment. When a model is released, the immutable revision defines the logical state of the project, while the checksums validate the physical integrity of the assets within that state. This dual-layer approach allows for deterministic reproduction, meaning that running the same pipeline with the same inputs will yield the same outputs, provided the hardware and software environments are compatible, thereby fostering trust in the model’s performance metrics and decision-making logic.\n\nUltimately, the integration of checksums and immutable revisions transforms machine learning releases from opaque black boxes into transparent, auditable engineering products. This transparency satisfies regulatory requirements for explainability and accountability, particularly in high-stakes domains like healthcare or finance, where model drift or unexpected behavior can have severe consequences. By guaranteeing that every release is a faithful, verifiable replica of its training environment, organizations can confidently scale their AI systems while maintaining rigorous standards of quality control and scientific rigor.",
"finish_reason": "stop",
"prompt_tokens": 31,
"generation_tokens": 402,
"generation_tps": 6.735802770366746,
"peak_memory_gb": 17.850597815,
"wall_seconds": 60.33028987504076,
"loop_score": 0.0
},
{
"id": "non_english",
"category": "non_english",
"passed": true,
"failures": [],
"text": "太陽光が大気中の分子に当たると、波長の短い青い光が他の色よりも強く散乱されます。そのため、私たちの目には空全体が青く見えるのです。",
"finish_reason": "stop",
"prompt_tokens": 28,
"generation_tokens": 40,
"generation_tps": 7.6589071248005105,
"peak_memory_gb": 17.850597815,
"wall_seconds": 5.75853370805271,
"loop_score": 0.0
},
{
"id": "json",
"category": "structured_json",
"passed": true,
"failures": [],
"text": "{\n \"name\": \"sample\",\n \"primes\": [2, 3, 5],\n \"active\": true\n}",
"finish_reason": "stop",
"prompt_tokens": 41,
"generation_tokens": 33,
"generation_tps": 7.827532342230347,
"peak_memory_gb": 17.850597815,
"wall_seconds": 4.699750334024429,
"loop_score": 0.0
}
],
"tool_tests": [
{
"id": "one_tool",
"passed": true,
"failures": [],
"calls": [
{
"name": "weather_at",
"arguments": {
"location": "Paris",
"units": "celsius"
}
}
],
"raw": "The user is asking to call weather_at for Paris in Celsius units.\n\n\n\n\n\nParis\n\n\ncelsius\n\n\n",
"finish_reason": "stop",
"dependent_raw": null
},
{
"id": "parallel_tools",
"passed": true,
"failures": [],
"calls": [
{
"name": "weather_at",
"arguments": {
"location": "Paris",
"units": "celsius"
}
},
{
"name": "weather_at",
"arguments": {
"location": "Austin",
"units": "fahrenheit"
}
}
],
"raw": "The user is asking to call weather_at in parallel for Paris (Celsius) and Austin (Fahrenheit). Since these are independent calls, they should be made within the same block.\n\n\n\n\n\nParis\n\n\ncelsius\n\n\n\n\n\n\nAustin\n\n\nfahrenheit\n\n\n",
"finish_reason": "stop",
"dependent_raw": null
},
{
"id": "sequential_dependency",
"passed": true,
"failures": [],
"calls": [
{
"name": "weather_at",
"arguments": {
"location": "Paris",
"units": "celsius"
}
}
],
"raw": "The user is asking me to do the following:\n1. First, call weather_at for Paris in Celsius\n2. After receiving the result, create an event whose title is exactly the returned condition string\n\nSince the second step depends on the result of the first step, I need to call weather_at first and wait for the result before creating the event.\n\n\n\n\n\nParis\n\n\ncelsius\n\n\n",
"finish_reason": "stop",
"dependent_raw": "The weather condition returned is \"sunny\". Now I need to create an event with the title exactly \"sunny\".\n\n\n\n\n\nsunny\n\n\n"
},
{
"id": "nested_json",
"passed": true,
"failures": [],
"calls": [
{
"name": "add_event",
"arguments": {
"title": "Release review",
"attendees": [
"ana@example.test",
"li@example.test"
],
"metadata": {
"release": {
"model": "Qwen3.8-27B",
"slot": 21
}
}
}
}
],
"raw": "The user is asking to call add_event with specific parameters:\n- title: \"Release review\"\n- attendees: [\"ana@example.test\", \"li@example.test\"]\n- metadata: {\"release\":{\"model\":\"Qwen3.8-27B\",\"slot\":21}}\n\nThis is a simple, direct request. Let's make the function call.\n\n\n\n\n\nRelease review\n\n\n[\"ana@example.test\", \"li@example.test\"]\n\n\n{\"release\": {\"model\": \"Qwen3.8-27B\", \"slot\": 21}}\n\n\n",
"finish_reason": "stop",
"dependent_raw": null
},
{
"id": "invalid_recovery",
"passed": true,
"failures": [],
"calls": [
{
"name": "weather_at",
"arguments": {
"location": "Paris",
"units": "celsius"
}
}
],
"raw": "The user is asking to call weather_at with location=Paris, units=kelvin. However, the schema only allows \"celsius\" or \"fahrenheit\" for units. Therefore, kelvin violates the schema. The user says: \"If that violates the schema, recover by using celsius.\"\n\nSo I should call weather_at with location=\"Paris\", units=\"celsius\".\n\nLet me make the call.\n\n\n\n\n\nParis\n\n\ncelsius\n\n\n",
"finish_reason": "stop",
"dependent_raw": null
}
],
"vision_tests": [
{
"id": "shapes_colors",
"passed": true,
"missing_patterns": [],
"text": "From left to right, the three large shapes are:\n\n1. **Red square** \n2. **Blue circle** \n3. **Green triangle**\n\nThese correspond to the labels “RED”, “BLUE”, and “GREEN” shown beneath each shape in the image.",
"finish_reason": "stop"
},
{
"id": "printed_text",
"passed": true,
"missing_patterns": [],
"text": "VISION CHECK 27B",
"finish_reason": "stop"
},
{
"id": "chart",
"passed": true,
"missing_patterns": [],
"text": "Looking at the bar chart:\n\n- Bar **A** has a value of **60**\n- Bar **B** has a value of **105**\n- Bar **C** has a value of **135**\n\nThe tallest bar is **C**, and the number printed above it is **135**.\n\n✅ **Answer: C, 135**",
"finish_reason": "stop"
}
],
"mtp": {
"passed": true,
"drafter_kind": "mtp",
"output_equivalent_temperature_zero": true,
"accepted_drafts": 83,
"drafted_tokens": 90,
"acceptance_rate": 0.9222222222222223,
"baseline_tps": 7.73703740076071,
"mtp_tps": 7.240511282952493,
"speedup": 0.9358247747698107,
"measured_improvement": false,
"baseline_wall_seconds": 16.956261500017717,
"mtp_wall_seconds": 18.137535667046905,
"advertise_acceleration": false
},
"warnings": [],
"phases": [
"mtp",
"text",
"tools",
"vision"
],
"validation_inputs": {
"prompts_sha256": "136a918e5fee962f2b52f8e520a0275fd5ec5569181e0f5fdf4910ee3c34d528",
"tools_sha256": "19ddc6236b18ec34f3f50b84ef959167b14baa2b9bab7b4145e7d5855d7c98c9",
"image_sha256": "0b1ae6badbe19a6049305c36d165a34cdf36df00033842e265339b2b7f057295"
},
"metal_memory_policy": {
"device": {
"device_name": "Apple M5 Pro",
"max_recommended_working_set_size": 55662788608,
"memory_size": 68719476736,
"architecture": "applegpu_g17s",
"max_buffer_length": 41747087360,
"resource_limit": 499000
},
"cache_limit_bytes": 256000000,
"wired_limit_bytes": 54549532835,
"previous_cache_limit_bytes": 65283502899,
"previous_wired_limit_bytes": 0,
"warnings": []
}
},
"warnings": [],
"overall_passed": true,
"runtime_failures": [],
"quality": {
"schema_version": 1,
"comparison_type": "cross-runtime output agreement against pinned BF16 source",
"passed": true,
"source_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0",
"embedding_model": {
"repo_id": "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2",
"revision": "e8f8c211226b894fcb81acc59f3b34ba3efd5f42",
"pooling": "attention-mask mean pooling followed by L2 normalization",
"maximum_tokens": 256
},
"thresholds": {
"mean_semantic_similarity": 0.55,
"per_case_severe_regression": 0.25
},
"baseline_valid": true,
"candidate_functional": true,
"semantic_gate_passed": true,
"validation_inputs_match": true,
"validation_inputs": {
"prompts_sha256": "136a918e5fee962f2b52f8e520a0275fd5ec5569181e0f5fdf4910ee3c34d528",
"tools_sha256": "19ddc6236b18ec34f3f50b84ef959167b14baa2b9bab7b4145e7d5855d7c98c9",
"image_sha256": "0b1ae6badbe19a6049305c36d165a34cdf36df00033842e265339b2b7f057295"
},
"mean_semantic_similarity": 0.948318076133728,
"exact_matches": 3,
"comparisons": [
{
"id": "fact",
"reference_passed": true,
"candidate_passed": true,
"exact_match": true,
"sequence_agreement": 1.0,
"semantic_similarity": 1.0,
"severe_regression": false,
"candidate_loop_score": 0.0
},
{
"id": "math",
"reference_passed": true,
"candidate_passed": true,
"exact_match": true,
"sequence_agreement": 1.0,
"semantic_similarity": 0.9999999403953552,
"severe_regression": false,
"candidate_loop_score": 0.0
},
{
"id": "code",
"reference_passed": true,
"candidate_passed": true,
"exact_match": false,
"sequence_agreement": 0.6566164154103853,
"semantic_similarity": 0.8689196109771729,
"severe_regression": false,
"candidate_loop_score": 0.0
},
{
"id": "multiturn",
"reference_passed": true,
"candidate_passed": true,
"exact_match": true,
"sequence_agreement": 1.0,
"semantic_similarity": 1.0,
"severe_regression": false,
"candidate_loop_score": 0.0
},
{
"id": "thinking_off",
"reference_passed": true,
"candidate_passed": true,
"exact_match": false,
"sequence_agreement": 0.9105691056910569,
"semantic_similarity": 0.9957865476608276,
"severe_regression": false,
"candidate_loop_score": 0.0
},
{
"id": "reasoning_low",
"reference_passed": true,
"candidate_passed": true,
"exact_match": false,
"sequence_agreement": 0.7906976744186046,
"semantic_similarity": 0.9233368635177612,
"severe_regression": false,
"candidate_loop_score": 0.0
},
{
"id": "reasoning_high",
"reference_passed": true,
"candidate_passed": true,
"exact_match": false,
"sequence_agreement": 0.605427974947808,
"semantic_similarity": 0.9447025060653687,
"severe_regression": false,
"candidate_loop_score": 0.0
},
{
"id": "long_form",
"reference_passed": true,
"candidate_passed": true,
"exact_match": false,
"sequence_agreement": 0.18007586344579757,
"semantic_similarity": 0.8539742827415466,
"severe_regression": false,
"candidate_loop_score": 0.0
},
{
"id": "non_english",
"reference_passed": true,
"candidate_passed": true,
"exact_match": false,
"sequence_agreement": 0.9302325581395349,
"semantic_similarity": 0.9963569641113281,
"severe_regression": false,
"candidate_loop_score": 0.0
},
{
"id": "json",
"reference_passed": true,
"candidate_passed": true,
"exact_match": false,
"sequence_agreement": 0.9130434782608695,
"semantic_similarity": 0.9001040458679199,
"severe_regression": false,
"candidate_loop_score": 0.0
}
],
"functional_results": {
"reference_text": {
"passed": 10,
"total": 10
},
"candidate_text": {
"passed": 10,
"total": 10
},
"reference_tools": {
"passed": 5,
"total": 5
},
"candidate_tools": {
"passed": 5,
"total": 5
},
"reference_vision": {
"passed": 3,
"total": 3
},
"candidate_vision": {
"passed": 3,
"total": 3
}
},
"measurements": {
"average_generation_tps": 7.383517108026069,
"peak_memory_gb": 17.850597815,
"artifact_bytes": 16946652592,
"maximum_prompt_tokens_tested": 73,
"loop_rate": 0.0
},
"warnings": [
"Semantic similarity is a measured embedding-model proxy, not ground-truth accuracy.",
"Raw-logit equality is unavailable across all target runtimes; exact functional gates and output agreement are used for portable release validation.",
"Sequence agreement is lexical and is reported diagnostically, not used as semantic accuracy."
]
},
"quality_failures": []
}