{ "suite_size": 200, "seed": 42, "models": [ { "name": "SAM", "is_sam": true, "params": "33.4M", "params_raw": 33400324, "size_mb": 127.4, "json_valid": 88.0, "op_correct": 63.5, "exact_match": 18.0, "latency_ms": 715.4, "n_cases": 200, "per_tier": [ { "tier": 1, "n": 40, "json_valid": 100.0, "op_correct": 82.5, "exact_match": 37.5 }, { "tier": 2, "n": 40, "json_valid": 100.0, "op_correct": 70.0, "exact_match": 27.5 }, { "tier": 3, "n": 40, "json_valid": 100.0, "op_correct": 100.0, "exact_match": 25.0 }, { "tier": 4, "n": 40, "json_valid": 75.0, "op_correct": 0.0, "exact_match": 0.0 }, { "tier": 5, "n": 40, "json_valid": 65.0, "op_correct": 65.0, "exact_match": 0.0 } ], "per_domain": [ { "domain": "CAL", "n": 2, "json_valid": 100.0, "op_correct": 100.0, "exact_match": 100.0 }, { "domain": "DB", "n": 27, "json_valid": 66.7, "op_correct": 66.7, "exact_match": 11.1 }, { "domain": "ECOMMERCE", "n": 8, "json_valid": 100.0, "op_correct": 87.5, "exact_match": 0.0 }, { "domain": "FILE", "n": 4, "json_valid": 100.0, "op_correct": 100.0, "exact_match": 75.0 }, { "domain": "HOME", "n": 7, "json_valid": 100.0, "op_correct": 28.6, "exact_match": 0.0 }, { "domain": "HTTP", "n": 25, "json_valid": 100.0, "op_correct": 64.0, "exact_match": 44.0 }, { "domain": "MQTT", "n": 15, "json_valid": 100.0, "op_correct": 100.0, "exact_match": 6.7 }, { "domain": "ROS", "n": 46, "json_valid": 97.8, "op_correct": 50.0, "exact_match": 10.9 }, { "domain": "VEHICLE", "n": 17, "json_valid": 100.0, "op_correct": 29.4, "exact_match": 17.6 }, { "domain": "WORKFLOW", "n": 49, "json_valid": 71.4, "op_correct": 71.4, "exact_match": 16.3 } ] }, { "name": "SmolLM2-360M", "is_sam": false, "params": "361.8M", "params_raw": 361821120, "size_mb": 690.1, "json_valid": 89.0, "op_correct": 0.0, "exact_match": 0.0, "latency_ms": 2042.9, "n_cases": 200, "per_tier": [ { "tier": 1, "n": 40, "json_valid": 100.0, "op_correct": 0.0, "exact_match": 0.0 }, { "tier": 2, "n": 40, "json_valid": 100.0, "op_correct": 0.0, "exact_match": 0.0 }, { "tier": 3, "n": 40, "json_valid": 100.0, "op_correct": 0.0, "exact_match": 0.0 }, { "tier": 4, "n": 40, "json_valid": 45.0, "op_correct": 0.0, "exact_match": 0.0 }, { "tier": 5, "n": 40, "json_valid": 100.0, "op_correct": 0.0, "exact_match": 0.0 } ], "per_domain": [ { "domain": "CAL", "n": 2, "json_valid": 100.0, "op_correct": 0.0, "exact_match": 0.0 }, { "domain": "DB", "n": 27, "json_valid": 100.0, "op_correct": 0.0, "exact_match": 0.0 }, { "domain": "ECOMMERCE", "n": 8, "json_valid": 100.0, "op_correct": 0.0, "exact_match": 0.0 }, { "domain": "FILE", "n": 4, "json_valid": 100.0, "op_correct": 0.0, "exact_match": 0.0 }, { "domain": "HOME", "n": 7, "json_valid": 100.0, "op_correct": 0.0, "exact_match": 0.0 }, { "domain": "HTTP", "n": 25, "json_valid": 100.0, "op_correct": 0.0, "exact_match": 0.0 }, { "domain": "MQTT", "n": 15, "json_valid": 100.0, "op_correct": 0.0, "exact_match": 0.0 }, { "domain": "ROS", "n": 46, "json_valid": 76.1, "op_correct": 0.0, "exact_match": 0.0 }, { "domain": "VEHICLE", "n": 17, "json_valid": 35.3, "op_correct": 0.0, "exact_match": 0.0 }, { "domain": "WORKFLOW", "n": 49, "json_valid": 100.0, "op_correct": 0.0, "exact_match": 0.0 } ] }, { "name": "Qwen2.5-0.5B", "is_sam": false, "params": "494.0M", "params_raw": 494032768, "size_mb": 942.3, "json_valid": 70.5, "op_correct": 30.0, "exact_match": 2.5, "latency_ms": 1937.2, "n_cases": 200, "per_tier": [ { "tier": 1, "n": 40, "json_valid": 60.0, "op_correct": 42.5, "exact_match": 12.5 }, { "tier": 2, "n": 40, "json_valid": 65.0, "op_correct": 35.0, "exact_match": 0.0 }, { "tier": 3, "n": 40, "json_valid": 67.5, "op_correct": 45.0, "exact_match": 0.0 }, { "tier": 4, "n": 40, "json_valid": 95.0, "op_correct": 27.5, "exact_match": 0.0 }, { "tier": 5, "n": 40, "json_valid": 65.0, "op_correct": 0.0, "exact_match": 0.0 } ], "per_domain": [ { "domain": "CAL", "n": 2, "json_valid": 100.0, "op_correct": 50.0, "exact_match": 0.0 }, { "domain": "DB", "n": 27, "json_valid": 44.4, "op_correct": 7.4, "exact_match": 0.0 }, { "domain": "ECOMMERCE", "n": 8, "json_valid": 25.0, "op_correct": 25.0, "exact_match": 0.0 }, { "domain": "FILE", "n": 4, "json_valid": 100.0, "op_correct": 100.0, "exact_match": 0.0 }, { "domain": "HOME", "n": 7, "json_valid": 100.0, "op_correct": 42.9, "exact_match": 0.0 }, { "domain": "HTTP", "n": 25, "json_valid": 72.0, "op_correct": 36.0, "exact_match": 0.0 }, { "domain": "MQTT", "n": 15, "json_valid": 86.7, "op_correct": 26.7, "exact_match": 0.0 }, { "domain": "ROS", "n": 46, "json_valid": 73.9, "op_correct": 43.5, "exact_match": 4.3 }, { "domain": "VEHICLE", "n": 17, "json_valid": 82.4, "op_correct": 82.4, "exact_match": 17.6 }, { "domain": "WORKFLOW", "n": 49, "json_valid": 71.4, "op_correct": 2.0, "exact_match": 0.0 } ] }, { "name": "TinyLlama-1.1B", "is_sam": false, "params": "1100.0M", "params_raw": 1100048384, "size_mb": 2098.2, "json_valid": 92.0, "op_correct": 55.5, "exact_match": 9.0, "latency_ms": 4199.3, "n_cases": 200, "per_tier": [ { "tier": 1, "n": 40, "json_valid": 100.0, "op_correct": 95.0, "exact_match": 27.5 }, { "tier": 2, "n": 40, "json_valid": 90.0, "op_correct": 82.5, "exact_match": 12.5 }, { "tier": 3, "n": 40, "json_valid": 100.0, "op_correct": 77.5, "exact_match": 5.0 }, { "tier": 4, "n": 40, "json_valid": 70.0, "op_correct": 22.5, "exact_match": 0.0 }, { "tier": 5, "n": 40, "json_valid": 100.0, "op_correct": 0.0, "exact_match": 0.0 } ], "per_domain": [ { "domain": "CAL", "n": 2, "json_valid": 100.0, "op_correct": 100.0, "exact_match": 0.0 }, { "domain": "DB", "n": 27, "json_valid": 100.0, "op_correct": 66.7, "exact_match": 18.5 }, { "domain": "ECOMMERCE", "n": 8, "json_valid": 100.0, "op_correct": 100.0, "exact_match": 0.0 }, { "domain": "FILE", "n": 4, "json_valid": 100.0, "op_correct": 100.0, "exact_match": 25.0 }, { "domain": "HOME", "n": 7, "json_valid": 100.0, "op_correct": 100.0, "exact_match": 0.0 }, { "domain": "HTTP", "n": 25, "json_valid": 84.0, "op_correct": 80.0, "exact_match": 0.0 }, { "domain": "MQTT", "n": 15, "json_valid": 100.0, "op_correct": 100.0, "exact_match": 0.0 }, { "domain": "ROS", "n": 46, "json_valid": 76.1, "op_correct": 47.8, "exact_match": 17.4 }, { "domain": "VEHICLE", "n": 17, "json_valid": 94.1, "op_correct": 35.3, "exact_match": 23.5 }, { "domain": "WORKFLOW", "n": 49, "json_valid": 100.0, "op_correct": 18.4, "exact_match": 0.0 } ] }, { "name": "Llama-3.2-1B", "is_sam": false, "params": "?", "params_raw": 0, "size_mb": 0, "json_valid": 0, "op_correct": 0, "exact_match": 0, "latency_ms": 0, "error": "load_failed" }, { "name": "Qwen2.5-1.5B", "is_sam": false, "params": "1543.7M", "params_raw": 1543714304, "size_mb": 2944.4, "json_valid": 100.0, "op_correct": 78.5, "exact_match": 9.0, "latency_ms": 1852.4, "n_cases": 200, "per_tier": [ { "tier": 1, "n": 40, "json_valid": 100.0, "op_correct": 95.0, "exact_match": 30.0 }, { "tier": 2, "n": 40, "json_valid": 100.0, "op_correct": 85.0, "exact_match": 12.5 }, { "tier": 3, "n": 40, "json_valid": 100.0, "op_correct": 100.0, "exact_match": 2.5 }, { "tier": 4, "n": 40, "json_valid": 100.0, "op_correct": 72.5, "exact_match": 0.0 }, { "tier": 5, "n": 40, "json_valid": 100.0, "op_correct": 40.0, "exact_match": 0.0 } ], "per_domain": [ { "domain": "CAL", "n": 2, "json_valid": 100.0, "op_correct": 100.0, "exact_match": 0.0 }, { "domain": "DB", "n": 27, "json_valid": 100.0, "op_correct": 100.0, "exact_match": 18.5 }, { "domain": "ECOMMERCE", "n": 8, "json_valid": 100.0, "op_correct": 100.0, "exact_match": 0.0 }, { "domain": "FILE", "n": 4, "json_valid": 100.0, "op_correct": 100.0, "exact_match": 0.0 }, { "domain": "HOME", "n": 7, "json_valid": 100.0, "op_correct": 85.7, "exact_match": 0.0 }, { "domain": "HTTP", "n": 25, "json_valid": 100.0, "op_correct": 84.0, "exact_match": 4.0 }, { "domain": "MQTT", "n": 15, "json_valid": 100.0, "op_correct": 93.3, "exact_match": 0.0 }, { "domain": "ROS", "n": 46, "json_valid": 100.0, "op_correct": 71.7, "exact_match": 19.6 }, { "domain": "VEHICLE", "n": 17, "json_valid": 100.0, "op_correct": 100.0, "exact_match": 17.6 }, { "domain": "WORKFLOW", "n": 49, "json_valid": 100.0, "op_correct": 51.0, "exact_match": 0.0 } ] }, { "name": "SmolLM2-1.7B", "is_sam": false, "params": "1711.4M", "params_raw": 1711376384, "size_mb": 3264.2, "json_valid": 89.0, "op_correct": 38.5, "exact_match": 2.5, "latency_ms": 1694.9, "n_cases": 200, "per_tier": [ { "tier": 1, "n": 40, "json_valid": 92.5, "op_correct": 75.0, "exact_match": 12.5 }, { "tier": 2, "n": 40, "json_valid": 80.0, "op_correct": 62.5, "exact_match": 0.0 }, { "tier": 3, "n": 40, "json_valid": 100.0, "op_correct": 55.0, "exact_match": 0.0 }, { "tier": 4, "n": 40, "json_valid": 72.5, "op_correct": 0.0, "exact_match": 0.0 }, { "tier": 5, "n": 40, "json_valid": 100.0, "op_correct": 0.0, "exact_match": 0.0 } ], "per_domain": [ { "domain": "CAL", "n": 2, "json_valid": 100.0, "op_correct": 100.0, "exact_match": 0.0 }, { "domain": "DB", "n": 27, "json_valid": 66.7, "op_correct": 59.3, "exact_match": 0.0 }, { "domain": "ECOMMERCE", "n": 8, "json_valid": 100.0, "op_correct": 100.0, "exact_match": 0.0 }, { "domain": "FILE", "n": 4, "json_valid": 100.0, "op_correct": 100.0, "exact_match": 0.0 }, { "domain": "HOME", "n": 7, "json_valid": 85.7, "op_correct": 57.1, "exact_match": 0.0 }, { "domain": "HTTP", "n": 25, "json_valid": 100.0, "op_correct": 4.0, "exact_match": 0.0 }, { "domain": "MQTT", "n": 15, "json_valid": 100.0, "op_correct": 100.0, "exact_match": 0.0 }, { "domain": "ROS", "n": 46, "json_valid": 95.7, "op_correct": 47.8, "exact_match": 4.3 }, { "domain": "VEHICLE", "n": 17, "json_valid": 88.2, "op_correct": 23.5, "exact_match": 17.6 }, { "domain": "WORKFLOW", "n": 49, "json_valid": 83.7, "op_correct": 2.0, "exact_match": 0.0 } ] }, { "name": "Gemma-2-2B", "is_sam": false, "params": "?", "params_raw": 0, "size_mb": 0, "json_valid": 0, "op_correct": 0, "exact_match": 0, "latency_ms": 0, "error": "load_failed" }, { "name": "Qwen2.5-3B", "is_sam": false, "params": "3085.9M", "params_raw": 3085938688, "size_mb": 5886.0, "json_valid": 100.0, "op_correct": 82.5, "exact_match": 16.0, "latency_ms": 2061.1, "n_cases": 200, "per_tier": [ { "tier": 1, "n": 40, "json_valid": 100.0, "op_correct": 97.5, "exact_match": 40.0 }, { "tier": 2, "n": 40, "json_valid": 100.0, "op_correct": 100.0, "exact_match": 12.5 }, { "tier": 3, "n": 40, "json_valid": 100.0, "op_correct": 100.0, "exact_match": 27.5 }, { "tier": 4, "n": 40, "json_valid": 100.0, "op_correct": 75.0, "exact_match": 0.0 }, { "tier": 5, "n": 40, "json_valid": 100.0, "op_correct": 40.0, "exact_match": 0.0 } ], "per_domain": [ { "domain": "CAL", "n": 2, "json_valid": 100.0, "op_correct": 100.0, "exact_match": 50.0 }, { "domain": "DB", "n": 27, "json_valid": 100.0, "op_correct": 100.0, "exact_match": 18.5 }, { "domain": "ECOMMERCE", "n": 8, "json_valid": 100.0, "op_correct": 100.0, "exact_match": 0.0 }, { "domain": "FILE", "n": 4, "json_valid": 100.0, "op_correct": 75.0, "exact_match": 0.0 }, { "domain": "HOME", "n": 7, "json_valid": 100.0, "op_correct": 100.0, "exact_match": 0.0 }, { "domain": "HTTP", "n": 25, "json_valid": 100.0, "op_correct": 100.0, "exact_match": 8.0 }, { "domain": "MQTT", "n": 15, "json_valid": 100.0, "op_correct": 100.0, "exact_match": 73.3 }, { "domain": "ROS", "n": 46, "json_valid": 100.0, "op_correct": 78.3, "exact_match": 21.7 }, { "domain": "VEHICLE", "n": 17, "json_valid": 100.0, "op_correct": 100.0, "exact_match": 17.6 }, { "domain": "WORKFLOW", "n": 49, "json_valid": 100.0, "op_correct": 51.0, "exact_match": 0.0 } ] }, { "name": "Llama-3.2-3B", "is_sam": false, "params": "?", "params_raw": 0, "size_mb": 0, "json_valid": 0, "op_correct": 0, "exact_match": 0, "latency_ms": 0, "error": "load_failed" }, { "name": "Phi-3.5-mini-3.8B", "is_sam": false, "params": "?", "params_raw": 0, "size_mb": 0, "json_valid": 0, "op_correct": 0, "exact_match": 0, "latency_ms": 0, "error": "load_failed" }, { "name": "Llama-3.2-3B-uns", "is_sam": false, "params": "?", "params_raw": 0, "size_mb": 0, "json_valid": 0, "op_correct": 0, "exact_match": 0, "latency_ms": 0, "error": "load_failed" }, { "name": "Gemma-3-270M-GGUF", "is_sam": false, "params": "270.0M", "params_raw": 270000000, "size_mb": 241.4, "json_valid": 92.0, "op_correct": 2.0, "exact_match": 0.0, "latency_ms": 6322.5, "n_cases": 200, "per_tier": [ { "tier": 1, "n": 40, "json_valid": 85.0, "op_correct": 5.0, "exact_match": 0.0 }, { "tier": 2, "n": 40, "json_valid": 97.5, "op_correct": 5.0, "exact_match": 0.0 }, { "tier": 3, "n": 40, "json_valid": 90.0, "op_correct": 0.0, "exact_match": 0.0 }, { "tier": 4, "n": 40, "json_valid": 87.5, "op_correct": 0.0, "exact_match": 0.0 }, { "tier": 5, "n": 40, "json_valid": 100.0, "op_correct": 0.0, "exact_match": 0.0 } ], "per_domain": [ { "domain": "CAL", "n": 2, "json_valid": 100.0, "op_correct": 0.0, "exact_match": 0.0 }, { "domain": "DB", "n": 27, "json_valid": 88.9, "op_correct": 0.0, "exact_match": 0.0 }, { "domain": "ECOMMERCE", "n": 8, "json_valid": 100.0, "op_correct": 12.5, "exact_match": 0.0 }, { "domain": "FILE", "n": 4, "json_valid": 100.0, "op_correct": 0.0, "exact_match": 0.0 }, { "domain": "HOME", "n": 7, "json_valid": 100.0, "op_correct": 42.9, "exact_match": 0.0 }, { "domain": "HTTP", "n": 25, "json_valid": 92.0, "op_correct": 0.0, "exact_match": 0.0 }, { "domain": "MQTT", "n": 15, "json_valid": 66.7, "op_correct": 0.0, "exact_match": 0.0 }, { "domain": "ROS", "n": 46, "json_valid": 87.0, "op_correct": 0.0, "exact_match": 0.0 }, { "domain": "VEHICLE", "n": 17, "json_valid": 100.0, "op_correct": 0.0, "exact_match": 0.0 }, { "domain": "WORKFLOW", "n": 49, "json_valid": 100.0, "op_correct": 0.0, "exact_match": 0.0 } ] }, { "name": "Ministral-3-8B-GGUF", "is_sam": false, "params": "?", "params_raw": 0, "size_mb": 0, "json_valid": 0, "op_correct": 0, "exact_match": 0, "latency_ms": 0, "error": "load_failed" }, { "name": "DeepSeek-R1-Qwen3-8B-GGUF", "is_sam": false, "params": "8000.0M", "params_raw": 8000000000, "size_mb": 4794.9, "json_valid": 98.0, "op_correct": 79.0, "exact_match": 11.5, "latency_ms": 31737.9, "n_cases": 200, "per_tier": [ { "tier": 1, "n": 40, "json_valid": 100.0, "op_correct": 90.0, "exact_match": 27.5 }, { "tier": 2, "n": 40, "json_valid": 95.0, "op_correct": 95.0, "exact_match": 7.5 }, { "tier": 3, "n": 40, "json_valid": 100.0, "op_correct": 77.5, "exact_match": 22.5 }, { "tier": 4, "n": 40, "json_valid": 95.0, "op_correct": 92.5, "exact_match": 0.0 }, { "tier": 5, "n": 40, "json_valid": 100.0, "op_correct": 40.0, "exact_match": 0.0 } ], "per_domain": [ { "domain": "CAL", "n": 2, "json_valid": 100.0, "op_correct": 100.0, "exact_match": 50.0 }, { "domain": "DB", "n": 27, "json_valid": 100.0, "op_correct": 100.0, "exact_match": 0.0 }, { "domain": "ECOMMERCE", "n": 8, "json_valid": 100.0, "op_correct": 87.5, "exact_match": 0.0 }, { "domain": "FILE", "n": 4, "json_valid": 100.0, "op_correct": 100.0, "exact_match": 25.0 }, { "domain": "HOME", "n": 7, "json_valid": 100.0, "op_correct": 85.7, "exact_match": 0.0 }, { "domain": "HTTP", "n": 25, "json_valid": 92.0, "op_correct": 92.0, "exact_match": 8.0 }, { "domain": "MQTT", "n": 15, "json_valid": 100.0, "op_correct": 100.0, "exact_match": 73.3 }, { "domain": "ROS", "n": 46, "json_valid": 95.7, "op_correct": 69.6, "exact_match": 10.9 }, { "domain": "VEHICLE", "n": 17, "json_valid": 100.0, "op_correct": 100.0, "exact_match": 17.6 }, { "domain": "WORKFLOW", "n": 49, "json_valid": 100.0, "op_correct": 51.0, "exact_match": 0.0 } ] } ], "per_tier": [ { "tier": 1, "n": 40, "json_valid": 100.0, "op_correct": 82.5, "exact_match": 37.5, "description": "Atomic \u2014 single op" }, { "tier": 2, "n": 40, "json_valid": 100.0, "op_correct": 70.0, "exact_match": 27.5, "description": "Compound \u2014 2-3 ops same domain" }, { "tier": 3, "n": 40, "json_valid": 100.0, "op_correct": 100.0, "exact_match": 25.0, "description": "Noisy \u2014 typos / casing / fillers" }, { "tier": 4, "n": 40, "json_valid": 75.0, "op_correct": 0.0, "exact_match": 0.0, "description": "Long chain \u2014 4+ ops" }, { "tier": 5, "n": 40, "json_valid": 65.0, "op_correct": 65.0, "exact_match": 0.0, "description": "Cross-domain workflow" } ], "per_domain": [ { "domain": "CAL", "n": 2, "json_valid": 100.0, "op_correct": 100.0, "exact_match": 100.0 }, { "domain": "DB", "n": 27, "json_valid": 66.7, "op_correct": 66.7, "exact_match": 11.1 }, { "domain": "ECOMMERCE", "n": 8, "json_valid": 100.0, "op_correct": 87.5, "exact_match": 0.0 }, { "domain": "FILE", "n": 4, "json_valid": 100.0, "op_correct": 100.0, "exact_match": 75.0 }, { "domain": "HOME", "n": 7, "json_valid": 100.0, "op_correct": 28.6, "exact_match": 0.0 }, { "domain": "HTTP", "n": 25, "json_valid": 100.0, "op_correct": 64.0, "exact_match": 44.0 }, { "domain": "MQTT", "n": 15, "json_valid": 100.0, "op_correct": 100.0, "exact_match": 6.7 }, { "domain": "ROS", "n": 46, "json_valid": 97.8, "op_correct": 50.0, "exact_match": 10.9 }, { "domain": "VEHICLE", "n": 17, "json_valid": 100.0, "op_correct": 29.4, "exact_match": 17.6 }, { "domain": "WORKFLOW", "n": 49, "json_valid": 71.4, "op_correct": 71.4, "exact_match": 16.3 } ], "tier_distribution": { "1": 40, "2": 40, "3": 40, "4": 40, "5": 40 }, "domain_distribution": { "CAL": 2, "DB": 27, "ECOMMERCE": 8, "FILE": 4, "HOME": 7, "HTTP": 25, "MQTT": 15, "ROS": 46, "VEHICLE": 17, "WORKFLOW": 49 } }