| { | |
| "suite_size": 200, | |
| "seed": 42, | |
| "models": [ | |
| { | |
| "name": "SAM", | |
| "is_sam": true, | |
| "params": "33.4M", | |
| "params_raw": 33400324, | |
| "size_mb": 127.4, | |
| "json_valid": 88.0, | |
| "op_correct": 63.5, | |
| "exact_match": 18.0, | |
| "latency_ms": 715.4, | |
| "n_cases": 200, | |
| "per_tier": [ | |
| { | |
| "tier": 1, | |
| "n": 40, | |
| "json_valid": 100.0, | |
| "op_correct": 82.5, | |
| "exact_match": 37.5 | |
| }, | |
| { | |
| "tier": 2, | |
| "n": 40, | |
| "json_valid": 100.0, | |
| "op_correct": 70.0, | |
| "exact_match": 27.5 | |
| }, | |
| { | |
| "tier": 3, | |
| "n": 40, | |
| "json_valid": 100.0, | |
| "op_correct": 100.0, | |
| "exact_match": 25.0 | |
| }, | |
| { | |
| "tier": 4, | |
| "n": 40, | |
| "json_valid": 75.0, | |
| "op_correct": 0.0, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "tier": 5, | |
| "n": 40, | |
| "json_valid": 65.0, | |
| "op_correct": 65.0, | |
| "exact_match": 0.0 | |
| } | |
| ], | |
| "per_domain": [ | |
| { | |
| "domain": "CAL", | |
| "n": 2, | |
| "json_valid": 100.0, | |
| "op_correct": 100.0, | |
| "exact_match": 100.0 | |
| }, | |
| { | |
| "domain": "DB", | |
| "n": 27, | |
| "json_valid": 66.7, | |
| "op_correct": 66.7, | |
| "exact_match": 11.1 | |
| }, | |
| { | |
| "domain": "ECOMMERCE", | |
| "n": 8, | |
| "json_valid": 100.0, | |
| "op_correct": 87.5, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "domain": "FILE", | |
| "n": 4, | |
| "json_valid": 100.0, | |
| "op_correct": 100.0, | |
| "exact_match": 75.0 | |
| }, | |
| { | |
| "domain": "HOME", | |
| "n": 7, | |
| "json_valid": 100.0, | |
| "op_correct": 28.6, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "domain": "HTTP", | |
| "n": 25, | |
| "json_valid": 100.0, | |
| "op_correct": 64.0, | |
| "exact_match": 44.0 | |
| }, | |
| { | |
| "domain": "MQTT", | |
| "n": 15, | |
| "json_valid": 100.0, | |
| "op_correct": 100.0, | |
| "exact_match": 6.7 | |
| }, | |
| { | |
| "domain": "ROS", | |
| "n": 46, | |
| "json_valid": 97.8, | |
| "op_correct": 50.0, | |
| "exact_match": 10.9 | |
| }, | |
| { | |
| "domain": "VEHICLE", | |
| "n": 17, | |
| "json_valid": 100.0, | |
| "op_correct": 29.4, | |
| "exact_match": 17.6 | |
| }, | |
| { | |
| "domain": "WORKFLOW", | |
| "n": 49, | |
| "json_valid": 71.4, | |
| "op_correct": 71.4, | |
| "exact_match": 16.3 | |
| } | |
| ] | |
| }, | |
| { | |
| "name": "SmolLM2-360M", | |
| "is_sam": false, | |
| "params": "361.8M", | |
| "params_raw": 361821120, | |
| "size_mb": 690.1, | |
| "json_valid": 89.0, | |
| "op_correct": 0.0, | |
| "exact_match": 0.0, | |
| "latency_ms": 2042.9, | |
| "n_cases": 200, | |
| "per_tier": [ | |
| { | |
| "tier": 1, | |
| "n": 40, | |
| "json_valid": 100.0, | |
| "op_correct": 0.0, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "tier": 2, | |
| "n": 40, | |
| "json_valid": 100.0, | |
| "op_correct": 0.0, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "tier": 3, | |
| "n": 40, | |
| "json_valid": 100.0, | |
| "op_correct": 0.0, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "tier": 4, | |
| "n": 40, | |
| "json_valid": 45.0, | |
| "op_correct": 0.0, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "tier": 5, | |
| "n": 40, | |
| "json_valid": 100.0, | |
| "op_correct": 0.0, | |
| "exact_match": 0.0 | |
| } | |
| ], | |
| "per_domain": [ | |
| { | |
| "domain": "CAL", | |
| "n": 2, | |
| "json_valid": 100.0, | |
| "op_correct": 0.0, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "domain": "DB", | |
| "n": 27, | |
| "json_valid": 100.0, | |
| "op_correct": 0.0, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "domain": "ECOMMERCE", | |
| "n": 8, | |
| "json_valid": 100.0, | |
| "op_correct": 0.0, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "domain": "FILE", | |
| "n": 4, | |
| "json_valid": 100.0, | |
| "op_correct": 0.0, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "domain": "HOME", | |
| "n": 7, | |
| "json_valid": 100.0, | |
| "op_correct": 0.0, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "domain": "HTTP", | |
| "n": 25, | |
| "json_valid": 100.0, | |
| "op_correct": 0.0, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "domain": "MQTT", | |
| "n": 15, | |
| "json_valid": 100.0, | |
| "op_correct": 0.0, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "domain": "ROS", | |
| "n": 46, | |
| "json_valid": 76.1, | |
| "op_correct": 0.0, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "domain": "VEHICLE", | |
| "n": 17, | |
| "json_valid": 35.3, | |
| "op_correct": 0.0, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "domain": "WORKFLOW", | |
| "n": 49, | |
| "json_valid": 100.0, | |
| "op_correct": 0.0, | |
| "exact_match": 0.0 | |
| } | |
| ] | |
| }, | |
| { | |
| "name": "Qwen2.5-0.5B", | |
| "is_sam": false, | |
| "params": "494.0M", | |
| "params_raw": 494032768, | |
| "size_mb": 942.3, | |
| "json_valid": 70.5, | |
| "op_correct": 30.0, | |
| "exact_match": 2.5, | |
| "latency_ms": 1937.2, | |
| "n_cases": 200, | |
| "per_tier": [ | |
| { | |
| "tier": 1, | |
| "n": 40, | |
| "json_valid": 60.0, | |
| "op_correct": 42.5, | |
| "exact_match": 12.5 | |
| }, | |
| { | |
| "tier": 2, | |
| "n": 40, | |
| "json_valid": 65.0, | |
| "op_correct": 35.0, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "tier": 3, | |
| "n": 40, | |
| "json_valid": 67.5, | |
| "op_correct": 45.0, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "tier": 4, | |
| "n": 40, | |
| "json_valid": 95.0, | |
| "op_correct": 27.5, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "tier": 5, | |
| "n": 40, | |
| "json_valid": 65.0, | |
| "op_correct": 0.0, | |
| "exact_match": 0.0 | |
| } | |
| ], | |
| "per_domain": [ | |
| { | |
| "domain": "CAL", | |
| "n": 2, | |
| "json_valid": 100.0, | |
| "op_correct": 50.0, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "domain": "DB", | |
| "n": 27, | |
| "json_valid": 44.4, | |
| "op_correct": 7.4, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "domain": "ECOMMERCE", | |
| "n": 8, | |
| "json_valid": 25.0, | |
| "op_correct": 25.0, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "domain": "FILE", | |
| "n": 4, | |
| "json_valid": 100.0, | |
| "op_correct": 100.0, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "domain": "HOME", | |
| "n": 7, | |
| "json_valid": 100.0, | |
| "op_correct": 42.9, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "domain": "HTTP", | |
| "n": 25, | |
| "json_valid": 72.0, | |
| "op_correct": 36.0, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "domain": "MQTT", | |
| "n": 15, | |
| "json_valid": 86.7, | |
| "op_correct": 26.7, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "domain": "ROS", | |
| "n": 46, | |
| "json_valid": 73.9, | |
| "op_correct": 43.5, | |
| "exact_match": 4.3 | |
| }, | |
| { | |
| "domain": "VEHICLE", | |
| "n": 17, | |
| "json_valid": 82.4, | |
| "op_correct": 82.4, | |
| "exact_match": 17.6 | |
| }, | |
| { | |
| "domain": "WORKFLOW", | |
| "n": 49, | |
| "json_valid": 71.4, | |
| "op_correct": 2.0, | |
| "exact_match": 0.0 | |
| } | |
| ] | |
| }, | |
| { | |
| "name": "TinyLlama-1.1B", | |
| "is_sam": false, | |
| "params": "1100.0M", | |
| "params_raw": 1100048384, | |
| "size_mb": 2098.2, | |
| "json_valid": 92.0, | |
| "op_correct": 55.5, | |
| "exact_match": 9.0, | |
| "latency_ms": 4199.3, | |
| "n_cases": 200, | |
| "per_tier": [ | |
| { | |
| "tier": 1, | |
| "n": 40, | |
| "json_valid": 100.0, | |
| "op_correct": 95.0, | |
| "exact_match": 27.5 | |
| }, | |
| { | |
| "tier": 2, | |
| "n": 40, | |
| "json_valid": 90.0, | |
| "op_correct": 82.5, | |
| "exact_match": 12.5 | |
| }, | |
| { | |
| "tier": 3, | |
| "n": 40, | |
| "json_valid": 100.0, | |
| "op_correct": 77.5, | |
| "exact_match": 5.0 | |
| }, | |
| { | |
| "tier": 4, | |
| "n": 40, | |
| "json_valid": 70.0, | |
| "op_correct": 22.5, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "tier": 5, | |
| "n": 40, | |
| "json_valid": 100.0, | |
| "op_correct": 0.0, | |
| "exact_match": 0.0 | |
| } | |
| ], | |
| "per_domain": [ | |
| { | |
| "domain": "CAL", | |
| "n": 2, | |
| "json_valid": 100.0, | |
| "op_correct": 100.0, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "domain": "DB", | |
| "n": 27, | |
| "json_valid": 100.0, | |
| "op_correct": 66.7, | |
| "exact_match": 18.5 | |
| }, | |
| { | |
| "domain": "ECOMMERCE", | |
| "n": 8, | |
| "json_valid": 100.0, | |
| "op_correct": 100.0, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "domain": "FILE", | |
| "n": 4, | |
| "json_valid": 100.0, | |
| "op_correct": 100.0, | |
| "exact_match": 25.0 | |
| }, | |
| { | |
| "domain": "HOME", | |
| "n": 7, | |
| "json_valid": 100.0, | |
| "op_correct": 100.0, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "domain": "HTTP", | |
| "n": 25, | |
| "json_valid": 84.0, | |
| "op_correct": 80.0, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "domain": "MQTT", | |
| "n": 15, | |
| "json_valid": 100.0, | |
| "op_correct": 100.0, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "domain": "ROS", | |
| "n": 46, | |
| "json_valid": 76.1, | |
| "op_correct": 47.8, | |
| "exact_match": 17.4 | |
| }, | |
| { | |
| "domain": "VEHICLE", | |
| "n": 17, | |
| "json_valid": 94.1, | |
| "op_correct": 35.3, | |
| "exact_match": 23.5 | |
| }, | |
| { | |
| "domain": "WORKFLOW", | |
| "n": 49, | |
| "json_valid": 100.0, | |
| "op_correct": 18.4, | |
| "exact_match": 0.0 | |
| } | |
| ] | |
| }, | |
| { | |
| "name": "Llama-3.2-1B", | |
| "is_sam": false, | |
| "params": "?", | |
| "params_raw": 0, | |
| "size_mb": 0, | |
| "json_valid": 0, | |
| "op_correct": 0, | |
| "exact_match": 0, | |
| "latency_ms": 0, | |
| "error": "load_failed" | |
| }, | |
| { | |
| "name": "Qwen2.5-1.5B", | |
| "is_sam": false, | |
| "params": "1543.7M", | |
| "params_raw": 1543714304, | |
| "size_mb": 2944.4, | |
| "json_valid": 100.0, | |
| "op_correct": 78.5, | |
| "exact_match": 9.0, | |
| "latency_ms": 1852.4, | |
| "n_cases": 200, | |
| "per_tier": [ | |
| { | |
| "tier": 1, | |
| "n": 40, | |
| "json_valid": 100.0, | |
| "op_correct": 95.0, | |
| "exact_match": 30.0 | |
| }, | |
| { | |
| "tier": 2, | |
| "n": 40, | |
| "json_valid": 100.0, | |
| "op_correct": 85.0, | |
| "exact_match": 12.5 | |
| }, | |
| { | |
| "tier": 3, | |
| "n": 40, | |
| "json_valid": 100.0, | |
| "op_correct": 100.0, | |
| "exact_match": 2.5 | |
| }, | |
| { | |
| "tier": 4, | |
| "n": 40, | |
| "json_valid": 100.0, | |
| "op_correct": 72.5, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "tier": 5, | |
| "n": 40, | |
| "json_valid": 100.0, | |
| "op_correct": 40.0, | |
| "exact_match": 0.0 | |
| } | |
| ], | |
| "per_domain": [ | |
| { | |
| "domain": "CAL", | |
| "n": 2, | |
| "json_valid": 100.0, | |
| "op_correct": 100.0, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "domain": "DB", | |
| "n": 27, | |
| "json_valid": 100.0, | |
| "op_correct": 100.0, | |
| "exact_match": 18.5 | |
| }, | |
| { | |
| "domain": "ECOMMERCE", | |
| "n": 8, | |
| "json_valid": 100.0, | |
| "op_correct": 100.0, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "domain": "FILE", | |
| "n": 4, | |
| "json_valid": 100.0, | |
| "op_correct": 100.0, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "domain": "HOME", | |
| "n": 7, | |
| "json_valid": 100.0, | |
| "op_correct": 85.7, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "domain": "HTTP", | |
| "n": 25, | |
| "json_valid": 100.0, | |
| "op_correct": 84.0, | |
| "exact_match": 4.0 | |
| }, | |
| { | |
| "domain": "MQTT", | |
| "n": 15, | |
| "json_valid": 100.0, | |
| "op_correct": 93.3, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "domain": "ROS", | |
| "n": 46, | |
| "json_valid": 100.0, | |
| "op_correct": 71.7, | |
| "exact_match": 19.6 | |
| }, | |
| { | |
| "domain": "VEHICLE", | |
| "n": 17, | |
| "json_valid": 100.0, | |
| "op_correct": 100.0, | |
| "exact_match": 17.6 | |
| }, | |
| { | |
| "domain": "WORKFLOW", | |
| "n": 49, | |
| "json_valid": 100.0, | |
| "op_correct": 51.0, | |
| "exact_match": 0.0 | |
| } | |
| ] | |
| }, | |
| { | |
| "name": "SmolLM2-1.7B", | |
| "is_sam": false, | |
| "params": "1711.4M", | |
| "params_raw": 1711376384, | |
| "size_mb": 3264.2, | |
| "json_valid": 89.0, | |
| "op_correct": 38.5, | |
| "exact_match": 2.5, | |
| "latency_ms": 1694.9, | |
| "n_cases": 200, | |
| "per_tier": [ | |
| { | |
| "tier": 1, | |
| "n": 40, | |
| "json_valid": 92.5, | |
| "op_correct": 75.0, | |
| "exact_match": 12.5 | |
| }, | |
| { | |
| "tier": 2, | |
| "n": 40, | |
| "json_valid": 80.0, | |
| "op_correct": 62.5, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "tier": 3, | |
| "n": 40, | |
| "json_valid": 100.0, | |
| "op_correct": 55.0, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "tier": 4, | |
| "n": 40, | |
| "json_valid": 72.5, | |
| "op_correct": 0.0, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "tier": 5, | |
| "n": 40, | |
| "json_valid": 100.0, | |
| "op_correct": 0.0, | |
| "exact_match": 0.0 | |
| } | |
| ], | |
| "per_domain": [ | |
| { | |
| "domain": "CAL", | |
| "n": 2, | |
| "json_valid": 100.0, | |
| "op_correct": 100.0, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "domain": "DB", | |
| "n": 27, | |
| "json_valid": 66.7, | |
| "op_correct": 59.3, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "domain": "ECOMMERCE", | |
| "n": 8, | |
| "json_valid": 100.0, | |
| "op_correct": 100.0, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "domain": "FILE", | |
| "n": 4, | |
| "json_valid": 100.0, | |
| "op_correct": 100.0, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "domain": "HOME", | |
| "n": 7, | |
| "json_valid": 85.7, | |
| "op_correct": 57.1, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "domain": "HTTP", | |
| "n": 25, | |
| "json_valid": 100.0, | |
| "op_correct": 4.0, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "domain": "MQTT", | |
| "n": 15, | |
| "json_valid": 100.0, | |
| "op_correct": 100.0, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "domain": "ROS", | |
| "n": 46, | |
| "json_valid": 95.7, | |
| "op_correct": 47.8, | |
| "exact_match": 4.3 | |
| }, | |
| { | |
| "domain": "VEHICLE", | |
| "n": 17, | |
| "json_valid": 88.2, | |
| "op_correct": 23.5, | |
| "exact_match": 17.6 | |
| }, | |
| { | |
| "domain": "WORKFLOW", | |
| "n": 49, | |
| "json_valid": 83.7, | |
| "op_correct": 2.0, | |
| "exact_match": 0.0 | |
| } | |
| ] | |
| }, | |
| { | |
| "name": "Gemma-2-2B", | |
| "is_sam": false, | |
| "params": "?", | |
| "params_raw": 0, | |
| "size_mb": 0, | |
| "json_valid": 0, | |
| "op_correct": 0, | |
| "exact_match": 0, | |
| "latency_ms": 0, | |
| "error": "load_failed" | |
| }, | |
| { | |
| "name": "Qwen2.5-3B", | |
| "is_sam": false, | |
| "params": "3085.9M", | |
| "params_raw": 3085938688, | |
| "size_mb": 5886.0, | |
| "json_valid": 100.0, | |
| "op_correct": 82.5, | |
| "exact_match": 16.0, | |
| "latency_ms": 2061.1, | |
| "n_cases": 200, | |
| "per_tier": [ | |
| { | |
| "tier": 1, | |
| "n": 40, | |
| "json_valid": 100.0, | |
| "op_correct": 97.5, | |
| "exact_match": 40.0 | |
| }, | |
| { | |
| "tier": 2, | |
| "n": 40, | |
| "json_valid": 100.0, | |
| "op_correct": 100.0, | |
| "exact_match": 12.5 | |
| }, | |
| { | |
| "tier": 3, | |
| "n": 40, | |
| "json_valid": 100.0, | |
| "op_correct": 100.0, | |
| "exact_match": 27.5 | |
| }, | |
| { | |
| "tier": 4, | |
| "n": 40, | |
| "json_valid": 100.0, | |
| "op_correct": 75.0, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "tier": 5, | |
| "n": 40, | |
| "json_valid": 100.0, | |
| "op_correct": 40.0, | |
| "exact_match": 0.0 | |
| } | |
| ], | |
| "per_domain": [ | |
| { | |
| "domain": "CAL", | |
| "n": 2, | |
| "json_valid": 100.0, | |
| "op_correct": 100.0, | |
| "exact_match": 50.0 | |
| }, | |
| { | |
| "domain": "DB", | |
| "n": 27, | |
| "json_valid": 100.0, | |
| "op_correct": 100.0, | |
| "exact_match": 18.5 | |
| }, | |
| { | |
| "domain": "ECOMMERCE", | |
| "n": 8, | |
| "json_valid": 100.0, | |
| "op_correct": 100.0, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "domain": "FILE", | |
| "n": 4, | |
| "json_valid": 100.0, | |
| "op_correct": 75.0, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "domain": "HOME", | |
| "n": 7, | |
| "json_valid": 100.0, | |
| "op_correct": 100.0, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "domain": "HTTP", | |
| "n": 25, | |
| "json_valid": 100.0, | |
| "op_correct": 100.0, | |
| "exact_match": 8.0 | |
| }, | |
| { | |
| "domain": "MQTT", | |
| "n": 15, | |
| "json_valid": 100.0, | |
| "op_correct": 100.0, | |
| "exact_match": 73.3 | |
| }, | |
| { | |
| "domain": "ROS", | |
| "n": 46, | |
| "json_valid": 100.0, | |
| "op_correct": 78.3, | |
| "exact_match": 21.7 | |
| }, | |
| { | |
| "domain": "VEHICLE", | |
| "n": 17, | |
| "json_valid": 100.0, | |
| "op_correct": 100.0, | |
| "exact_match": 17.6 | |
| }, | |
| { | |
| "domain": "WORKFLOW", | |
| "n": 49, | |
| "json_valid": 100.0, | |
| "op_correct": 51.0, | |
| "exact_match": 0.0 | |
| } | |
| ] | |
| }, | |
| { | |
| "name": "Llama-3.2-3B", | |
| "is_sam": false, | |
| "params": "?", | |
| "params_raw": 0, | |
| "size_mb": 0, | |
| "json_valid": 0, | |
| "op_correct": 0, | |
| "exact_match": 0, | |
| "latency_ms": 0, | |
| "error": "load_failed" | |
| }, | |
| { | |
| "name": "Phi-3.5-mini-3.8B", | |
| "is_sam": false, | |
| "params": "?", | |
| "params_raw": 0, | |
| "size_mb": 0, | |
| "json_valid": 0, | |
| "op_correct": 0, | |
| "exact_match": 0, | |
| "latency_ms": 0, | |
| "error": "load_failed" | |
| }, | |
| { | |
| "name": "Llama-3.2-3B-uns", | |
| "is_sam": false, | |
| "params": "?", | |
| "params_raw": 0, | |
| "size_mb": 0, | |
| "json_valid": 0, | |
| "op_correct": 0, | |
| "exact_match": 0, | |
| "latency_ms": 0, | |
| "error": "load_failed" | |
| }, | |
| { | |
| "name": "Gemma-3-270M-GGUF", | |
| "is_sam": false, | |
| "params": "270.0M", | |
| "params_raw": 270000000, | |
| "size_mb": 241.4, | |
| "json_valid": 92.0, | |
| "op_correct": 2.0, | |
| "exact_match": 0.0, | |
| "latency_ms": 6322.5, | |
| "n_cases": 200, | |
| "per_tier": [ | |
| { | |
| "tier": 1, | |
| "n": 40, | |
| "json_valid": 85.0, | |
| "op_correct": 5.0, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "tier": 2, | |
| "n": 40, | |
| "json_valid": 97.5, | |
| "op_correct": 5.0, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "tier": 3, | |
| "n": 40, | |
| "json_valid": 90.0, | |
| "op_correct": 0.0, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "tier": 4, | |
| "n": 40, | |
| "json_valid": 87.5, | |
| "op_correct": 0.0, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "tier": 5, | |
| "n": 40, | |
| "json_valid": 100.0, | |
| "op_correct": 0.0, | |
| "exact_match": 0.0 | |
| } | |
| ], | |
| "per_domain": [ | |
| { | |
| "domain": "CAL", | |
| "n": 2, | |
| "json_valid": 100.0, | |
| "op_correct": 0.0, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "domain": "DB", | |
| "n": 27, | |
| "json_valid": 88.9, | |
| "op_correct": 0.0, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "domain": "ECOMMERCE", | |
| "n": 8, | |
| "json_valid": 100.0, | |
| "op_correct": 12.5, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "domain": "FILE", | |
| "n": 4, | |
| "json_valid": 100.0, | |
| "op_correct": 0.0, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "domain": "HOME", | |
| "n": 7, | |
| "json_valid": 100.0, | |
| "op_correct": 42.9, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "domain": "HTTP", | |
| "n": 25, | |
| "json_valid": 92.0, | |
| "op_correct": 0.0, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "domain": "MQTT", | |
| "n": 15, | |
| "json_valid": 66.7, | |
| "op_correct": 0.0, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "domain": "ROS", | |
| "n": 46, | |
| "json_valid": 87.0, | |
| "op_correct": 0.0, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "domain": "VEHICLE", | |
| "n": 17, | |
| "json_valid": 100.0, | |
| "op_correct": 0.0, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "domain": "WORKFLOW", | |
| "n": 49, | |
| "json_valid": 100.0, | |
| "op_correct": 0.0, | |
| "exact_match": 0.0 | |
| } | |
| ] | |
| }, | |
| { | |
| "name": "Ministral-3-8B-GGUF", | |
| "is_sam": false, | |
| "params": "?", | |
| "params_raw": 0, | |
| "size_mb": 0, | |
| "json_valid": 0, | |
| "op_correct": 0, | |
| "exact_match": 0, | |
| "latency_ms": 0, | |
| "error": "load_failed" | |
| }, | |
| { | |
| "name": "DeepSeek-R1-Qwen3-8B-GGUF", | |
| "is_sam": false, | |
| "params": "8000.0M", | |
| "params_raw": 8000000000, | |
| "size_mb": 4794.9, | |
| "json_valid": 98.0, | |
| "op_correct": 79.0, | |
| "exact_match": 11.5, | |
| "latency_ms": 31737.9, | |
| "n_cases": 200, | |
| "per_tier": [ | |
| { | |
| "tier": 1, | |
| "n": 40, | |
| "json_valid": 100.0, | |
| "op_correct": 90.0, | |
| "exact_match": 27.5 | |
| }, | |
| { | |
| "tier": 2, | |
| "n": 40, | |
| "json_valid": 95.0, | |
| "op_correct": 95.0, | |
| "exact_match": 7.5 | |
| }, | |
| { | |
| "tier": 3, | |
| "n": 40, | |
| "json_valid": 100.0, | |
| "op_correct": 77.5, | |
| "exact_match": 22.5 | |
| }, | |
| { | |
| "tier": 4, | |
| "n": 40, | |
| "json_valid": 95.0, | |
| "op_correct": 92.5, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "tier": 5, | |
| "n": 40, | |
| "json_valid": 100.0, | |
| "op_correct": 40.0, | |
| "exact_match": 0.0 | |
| } | |
| ], | |
| "per_domain": [ | |
| { | |
| "domain": "CAL", | |
| "n": 2, | |
| "json_valid": 100.0, | |
| "op_correct": 100.0, | |
| "exact_match": 50.0 | |
| }, | |
| { | |
| "domain": "DB", | |
| "n": 27, | |
| "json_valid": 100.0, | |
| "op_correct": 100.0, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "domain": "ECOMMERCE", | |
| "n": 8, | |
| "json_valid": 100.0, | |
| "op_correct": 87.5, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "domain": "FILE", | |
| "n": 4, | |
| "json_valid": 100.0, | |
| "op_correct": 100.0, | |
| "exact_match": 25.0 | |
| }, | |
| { | |
| "domain": "HOME", | |
| "n": 7, | |
| "json_valid": 100.0, | |
| "op_correct": 85.7, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "domain": "HTTP", | |
| "n": 25, | |
| "json_valid": 92.0, | |
| "op_correct": 92.0, | |
| "exact_match": 8.0 | |
| }, | |
| { | |
| "domain": "MQTT", | |
| "n": 15, | |
| "json_valid": 100.0, | |
| "op_correct": 100.0, | |
| "exact_match": 73.3 | |
| }, | |
| { | |
| "domain": "ROS", | |
| "n": 46, | |
| "json_valid": 95.7, | |
| "op_correct": 69.6, | |
| "exact_match": 10.9 | |
| }, | |
| { | |
| "domain": "VEHICLE", | |
| "n": 17, | |
| "json_valid": 100.0, | |
| "op_correct": 100.0, | |
| "exact_match": 17.6 | |
| }, | |
| { | |
| "domain": "WORKFLOW", | |
| "n": 49, | |
| "json_valid": 100.0, | |
| "op_correct": 51.0, | |
| "exact_match": 0.0 | |
| } | |
| ] | |
| } | |
| ], | |
| "per_tier": [ | |
| { | |
| "tier": 1, | |
| "n": 40, | |
| "json_valid": 100.0, | |
| "op_correct": 82.5, | |
| "exact_match": 37.5, | |
| "description": "Atomic \u2014 single op" | |
| }, | |
| { | |
| "tier": 2, | |
| "n": 40, | |
| "json_valid": 100.0, | |
| "op_correct": 70.0, | |
| "exact_match": 27.5, | |
| "description": "Compound \u2014 2-3 ops same domain" | |
| }, | |
| { | |
| "tier": 3, | |
| "n": 40, | |
| "json_valid": 100.0, | |
| "op_correct": 100.0, | |
| "exact_match": 25.0, | |
| "description": "Noisy \u2014 typos / casing / fillers" | |
| }, | |
| { | |
| "tier": 4, | |
| "n": 40, | |
| "json_valid": 75.0, | |
| "op_correct": 0.0, | |
| "exact_match": 0.0, | |
| "description": "Long chain \u2014 4+ ops" | |
| }, | |
| { | |
| "tier": 5, | |
| "n": 40, | |
| "json_valid": 65.0, | |
| "op_correct": 65.0, | |
| "exact_match": 0.0, | |
| "description": "Cross-domain workflow" | |
| } | |
| ], | |
| "per_domain": [ | |
| { | |
| "domain": "CAL", | |
| "n": 2, | |
| "json_valid": 100.0, | |
| "op_correct": 100.0, | |
| "exact_match": 100.0 | |
| }, | |
| { | |
| "domain": "DB", | |
| "n": 27, | |
| "json_valid": 66.7, | |
| "op_correct": 66.7, | |
| "exact_match": 11.1 | |
| }, | |
| { | |
| "domain": "ECOMMERCE", | |
| "n": 8, | |
| "json_valid": 100.0, | |
| "op_correct": 87.5, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "domain": "FILE", | |
| "n": 4, | |
| "json_valid": 100.0, | |
| "op_correct": 100.0, | |
| "exact_match": 75.0 | |
| }, | |
| { | |
| "domain": "HOME", | |
| "n": 7, | |
| "json_valid": 100.0, | |
| "op_correct": 28.6, | |
| "exact_match": 0.0 | |
| }, | |
| { | |
| "domain": "HTTP", | |
| "n": 25, | |
| "json_valid": 100.0, | |
| "op_correct": 64.0, | |
| "exact_match": 44.0 | |
| }, | |
| { | |
| "domain": "MQTT", | |
| "n": 15, | |
| "json_valid": 100.0, | |
| "op_correct": 100.0, | |
| "exact_match": 6.7 | |
| }, | |
| { | |
| "domain": "ROS", | |
| "n": 46, | |
| "json_valid": 97.8, | |
| "op_correct": 50.0, | |
| "exact_match": 10.9 | |
| }, | |
| { | |
| "domain": "VEHICLE", | |
| "n": 17, | |
| "json_valid": 100.0, | |
| "op_correct": 29.4, | |
| "exact_match": 17.6 | |
| }, | |
| { | |
| "domain": "WORKFLOW", | |
| "n": 49, | |
| "json_valid": 71.4, | |
| "op_correct": 71.4, | |
| "exact_match": 16.3 | |
| } | |
| ], | |
| "tier_distribution": { | |
| "1": 40, | |
| "2": 40, | |
| "3": 40, | |
| "4": 40, | |
| "5": 40 | |
| }, | |
| "domain_distribution": { | |
| "CAL": 2, | |
| "DB": 27, | |
| "ECOMMERCE": 8, | |
| "FILE": 4, | |
| "HOME": 7, | |
| "HTTP": 25, | |
| "MQTT": 15, | |
| "ROS": 46, | |
| "VEHICLE": 17, | |
| "WORKFLOW": 49 | |
| } | |
| } |