| { |
| "model": "Qwen/Qwen2.5-Coder-14B-Instruct", |
| "adapter": "cmndcntrlcyber/qwen14b-code-trainer-v7_mixed", |
| "eval_type": "nexus-aligned", |
| "scenarios": 5, |
| "progressing": 5, |
| "progress_rate": 1.0, |
| "target_threshold": 3, |
| "meets_target": true, |
| "tools_tested": [ |
| "Read", |
| "Write", |
| "Edit", |
| "LS", |
| "Bash", |
| "Grep", |
| "Glob" |
| ], |
| "results": [ |
| { |
| "scenario": "read_and_analyze", |
| "description": "Read a file and explain what it does", |
| "making_progress": true, |
| "used_tools": true, |
| "gave_explanation": true, |
| "not_refusing": true, |
| "not_looping": true, |
| "reason": "used tools; gave explanation", |
| "n_turns": 5 |
| }, |
| { |
| "scenario": "find_and_fix_bug", |
| "description": "Find a bug in code and propose a fix", |
| "making_progress": true, |
| "used_tools": true, |
| "gave_explanation": true, |
| "not_refusing": true, |
| "not_looping": true, |
| "reason": "used tools; gave explanation", |
| "n_turns": 5 |
| }, |
| { |
| "scenario": "multi_file_investigation", |
| "description": "Investigate across multiple files", |
| "making_progress": true, |
| "used_tools": true, |
| "gave_explanation": true, |
| "not_refusing": true, |
| "not_looping": true, |
| "reason": "used tools; gave explanation", |
| "n_turns": 5 |
| }, |
| { |
| "scenario": "write_new_code", |
| "description": "Write a new function based on requirements", |
| "making_progress": true, |
| "used_tools": true, |
| "gave_explanation": true, |
| "not_refusing": true, |
| "not_looping": true, |
| "reason": "used tools; gave explanation", |
| "n_turns": 3 |
| }, |
| { |
| "scenario": "run_and_debug", |
| "description": "Run a command and interpret the output", |
| "making_progress": true, |
| "used_tools": true, |
| "gave_explanation": true, |
| "not_refusing": true, |
| "not_looping": true, |
| "reason": "used tools; gave explanation", |
| "n_turns": 5 |
| } |
| ] |
| } |
