File size: 698 Bytes
c981f27 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 | # Evaluation for checking appropriate todo_write tool usage
run:
- git clone --depth=1 --branch main https://github.com/tailcallhq/forgecode .
- FORGE_OVERRIDE_PROVIDER=open_router FORGE_OVERRIDE_MODEL={{model}} FORGE_DEBUG_REQUESTS='{{dir}}/context.json' forgee -p '{{task}}'
parallelism: 3
timeout: 240
early_exit: true
validations:
- name: "Uses todo_write tool for multi-step tasks"
type: shell
command: |
# Check if todo_write was called at least once
jq -e '[.messages[]?.tool_calls[]? | select(.function?.name == "todo_write")] | length > 0' {{dir}}/context.json
sources:
- value:
- model: "anthropic/claude-sonnet-4.5"
- csv: todo_write_usage_tasks.csv
|