File size: 1,044 Bytes
1244914 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 | run:
# Clone into `tmp/task` dir
- git clone --depth=1 --branch main https://github.com/tailcallhq/forgecode .
- FORGE_DEBUG_REQUESTS='{{dir}}/context.json' forgee --provider open_router --model {{model}} -p '{{task}}'
parallelism: 50
timeout: 120
early_exit: true
validations:
- name: "Should use patch tool (not git apply)"
type: shell
command: "jq -e '[.messages[]?.tool_calls[]? | select(.function.name == \"patch\")] | length > 0' {{dir}}/context.json"
- name: "Should NOT use shell with git apply"
type: shell
command: "jq -e '[.messages[]?.tool_calls[]? | select(.function.name == \"shell\" and (.function.arguments | contains(\"git apply\") or contains(\"git patch\")))] | length == 0' {{dir}}/context.json"
- name: "Should call patch tool multiple times for multi-file changes"
type: shell
command: "jq -e '[.messages[]?.tool_calls[]? | select(.function.name == \"patch\")] | length >= 3' {{dir}}/context.json"
sources:
- value:
- model: "openai/gpt-5.2"
- csv: multi_file_patch_tasks.csv
|