benchmarks/evals/multi_file_patch/task.yml

run:
  # Clone into `tmp/task` dir
  - git clone --depth=1 --branch main https://github.com/tailcallhq/forgecode .
  - FORGE_DEBUG_REQUESTS='{{dir}}/context.json' forgee --provider open_router --model {{model}} -p '{{task}}'
parallelism: 50
timeout: 120
early_exit: true
validations:
  - name: "Should use patch tool (not git apply)"
    type: shell
    command: "jq -e '[.messages[]?.tool_calls[]? | select(.function.name == \"patch\")] | length > 0' {{dir}}/context.json"
  - name: "Should NOT use shell with git apply"
    type: shell
    command: "jq -e '[.messages[]?.tool_calls[]? | select(.function.name == \"shell\" and (.function.arguments | contains(\"git apply\") or contains(\"git patch\")))] | length == 0' {{dir}}/context.json"
  - name: "Should call patch tool multiple times for multi-file changes"
    type: shell
    command: "jq -e '[.messages[]?.tool_calls[]? | select(.function.name == \"patch\")] | length >= 3' {{dir}}/context.json"
sources:
  - value:
      - model: "openai/gpt-5.2"
  - csv: multi_file_patch_tasks.csv