benchmarks/evals/create_skill/task.yml

# No before_run needed - binary should be pre-built
run: FORGE_DEBUG_REQUESTS='{{dir}}/context.json' forgee -p '{{skill}}'
parallelism: 10
timeout: 60
early_exit: true
validations:
  - name: "Uses create-skill tool"
    type: shell
    command: "jq -e '[.messages[]?.tool_calls[]? | select(.function.name == \"skill\") | .function.arguments | fromjson | .name == \"create-skill\"] | any' {{dir}}/context.json"
sources:
  - csv: create_skill_tasks.csv