name: SWE-bench Pro
description: >
  SWE-bench Pro is a benchmark for evaluating AI models on real-world software
  engineering tasks. It contains 100 problems from various open-source repositories
  across multiple programming languages (Go, Python, JavaScript, TypeScript).
  Each problem includes a repository, base commit, patch, test patch, and problem
  statement.
evaluation_framework: inspect-ai

tasks:
  - id: swe-bench-pro
    config: default
    split: test

    field_spec:
      input: problem_statement
      target: patch
      choices:
        - repo
        - instance_id
        - base_commit
        - test_patch
        - requirements
        - interface
        - repo_language
        - fail_to_pass
        - pass_to_pass
        - issue_specificity
        - issue_categories
        - before_repo_set_cmd
        - selected_test_files_to_run
        - dockerhub_tag

    solvers:
      - name: system_message
        args:
          template: |
            You are an expert software engineer. Given a problem statement and repository context,
            generate the correct patch to fix the issue.
            
            Consider:
            1. The repository structure and language
            2. The base commit and changes needed
            3. Test requirements and validation
            
      - name: generate

    scorers:
      - name: model_graded_fact
        args:
          model: openai/o3-mini