build_data.py

"""Construit le petit jeu de données embarqué à partir du dump public (12 MB).

Source : Large Language Monkeys (Brown et al. 2024), agent moatless-tools +
DeepSeek-Coder-V2-Instruct, ~250 échantillons / tâche sur SWE-bench Lite.
On ne garde que (task_id, n, c) par tâche -> quelques Ko, versionnés dans data/.

Usage (une seule fois) :  uv run build_data.py
"""

from __future__ import annotations

import json
import os
import urllib.request

URL = ("https://raw.githubusercontent.com/ScalingIntelligence/"
       "swe-bench-lite-samples/main/summary.json")
OUT = os.path.join(os.path.dirname(__file__), "data", "swebench_lite_samples.json")


def main() -> None:
    print(f"Téléchargement {URL} ...")
    with urllib.request.urlopen(URL) as r:
        data = json.load(r)

    tasks = []
    for group, flaky in [("instances_without_flaky_tests", False),
                         ("instances_with_flaky_tests", True)]:
        for task_id, samples in data.get(group, {}).items():
            n = len(samples)
            c = sum(1 for s in samples.values() if s.get("resolved"))
            tasks.append({"task_id": task_id, "n": n, "c": c, "flaky": flaky})
    tasks.sort(key=lambda t: t["task_id"])

    out = {
        "benchmark": "SWE-bench Lite",
        "model": "DeepSeek-Coder-V2-Instruct (agent : moatless-tools)",
        "paper": "Large Language Monkeys (Brown et al. 2024), arXiv:2407.21787",
        "source": URL,
        "n_tasks": len(tasks),
        "tasks": tasks,
    }
    os.makedirs(os.path.dirname(OUT), exist_ok=True)
    with open(OUT, "w") as f:
        json.dump(out, f, separators=(",", ":"))

    ns = [t["n"] for t in tasks]
    cs = [t["c"] for t in tasks]
    print(f"{len(tasks)} tâches  ->  {OUT}")
    print(f"n : min={min(ns)} max={max(ns)}   "
          f"tâches résolues au moins 1 fois : {sum(c > 0 for c in cs)}/{len(cs)}   "
          f"tâches avec p>0.5 : {sum(c / n > 0.5 for c, n in zip(cs, ns))}/{len(cs)}")


if __name__ == "__main__":
    main()