"""Construit le petit jeu de données embarqué à partir du dump public (12 MB). Source : Large Language Monkeys (Brown et al. 2024), agent moatless-tools + DeepSeek-Coder-V2-Instruct, ~250 échantillons / tâche sur SWE-bench Lite. On ne garde que (task_id, n, c) par tâche -> quelques Ko, versionnés dans data/. Usage (une seule fois) : uv run build_data.py """ from __future__ import annotations import json import os import urllib.request URL = ("https://raw.githubusercontent.com/ScalingIntelligence/" "swe-bench-lite-samples/main/summary.json") OUT = os.path.join(os.path.dirname(__file__), "data", "swebench_lite_samples.json") def main() -> None: print(f"Téléchargement {URL} ...") with urllib.request.urlopen(URL) as r: data = json.load(r) tasks = [] for group, flaky in [("instances_without_flaky_tests", False), ("instances_with_flaky_tests", True)]: for task_id, samples in data.get(group, {}).items(): n = len(samples) c = sum(1 for s in samples.values() if s.get("resolved")) tasks.append({"task_id": task_id, "n": n, "c": c, "flaky": flaky}) tasks.sort(key=lambda t: t["task_id"]) out = { "benchmark": "SWE-bench Lite", "model": "DeepSeek-Coder-V2-Instruct (agent : moatless-tools)", "paper": "Large Language Monkeys (Brown et al. 2024), arXiv:2407.21787", "source": URL, "n_tasks": len(tasks), "tasks": tasks, } os.makedirs(os.path.dirname(OUT), exist_ok=True) with open(OUT, "w") as f: json.dump(out, f, separators=(",", ":")) ns = [t["n"] for t in tasks] cs = [t["c"] for t in tasks] print(f"{len(tasks)} tâches -> {OUT}") print(f"n : min={min(ns)} max={max(ns)} " f"tâches résolues au moins 1 fois : {sum(c > 0 for c in cs)}/{len(cs)} " f"tâches avec p>0.5 : {sum(c / n > 0.5 for c, n in zip(cs, ns))}/{len(cs)}") if __name__ == "__main__": main()