eval_tuned.json

{
  "evaluated_at": "2026-07-02T13:26:41+00:00",
  "base_model": "mlx-community/Qwen2.5-0.5B-Instruct-4bit",
  "task": "ratchet-loop-demo",
  "adapter": "runs/ratchet-loop-demo/adapters",
  "test_set": "runs/ratchet-loop-demo/data/test.jsonl",
  "decoding": {
    "temperature": 0.0,
    "max_tokens": 180,
    "chat_template_kwargs": {}
  },
  "metrics": {
    "examples": 50,
    "exact_match": 1.0,
    "json_examples": 50,
    "json_validity": 1.0,
    "field_accuracy": 1.0,
    "mean_latency_s": 0.315
  },
  "samples": [
    {
      "expected": "{\"diagnosis\": \"The proposed step is too broad for a single ratchet iteration.\", \"target_files\": [\"ratchet_driver.py\", \"README.md\"], \"patch_strategy\": \"Reject the broad patch and request a narrower change touching only files tied to the failing signal.\", \"verification_command\": \"git diff --check && make test\", \"acceptance_signal\": \"manual_review\"}",
      "generated": "{\"diagnosis\": \"The proposed step is too broad for a single ratchet iteration.\", \"target_files\": [\"ratchet_driver.py\", \"README.md\"], \"patch_strategy\": \"Reject the broad patch and request a narrower change touching only files tied to the failing signal.\", \"verification_command\": \"git diff --check && make test\", \"acceptance_signal\": \"manual_review\"}"
    },
    {
      "expected": "{\"diagnosis\": \"The JSON extractor parses only raw objects and misses fenced responses.\", \"target_files\": [\"finetune/pipeline/metrics.py\", \"finetune/tests/test_metrics.py\"], \"patch_strategy\": \"Strip an optional json code fence before locating the first balanced JSON object.\", \"verification_command\": \"make test\", \"acceptance_signal\": \"tests_pass\"}",
      "generated": "{\"diagnosis\": \"The JSON extractor parses only raw objects and misses fenced responses.\", \"target_files\": [\"finetune/pipeline/metrics.py\", \"finetune/tests/test_metrics.py\"], \"patch_strategy\": \"Strip an optional json code fence before locating the first balanced JSON object.\", \"verification_command\": \"make test\", \"acceptance_signal\": \"tests_pass\"}"
    },
    {
      "expected": "{\"diagnosis\": \"The generator uses global randomness or unordered data while building examples.\", \"target_files\": [\"finetune/tasks/example/generate_data.py\"], \"patch_strategy\": \"Use a local random.Random(seed) instance and sort any template collections before sampling.\", \"verification_command\": \"make test\", \"acceptance_signal\": \"tests_pass\"}",
      "generated": "{\"diagnosis\": \"The generator uses global randomness or unordered data while building examples.\", \"target_files\": [\"finetune/tasks/example/generate_data.py\"], \"patch_strategy\": \"Use a local random.Random(seed) instance and sort any template collections before sampling.\", \"verification_command\": \"make test\", \"acceptance_signal\": \"tests_pass\"}"
    },
    {
      "expected": "{\"diagnosis\": \"The JSON extractor parses only raw objects and misses fenced responses.\", \"target_files\": [\"finetune/pipeline/metrics.py\", \"finetune/tests/test_metrics.py\"], \"patch_strategy\": \"Strip an optional json code fence before locating the first balanced JSON object.\", \"verification_command\": \"make test\", \"acceptance_signal\": \"tests_pass\"}",
      "generated": "{\"diagnosis\": \"The JSON extractor parses only raw objects and misses fenced responses.\", \"target_files\": [\"finetune/pipeline/metrics.py\", \"finetune/tests/test_metrics.py\"], \"patch_strategy\": \"Strip an optional json code fence before locating the first balanced JSON object.\", \"verification_command\": \"make test\", \"acceptance_signal\": \"tests_pass\"}"
    },
    {
      "expected": "{\"diagnosis\": \"Documentation still references a previous task path after a rename.\", \"target_files\": [\"README.md\", \"RUNNING.md\", \"finetune/README.md\"], \"patch_strategy\": \"Replace stale task references in docs and keep the commands copy-pasteable.\", \"verification_command\": \"git diff --check && make test\", \"acceptance_signal\": \"lint_clean\"}",
      "generated": "{\"diagnosis\": \"Documentation still references a previous task path after a rename.\", \"target_files\": [\"README.md\", \"RUNNING.md\", \"finetune/README.md\"], \"patch_strategy\": \"Replace stale task references in docs and keep the commands copy-pasteable.\", \"verification_command\": \"git diff --check && make test\", \"acceptance_signal\": \"lint_clean\"}"
    }
  ]
}