eval_base.json

{
  "evaluated_at": "2026-07-02T13:26:24+00:00",
  "base_model": "mlx-community/Qwen2.5-0.5B-Instruct-4bit",
  "task": "ratchet-loop-demo",
  "adapter": null,
  "test_set": "runs/ratchet-loop-demo/data/test.jsonl",
  "decoding": {
    "temperature": 0.0,
    "max_tokens": 180,
    "chat_template_kwargs": {}
  },
  "metrics": {
    "examples": 50,
    "exact_match": 0.0,
    "json_examples": 50,
    "json_validity": 0.92,
    "field_accuracy": 0.112,
    "mean_latency_s": 0.277
  },
  "samples": [
    {
      "expected": "{\"diagnosis\": \"The proposed step is too broad for a single ratchet iteration.\", \"target_files\": [\"ratchet_driver.py\", \"README.md\"], \"patch_strategy\": \"Reject the broad patch and request a narrower change touching only files tied to the failing signal.\", \"verification_command\": \"git diff --check && make test\", \"acceptance_signal\": \"manual_review\"}",
      "generated": "```json\n{\n  \"diagnosis\": \"Manual review finds the patch changes unrelated files\",\n  \"target_files\": [\"ratchet_driver.py\", \"README.md\"],\n  \"patch_strategy\": \"Keep the patch under two files\",\n  \"verification_command\": \"git status\",\n  \"acceptance_signal\": \"Manual review finds the patch changes unrelated files\"\n}\n```"
    },
    {
      "expected": "{\"diagnosis\": \"The JSON extractor parses only raw objects and misses fenced responses.\", \"target_files\": [\"finetune/pipeline/metrics.py\", \"finetune/tests/test_metrics.py\"], \"patch_strategy\": \"Strip an optional json code fence before locating the first balanced JSON object.\", \"verification_command\": \"make test\", \"acceptance_signal\": \"tests_pass\"}",
      "generated": "```json\n{\n  \"diagnosis\": \"json_field_score returns zero when the model wraps valid JSON in a fenced block\",\n  \"target_files\": [\"finetune/pipeline/metrics.py\", \"finetune/tests/test_metrics.py\"],\n  \"patch_strategy\": \"replace json_field_score with a simple check for valid JSON\",\n  \"verification_command\": \"check if the model's output is valid JSON in a fenced block\",\n  \"acceptance_signal\": \"if json_field_score returns zero, replace it with a simple check\"\n}\n```"
    },
    {
      "expected": "{\"diagnosis\": \"The generator uses global randomness or unordered data while building examples.\", \"target_files\": [\"finetune/tasks/example/generate_data.py\"], \"patch_strategy\": \"Use a local random.Random(seed) instance and sort any template collections before sampling.\", \"verification_command\": \"make test\", \"acceptance_signal\": \"tests_pass\"}",
      "generated": "```json\n{\n  \"diagnosis\": \"The CLI task runner is not deterministic. This is a failing signal.\",\n  \"target_files\": [\"finetune/tasks/example/generate_data.py\"],\n  \"patch_strategy\": \"Upgrade the code generation logic to ensure deterministic behavior.\",\n  \"verification_command\": \"Run the code generation logic with the new version of `generate_data.py`.\",\n  \"acceptance_signal\": \"Ensure the CLI task runner is deterministic by testing the code generation logic.\"\n}\n```"
    },
    {
      "expected": "{\"diagnosis\": \"The JSON extractor parses only raw objects and misses fenced responses.\", \"target_files\": [\"finetune/pipeline/metrics.py\", \"finetune/tests/test_metrics.py\"], \"patch_strategy\": \"Strip an optional json code fence before locating the first balanced JSON object.\", \"verification_command\": \"make test\", \"acceptance_signal\": \"tests_pass\"}",
      "generated": "```json\n{\n  \"diagnosis\": \"json_field_score returns zero when the model wraps valid JSON in a fenced block\",\n  \"target_files\": [\"finetune/pipeline/metrics.py\", \"finetune/tests/test_metrics.py\"],\n  \"patch_strategy\": \"use a single patch file\",\n  \"verification_command\": \"check if the patch is small and verifiable\",\n  \"acceptance_signal\": \"if the patch is small and verifiable, the evaluation should tolerate fenced JSON replies.\"\n}\n```"
    },
    {
      "expected": "{\"diagnosis\": \"Documentation still references a previous task path after a rename.\", \"target_files\": [\"README.md\", \"RUNNING.md\", \"finetune/README.md\"], \"patch_strategy\": \"Replace stale task references in docs and keep the commands copy-pasteable.\", \"verification_command\": \"git diff --check && make test\", \"acceptance_signal\": \"lint_clean\"}",
      "generated": "```json\n{\n  \"diagnosis\": \"The old task name is not found in the current task name file.\",\n  \"target_files\": [\"README.md\", \"RUNNING.md\", \"finetune/README.md\"],\n  \"patch_strategy\": \"Replace the old task name with the current task name file.\",\n  \"verification_command\": \"Compare the old task name with the current task name file.\",\n  \"acceptance_signal\": \"Ensure the old task name is not present in the current task name file.\"\n}\n```"
    }
  ]
}