🧠 disable thinking mode: empty <think> block + faster eval with eos_token_id

notebooks/qwen35-2b-bash-finetune.ipynb

@@ -188,35 +188,7 @@
    "execution_count": null,
    "metadata": {},
    "outputs": [],
-   "source": [
-    "FastLanguageModel.for_inference(model)\n",
-    "\n",
-    "test_cases = [\n",
-    "    \"git space push space dash u space origin space main\",\n",
-    "    \"find dot dash name star dot txt\",\n",
-    "    \"docker space run space dash dash rm space dash p space eight zero eight zero colon eight zero space nginx\",\n",
-    "    \"chmod space zero seven five five space slash usr slash local slash bin slash deploy dot sh\",\n",
-    "    \"ssh space dash i space tilde slash dot ssh slash id underscore rsa space root at one nine two dot one six eight dot one dot one\",\n",
-    "]\n",
-    "\n",
-    "for dictated in test_cases:\n",
-    "    messages = [\n",
-    "        {\"role\": \"system\", \"content\": \"Reconstruct the intended syntax from the dictated text. Output only the result.\"},\n",
-    "        {\"role\": \"user\", \"content\": dictated},\n",
-    "    ]\n",
-    "    inputs = tokenizer.apply_chat_template(\n",
-    "        messages, tokenize=True, add_generation_prompt=True, return_tensors=\"pt\"\n",
-    "    ).to(model.device)\n",
-    "\n",
-    "    outputs = model.generate(\n",
-    "        input_ids=inputs, max_new_tokens=128, temperature=0.0,\n",
-    "        do_sample=False, use_cache=True,\n",
-    "    )\n",
-    "    response = tokenizer.decode(outputs[0][inputs.shape[-1]:], skip_special_tokens=True)\n",
-    "    print(f\"IN:  {dictated}\")\n",
-    "    print(f\"OUT: {response}\")\n",
-    "    print()"
-   ]
+   "source": "FastLanguageModel.for_inference(model)\n\ntext_tokenizer = tokenizer.tokenizer if hasattr(tokenizer, 'tokenizer') else tokenizer\neos_id = text_tokenizer.convert_tokens_to_ids(\"<|im_end|>\")\n\ntest_cases = [\n    \"git space push space dash u space origin space main\",\n    \"find dot dash name star dot txt\",\n    \"docker space run space dash dash rm space dash p space eight zero eight zero colon eight zero space nginx\",\n    \"chmod space zero seven five five space slash usr slash local slash bin slash deploy dot sh\",\n    \"ssh space dash i space tilde slash dot ssh slash id underscore rsa space root at one nine two dot one six eight dot one dot one\",\n]\n\nfor dictated in test_cases:\n    prompt = \"<|im_start|>system\\nReconstruct the intended syntax from the dictated text. Output only the result.<|im_end|>\\n\"\n    prompt += f\"<|im_start|>user\\n{dictated}<|im_end|>\\n\"\n    prompt += \"<|im_start|>assistant\\n<think>\\n</think>\\n\"\n\n    inputs = text_tokenizer(prompt, return_tensors=\"pt\").to(model.device)\n\n    outputs = model.generate(\n        **inputs, max_new_tokens=64, temperature=0.0,\n        do_sample=False, use_cache=True, eos_token_id=eos_id,\n    )\n    response = text_tokenizer.decode(outputs[0][inputs[\"input_ids\"].shape[-1]:], skip_special_tokens=True)\n    print(f\"IN:  {dictated}\")\n    print(f\"OUT: {response}\")\n    print()"
   },
   {
    "cell_type": "markdown",
@@ -230,77 +202,7 @@
    "execution_count": null,
    "metadata": {},
    "outputs": [],
-   "source": [
-    "from difflib import SequenceMatcher\n",
-    "import time\n",
-    "\n",
-    "test_dataset = load_dataset(\n",
-    "    \"json\",\n",
-    "    data_files=\"./data/training/data/bash-v2/minimal/test.jsonl\",\n",
-    "    split=\"train\",\n",
-    ")\n",
-    "\n",
-    "results = {\"exact\": 0, \"near\": 0, \"partial\": 0, \"wrong\": 0}\n",
-    "total_time = 0\n",
-    "errors = []\n",
-    "\n",
-    "for i, example in enumerate(test_dataset):\n",
-    "    msgs = example[\"messages\"]\n",
-    "    dictated = msgs[1][\"content\"]\n",
-    "    expected = msgs[2][\"content\"]\n",
-    "\n",
-    "    messages = [\n",
-    "        {\"role\": \"system\", \"content\": msgs[0][\"content\"]},\n",
-    "        {\"role\": \"user\", \"content\": dictated},\n",
-    "    ]\n",
-    "    inputs = tokenizer.apply_chat_template(\n",
-    "        messages, tokenize=True, add_generation_prompt=True, return_tensors=\"pt\"\n",
-    "    ).to(model.device)\n",
-    "\n",
-    "    t0 = time.time()\n",
-    "    outputs = model.generate(\n",
-    "        input_ids=inputs, max_new_tokens=256, temperature=0.0,\n",
-    "        do_sample=False, use_cache=True,\n",
-    "    )\n",
-    "    t1 = time.time()\n",
-    "    total_time += (t1 - t0)\n",
-    "\n",
-    "    response = tokenizer.decode(outputs[0][inputs.shape[-1]:], skip_special_tokens=True).strip()\n",
-    "    similarity = SequenceMatcher(None, response, expected).ratio()\n",
-    "\n",
-    "    if response == expected:\n",
-    "        results[\"exact\"] += 1\n",
-    "    elif similarity >= 0.9:\n",
-    "        results[\"near\"] += 1\n",
-    "    elif similarity >= 0.7:\n",
-    "        results[\"partial\"] += 1\n",
-    "    else:\n",
-    "        results[\"wrong\"] += 1\n",
-    "        errors.append((dictated[:60], expected[:60], response[:60]))\n",
-    "\n",
-    "    if (i + 1) % 50 == 0:\n",
-    "        print(f\"  {i+1}/{len(test_dataset)}...\")\n",
-    "\n",
-    "n = len(test_dataset)\n",
-    "print(f\"\\n{'='*50}\")\n",
-    "print(f\"QWEN3.5-2B EVALUATION ({n} cases)\")\n",
-    "print(f\"{'='*50}\")\n",
-    "print(f\"Exact match:    {results['exact']:4d} / {n} ({results['exact']/n*100:.1f}%)\")\n",
-    "print(f\"Near (>90%):    {results['near']:4d} / {n} ({results['near']/n*100:.1f}%)\")\n",
-    "print(f\"Partial (70-90): {results['partial']:3d} / {n} ({results['partial']/n*100:.1f}%)\")\n",
-    "print(f\"Wrong (<70%):    {results['wrong']:3d} / {n} ({results['wrong']/n*100:.1f}%)\")\n",
-    "print(f\"\\nEffective accuracy: {(results['exact']+results['near'])/n*100:.1f}%\")\n",
-    "print(f\"Avg inference time: {total_time/n:.2f}s\")\n",
-    "print(f\"Peak memory: {torch.cuda.max_memory_allocated() / 1e9:.2f} GB\")\n",
-    "\n",
-    "if errors:\n",
-    "    print(f\"\\nSample errors ({len(errors)} total):\")\n",
-    "    for d, e, r in errors[:5]:\n",
-    "        print(f\"  IN:  {d}\")\n",
-    "        print(f\"  EXP: {e}\")\n",
-    "        print(f\"  GOT: {r}\")\n",
-    "        print()"
-   ]
+   "source": "from difflib import SequenceMatcher\nimport time\n\ntext_tokenizer = tokenizer.tokenizer if hasattr(tokenizer, 'tokenizer') else tokenizer\neos_id = text_tokenizer.convert_tokens_to_ids(\"<|im_end|>\")\n\ntest_dataset = load_dataset(\n    \"json\",\n    data_files=\"./data/training/data/bash-v2/minimal/test.jsonl\",\n    split=\"train\",\n)\n\nresults = {\"exact\": 0, \"near\": 0, \"partial\": 0, \"wrong\": 0}\ntotal_time = 0\nerrors = []\n\nfor i, example in enumerate(test_dataset):\n    msgs = example[\"messages\"]\n    dictated = msgs[1][\"content\"]\n    expected = msgs[2][\"content\"]\n\n    prompt = f\"<|im_start|>{msgs[0]['role']}\\n{msgs[0]['content']}<|im_end|>\\n\"\n    prompt += f\"<|im_start|>user\\n{dictated}<|im_end|>\\n\"\n    prompt += \"<|im_start|>assistant\\n<think>\\n</think>\\n\"\n\n    inputs = text_tokenizer(prompt, return_tensors=\"pt\").to(model.device)\n\n    t0 = time.time()\n    outputs = model.generate(\n        **inputs, max_new_tokens=64, temperature=0.0,\n        do_sample=False, use_cache=True, eos_token_id=eos_id,\n    )\n    t1 = time.time()\n    total_time += (t1 - t0)\n\n    response = text_tokenizer.decode(outputs[0][inputs[\"input_ids\"].shape[-1]:], skip_special_tokens=True).strip()\n    similarity = SequenceMatcher(None, response, expected).ratio()\n\n    if response == expected:\n        results[\"exact\"] += 1\n    elif similarity >= 0.9:\n        results[\"near\"] += 1\n    elif similarity >= 0.7:\n        results[\"partial\"] += 1\n    else:\n        results[\"wrong\"] += 1\n        errors.append((dictated[:60], expected[:60], response[:60]))\n\n    if (i + 1) % 10 == 0:\n        elapsed = total_time / (i + 1)\n        print(f\"  {i+1}/{len(test_dataset)} | avg {elapsed:.1f}s/ex | exact so far: {results['exact']}/{i+1}\")\n\nn = len(test_dataset)\nprint(f\"\\n{'='*50}\")\nprint(f\"QWEN3.5-2B EVALUATION ({n} cases)\")\nprint(f\"{'='*50}\")\nprint(f\"Exact match:    {results['exact']:4d} / {n} ({results['exact']/n*100:.1f}%)\")\nprint(f\"Near (>90%):    {results['near']:4d} / {n} ({results['near']/n*100:.1f}%)\")\nprint(f\"Partial (70-90): {results['partial']:3d} / {n} ({results['partial']/n*100:.1f}%)\")\nprint(f\"Wrong (<70%):    {results['wrong']:3d} / {n} ({results['wrong']/n*100:.1f}%)\")\nprint(f\"\\nEffective accuracy: {(results['exact']+results['near'])/n*100:.1f}%\")\nprint(f\"Avg inference time: {total_time/n:.2f}s\")\nprint(f\"Peak memory: {torch.cuda.max_memory_allocated() / 1e9:.2f} GB\")\n\nif errors:\n    print(f\"\\nSample errors ({len(errors)} total):\")\n    for d, e, r in errors[:5]:\n        print(f\"  IN:  {d}\")\n        print(f\"  EXP: {e}\")\n        print(f\"  GOT: {r}\")\n        print()"
   },
   {
    "cell_type": "markdown",