Update code/codellama_7b_instruct_gguf_q4_k_m.ipynb

code/codellama_7b_instruct_gguf_q4_k_m.ipynb

@@ -1,153 +1,264 @@
-#
-# 🚀 CodeLlama 7B Instruct (GGUF Q4_K_M) — Colab Notebook
-# This notebook runs TheBloke/CodeLlama-7B-Instruct-GGUF in Google Colab
-# using llama-cpp-python with automatic GPU offloading.
-#
-
-# Cell 1: Check environment
-#@title 🔧 Check environment
-!nvidia-smi || echo "No NVIDIA GPU detected (CPU mode will be used)"
-!python --version
-
-# Cell 2: Install dependencies
-#@title ⬇️ Install dependencies (GPU wheel if possible; fallback to CPU)
-import sys, subprocess
-
-def pip_install(args):
-    print("pip install", " ".join(args))
-    return subprocess.call([sys.executable, "-m", "pip", "install", "-qU"] + args)
-
-cuda_spec = "cu121"
-gpu_index = f"https://abetlen.github.io/llama-cpp-python/whl/{cuda_spec}"
-# Try GPU wheel first
-rc = pip_install([f"--extra-index-url={gpu_index}", "llama-cpp-python>=0.2.90", "huggingface_hub>=0.23.0", "ipywidgets"])
-if rc != 0:
-    print("⚠️ GPU wheel failed, trying CPU wheel...")
-    rc2 = pip_install(["llama-cpp-python>=0.2.90", "huggingface_hub>=0.23.0", "ipywidgets"])
-    if rc2 != 0:
-        raise RuntimeError("Failed to install llama-cpp-python")
-print("✅ Installation complete")
-
-# Cell 3: (Optional) Hugging Face login
-#@title 🔐 (Optional) Hugging Face login
-from google.colab import userdata
-from huggingface_hub import login
-
-# Use Colab secrets to store your HF token
-try:
-    HF_TOKEN = userdata.get('HF_TOKEN')
-    login(token=HF_TOKEN, add_to_git_credential=True)
-    print("✅ Logged in to Hugging Face")
-except userdata.SecretNotFoundError:
-    print("Skipping login (HF_TOKEN secret not found)")
-except Exception as e:
-    print(f"Login failed: {e}")
-
-
-# Cell 4: Download model (GGUF) from Hugging Face
-#@title 📦 Download model (GGUF) from Hugging Face
-from huggingface_hub import hf_hub_download
-
-REPO_ID = "TheBloke/CodeLlama-7B-Instruct-GGUF"
-FILENAME = "codellama-7b-instruct.Q4_K_M.gguf"
-
-print(f"Downloading {FILENAME} from {REPO_ID}...")
-model_path = hf_hub_download(
-    repo_id=REPO_ID,
-    filename=FILENAME,
-    local_dir="models",
-    local_dir_use_symlinks=False
-)
-print("✅ Downloaded:", model_path)
-
-
-# Cell 5: Load model with llama.cpp (auto GPU offload)
-#@title ⚙️ Load model with llama.cpp (auto GPU offload)
-from llama_cpp import Llama
-
-def try_load(n_gpu_layers):
-    print(f"Trying to load model with n_gpu_layers={n_gpu_layers} ...")
-    return Llama(
-        model_path=model_path,
-        n_ctx=4096,
-        n_threads=None, # Auto-detect
-        n_gpu_layers=n_gpu_layers,  # -1 = all layers on GPU (if possible)
-        verbose=False,
-    )
-
-llm = None
-# Attempt to load with max GPU layers, then fewer, then CPU only
-for attempt in (-1, 40, 0):
-    try:
-        llm = try_load(attempt)
-        print(f"✅ Model loaded successfully with n_gpu_layers = {attempt}")
-        break
-    except Exception as e:
-        print(f"Load failed with {attempt} GPU layers: {e}")
-
-if llm is None:
-    raise RuntimeError("Could not load the model. Ensure you have enough RAM/VRAM.")
-
-
-# Cell 6: Prompt builder
-#@title 🧩 Prompt builder (code-first templates)
-from textwrap import dedent
-
-def build_prompt(user_query, system="You are an expert software engineer. Output concise, correct code. If possible, return code only."):
-    return dedent(f"""
-        <|system|>
-        {system}
-        <|user|>
-        {user_query}
-        <|assistant|>
-    """).strip()
-
-# Example of a built prompt
-print("--- Example Prompt ---")
-print(build_prompt("Write a Python function `is_prime(n)`."))
-print("----------------------")
-
-
-# Cell 7: Interactive code chat UI
-#@title 💬 Interactive code chat (UI)
-from google.colab import output
-output.enable_custom_widget_manager() # Enable widgets in Colab
-
-import ipywidgets as widgets
-from IPython.display import display, Markdown
-
-sys_area = widgets.Textarea(
-    value="You are an expert programmer. Prefer minimal, correct code. If possible, output only code.",
-    description="System",
-    layout=widgets.Layout(width="100%", height="80px")
-)
-user_area = widgets.Textarea(
-    value="Write a Python function to parse a CSV file and compute the average of a column named 'score'.",
-    description="Prompt",
-    layout=widgets.Layout(width="100%", height="100px")
-)
-temp = widgets.FloatSlider(value=0.2, min=0.0, max=1.2, step=0.05, description="Temperature")
-maxtok = widgets.IntSlider(value=512, min=64, max=2048, step=32, description="Max tokens")
-code_only_box = widgets.Checkbox(value=True, description="Code only")
-run_btn = widgets.Button(description="Generate", button_style="success")
-out_area = widgets.Output()
-
-def on_run(_):
-    out_area.clear_output()
-    with out_area:
-        print("Generating response...")
-        prompt = build_prompt(user_area.value, system=sys_area.value)
-        stops = ["<|user|>", "<|system|>", "</s>", "```"] if code_only_box.value else ["<|user|>", "<|system|>", "</s>"]
-        
-        result = llm(prompt, max_tokens=maxtok.value, temperature=temp.value, stop=stops)
-        text = result["choices"][0]["text"]
-        
-        # Post-processing to ensure it's a clean code block
-        if code_only_box.value and "```" not in text:
-            text = "```python\n" + text.strip() + "\n```"
-        
-        out_area.clear_output() # Clear "Generating..." message
-        display(Markdown(text))
-
-run_btn.on_click(on_run)
-display(widgets.VBox([sys_area, user_area, temp, maxtok, code_only_box, run_btn, out_area]))
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "d5c1cbb7",
+   "metadata": {},
+   "source": [
+    "# 🚀 CodeLlama 7B Instruct (GGUF Q4_K_M) — Colab (GGUF via llama.cpp)\n",
+    "\n",
+    "**One-click notebook** to run `TheBloke/CodeLlama-7B-Instruct-GGUF` (`codellama-7b-instruct.Q4_K_M.gguf`) in Google Colab using **llama-cpp-python**.\n",
+    "\n",
+    "**Features**\n",
+    "- Hugging Face login (optional for gated repos)\n",
+    "- Automatic GPU offload (T4/A100) with CPU fallback\n",
+    "- Download GGUF to Colab temp disk (no Drive required)\n",
+    "- Prompt templates optimized for **code generation**\n",
+    "- Interactive chat UI (code-focused)\n",
+    "- Optional local API server\n",
+    "\n",
+    "Best for general coding tasks (Python/JS/C++).\n",
+    "\n",
+    "> Tip: In Colab use **Runtime → Change runtime type → GPU (T4)** for speed.\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5b152f1d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#@title 🔧 Check environment\n",
+    "!nvidia-smi || echo \"No NVIDIA GPU detected (CPU mode will be used)\"\n",
+    "!python --version"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5285c04d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#@title ⬇️ Install dependencies (GPU wheel if possible; fallback to CPU)\n",
+    "import sys, subprocess\n",
+    "\n",
+    "def pip_install(args):\n",
+    "    print(\"pip install\", \" \".join(args))\n",
+    "    return subprocess.call([sys.executable, \"-m\", \"pip\", \"install\", \"-qU\"] + args)\n",
+    "\n",
+    "cuda_spec = \"cu121\"\n",
+    "gpu_index = f\"https://abetlen.github.io/llama-cpp-python/whl/{cuda_spec}\"\n",
+    "# Try GPU wheel first\n",
+    "rc = pip_install([f\"--extra-index-url={gpu_index}\", \"llama-cpp-python>=0.2.90\", \"huggingface_hub>=0.23.0\",\n",
+    "                  \"ipywidgets\", \"pydantic<3\", \"uvicorn\", \"fastapi\"])\n",
+    "if rc != 0:\n",
+    "    print(\"⚠️ GPU wheel failed, trying CPU wheel...\")\n",
+    "    rc2 = pip_install([\"llama-cpp-python>=0.2.90\", \"huggingface_hub>=0.23.0\",\n",
+    "                       \"ipywidgets\", \"pydantic<3\", \"uvicorn\", \"fastapi\"])\n",
+    "    if rc2 != 0:\n",
+    "        raise RuntimeError(\"Failed to install llama-cpp-python\")\n",
+    "print(\"✅ Installation complete\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "80157423",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#@title 🔐 (Optional) Hugging Face login\n",
+    "HF_TOKEN = \"\"  #@param {type:\"string\"}\n",
+    "from huggingface_hub import login\n",
+    "if HF_TOKEN.strip():\n",
+    "    login(token=HF_TOKEN.strip(), add_to_git_credential=True)\n",
+    "    print(\"Logged in to Hugging Face\")\n",
+    "else:\n",
+    "    print(\"Skipping login (no token provided)\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "82dd88aa",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#@title 📦 Download model (GGUF) from Hugging Face\n",
+    "from huggingface_hub import hf_hub_download\n",
+    "\n",
+    "REPO_ID = \"TheBloke/CodeLlama-7B-Instruct-GGUF\"  #@param [\"TheBloke/CodeLlama-7B-Instruct-GGUF\"] {allow-input: true}\n",
+    "FILENAME = \"codellama-7b-instruct.Q4_K_M.gguf\"  #@param [\"codellama-7b-instruct.Q4_K_M.gguf\"] {allow-input: true}\n",
+    "\n",
+    "model_path = hf_hub_download(\n",
+    "    repo_id=REPO_ID,\n",
+    "    filename=FILENAME,\n",
+    "    local_dir=\"models\",\n",
+    "    local_dir_use_symlinks=False\n",
+    ")\n",
+    "print(\"✅ Downloaded:\", model_path)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7862b7f1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#@title ⚙️ Load model with llama.cpp (auto GPU offload)\n",
+    "from llama_cpp import Llama\n",
+    "\n",
+    "def try_load(n_gpu_layers):\n",
+    "    print(f\"Trying n_gpu_layers={n_gpu_layers} ...\")\n",
+    "    return Llama(\n",
+    "        model_path=model_path,\n",
+    "        n_ctx=4096,\n",
+    "        n_threads=None,\n",
+    "        n_gpu_layers=n_gpu_layers,  # -1 = all layers on GPU (if possible)\n",
+    "        logits_all=False,\n",
+    "        verbose=False,\n",
+    "    )\n",
+    "\n",
+    "llm = None\n",
+    "for attempt in (-1, 40, 20, 0):\n",
+    "    try:\n",
+    "        llm = try_load(attempt)\n",
+    "        print(\"✅ Loaded with n_gpu_layers =\", attempt)\n",
+    "        break\n",
+    "    except Exception as e:\n",
+    "        print(\"Load failed:\", e)\n",
+    "\n",
+    "if llm is None:\n",
+    "    raise RuntimeError(\"Could not load the model. Try a smaller quant or reduce context.\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ab41cbde",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#@title 🧩 Prompt builder (code-first templates)\n",
+    "from textwrap import dedent\n",
+    "\n",
+    "def build_prompt(user_query, system=\"You are an expert software engineer. Output concise, correct code. If possible, return code only.\"):\n",
+    "    instruct = dedent(f\"\"\"\n",
+    "    <|system|>\n",
+    "    {system}\n",
+    "    <|user|>\n",
+    "    {user_query}\n",
+    "    <|assistant|>\n",
+    "    \"\"\").strip()\n",
+    "    return instruct\n",
+    "\n",
+    "print(build_prompt(\"Write a Python function `is_prime(n)`.\"))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c71af4c4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#@title 🧪 Generate (single turn)\n",
+    "user_request = \"Write a Python function `two_sum(nums, target)` returning indices.\"  #@param {type:\"string\"}\n",
+    "max_tokens = 512  #@param {type:\"slider\", min:64, max:2048, step:32}\n",
+    "temperature = 0.2  #@param {type:\"number\"}\n",
+    "code_only = True  #@param {type:\"boolean\"}\n",
+    "\n",
+    "sys_prompt = \"You are an expert programmer. Prefer minimal, correct code. If possible, output only code.\"\n",
+    "prompt = build_prompt(user_request, system=sys_prompt)\n",
+    "\n",
+    "stops = [\"<|user|>\", \"<|system|>\", \"</s>\", \"```\"] if code_only else [\"<|user|>\", \"<|system|>\", \"</s>\"]\n",
+    "out = llm(prompt, max_tokens=max_tokens, temperature=temperature, stop=stops)\n",
+    "text = out[\"choices\"][0][\"text\"]\n",
+    "\n",
+    "if code_only and \"```\" not in text:\n",
+    "    text = \"```python\\n\" + text.strip() + \"\\n```\"\n",
+    "\n",
+    "print(text)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2701cdb8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#@title 💬 Interactive code chat (UI)\n",
+    "import ipywidgets as widgets\n",
+    "from IPython.display import display, Markdown\n",
+    "\n",
+    "sys_area = widgets.Textarea(\n",
+    "    value=\"You are an expert programmer. Prefer minimal, correct code. If possible, output only code.\",\n",
+    "    description=\"System\",\n",
+    "    layout=widgets.Layout(width=\"100%\", height=\"80px\")\n",
+    ")\n",
+    "user_area = widgets.Textarea(\n",
+    "    value=\"Write a Python function to parse a CSV file and compute average of a column named 'score'.\",\n",
+    "    description=\"Prompt\",\n",
+    "    layout=widgets.Layout(width=\"100%\", height=\"100px\")\n",
+    ")\n",
+    "temp = widgets.FloatSlider(value=0.2, min=0.0, max=1.2, step=0.05, description=\"Temperature\")\n",
+    "maxtok = widgets.IntSlider(value=512, min=64, max=2048, step=32, description=\"Max tokens\")\n",
+    "code_only_box = widgets.Checkbox(value=True, description=\"Code only\")\n",
+    "run_btn = widgets.Button(description=\"Generate\", button_style=\"success\")\n",
+    "out_area = widgets.Output()\n",
+    "\n",
+    "def on_run(_):\n",
+    "    out_area.clear_output()\n",
+    "    with out_area:\n",
+    "        prompt = build_prompt(user_area.value, system=sys_area.value)\n",
+    "        stops = [\"<|user|>\", \"<|system|>\", \"</s>\", \"```\"] if code_only_box.value else [\"<|user|>\", \"<|system|>\", \"</s>\"]\n",
+    "        result = llm(prompt, max_tokens=maxtok.value, temperature=temp.value, stop=stops)\n",
+    "        text = result[\"choices\"][0][\"text\"]\n",
+    "        if code_only_box.value and \"```\" not in text:\n",
+    "            text = \"```python\\n\" + text.strip() + \"\\n```\"\n",
+    "        display(Markdown(text))\n",
+    "\n",
+    "run_btn.on_click(on_run)\n",
+    "display(widgets.VBox([sys_area, user_area, temp, maxtok, code_only_box, run_btn, out_area]))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "37a7a7f9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#@title 🌐 Optional: start local API server (OpenAI-like)\n",
+    "# After running, open http://127.0.0.1:8000/docs inside Colab to test.\n",
+    "import threading\n",
+    "from llama_cpp.server.app import create_app\n",
+    "from fastapi.middleware.cors import CORSMiddleware\n",
+    "import uvicorn\n",
+    "\n",
+    "app = create_app(llm)\n",
+    "app.add_middleware(\n",
+    "    CORSMiddleware,\n",
+    "    allow_origins=[\"*\"],\n",
+    "    allow_credentials=True,\n",
+    "    allow_methods=[\"*\"],\n",
+    "    allow_headers=[\"*\"],\n",
+    ")\n",
+    "\n",
+    "def run_server():\n",
+    "    uvicorn.run(app, host=\"0.0.0.0\", port=8000, log_level=\"info\")\n",
+    "\n",
+    "thread = threading.Thread(target=run_server, daemon=True)\n",
+    "thread.start()\n",
+    "print(\"Server starting on http://127.0.0.1:8000\")"
+   ]
+  }
+ ],
+ "metadata": {},
+ "nbformat": 4,
+ "nbformat_minor": 5
+}