"""
Component 8: Local chat interface using Gradio.
- Clean dark-themed UI.
- Prompt input box.
- Syntax-highlighted code output (Python + JavaScript).
- Copy button for each code response.
- Generation time + token count.
- Conversation history in session.
- Clear button to reset history.
- Live model selector: Base / LoRA / INT8 (no restart).
"""
from __future__ import annotations
import html
import re
import time
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple
import gradio as gr
import torch
import torch.nn as nn
import yaml
from pygments import highlight
from pygments.formatters import HtmlFormatter
from pygments.lexers import JavascriptLexer, PythonLexer, TextLexer
from src.finetuning_system.lora_adapter import LoRAConfig, apply_lora, load_lora_state_dict
from src.inference_engine.inference_engine import DecodingConfig, InferenceEngine
from src.model_architecture.code_transformer import CodeTransformerLM, ModelConfig, get_model_presets
from src.tokenizer.code_tokenizer import CodeTokenizer
def _load_yaml(path: Path) -> Dict[str, Any]:
if not path.exists():
raise FileNotFoundError(f"Config file not found: {path}")
data = yaml.safe_load(path.read_text(encoding="utf-8-sig"))
if not isinstance(data, dict):
raise ValueError("Invalid YAML format.")
return data
def _build_model_config(path: Path) -> ModelConfig:
cfg = _load_yaml(path)
preset = cfg.get("preset")
model_cfg = cfg.get("model", {})
if preset:
presets = get_model_presets()
if preset not in presets:
raise ValueError(f"Unknown preset: {preset}")
merged = presets[preset].__dict__.copy()
merged.update(model_cfg)
return ModelConfig(**merged)
return ModelConfig(**model_cfg)
def _guess_language(prompt: str, default_lang: str = "python") -> str:
p = prompt.lower()
if "javascript" in p or " js " in f" {p} " or "node" in p:
return "javascript"
if "python" in p:
return "python"
return default_lang
def _is_coding_prompt(prompt: str) -> bool:
p = prompt.lower().strip()
coding_keywords = [
"code",
"python",
"javascript",
"function",
"bug",
"error",
"algorithm",
"sort",
"loop",
"class",
"api",
"sql",
"regex",
"debug",
"implement",
"write",
]
if any(k in p for k in coding_keywords):
return True
if re.fullmatch(r"(hi|hello|hey|yo|hola)[!. ]*", p):
return False
return False
def _highlight_code(code: str, language: str) -> str:
code = code or ""
if language == "javascript":
lexer = JavascriptLexer()
elif language == "python":
lexer = PythonLexer()
else:
lexer = TextLexer()
formatter = HtmlFormatter(nowrap=True)
return highlight(code, lexer, formatter)
def _render_history(history: List[Dict[str, Any]]) -> str:
formatter = HtmlFormatter(style="monokai")
css = formatter.get_style_defs(".codehilite")
blocks = [
"",
"""
""",
'
',
]
if not history:
blocks.append('
No messages yet. Ask a coding question to begin.
')
for i, item in enumerate(history, start=1):
lang = item.get("language", "python")
prompt = html.escape(str(item.get("prompt", "")))
highlighted = _highlight_code(str(item.get("code", "")), lang)
code_id = f"code-{i}"
syntax_ok = "yes" if item.get("syntax_ok", False) else "n/a"
mode = item.get("mode", "base")
blocks.append('
')
blocks.append(f'
User: {prompt}
')
blocks.append(f'
Assistant ({lang})
')
blocks.append(f'
')
blocks.append('
')
blocks.append('
')
blocks.append(f'
{highlighted}')
blocks.append('
')
blocks.append(
f'
mode={mode} | time={item.get("time_sec", 0):.2f}s | '
f'tokens={item.get("tokens", 0)} | syntax_ok={syntax_ok} | '
f'attempt={item.get("attempt", 1)}
'
)
blocks.append('
')
blocks.append('
')
return "\n".join(blocks)
class ChatRuntime:
def __init__(self, config_path: str) -> None:
self.project_root = Path(__file__).resolve().parents[2]
self.cfg = _load_yaml(self.project_root / config_path)
self.model_cfg = _build_model_config(self.project_root / self.cfg["model"]["model_config_path"])
self.cuda_device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
if self.cuda_device.type != "cuda":
raise RuntimeError("CUDA GPU is required for this chat interface setup.")
self.tokenizer = CodeTokenizer.load(str(self.project_root / self.cfg["model"]["tokenizer_dir"]))
self.decode_cfg = DecodingConfig(
max_new_tokens=int(self.cfg["inference"].get("max_new_tokens", 300)),
greedy_temperature=float(self.cfg["inference"].get("greedy_temperature", 0.0)),
retry2_temperature=float(self.cfg["inference"].get("retry2_temperature", 0.25)),
retry2_top_p=float(self.cfg["inference"].get("retry2_top_p", 0.85)),
retry3_temperature=float(self.cfg["inference"].get("retry3_temperature", 0.35)),
retry3_top_p=float(self.cfg["inference"].get("retry3_top_p", 0.90)),
max_retries=int(self.cfg["inference"].get("max_retries", 3)),
min_tokens_before_stop_check=int(self.cfg["inference"].get("min_tokens_before_stop_check", 64)),
)
self.current_mode: Optional[str] = None
self.engine: Optional[InferenceEngine] = None
def _release_current(self) -> None:
self.engine = None
self.current_mode = None
if torch.cuda.is_available():
torch.cuda.empty_cache()
def _current_vram_gb(self) -> float:
if not torch.cuda.is_available():
return 0.0
return float(torch.cuda.memory_allocated() / (1024**3))
def _status_text(self, mode: str, load_sec: float) -> str:
return f"MINDI 1.0 420M | mode={mode} | load={load_sec:.2f}s | vram={self._current_vram_gb():.2f}GB"
def _load_base_model(self) -> InferenceEngine:
model = CodeTransformerLM(self.model_cfg).to(self.cuda_device)
payload = torch.load(self.project_root / self.cfg["model"]["base_checkpoint_path"], map_location=self.cuda_device)
model.load_state_dict(payload["model_state"])
model.half()
return InferenceEngine(model=model, tokenizer=self.tokenizer, device=self.cuda_device)
def _load_lora_model(self) -> InferenceEngine:
model = CodeTransformerLM(self.model_cfg).to(self.cuda_device)
payload = torch.load(self.project_root / self.cfg["model"]["base_checkpoint_path"], map_location=self.cuda_device)
model.load_state_dict(payload["model_state"])
lora_cfg = LoRAConfig(
r=int(self.cfg.get("lora", {}).get("r", 8)),
alpha=int(self.cfg.get("lora", {}).get("alpha", 16)),
dropout=float(self.cfg.get("lora", {}).get("dropout", 0.05)),
target_keywords=list(self.cfg.get("lora", {}).get("target_keywords", ["q_proj", "k_proj", "v_proj", "o_proj", "fc1", "fc2"])),
)
apply_lora(model, lora_cfg)
model = model.to(self.cuda_device)
lora_payload = torch.load(self.project_root / self.cfg["model"]["lora_adapter_path"], map_location=self.cuda_device)
lora_state = lora_payload.get("lora_state", lora_payload)
load_lora_state_dict(model, lora_state)
model.half()
return InferenceEngine(model=model, tokenizer=self.tokenizer, device=self.cuda_device)
def _load_int8_model(self) -> InferenceEngine:
cpu = torch.device("cpu")
model = CodeTransformerLM(self.model_cfg).to(cpu).float()
model = torch.quantization.quantize_dynamic(model, {nn.Linear}, dtype=torch.qint8)
q_state = torch.load(self.project_root / self.cfg["model"]["quantized_state_path"], map_location=cpu)
model.load_state_dict(q_state)
return InferenceEngine(model=model, tokenizer=self.tokenizer, device=cpu)
def _ensure_mode(self, mode: str) -> str:
mode = (mode or "base").lower().strip()
if mode not in {"base", "lora", "int8"}:
mode = "base"
if self.current_mode == mode and self.engine is not None:
return self._status_text(mode, load_sec=0.0)
t0 = time.perf_counter()
self._release_current()
if mode == "base":
self.engine = self._load_base_model()
elif mode == "lora":
self.engine = self._load_lora_model()
else:
self.engine = self._load_int8_model()
self.current_mode = mode
load_sec = time.perf_counter() - t0
return self._status_text(mode, load_sec=load_sec)
def switch_mode(self, mode: str) -> str:
return self._ensure_mode(mode)
def respond(self, prompt: str, history: List[Dict[str, Any]], mode: str) -> Tuple[str, List[Dict[str, Any]], str, str]:
prompt = (prompt or "").strip()
if not prompt:
status = self._ensure_mode(mode)
return _render_history(history), history, "", status
status = self._ensure_mode(mode)
if not _is_coding_prompt(prompt):
fallback = "Please ask a coding question (for example: 'Write a Python function to ...' or 'Fix this JavaScript bug ...')."
history.append(
{
"prompt": prompt,
"code": fallback,
"language": "text",
"tokens": 0,
"time_sec": 0.0,
"syntax_ok": None,
"attempt": 0,
"mode": self.current_mode or "base",
}
)
return _render_history(history), history, "", status
lang_default = str(self.cfg["inference"].get("language_default", "python"))
language = _guess_language(prompt, default_lang=lang_default)
start = time.perf_counter()
result = self.engine.generate_with_retry(prompt=prompt, language=language, cfg=self.decode_cfg) # type: ignore[union-attr]
elapsed = time.perf_counter() - start
final = result["final"]
history.append(
{
"prompt": prompt,
"code": final["code"],
"language": language,
"tokens": int(final.get("generated_tokens", 0)),
"time_sec": float(elapsed),
"syntax_ok": bool(final.get("syntax_ok", False)) if language == "python" else None,
"attempt": int(final.get("attempt", 1)),
"mode": self.current_mode or "base",
}
)
return _render_history(history), history, "", status
def clear(self, mode: str) -> Tuple[str, List[Dict[str, Any]], str, str]:
history: List[Dict[str, Any]] = []
status = self._ensure_mode(mode)
return _render_history(history), history, "", status
def create_demo(config_path: str = "configs/component8_chat_config.yaml") -> gr.Blocks:
runtime = ChatRuntime(config_path=config_path)
with gr.Blocks(title="MINDI 1.0 420M", theme=gr.themes.Base()) as demo:
gr.Markdown("## MINDI 1.0 420M\nYour local coding intelligence — 420M parameters, fully offline")
history_state = gr.State([])
chat_html = gr.HTML(value=_render_history([]))
with gr.Row():
mode_dropdown = gr.Dropdown(
label="Model Mode",
choices=["base", "lora", "int8"],
value="base",
interactive=True,
)
status_box = gr.Textbox(label="Status", value="MINDI 1.0 420M | mode=base | load=0.00s | vram=0.00GB", interactive=False)
prompt_box = gr.Textbox(
label="Your Prompt",
lines=4,
placeholder="Ask MINDI anything about code",
)
with gr.Row():
send_btn = gr.Button("Generate", variant="primary")
clear_btn = gr.Button("Clear Conversation")
switch_btn = gr.Button("Apply Mode")
switch_btn.click(fn=runtime.switch_mode, inputs=[mode_dropdown], outputs=[status_box])
send_btn.click(
fn=runtime.respond,
inputs=[prompt_box, history_state, mode_dropdown],
outputs=[chat_html, history_state, prompt_box, status_box],
queue=True,
)
prompt_box.submit(
fn=runtime.respond,
inputs=[prompt_box, history_state, mode_dropdown],
outputs=[chat_html, history_state, prompt_box, status_box],
queue=True,
)
clear_btn.click(
fn=runtime.clear,
inputs=[mode_dropdown],
outputs=[chat_html, history_state, prompt_box, status_box],
)
return demo