# filename: okemo_backend.py # OkemoLLM backend — Streaming responses + DuckDuckGo + Increased Web Retention import os import re import math import unicodedata import requests from typing import Dict, List, Tuple, Optional, Any, Iterator import torch import torch.nn as nn import torch.nn.functional as F import torch.optim as optim import gradio as gr os.environ["PYTORCH_ENABLE_MPS_FALLBACK"] = "1" # --- SYSTEM INSTRUCTION MODIFIED FOR CHATGPT-LIKE BEHAVIOR --- SYSTEM_INSTRUCTION = ( "You are Okemo Language Model (OLM), a highly **helpful, verbose, and comprehensive** AI assistant, similar to ChatGPT. " "Your goal is to provide detailed and well-structured answers in a conversational tone, ensuring **clarity and fluency**." "For complex queries, first generate an internal reasoning chain (DO NOT show it to the user). " "If external context is provided, naturally integrate that knowledge into your answer." ) # ------------------------------------------------------------------------- CHECKPOINT_PATH = "checkpoint.pth" DEFAULTS = { "d_model": 768, "n_head": 48, "d_ff": 768, "n_layers": 12, "dropout": 0.1, "max_length": 1024, "k_sampling": 200, # Using 200 for diversity "LEARNING_RATE": 1e-6, } WEB_TAG_RE = re.compile(r"<\s*web\s*>", flags=re.IGNORECASE) TRAIN_INPUT_STATE = {} # Initial value for the Gradio State # ------------------------ # Helpers # ------------------------ def to_ascii(text: Optional[str]) -> str: return unicodedata.normalize("NFKC", text or "") _TOKEN_RE = re.compile(r"\w+|[^\w\s]", flags=re.UNICODE) def tokenize(text: str) -> List[str]: return _TOKEN_RE.findall(to_ascii(text).lower()) def is_punctuation_token(token: str) -> bool: """Checks if a token is a punctuation mark that typically follows a word without a space.""" # Added open parenthesis and quotation marks as tokens that may start a new space-separated unit # Added common closing punctuation that should NOT have a preceding space return token in [".", ",", "!", "?", ":", ";", ")", "]", "}", "'s", "'", "\""] # ------------------------ # Vocabulary # ------------------------ class Voc: def __init__(self) -> None: self.PAD, self.SOS, self.EOS, self.UNK, self.SYSTEM = "", "", "", "", "" self.word2index = {self.PAD: 0, self.SOS: 1, self.EOS: 2, self.UNK: 3, self.SYSTEM: 4} self.index2word = {i: w for w, i in self.word2index.items()} self.n_words = len(self.word2index) @staticmethod def load_from_checkpoint(ckpt: Dict[str, Any]) -> "Voc": voc = Voc() if "voc_word2index" in ckpt: voc.word2index = ckpt["voc_word2index"] voc.index2word = ckpt["voc_index2word"] voc.n_words = len(voc.word2index) return voc def add_word(self, word: str) -> None: if word not in self.word2index: self.word2index[word] = self.n_words self.index2word[self.n_words] = word self.n_words += 1 # ------------------------ # Model # ------------------------ class PositionalEncoding(nn.Module): def __init__(self, d_model: int, max_len: int = 5000): super().__init__() pe = torch.zeros(max_len, d_model) position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1) div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model)) pe[:, 0::2] = torch.sin(position * div_term) pe[:, 1::2] = torch.cos(position * div_term) self.register_buffer("pe", pe.unsqueeze(0)) def forward(self, x: torch.Tensor) -> torch.Tensor: return x + self.pe[:, : x.size(1), :] class GPT(nn.Module): def __init__(self, n_words, d_model, n_layers, n_head, d_ff, dropout=0.1): super().__init__() self.embedding = nn.Embedding(n_words, d_model) self.pos = PositionalEncoding(d_model) self.norm = nn.LayerNorm(d_model) layer = nn.TransformerEncoderLayer(d_model, n_head, d_ff, dropout, batch_first=True) self.transformer = nn.TransformerEncoder(layer, n_layers) self.fc = nn.Linear(d_model, n_words, bias=False) self.fc.weight = self.embedding.weight def forward(self, x, mask=None, key_padding_mask=None): x = self.embedding(x) x = self.pos(x) x = self.norm(x) out = self.transformer(x, mask=mask, src_key_padding_mask=key_padding_mask) return self.fc(out) # ------------------------ # Device + Checkpoint + Resizing Fix # ------------------------ device = torch.device("cpu") def resize_model_if_needed(model: nn.Module, new_n_words: int, d_model: int) -> bool: """ Checks if the model's vocabulary size needs to be updated and resizes the layers. Returns True if resize occurred, False otherwise. """ if new_n_words > model.embedding.num_embeddings: print(f"Resizing model from {model.embedding.num_embeddings} to {new_n_words} vocabulary size...") # 1. Save old embedding and fc weights old_embedding_weight = model.embedding.weight.data.clone() # 2. Create new embedding layer new_embedding = nn.Embedding(new_n_words, d_model) new_embedding.to(model.embedding.weight.device) # ensure device consistency # 3. Copy old weights to new layer copy_len = min(new_n_words, old_embedding_weight.size(0)) new_embedding.weight.data[:copy_len] = old_embedding_weight[:copy_len] # 4. Replace old layers model.embedding = new_embedding model.fc = nn.Linear(d_model, new_n_words, bias=False) # 5. Re-tie weights model.fc.weight = model.embedding.weight return True return False def save_checkpoint(path: str, hp: Dict[str, Any], voc: "Voc", model: nn.Module, optimizer: optim.Optimizer, iteration: int = 0) -> None: """Saves the current model and vocabulary state.""" print(f"Saving checkpoint to {path} at iteration {iteration}...") current_hp = {k: hp[k] for k in hp} current_hp["n_words"] = voc.n_words torch.save( { **current_hp, "iteration": iteration, "gpt_model_state_dict": model.state_dict(), "optimizer_state_dict": optimizer.state_dict(), "voc_word2index": voc.word2index, "voc_index2word": voc.index2word, "voc_n_words": voc.n_words, # Added for robustness }, path, ) def load_checkpoint_or_defaults(path: str): global device # Attempt to use MPS/CUDA if available if torch.backends.mps.is_available(): device = torch.device("mps") elif torch.cuda.is_available(): device = torch.device("cuda") else: device = torch.device("cpu") print(f"Using device: {device}") if os.path.exists(path): ckpt = torch.load(path, map_location=device) hp = {k: ckpt.get(k, v) for k, v in DEFAULTS.items()} voc = Voc.load_from_checkpoint(ckpt) n_words = ckpt.get("n_words", voc.n_words) # Use n_words from checkpoint if available model = GPT(n_words, hp["d_model"], hp["n_layers"], hp["n_head"], hp["d_ff"], hp["dropout"]).to(device) model.load_state_dict(ckpt["gpt_model_state_dict"], strict=False) optimizer = optim.Adam(model.parameters(), lr=hp["LEARNING_RATE"]) if "optimizer_state_dict" in ckpt: optimizer.load_state_dict(ckpt["optimizer_state_dict"]) return ckpt, hp, voc, model, optimizer else: voc = Voc() model = GPT(voc.n_words, **{k: v for k, v in DEFAULTS.items() if k in ["d_model", "n_layers", "n_head", "d_ff", "dropout"]}).to(device) optimizer = optim.Adam(model.parameters(), lr=DEFAULTS["LEARNING_RATE"]) return {}, DEFAULTS, voc, model, optimizer checkpoint, HP, voc, gpt_model, optimizer = load_checkpoint_or_defaults(CHECKPOINT_PATH) start_iteration = checkpoint.get("iteration", 0) # ------------------------ # Text utilities # ------------------------ def indexes_from_sentence(voc: Voc, sentence: str) -> List[int]: tokens = [voc.SYSTEM] + tokenize(sentence) return [voc.word2index.get(tok, voc.word2index[voc.UNK]) for tok in tokens] def make_bool_causal_mask(sz: int, device: torch.device): # Causal mask (upper triangle is True/ignored) return torch.triu(torch.ones((sz, sz), dtype=torch.bool, device=device), diagonal=1) # ------------------------ # Training Function (Manual Feedback Training) # ------------------------ def train_model( full_prompt_text: str, desired_response_text: str, voc: "Voc", model: nn.Module, optimizer: optim.Optimizer ) -> Tuple[float, int]: """Trains the model to produce the desired response given the full context prompt.""" global start_iteration, checkpoint, HP if not full_prompt_text or not desired_response_text: # NOTE: This should only be called when desired_response_text is provided by the GUI return 0.0, start_iteration old_n_words = voc.n_words # 1. Update Voc for word in tokenize(full_prompt_text): voc.add_word(word) for word in tokenize(desired_response_text): # Also add words from the target voc.add_word(word) # FIX: Resize model if vocabulary grew if voc.n_words > old_n_words: resize_model_if_needed(model, voc.n_words, HP["d_model"]) # 2. Prepare Data: Input is Full Prompt, Target is Desired Response input_ids = indexes_from_sentence(voc, full_prompt_text) target_ids = indexes_from_sentence(voc, desired_response_text) # Target sequence starts after the SYSTEM token target_ids = target_ids[1:] + [voc.word2index[voc.EOS]] combined_ids = input_ids + target_ids prompt_len = len(input_ids) sequence_len = len(combined_ids) if sequence_len > HP["max_length"]: combined_ids = combined_ids[:HP["max_length"]] sequence_len = HP["max_length"] prompt_len = min(prompt_len, HP["max_length"] - 1) # --- FIX APPLIED PREVIOUSLY: Ensure input_tensor is [1, L] (2D tensor of indices) --- input_tensor = torch.tensor([combined_ids[:-1]], dtype=torch.long, device=device) target_tensor = torch.tensor([combined_ids[1:]], dtype=torch.long, device=device) # ----------------------------------------------------------------------------------- # 3. Forward Pass model.train() optimizer.zero_grad() L = input_tensor.size(1) mask = make_bool_causal_mask(L, device) padmask = (input_tensor == voc.word2index[voc.PAD]) logits = model(input_tensor, mask=mask, key_padding_mask=padmask) # Loss calculation: We only calculate loss on the target tokens (Desired Response) # Loss mask needs to be the same size as the flattened logits/targets loss_mask = torch.zeros(target_tensor.size(1), dtype=torch.float, device=device) # Start the loss calculation from the first target token (index prompt_len - 1) loss_mask[prompt_len - 1:] = 1.0 loss_mask_flat = loss_mask.view(-1) # Calculate Cross Entropy Loss loss = F.cross_entropy( logits.view(-1, voc.n_words), target_tensor.view(-1), ignore_index=voc.word2index[voc.PAD], reduction='none' ) # Apply mask and re-reduce (average loss over target tokens only) sum_mask = loss_mask_flat.sum() if sum_mask > 0: loss = (loss * loss_mask_flat).sum() / sum_mask else: # If the sequence was truncated and contains only the prompt, loss is zero loss = torch.tensor(0.0, device=device) # 4. Backward Pass & Optimization loss.backward() torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0) optimizer.step() start_iteration += 1 # 5. Save Checkpoint save_checkpoint(CHECKPOINT_PATH, HP, voc, model, optimizer, start_iteration) return loss.item(), start_iteration # ------------------------ # DuckDuckGo Web Search (Unchanged) # ------------------------ def perform_web_search(query: str) -> str: """Fetch text from DuckDuckGo instant answers.""" query = to_ascii(query.strip()) if not query: return "" try: resp = requests.get( "https://api.duckduckgo.com/", params={"q": query, "format": "json", "no_html": 1, "skip_disambig": 1}, timeout=6, ) resp.raise_for_status() data = resp.json() if data.get("AbstractText"): return to_ascii(data["AbstractText"]) elif data.get("RelatedTopics"): texts = [t["Text"] for t in data["RelatedTopics"] if isinstance(t, dict) and "Text" in t] return to_ascii(" ".join(texts[:5])) except Exception as e: print("Search failed:", e) return "" # ------------------------ # Web Content Preparation (Max Retention) # ------------------------ def prepare_web_content(text: str) -> str: """ Prepare raw web text by truncating it to a maximal safe length (500 characters) to ensure maximum retention while avoiding context overflow. """ MAX_WEB_CONTENT_LENGTH = 500 if not text.strip(): return "" prepared_text = to_ascii(text.strip()) if len(prepared_text) > MAX_WEB_CONTENT_LENGTH: prepared_text = prepared_text[:MAX_WEB_CONTENT_LENGTH] + "..." return prepared_text # ------------------------ # Streaming Generator (STABLE SAMPLER) - REVISED FOR READABILITY # ------------------------ def generate_response(prompt: str) -> Iterator[str]: """Token-by-token streaming response generator with improved punctuation handling.""" gpt_model.eval() # NOTE: The full prompt string is passed, and indexing prepends the SYSTEM token input_ids = indexes_from_sentence(voc, prompt) # --- FIX APPLIED PREVIOUSLY: Ensure input_tensor is [1, L] (2D tensor of indices) --- input_tensor = torch.tensor([input_ids], dtype=torch.long, device=device) # ----------------------------------------------------------------------------------- # --- Inference Parameters --- temperature = 0.8 top_k = HP.get("k_sampling", 200) # ---------------------------- eos_id = voc.word2index[voc.EOS] # Get the index for "olm" tokens for strong penalty (still helpful) # Filter for tokens actually in the current vocabulary olm_tokens = [voc.word2index.get(t, -1) for t in ['o', 'l', 'm', 'olm'] if t in voc.word2index] olm_tokens = [t_id for t_id in olm_tokens if t_id != -1] # Buffer to hold the last yielded token (for conditional spacing) last_yielded_token = "" with torch.no_grad(): for _ in range(HP["max_length"]): L = input_tensor.size(1) mask = make_bool_causal_mask(L, device) padmask = (input_tensor == voc.word2index[voc.PAD]) logits = gpt_model(input_tensor, mask=mask, key_padding_mask=padmask) # Extract last logits and enforce 1D shape logits = logits[:, -1, :].squeeze(0) / temperature # Shape: [N_vocab] # --- STABILIZED PENALTIES (operates on 1D logits) --- # Strong penalty against spamming OLM name for t_id in olm_tokens: if t_id < logits.size(-1): logits[t_id] = -float('inf') # Top-K Sampling if top_k > 0: v, _ = torch.topk(logits, min(top_k, logits.size(-1))) logits[logits < v[-1]] = -float('inf') # ---------------------- # Softmax on the penalized logits probs = F.softmax(logits, dim=-1).unsqueeze(0) # Unsqueeze to add batch dim for multinomial # Runtime safety checks if probs.size(-1) < gpt_model.embedding.num_embeddings: probs = probs[..., :gpt_model.embedding.num_embeddings] next_id = torch.multinomial(probs, 1).item() if next_id == eos_id: break if next_id >= gpt_model.embedding.num_embeddings: print(f"TERMINATING: Generated token ID {next_id} is out of bounds for current embeddings.") break next_token_word = voc.index2word.get(next_id, voc.UNK) if next_token_word not in [voc.PAD, voc.SOS, voc.EOS, voc.UNK, voc.SYSTEM]: # Logic: Add space if (it's not the first token) AND # (the previous token wasn't punctuation) AND # (the current token isn't punctuation or an opening bracket/quote) should_add_space = ( last_yielded_token and not is_punctuation_token(last_yielded_token) and not is_punctuation_token(next_token_word) and next_token_word not in ["(", "[", "{", "\""] ) if should_add_space: yield " " # Yield the word/token yield next_token_word last_yielded_token = next_token_word # Update the buffer next_token = torch.tensor([[next_id]], device=device) input_tensor = torch.cat([input_tensor, next_token], dim=1) # ------------------------ # Core Chat Handler (REGULAR INFERENCE ONLY) # ------------------------ def on_submit(user_message: str, history: Optional[List[List[str]]], train_input_state: Dict[str, str], *_): """ Token-by-token chat handler. Performs regular inference. """ history = history or [] # Strip image/attachment tags user_message = re.sub(r'\(Attached:.*?\)|\[IMAGE_PREVIEW_URL:.*?]', '', user_message or "").strip() current_state = train_input_state.copy() search_summary = "" is_search = WEB_TAG_RE.search(user_message) if is_search: query = WEB_TAG_RE.sub("", user_message).strip() raw = perform_web_search(query) search_summary = prepare_web_content(raw) # Prepare the FULL PROMPT for generation prefix = f"The provided web context is: {search_summary}. " if search_summary else "" full_prompt = f"{SYSTEM_INSTRUCTION} {prefix}{user_message}" # Store state data for the training interface and next turns current_state["full_prompt"] = full_prompt current_state["user_message"] = user_message current_state["search_summary"] = search_summary current_state["last_model_response"] = "" # Will be filled by stream response_stream = generate_response(full_prompt) partial_reply = "" # Status bar shows search status search_status = "🔍 Web search active." if search_summary else "No web info used." if not history or (history and history[-1][1] is not None): history.append([user_message, None]) for token in response_stream: partial_reply += token history[-1][1] = partial_reply yield history, search_status, current_state final_reply = partial_reply.strip() if history: history[-1][1] = final_reply current_state["last_model_response"] = final_reply # Final yield yield history, "Response complete.", current_state # ------------------------ # API Endpoint for Feedback Training # ------------------------ def feedback_train(history: List[List[str]], desired_response: str) -> str: """ Endpoint for training the model based on explicit bad feedback and a desired response. This function is used by the Manual Training panel and is called by the JS Bad Feedback button. """ global start_iteration if not desired_response.strip(): # NOTE: This only logs a status if the desired response is empty. # The JS client should handle simple logging if needed. return "Warning: Training skipped. Provide a desired response to train." if not history or not history[-1][0]: return "Error: Chat history is empty or missing user prompt. Cannot train." # Retrieve the full context from the last state, if available, otherwise reconstruct the simplest form last_user_prompt_raw = history[-1][0] # Reconstruct the full input prompt in the safest way possible for training full_training_input = f"{SYSTEM_INSTRUCTION} {last_user_prompt_raw}" try: loss_value, new_iteration = train_model(full_training_input, desired_response, voc, gpt_model, optimizer) return f"✅ Training Complete! Iteration: {new_iteration}, Loss: {loss_value:.4f}. Model updated." except Exception as e: return f"❌ Training Failed: {e}. Check server logs." # ------------------------ # Gradio Interface # ------------------------ with gr.Blocks(title="OkemoLLM") as demo: gr.Markdown("# 🌐 OkemoLLM — Streaming DuckDuckGo + Stable Inference") # Status bar status = gr.Markdown(f"Ready. **Current Checkpoint Iteration: {start_iteration}**. (Training is manual.)") chat = gr.Chatbot(height=400) train_state = gr.State(TRAIN_INPUT_STATE) with gr.Row(): msg = gr.Textbox(placeholder='Type your message (add for search).', label="Message", scale=4) clear = gr.Button("Clear", scale=1) gr.Markdown("---") gr.Markdown("## ⚙️ Manual Feedback Training") gr.Markdown("Use this section to train the model with a **correct answer** after a poor response. The Bad Feedback button in the chat also triggers this function.") with gr.Column(): # Defining the train_status component here train_status = gr.Markdown("Status will update after clicking 'Train'.") # This displays the last user input prompt_display = gr.Textbox(label="Last User Message (Input for Training)", interactive=False) last_response_display = gr.Textbox(label="Model's Last Response", interactive=False) # User provides the ground truth here manual_target_response = gr.Textbox(label="Desired Response (Ground Truth for Training)", placeholder='Enter the correct/better response for the prompt above.') train_button = gr.Button("Perform Manual Feedback Training") # Connect Chat Handlers (Inference only) msg.submit( on_submit, [msg, chat, train_state], [chat, status, train_state] ).then( # Update context displays after submission lambda s: (s.get("user_message", "No message."), s.get("last_model_response", "")), train_state, [prompt_display, last_response_display] ) clear.click(lambda: ([], f"Cleared. Current Checkpoint Iteration: {start_iteration}."), None, [chat, status], queue=False) # Connect the Manual Training Button (This implicitly registers the /feedback_train endpoint) train_button.click( feedback_train, [chat, manual_target_response], # chat component passes the history train_status ) # --- Removing the simple logging endpoint and relying on 'feedback_train' --- # The JS will send an empty string to /feedback_train, which the Python function handles by skipping training. if __name__ == "__main__": demo.queue(api_open=True).launch(ssr_mode=False)