okemo2 / app.py
ar12c's picture
Update app.py
ca72be1 verified
Raw History Blame Contribute Delete
23.5 kB
# filename: okemo_backend.py
# OkemoLLM backend β€” Streaming responses + DuckDuckGo + Increased Web Retention
import os
import re
import math
import unicodedata
import requests
from typing import Dict, List, Tuple, Optional, Any, Iterator
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import gradio as gr
os.environ["PYTORCH_ENABLE_MPS_FALLBACK"] = "1"
# --- SYSTEM INSTRUCTION MODIFIED FOR CHATGPT-LIKE BEHAVIOR ---
SYSTEM_INSTRUCTION = (
"You are Okemo Language Model (OLM), a highly **helpful, verbose, and comprehensive** AI assistant, similar to ChatGPT. "
"Your goal is to provide detailed and well-structured answers in a conversational tone, ensuring **clarity and fluency**."
"For complex queries, first generate an internal reasoning chain (DO NOT show it to the user). "
"If external context is provided, naturally integrate that knowledge into your answer."
)
# -------------------------------------------------------------------------
CHECKPOINT_PATH = "checkpoint.pth"
DEFAULTS = {
"d_model": 768,
"n_head": 48,
"d_ff": 768,
"n_layers": 12,
"dropout": 0.1,
"max_length": 1024,
"k_sampling": 200, # Using 200 for diversity
"LEARNING_RATE": 1e-6,
}
WEB_TAG_RE = re.compile(r"<\s*web\s*>", flags=re.IGNORECASE)
TRAIN_INPUT_STATE = {} # Initial value for the Gradio State
# ------------------------
# Helpers
# ------------------------
def to_ascii(text: Optional[str]) -> str:
return unicodedata.normalize("NFKC", text or "")
_TOKEN_RE = re.compile(r"\w+|[^\w\s]", flags=re.UNICODE)
def tokenize(text: str) -> List[str]:
return _TOKEN_RE.findall(to_ascii(text).lower())
def is_punctuation_token(token: str) -> bool:
"""Checks if a token is a punctuation mark that typically follows a word without a space."""
# Added open parenthesis and quotation marks as tokens that may start a new space-separated unit
# Added common closing punctuation that should NOT have a preceding space
return token in [".", ",", "!", "?", ":", ";", ")", "]", "}", "'s", "'", "\""]
# ------------------------
# Vocabulary
# ------------------------
class Voc:
def __init__(self) -> None:
self.PAD, self.SOS, self.EOS, self.UNK, self.SYSTEM = "<PAD>", "<SOS>", "<EOS>", "<UNK>", "<SYSTEM>"
self.word2index = {self.PAD: 0, self.SOS: 1, self.EOS: 2, self.UNK: 3, self.SYSTEM: 4}
self.index2word = {i: w for w, i in self.word2index.items()}
self.n_words = len(self.word2index)
@staticmethod
def load_from_checkpoint(ckpt: Dict[str, Any]) -> "Voc":
voc = Voc()
if "voc_word2index" in ckpt:
voc.word2index = ckpt["voc_word2index"]
voc.index2word = ckpt["voc_index2word"]
voc.n_words = len(voc.word2index)
return voc
def add_word(self, word: str) -> None:
if word not in self.word2index:
self.word2index[word] = self.n_words
self.index2word[self.n_words] = word
self.n_words += 1
# ------------------------
# Model
# ------------------------
class PositionalEncoding(nn.Module):
def __init__(self, d_model: int, max_len: int = 5000):
super().__init__()
pe = torch.zeros(max_len, d_model)
position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)
div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model))
pe[:, 0::2] = torch.sin(position * div_term)
pe[:, 1::2] = torch.cos(position * div_term)
self.register_buffer("pe", pe.unsqueeze(0))
def forward(self, x: torch.Tensor) -> torch.Tensor:
return x + self.pe[:, : x.size(1), :]
class GPT(nn.Module):
def __init__(self, n_words, d_model, n_layers, n_head, d_ff, dropout=0.1):
super().__init__()
self.embedding = nn.Embedding(n_words, d_model)
self.pos = PositionalEncoding(d_model)
self.norm = nn.LayerNorm(d_model)
layer = nn.TransformerEncoderLayer(d_model, n_head, d_ff, dropout, batch_first=True)
self.transformer = nn.TransformerEncoder(layer, n_layers)
self.fc = nn.Linear(d_model, n_words, bias=False)
self.fc.weight = self.embedding.weight
def forward(self, x, mask=None, key_padding_mask=None):
x = self.embedding(x)
x = self.pos(x)
x = self.norm(x)
out = self.transformer(x, mask=mask, src_key_padding_mask=key_padding_mask)
return self.fc(out)
# ------------------------
# Device + Checkpoint + Resizing Fix
# ------------------------
device = torch.device("cpu")
def resize_model_if_needed(model: nn.Module, new_n_words: int, d_model: int) -> bool:
"""
Checks if the model's vocabulary size needs to be updated and resizes the layers.
Returns True if resize occurred, False otherwise.
"""
if new_n_words > model.embedding.num_embeddings:
print(f"Resizing model from {model.embedding.num_embeddings} to {new_n_words} vocabulary size...")
# 1. Save old embedding and fc weights
old_embedding_weight = model.embedding.weight.data.clone()
# 2. Create new embedding layer
new_embedding = nn.Embedding(new_n_words, d_model)
new_embedding.to(model.embedding.weight.device) # ensure device consistency
# 3. Copy old weights to new layer
copy_len = min(new_n_words, old_embedding_weight.size(0))
new_embedding.weight.data[:copy_len] = old_embedding_weight[:copy_len]
# 4. Replace old layers
model.embedding = new_embedding
model.fc = nn.Linear(d_model, new_n_words, bias=False)
# 5. Re-tie weights
model.fc.weight = model.embedding.weight
return True
return False
def save_checkpoint(path: str, hp: Dict[str, Any], voc: "Voc", model: nn.Module, optimizer: optim.Optimizer, iteration: int = 0) -> None:
"""Saves the current model and vocabulary state."""
print(f"Saving checkpoint to {path} at iteration {iteration}...")
current_hp = {k: hp[k] for k in hp}
current_hp["n_words"] = voc.n_words
torch.save(
{
**current_hp,
"iteration": iteration,
"gpt_model_state_dict": model.state_dict(),
"optimizer_state_dict": optimizer.state_dict(),
"voc_word2index": voc.word2index,
"voc_index2word": voc.index2word,
"voc_n_words": voc.n_words, # Added for robustness
},
path,
)
def load_checkpoint_or_defaults(path: str):
global device
# Attempt to use MPS/CUDA if available
if torch.backends.mps.is_available():
device = torch.device("mps")
elif torch.cuda.is_available():
device = torch.device("cuda")
else:
device = torch.device("cpu")
print(f"Using device: {device}")
if os.path.exists(path):
ckpt = torch.load(path, map_location=device)
hp = {k: ckpt.get(k, v) for k, v in DEFAULTS.items()}
voc = Voc.load_from_checkpoint(ckpt)
n_words = ckpt.get("n_words", voc.n_words) # Use n_words from checkpoint if available
model = GPT(n_words, hp["d_model"], hp["n_layers"], hp["n_head"], hp["d_ff"], hp["dropout"]).to(device)
model.load_state_dict(ckpt["gpt_model_state_dict"], strict=False)
optimizer = optim.Adam(model.parameters(), lr=hp["LEARNING_RATE"])
if "optimizer_state_dict" in ckpt:
optimizer.load_state_dict(ckpt["optimizer_state_dict"])
return ckpt, hp, voc, model, optimizer
else:
voc = Voc()
model = GPT(voc.n_words, **{k: v for k, v in DEFAULTS.items() if k in ["d_model", "n_layers", "n_head", "d_ff", "dropout"]}).to(device)
optimizer = optim.Adam(model.parameters(), lr=DEFAULTS["LEARNING_RATE"])
return {}, DEFAULTS, voc, model, optimizer
checkpoint, HP, voc, gpt_model, optimizer = load_checkpoint_or_defaults(CHECKPOINT_PATH)
start_iteration = checkpoint.get("iteration", 0)
# ------------------------
# Text utilities
# ------------------------
def indexes_from_sentence(voc: Voc, sentence: str) -> List[int]:
tokens = [voc.SYSTEM] + tokenize(sentence)
return [voc.word2index.get(tok, voc.word2index[voc.UNK]) for tok in tokens]
def make_bool_causal_mask(sz: int, device: torch.device):
# Causal mask (upper triangle is True/ignored)
return torch.triu(torch.ones((sz, sz), dtype=torch.bool, device=device), diagonal=1)
# ------------------------
# Training Function (Manual Feedback Training)
# ------------------------
def train_model(
full_prompt_text: str, desired_response_text: str, voc: "Voc", model: nn.Module, optimizer: optim.Optimizer
) -> Tuple[float, int]:
"""Trains the model to produce the desired response given the full context prompt."""
global start_iteration, checkpoint, HP
if not full_prompt_text or not desired_response_text:
# NOTE: This should only be called when desired_response_text is provided by the GUI
return 0.0, start_iteration
old_n_words = voc.n_words
# 1. Update Voc
for word in tokenize(full_prompt_text):
voc.add_word(word)
for word in tokenize(desired_response_text): # Also add words from the target
voc.add_word(word)
# FIX: Resize model if vocabulary grew
if voc.n_words > old_n_words:
resize_model_if_needed(model, voc.n_words, HP["d_model"])
# 2. Prepare Data: Input is Full Prompt, Target is Desired Response
input_ids = indexes_from_sentence(voc, full_prompt_text)
target_ids = indexes_from_sentence(voc, desired_response_text)
# Target sequence starts after the SYSTEM token
target_ids = target_ids[1:] + [voc.word2index[voc.EOS]]
combined_ids = input_ids + target_ids
prompt_len = len(input_ids)
sequence_len = len(combined_ids)
if sequence_len > HP["max_length"]:
combined_ids = combined_ids[:HP["max_length"]]
sequence_len = HP["max_length"]
prompt_len = min(prompt_len, HP["max_length"] - 1)
# --- FIX APPLIED PREVIOUSLY: Ensure input_tensor is [1, L] (2D tensor of indices) ---
input_tensor = torch.tensor([combined_ids[:-1]], dtype=torch.long, device=device)
target_tensor = torch.tensor([combined_ids[1:]], dtype=torch.long, device=device)
# -----------------------------------------------------------------------------------
# 3. Forward Pass
model.train()
optimizer.zero_grad()
L = input_tensor.size(1)
mask = make_bool_causal_mask(L, device)
padmask = (input_tensor == voc.word2index[voc.PAD])
logits = model(input_tensor, mask=mask, key_padding_mask=padmask)
# Loss calculation: We only calculate loss on the target tokens (Desired Response)
# Loss mask needs to be the same size as the flattened logits/targets
loss_mask = torch.zeros(target_tensor.size(1), dtype=torch.float, device=device)
# Start the loss calculation from the first target token (index prompt_len - 1)
loss_mask[prompt_len - 1:] = 1.0
loss_mask_flat = loss_mask.view(-1)
# Calculate Cross Entropy Loss
loss = F.cross_entropy(
logits.view(-1, voc.n_words),
target_tensor.view(-1),
ignore_index=voc.word2index[voc.PAD],
reduction='none'
)
# Apply mask and re-reduce (average loss over target tokens only)
sum_mask = loss_mask_flat.sum()
if sum_mask > 0:
loss = (loss * loss_mask_flat).sum() / sum_mask
else:
# If the sequence was truncated and contains only the prompt, loss is zero
loss = torch.tensor(0.0, device=device)
# 4. Backward Pass & Optimization
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
optimizer.step()
start_iteration += 1
# 5. Save Checkpoint
save_checkpoint(CHECKPOINT_PATH, HP, voc, model, optimizer, start_iteration)
return loss.item(), start_iteration
# ------------------------
# DuckDuckGo Web Search (Unchanged)
# ------------------------
def perform_web_search(query: str) -> str:
"""Fetch text from DuckDuckGo instant answers."""
query = to_ascii(query.strip())
if not query:
return ""
try:
resp = requests.get(
"https://api.duckduckgo.com/",
params={"q": query, "format": "json", "no_html": 1, "skip_disambig": 1},
timeout=6,
)
resp.raise_for_status()
data = resp.json()
if data.get("AbstractText"):
return to_ascii(data["AbstractText"])
elif data.get("RelatedTopics"):
texts = [t["Text"] for t in data["RelatedTopics"] if isinstance(t, dict) and "Text" in t]
return to_ascii(" ".join(texts[:5]))
except Exception as e:
print("Search failed:", e)
return ""
# ------------------------
# Web Content Preparation (Max Retention)
# ------------------------
def prepare_web_content(text: str) -> str:
"""
Prepare raw web text by truncating it to a maximal safe length
(500 characters) to ensure maximum retention while avoiding context overflow.
"""
MAX_WEB_CONTENT_LENGTH = 500
if not text.strip():
return ""
prepared_text = to_ascii(text.strip())
if len(prepared_text) > MAX_WEB_CONTENT_LENGTH:
prepared_text = prepared_text[:MAX_WEB_CONTENT_LENGTH] + "..."
return prepared_text
# ------------------------
# Streaming Generator (STABLE SAMPLER) - REVISED FOR READABILITY
# ------------------------
def generate_response(prompt: str) -> Iterator[str]:
"""Token-by-token streaming response generator with improved punctuation handling."""
gpt_model.eval()
# NOTE: The full prompt string is passed, and indexing prepends the SYSTEM token
input_ids = indexes_from_sentence(voc, prompt)
# --- FIX APPLIED PREVIOUSLY: Ensure input_tensor is [1, L] (2D tensor of indices) ---
input_tensor = torch.tensor([input_ids], dtype=torch.long, device=device)
# -----------------------------------------------------------------------------------
# --- Inference Parameters ---
temperature = 0.8
top_k = HP.get("k_sampling", 200)
# ----------------------------
eos_id = voc.word2index[voc.EOS]
# Get the index for "olm" tokens for strong penalty (still helpful)
# Filter for tokens actually in the current vocabulary
olm_tokens = [voc.word2index.get(t, -1) for t in ['o', 'l', 'm', 'olm'] if t in voc.word2index]
olm_tokens = [t_id for t_id in olm_tokens if t_id != -1]
# Buffer to hold the last yielded token (for conditional spacing)
last_yielded_token = ""
with torch.no_grad():
for _ in range(HP["max_length"]):
L = input_tensor.size(1)
mask = make_bool_causal_mask(L, device)
padmask = (input_tensor == voc.word2index[voc.PAD])
logits = gpt_model(input_tensor, mask=mask, key_padding_mask=padmask)
# Extract last logits and enforce 1D shape
logits = logits[:, -1, :].squeeze(0) / temperature # Shape: [N_vocab]
# --- STABILIZED PENALTIES (operates on 1D logits) ---
# Strong penalty against spamming OLM name
for t_id in olm_tokens:
if t_id < logits.size(-1):
logits[t_id] = -float('inf')
# Top-K Sampling
if top_k > 0:
v, _ = torch.topk(logits, min(top_k, logits.size(-1)))
logits[logits < v[-1]] = -float('inf')
# ----------------------
# Softmax on the penalized logits
probs = F.softmax(logits, dim=-1).unsqueeze(0) # Unsqueeze to add batch dim for multinomial
# Runtime safety checks
if probs.size(-1) < gpt_model.embedding.num_embeddings:
probs = probs[..., :gpt_model.embedding.num_embeddings]
next_id = torch.multinomial(probs, 1).item()
if next_id == eos_id:
break
if next_id >= gpt_model.embedding.num_embeddings:
print(f"TERMINATING: Generated token ID {next_id} is out of bounds for current embeddings.")
break
next_token_word = voc.index2word.get(next_id, voc.UNK)
if next_token_word not in [voc.PAD, voc.SOS, voc.EOS, voc.UNK, voc.SYSTEM]:
# Logic: Add space if (it's not the first token) AND
# (the previous token wasn't punctuation) AND
# (the current token isn't punctuation or an opening bracket/quote)
should_add_space = (
last_yielded_token and
not is_punctuation_token(last_yielded_token) and
not is_punctuation_token(next_token_word) and
next_token_word not in ["(", "[", "{", "\""]
)
if should_add_space:
yield " "
# Yield the word/token
yield next_token_word
last_yielded_token = next_token_word # Update the buffer
next_token = torch.tensor([[next_id]], device=device)
input_tensor = torch.cat([input_tensor, next_token], dim=1)
# ------------------------
# Core Chat Handler (REGULAR INFERENCE ONLY)
# ------------------------
def on_submit(user_message: str, history: Optional[List[List[str]]], train_input_state: Dict[str, str], *_):
"""
Token-by-token chat handler. Performs regular inference.
"""
history = history or []
# Strip image/attachment tags
user_message = re.sub(r'\(Attached:.*?\)|\[IMAGE_PREVIEW_URL:.*?]', '', user_message or "").strip()
current_state = train_input_state.copy()
search_summary = ""
is_search = WEB_TAG_RE.search(user_message)
if is_search:
query = WEB_TAG_RE.sub("", user_message).strip()
raw = perform_web_search(query)
search_summary = prepare_web_content(raw)
# Prepare the FULL PROMPT for generation
prefix = f"The provided web context is: {search_summary}. " if search_summary else ""
full_prompt = f"{SYSTEM_INSTRUCTION} {prefix}{user_message}"
# Store state data for the training interface and next turns
current_state["full_prompt"] = full_prompt
current_state["user_message"] = user_message
current_state["search_summary"] = search_summary
current_state["last_model_response"] = "" # Will be filled by stream
response_stream = generate_response(full_prompt)
partial_reply = ""
# Status bar shows search status
search_status = "πŸ” Web search active." if search_summary else "No web info used."
if not history or (history and history[-1][1] is not None):
history.append([user_message, None])
for token in response_stream:
partial_reply += token
history[-1][1] = partial_reply
yield history, search_status, current_state
final_reply = partial_reply.strip()
if history:
history[-1][1] = final_reply
current_state["last_model_response"] = final_reply
# Final yield
yield history, "Response complete.", current_state
# ------------------------
# API Endpoint for Feedback Training
# ------------------------
def feedback_train(history: List[List[str]], desired_response: str) -> str:
"""
Endpoint for training the model based on explicit bad feedback and a desired response.
This function is used by the Manual Training panel and is called by the JS Bad Feedback button.
"""
global start_iteration
if not desired_response.strip():
# NOTE: This only logs a status if the desired response is empty.
# The JS client should handle simple logging if needed.
return "Warning: Training skipped. Provide a desired response to train."
if not history or not history[-1][0]:
return "Error: Chat history is empty or missing user prompt. Cannot train."
# Retrieve the full context from the last state, if available, otherwise reconstruct the simplest form
last_user_prompt_raw = history[-1][0]
# Reconstruct the full input prompt in the safest way possible for training
full_training_input = f"{SYSTEM_INSTRUCTION} {last_user_prompt_raw}"
try:
loss_value, new_iteration = train_model(full_training_input, desired_response, voc, gpt_model, optimizer)
return f"βœ… Training Complete! Iteration: {new_iteration}, Loss: {loss_value:.4f}. Model updated."
except Exception as e:
return f"❌ Training Failed: {e}. Check server logs."
# ------------------------
# Gradio Interface
# ------------------------
with gr.Blocks(title="OkemoLLM") as demo:
gr.Markdown("# 🌐 OkemoLLM β€” Streaming DuckDuckGo + Stable Inference")
# Status bar
status = gr.Markdown(f"Ready. **Current Checkpoint Iteration: {start_iteration}**. (Training is manual.)")
chat = gr.Chatbot(height=400)
train_state = gr.State(TRAIN_INPUT_STATE)
with gr.Row():
msg = gr.Textbox(placeholder='Type your message (add <web> for search).', label="Message", scale=4)
clear = gr.Button("Clear", scale=1)
gr.Markdown("---")
gr.Markdown("## βš™οΈ Manual Feedback Training")
gr.Markdown("Use this section to train the model with a **correct answer** after a poor response. The Bad Feedback button in the chat also triggers this function.")
with gr.Column():
# Defining the train_status component here
train_status = gr.Markdown("Status will update after clicking 'Train'.")
# This displays the last user input
prompt_display = gr.Textbox(label="Last User Message (Input for Training)", interactive=False)
last_response_display = gr.Textbox(label="Model's Last Response", interactive=False)
# User provides the ground truth here
manual_target_response = gr.Textbox(label="Desired Response (Ground Truth for Training)", placeholder='Enter the correct/better response for the prompt above.')
train_button = gr.Button("Perform Manual Feedback Training")
# Connect Chat Handlers (Inference only)
msg.submit(
on_submit,
[msg, chat, train_state],
[chat, status, train_state]
).then(
# Update context displays after submission
lambda s: (s.get("user_message", "No message."), s.get("last_model_response", "")),
train_state,
[prompt_display, last_response_display]
)
clear.click(lambda: ([], f"Cleared. Current Checkpoint Iteration: {start_iteration}."), None, [chat, status], queue=False)
# Connect the Manual Training Button (This implicitly registers the /feedback_train endpoint)
train_button.click(
feedback_train,
[chat, manual_target_response], # chat component passes the history
train_status
)
# --- Removing the simple logging endpoint and relying on 'feedback_train' ---
# The JS will send an empty string to /feedback_train, which the Python function handles by skipping training.
if __name__ == "__main__":
demo.queue(api_open=True).launch(ssr_mode=False)