OkemoLLM / app.py
ar12c's picture
Update app.py
26a3106
Raw History Blame Contribute Delete
11.8 kB
import os
import torch
import torch.nn as nn
import torch.nn.functional as F
import gradio as gr
from transformers import GPTNeoXTokenizerFast
try:
from duckduckgo_search import DDGS
except ImportError:
DDGS = None
# ================= CONFIG =================
D_MODEL = 832
N_HEADS = 13
N_LAYERS = 18
VOCAB_SIZE = 51200
SEQ_LEN = 512
DEVICE = "cpu"
WEIGHTS_PATH = "OLM_NORTH_STAR.pt"
TOKENIZER_NAME = "EleutherAI/gpt-neox-20b"
# ================= MODEL =================
class RotaryEmbedding(nn.Module):
def __init__(self, dim, base=10000):
super().__init__()
self.dim = dim
self.base = base
def forward(self, x, seq_len):
device = x.device
inv_freq = 1.0 / (self.base ** (torch.arange(0, self.dim, 2, device=device).float() / self.dim))
t = torch.arange(seq_len, device=device).type_as(inv_freq)
freqs = torch.outer(t, inv_freq)
cos = freqs.cos()[None, None, :, :]
sin = freqs.sin()[None, None, :, :]
x1 = x[..., 0::2]
x2 = x[..., 1::2]
out = torch.stack([x1 * cos - x2 * sin, x1 * sin + x2 * cos], dim=-1)
return out.flatten(-2)
class Attention(nn.Module):
def __init__(self):
super().__init__()
self.qkv = nn.Linear(D_MODEL, 3 * D_MODEL, bias=False)
self.proj = nn.Linear(D_MODEL, D_MODEL, bias=False)
self.head_dim = D_MODEL // N_HEADS
self.rope = RotaryEmbedding(self.head_dim)
def forward(self, x, mask=None):
B, T, C = x.shape
qkv = self.qkv(x)
q, k, v = qkv.chunk(3, dim=-1)
q = q.view(B, T, N_HEADS, self.head_dim).transpose(1, 2)
k = k.view(B, T, N_HEADS, self.head_dim).transpose(1, 2)
v = v.view(B, T, N_HEADS, self.head_dim).transpose(1, 2)
q = self.rope(q, T)
k = self.rope(k, T)
y = F.scaled_dot_product_attention(q, k, v, attn_mask=mask, is_causal=True)
y = y.transpose(1, 2).contiguous().view(B, T, C)
return self.proj(y)
class FeedForward(nn.Module):
def __init__(self):
super().__init__()
self.layers = nn.ModuleList([
nn.Linear(D_MODEL, int(4.5 * D_MODEL), bias=False),
nn.GELU(),
nn.Linear(int(4.5 * D_MODEL), D_MODEL, bias=False)
])
def forward(self, x):
for layer in self.layers:
x = layer(x)
return x
class Block(nn.Module):
def __init__(self):
super().__init__()
self.ln1 = nn.LayerNorm(D_MODEL)
self.attn = Attention()
self.ln2 = nn.LayerNorm(D_MODEL)
self.ff = FeedForward()
def forward(self, x):
x = x + self.attn(self.ln1(x))
x = x + self.ff(self.ln2(x))
return x
class PyTorchGPT(nn.Module):
def __init__(self):
super().__init__()
self.embed = nn.Embedding(VOCAB_SIZE, D_MODEL)
self.blocks = nn.ModuleList([Block() for _ in range(N_LAYERS)])
self.ln = nn.LayerNorm(D_MODEL)
self.head = nn.Linear(D_MODEL, VOCAB_SIZE, bias=False)
def forward(self, idx):
x = self.embed(idx)
for block in self.blocks:
x = block(x)
x = self.ln(x)
return self.head(x)
# ================= LOAD =================
model = None
tokenizer = None
def load_model():
global model, tokenizer
print(f"--- Initializing NorthStar on {DEVICE} ---")
model = PyTorchGPT().to(DEVICE)
if os.path.exists(WEIGHTS_PATH):
print(f"✓ Loading weights from: {WEIGHTS_PATH}")
state_dict = torch.load(WEIGHTS_PATH, map_location=DEVICE)
if "model_state_dict" in state_dict:
state_dict = state_dict["model_state_dict"]
model.load_state_dict(state_dict)
model.eval()
print("✓ Weights loaded successfully!")
else:
print(f"✗ ERROR: {WEIGHTS_PATH} not found!")
tokenizer = GPTNeoXTokenizerFast.from_pretrained(TOKENIZER_NAME)
print("✓ Tokenizer loaded")
load_model()
# ================= GENERATION =================
def get_system_prompt(use_thought):
"""Selects the appropriate system prompt based on the thought toggle."""
if use_thought:
return "You are Polaris, a creative and analytical AI created by Okemo. Think deeply inside <thought> tags, then provide a helpful and natural answer."
return "You are Polaris, a creative and analytical AI created by Okemo. Provide a helpful and natural answer directly."
def search_web(query):
"""Performs a DuckDuckGo web search and returns formatted results."""
if DDGS is None:
return "Error: duckduckgo_search library not installed."
try:
with DDGS() as ddgs:
results = list(ddgs.text(query, max_results=5))
if not results:
return "No search results found."
formatted_results = "Web Search Results:\n\n"
for i, res in enumerate(results, 1):
formatted_results += f"{i}. {res['title']}\n {res['body']}\n Source: {res['href']}\n\n"
return formatted_results
except Exception as e:
return f"Error performing search: {str(e)}"
def extract_text(content):
"""Extract plain text from various Gradio content formats."""
if isinstance(content, str):
return content
if isinstance(content, list):
# List of content parts like [{"type": "text", "text": "..."}]
parts = []
for part in content:
if isinstance(part, str):
parts.append(part)
elif isinstance(part, dict):
parts.append(part.get("text", part.get("value", str(part))))
else:
parts.append(getattr(part, "text", getattr(part, "value", str(part))))
return "".join(parts)
return str(content)
def stream_chat(message, history, use_thought=True, use_search=False):
"""Streaming chat function - yields updated history as tokens are generated."""
if not message.strip():
yield history
return
model.eval()
# Select system prompt based on thought toggle
instruction = get_system_prompt(use_thought)
# Perform web search if enabled
context_injection = ""
if use_search:
search_results = search_web(message)
context_injection = f"\n\nContext from Web Search:\n{search_results}\n\nUse this information to answer the user's question if relevant."
instruction += context_injection
# Build prompt with ChatML format
full_prompt = f"<|im_start|>system\n{instruction}<|im_end|>\n"
# Add conversation history - handle both dict and ChatMessage objects
for turn in history:
role = ""
content = ""
if isinstance(turn, dict):
role = turn.get("role", "user")
content = extract_text(turn.get("content", ""))
else:
# Gradio ChatMessage objects
role = getattr(turn, "role", "user")
content = extract_text(getattr(turn, "content", ""))
if role and content:
full_prompt += f"<|im_start|>{role}\n{content}<|im_end|>\n"
# Add current user input
full_prompt += f"<|im_start|>user\n{message}<|im_end|>\n<|im_start|>assistant\n"
# Encode
idx = tokenizer.encode(full_prompt, return_tensors="pt").to(DEVICE)
generated_text = ""
# Build clean history as plain dicts for output
clean_history = []
for turn in history:
if isinstance(turn, dict):
clean_history.append({"role": turn.get("role", "user"), "content": extract_text(turn.get("content", ""))})
else:
clean_history.append({"role": getattr(turn, "role", "user"), "content": extract_text(getattr(turn, "content", ""))})
# Add user message to history first
new_history = clean_history + [
{"role": "user", "content": message},
{"role": "assistant", "content": ""}
]
max_new_tokens = 512
temperature = 0.35
top_p = 0.75
rep_penalty = 1.25
pres_penalty = 0.3
generated_tokens = []
for _ in range(max_new_tokens):
idx_cond = idx if idx.size(1) <= SEQ_LEN else idx[:, -SEQ_LEN:]
with torch.no_grad():
logits = model(idx_cond)
logits = logits[:, -1, :] / temperature
# Repetition & Presence Penalty
if generated_tokens:
unique_tokens = list(set(generated_tokens))
for t_id in unique_tokens:
if logits[0, t_id] > 0:
logits[0, t_id] /= rep_penalty
else:
logits[0, t_id] *= rep_penalty
logits[0, t_id] -= pres_penalty
# Top-P (nucleus) sampling
probs = F.softmax(logits, dim=-1)
sorted_probs, sorted_indices = torch.sort(probs, descending=True)
cumulative_probs = torch.cumsum(sorted_probs, dim=-1)
sorted_indices_to_remove = cumulative_probs > top_p
sorted_indices_to_remove[..., 1:] = sorted_indices_to_remove[..., :-1].clone()
sorted_indices_to_remove[..., 0] = 0
indices_to_remove = sorted_indices_to_remove.scatter(1, sorted_indices, sorted_indices_to_remove)
probs[indices_to_remove] = 0
probs = probs / probs.sum(dim=-1, keepdim=True)
idx_next = torch.multinomial(probs, num_samples=1)
# Decode just the new token
new_token = tokenizer.decode(idx_next[0])
# Check for stop token before adding
if "<|im_end" in new_token or "<|im_end|>" in generated_text + new_token:
# Clean up the response
final_text = generated_text
for stop in ["<|im_end|>", "<|im_end", "<|im_", "<|im", "<|i", "<|"]:
if stop in final_text:
final_text = final_text.split(stop)[0]
break
new_history[-1]["content"] = final_text.strip() + "__DONE__"
yield new_history
return
generated_text += new_token
# Update assistant message and yield
new_history[-1]["content"] = generated_text.strip()
yield new_history
idx = torch.cat((idx, idx_next), dim=1)
generated_tokens.append(idx_next.item())
if idx_next.item() == tokenizer.eos_token_id:
break
# Final cleanup
final_text = generated_text
for stop in ["<|im_end|>", "<|im_end", "<|im_", "<|im", "<|i", "<|"]:
if stop in final_text:
final_text = final_text.split(stop)[0]
break
new_history[-1]["content"] = final_text.strip() + "__DONE__"
yield new_history
def clear_chat():
return []
# Simple Blocks interface
with gr.Blocks(title="Polaris") as demo:
gr.Markdown("# 🌠 Polaris")
chatbot = gr.Chatbot(height=500, label="Chat")
thought_toggle = gr.Checkbox(value=True, visible=False, elem_id="thought-toggle", label="Use Thought")
search_toggle = gr.Checkbox(value=False, visible=False, elem_id="search-toggle", label="Use Search")
with gr.Row():
msg = gr.Textbox(placeholder="Type message...", show_label=False, scale=9)
clear = gr.Button("Clear", scale=1)
# API endpoint with explicit name - streaming
msg.submit(stream_chat, [msg, chatbot, thought_toggle, search_toggle], [chatbot], api_name="chat").then(
lambda: "", None, [msg]
)
clear.click(clear_chat, None, [chatbot], api_name="clear_chat")
if __name__ == "__main__":
demo.queue().launch()