import os import torch import torch.nn as nn import torch.nn.functional as F import gradio as gr from transformers import GPTNeoXTokenizerFast try: from duckduckgo_search import DDGS except ImportError: DDGS = None # ================= CONFIG ================= D_MODEL = 832 N_HEADS = 13 N_LAYERS = 18 VOCAB_SIZE = 51200 SEQ_LEN = 512 DEVICE = "cpu" WEIGHTS_PATH = "OLM_NORTH_STAR.pt" TOKENIZER_NAME = "EleutherAI/gpt-neox-20b" # ================= MODEL ================= class RotaryEmbedding(nn.Module): def __init__(self, dim, base=10000): super().__init__() self.dim = dim self.base = base def forward(self, x, seq_len): device = x.device inv_freq = 1.0 / (self.base ** (torch.arange(0, self.dim, 2, device=device).float() / self.dim)) t = torch.arange(seq_len, device=device).type_as(inv_freq) freqs = torch.outer(t, inv_freq) cos = freqs.cos()[None, None, :, :] sin = freqs.sin()[None, None, :, :] x1 = x[..., 0::2] x2 = x[..., 1::2] out = torch.stack([x1 * cos - x2 * sin, x1 * sin + x2 * cos], dim=-1) return out.flatten(-2) class Attention(nn.Module): def __init__(self): super().__init__() self.qkv = nn.Linear(D_MODEL, 3 * D_MODEL, bias=False) self.proj = nn.Linear(D_MODEL, D_MODEL, bias=False) self.head_dim = D_MODEL // N_HEADS self.rope = RotaryEmbedding(self.head_dim) def forward(self, x, mask=None): B, T, C = x.shape qkv = self.qkv(x) q, k, v = qkv.chunk(3, dim=-1) q = q.view(B, T, N_HEADS, self.head_dim).transpose(1, 2) k = k.view(B, T, N_HEADS, self.head_dim).transpose(1, 2) v = v.view(B, T, N_HEADS, self.head_dim).transpose(1, 2) q = self.rope(q, T) k = self.rope(k, T) y = F.scaled_dot_product_attention(q, k, v, attn_mask=mask, is_causal=True) y = y.transpose(1, 2).contiguous().view(B, T, C) return self.proj(y) class FeedForward(nn.Module): def __init__(self): super().__init__() self.layers = nn.ModuleList([ nn.Linear(D_MODEL, int(4.5 * D_MODEL), bias=False), nn.GELU(), nn.Linear(int(4.5 * D_MODEL), D_MODEL, bias=False) ]) def forward(self, x): for layer in self.layers: x = layer(x) return x class Block(nn.Module): def __init__(self): super().__init__() self.ln1 = nn.LayerNorm(D_MODEL) self.attn = Attention() self.ln2 = nn.LayerNorm(D_MODEL) self.ff = FeedForward() def forward(self, x): x = x + self.attn(self.ln1(x)) x = x + self.ff(self.ln2(x)) return x class PyTorchGPT(nn.Module): def __init__(self): super().__init__() self.embed = nn.Embedding(VOCAB_SIZE, D_MODEL) self.blocks = nn.ModuleList([Block() for _ in range(N_LAYERS)]) self.ln = nn.LayerNorm(D_MODEL) self.head = nn.Linear(D_MODEL, VOCAB_SIZE, bias=False) def forward(self, idx): x = self.embed(idx) for block in self.blocks: x = block(x) x = self.ln(x) return self.head(x) # ================= LOAD ================= model = None tokenizer = None def load_model(): global model, tokenizer print(f"--- Initializing NorthStar on {DEVICE} ---") model = PyTorchGPT().to(DEVICE) if os.path.exists(WEIGHTS_PATH): print(f"✓ Loading weights from: {WEIGHTS_PATH}") state_dict = torch.load(WEIGHTS_PATH, map_location=DEVICE) if "model_state_dict" in state_dict: state_dict = state_dict["model_state_dict"] model.load_state_dict(state_dict) model.eval() print("✓ Weights loaded successfully!") else: print(f"✗ ERROR: {WEIGHTS_PATH} not found!") tokenizer = GPTNeoXTokenizerFast.from_pretrained(TOKENIZER_NAME) print("✓ Tokenizer loaded") load_model() # ================= GENERATION ================= def get_system_prompt(use_thought): """Selects the appropriate system prompt based on the thought toggle.""" if use_thought: return "You are Polaris, a creative and analytical AI created by Okemo. Think deeply inside tags, then provide a helpful and natural answer." return "You are Polaris, a creative and analytical AI created by Okemo. Provide a helpful and natural answer directly." def search_web(query): """Performs a DuckDuckGo web search and returns formatted results.""" if DDGS is None: return "Error: duckduckgo_search library not installed." try: with DDGS() as ddgs: results = list(ddgs.text(query, max_results=5)) if not results: return "No search results found." formatted_results = "Web Search Results:\n\n" for i, res in enumerate(results, 1): formatted_results += f"{i}. {res['title']}\n {res['body']}\n Source: {res['href']}\n\n" return formatted_results except Exception as e: return f"Error performing search: {str(e)}" def extract_text(content): """Extract plain text from various Gradio content formats.""" if isinstance(content, str): return content if isinstance(content, list): # List of content parts like [{"type": "text", "text": "..."}] parts = [] for part in content: if isinstance(part, str): parts.append(part) elif isinstance(part, dict): parts.append(part.get("text", part.get("value", str(part)))) else: parts.append(getattr(part, "text", getattr(part, "value", str(part)))) return "".join(parts) return str(content) def stream_chat(message, history, use_thought=True, use_search=False): """Streaming chat function - yields updated history as tokens are generated.""" if not message.strip(): yield history return model.eval() # Select system prompt based on thought toggle instruction = get_system_prompt(use_thought) # Perform web search if enabled context_injection = "" if use_search: search_results = search_web(message) context_injection = f"\n\nContext from Web Search:\n{search_results}\n\nUse this information to answer the user's question if relevant." instruction += context_injection # Build prompt with ChatML format full_prompt = f"<|im_start|>system\n{instruction}<|im_end|>\n" # Add conversation history - handle both dict and ChatMessage objects for turn in history: role = "" content = "" if isinstance(turn, dict): role = turn.get("role", "user") content = extract_text(turn.get("content", "")) else: # Gradio ChatMessage objects role = getattr(turn, "role", "user") content = extract_text(getattr(turn, "content", "")) if role and content: full_prompt += f"<|im_start|>{role}\n{content}<|im_end|>\n" # Add current user input full_prompt += f"<|im_start|>user\n{message}<|im_end|>\n<|im_start|>assistant\n" # Encode idx = tokenizer.encode(full_prompt, return_tensors="pt").to(DEVICE) generated_text = "" # Build clean history as plain dicts for output clean_history = [] for turn in history: if isinstance(turn, dict): clean_history.append({"role": turn.get("role", "user"), "content": extract_text(turn.get("content", ""))}) else: clean_history.append({"role": getattr(turn, "role", "user"), "content": extract_text(getattr(turn, "content", ""))}) # Add user message to history first new_history = clean_history + [ {"role": "user", "content": message}, {"role": "assistant", "content": ""} ] max_new_tokens = 512 temperature = 0.35 top_p = 0.75 rep_penalty = 1.25 pres_penalty = 0.3 generated_tokens = [] for _ in range(max_new_tokens): idx_cond = idx if idx.size(1) <= SEQ_LEN else idx[:, -SEQ_LEN:] with torch.no_grad(): logits = model(idx_cond) logits = logits[:, -1, :] / temperature # Repetition & Presence Penalty if generated_tokens: unique_tokens = list(set(generated_tokens)) for t_id in unique_tokens: if logits[0, t_id] > 0: logits[0, t_id] /= rep_penalty else: logits[0, t_id] *= rep_penalty logits[0, t_id] -= pres_penalty # Top-P (nucleus) sampling probs = F.softmax(logits, dim=-1) sorted_probs, sorted_indices = torch.sort(probs, descending=True) cumulative_probs = torch.cumsum(sorted_probs, dim=-1) sorted_indices_to_remove = cumulative_probs > top_p sorted_indices_to_remove[..., 1:] = sorted_indices_to_remove[..., :-1].clone() sorted_indices_to_remove[..., 0] = 0 indices_to_remove = sorted_indices_to_remove.scatter(1, sorted_indices, sorted_indices_to_remove) probs[indices_to_remove] = 0 probs = probs / probs.sum(dim=-1, keepdim=True) idx_next = torch.multinomial(probs, num_samples=1) # Decode just the new token new_token = tokenizer.decode(idx_next[0]) # Check for stop token before adding if "<|im_end" in new_token or "<|im_end|>" in generated_text + new_token: # Clean up the response final_text = generated_text for stop in ["<|im_end|>", "<|im_end", "<|im_", "<|im", "<|i", "<|"]: if stop in final_text: final_text = final_text.split(stop)[0] break new_history[-1]["content"] = final_text.strip() + "__DONE__" yield new_history return generated_text += new_token # Update assistant message and yield new_history[-1]["content"] = generated_text.strip() yield new_history idx = torch.cat((idx, idx_next), dim=1) generated_tokens.append(idx_next.item()) if idx_next.item() == tokenizer.eos_token_id: break # Final cleanup final_text = generated_text for stop in ["<|im_end|>", "<|im_end", "<|im_", "<|im", "<|i", "<|"]: if stop in final_text: final_text = final_text.split(stop)[0] break new_history[-1]["content"] = final_text.strip() + "__DONE__" yield new_history def clear_chat(): return [] # Simple Blocks interface with gr.Blocks(title="Polaris") as demo: gr.Markdown("# 🌠 Polaris") chatbot = gr.Chatbot(height=500, label="Chat") thought_toggle = gr.Checkbox(value=True, visible=False, elem_id="thought-toggle", label="Use Thought") search_toggle = gr.Checkbox(value=False, visible=False, elem_id="search-toggle", label="Use Search") with gr.Row(): msg = gr.Textbox(placeholder="Type message...", show_label=False, scale=9) clear = gr.Button("Clear", scale=1) # API endpoint with explicit name - streaming msg.submit(stream_chat, [msg, chatbot, thought_toggle, search_toggle], [chatbot], api_name="chat").then( lambda: "", None, [msg] ) clear.click(clear_chat, None, [chatbot], api_name="clear_chat") if __name__ == "__main__": demo.queue().launch()