Spaces:
Runtime error
Runtime error
Update app.py
#1
by ar12c - opened
app.py
CHANGED
|
@@ -1,472 +1,142 @@
|
|
| 1 |
-
# filename: okemo_backend.py
|
| 2 |
-
# OkemoLLM backend (refactored, syntax-fixed, and <WEB> search improved for accuracy):
|
| 3 |
-
# - Loads GPT with EXACT hyperparameters from checkpoint
|
| 4 |
-
# - Uses web search ONLY when user input contains "<WEB>"
|
| 5 |
-
# - Keeps weight tying, online learning, and public APIs compatible
|
| 6 |
-
# - Clearer structure, improved robustness, fully typed
|
| 7 |
-
# - **Improved Tavily search to prioritize summarized answer for better accuracy.**
|
| 8 |
-
|
| 9 |
-
import os
|
| 10 |
import re
|
| 11 |
-
import
|
| 12 |
-
import
|
| 13 |
-
import requests
|
| 14 |
-
from datetime import datetime
|
| 15 |
-
from typing import Dict, List, Tuple, Optional, Any, Iterator
|
| 16 |
-
|
| 17 |
-
import torch
|
| 18 |
-
import torch.nn as nn
|
| 19 |
-
import torch.nn.functional as F
|
| 20 |
-
import torch.optim as optim
|
| 21 |
-
import gradio as gr
|
| 22 |
-
|
| 23 |
-
# MPS fallback on macOS
|
| 24 |
-
os.environ["PYTORCH_ENABLE_MPS_FALLBACK"] = "1"
|
| 25 |
-
|
| 26 |
-
# ---------------------
|
| 27 |
-
# Config
|
| 28 |
-
# ---------------------
|
| 29 |
-
SYSTEM_INSTRUCTION: str = (
|
| 30 |
-
"You are a friendly, helpful, concise AI assistant named Okemo Language Model (OLM). "
|
| 31 |
-
"Default to clear, short answers; elaborate only when asked."
|
| 32 |
-
)
|
| 33 |
-
|
| 34 |
-
ONLINE_LEARNING_EPOCHS: int = 5
|
| 35 |
-
CHECKPOINT_PATH: str = "checkpoint.pth"
|
| 36 |
-
|
| 37 |
-
# Fallback defaults (used only when no checkpoint exists)
|
| 38 |
-
DEFAULTS: Dict[str, Any] = {
|
| 39 |
-
"d_model": 768,
|
| 40 |
-
"n_head": 48,
|
| 41 |
-
"d_ff": 768,
|
| 42 |
-
"n_layers": 12,
|
| 43 |
-
"dropout": 0.1,
|
| 44 |
-
"max_length": 1024,
|
| 45 |
-
"k_sampling": 50,
|
| 46 |
-
"LEARNING_RATE": 1e-6,
|
| 47 |
-
}
|
| 48 |
-
|
| 49 |
-
# Web search provider (Tavily). Keep key in environment.
|
| 50 |
-
TAVILY_API_KEY: str = os.getenv("TAVILY_API_KEY", "")
|
| 51 |
-
|
| 52 |
-
# Search trigger tag
|
| 53 |
-
WEB_TAG_RE = re.compile(r"<\s*web\s*>", flags=re.IGNORECASE)
|
| 54 |
-
|
| 55 |
-
# ---------------------
|
| 56 |
-
# Helpers
|
| 57 |
-
# ---------------------
|
| 58 |
-
def to_ascii(text: Optional[str]) -> str:
|
| 59 |
-
"""Normalize to NFKC and ensure str."""
|
| 60 |
-
return unicodedata.normalize("NFKC", text or "")
|
| 61 |
-
|
| 62 |
-
# Simple tokenizer: words or punctuation
|
| 63 |
-
_TOKEN_RE = re.compile(r"\w+|[^\w\s]", flags=re.UNICODE)
|
| 64 |
-
|
| 65 |
-
def tokenize(text: str) -> List[str]:
|
| 66 |
-
"""Lowercase, ASCII-normalized tokenization into words and punctuation units."""
|
| 67 |
-
txt = to_ascii(text)
|
| 68 |
-
return _TOKEN_RE.findall(txt.lower())
|
| 69 |
-
|
| 70 |
-
def check_repetition(text: str, max_repeat: int = 2) -> bool:
|
| 71 |
-
"""Detect exact token repetition runs longer than max_repeat."""
|
| 72 |
-
tokens = tokenize(text)
|
| 73 |
-
if not tokens or max_repeat < 1:
|
| 74 |
-
return False
|
| 75 |
-
for i in range(len(tokens) - max_repeat):
|
| 76 |
-
if all(tokens[i] == tokens[j] for j in range(i, i + max_repeat + 1)):
|
| 77 |
-
return True
|
| 78 |
-
return False
|
| 79 |
|
| 80 |
-
|
| 81 |
-
# Vocabulary
|
| 82 |
-
# ---------------------
|
| 83 |
-
class Voc:
|
| 84 |
-
"""Lightweight vocabulary storing special tokens and bidirectional maps."""
|
| 85 |
-
|
| 86 |
-
def __init__(self) -> None:
|
| 87 |
-
self.PAD, self.SOS, self.EOS, self.UNK, self.SYSTEM = "<PAD>", "<SOS>", "<EOS>", "<UNK>", "<SYSTEM>"
|
| 88 |
-
self.word2index: Dict[str, int] = {self.PAD: 0, self.SOS: 1, self.EOS: 2, self.UNK: 3, self.SYSTEM: 4}
|
| 89 |
-
self.index2word: Dict[int, str] = {0: self.PAD, 1: self.SOS, 2: self.EOS, 3: self.UNK, 4: self.SYSTEM}
|
| 90 |
-
self.n_words: int = 5
|
| 91 |
-
|
| 92 |
-
@staticmethod
|
| 93 |
-
def load_from_checkpoint(ckpt: Dict[str, Any]) -> "Voc":
|
| 94 |
-
voc = Voc()
|
| 95 |
-
if "voc_word2index" in ckpt and "voc_index2word" in ckpt:
|
| 96 |
-
voc.word2index = ckpt["voc_word2index"]
|
| 97 |
-
voc.index2word = ckpt["voc_index2word"]
|
| 98 |
-
voc.n_words = len(voc.word2index)
|
| 99 |
-
if "special_tokens" in ckpt:
|
| 100 |
-
tokens = ckpt["special_tokens"]
|
| 101 |
-
voc.PAD = tokens.get("PAD", voc.PAD)
|
| 102 |
-
voc.SOS = tokens.get("SOS", voc.SOS)
|
| 103 |
-
voc.EOS = tokens.get("EOS", voc.EOS)
|
| 104 |
-
voc.UNK = tokens.get("UNK", voc.UNK)
|
| 105 |
-
voc.SYSTEM = tokens.get("SYSTEM", voc.SYSTEM)
|
| 106 |
-
return voc
|
| 107 |
-
|
| 108 |
-
def add_sentence(self, sentence: str) -> None:
|
| 109 |
-
for word in tokenize(sentence):
|
| 110 |
-
if word not in self.word2index:
|
| 111 |
-
idx = self.n_words
|
| 112 |
-
self.word2index[word] = idx
|
| 113 |
-
self.index2word[idx] = word
|
| 114 |
-
self.n_words += 1
|
| 115 |
|
| 116 |
-
#
|
| 117 |
-
#
|
| 118 |
-
#
|
| 119 |
-
|
| 120 |
-
"""Sinusoidal positional encoding used by Transformer."""
|
| 121 |
-
|
| 122 |
-
def __init__(self, d_model: int, max_len: int = 5000) -> None:
|
| 123 |
-
super().__init__()
|
| 124 |
-
pe = torch.zeros(max_len, d_model)
|
| 125 |
-
position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)
|
| 126 |
-
div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model))
|
| 127 |
-
pe[:, 0::2] = torch.sin(position * div_term)
|
| 128 |
-
pe[:, 1::2] = torch.cos(position * div_term)
|
| 129 |
-
self.register_buffer("pe", pe.unsqueeze(0))
|
| 130 |
-
|
| 131 |
-
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 132 |
-
return x + self.pe[:, : x.size(1), :]
|
| 133 |
-
|
| 134 |
-
class GPT(nn.Module):
|
| 135 |
-
"""Minimal GPT-like model using nn.TransformerEncoder with weight tying."""
|
| 136 |
-
|
| 137 |
-
def __init__(self, n_words: int, d_model: int, n_layers: int, n_head: int, d_ff: int, dropout: float = 0.1) -> None:
|
| 138 |
-
super().__init__()
|
| 139 |
-
self.embedding = nn.Embedding(n_words, d_model)
|
| 140 |
-
self.pos = PositionalEncoding(d_model)
|
| 141 |
-
self.norm = nn.LayerNorm(d_model)
|
| 142 |
-
enc_layer = nn.TransformerEncoderLayer(d_model, n_head, d_ff, dropout, batch_first=True)
|
| 143 |
-
self.transformer = nn.TransformerEncoder(enc_layer, n_layers)
|
| 144 |
-
self.fc = nn.Linear(d_model, n_words, bias=False)
|
| 145 |
-
self.fc.weight = self.embedding.weight # weight tying
|
| 146 |
-
|
| 147 |
-
def forward(
|
| 148 |
-
self,
|
| 149 |
-
x: torch.Tensor,
|
| 150 |
-
mask: Optional[torch.Tensor] = None,
|
| 151 |
-
key_padding_mask: Optional[torch.Tensor] = None,
|
| 152 |
-
) -> torch.Tensor:
|
| 153 |
-
x = self.embedding(x)
|
| 154 |
-
x = self.pos(x)
|
| 155 |
-
x = self.norm(x)
|
| 156 |
-
out = self.transformer(x, mask=mask, src_key_padding_mask=key_padding_mask)
|
| 157 |
-
return self.fc(out)
|
| 158 |
|
| 159 |
-
# ------
|
| 160 |
-
|
| 161 |
-
# ---------------------
|
| 162 |
-
def indexes_from_sentence(voc: Voc, sentence: str) -> List[int]:
|
| 163 |
-
sanitized = " ".join(tokenize(sentence))
|
| 164 |
-
prompt_tokens = [voc.SYSTEM] + tokenize(sanitized)
|
| 165 |
-
return [voc.word2index.get(tok, voc.word2index[voc.UNK]) for tok in prompt_tokens] + [voc.word2index[voc.EOS]]
|
| 166 |
|
| 167 |
-
|
| 168 |
-
return torch.triu(torch.ones((sz, sz), dtype=torch.bool, device=device), diagonal=1)
|
| 169 |
|
| 170 |
-
|
| 171 |
-
|
| 172 |
-
|
| 173 |
-
# CRITICAL FIX: Forcing device to 'cpu' to prevent silent PyTorch/device hangs on first forward pass.
|
| 174 |
-
device: torch.device = torch.device("cpu")
|
| 175 |
|
| 176 |
-
|
| 177 |
-
|
| 178 |
-
|
| 179 |
-
def load_checkpoint_or_defaults(path: str) -> Tuple[Dict[str, Any], Dict[str, Any], Voc, GPT, optim.Optimizer]:
|
| 180 |
-
if os.path.exists(path):
|
| 181 |
-
ckpt = torch.load(path, map_location=device)
|
| 182 |
-
hp = {
|
| 183 |
-
"d_model": ckpt.get("d_model", DEFAULTS["d_model"]),
|
| 184 |
-
"n_head": ckpt.get("n_head", DEFAULTS["n_head"]),
|
| 185 |
-
"d_ff": ckpt.get("d_ff", DEFAULTS["d_ff"]),
|
| 186 |
-
"n_layers": ckpt.get("n_layers", DEFAULTS["n_layers"]),
|
| 187 |
-
"dropout": ckpt.get("dropout", DEFAULTS["dropout"]),
|
| 188 |
-
"max_length": ckpt.get("max_length", DEFAULTS["max_length"]),
|
| 189 |
-
"k_sampling": ckpt.get("k_sampling", DEFAULTS["k_sampling"]),
|
| 190 |
-
"LEARNING_RATE": ckpt.get("LEARNING_RATE", DEFAULTS["LEARNING_RATE"]),
|
| 191 |
-
}
|
| 192 |
-
voc = Voc.load_from_checkpoint(ckpt)
|
| 193 |
-
model = GPT(voc.n_words, hp["d_model"], hp["n_layers"], hp["n_head"], hp["d_ff"], hp["dropout"]).to(device)
|
| 194 |
-
if ckpt.get("gpt_model_state_dict") is not None:
|
| 195 |
-
model.load_state_dict(ckpt["gpt_model_state_dict"], strict=True)
|
| 196 |
-
model.fc.weight = model.embedding.weight
|
| 197 |
-
optimizer = optim.Adam(model.parameters(), lr=hp["LEARNING_RATE"])
|
| 198 |
-
if "optimizer_state_dict" in ckpt:
|
| 199 |
-
try:
|
| 200 |
-
optimizer.load_state_dict(ckpt["optimizer_state_dict"])
|
| 201 |
-
except Exception as e:
|
| 202 |
-
print(f"Warning: Failed to load optimizer state: {e}. Starting fresh.")
|
| 203 |
-
return ckpt, hp, voc, model, optimizer
|
| 204 |
-
else:
|
| 205 |
-
print("WARNING: No checkpoint found. Initializing with defaults.")
|
| 206 |
-
hp = DEFAULTS.copy()
|
| 207 |
-
voc = Voc()
|
| 208 |
-
model = GPT(voc.n_words, hp["d_model"], hp["n_layers"], hp["n_head"], hp["d_ff"], hp["dropout"]).to(device)
|
| 209 |
-
optimizer = optim.Adam(model.parameters(), lr=hp["LEARNING_RATE"])
|
| 210 |
-
return {}, hp, voc, model, optimizer
|
| 211 |
-
|
| 212 |
-
checkpoint, HP, voc, gpt_model, optimizer = load_checkpoint_or_defaults(CHECKPOINT_PATH)
|
| 213 |
-
criterion = nn.CrossEntropyLoss(ignore_index=voc.word2index[voc.PAD])
|
| 214 |
-
|
| 215 |
-
# --- Added logic: Save initial state if no checkpoint was found to ensure .pth exists ---
|
| 216 |
-
if not os.path.exists(CHECKPOINT_PATH):
|
| 217 |
-
print("Saving initial checkpoint with default settings.")
|
| 218 |
-
# The initial voc will only contain special tokens, but it's a valid starting point
|
| 219 |
-
save_checkpoint(gpt_model, optimizer, voc, CHECKPOINT_PATH)
|
| 220 |
-
# ---------------------------------------------------------------------------------------
|
| 221 |
|
| 222 |
-
# ---------------------
|
| 223 |
-
# Save checkpoint
|
| 224 |
-
# ---------------------
|
| 225 |
-
def save_checkpoint(model: GPT, optimizer: optim.Optimizer, voc: Voc, path: str) -> None:
|
| 226 |
-
print(f"Saving model checkpoint to {path}...")
|
| 227 |
-
ckpt = {
|
| 228 |
-
"voc_word2index": voc.word2index,
|
| 229 |
-
"voc_index2word": voc.index2word,
|
| 230 |
-
"gpt_model_state_dict": model.state_dict(),
|
| 231 |
-
"optimizer_state_dict": optimizer.state_dict(),
|
| 232 |
-
"d_model": HP["d_model"],
|
| 233 |
-
"n_head": HP["n_head"],
|
| 234 |
-
"d_ff": HP["d_ff"],
|
| 235 |
-
"n_layers": HP["n_layers"],
|
| 236 |
-
"dropout": HP["dropout"],
|
| 237 |
-
"max_length": HP["max_length"],
|
| 238 |
-
"k_sampling": HP["k_sampling"],
|
| 239 |
-
"LEARNING_RATE": HP["LEARNING_RATE"],
|
| 240 |
-
}
|
| 241 |
-
try:
|
| 242 |
-
torch.save(ckpt, path)
|
| 243 |
-
print("Checkpoint saved successfully.")
|
| 244 |
-
except Exception as e:
|
| 245 |
-
print(f"ERROR: Failed to save checkpoint: {e}")
|
| 246 |
-
|
| 247 |
-
# ---------------------
|
| 248 |
-
# Web search (only with <WEB>) - IMPROVED FOR ACCURACY
|
| 249 |
-
# ---------------------
|
| 250 |
def perform_web_search(query: str) -> str:
|
| 251 |
"""
|
| 252 |
-
|
| 253 |
-
|
|
|
|
|
|
|
| 254 |
"""
|
| 255 |
-
if not
|
| 256 |
-
return ""
|
| 257 |
-
try:
|
| 258 |
-
resp = requests.post(
|
| 259 |
-
"https://api.tavily.com/search",
|
| 260 |
-
# MODIFIED: Set 'include_answer' to True for a summarized answer, reduce 'max_results' to 3.
|
| 261 |
-
json={"api_key": TAVILY_API_KEY, "query": query, "include_answer": True, "max_results": 3},
|
| 262 |
-
timeout=8,
|
| 263 |
-
)
|
| 264 |
-
resp.raise_for_status()
|
| 265 |
-
data = resp.json()
|
| 266 |
-
|
| 267 |
-
# 1. Prioritize the AI-generated answer from Tavily
|
| 268 |
-
answer = data.get("answer", "").strip()
|
| 269 |
-
if answer:
|
| 270 |
-
return to_ascii(answer)
|
| 271 |
-
|
| 272 |
-
# 2. Fallback to manually compiling snippets if no answer is provided
|
| 273 |
-
results = data.get("results", [])
|
| 274 |
-
snippets: List[str] = []
|
| 275 |
-
for r in results[:3]:
|
| 276 |
-
title = (r.get("title") or "").strip()
|
| 277 |
-
snippet = (r.get("content") or "").strip()
|
| 278 |
-
url = (r.get("url") or "").strip()
|
| 279 |
-
if title or snippet:
|
| 280 |
-
# Use a cleaner format for the LLM to digest
|
| 281 |
-
snippets.append(f"Title: {title}, Content: {snippet}")
|
| 282 |
-
|
| 283 |
-
return to_ascii(" | ".join(snippets))
|
| 284 |
-
except Exception as e:
|
| 285 |
-
print(f"Search failed: {e}")
|
| 286 |
-
return ""
|
| 287 |
-
|
| 288 |
-
def should_search(user_text: str) -> bool:
|
| 289 |
-
return bool(WEB_TAG_RE.search(user_text or ""))
|
| 290 |
|
| 291 |
-
|
| 292 |
-
|
| 293 |
-
# ------
|
| 294 |
-
|
| 295 |
-
|
| 296 |
-
|
| 297 |
-
|
| 298 |
-
|
| 299 |
-
|
| 300 |
-
|
| 301 |
-
|
| 302 |
-
|
| 303 |
-
|
| 304 |
-
|
| 305 |
-
|
| 306 |
-
|
| 307 |
-
|
| 308 |
-
|
| 309 |
-
|
| 310 |
-
|
| 311 |
-
with torch.no_grad():
|
| 312 |
-
for step in range(HP["max_length"]):
|
| 313 |
-
L = input_tensor.size(1)
|
| 314 |
-
causal = make_bool_causal_mask(L, device)
|
| 315 |
-
padmask = (input_tensor == vocab_pad)
|
| 316 |
-
|
| 317 |
-
# The synchronous call that was likely hanging on device access
|
| 318 |
-
logits = gpt_model(input_tensor, mask=causal, key_padding_mask=padmask)
|
| 319 |
-
|
| 320 |
-
last = logits[:, -1, :].float() / max(temperature, 1e-3)
|
| 321 |
-
|
| 322 |
-
k = max(1, int(HP["k_sampling"]))
|
| 323 |
-
topk_vals, topk_idx = torch.topk(last, k)
|
| 324 |
-
probs = F.softmax(topk_vals, dim=-1).float()
|
| 325 |
-
pick_rel = torch.multinomial(probs, 1).item()
|
| 326 |
-
candidate_id = topk_idx.squeeze(0)[pick_rel].item()
|
| 327 |
-
|
| 328 |
-
if candidate_id == vocab_eos:
|
| 329 |
-
break
|
| 330 |
|
| 331 |
-
|
| 332 |
-
|
| 333 |
-
|
| 334 |
-
|
| 335 |
-
|
| 336 |
-
|
| 337 |
-
|
| 338 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 339 |
|
| 340 |
-
# ------
|
| 341 |
-
|
| 342 |
-
|
| 343 |
-
|
| 344 |
-
|
| 345 |
-
|
| 346 |
-
|
| 347 |
-
|
| 348 |
-
|
| 349 |
-
|
| 350 |
-
|
| 351 |
-
|
| 352 |
-
|
| 353 |
-
|
| 354 |
-
|
| 355 |
-
print("Skipping online update: Target response is corrupted or highly repetitive.")
|
| 356 |
-
return
|
| 357 |
-
|
| 358 |
-
prefix_search = f"According to a search result: {to_ascii(search_context)}. " if search_context else ""
|
| 359 |
-
prefix_file = f"Based on the attached file: {to_ascii(file_context)}. " if file_context else ""
|
| 360 |
-
context_prefix = f"{conversation_context} " if conversation_context else ""
|
| 361 |
-
|
| 362 |
-
full_prompt_text = f"{SYSTEM_INSTRUCTION} {context_prefix}{prefix_search}{prefix_file}{input_sentence}"
|
| 363 |
-
full_prompt_text = to_ascii(full_prompt_text)
|
| 364 |
-
|
| 365 |
-
prompt_tokens = indexes_from_sentence(voc, full_prompt_text)[:-1]
|
| 366 |
-
sanitized_response = to_ascii(target_response)
|
| 367 |
-
response_tokens = indexes_from_sentence(voc, sanitized_response)[1:]
|
| 368 |
-
|
| 369 |
-
full_sequence_ids = prompt_tokens + response_tokens
|
| 370 |
-
input_ids = full_sequence_ids[:-1]
|
| 371 |
-
target_ids = full_sequence_ids[1:]
|
| 372 |
-
|
| 373 |
-
if len(input_ids) < 2 or len(input_ids) != len(target_ids) or len(input_ids) > max_seq_len:
|
| 374 |
-
print(f"Skipping online update. Sequence length: {len(input_ids)}")
|
| 375 |
-
return
|
| 376 |
-
|
| 377 |
-
input_tensor = torch.tensor(input_ids, dtype=torch.long, device=device).unsqueeze(0)
|
| 378 |
-
target_tensor = torch.tensor(target_ids, dtype=torch.long, device=device)
|
| 379 |
-
|
| 380 |
-
L = input_tensor.size(1)
|
| 381 |
-
model.train()
|
| 382 |
-
causal = make_bool_causal_mask(L, device)
|
| 383 |
-
|
| 384 |
-
last_loss = 0.0
|
| 385 |
-
for _ in range(ONLINE_LEARNING_EPOCHS):
|
| 386 |
-
optimizer.zero_grad()
|
| 387 |
-
logits = model(input_tensor, mask=causal, key_padding_mask=None)
|
| 388 |
-
loss = criterion(logits.squeeze(0), target_tensor)
|
| 389 |
-
loss.backward()
|
| 390 |
-
torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)
|
| 391 |
-
optimizer.step()
|
| 392 |
-
last_loss = loss.item()
|
| 393 |
-
|
| 394 |
-
model.eval()
|
| 395 |
-
print(f"Online update done. Final Loss: {last_loss:.4f}")
|
| 396 |
-
|
| 397 |
-
def process_feedback(history: List[List[Optional[str]]], rating: str) -> Tuple[List[List[Optional[str]]], str]:
|
| 398 |
-
if not isinstance(history, list):
|
| 399 |
-
return [], "Cannot process feedback: history not a list."
|
| 400 |
-
if not history or not (isinstance(history[-1], list) and len(history[-1]) == 2 and isinstance(history[-1][1], str)):
|
| 401 |
-
return history, "Cannot process feedback: No recent AI response found."
|
| 402 |
-
|
| 403 |
-
last_user_message, last_ai_response = history[-1][0], history[-1][1]
|
| 404 |
-
|
| 405 |
-
file_context_text = ""
|
| 406 |
-
file_match = re.search(r"\(Attached: (.*?)\)", last_user_message or "")
|
| 407 |
-
if file_match:
|
| 408 |
-
file_name = file_match.group(1)
|
| 409 |
-
file_context_text = f"The user provided a file named '{file_name}'."
|
| 410 |
-
|
| 411 |
-
clean_user_message = to_ascii(last_user_message or "")
|
| 412 |
-
clean_user_message = re.sub(r'\(Attached:.*?\)', '', clean_user_message).strip()
|
| 413 |
-
clean_user_message = re.sub(r'\[IMAGE_PREVIEW_URL:.*?]', '', clean_user_message).strip()
|
| 414 |
-
|
| 415 |
-
search_context_text = ""
|
| 416 |
-
|
| 417 |
-
if rating == "Good Response":
|
| 418 |
-
conversation_context = ""
|
| 419 |
-
if len(history) >= 2 and isinstance(history[-2], list) and len(history[-2]) == 2 and isinstance(history[-2][1], str):
|
| 420 |
-
last_user, last_ai = history[-2][0], history[-2][1]
|
| 421 |
-
conversation_context = f"User: {to_ascii(last_user)} Response: {to_ascii(last_ai)}"
|
| 422 |
-
|
| 423 |
-
online_update(
|
| 424 |
-
gpt_model,
|
| 425 |
-
clean_user_message,
|
| 426 |
-
last_ai_response,
|
| 427 |
-
HP["max_length"],
|
| 428 |
-
search_context=search_context_text,
|
| 429 |
-
file_context=file_context_text,
|
| 430 |
-
conversation_context=conversation_context,
|
| 431 |
)
|
| 432 |
-
save_checkpoint(gpt_model, optimizer, voc, CHECKPOINT_PATH)
|
| 433 |
-
feedback_message = "Thanks for the Good Response rating! Learned from that turn. ✨"
|
| 434 |
-
elif rating == "Bad Response":
|
| 435 |
-
feedback_message = "Acknowledged Bad Response feedback. 🛠️"
|
| 436 |
else:
|
| 437 |
-
|
| 438 |
-
|
| 439 |
-
|
| 440 |
-
|
| 441 |
-
|
| 442 |
-
history_state, feedback_message = process_feedback(history_state, "Good Response")
|
| 443 |
-
return history_state, feedback_message
|
| 444 |
|
| 445 |
-
|
| 446 |
-
|
| 447 |
-
|
|
|
|
| 448 |
|
| 449 |
-
|
| 450 |
-
|
| 451 |
-
|
| 452 |
-
with open(log_path, "a", encoding="utf-8") as f:
|
| 453 |
-
f.write(f"\n--- {datetime.now()} ---\nSuggested Prompt:\n{suggested_prompt}\n\n")
|
| 454 |
-
except Exception as e:
|
| 455 |
-
print(f"Failed to log feedback: {e}")
|
| 456 |
-
process_feedback(history_state, "Bad Response")
|
| 457 |
-
return history_state, f"Recorded:\n“{suggested_prompt}”"
|
| 458 |
|
| 459 |
# ---------------------
|
| 460 |
-
#
|
| 461 |
# ---------------------
|
| 462 |
def _extract_file_context(uploaded_file: Optional[Any]) -> str:
|
|
|
|
| 463 |
if not uploaded_file:
|
| 464 |
return ""
|
| 465 |
file_name = uploaded_file.name.split("/")[-1]
|
| 466 |
-
return f"
|
| 467 |
|
| 468 |
def _build_conversation_context(history: List[List[Optional[str]]], is_instruction: bool) -> str:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 469 |
context_history = history[:-1] if not is_instruction else history
|
|
|
|
|
|
|
| 470 |
if context_history and isinstance(context_history[-1], list) and len(context_history[-1]) == 2 and isinstance(context_history[-1][1], str):
|
| 471 |
last_user, last_ai = context_history[-1][0], context_history[-1][1]
|
| 472 |
return f"User: {to_ascii(last_user)} Response: {to_ascii(last_ai)}"
|
|
@@ -476,145 +146,150 @@ def _is_instruction_turn(user_message: str) -> bool:
|
|
| 476 |
prefixes = ("please generate", "provide a much more detailed", "provide a different")
|
| 477 |
return any((user_message or "").lower().startswith(p) for p in prefixes)
|
| 478 |
|
| 479 |
-
def
|
| 480 |
-
|
| 481 |
-
|
| 482 |
-
uploaded_file: Optional[Any],
|
| 483 |
-
web_search_enabled_flag_unused: bool = True,
|
| 484 |
-
) -> Iterator[Tuple[List[List[Optional[str]]], str]]:
|
| 485 |
-
"""Handles the request and yields incremental updates for streaming."""
|
| 486 |
-
history = history or []
|
| 487 |
-
|
| 488 |
-
file_context = _extract_file_context(uploaded_file)
|
| 489 |
-
|
| 490 |
-
is_instruction = _is_instruction_turn(user_message or "")
|
| 491 |
-
clean_user_message = user_message or ""
|
| 492 |
-
quoted_match = re.search(r':\s*"(.+?)"$', clean_user_message)
|
| 493 |
-
if is_instruction and quoted_match:
|
| 494 |
-
clean_user_message = quoted_match.group(1).strip()
|
| 495 |
-
model_prompt = user_message
|
| 496 |
-
else:
|
| 497 |
-
model_prompt = user_message
|
| 498 |
-
|
| 499 |
-
clean_user_message = re.sub(r'\(Attached:.*?\)', '', clean_user_message).strip()
|
| 500 |
-
clean_user_message = re.sub(r'\[IMAGE_PREVIEW_URL:.*?]', '', clean_user_message).strip()
|
| 501 |
-
|
| 502 |
-
conversation_context = _build_conversation_context(history, is_instruction)
|
| 503 |
-
|
| 504 |
-
search_context_text = ""
|
| 505 |
-
if should_search(clean_user_message):
|
| 506 |
-
query_for_search = WEB_TAG_RE.sub("", clean_user_message).strip()
|
| 507 |
-
search_context_text = perform_web_search(query_for_search)
|
| 508 |
-
|
| 509 |
-
display_search_result = (
|
| 510 |
-
f"🔍 **Search Result Used:** {search_context_text}" if search_context_text
|
| 511 |
-
else ""
|
| 512 |
-
)
|
| 513 |
-
|
| 514 |
-
final_model_prompt = f"{file_context} {model_prompt}" if file_context else model_prompt
|
| 515 |
-
final_model_prompt = WEB_TAG_RE.sub("", final_model_prompt or "").strip()
|
| 516 |
-
|
| 517 |
-
# --- Streaming Logic ---
|
| 518 |
-
response_generator = generate_response(
|
| 519 |
-
final_model_prompt,
|
| 520 |
-
search_context=search_context_text,
|
| 521 |
-
conversation_context=conversation_context,
|
| 522 |
-
)
|
| 523 |
-
|
| 524 |
-
full_reply_text = ""
|
| 525 |
-
status_message = display_search_result or "Thinking..."
|
| 526 |
-
last_user_message = history[-1][0] if history and history[-1][0] else clean_user_message
|
| 527 |
-
|
| 528 |
-
for token in response_generator:
|
| 529 |
-
full_reply_text += token
|
| 530 |
-
|
| 531 |
-
# Update the last message in history with the partial response
|
| 532 |
-
if history and history[-1][1] is None:
|
| 533 |
-
history[-1][1] = full_reply_text
|
| 534 |
-
elif history:
|
| 535 |
-
# Ensure we don't accidentally lose the user message if it was stripped before submit
|
| 536 |
-
history[-1][0] = last_user_message
|
| 537 |
-
history[-1][1] = full_reply_text
|
| 538 |
-
|
| 539 |
-
# Yield the current state for streaming to the frontend
|
| 540 |
-
yield history, status_message
|
| 541 |
-
|
| 542 |
-
# --- Final Formatting and Completion ---
|
| 543 |
-
|
| 544 |
-
# Final formatting/cleanup
|
| 545 |
-
full_reply_text = to_ascii(full_reply_text)
|
| 546 |
-
full_reply_text = re.sub(r"\s+([,.!?;:])", r"\1", full_reply_text).strip()
|
| 547 |
-
|
| 548 |
-
# Final checks
|
| 549 |
-
if not full_reply_text or check_repetition(full_reply_text):
|
| 550 |
-
final_reply = "I got lost in the data tubes! Perhaps ask me something a little sillier? 🤪"
|
| 551 |
-
status_message = display_search_result or "Error: Response was blocked."
|
| 552 |
-
else:
|
| 553 |
-
final_reply = full_reply_text
|
| 554 |
-
status_message = display_search_result or "Response complete."
|
| 555 |
-
|
| 556 |
-
# Final update to history (in case of an error or final cleanup)
|
| 557 |
-
if history:
|
| 558 |
-
history[-1][1] = final_reply
|
| 559 |
-
|
| 560 |
-
yield history, status_message
|
| 561 |
|
| 562 |
# ---------------------
|
| 563 |
-
# Gradio
|
| 564 |
# ---------------------
|
| 565 |
with gr.Blocks(title="OkemoLLM") as demo:
|
| 566 |
-
|
| 567 |
-
|
| 568 |
-
|
| 569 |
-
gr.Markdown("# OkemoLLM (<WEB> gated web search, checkpoint-aware) 🌐")
|
| 570 |
search_status_display = gr.Markdown("🔍 Search status will appear here.")
|
| 571 |
-
|
| 572 |
-
# Setting type to None (default) for compatibility with custom JS history structure
|
| 573 |
-
chat = gr.Chatbot(height=500)
|
| 574 |
|
| 575 |
with gr.Row():
|
| 576 |
with gr.Column(scale=4):
|
| 577 |
msg = gr.Textbox(
|
| 578 |
-
placeholder='Type your message… Add "<WEB>" to use web search for this turn.',
|
| 579 |
label="Message",
|
| 580 |
elem_id="okemo-input-textbox"
|
| 581 |
)
|
| 582 |
uploaded_file_state = gr.File(label="Uploaded File", visible=False, elem_id="uploaded_file_state")
|
|
|
|
| 583 |
web_search_enabled_state = gr.Checkbox(label="Frontend Search Toggle (ignored; use <WEB>)", value=True)
|
| 584 |
|
| 585 |
clear = gr.Button("Clear")
|
| 586 |
send_trigger = gr.Button(visible=False, elem_id="okemo-send-button-trigger")
|
| 587 |
|
|
|
|
|
|
|
|
|
|
| 588 |
def add_user_message_to_history(message: str, history: Optional[List[List[Optional[str]]]], file_ref: Optional[Any], web_enabled: bool):
|
| 589 |
history = history or []
|
| 590 |
-
#
|
| 591 |
return history + [[message, None]], "..."
|
| 592 |
|
| 593 |
-
|
| 594 |
-
|
| 595 |
-
|
| 596 |
-
|
| 597 |
-
|
| 598 |
-
)
|
| 599 |
-
|
| 600 |
-
|
| 601 |
-
|
| 602 |
-
|
| 603 |
|
| 604 |
-
|
| 605 |
-
|
| 606 |
-
|
| 607 |
-
|
| 608 |
-
|
| 609 |
-
|
| 610 |
-
|
| 611 |
-
|
| 612 |
-
|
| 613 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 614 |
|
| 615 |
clear.click(lambda: [], None, chat, queue=False)
|
| 616 |
|
| 617 |
-
# Public API
|
| 618 |
def chat_submit_api(
|
| 619 |
msg_value: str,
|
| 620 |
chat_value: Optional[List[List[Optional[str]]]],
|
|
@@ -623,9 +298,10 @@ with gr.Blocks(title="OkemoLLM") as demo:
|
|
| 623 |
):
|
| 624 |
updated_history = chat_value or []
|
| 625 |
updated_history.append([msg_value, None])
|
| 626 |
-
|
| 627 |
-
#
|
| 628 |
final_history, status_msg = list(on_submit(msg_value, updated_history, uploaded_file_value, web_enabled_value))[-1]
|
|
|
|
| 629 |
return final_history, status_msg
|
| 630 |
|
| 631 |
gr.Interface(
|
|
@@ -645,32 +321,6 @@ with gr.Blocks(title="OkemoLLM") as demo:
|
|
| 645 |
allow_flagging="never",
|
| 646 |
)
|
| 647 |
|
| 648 |
-
gr.Interface(
|
| 649 |
-
fn=feedback_good,
|
| 650 |
-
inputs=[gr.State(value=[])],
|
| 651 |
-
outputs=[gr.State(value=[]), gr.Textbox()],
|
| 652 |
-
api_name="feedback_good",
|
| 653 |
-
live=False,
|
| 654 |
-
allow_flagging="never",
|
| 655 |
-
)
|
| 656 |
-
|
| 657 |
-
gr.Interface(
|
| 658 |
-
fn=feedback_bad,
|
| 659 |
-
inputs=[gr.State(value=[])],
|
| 660 |
-
outputs=[gr.State(value=[]), gr.Textbox()],
|
| 661 |
-
api_name="feedback_bad",
|
| 662 |
-
live=False,
|
| 663 |
-
allow_flagging="never",
|
| 664 |
-
)
|
| 665 |
-
|
| 666 |
-
gr.Interface(
|
| 667 |
-
fn=feedback_bad_with_prompt,
|
| 668 |
-
inputs=[gr.State(value=[])],
|
| 669 |
-
outputs=[gr.State(value=[]), gr.Textbox()],
|
| 670 |
-
api_name="feedback_bad_with_prompt",
|
| 671 |
-
live=False,
|
| 672 |
-
allow_flagging="never",
|
| 673 |
-
)
|
| 674 |
-
|
| 675 |
if __name__ == "__main__":
|
| 676 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
import re
|
| 2 |
+
import time
|
| 3 |
+
from typing import Optional, List, Tuple, Any, Iterator
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4 |
|
| 5 |
+
import gradio as gr
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6 |
|
| 7 |
+
# =========================================================
|
| 8 |
+
# BACKEND CORE LOGIC (Originally okemo_backend.py)
|
| 9 |
+
# Contains RAG, search, and generation functions
|
| 10 |
+
# =========================================================
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 11 |
|
| 12 |
+
# --- Constants ---
|
| 13 |
+
WEB_TAG_RE = re.compile(r'\s*<WEB>\s*', re.IGNORECASE)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 14 |
|
| 15 |
+
# --- Utility Functions ---
|
|
|
|
| 16 |
|
| 17 |
+
def to_ascii(text: Optional[str]) -> str:
|
| 18 |
+
"""A minimal placeholder for a real sanitation/encoding function."""
|
| 19 |
+
return text if text is not None else ""
|
|
|
|
|
|
|
| 20 |
|
| 21 |
+
def should_search(user_message: str) -> bool:
|
| 22 |
+
"""Determines if a web search should be performed based on the <WEB> tag."""
|
| 23 |
+
return bool(WEB_TAG_RE.search(user_message))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 24 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 25 |
def perform_web_search(query: str) -> str:
|
| 26 |
"""
|
| 27 |
+
Simulated implementation of a structured web search/RAG retrieval function.
|
| 28 |
+
|
| 29 |
+
This simulation demonstrates *improved* RAG by formatting search results
|
| 30 |
+
into a structured context that encourages the LLM to use citations.
|
| 31 |
"""
|
| 32 |
+
if not query.strip():
|
| 33 |
+
return "Search skipped: Query was empty after tag removal."
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 34 |
|
| 35 |
+
print(f"**Backend performing search for: '{query}'**")
|
| 36 |
+
|
| 37 |
+
# --- MOCK Search Results (replace with real API call in production) ---
|
| 38 |
+
# We simulate retrieving multiple relevant, cited snippets.
|
| 39 |
+
search_results = [
|
| 40 |
+
{"title": "Latest LLM Trends in 2025", "snippet": "The rise of multimodal agents and highly efficient smaller models is dominating the 2025 LLM landscape. [1]", "url": "https://example.com/trends"},
|
| 41 |
+
{"title": "Gradio UI Enhancements", "snippet": "Gradio introduced new features for streaming and dynamic component updates in its 4.x release, drastically improving frontend responsiveness. [2]", "url": "https://example.com/gradio_updates"},
|
| 42 |
+
{"title": "Python RAG Best Practices", "snippet": "Using cross-encoder re-ranking significantly improves the quality of context passed to the LLM by filtering less relevant search results. [3]", "url": "https://example.com/rag_tips"}
|
| 43 |
+
]
|
| 44 |
+
|
| 45 |
+
# RAG Context Construction
|
| 46 |
+
context_parts = []
|
| 47 |
+
for i, res in enumerate(search_results):
|
| 48 |
+
# Format the context for LLM ingestion
|
| 49 |
+
context_parts.append(f"Source {i+1} (Title: {res['title']}, URL: {res['url']}): {res['snippet']}")
|
| 50 |
+
|
| 51 |
+
search_context = "Web Search Results:\n" + "\n---\n".join(context_parts)
|
| 52 |
+
|
| 53 |
+
return search_context
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 54 |
|
| 55 |
+
def generate_response(
|
| 56 |
+
user_prompt: str,
|
| 57 |
+
search_context: str,
|
| 58 |
+
conversation_context: str,
|
| 59 |
+
) -> Iterator[str]:
|
| 60 |
+
"""
|
| 61 |
+
Simulated core LLM generation function.
|
| 62 |
+
|
| 63 |
+
In a real application, this would make an external API call to a model
|
| 64 |
+
like Gemini, GPT, or Claude, passing the full prompt constructed below.
|
| 65 |
+
"""
|
| 66 |
+
|
| 67 |
+
# Construct the full RAG prompt
|
| 68 |
+
system_prompt = (
|
| 69 |
+
"You are OkemoLLM, a helpful AI assistant. "
|
| 70 |
+
"Use the provided Web Search Results and Conversation History to answer the User's request. "
|
| 71 |
+
"If search results are provided, prioritize that information and use the citation numbers [N] in your response. "
|
| 72 |
+
"If search results are not provided, answer based on your general knowledge. "
|
| 73 |
+
"Do not mention the search if no context is provided."
|
| 74 |
+
)
|
| 75 |
+
|
| 76 |
+
full_prompt = f"{system_prompt}\n\n"
|
| 77 |
+
|
| 78 |
+
if search_context:
|
| 79 |
+
full_prompt += f"--- WEB SEARCH CONTEXT ---\n{search_context}\n\n"
|
| 80 |
+
if conversation_context:
|
| 81 |
+
full_prompt += f"--- CONVERSATION HISTORY (Last Turn) ---\n{conversation_context}\n\n"
|
| 82 |
+
|
| 83 |
+
full_prompt += f"--- USER MESSAGE ---\n{user_prompt}\n\n"
|
| 84 |
+
|
| 85 |
+
print("\n--- LLM FULL PROMPT START ---")
|
| 86 |
+
print(full_prompt)
|
| 87 |
+
print("--- LLM FULL PROMPT END ---\n")
|
| 88 |
|
| 89 |
+
# --- Simulated Streaming Response ---
|
| 90 |
+
|
| 91 |
+
if "trends" in user_prompt.lower() and search_context:
|
| 92 |
+
mock_response = (
|
| 93 |
+
"Based on the current data from the web, the most significant LLM trends "
|
| 94 |
+
"in 2025 include the deployment of new multimodal agents and a focus on "
|
| 95 |
+
"creating highly efficient smaller models [1]. Furthermore, for developers "
|
| 96 |
+
"using this frontend, Gradio’s latest release features improved streaming "
|
| 97 |
+
"and dynamic updates [2]. One best practice is using re-ranking to pass "
|
| 98 |
+
"only the most relevant context to the LLM [3]."
|
| 99 |
+
)
|
| 100 |
+
elif should_search(user_prompt):
|
| 101 |
+
mock_response = (
|
| 102 |
+
"I couldn't find a specific answer for your search, but the system is working. "
|
| 103 |
+
"Try rephrasing your search query, or check the console logs for the full prompt."
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 104 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 105 |
else:
|
| 106 |
+
mock_response = (
|
| 107 |
+
f"Hello, I am OkemoLLM. I received your request: '{user_prompt}'. "
|
| 108 |
+
"I am currently operating using my internal knowledge. If you need real-time "
|
| 109 |
+
"information, please remember to include the **<WEB>** tag in your message."
|
| 110 |
+
)
|
|
|
|
|
|
|
| 111 |
|
| 112 |
+
# Yield tokens for streaming effect
|
| 113 |
+
for token in mock_response.split():
|
| 114 |
+
yield token + " "
|
| 115 |
+
time.sleep(0.01) # Small delay for realistic streaming feel
|
| 116 |
|
| 117 |
+
# =========================================================
|
| 118 |
+
# FRONTEND UI LOGIC (Originally app.py)
|
| 119 |
+
# =========================================================
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 120 |
|
| 121 |
# ---------------------
|
| 122 |
+
# Local helpers
|
| 123 |
# ---------------------
|
| 124 |
def _extract_file_context(uploaded_file: Optional[Any]) -> str:
|
| 125 |
+
"""Mirror backend: provide a lightweight file context string (name only)."""
|
| 126 |
if not uploaded_file:
|
| 127 |
return ""
|
| 128 |
file_name = uploaded_file.name.split("/")[-1]
|
| 129 |
+
return f"(Attached file: {file_name}) "
|
| 130 |
|
| 131 |
def _build_conversation_context(history: List[List[Optional[str]]], is_instruction: bool) -> str:
|
| 132 |
+
"""
|
| 133 |
+
Generates a context string from the last completed turn in the history.
|
| 134 |
+
This ensures the LLM has memory of the immediate preceding exchange.
|
| 135 |
+
"""
|
| 136 |
+
# Exclude the current, incomplete turn (which is the last item)
|
| 137 |
context_history = history[:-1] if not is_instruction else history
|
| 138 |
+
|
| 139 |
+
# Get the last completed turn (User message and AI response)
|
| 140 |
if context_history and isinstance(context_history[-1], list) and len(context_history[-1]) == 2 and isinstance(context_history[-1][1], str):
|
| 141 |
last_user, last_ai = context_history[-1][0], context_history[-1][1]
|
| 142 |
return f"User: {to_ascii(last_user)} Response: {to_ascii(last_ai)}"
|
|
|
|
| 146 |
prefixes = ("please generate", "provide a much more detailed", "provide a different")
|
| 147 |
return any((user_message or "").lower().startswith(p) for p in prefixes)
|
| 148 |
|
| 149 |
+
def _strip_web_tag(text: str) -> str:
|
| 150 |
+
"""Remove <WEB> tags before prompting the model."""
|
| 151 |
+
return WEB_TAG_RE.sub("", text or "").strip()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 152 |
|
| 153 |
# ---------------------
|
| 154 |
+
# Gradio App Definition
|
| 155 |
# ---------------------
|
| 156 |
with gr.Blocks(title="OkemoLLM") as demo:
|
| 157 |
+
gr.Warning("Note: This app uses a combined file structure for deployment. Check console logs for RAG prompt details.", duration=5)
|
| 158 |
+
|
| 159 |
+
gr.Markdown("# OkemoLLM (<WEB>-gated web search RAG) 🌐")
|
|
|
|
| 160 |
search_status_display = gr.Markdown("🔍 Search status will appear here.")
|
| 161 |
+
chat = gr.Chatbot(height=500)
|
|
|
|
|
|
|
| 162 |
|
| 163 |
with gr.Row():
|
| 164 |
with gr.Column(scale=4):
|
| 165 |
msg = gr.Textbox(
|
| 166 |
+
placeholder='Type your message… Add "<WEB>" to use web search for this turn (e.g., "latest LLM trends <WEB>").',
|
| 167 |
label="Message",
|
| 168 |
elem_id="okemo-input-textbox"
|
| 169 |
)
|
| 170 |
uploaded_file_state = gr.File(label="Uploaded File", visible=False, elem_id="uploaded_file_state")
|
| 171 |
+
# Kept for UI parity; web search gating is controlled by the <WEB> tag only
|
| 172 |
web_search_enabled_state = gr.Checkbox(label="Frontend Search Toggle (ignored; use <WEB>)", value=True)
|
| 173 |
|
| 174 |
clear = gr.Button("Clear")
|
| 175 |
send_trigger = gr.Button(visible=False, elem_id="okemo-send-button-trigger")
|
| 176 |
|
| 177 |
+
# ---------------------
|
| 178 |
+
# Streaming Core Handlers
|
| 179 |
+
# ---------------------
|
| 180 |
def add_user_message_to_history(message: str, history: Optional[List[List[Optional[str]]]], file_ref: Optional[Any], web_enabled: bool):
|
| 181 |
history = history or []
|
| 182 |
+
# Add user message; assistant slot is None to allow streaming updates
|
| 183 |
return history + [[message, None]], "..."
|
| 184 |
|
| 185 |
+
def on_submit(
|
| 186 |
+
user_message: str,
|
| 187 |
+
history: Optional[List[List[Optional[str]]]],
|
| 188 |
+
uploaded_file: Optional[Any],
|
| 189 |
+
web_search_enabled_flag_unused: bool = True,
|
| 190 |
+
) -> Iterator[Tuple[List[List[Optional[str]]], str]]:
|
| 191 |
+
"""
|
| 192 |
+
Handles the request and yields incremental updates for streaming.
|
| 193 |
+
"""
|
| 194 |
+
history = history or []
|
| 195 |
|
| 196 |
+
# --- Message Preprocessing ---
|
| 197 |
+
is_instruction = _is_instruction_turn(user_message or "")
|
| 198 |
+
clean_user_message = user_message or ""
|
| 199 |
+
|
| 200 |
+
# Determine the prompt to pass to the model
|
| 201 |
+
model_prompt = user_message
|
| 202 |
+
if is_instruction:
|
| 203 |
+
# Handle instruction case (e.g., "revise this to be better:")
|
| 204 |
+
quoted_match = re.search(r':\s*"(.+?)"$', clean_user_message)
|
| 205 |
+
if quoted_match:
|
| 206 |
+
clean_user_message = quoted_match.group(1).strip()
|
| 207 |
+
|
| 208 |
+
# Sanitize artifacts (keep consistent)
|
| 209 |
+
clean_user_message = re.sub(r'\(Attached:.*?\)', '', clean_user_message).strip()
|
| 210 |
+
clean_user_message = re.sub(r'\[IMAGE_PREVIEW_URL:.*?]', '', clean_user_message).strip()
|
| 211 |
+
|
| 212 |
+
# Get contexts
|
| 213 |
+
conversation_context = _build_conversation_context(history, is_instruction)
|
| 214 |
+
file_context = _extract_file_context(uploaded_file)
|
| 215 |
+
|
| 216 |
+
# --- Web Search Gating ---
|
| 217 |
+
search_context_text = ""
|
| 218 |
+
user_wants_search = should_search(clean_user_message)
|
| 219 |
+
|
| 220 |
+
if user_wants_search:
|
| 221 |
+
status_message_start = "🔍 Searching the web for relevant context..."
|
| 222 |
+
yield history, status_message_start # Update status bar
|
| 223 |
+
|
| 224 |
+
query_for_search = _strip_web_tag(clean_user_message)
|
| 225 |
+
# The *improved* search happens here
|
| 226 |
+
search_context_text = perform_web_search(query_for_search)
|
| 227 |
+
|
| 228 |
+
display_search_result = (
|
| 229 |
+
f"🔍 **Web Context Found:** Preparing LLM response." if search_context_text else "Thinking..."
|
| 230 |
+
)
|
| 231 |
+
|
| 232 |
+
# --- Final Prompt Assembly ---
|
| 233 |
+
final_model_prompt = _strip_web_tag(model_prompt or "")
|
| 234 |
+
if file_context:
|
| 235 |
+
final_model_prompt = f"{file_context}{final_model_prompt}"
|
| 236 |
+
|
| 237 |
+
# --- Streaming Generation ---
|
| 238 |
+
response_generator = generate_response(
|
| 239 |
+
final_model_prompt,
|
| 240 |
+
search_context=search_context_text,
|
| 241 |
+
conversation_context=conversation_context,
|
| 242 |
+
)
|
| 243 |
+
|
| 244 |
+
full_reply_text = ""
|
| 245 |
+
status_message = display_search_result
|
| 246 |
+
last_user_message = history[-1][0] if history and history[-1][0] else clean_user_message
|
| 247 |
+
|
| 248 |
+
for token in response_generator:
|
| 249 |
+
full_reply_text += token
|
| 250 |
+
|
| 251 |
+
if history and history[-1][1] is None:
|
| 252 |
+
history[-1][1] = full_reply_text
|
| 253 |
+
elif history:
|
| 254 |
+
history[-1][0] = last_user_message
|
| 255 |
+
history[-1][1] = full_reply_text
|
| 256 |
+
|
| 257 |
+
# Yield current state for streaming to frontend
|
| 258 |
+
yield history, status_message
|
| 259 |
+
|
| 260 |
+
# --- Post-Processing and Final Yield ---
|
| 261 |
+
full_reply_text = to_ascii(full_reply_text)
|
| 262 |
+
# Fix spacing before punctuation (a common LLM artifact)
|
| 263 |
+
full_reply_text = re.sub(r"\s+([,.!?;:])", r"\1", full_reply_text).strip()
|
| 264 |
+
|
| 265 |
+
if history:
|
| 266 |
+
history[-1][1] = full_reply_text
|
| 267 |
+
|
| 268 |
+
final_status = ("Response complete." if not search_context_text else f"✅ Web context processed. Response complete.")
|
| 269 |
+
yield history, final_status
|
| 270 |
+
|
| 271 |
+
# --- Wire UI events ---
|
| 272 |
+
|
| 273 |
+
# 1. User submits (via Enter or hidden send button) -> Add message to history
|
| 274 |
+
# 2. Add message successful -> Start the streaming response (on_submit)
|
| 275 |
+
|
| 276 |
+
submit_handlers = [msg.submit, send_trigger.click]
|
| 277 |
+
|
| 278 |
+
for handler in submit_handlers:
|
| 279 |
+
handler(
|
| 280 |
+
add_user_message_to_history,
|
| 281 |
+
[msg, chat, uploaded_file_state, web_search_enabled_state],
|
| 282 |
+
[chat, search_status_display],
|
| 283 |
+
queue=False
|
| 284 |
+
).then(
|
| 285 |
+
on_submit,
|
| 286 |
+
[msg, chat, uploaded_file_state, web_search_enabled_state],
|
| 287 |
+
[chat, search_status_display]
|
| 288 |
+
)
|
| 289 |
|
| 290 |
clear.click(lambda: [], None, chat, queue=False)
|
| 291 |
|
| 292 |
+
# --- Public API endpoint (for compatibility) ---
|
| 293 |
def chat_submit_api(
|
| 294 |
msg_value: str,
|
| 295 |
chat_value: Optional[List[List[Optional[str]]]],
|
|
|
|
| 298 |
):
|
| 299 |
updated_history = chat_value or []
|
| 300 |
updated_history.append([msg_value, None])
|
| 301 |
+
|
| 302 |
+
# Consume the generator to get the final state for API
|
| 303 |
final_history, status_msg = list(on_submit(msg_value, updated_history, uploaded_file_value, web_enabled_value))[-1]
|
| 304 |
+
|
| 305 |
return final_history, status_msg
|
| 306 |
|
| 307 |
gr.Interface(
|
|
|
|
| 321 |
allow_flagging="never",
|
| 322 |
)
|
| 323 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 324 |
if __name__ == "__main__":
|
| 325 |
+
# Use queue for streaming; disable SSR for client-side rendering
|
| 326 |
+
demo.queue(api_open=True).launch(ssr_mode=False)
|