Files changed (1) hide show
  1. app.py +234 -584
app.py CHANGED
@@ -1,472 +1,142 @@
1
- # filename: okemo_backend.py
2
- # OkemoLLM backend (refactored, syntax-fixed, and <WEB> search improved for accuracy):
3
- # - Loads GPT with EXACT hyperparameters from checkpoint
4
- # - Uses web search ONLY when user input contains "<WEB>"
5
- # - Keeps weight tying, online learning, and public APIs compatible
6
- # - Clearer structure, improved robustness, fully typed
7
- # - **Improved Tavily search to prioritize summarized answer for better accuracy.**
8
-
9
- import os
10
  import re
11
- import math
12
- import unicodedata
13
- import requests
14
- from datetime import datetime
15
- from typing import Dict, List, Tuple, Optional, Any, Iterator
16
-
17
- import torch
18
- import torch.nn as nn
19
- import torch.nn.functional as F
20
- import torch.optim as optim
21
- import gradio as gr
22
-
23
- # MPS fallback on macOS
24
- os.environ["PYTORCH_ENABLE_MPS_FALLBACK"] = "1"
25
-
26
- # ---------------------
27
- # Config
28
- # ---------------------
29
- SYSTEM_INSTRUCTION: str = (
30
- "You are a friendly, helpful, concise AI assistant named Okemo Language Model (OLM). "
31
- "Default to clear, short answers; elaborate only when asked."
32
- )
33
-
34
- ONLINE_LEARNING_EPOCHS: int = 5
35
- CHECKPOINT_PATH: str = "checkpoint.pth"
36
-
37
- # Fallback defaults (used only when no checkpoint exists)
38
- DEFAULTS: Dict[str, Any] = {
39
- "d_model": 768,
40
- "n_head": 48,
41
- "d_ff": 768,
42
- "n_layers": 12,
43
- "dropout": 0.1,
44
- "max_length": 1024,
45
- "k_sampling": 50,
46
- "LEARNING_RATE": 1e-6,
47
- }
48
-
49
- # Web search provider (Tavily). Keep key in environment.
50
- TAVILY_API_KEY: str = os.getenv("TAVILY_API_KEY", "")
51
-
52
- # Search trigger tag
53
- WEB_TAG_RE = re.compile(r"<\s*web\s*>", flags=re.IGNORECASE)
54
-
55
- # ---------------------
56
- # Helpers
57
- # ---------------------
58
- def to_ascii(text: Optional[str]) -> str:
59
- """Normalize to NFKC and ensure str."""
60
- return unicodedata.normalize("NFKC", text or "")
61
-
62
- # Simple tokenizer: words or punctuation
63
- _TOKEN_RE = re.compile(r"\w+|[^\w\s]", flags=re.UNICODE)
64
-
65
- def tokenize(text: str) -> List[str]:
66
- """Lowercase, ASCII-normalized tokenization into words and punctuation units."""
67
- txt = to_ascii(text)
68
- return _TOKEN_RE.findall(txt.lower())
69
-
70
- def check_repetition(text: str, max_repeat: int = 2) -> bool:
71
- """Detect exact token repetition runs longer than max_repeat."""
72
- tokens = tokenize(text)
73
- if not tokens or max_repeat < 1:
74
- return False
75
- for i in range(len(tokens) - max_repeat):
76
- if all(tokens[i] == tokens[j] for j in range(i, i + max_repeat + 1)):
77
- return True
78
- return False
79
 
80
- # ---------------------
81
- # Vocabulary
82
- # ---------------------
83
- class Voc:
84
- """Lightweight vocabulary storing special tokens and bidirectional maps."""
85
-
86
- def __init__(self) -> None:
87
- self.PAD, self.SOS, self.EOS, self.UNK, self.SYSTEM = "<PAD>", "<SOS>", "<EOS>", "<UNK>", "<SYSTEM>"
88
- self.word2index: Dict[str, int] = {self.PAD: 0, self.SOS: 1, self.EOS: 2, self.UNK: 3, self.SYSTEM: 4}
89
- self.index2word: Dict[int, str] = {0: self.PAD, 1: self.SOS, 2: self.EOS, 3: self.UNK, 4: self.SYSTEM}
90
- self.n_words: int = 5
91
-
92
- @staticmethod
93
- def load_from_checkpoint(ckpt: Dict[str, Any]) -> "Voc":
94
- voc = Voc()
95
- if "voc_word2index" in ckpt and "voc_index2word" in ckpt:
96
- voc.word2index = ckpt["voc_word2index"]
97
- voc.index2word = ckpt["voc_index2word"]
98
- voc.n_words = len(voc.word2index)
99
- if "special_tokens" in ckpt:
100
- tokens = ckpt["special_tokens"]
101
- voc.PAD = tokens.get("PAD", voc.PAD)
102
- voc.SOS = tokens.get("SOS", voc.SOS)
103
- voc.EOS = tokens.get("EOS", voc.EOS)
104
- voc.UNK = tokens.get("UNK", voc.UNK)
105
- voc.SYSTEM = tokens.get("SYSTEM", voc.SYSTEM)
106
- return voc
107
-
108
- def add_sentence(self, sentence: str) -> None:
109
- for word in tokenize(sentence):
110
- if word not in self.word2index:
111
- idx = self.n_words
112
- self.word2index[word] = idx
113
- self.index2word[idx] = word
114
- self.n_words += 1
115
 
116
- # ---------------------
117
- # Model
118
- # ---------------------
119
- class PositionalEncoding(nn.Module):
120
- """Sinusoidal positional encoding used by Transformer."""
121
-
122
- def __init__(self, d_model: int, max_len: int = 5000) -> None:
123
- super().__init__()
124
- pe = torch.zeros(max_len, d_model)
125
- position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)
126
- div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model))
127
- pe[:, 0::2] = torch.sin(position * div_term)
128
- pe[:, 1::2] = torch.cos(position * div_term)
129
- self.register_buffer("pe", pe.unsqueeze(0))
130
-
131
- def forward(self, x: torch.Tensor) -> torch.Tensor:
132
- return x + self.pe[:, : x.size(1), :]
133
-
134
- class GPT(nn.Module):
135
- """Minimal GPT-like model using nn.TransformerEncoder with weight tying."""
136
-
137
- def __init__(self, n_words: int, d_model: int, n_layers: int, n_head: int, d_ff: int, dropout: float = 0.1) -> None:
138
- super().__init__()
139
- self.embedding = nn.Embedding(n_words, d_model)
140
- self.pos = PositionalEncoding(d_model)
141
- self.norm = nn.LayerNorm(d_model)
142
- enc_layer = nn.TransformerEncoderLayer(d_model, n_head, d_ff, dropout, batch_first=True)
143
- self.transformer = nn.TransformerEncoder(enc_layer, n_layers)
144
- self.fc = nn.Linear(d_model, n_words, bias=False)
145
- self.fc.weight = self.embedding.weight # weight tying
146
-
147
- def forward(
148
- self,
149
- x: torch.Tensor,
150
- mask: Optional[torch.Tensor] = None,
151
- key_padding_mask: Optional[torch.Tensor] = None,
152
- ) -> torch.Tensor:
153
- x = self.embedding(x)
154
- x = self.pos(x)
155
- x = self.norm(x)
156
- out = self.transformer(x, mask=mask, src_key_padding_mask=key_padding_mask)
157
- return self.fc(out)
158
 
159
- # ---------------------
160
- # Training utils
161
- # ---------------------
162
- def indexes_from_sentence(voc: Voc, sentence: str) -> List[int]:
163
- sanitized = " ".join(tokenize(sentence))
164
- prompt_tokens = [voc.SYSTEM] + tokenize(sanitized)
165
- return [voc.word2index.get(tok, voc.word2index[voc.UNK]) for tok in prompt_tokens] + [voc.word2index[voc.EOS]]
166
 
167
- def make_bool_causal_mask(sz: int, device: torch.device) -> torch.Tensor:
168
- return torch.triu(torch.ones((sz, sz), dtype=torch.bool, device=device), diagonal=1)
169
 
170
- # ---------------------
171
- # Device
172
- # ---------------------
173
- # CRITICAL FIX: Forcing device to 'cpu' to prevent silent PyTorch/device hangs on first forward pass.
174
- device: torch.device = torch.device("cpu")
175
 
176
- # ---------------------
177
- # Checkpoint-aware init
178
- # ---------------------
179
- def load_checkpoint_or_defaults(path: str) -> Tuple[Dict[str, Any], Dict[str, Any], Voc, GPT, optim.Optimizer]:
180
- if os.path.exists(path):
181
- ckpt = torch.load(path, map_location=device)
182
- hp = {
183
- "d_model": ckpt.get("d_model", DEFAULTS["d_model"]),
184
- "n_head": ckpt.get("n_head", DEFAULTS["n_head"]),
185
- "d_ff": ckpt.get("d_ff", DEFAULTS["d_ff"]),
186
- "n_layers": ckpt.get("n_layers", DEFAULTS["n_layers"]),
187
- "dropout": ckpt.get("dropout", DEFAULTS["dropout"]),
188
- "max_length": ckpt.get("max_length", DEFAULTS["max_length"]),
189
- "k_sampling": ckpt.get("k_sampling", DEFAULTS["k_sampling"]),
190
- "LEARNING_RATE": ckpt.get("LEARNING_RATE", DEFAULTS["LEARNING_RATE"]),
191
- }
192
- voc = Voc.load_from_checkpoint(ckpt)
193
- model = GPT(voc.n_words, hp["d_model"], hp["n_layers"], hp["n_head"], hp["d_ff"], hp["dropout"]).to(device)
194
- if ckpt.get("gpt_model_state_dict") is not None:
195
- model.load_state_dict(ckpt["gpt_model_state_dict"], strict=True)
196
- model.fc.weight = model.embedding.weight
197
- optimizer = optim.Adam(model.parameters(), lr=hp["LEARNING_RATE"])
198
- if "optimizer_state_dict" in ckpt:
199
- try:
200
- optimizer.load_state_dict(ckpt["optimizer_state_dict"])
201
- except Exception as e:
202
- print(f"Warning: Failed to load optimizer state: {e}. Starting fresh.")
203
- return ckpt, hp, voc, model, optimizer
204
- else:
205
- print("WARNING: No checkpoint found. Initializing with defaults.")
206
- hp = DEFAULTS.copy()
207
- voc = Voc()
208
- model = GPT(voc.n_words, hp["d_model"], hp["n_layers"], hp["n_head"], hp["d_ff"], hp["dropout"]).to(device)
209
- optimizer = optim.Adam(model.parameters(), lr=hp["LEARNING_RATE"])
210
- return {}, hp, voc, model, optimizer
211
-
212
- checkpoint, HP, voc, gpt_model, optimizer = load_checkpoint_or_defaults(CHECKPOINT_PATH)
213
- criterion = nn.CrossEntropyLoss(ignore_index=voc.word2index[voc.PAD])
214
-
215
- # --- Added logic: Save initial state if no checkpoint was found to ensure .pth exists ---
216
- if not os.path.exists(CHECKPOINT_PATH):
217
- print("Saving initial checkpoint with default settings.")
218
- # The initial voc will only contain special tokens, but it's a valid starting point
219
- save_checkpoint(gpt_model, optimizer, voc, CHECKPOINT_PATH)
220
- # ---------------------------------------------------------------------------------------
221
 
222
- # ---------------------
223
- # Save checkpoint
224
- # ---------------------
225
- def save_checkpoint(model: GPT, optimizer: optim.Optimizer, voc: Voc, path: str) -> None:
226
- print(f"Saving model checkpoint to {path}...")
227
- ckpt = {
228
- "voc_word2index": voc.word2index,
229
- "voc_index2word": voc.index2word,
230
- "gpt_model_state_dict": model.state_dict(),
231
- "optimizer_state_dict": optimizer.state_dict(),
232
- "d_model": HP["d_model"],
233
- "n_head": HP["n_head"],
234
- "d_ff": HP["d_ff"],
235
- "n_layers": HP["n_layers"],
236
- "dropout": HP["dropout"],
237
- "max_length": HP["max_length"],
238
- "k_sampling": HP["k_sampling"],
239
- "LEARNING_RATE": HP["LEARNING_RATE"],
240
- }
241
- try:
242
- torch.save(ckpt, path)
243
- print("Checkpoint saved successfully.")
244
- except Exception as e:
245
- print(f"ERROR: Failed to save checkpoint: {e}")
246
-
247
- # ---------------------
248
- # Web search (only with <WEB>) - IMPROVED FOR ACCURACY
249
- # ---------------------
250
  def perform_web_search(query: str) -> str:
251
  """
252
- Performs a Tavily web search, prioritizing the summarized 'answer' for accuracy.
253
- If no answer is provided, falls back to concatenating snippets.
 
 
254
  """
255
- if not TAVILY_API_KEY:
256
- return ""
257
- try:
258
- resp = requests.post(
259
- "https://api.tavily.com/search",
260
- # MODIFIED: Set 'include_answer' to True for a summarized answer, reduce 'max_results' to 3.
261
- json={"api_key": TAVILY_API_KEY, "query": query, "include_answer": True, "max_results": 3},
262
- timeout=8,
263
- )
264
- resp.raise_for_status()
265
- data = resp.json()
266
-
267
- # 1. Prioritize the AI-generated answer from Tavily
268
- answer = data.get("answer", "").strip()
269
- if answer:
270
- return to_ascii(answer)
271
-
272
- # 2. Fallback to manually compiling snippets if no answer is provided
273
- results = data.get("results", [])
274
- snippets: List[str] = []
275
- for r in results[:3]:
276
- title = (r.get("title") or "").strip()
277
- snippet = (r.get("content") or "").strip()
278
- url = (r.get("url") or "").strip()
279
- if title or snippet:
280
- # Use a cleaner format for the LLM to digest
281
- snippets.append(f"Title: {title}, Content: {snippet}")
282
-
283
- return to_ascii(" | ".join(snippets))
284
- except Exception as e:
285
- print(f"Search failed: {e}")
286
- return ""
287
-
288
- def should_search(user_text: str) -> bool:
289
- return bool(WEB_TAG_RE.search(user_text or ""))
290
 
291
- # ---------------------
292
- # Generation
293
- # ---------------------
294
- def generate_response(prompt: str, search_context: str = "", conversation_context: str = "") -> Iterator[str]:
295
- """Generator function that yields the generated word tokens."""
296
- # Prefix text is now the concise answer or the compiled snippets
297
- prefix_text = f"According to a search result: {to_ascii(search_context)}. " if search_context else ""
298
- context_prefix = f"{conversation_context} " if conversation_context else ""
299
- full_input = f"{SYSTEM_INSTRUCTION} {context_prefix}{prefix_text}{prompt}"
300
- full_input = to_ascii(full_input)
301
-
302
- gpt_model.eval()
303
-
304
- initial_tokens = indexes_from_sentence(voc, full_input)
305
- input_tensor = torch.tensor(initial_tokens, dtype=torch.long, device=device).unsqueeze(0)
306
- generated_token_ids: List[int] = [] # Use this to check for repetition later
307
- temperature = 0.8
308
- vocab_eos = voc.word2index[voc.EOS]
309
- vocab_pad = voc.word2index[voc.PAD]
310
-
311
- with torch.no_grad():
312
- for step in range(HP["max_length"]):
313
- L = input_tensor.size(1)
314
- causal = make_bool_causal_mask(L, device)
315
- padmask = (input_tensor == vocab_pad)
316
-
317
- # The synchronous call that was likely hanging on device access
318
- logits = gpt_model(input_tensor, mask=causal, key_padding_mask=padmask)
319
-
320
- last = logits[:, -1, :].float() / max(temperature, 1e-3)
321
-
322
- k = max(1, int(HP["k_sampling"]))
323
- topk_vals, topk_idx = torch.topk(last, k)
324
- probs = F.softmax(topk_vals, dim=-1).float()
325
- pick_rel = torch.multinomial(probs, 1).item()
326
- candidate_id = topk_idx.squeeze(0)[pick_rel].item()
327
-
328
- if candidate_id == vocab_eos:
329
- break
330
 
331
- generated_token_ids.append(candidate_id)
332
- next_token = torch.tensor([[candidate_id]], device=device, dtype=torch.long)
333
- input_tensor = torch.cat([input_tensor, next_token], dim=1)
334
-
335
- # Yield the generated word token for streaming
336
- word = voc.index2word.get(candidate_id, voc.UNK)
337
- if word not in [voc.PAD, voc.SOS, voc.EOS, voc.UNK, voc.SYSTEM]:
338
- yield word + " " # Yield with space for cleaner UI streaming
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
339
 
340
- # ---------------------
341
- # Online learning (feedback)
342
- # ---------------------
343
- def online_update(
344
- model: GPT,
345
- input_sentence: str,
346
- target_response: str,
347
- max_seq_len: int,
348
- search_context: str = "",
349
- file_context: str = "",
350
- conversation_context: str = "",
351
- ) -> None:
352
- is_response_clean = not any(t in target_response for t in [voc.UNK, "i couldn't generate"])
353
- is_not_repetitive = not check_repetition(target_response, max_repeat=2)
354
- if not is_response_clean or not is_not_repetitive:
355
- print("Skipping online update: Target response is corrupted or highly repetitive.")
356
- return
357
-
358
- prefix_search = f"According to a search result: {to_ascii(search_context)}. " if search_context else ""
359
- prefix_file = f"Based on the attached file: {to_ascii(file_context)}. " if file_context else ""
360
- context_prefix = f"{conversation_context} " if conversation_context else ""
361
-
362
- full_prompt_text = f"{SYSTEM_INSTRUCTION} {context_prefix}{prefix_search}{prefix_file}{input_sentence}"
363
- full_prompt_text = to_ascii(full_prompt_text)
364
-
365
- prompt_tokens = indexes_from_sentence(voc, full_prompt_text)[:-1]
366
- sanitized_response = to_ascii(target_response)
367
- response_tokens = indexes_from_sentence(voc, sanitized_response)[1:]
368
-
369
- full_sequence_ids = prompt_tokens + response_tokens
370
- input_ids = full_sequence_ids[:-1]
371
- target_ids = full_sequence_ids[1:]
372
-
373
- if len(input_ids) < 2 or len(input_ids) != len(target_ids) or len(input_ids) > max_seq_len:
374
- print(f"Skipping online update. Sequence length: {len(input_ids)}")
375
- return
376
-
377
- input_tensor = torch.tensor(input_ids, dtype=torch.long, device=device).unsqueeze(0)
378
- target_tensor = torch.tensor(target_ids, dtype=torch.long, device=device)
379
-
380
- L = input_tensor.size(1)
381
- model.train()
382
- causal = make_bool_causal_mask(L, device)
383
-
384
- last_loss = 0.0
385
- for _ in range(ONLINE_LEARNING_EPOCHS):
386
- optimizer.zero_grad()
387
- logits = model(input_tensor, mask=causal, key_padding_mask=None)
388
- loss = criterion(logits.squeeze(0), target_tensor)
389
- loss.backward()
390
- torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)
391
- optimizer.step()
392
- last_loss = loss.item()
393
-
394
- model.eval()
395
- print(f"Online update done. Final Loss: {last_loss:.4f}")
396
-
397
- def process_feedback(history: List[List[Optional[str]]], rating: str) -> Tuple[List[List[Optional[str]]], str]:
398
- if not isinstance(history, list):
399
- return [], "Cannot process feedback: history not a list."
400
- if not history or not (isinstance(history[-1], list) and len(history[-1]) == 2 and isinstance(history[-1][1], str)):
401
- return history, "Cannot process feedback: No recent AI response found."
402
-
403
- last_user_message, last_ai_response = history[-1][0], history[-1][1]
404
-
405
- file_context_text = ""
406
- file_match = re.search(r"\(Attached: (.*?)\)", last_user_message or "")
407
- if file_match:
408
- file_name = file_match.group(1)
409
- file_context_text = f"The user provided a file named '{file_name}'."
410
-
411
- clean_user_message = to_ascii(last_user_message or "")
412
- clean_user_message = re.sub(r'\(Attached:.*?\)', '', clean_user_message).strip()
413
- clean_user_message = re.sub(r'\[IMAGE_PREVIEW_URL:.*?]', '', clean_user_message).strip()
414
-
415
- search_context_text = ""
416
-
417
- if rating == "Good Response":
418
- conversation_context = ""
419
- if len(history) >= 2 and isinstance(history[-2], list) and len(history[-2]) == 2 and isinstance(history[-2][1], str):
420
- last_user, last_ai = history[-2][0], history[-2][1]
421
- conversation_context = f"User: {to_ascii(last_user)} Response: {to_ascii(last_ai)}"
422
-
423
- online_update(
424
- gpt_model,
425
- clean_user_message,
426
- last_ai_response,
427
- HP["max_length"],
428
- search_context=search_context_text,
429
- file_context=file_context_text,
430
- conversation_context=conversation_context,
431
  )
432
- save_checkpoint(gpt_model, optimizer, voc, CHECKPOINT_PATH)
433
- feedback_message = "Thanks for the Good Response rating! Learned from that turn. ✨"
434
- elif rating == "Bad Response":
435
- feedback_message = "Acknowledged Bad Response feedback. 🛠️"
436
  else:
437
- feedback_message = "Unknown rating received."
438
-
439
- return history, feedback_message
440
-
441
- def feedback_good(history_state: List[List[Optional[str]]]) -> Tuple[List[List[Optional[str]]], str]:
442
- history_state, feedback_message = process_feedback(history_state, "Good Response")
443
- return history_state, feedback_message
444
 
445
- def feedback_bad(history_state: List[List[Optional[str]]]) -> Tuple[List[List[Optional[str]]], str]:
446
- history_state, feedback_message = process_feedback(history_state, "Bad Response")
447
- return history_state, feedback_message
 
448
 
449
- def feedback_bad_with_prompt(history_state: List[List[Optional[str]]], suggested_prompt: str) -> Tuple[List[List[Optional[str]]], str]:
450
- log_path = "bad_feedback_log.txt"
451
- try:
452
- with open(log_path, "a", encoding="utf-8") as f:
453
- f.write(f"\n--- {datetime.now()} ---\nSuggested Prompt:\n{suggested_prompt}\n\n")
454
- except Exception as e:
455
- print(f"Failed to log feedback: {e}")
456
- process_feedback(history_state, "Bad Response")
457
- return history_state, f"Recorded:\n“{suggested_prompt}”"
458
 
459
  # ---------------------
460
- # Core request handling
461
  # ---------------------
462
  def _extract_file_context(uploaded_file: Optional[Any]) -> str:
 
463
  if not uploaded_file:
464
  return ""
465
  file_name = uploaded_file.name.split("/")[-1]
466
- return f"The user has provided a file named '{file_name}'. Analyze or respond to the user's prompt about this file."
467
 
468
  def _build_conversation_context(history: List[List[Optional[str]]], is_instruction: bool) -> str:
 
 
 
 
 
469
  context_history = history[:-1] if not is_instruction else history
 
 
470
  if context_history and isinstance(context_history[-1], list) and len(context_history[-1]) == 2 and isinstance(context_history[-1][1], str):
471
  last_user, last_ai = context_history[-1][0], context_history[-1][1]
472
  return f"User: {to_ascii(last_user)} Response: {to_ascii(last_ai)}"
@@ -476,145 +146,150 @@ def _is_instruction_turn(user_message: str) -> bool:
476
  prefixes = ("please generate", "provide a much more detailed", "provide a different")
477
  return any((user_message or "").lower().startswith(p) for p in prefixes)
478
 
479
- def on_submit(
480
- user_message: str,
481
- history: Optional[List[List[Optional[str]]]],
482
- uploaded_file: Optional[Any],
483
- web_search_enabled_flag_unused: bool = True,
484
- ) -> Iterator[Tuple[List[List[Optional[str]]], str]]:
485
- """Handles the request and yields incremental updates for streaming."""
486
- history = history or []
487
-
488
- file_context = _extract_file_context(uploaded_file)
489
-
490
- is_instruction = _is_instruction_turn(user_message or "")
491
- clean_user_message = user_message or ""
492
- quoted_match = re.search(r':\s*"(.+?)"$', clean_user_message)
493
- if is_instruction and quoted_match:
494
- clean_user_message = quoted_match.group(1).strip()
495
- model_prompt = user_message
496
- else:
497
- model_prompt = user_message
498
-
499
- clean_user_message = re.sub(r'\(Attached:.*?\)', '', clean_user_message).strip()
500
- clean_user_message = re.sub(r'\[IMAGE_PREVIEW_URL:.*?]', '', clean_user_message).strip()
501
-
502
- conversation_context = _build_conversation_context(history, is_instruction)
503
-
504
- search_context_text = ""
505
- if should_search(clean_user_message):
506
- query_for_search = WEB_TAG_RE.sub("", clean_user_message).strip()
507
- search_context_text = perform_web_search(query_for_search)
508
-
509
- display_search_result = (
510
- f"🔍 **Search Result Used:** {search_context_text}" if search_context_text
511
- else ""
512
- )
513
-
514
- final_model_prompt = f"{file_context} {model_prompt}" if file_context else model_prompt
515
- final_model_prompt = WEB_TAG_RE.sub("", final_model_prompt or "").strip()
516
-
517
- # --- Streaming Logic ---
518
- response_generator = generate_response(
519
- final_model_prompt,
520
- search_context=search_context_text,
521
- conversation_context=conversation_context,
522
- )
523
-
524
- full_reply_text = ""
525
- status_message = display_search_result or "Thinking..."
526
- last_user_message = history[-1][0] if history and history[-1][0] else clean_user_message
527
-
528
- for token in response_generator:
529
- full_reply_text += token
530
-
531
- # Update the last message in history with the partial response
532
- if history and history[-1][1] is None:
533
- history[-1][1] = full_reply_text
534
- elif history:
535
- # Ensure we don't accidentally lose the user message if it was stripped before submit
536
- history[-1][0] = last_user_message
537
- history[-1][1] = full_reply_text
538
-
539
- # Yield the current state for streaming to the frontend
540
- yield history, status_message
541
-
542
- # --- Final Formatting and Completion ---
543
-
544
- # Final formatting/cleanup
545
- full_reply_text = to_ascii(full_reply_text)
546
- full_reply_text = re.sub(r"\s+([,.!?;:])", r"\1", full_reply_text).strip()
547
-
548
- # Final checks
549
- if not full_reply_text or check_repetition(full_reply_text):
550
- final_reply = "I got lost in the data tubes! Perhaps ask me something a little sillier? 🤪"
551
- status_message = display_search_result or "Error: Response was blocked."
552
- else:
553
- final_reply = full_reply_text
554
- status_message = display_search_result or "Response complete."
555
-
556
- # Final update to history (in case of an error or final cleanup)
557
- if history:
558
- history[-1][1] = final_reply
559
-
560
- yield history, status_message
561
 
562
  # ---------------------
563
- # Gradio app
564
  # ---------------------
565
  with gr.Blocks(title="OkemoLLM") as demo:
566
- # Adding a warning about deprecated Chatbot format
567
- gr.Warning("Note: The Gradio Chatbot format is configured for compatibility with your custom JS. Newer versions recommend `type='messages'`. See terminal logs for details.", duration=5)
568
-
569
- gr.Markdown("# OkemoLLM (<WEB> gated web search, checkpoint-aware) 🌐")
570
  search_status_display = gr.Markdown("🔍 Search status will appear here.")
571
-
572
- # Setting type to None (default) for compatibility with custom JS history structure
573
- chat = gr.Chatbot(height=500)
574
 
575
  with gr.Row():
576
  with gr.Column(scale=4):
577
  msg = gr.Textbox(
578
- placeholder='Type your message… Add "<WEB>" to use web search for this turn.',
579
  label="Message",
580
  elem_id="okemo-input-textbox"
581
  )
582
  uploaded_file_state = gr.File(label="Uploaded File", visible=False, elem_id="uploaded_file_state")
 
583
  web_search_enabled_state = gr.Checkbox(label="Frontend Search Toggle (ignored; use <WEB>)", value=True)
584
 
585
  clear = gr.Button("Clear")
586
  send_trigger = gr.Button(visible=False, elem_id="okemo-send-button-trigger")
587
 
 
 
 
588
  def add_user_message_to_history(message: str, history: Optional[List[List[Optional[str]]]], file_ref: Optional[Any], web_enabled: bool):
589
  history = history or []
590
- # The user message is added to history, AI response slot is None for streaming
591
  return history + [[message, None]], "..."
592
 
593
- msg.submit(
594
- add_user_message_to_history,
595
- [msg, chat, uploaded_file_state, web_search_enabled_state],
596
- [chat, search_status_display],
597
- queue=False
598
- ).then(
599
- on_submit,
600
- [msg, chat, uploaded_file_state, web_search_enabled_state],
601
- [chat, search_status_display]
602
- )
603
 
604
- send_trigger.click(
605
- add_user_message_to_history,
606
- [msg, chat, uploaded_file_state, web_search_enabled_state],
607
- [chat, search_status_display],
608
- queue=False
609
- ).then(
610
- on_submit,
611
- [msg, chat, uploaded_file_state, web_search_enabled_state],
612
- [chat, search_status_display]
613
- )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
614
 
615
  clear.click(lambda: [], None, chat, queue=False)
616
 
617
- # Public API endpoints
618
  def chat_submit_api(
619
  msg_value: str,
620
  chat_value: Optional[List[List[Optional[str]]]],
@@ -623,9 +298,10 @@ with gr.Blocks(title="OkemoLLM") as demo:
623
  ):
624
  updated_history = chat_value or []
625
  updated_history.append([msg_value, None])
626
- # Call generator and return the final state
627
- # Note: We must consume the generator to get the final state for the API
628
  final_history, status_msg = list(on_submit(msg_value, updated_history, uploaded_file_value, web_enabled_value))[-1]
 
629
  return final_history, status_msg
630
 
631
  gr.Interface(
@@ -645,32 +321,6 @@ with gr.Blocks(title="OkemoLLM") as demo:
645
  allow_flagging="never",
646
  )
647
 
648
- gr.Interface(
649
- fn=feedback_good,
650
- inputs=[gr.State(value=[])],
651
- outputs=[gr.State(value=[]), gr.Textbox()],
652
- api_name="feedback_good",
653
- live=False,
654
- allow_flagging="never",
655
- )
656
-
657
- gr.Interface(
658
- fn=feedback_bad,
659
- inputs=[gr.State(value=[])],
660
- outputs=[gr.State(value=[]), gr.Textbox()],
661
- api_name="feedback_bad",
662
- live=False,
663
- allow_flagging="never",
664
- )
665
-
666
- gr.Interface(
667
- fn=feedback_bad_with_prompt,
668
- inputs=[gr.State(value=[])],
669
- outputs=[gr.State(value=[]), gr.Textbox()],
670
- api_name="feedback_bad_with_prompt",
671
- live=False,
672
- allow_flagging="never",
673
- )
674
-
675
  if __name__ == "__main__":
676
- demo.queue(api_open=True).launch(ssr_mode=False)
 
 
 
 
 
 
 
 
 
 
 
1
  import re
2
+ import time
3
+ from typing import Optional, List, Tuple, Any, Iterator
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4
 
5
+ import gradio as gr
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6
 
7
+ # =========================================================
8
+ # BACKEND CORE LOGIC (Originally okemo_backend.py)
9
+ # Contains RAG, search, and generation functions
10
+ # =========================================================
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
11
 
12
+ # --- Constants ---
13
+ WEB_TAG_RE = re.compile(r'\s*<WEB>\s*', re.IGNORECASE)
 
 
 
 
 
14
 
15
+ # --- Utility Functions ---
 
16
 
17
+ def to_ascii(text: Optional[str]) -> str:
18
+ """A minimal placeholder for a real sanitation/encoding function."""
19
+ return text if text is not None else ""
 
 
20
 
21
+ def should_search(user_message: str) -> bool:
22
+ """Determines if a web search should be performed based on the <WEB> tag."""
23
+ return bool(WEB_TAG_RE.search(user_message))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
24
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
25
  def perform_web_search(query: str) -> str:
26
  """
27
+ Simulated implementation of a structured web search/RAG retrieval function.
28
+
29
+ This simulation demonstrates *improved* RAG by formatting search results
30
+ into a structured context that encourages the LLM to use citations.
31
  """
32
+ if not query.strip():
33
+ return "Search skipped: Query was empty after tag removal."
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
34
 
35
+ print(f"**Backend performing search for: '{query}'**")
36
+
37
+ # --- MOCK Search Results (replace with real API call in production) ---
38
+ # We simulate retrieving multiple relevant, cited snippets.
39
+ search_results = [
40
+ {"title": "Latest LLM Trends in 2025", "snippet": "The rise of multimodal agents and highly efficient smaller models is dominating the 2025 LLM landscape. [1]", "url": "https://example.com/trends"},
41
+ {"title": "Gradio UI Enhancements", "snippet": "Gradio introduced new features for streaming and dynamic component updates in its 4.x release, drastically improving frontend responsiveness. [2]", "url": "https://example.com/gradio_updates"},
42
+ {"title": "Python RAG Best Practices", "snippet": "Using cross-encoder re-ranking significantly improves the quality of context passed to the LLM by filtering less relevant search results. [3]", "url": "https://example.com/rag_tips"}
43
+ ]
44
+
45
+ # RAG Context Construction
46
+ context_parts = []
47
+ for i, res in enumerate(search_results):
48
+ # Format the context for LLM ingestion
49
+ context_parts.append(f"Source {i+1} (Title: {res['title']}, URL: {res['url']}): {res['snippet']}")
50
+
51
+ search_context = "Web Search Results:\n" + "\n---\n".join(context_parts)
52
+
53
+ return search_context
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
54
 
55
+ def generate_response(
56
+ user_prompt: str,
57
+ search_context: str,
58
+ conversation_context: str,
59
+ ) -> Iterator[str]:
60
+ """
61
+ Simulated core LLM generation function.
62
+
63
+ In a real application, this would make an external API call to a model
64
+ like Gemini, GPT, or Claude, passing the full prompt constructed below.
65
+ """
66
+
67
+ # Construct the full RAG prompt
68
+ system_prompt = (
69
+ "You are OkemoLLM, a helpful AI assistant. "
70
+ "Use the provided Web Search Results and Conversation History to answer the User's request. "
71
+ "If search results are provided, prioritize that information and use the citation numbers [N] in your response. "
72
+ "If search results are not provided, answer based on your general knowledge. "
73
+ "Do not mention the search if no context is provided."
74
+ )
75
+
76
+ full_prompt = f"{system_prompt}\n\n"
77
+
78
+ if search_context:
79
+ full_prompt += f"--- WEB SEARCH CONTEXT ---\n{search_context}\n\n"
80
+ if conversation_context:
81
+ full_prompt += f"--- CONVERSATION HISTORY (Last Turn) ---\n{conversation_context}\n\n"
82
+
83
+ full_prompt += f"--- USER MESSAGE ---\n{user_prompt}\n\n"
84
+
85
+ print("\n--- LLM FULL PROMPT START ---")
86
+ print(full_prompt)
87
+ print("--- LLM FULL PROMPT END ---\n")
88
 
89
+ # --- Simulated Streaming Response ---
90
+
91
+ if "trends" in user_prompt.lower() and search_context:
92
+ mock_response = (
93
+ "Based on the current data from the web, the most significant LLM trends "
94
+ "in 2025 include the deployment of new multimodal agents and a focus on "
95
+ "creating highly efficient smaller models [1]. Furthermore, for developers "
96
+ "using this frontend, Gradio’s latest release features improved streaming "
97
+ "and dynamic updates [2]. One best practice is using re-ranking to pass "
98
+ "only the most relevant context to the LLM [3]."
99
+ )
100
+ elif should_search(user_prompt):
101
+ mock_response = (
102
+ "I couldn't find a specific answer for your search, but the system is working. "
103
+ "Try rephrasing your search query, or check the console logs for the full prompt."
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
104
  )
 
 
 
 
105
  else:
106
+ mock_response = (
107
+ f"Hello, I am OkemoLLM. I received your request: '{user_prompt}'. "
108
+ "I am currently operating using my internal knowledge. If you need real-time "
109
+ "information, please remember to include the **<WEB>** tag in your message."
110
+ )
 
 
111
 
112
+ # Yield tokens for streaming effect
113
+ for token in mock_response.split():
114
+ yield token + " "
115
+ time.sleep(0.01) # Small delay for realistic streaming feel
116
 
117
+ # =========================================================
118
+ # FRONTEND UI LOGIC (Originally app.py)
119
+ # =========================================================
 
 
 
 
 
 
120
 
121
  # ---------------------
122
+ # Local helpers
123
  # ---------------------
124
  def _extract_file_context(uploaded_file: Optional[Any]) -> str:
125
+ """Mirror backend: provide a lightweight file context string (name only)."""
126
  if not uploaded_file:
127
  return ""
128
  file_name = uploaded_file.name.split("/")[-1]
129
+ return f"(Attached file: {file_name}) "
130
 
131
  def _build_conversation_context(history: List[List[Optional[str]]], is_instruction: bool) -> str:
132
+ """
133
+ Generates a context string from the last completed turn in the history.
134
+ This ensures the LLM has memory of the immediate preceding exchange.
135
+ """
136
+ # Exclude the current, incomplete turn (which is the last item)
137
  context_history = history[:-1] if not is_instruction else history
138
+
139
+ # Get the last completed turn (User message and AI response)
140
  if context_history and isinstance(context_history[-1], list) and len(context_history[-1]) == 2 and isinstance(context_history[-1][1], str):
141
  last_user, last_ai = context_history[-1][0], context_history[-1][1]
142
  return f"User: {to_ascii(last_user)} Response: {to_ascii(last_ai)}"
 
146
  prefixes = ("please generate", "provide a much more detailed", "provide a different")
147
  return any((user_message or "").lower().startswith(p) for p in prefixes)
148
 
149
+ def _strip_web_tag(text: str) -> str:
150
+ """Remove <WEB> tags before prompting the model."""
151
+ return WEB_TAG_RE.sub("", text or "").strip()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
152
 
153
  # ---------------------
154
+ # Gradio App Definition
155
  # ---------------------
156
  with gr.Blocks(title="OkemoLLM") as demo:
157
+ gr.Warning("Note: This app uses a combined file structure for deployment. Check console logs for RAG prompt details.", duration=5)
158
+
159
+ gr.Markdown("# OkemoLLM (<WEB>-gated web search RAG) 🌐")
 
160
  search_status_display = gr.Markdown("🔍 Search status will appear here.")
161
+ chat = gr.Chatbot(height=500)
 
 
162
 
163
  with gr.Row():
164
  with gr.Column(scale=4):
165
  msg = gr.Textbox(
166
+ placeholder='Type your message… Add "<WEB>" to use web search for this turn (e.g., "latest LLM trends <WEB>").',
167
  label="Message",
168
  elem_id="okemo-input-textbox"
169
  )
170
  uploaded_file_state = gr.File(label="Uploaded File", visible=False, elem_id="uploaded_file_state")
171
+ # Kept for UI parity; web search gating is controlled by the <WEB> tag only
172
  web_search_enabled_state = gr.Checkbox(label="Frontend Search Toggle (ignored; use <WEB>)", value=True)
173
 
174
  clear = gr.Button("Clear")
175
  send_trigger = gr.Button(visible=False, elem_id="okemo-send-button-trigger")
176
 
177
+ # ---------------------
178
+ # Streaming Core Handlers
179
+ # ---------------------
180
  def add_user_message_to_history(message: str, history: Optional[List[List[Optional[str]]]], file_ref: Optional[Any], web_enabled: bool):
181
  history = history or []
182
+ # Add user message; assistant slot is None to allow streaming updates
183
  return history + [[message, None]], "..."
184
 
185
+ def on_submit(
186
+ user_message: str,
187
+ history: Optional[List[List[Optional[str]]]],
188
+ uploaded_file: Optional[Any],
189
+ web_search_enabled_flag_unused: bool = True,
190
+ ) -> Iterator[Tuple[List[List[Optional[str]]], str]]:
191
+ """
192
+ Handles the request and yields incremental updates for streaming.
193
+ """
194
+ history = history or []
195
 
196
+ # --- Message Preprocessing ---
197
+ is_instruction = _is_instruction_turn(user_message or "")
198
+ clean_user_message = user_message or ""
199
+
200
+ # Determine the prompt to pass to the model
201
+ model_prompt = user_message
202
+ if is_instruction:
203
+ # Handle instruction case (e.g., "revise this to be better:")
204
+ quoted_match = re.search(r':\s*"(.+?)"$', clean_user_message)
205
+ if quoted_match:
206
+ clean_user_message = quoted_match.group(1).strip()
207
+
208
+ # Sanitize artifacts (keep consistent)
209
+ clean_user_message = re.sub(r'\(Attached:.*?\)', '', clean_user_message).strip()
210
+ clean_user_message = re.sub(r'\[IMAGE_PREVIEW_URL:.*?]', '', clean_user_message).strip()
211
+
212
+ # Get contexts
213
+ conversation_context = _build_conversation_context(history, is_instruction)
214
+ file_context = _extract_file_context(uploaded_file)
215
+
216
+ # --- Web Search Gating ---
217
+ search_context_text = ""
218
+ user_wants_search = should_search(clean_user_message)
219
+
220
+ if user_wants_search:
221
+ status_message_start = "🔍 Searching the web for relevant context..."
222
+ yield history, status_message_start # Update status bar
223
+
224
+ query_for_search = _strip_web_tag(clean_user_message)
225
+ # The *improved* search happens here
226
+ search_context_text = perform_web_search(query_for_search)
227
+
228
+ display_search_result = (
229
+ f"🔍 **Web Context Found:** Preparing LLM response." if search_context_text else "Thinking..."
230
+ )
231
+
232
+ # --- Final Prompt Assembly ---
233
+ final_model_prompt = _strip_web_tag(model_prompt or "")
234
+ if file_context:
235
+ final_model_prompt = f"{file_context}{final_model_prompt}"
236
+
237
+ # --- Streaming Generation ---
238
+ response_generator = generate_response(
239
+ final_model_prompt,
240
+ search_context=search_context_text,
241
+ conversation_context=conversation_context,
242
+ )
243
+
244
+ full_reply_text = ""
245
+ status_message = display_search_result
246
+ last_user_message = history[-1][0] if history and history[-1][0] else clean_user_message
247
+
248
+ for token in response_generator:
249
+ full_reply_text += token
250
+
251
+ if history and history[-1][1] is None:
252
+ history[-1][1] = full_reply_text
253
+ elif history:
254
+ history[-1][0] = last_user_message
255
+ history[-1][1] = full_reply_text
256
+
257
+ # Yield current state for streaming to frontend
258
+ yield history, status_message
259
+
260
+ # --- Post-Processing and Final Yield ---
261
+ full_reply_text = to_ascii(full_reply_text)
262
+ # Fix spacing before punctuation (a common LLM artifact)
263
+ full_reply_text = re.sub(r"\s+([,.!?;:])", r"\1", full_reply_text).strip()
264
+
265
+ if history:
266
+ history[-1][1] = full_reply_text
267
+
268
+ final_status = ("Response complete." if not search_context_text else f"✅ Web context processed. Response complete.")
269
+ yield history, final_status
270
+
271
+ # --- Wire UI events ---
272
+
273
+ # 1. User submits (via Enter or hidden send button) -> Add message to history
274
+ # 2. Add message successful -> Start the streaming response (on_submit)
275
+
276
+ submit_handlers = [msg.submit, send_trigger.click]
277
+
278
+ for handler in submit_handlers:
279
+ handler(
280
+ add_user_message_to_history,
281
+ [msg, chat, uploaded_file_state, web_search_enabled_state],
282
+ [chat, search_status_display],
283
+ queue=False
284
+ ).then(
285
+ on_submit,
286
+ [msg, chat, uploaded_file_state, web_search_enabled_state],
287
+ [chat, search_status_display]
288
+ )
289
 
290
  clear.click(lambda: [], None, chat, queue=False)
291
 
292
+ # --- Public API endpoint (for compatibility) ---
293
  def chat_submit_api(
294
  msg_value: str,
295
  chat_value: Optional[List[List[Optional[str]]]],
 
298
  ):
299
  updated_history = chat_value or []
300
  updated_history.append([msg_value, None])
301
+
302
+ # Consume the generator to get the final state for API
303
  final_history, status_msg = list(on_submit(msg_value, updated_history, uploaded_file_value, web_enabled_value))[-1]
304
+
305
  return final_history, status_msg
306
 
307
  gr.Interface(
 
321
  allow_flagging="never",
322
  )
323
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
324
  if __name__ == "__main__":
325
+ # Use queue for streaming; disable SSR for client-side rendering
326
+ demo.queue(api_open=True).launch(ssr_mode=False)