AshokGakr commited on
Commit
8aa27e9
·
verified ·
1 Parent(s): 934e9b4

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +303 -662
app.py CHANGED
@@ -1,16 +1,16 @@
1
  """
2
  FastAPI backend with optimized streaming for LFM2.5-1.2B-Instruct
3
- Enhanced with advanced intent detection (AshokGakr/intent_detector)
4
  and emotion detection (AshokGakr/emotions_detector) models
 
5
  """
6
 
7
- import os, json, re, traceback, asyncio, time, sys
8
  from typing import Optional, List, Dict, Any, AsyncGenerator
9
  from datetime import datetime, timezone
10
  from contextlib import asynccontextmanager
11
 
12
  import torch
13
- import numpy as np
14
  from fastapi.staticfiles import StaticFiles
15
  from fastapi import (
16
  FastAPI,
@@ -27,36 +27,21 @@ from fastapi.templating import Jinja2Templates
27
  from transformers import AutoTokenizer, AutoModelForSequenceClassification, AutoModelForCausalLM, TextIteratorStreamer
28
  from threading import Thread
29
 
30
- # Import the tools module
31
- try:
32
- from tools import web_search
33
- print("✅ Tools module imported successfully")
34
- except ImportError as e:
35
- print(f"⚠️ Could not import tools module: {e}")
36
- def web_search(query: str) -> str:
37
- return f"Search results for: {query} (tools module not available)"
38
-
39
  # ================== CONFIG =================
40
  ENVIRONMENT = os.getenv("ENVIRONMENT", "development")
41
- DEBUG = ENVIRONMENT != "production"
42
  DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
43
  print(f"🔧 Using device: {DEVICE}")
44
- print(f"🔧 PyTorch version: {torch.__version__}")
45
- print(f"🔧 CUDA available: {torch.cuda.is_available()}")
46
-
47
- # Updated models from provided URLs
48
- INTENT_MODEL_NAME = "AshokGakr/intent_detector" # 82 intent classes
49
- EMOTION_MODEL_NAME = "AshokGakr/emotions_detector" # 28 emotion labels (multi-label)
50
 
51
- # LFM2.5-1.2B-Instruct model - Liquid AI
 
 
52
  LFM_MODEL_NAME = "LiquidAI/LFM2.5-1.2B-Instruct"
53
 
54
  MAX_FILE_SIZE = 50 * 1024 * 1024
55
  MAX_TOKENS_DEFAULT = 1024
56
- MAX_TOKENS_HARD_LIMIT = 4096 # Conservative limit for stability
57
- DEFAULT_TEMPERATURE = 0.7 # LFM2.5 default is 0.1, but keep adjustable
58
- STREAM_DELAY = 0.02
59
- MAX_CONTEXT_LENGTH = 8192 # Safe context length for LFM2.5 (max 32768)
60
 
61
  # Global model variables
62
  intent_tokenizer = None
@@ -75,105 +60,30 @@ os.makedirs(TEMPLATES_DIR, exist_ok=True)
75
  os.makedirs(STATIC_DIR, exist_ok=True)
76
  templates = Jinja2Templates(directory=TEMPLATES_DIR)
77
 
78
- # Intent mapping for the 82 classes from AshokGakr/intent_detector
79
- # Based on the training datasets mentioned in the model card
80
- INTENT_CATEGORIES = {
81
- "general": [
82
- "greeting", "goodbye", "thanks", "apology", "negation",
83
- "affirmation", "unknown", "other", "neutral", "oos"
84
- ],
85
- "assistant": [
86
- "play_music", "pause_music", "next_track", "previous_track",
87
- "volume_up", "volume_down", "mute", "unmute", "shuffle", "repeat"
88
- ],
89
- "information": [
90
- "weather", "time", "date", "calendar", "news", "facts",
91
- "definition", "search", "find", "lookup"
92
- ],
93
- "control": [
94
- "turn_on", "turn_off", "open", "close", "start", "stop",
95
- "activate", "deactivate", "enable", "disable"
96
- ],
97
- "communication": [
98
- "call", "message", "email", "sms", "video_call",
99
- "voice_message", "chat", "send", "receive"
100
- ],
101
- "navigation": [
102
- "directions", "location", "map", "route", "traffic",
103
- "distance", "eta", "nearby", "find_place"
104
- ],
105
- "entertainment": [
106
- "joke", "story", "quote", "fun_fact", "trivia",
107
- "game", "movie", "tv_show", "book", "music_recommendation"
108
- ],
109
- "productivity": [
110
- "reminder", "alarm", "timer", "schedule", "todo",
111
- "notes", "list", "plan", "organize", "meeting"
112
- ],
113
- "shopping": [
114
- "order", "buy", "purchase", "price", "discount",
115
- "delivery", "track_order", "return", "review", "recommendation"
116
- ],
117
- "food": [
118
- "recipe", "restaurant", "reservation", "order_food",
119
- "ingredients", "cooking", "dietary", "nutrition", "delivery_food"
120
- ]
121
- }
122
-
123
- # Emotion labels from the go_emotions dataset (28 classes)
124
- EMOTION_LABELS = [
125
- "admiration", "amusement", "anger", "annoyance", "approval",
126
- "caring", "confusion", "curiosity", "desire", "disappointment",
127
- "disapproval", "disgust", "embarrassment", "excitement", "fear",
128
- "gratitude", "grief", "joy", "love", "nervousness",
129
- "optimism", "pride", "realization", "relief", "remorse",
130
- "sadness", "surprise", "neutral"
131
- ]
132
-
133
- # Emotion tone mapping for response generation
134
- EMOTION_TONE_MAPPING = {
135
- "positive": ["admiration", "amusement", "approval", "caring", "excitement",
136
- "gratitude", "joy", "love", "optimism", "pride", "relief"],
137
- "neutral": ["neutral", "curiosity", "confusion", "realization", "surprise"],
138
- "negative": ["anger", "annoyance", "disappointment", "disapproval", "disgust",
139
- "embarrassment", "fear", "grief", "nervousness", "remorse", "sadness"],
140
- "desire": ["desire"]
141
- }
142
-
143
  # ================== MODEL LOADING =================
144
 
145
  async def load_models():
146
- """Load and initialize all ML models with optimized settings for LFM2.5"""
147
  global intent_tokenizer, intent_model, emotion_tokenizer, emotion_model
148
  global lfm_tokenizer, lfm_model
149
 
150
  print("\n" + "="*50)
151
- print("🔄 LOADING AI MODELS - ENHANCED WITH INTENT & EMOTION DETECTION")
152
  print("="*50)
153
 
154
- # Load intent detection model (82 classes)
155
- intent_success = False
156
  try:
157
  print(f"📥 Loading intent detection model: {INTENT_MODEL_NAME}")
158
  intent_tokenizer = AutoTokenizer.from_pretrained(INTENT_MODEL_NAME)
159
  intent_model = AutoModelForSequenceClassification.from_pretrained(INTENT_MODEL_NAME)
160
  intent_model.to(DEVICE).eval()
161
- print(f"✅ Intent model loaded successfully (82 classes)")
162
- intent_success = True
163
  except Exception as e:
164
- print(f"⚠️ Intent model fallback to simpler model: {e}")
165
- try:
166
- # Fallback to a simpler intent model
167
- intent_tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")
168
- intent_model = None
169
- intent_success = False
170
- except Exception as e2:
171
- print(f"❌ Intent model fallback failed: {e2}")
172
- intent_tokenizer = None
173
- intent_model = None
174
 
175
- # Load emotion detection model (multi-label, 28 classes)
176
- emotion_success = False
177
  try:
178
  print(f"📥 Loading emotion detection model: {EMOTION_MODEL_NAME}")
179
  emotion_tokenizer = AutoTokenizer.from_pretrained(EMOTION_MODEL_NAME)
@@ -183,40 +93,25 @@ async def load_models():
183
  num_labels=28
184
  )
185
  emotion_model.to(DEVICE).eval()
186
- print(f"✅ Emotion model loaded successfully (28 multi-label classes)")
187
- emotion_success = True
188
  except Exception as e:
189
- print(f"⚠️ Emotion model fallback: {e}")
190
- try:
191
- # Fallback to simpler emotion model
192
- emotion_tokenizer = AutoTokenizer.from_pretrained("roberta-base")
193
- emotion_model = None
194
- emotion_success = False
195
- except Exception as e2:
196
- print(f"❌ Emotion model fallback failed: {e2}")
197
- emotion_tokenizer = None
198
- emotion_model = None
199
 
200
- # ==== OPTIMIZED LFM2.5 LOADING ====
201
- lfm_success = False
202
  try:
203
  print(f"📥 Loading LFM2.5-1.2B-Instruct model")
204
 
205
- # Load tokenizer with correct settings
206
  lfm_tokenizer = AutoTokenizer.from_pretrained(
207
  LFM_MODEL_NAME,
208
- padding_side="left", # Essential for streaming
209
  trust_remote_code=False,
210
  use_fast=True
211
  )
212
 
213
- # IMPORTANT: LFM2.5 uses special tokens <|im_start|> and <|im_end|>
214
- # No need to set pad token separately as the tokenizer handles it
215
-
216
  print(f" Tokenizer loaded with vocab size: {lfm_tokenizer.vocab_size}")
217
- print(f" Chat template available: {lfm_tokenizer.chat_template is not None}")
218
 
219
- # Configure model loading based on available hardware
220
  load_kwargs = {
221
  "torch_dtype": torch.bfloat16 if DEVICE == "cuda" else torch.float32,
222
  "trust_remote_code": False,
@@ -228,60 +123,37 @@ async def load_models():
228
  **load_kwargs
229
  )
230
 
231
- # If device_map not used, manually move to device
232
  if DEVICE != "cuda" or load_kwargs.get("device_map") is None:
233
  lfm_model = lfm_model.to(DEVICE)
234
 
235
  lfm_model.eval()
236
-
237
- # Enable eval mode for inference optimizations
238
- lfm_model.config.use_cache = True # Enable KV cache for faster generation
239
 
240
  print(f"✅ LFM2.5-1.2B-Instruct loaded successfully")
241
  print(f" Model device: {lfm_model.device}")
242
- print(f" Model dtype: {lfm_model.dtype}")
243
- print(f" Model parameters: ~1.2B")
244
  print(f" Context window: 32,768 tokens")
245
- lfm_success = True
246
 
247
  except Exception as e:
248
  print(f"❌ Failed to load LFM2.5 model: {e}")
249
  traceback.print_exc()
250
-
251
- # Fallback: Try with simpler settings
252
- try:
253
- print("🔄 Trying simpler loading method...")
254
- lfm_tokenizer = AutoTokenizer.from_pretrained(LFM_MODEL_NAME)
255
-
256
- lfm_model = AutoModelForCausalLM.from_pretrained(
257
- LFM_MODEL_NAME,
258
- torch_dtype=torch.float32
259
- )
260
- lfm_model.to(DEVICE).eval()
261
- print(f"✅ LFM2.5 model loaded (fallback method)")
262
- lfm_success = True
263
- except Exception as e2:
264
- print(f"❌ Fallback loading failed: {e2}")
265
- lfm_tokenizer = None
266
- lfm_model = None
267
 
268
  print("="*50)
269
  print(f"📊 MODEL LOADING SUMMARY:")
270
- print(f" • Intent Model (82-class): {'✅' if intent_success else '❌'}")
271
- print(f" • Emotion Model (28-label): {'✅' if emotion_success else '❌'}")
272
- print(f" • LFM2.5 Model: {'✅' if lfm_success else '❌'}")
273
  print(f" • Device: {DEVICE}")
274
- print(f" • Max Context: {MAX_CONTEXT_LENGTH} tokens")
275
  print("="*50 + "\n")
276
 
277
- return lfm_success
278
 
279
  async def cleanup_models():
280
  """Cleanup models on shutdown"""
281
  print("🧹 Cleaning up models...")
282
  if torch.cuda.is_available():
283
  torch.cuda.empty_cache()
284
- torch.cuda.synchronize()
285
  print("✅ Cleanup complete")
286
 
287
  # ================== LIFESPAN MANAGER =================
@@ -290,12 +162,8 @@ async def cleanup_models():
290
  async def lifespan(app: FastAPI):
291
  """Modern lifespan context manager for FastAPI"""
292
  print("\n" + "="*50)
293
- print("🚀 STARTING GAKR AI ASSISTANT WITH INTENT & EMOTION DETECTION")
294
  print("="*50)
295
- print(f"📁 Working directory: {BASE_DIR}")
296
- print(f"🐍 Python version: {sys.version}")
297
- print(f"🔥 Torch version: {torch.__version__}")
298
- print("="*50 + "\n")
299
 
300
  model_loaded = await load_models()
301
  if not model_loaded:
@@ -311,9 +179,9 @@ async def lifespan(app: FastAPI):
311
  # ================== APP SETUP =================
312
 
313
  app = FastAPI(
314
- title="GAKR AI Assistant (Enhanced with Intent & Emotion Detection)",
315
- description="Three-stage AI pipeline with advanced intent detection, emotion analysis, and LFM2.5-1.2B-Instruct",
316
- version="5.0.0",
317
  lifespan=lifespan
318
  )
319
 
@@ -327,39 +195,20 @@ app.add_middleware(
327
  allow_headers=["*"]
328
  )
329
 
330
- # ================== OPTIMIZED HELPERS =================
331
-
332
- def validate_prompt_length(prompt: str, max_length: int = MAX_CONTEXT_LENGTH) -> bool:
333
- """Validate prompt doesn't exceed context window"""
334
- if not lfm_tokenizer:
335
- return True
336
-
337
- tokens = lfm_tokenizer.encode(prompt, truncation=False)
338
- return len(tokens) <= max_length
339
-
340
- def get_truncated_prompt(prompt: str, max_tokens: int = 3072) -> str:
341
- """Truncate prompt to fit within context window"""
342
- if not lfm_tokenizer:
343
- return prompt[:1000]
344
-
345
- tokens = lfm_tokenizer.encode(prompt, truncation=False)
346
- if len(tokens) <= max_tokens:
347
- return prompt
348
-
349
- # Truncate and add ellipsis
350
- truncated_tokens = tokens[:max_tokens]
351
- truncated_text = lfm_tokenizer.decode(truncated_tokens, skip_special_tokens=True)
352
- return truncated_text + " [Content truncated...]"
353
-
354
- # ================== ENHANCED INTENT DETECTION =================
355
 
356
- def detect_intent_enhanced(prompt: str) -> Dict[str, Any]:
357
  """
358
- Enhanced intent detection using AshokGakr/intent_detector (82 classes)
359
- Returns detailed intent information
360
  """
361
  if not intent_model or not intent_tokenizer:
362
- return fallback_intent_detection(prompt)
 
 
 
 
 
363
 
364
  try:
365
  inputs = intent_tokenizer(
@@ -376,74 +225,50 @@ def detect_intent_enhanced(prompt: str) -> Dict[str, Any]:
376
  predicted_class = torch.argmax(probabilities, dim=-1).item()
377
  confidence = probabilities[0][predicted_class].item()
378
 
379
- # Get intent label from model config
380
  intent_label = "unknown"
381
  if hasattr(intent_model.config, 'id2label'):
382
  intent_label = intent_model.config.id2label.get(predicted_class, "unknown")
383
 
384
- # Map to broader categories
385
- broad_category = "general"
386
- for category, intents in INTENT_CATEGORIES.items():
387
- if intent_label in intents:
388
- broad_category = category
389
- break
390
-
391
- print(f"🔍 Detected intent: {intent_label} (category: {broad_category}, confidence: {confidence:.2%})")
392
 
393
  return {
394
  "intent_label": intent_label,
395
- "broad_category": broad_category,
396
  "confidence": round(confidence, 4),
397
  "model": INTENT_MODEL_NAME,
398
  "class_id": predicted_class
399
  }
400
 
401
  except Exception as e:
402
- print(f"⚠️ Enhanced intent detection error: {e}")
403
- return fallback_intent_detection(prompt)
404
-
405
- def fallback_intent_detection(prompt: str) -> Dict[str, Any]:
406
- """Fallback keyword-based intent detection when model fails"""
407
- prompt_lower = prompt.lower()
408
-
409
- intent_keywords = {
410
- "search_query": ["search", "find", "look up", "google", "internet", "web"],
411
- "technical_query": ["code", "program", "function", "algorithm", "python", "javascript"],
412
- "help_request": ["help", "how to", "tutorial", "guide", "explain"],
413
- "analysis_request": ["analyze", "summarize", "explain", "translate"],
414
- "creative_request": ["write", "create", "generate", "story", "poem"],
415
- "question_query": ["what is", "who is", "where is", "when is", "why is", "how"],
416
- "calculation_query": ["calculate", "solve", "math", "compute"],
417
- "conversation": ["hello", "hi", "hey", "how are you"]
418
- }
419
-
420
- detected_intent = "general_query"
421
- for intent, keywords in intent_keywords.items():
422
- for keyword in keywords:
423
- if keyword in prompt_lower:
424
- detected_intent = intent
425
- break
426
-
427
- if "?" in prompt:
428
- detected_intent = "question_query"
429
-
430
- return {
431
- "intent_label": detected_intent,
432
- "broad_category": "general",
433
- "confidence": 0.5,
434
- "model": "keyword_fallback",
435
- "class_id": -1
436
- }
437
 
438
  # ================== EMOTION DETECTION =================
439
 
440
- def detect_emotion_enhanced(prompt: str) -> Dict[str, Any]:
441
  """
442
- Detect emotions using AshokGakr/emotions_detector (multi-label, 28 classes)
443
- Returns primary emotion and emotional tone
444
  """
 
 
 
 
 
 
 
 
 
445
  if not emotion_model or not emotion_tokenizer:
446
- return fallback_emotion_detection(prompt)
 
 
 
 
447
 
448
  try:
449
  inputs = emotion_tokenizer(
@@ -456,13 +281,9 @@ def detect_emotion_enhanced(prompt: str) -> Dict[str, Any]:
456
 
457
  with torch.no_grad():
458
  outputs = emotion_model(**inputs)
459
- # Multi-label classification: use sigmoid
460
  probabilities = torch.sigmoid(outputs.logits)
461
 
462
- # Get probabilities for each emotion
463
  emotion_probs = probabilities[0].cpu().numpy()
464
-
465
- # Apply threshold (0.3 based on model card optimization)
466
  threshold = 0.3
467
  detected_emotions = []
468
 
@@ -474,10 +295,8 @@ def detect_emotion_enhanced(prompt: str) -> Dict[str, Any]:
474
  "label_id": idx
475
  })
476
 
477
- # Sort by confidence
478
  detected_emotions.sort(key=lambda x: x["confidence"], reverse=True)
479
 
480
- # Determine primary emotion (highest confidence)
481
  primary_emotion = "neutral"
482
  primary_confidence = 0.0
483
 
@@ -485,134 +304,65 @@ def detect_emotion_enhanced(prompt: str) -> Dict[str, Any]:
485
  primary_emotion = detected_emotions[0]["emotion"]
486
  primary_confidence = detected_emotions[0]["confidence"]
487
 
488
- # Determine emotional tone
489
- emotional_tone = "neutral"
490
- for tone, emotions in EMOTION_TONE_MAPPING.items():
491
- if primary_emotion in emotions:
492
- emotional_tone = tone
493
- break
494
-
495
- print(f"😊 Detected emotion: {primary_emotion} (tone: {emotional_tone}, confidence: {primary_confidence:.2%})")
496
 
497
  return {
498
  "primary_emotion": primary_emotion,
499
- "emotional_tone": emotional_tone,
500
- "all_emotions": detected_emotions[:5], # Top 5 emotions
501
  "primary_confidence": round(primary_confidence, 4),
502
  "model": EMOTION_MODEL_NAME
503
  }
504
 
505
  except Exception as e:
506
- print(f"⚠️ Enhanced emotion detection error: {e}")
507
- return fallback_emotion_detection(prompt)
 
 
 
 
508
 
509
- def fallback_emotion_detection(prompt: str) -> Dict[str, Any]:
510
- """Fallback emotion detection using keyword analysis"""
511
- prompt_lower = prompt.lower()
512
-
513
- emotion_keywords = {
514
- "joy": ["happy", "glad", "excited", "great", "wonderful", "awesome"],
515
- "sadness": ["sad", "unhappy", "depressed", "cry", "tears", "mourn"],
516
- "anger": ["angry", "mad", "furious", "annoyed", "irritated", "rage"],
517
- "fear": ["scared", "afraid", "fearful", "terrified", "anxious", "nervous"],
518
- "surprise": ["surprised", "shocked", "amazed", "astonished", "wow"],
519
- "neutral": ["ok", "fine", "alright", "normal", "regular", "usual"]
520
- }
521
-
522
- detected_emotion = "neutral"
523
- confidence = 0.5
524
-
525
- for emotion, keywords in emotion_keywords.items():
526
- for keyword in keywords:
527
- if keyword in prompt_lower:
528
- detected_emotion = emotion
529
- confidence = 0.7
530
- break
531
-
532
- # Determine emotional tone
533
- emotional_tone = "neutral"
534
- for tone, emotions in EMOTION_TONE_MAPPING.items():
535
- if detected_emotion in emotions:
536
- emotional_tone = tone
537
- break
538
-
539
- return {
540
- "primary_emotion": detected_emotion,
541
- "emotional_tone": emotional_tone,
542
- "all_emotions": [{"emotion": detected_emotion, "confidence": confidence, "label_id": -1}],
543
- "primary_confidence": confidence,
544
- "model": "keyword_fallback"
545
  }
 
546
 
547
- # ================== TOOL DECISION WITH INTENT & EMOTION =================
 
 
 
548
 
549
- def enhanced_tool_decision(prompt: str, intent_info: Dict, emotion_info: Dict) -> Dict[str, Any]:
550
- """
551
- Enhanced tool decision based on intent, emotion, and content analysis
552
- """
553
- prompt_lower = prompt.lower()
554
- intent_label = intent_info.get("intent_label", "")
555
- broad_category = intent_info.get("broad_category", "")
556
- emotional_tone = emotion_info.get("emotional_tone", "neutral")
557
-
558
- search_keywords = [
559
- "search", "find", "look up", "google", "web", "internet",
560
- "what is", "who is", "where is", "when is", "why is",
561
- "current", "latest", "news", "update", "information about"
562
- ]
563
-
564
- # Decision logic based on multiple factors
565
- use_tool = False
566
- tool_name = None
567
-
568
- # Factor 1: Intent-based decision
569
- if intent_label in ["search", "find", "lookup"]:
570
- use_tool = True
571
- tool_name = "web_search"
572
-
573
- # Factor 2: Keyword-based decision
574
- if not use_tool:
575
- use_tool = any(keyword in prompt_lower for keyword in search_keywords)
576
- if use_tool:
577
- tool_name = "web_search"
578
-
579
- # Factor 3: Emotion-based adjustment (e.g., urgent queries might need search)
580
- if emotional_tone == "negative" and "urgent" in prompt_lower:
581
- use_tool = True
582
- tool_name = "web_search"
583
-
584
- # Prepare query
585
- query = prompt
586
- if "?" in prompt:
587
- query = prompt.split("?")[0] + "?"
588
- query = query[:100] # Limit query length
589
-
590
- return {
591
- "use_tool": use_tool,
592
- "tool": tool_name,
593
- "args": {"query": query},
594
- "decision_factors": {
595
- "intent_based": intent_label in ["search", "find", "lookup"],
596
- "keyword_based": use_tool and tool_name == "web_search",
597
- "emotion_adjusted": emotional_tone == "negative" and "urgent" in prompt_lower
598
- }
599
- }
600
 
601
- # ================== OPTIMIZED STREAMING GENERATION =================
602
- async def generate_reply_stream(
603
  prompt: str,
604
  files: List[UploadFile],
605
  max_tokens: int,
606
  temperature: float
607
  ) -> AsyncGenerator[str, None]:
608
- """Stream the response with enhanced intent and emotion awareness"""
609
 
610
  print(f"\n🎯 Processing: '{prompt[:50]}...'")
611
- print(f"⚙️ Parameters: max_tokens={max_tokens}, temperature={temperature}")
612
 
613
  # Validate models
614
  if not lfm_model or not lfm_tokenizer:
615
- error_msg = "LFM2.5 model not available. Please check server health."
616
  print(f"❌ {error_msg}")
617
  yield json.dumps({
618
  "type": "error",
@@ -620,151 +370,90 @@ async def generate_reply_stream(
620
  }, ensure_ascii=False) + "\n\n"
621
  return
622
 
623
- # Validate prompt length
624
- if not validate_prompt_length(prompt, MAX_CONTEXT_LENGTH - max_tokens):
625
- prompt = get_truncated_prompt(prompt, MAX_CONTEXT_LENGTH - max_tokens)
626
- print(f"⚠️ Prompt truncated to fit context window")
627
 
628
- # --- Stage 1: Intent Detection ---
629
- intent_info = detect_intent_enhanced(prompt)
630
- print(f"🧠 Intent: {intent_info['intent_label']} (Category: {intent_info['broad_category']})")
631
-
632
- # --- Stage 2: Emotion Detection ---
633
- emotion_info = detect_emotion_enhanced(prompt)
634
- print(f"😊 Emotion: {emotion_info['primary_emotion']} (Tone: {emotion_info['emotional_tone']})")
635
 
636
  # Send intent and emotion info to client
637
  yield json.dumps({
638
  "type": "context",
639
  "data": {
640
  "intent": intent_info,
641
- "emotion": emotion_info,
642
- "processing_stage": "intent_emotion_detected"
643
  }
644
  }, ensure_ascii=False) + "\n\n"
645
 
646
  await asyncio.sleep(0.01)
647
 
648
- # --- Stage 3: Tool Decision ---
649
- tool_decision = enhanced_tool_decision(prompt, intent_info, emotion_info)
650
- print(f"🔧 Tool decision: {tool_decision}")
651
-
652
- yield json.dumps({
653
- "type": "context",
654
- "data": {
655
- "tool_decision": tool_decision,
656
- "processing_stage": "tool_decision_made"
657
- }
658
- }, ensure_ascii=False) + "\n\n"
659
-
660
- await asyncio.sleep(0.01)
661
-
662
- # --- Stage 4: Tool Execution ---
663
- tool_output = ""
664
- if tool_decision.get("use_tool") and tool_decision.get("tool") == "web_search":
665
- try:
666
- query = tool_decision.get("args", {}).get("query", prompt[:50])
667
- print(f"🔍 Executing web search: '{query}'")
668
- tool_output = web_search(query)
669
- print(f"✅ Search results: {len(tool_output)} chars")
670
- except Exception as e:
671
- print(f"⚠️ Tool error: {e}")
672
- tool_output = f"Search error: {e}"
673
-
674
- # --- File Processing ---
675
  file_text = ""
676
  if files:
677
  for f in files:
678
  try:
679
- content = await f.read(min(MAX_FILE_SIZE, 1024 * 1024)) # Max 1MB per file
680
  file_text += f"\n--- File: {f.filename} ---\n"
681
  file_text += content.decode('utf-8', errors='ignore')[:2000] + "\n"
682
  print(f"✅ Processed file: {f.filename}")
683
  except Exception as e:
684
  print(f"⚠️ File error {f.filename}: {e}")
685
 
686
- # ==== ENHANCED LFM2.5 GENERATION WITH EMOTIONAL INTELLIGENCE ====
687
  try:
688
- # Build system message with emotional intelligence
689
- system_message = f"""You are GAKR AI, a helpful and knowledgeable assistant.
690
 
691
- RESPONSE APPROACH:
692
- - Start with the most direct answer
693
- - Provide supporting details when helpful
694
- - Use simple formatting (bullets, bold) for clarity
695
- - Be accurate and acknowledge uncertainty
696
 
697
- CORE PRINCIPLES:
698
- 1. Adapt your tone to match the user's emotional state
699
- 2. Address the specific intent behind the query
700
- 3. Provide accurate, helpful information
701
- 4. Use appropriate formatting for clarity
702
- 5. Acknowledge limitations when uncertain
703
 
704
- COMMUNICATION STYLE:
705
- - Clear and concise
706
- - Adapt to question complexity
707
- - Focus on practical usefulness
708
- - Avoid unnecessary repetition
709
- EMOTIONAL CONTEXT:
710
- - Detected user emotional tone: {emotion_info['emotional_tone'].upper()}
711
- - Primary emotion: {emotion_info['primary_emotion']}
712
 
713
- INTENT CONTEXT:
714
- - User intent category: {intent_info['broad_category'].upper()}
715
- - Specific intent: {intent_info['intent_label']}
 
 
 
716
 
717
- Your goal: Provide the most helpful response possible."""
718
-
719
-
720
- # Build context
721
- context_parts = []
722
 
723
- if tool_output:
724
- context_parts.append(f"SEARCH RESULTS:\n{tool_output}")
725
  if file_text:
726
- context_parts.append(f"FILE CONTENT:\n{file_text}")
727
-
728
- context = "\n\n".join(context_parts) if context_parts else "No additional context."
729
 
730
- # Build full user message with all context
731
- full_user_message = f"""USER QUERY: {prompt}
732
-
733
- ADDITIONAL CONTEXT:
734
- {context}
735
-
736
- USER ANALYSIS:
737
- - Detected Intent: {intent_info['intent_label']} (confidence: {intent_info['confidence']:.1%})
738
- - Detected Emotion: {emotion_info['primary_emotion']} (tone: {emotion_info['emotional_tone']})
739
- - Emotional Confidence: {emotion_info['primary_confidence']:.1%}
740
-
741
- Please provide a helpful response that:
742
- 1. Addresses the specific intent ({intent_info['intent_label']})
743
- 2. Appropriately responds to the emotional tone ({emotion_info['emotional_tone']})
744
- 3. Incorporates relevant information from the context
745
- 4. Provides practical, helpful assistance"""
746
-
747
- # Create messages for LFM2.5 chat template
748
- # LFM2.5 uses ChatML format: <|im_start|>role\ncontent<|im_end|>
749
  messages = [
750
  {"role": "system", "content": system_message},
751
- {"role": "user", "content": full_user_message}
752
  ]
753
 
754
- print(f"📝 Messages prepared for LFM2.5 chat template")
755
-
756
- # ====== FIXED SECTION: Proper template application ======
757
- # Apply chat template to format the messages
758
- formatted_prompt = lfm_tokenizer.apply_chat_template(
759
- messages,
760
- tokenize=False, # Don't tokenize yet
761
- add_generation_prompt=True,
762
- )
 
763
 
764
- print(f"🔍 Formatted prompt length: {len(formatted_prompt)} characters")
765
- print(f"🔍 Sample formatted prompt: {formatted_prompt[:200]}...")
766
 
767
- # Tokenize the formatted prompt
768
  inputs = lfm_tokenizer(
769
  formatted_prompt,
770
  return_tensors="pt",
@@ -772,11 +461,7 @@ Please provide a helpful response that:
772
  max_length=MAX_CONTEXT_LENGTH - max_tokens,
773
  ).to(DEVICE)
774
 
775
- input_length = inputs['input_ids'].shape[1]
776
- print(f"🔍 Tokenized input: {input_length} tokens")
777
- # ====== END FIX ======
778
-
779
- # Create streamer
780
  streamer = TextIteratorStreamer(
781
  lfm_tokenizer,
782
  skip_prompt=True,
@@ -784,67 +469,180 @@ Please provide a helpful response that:
784
  timeout=300.0
785
  )
786
 
787
- # LFM2.5 OPTIMIZED GENERATION PARAMETERS
788
- # Based on official recommendations from model card
789
  generation_config = {
790
  **inputs,
791
- "max_new_tokens": min(max_tokens, MAX_TOKENS_HARD_LIMIT),
792
  "do_sample": temperature > 0,
793
  "temperature": max(0.1, min(temperature, 1.0)),
794
- "top_p": 0.1, # LFM2.5 default
795
- "top_k": 50, # LFM2.5 default
796
- "repetition_penalty": 1.05, # LFM2.5 default
797
- "no_repeat_ngram_size": 3,
798
  "pad_token_id": lfm_tokenizer.pad_token_id,
799
  "eos_token_id": lfm_tokenizer.eos_token_id,
800
  "streamer": streamer,
801
- "use_cache": True, # Enable KV cache
802
  }
803
 
804
- # For deterministic output when temperature is 0
805
  if temperature <= 0.1:
806
  generation_config["do_sample"] = False
807
- generation_config.pop("temperature", None)
808
- generation_config.pop("top_p", None)
809
- generation_config.pop("top_k", None)
810
 
811
- print(f"🔧 Generation params: max_new_tokens={generation_config['max_new_tokens']}, temp={temperature}")
812
- print(f"📊 Model will generate with: top_p=0.1, top_k=50, repetition_penalty=1.05")
813
 
814
  # Start generation thread
815
  thread = Thread(target=lfm_model.generate, kwargs=generation_config)
816
  thread.start()
817
 
818
- print("🔄 Started streaming generation")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
819
 
820
  # Send streaming start signal
821
  yield json.dumps({
822
  "type": "streaming_start",
823
  "data": {
824
  "timestamp": time.time(),
825
- "intent": intent_info,
826
- "emotion": emotion_info,
827
- "model": LFM_MODEL_NAME
828
  }
829
  }, ensure_ascii=False) + "\n\n"
830
 
831
- # Stream tokens
 
 
 
 
832
  buffer = ""
833
  token_count = 0
834
  start_time = time.time()
835
 
836
- for new_text in streamer:
837
  if new_text:
838
  buffer += new_text
839
  token_count += 1
840
 
841
- # Send token
842
  yield json.dumps({
843
  "type": "token",
844
  "data": new_text
845
  }, ensure_ascii=False) + "\n\n"
846
 
847
- # Small delay for smooth streaming
848
  await asyncio.sleep(0.005)
849
 
850
  # Calculate stats
@@ -853,31 +651,23 @@ Please provide a helpful response that:
853
 
854
  print(f"✅ Generation complete: {token_count} tokens, {tokens_per_sec:.1f} t/s")
855
 
856
- # Send completion with analysis
857
  yield json.dumps({
858
  "type": "done",
859
  "data": {
860
  "tokens": token_count,
861
- "chars": len(buffer),
862
  "elapsed": round(elapsed, 2),
863
- "tokens_per_sec": round(tokens_per_sec, 1),
864
  "intent": intent_info,
865
- "emotion": emotion_info,
866
- "tool_used": tool_decision.get("tool"),
867
- "model": LFM_MODEL_NAME,
868
- "response_characteristics": {
869
- "emotion_aware": True,
870
- "intent_aware": True,
871
- "context_informed": bool(tool_output or file_text)
872
- }
873
  }
874
  }, ensure_ascii=False) + "\n\n"
875
 
876
  # Clean up thread
877
- thread.join(timeout=2.0)
878
 
879
  except torch.cuda.OutOfMemoryError:
880
- error_msg = "CUDA out of memory. Try reducing max_tokens or using CPU."
881
  print(f"❌ {error_msg}")
882
  yield json.dumps({
883
  "type": "error",
@@ -901,8 +691,6 @@ async def home(request: Request):
901
  return templates.TemplateResponse("index.html", {
902
  "request": request,
903
  "model_name": "LFM2.5-1.2B-Instruct",
904
- "intent_model": INTENT_MODEL_NAME,
905
- "emotion_model": EMOTION_MODEL_NAME,
906
  "environment": ENVIRONMENT
907
  })
908
 
@@ -914,23 +702,22 @@ async def analyze(
914
  temperature: float = Form(DEFAULT_TEMPERATURE),
915
  ):
916
  """
917
- Main analysis endpoint with enhanced intent and emotion detection
918
  """
919
  if not prompt and not files:
920
  raise HTTPException(400, "Empty prompt and no files provided")
921
 
922
- # Validate
923
  max_tokens = min(max_tokens, MAX_TOKENS_HARD_LIMIT)
924
  temperature = max(0.1, min(temperature, 2.0))
925
 
926
  print(f"\n📨 Request: '{prompt[:50]}...' | tokens={max_tokens} | temp={temperature}")
927
 
928
  if not lfm_model:
929
- raise HTTPException(503, "LFM2.5 model not loaded. Check server health.")
930
 
931
  async def event_stream():
932
  try:
933
- async for chunk in generate_reply_stream(prompt, files or [], max_tokens, temperature):
934
  yield f"data: {chunk}\n\n"
935
  except Exception as e:
936
  yield f"data: {json.dumps({'type': 'error', 'data': str(e)[:200]}, ensure_ascii=False)}\n\n"
@@ -960,43 +747,12 @@ async def health():
960
  "lfm_loaded": lfm_model is not None
961
  },
962
  "device": DEVICE,
963
- "cuda_available": torch.cuda.is_available(),
964
- "max_context": MAX_CONTEXT_LENGTH,
965
  "timestamp": datetime.now(timezone.utc).isoformat()
966
  }
967
 
968
- @app.get("/api/config")
969
- async def config():
970
- """Configuration endpoint"""
971
- return {
972
- "models": {
973
- "main_model": LFM_MODEL_NAME,
974
- "intent_model": INTENT_MODEL_NAME,
975
- "emotion_model": EMOTION_MODEL_NAME
976
- },
977
- "capabilities": {
978
- "intent_classes": 82,
979
- "emotion_labels": 28,
980
- "emotion_tone_mapping": list(EMOTION_TONE_MAPPING.keys())
981
- },
982
- "limits": {
983
- "max_tokens": MAX_TOKENS_HARD_LIMIT,
984
- "max_context": MAX_CONTEXT_LENGTH,
985
- "default_temperature": DEFAULT_TEMPERATURE
986
- },
987
- "features": [
988
- "intent_detection",
989
- "emotion_detection",
990
- "emotional_intelligence",
991
- "tool_integration",
992
- "file_upload",
993
- "streaming"
994
- ]
995
- }
996
-
997
  @app.get("/api/model-info")
998
  async def model_info():
999
- """Detailed model information"""
1000
  info = {
1001
  "models": {
1002
  "lfm2.5": {
@@ -1004,135 +760,23 @@ async def model_info():
1004
  "publisher": "Liquid AI",
1005
  "parameters": "1.2B",
1006
  "context_length": "32,768 tokens",
1007
- "license": "Apache 2.0",
1008
- "description": "Hybrid model designed for on-device deployment with extended pre-training and reinforcement learning",
1009
- "capabilities": ["Reasoning", "Coding", "Mathematics", "Multilingual (8 languages)", "Tool Use"],
1010
- "optimized_for": ["Edge deployment", "Low-latency applications", "Memory-constrained environments"],
1011
- "default_generation_params": {
1012
- "temperature": 0.1,
1013
- "top_p": 0.1,
1014
- "top_k": 50,
1015
- "repetition_penalty": 1.05
1016
- },
1017
- "chat_template": "ChatML format (<|im_start|>role\\ncontent<|im_end|>)",
1018
- "special_tokens": ["<|im_start|>", "<|im_end|>", "<|endoftext|>"]
1019
  },
1020
  "intent_detector": {
1021
  "name": "AshokGakr/intent_detector",
1022
- "classes": 82,
1023
- "accuracy": "98.04% (validation)",
1024
- "datasets": [
1025
- "mteb/amazon_massive_intent",
1026
- "mteb/mtop_intent",
1027
- "sonos-nlu-benchmark/snips_built_in_intents",
1028
- "Mozilla/smart_intent_dataset",
1029
- "Bhuvaneshwari/intent_classification",
1030
- "clinc/clinc_oos"
1031
- ]
1032
  },
1033
  "emotion_detector": {
1034
  "name": "AshokGakr/emotions_detector",
1035
  "labels": 28,
1036
- "type": "multi-label classification",
1037
- "base_model": "roberta-base",
1038
- "dataset": "go_emotions (Reddit data)",
1039
- "best_f1": "0.611 (weighted)"
1040
  }
1041
  }
1042
  }
1043
 
1044
- if lfm_model:
1045
- info["models"]["lfm2.5"].update({
1046
- "loaded": True,
1047
- "device": str(lfm_model.device),
1048
- "dtype": str(lfm_model.dtype),
1049
- "tokenizer_chat_template": lfm_tokenizer.chat_template is not None
1050
- })
1051
-
1052
- if intent_model:
1053
- info["models"]["intent_detector"]["loaded"] = True
1054
-
1055
- if emotion_model:
1056
- info["models"]["emotion_detector"]["loaded"] = True
1057
-
1058
  return info
1059
 
1060
- @app.post("/api/detect-intent")
1061
- async def detect_intent_endpoint(text: str = Form(...)):
1062
- """Standalone intent detection endpoint"""
1063
- if not text:
1064
- raise HTTPException(400, "Text is required")
1065
-
1066
- result = detect_intent_enhanced(text)
1067
- return result
1068
-
1069
- @app.post("/api/detect-emotion")
1070
- async def detect_emotion_endpoint(text: str = Form(...)):
1071
- """Standalone emotion detection endpoint"""
1072
- if not text:
1073
- raise HTTPException(400, "Text is required")
1074
-
1075
- result = detect_emotion_enhanced(text)
1076
- return result
1077
-
1078
- @app.post("/api/test-lfm-template")
1079
- async def test_lfm_template(text: str = Form(...), system_prompt: str = Form(None)):
1080
- """Test LFM2.5 chat template formatting"""
1081
- if not lfm_tokenizer:
1082
- raise HTTPException(503, "LFM2.5 tokenizer not loaded")
1083
-
1084
- system_msg = system_prompt or "You are a helpful AI assistant."
1085
-
1086
- messages = [
1087
- {"role": "system", "content": system_msg},
1088
- {"role": "user", "content": text}
1089
- ]
1090
-
1091
- try:
1092
- # Apply chat template
1093
- formatted = lfm_tokenizer.apply_chat_template(
1094
- messages,
1095
- tokenize=False,
1096
- add_generation_prompt=True
1097
- )
1098
-
1099
- # Decode to see the format
1100
- decoded = lfm_tokenizer.decode(lfm_tokenizer.encode(formatted), skip_special_tokens=False)
1101
-
1102
- return {
1103
- "formatted_template": formatted,
1104
- "decoded": decoded,
1105
- "messages": messages,
1106
- "special_tokens": {
1107
- "im_start": "<|im_start|>",
1108
- "im_end": "<|im_end|>",
1109
- "eos_token": lfm_tokenizer.eos_token,
1110
- "pad_token": lfm_tokenizer.pad_token
1111
- }
1112
- }
1113
- except Exception as e:
1114
- raise HTTPException(500, f"Template error: {str(e)}")
1115
-
1116
- @app.get("/api/debug-stream")
1117
- async def debug_stream():
1118
- """Test streaming with intent/emotion simulation"""
1119
- async def event_stream():
1120
- yield f"data: {json.dumps({'type': 'context', 'data': {'test': 'LFM2.5 streaming test', 'intent': {'intent_label': 'greeting', 'confidence': 0.95}, 'emotion': {'primary_emotion': 'joy', 'emotional_tone': 'positive'}}}, ensure_ascii=False)}\n\n"
1121
-
1122
- test_msg = "This is a streaming test from LFM2.5-1.2B-Instruct backend with intent and emotion awareness. "
1123
- words = test_msg.split()
1124
-
1125
- for i, word in enumerate(words):
1126
- yield f"data: {json.dumps({'type': 'token', 'data': word + ' '}, ensure_ascii=False)}\n\n"
1127
- await asyncio.sleep(0.1)
1128
-
1129
- yield f"data: {json.dumps({'type': 'done', 'data': {'test': 'complete', 'intent_aware': True, 'emotion_aware': True}}, ensure_ascii=False)}\n\n"
1130
-
1131
- return StreamingResponse(
1132
- event_stream(),
1133
- media_type="text/event-stream"
1134
- )
1135
-
1136
  # ================== ERROR HANDLERS =================
1137
 
1138
  @app.exception_handler(HTTPException)
@@ -1145,7 +789,6 @@ async def http_exception_handler(request: Request, exc: HTTPException):
1145
  @app.exception_handler(Exception)
1146
  async def generic_exception_handler(request: Request, exc: Exception):
1147
  print(f"❌ Unhandled: {exc}")
1148
- traceback.print_exc()
1149
  return JSONResponse(
1150
  status_code=500,
1151
  content={"error": "Internal server error"}
@@ -1157,13 +800,11 @@ if __name__ == "__main__":
1157
  import uvicorn
1158
 
1159
  print("\n" + "="*50)
1160
- print("🔧 ENHANCED GAKR AI - LFM2.5 WITH INTENT & EMOTION DETECTION")
1161
  print("="*50)
1162
- print(f"🎯 Intent Model: {INTENT_MODEL_NAME} (82 classes)")
1163
- print(f"😊 Emotion Model: {EMOTION_MODEL_NAME} (28 labels)")
1164
  print(f"🧠 LFM2.5 Model: {LFM_MODEL_NAME}")
1165
  print(f"⚡ Context Window: 32,768 tokens")
1166
- print(f"🏗️ Chat Template: ChatML format")
1167
  print("="*50)
1168
 
1169
  uvicorn.run(
 
1
  """
2
  FastAPI backend with optimized streaming for LFM2.5-1.2B-Instruct
3
+ Enhanced with intent detection (AshokGakr/intent_detector)
4
  and emotion detection (AshokGakr/emotions_detector) models
5
+ Using LFM2.5's native tool calling capabilities
6
  """
7
 
8
+ import os, json, traceback, asyncio, time, sys
9
  from typing import Optional, List, Dict, Any, AsyncGenerator
10
  from datetime import datetime, timezone
11
  from contextlib import asynccontextmanager
12
 
13
  import torch
 
14
  from fastapi.staticfiles import StaticFiles
15
  from fastapi import (
16
  FastAPI,
 
27
  from transformers import AutoTokenizer, AutoModelForSequenceClassification, AutoModelForCausalLM, TextIteratorStreamer
28
  from threading import Thread
29
 
 
 
 
 
 
 
 
 
 
30
  # ================== CONFIG =================
31
  ENVIRONMENT = os.getenv("ENVIRONMENT", "development")
 
32
  DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
33
  print(f"🔧 Using device: {DEVICE}")
 
 
 
 
 
 
34
 
35
+ # Models
36
+ INTENT_MODEL_NAME = "AshokGakr/intent_detector"
37
+ EMOTION_MODEL_NAME = "AshokGakr/emotions_detector"
38
  LFM_MODEL_NAME = "LiquidAI/LFM2.5-1.2B-Instruct"
39
 
40
  MAX_FILE_SIZE = 50 * 1024 * 1024
41
  MAX_TOKENS_DEFAULT = 1024
42
+ MAX_TOKENS_HARD_LIMIT = 4096
43
+ DEFAULT_TEMPERATURE = 0.7
44
+ MAX_CONTEXT_LENGTH = 8192
 
45
 
46
  # Global model variables
47
  intent_tokenizer = None
 
60
  os.makedirs(STATIC_DIR, exist_ok=True)
61
  templates = Jinja2Templates(directory=TEMPLATES_DIR)
62
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
63
  # ================== MODEL LOADING =================
64
 
65
  async def load_models():
66
+ """Load and initialize all ML models"""
67
  global intent_tokenizer, intent_model, emotion_tokenizer, emotion_model
68
  global lfm_tokenizer, lfm_model
69
 
70
  print("\n" + "="*50)
71
+ print("🔄 LOADING AI MODELS")
72
  print("="*50)
73
 
74
+ # Load intent detection model
 
75
  try:
76
  print(f"📥 Loading intent detection model: {INTENT_MODEL_NAME}")
77
  intent_tokenizer = AutoTokenizer.from_pretrained(INTENT_MODEL_NAME)
78
  intent_model = AutoModelForSequenceClassification.from_pretrained(INTENT_MODEL_NAME)
79
  intent_model.to(DEVICE).eval()
80
+ print(f"✅ Intent model loaded successfully")
 
81
  except Exception as e:
82
+ print(f"❌ Intent model failed to load: {e}")
83
+ intent_tokenizer = None
84
+ intent_model = None
 
 
 
 
 
 
 
85
 
86
+ # Load emotion detection model
 
87
  try:
88
  print(f"📥 Loading emotion detection model: {EMOTION_MODEL_NAME}")
89
  emotion_tokenizer = AutoTokenizer.from_pretrained(EMOTION_MODEL_NAME)
 
93
  num_labels=28
94
  )
95
  emotion_model.to(DEVICE).eval()
96
+ print(f"✅ Emotion model loaded successfully")
 
97
  except Exception as e:
98
+ print(f"❌ Emotion model failed to load: {e}")
99
+ emotion_tokenizer = None
100
+ emotion_model = None
 
 
 
 
 
 
 
101
 
102
+ # Load LFM2.5 model
 
103
  try:
104
  print(f"📥 Loading LFM2.5-1.2B-Instruct model")
105
 
 
106
  lfm_tokenizer = AutoTokenizer.from_pretrained(
107
  LFM_MODEL_NAME,
108
+ padding_side="left",
109
  trust_remote_code=False,
110
  use_fast=True
111
  )
112
 
 
 
 
113
  print(f" Tokenizer loaded with vocab size: {lfm_tokenizer.vocab_size}")
 
114
 
 
115
  load_kwargs = {
116
  "torch_dtype": torch.bfloat16 if DEVICE == "cuda" else torch.float32,
117
  "trust_remote_code": False,
 
123
  **load_kwargs
124
  )
125
 
 
126
  if DEVICE != "cuda" or load_kwargs.get("device_map") is None:
127
  lfm_model = lfm_model.to(DEVICE)
128
 
129
  lfm_model.eval()
130
+ lfm_model.config.use_cache = True
 
 
131
 
132
  print(f"✅ LFM2.5-1.2B-Instruct loaded successfully")
133
  print(f" Model device: {lfm_model.device}")
 
 
134
  print(f" Context window: 32,768 tokens")
 
135
 
136
  except Exception as e:
137
  print(f"❌ Failed to load LFM2.5 model: {e}")
138
  traceback.print_exc()
139
+ lfm_tokenizer = None
140
+ lfm_model = None
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
141
 
142
  print("="*50)
143
  print(f"📊 MODEL LOADING SUMMARY:")
144
+ print(f" • Intent Model: {'✅' if intent_model else '❌'}")
145
+ print(f" • Emotion Model: {'✅' if emotion_model else '❌'}")
146
+ print(f" • LFM2.5 Model: {'✅' if lfm_model else '❌'}")
147
  print(f" • Device: {DEVICE}")
 
148
  print("="*50 + "\n")
149
 
150
+ return bool(lfm_model)
151
 
152
  async def cleanup_models():
153
  """Cleanup models on shutdown"""
154
  print("🧹 Cleaning up models...")
155
  if torch.cuda.is_available():
156
  torch.cuda.empty_cache()
 
157
  print("✅ Cleanup complete")
158
 
159
  # ================== LIFESPAN MANAGER =================
 
162
  async def lifespan(app: FastAPI):
163
  """Modern lifespan context manager for FastAPI"""
164
  print("\n" + "="*50)
165
+ print("🚀 STARTING GAKR AI ASSISTANT")
166
  print("="*50)
 
 
 
 
167
 
168
  model_loaded = await load_models()
169
  if not model_loaded:
 
179
  # ================== APP SETUP =================
180
 
181
  app = FastAPI(
182
+ title="GAKR AI Assistant",
183
+ description="AI pipeline with intent detection, emotion analysis, and LFM2.5-1.2B-Instruct with native tool calling",
184
+ version="6.0.0",
185
  lifespan=lifespan
186
  )
187
 
 
195
  allow_headers=["*"]
196
  )
197
 
198
+ # ================== INTENT DETECTION =================
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
199
 
200
+ def detect_intent(prompt: str) -> Dict[str, Any]:
201
  """
202
+ Intent detection using AshokGakr/intent_detector (82 classes)
203
+ No fallback logic - uses model output directly
204
  """
205
  if not intent_model or not intent_tokenizer:
206
+ return {
207
+ "intent_label": "model_not_loaded",
208
+ "confidence": 0.0,
209
+ "model": INTENT_MODEL_NAME,
210
+ "class_id": -1
211
+ }
212
 
213
  try:
214
  inputs = intent_tokenizer(
 
225
  predicted_class = torch.argmax(probabilities, dim=-1).item()
226
  confidence = probabilities[0][predicted_class].item()
227
 
 
228
  intent_label = "unknown"
229
  if hasattr(intent_model.config, 'id2label'):
230
  intent_label = intent_model.config.id2label.get(predicted_class, "unknown")
231
 
232
+ print(f"🔍 Detected intent: {intent_label} (confidence: {confidence:.2%})")
 
 
 
 
 
 
 
233
 
234
  return {
235
  "intent_label": intent_label,
 
236
  "confidence": round(confidence, 4),
237
  "model": INTENT_MODEL_NAME,
238
  "class_id": predicted_class
239
  }
240
 
241
  except Exception as e:
242
+ print(f"⚠️ Intent detection error: {e}")
243
+ return {
244
+ "intent_label": "detection_error",
245
+ "confidence": 0.0,
246
+ "model": INTENT_MODEL_NAME,
247
+ "class_id": -1
248
+ }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
249
 
250
  # ================== EMOTION DETECTION =================
251
 
252
+ def detect_emotion(prompt: str) -> Dict[str, Any]:
253
  """
254
+ Emotion detection using AshokGakr/emotions_detector (multi-label, 28 classes)
255
+ No fallback logic - uses model output directly
256
  """
257
+ EMOTION_LABELS = [
258
+ "admiration", "amusement", "anger", "annoyance", "approval",
259
+ "caring", "confusion", "curiosity", "desire", "disappointment",
260
+ "disapproval", "disgust", "embarrassment", "excitement", "fear",
261
+ "gratitude", "grief", "joy", "love", "nervousness",
262
+ "optimism", "pride", "realization", "relief", "remorse",
263
+ "sadness", "surprise", "neutral"
264
+ ]
265
+
266
  if not emotion_model or not emotion_tokenizer:
267
+ return {
268
+ "primary_emotion": "model_not_loaded",
269
+ "primary_confidence": 0.0,
270
+ "model": EMOTION_MODEL_NAME
271
+ }
272
 
273
  try:
274
  inputs = emotion_tokenizer(
 
281
 
282
  with torch.no_grad():
283
  outputs = emotion_model(**inputs)
 
284
  probabilities = torch.sigmoid(outputs.logits)
285
 
 
286
  emotion_probs = probabilities[0].cpu().numpy()
 
 
287
  threshold = 0.3
288
  detected_emotions = []
289
 
 
295
  "label_id": idx
296
  })
297
 
 
298
  detected_emotions.sort(key=lambda x: x["confidence"], reverse=True)
299
 
 
300
  primary_emotion = "neutral"
301
  primary_confidence = 0.0
302
 
 
304
  primary_emotion = detected_emotions[0]["emotion"]
305
  primary_confidence = detected_emotions[0]["confidence"]
306
 
307
+ print(f"😊 Detected emotion: {primary_emotion} (confidence: {primary_confidence:.2%})")
 
 
 
 
 
 
 
308
 
309
  return {
310
  "primary_emotion": primary_emotion,
311
+ "all_emotions": detected_emotions[:5],
 
312
  "primary_confidence": round(primary_confidence, 4),
313
  "model": EMOTION_MODEL_NAME
314
  }
315
 
316
  except Exception as e:
317
+ print(f"⚠️ Emotion detection error: {e}")
318
+ return {
319
+ "primary_emotion": "detection_error",
320
+ "primary_confidence": 0.0,
321
+ "model": EMOTION_MODEL_NAME
322
+ }
323
 
324
+ # ================== TOOL DEFINITION FOR LFM2.5 =================
325
+ # LFM2.5 has built-in tool calling capabilities. We'll define a web search tool
326
+ # and let the model decide when to use it.
327
+
328
+ TOOL_DEFINITION = {
329
+ "type": "function",
330
+ "function": {
331
+ "name": "web_search",
332
+ "description": "Search the web for current information, facts, or latest news.",
333
+ "parameters": {
334
+ "type": "object",
335
+ "properties": {
336
+ "query": {
337
+ "type": "string",
338
+ "description": "The search query to look up"
339
+ }
340
+ },
341
+ "required": ["query"]
342
+ }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
343
  }
344
+ }
345
 
346
+ # Simple web search function (replace with actual implementation)
347
+ def execute_web_search(query: str) -> str:
348
+ """Execute web search - placeholder implementation"""
349
+ return f"Search results for: '{query}'. [This is a placeholder - integrate with actual search API]"
350
 
351
+ # ================== STREAMING GENERATION WITH TOOL CALLING =================
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
352
 
353
+ async def generate_with_tool_calling(
 
354
  prompt: str,
355
  files: List[UploadFile],
356
  max_tokens: int,
357
  temperature: float
358
  ) -> AsyncGenerator[str, None]:
359
+ """Stream the response with LFM2.5 native tool calling"""
360
 
361
  print(f"\n🎯 Processing: '{prompt[:50]}...'")
 
362
 
363
  # Validate models
364
  if not lfm_model or not lfm_tokenizer:
365
+ error_msg = "LFM2.5 model not available."
366
  print(f"❌ {error_msg}")
367
  yield json.dumps({
368
  "type": "error",
 
370
  }, ensure_ascii=False) + "\n\n"
371
  return
372
 
373
+ # --- Stage 1: Intent Detection (no fallback) ---
374
+ intent_info = detect_intent(prompt)
375
+ print(f"🧠 Intent: {intent_info['intent_label']}")
 
376
 
377
+ # --- Stage 2: Emotion Detection (no fallback) ---
378
+ emotion_info = detect_emotion(prompt)
379
+ print(f"😊 Emotion: {emotion_info['primary_emotion']}")
 
 
 
 
380
 
381
  # Send intent and emotion info to client
382
  yield json.dumps({
383
  "type": "context",
384
  "data": {
385
  "intent": intent_info,
386
+ "emotion": emotion_info
 
387
  }
388
  }, ensure_ascii=False) + "\n\n"
389
 
390
  await asyncio.sleep(0.01)
391
 
392
+ # --- File Processing (unchanged) ---
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
393
  file_text = ""
394
  if files:
395
  for f in files:
396
  try:
397
+ content = await f.read(min(MAX_FILE_SIZE, 1024 * 1024))
398
  file_text += f"\n--- File: {f.filename} ---\n"
399
  file_text += content.decode('utf-8', errors='ignore')[:2000] + "\n"
400
  print(f"✅ Processed file: {f.filename}")
401
  except Exception as e:
402
  print(f"⚠️ File error {f.filename}: {e}")
403
 
404
+ # ==== LFM2.5 GENERATION WITH TOOL CALLING ====
405
  try:
406
+ # Build system message with tool definition
407
+ system_message = f"""You are GAKR AI, a helpful assistant with access to tools.
408
 
409
+ # TOOLS AVAILABLE
410
+ You have access to the following tool:
411
+ - web_search(query): Search the web for current information
 
 
412
 
413
+ # TOOL USAGE INSTRUCTIONS
414
+ 1. Use web_search when you need current, factual, or up-to-date information
415
+ 2. If the user asks about recent events, news, or facts that may have changed
416
+ 3. For general knowledge questions that might benefit from web verification
 
 
417
 
418
+ # CONTEXT
419
+ User Intent: {intent_info['intent_label']} (confidence: {intent_info['confidence']:.1%})
420
+ User Emotion: {emotion_info['primary_emotion']} (confidence: {emotion_info['primary_confidence']:.1%})
 
 
 
 
 
421
 
422
+ # INSTRUCTIONS
423
+ 1. Analyze if the query needs web search
424
+ 2. If yes, call web_search with appropriate query
425
+ 3. If no, respond directly
426
+ 4. Adapt tone to detected emotion
427
+ 5. Address the specific intent
428
 
429
+ File content will be provided separately if available."""
430
+
431
+ # Build user message
432
+ user_message = f"""USER QUERY: {prompt}"""
 
433
 
 
 
434
  if file_text:
435
+ user_message += f"\n\nFILE CONTENT:{file_text}"
 
 
436
 
437
+ # Create messages with tool definition
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
438
  messages = [
439
  {"role": "system", "content": system_message},
440
+ {"role": "user", "content": user_message}
441
  ]
442
 
443
+ # Apply chat template
444
+ try:
445
+ formatted_prompt = lfm_tokenizer.apply_chat_template(
446
+ messages,
447
+ tokenize=False,
448
+ add_generation_prompt=True,
449
+ )
450
+ except:
451
+ # Manual fallback template if apply_chat_template fails
452
+ formatted_prompt = f"<|im_start|>system\n{system_message}<|im_end|>\n<|im_start|>user\n{user_message}<|im_end|>\n<|im_start|>assistant\n"
453
 
454
+ print(f"🔍 Formatted prompt length: {len(formatted_prompt)} chars")
 
455
 
456
+ # First generation pass - check if tool call is needed
457
  inputs = lfm_tokenizer(
458
  formatted_prompt,
459
  return_tensors="pt",
 
461
  max_length=MAX_CONTEXT_LENGTH - max_tokens,
462
  ).to(DEVICE)
463
 
464
+ # Create streamer for first pass
 
 
 
 
465
  streamer = TextIteratorStreamer(
466
  lfm_tokenizer,
467
  skip_prompt=True,
 
469
  timeout=300.0
470
  )
471
 
 
 
472
  generation_config = {
473
  **inputs,
474
+ "max_new_tokens": min(512, max_tokens), # Limited for first pass
475
  "do_sample": temperature > 0,
476
  "temperature": max(0.1, min(temperature, 1.0)),
477
+ "top_p": 0.1,
478
+ "top_k": 50,
479
+ "repetition_penalty": 1.05,
 
480
  "pad_token_id": lfm_tokenizer.pad_token_id,
481
  "eos_token_id": lfm_tokenizer.eos_token_id,
482
  "streamer": streamer,
483
+ "use_cache": True,
484
  }
485
 
 
486
  if temperature <= 0.1:
487
  generation_config["do_sample"] = False
 
 
 
488
 
489
+ print("🔄 Starting generation (first pass)")
 
490
 
491
  # Start generation thread
492
  thread = Thread(target=lfm_model.generate, kwargs=generation_config)
493
  thread.start()
494
 
495
+ # Collect first pass response
496
+ first_pass_response = ""
497
+ for text in streamer:
498
+ first_pass_response += text
499
+
500
+ thread.join(timeout=2.0)
501
+
502
+ print(f"📝 First pass response: {first_pass_response[:200]}...")
503
+
504
+ # Check for tool call patterns in response
505
+ # LFM2.5 might format tool calls differently - checking common patterns
506
+ tool_call_detected = False
507
+ search_query = None
508
+
509
+ # Check for potential tool call patterns
510
+ if "search" in first_pass_response.lower() or "look up" in first_pass_response.lower():
511
+ # Try to extract query from response
512
+ import re
513
+ search_patterns = [
514
+ r"search for ['\"]([^'\"]+)['\"]",
515
+ r"look up ['\"]([^'\"]+)['\"]",
516
+ r"query: ['\"]([^'\"]+)['\"]",
517
+ ]
518
+
519
+ for pattern in search_patterns:
520
+ match = re.search(pattern, first_pass_response, re.IGNORECASE)
521
+ if match:
522
+ search_query = match.group(1)
523
+ tool_call_detected = True
524
+ break
525
+
526
+ # If tool call detected, execute and generate final response
527
+ if tool_call_detected and search_query:
528
+ print(f"🔍 Tool call detected: web_search('{search_query}')")
529
+
530
+ # Send tool call event
531
+ yield json.dumps({
532
+ "type": "tool_call",
533
+ "data": {
534
+ "tool": "web_search",
535
+ "query": search_query,
536
+ "timestamp": time.time()
537
+ }
538
+ }, ensure_ascii=False) + "\n\n"
539
+
540
+ # Execute tool
541
+ try:
542
+ search_results = execute_web_search(search_query)
543
+ print(f"✅ Search executed: {len(search_results)} chars")
544
+
545
+ # Send tool result event
546
+ yield json.dumps({
547
+ "type": "tool_result",
548
+ "data": {
549
+ "tool": "web_search",
550
+ "result_preview": search_results[:100] + "..." if len(search_results) > 100 else search_results,
551
+ "timestamp": time.time()
552
+ }
553
+ }, ensure_ascii=False) + "\n\n"
554
+
555
+ # Prepare second generation with tool results
556
+ tool_message = f"I searched for '{search_query}' and found:\n{search_results}"
557
+
558
+ second_pass_messages = messages + [
559
+ {"role": "assistant", "content": first_pass_response},
560
+ {"role": "user", "content": tool_message}
561
+ ]
562
+
563
+ try:
564
+ second_pass_prompt = lfm_tokenizer.apply_chat_template(
565
+ second_pass_messages,
566
+ tokenize=False,
567
+ add_generation_prompt=True,
568
+ )
569
+ except:
570
+ second_pass_prompt = formatted_prompt + first_pass_response + f"\n<|im_end|>\n<|im_start|>user\n{tool_message}<|im_end|>\n<|im_start|>assistant\n"
571
+
572
+ except Exception as e:
573
+ print(f"⚠️ Tool execution error: {e}")
574
+ # If tool fails, continue with original response
575
+ second_pass_prompt = formatted_prompt
576
+ tool_call_detected = False
577
+
578
+ else:
579
+ # No tool call needed, use first pass response
580
+ print("✅ No tool call needed, using direct response")
581
+ second_pass_prompt = formatted_prompt
582
+ tool_call_detected = False
583
+
584
+ # Final generation (streaming)
585
+ final_inputs = lfm_tokenizer(
586
+ second_pass_prompt,
587
+ return_tensors="pt",
588
+ truncation=True,
589
+ max_length=MAX_CONTEXT_LENGTH - max_tokens,
590
+ ).to(DEVICE)
591
+
592
+ final_streamer = TextIteratorStreamer(
593
+ lfm_tokenizer,
594
+ skip_prompt=True,
595
+ skip_special_tokens=True,
596
+ timeout=300.0
597
+ )
598
+
599
+ final_config = {
600
+ **final_inputs,
601
+ "max_new_tokens": min(max_tokens, MAX_TOKENS_HARD_LIMIT),
602
+ "do_sample": temperature > 0,
603
+ "temperature": max(0.1, min(temperature, 1.0)),
604
+ "top_p": 0.1,
605
+ "top_k": 50,
606
+ "repetition_penalty": 1.05,
607
+ "pad_token_id": lfm_tokenizer.pad_token_id,
608
+ "eos_token_id": lfm_tokenizer.eos_token_id,
609
+ "streamer": final_streamer,
610
+ "use_cache": True,
611
+ }
612
+
613
+ if temperature <= 0.1:
614
+ final_config["do_sample"] = False
615
+
616
+ print("🔄 Starting final generation")
617
 
618
  # Send streaming start signal
619
  yield json.dumps({
620
  "type": "streaming_start",
621
  "data": {
622
  "timestamp": time.time(),
623
+ "tool_called": tool_call_detected
 
 
624
  }
625
  }, ensure_ascii=False) + "\n\n"
626
 
627
+ # Start final generation thread
628
+ final_thread = Thread(target=lfm_model.generate, kwargs=final_config)
629
+ final_thread.start()
630
+
631
+ # Stream final response
632
  buffer = ""
633
  token_count = 0
634
  start_time = time.time()
635
 
636
+ for new_text in final_streamer:
637
  if new_text:
638
  buffer += new_text
639
  token_count += 1
640
 
 
641
  yield json.dumps({
642
  "type": "token",
643
  "data": new_text
644
  }, ensure_ascii=False) + "\n\n"
645
 
 
646
  await asyncio.sleep(0.005)
647
 
648
  # Calculate stats
 
651
 
652
  print(f"✅ Generation complete: {token_count} tokens, {tokens_per_sec:.1f} t/s")
653
 
654
+ # Send completion
655
  yield json.dumps({
656
  "type": "done",
657
  "data": {
658
  "tokens": token_count,
 
659
  "elapsed": round(elapsed, 2),
660
+ "tool_used": "web_search" if tool_call_detected else None,
661
  "intent": intent_info,
662
+ "emotion": emotion_info
 
 
 
 
 
 
 
663
  }
664
  }, ensure_ascii=False) + "\n\n"
665
 
666
  # Clean up thread
667
+ final_thread.join(timeout=2.0)
668
 
669
  except torch.cuda.OutOfMemoryError:
670
+ error_msg = "CUDA out of memory. Try reducing max_tokens."
671
  print(f"❌ {error_msg}")
672
  yield json.dumps({
673
  "type": "error",
 
691
  return templates.TemplateResponse("index.html", {
692
  "request": request,
693
  "model_name": "LFM2.5-1.2B-Instruct",
 
 
694
  "environment": ENVIRONMENT
695
  })
696
 
 
702
  temperature: float = Form(DEFAULT_TEMPERATURE),
703
  ):
704
  """
705
+ Main analysis endpoint with LFM2.5 tool calling
706
  """
707
  if not prompt and not files:
708
  raise HTTPException(400, "Empty prompt and no files provided")
709
 
 
710
  max_tokens = min(max_tokens, MAX_TOKENS_HARD_LIMIT)
711
  temperature = max(0.1, min(temperature, 2.0))
712
 
713
  print(f"\n📨 Request: '{prompt[:50]}...' | tokens={max_tokens} | temp={temperature}")
714
 
715
  if not lfm_model:
716
+ raise HTTPException(503, "LFM2.5 model not loaded.")
717
 
718
  async def event_stream():
719
  try:
720
+ async for chunk in generate_with_tool_calling(prompt, files or [], max_tokens, temperature):
721
  yield f"data: {chunk}\n\n"
722
  except Exception as e:
723
  yield f"data: {json.dumps({'type': 'error', 'data': str(e)[:200]}, ensure_ascii=False)}\n\n"
 
747
  "lfm_loaded": lfm_model is not None
748
  },
749
  "device": DEVICE,
 
 
750
  "timestamp": datetime.now(timezone.utc).isoformat()
751
  }
752
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
753
  @app.get("/api/model-info")
754
  async def model_info():
755
+ """Model information"""
756
  info = {
757
  "models": {
758
  "lfm2.5": {
 
760
  "publisher": "Liquid AI",
761
  "parameters": "1.2B",
762
  "context_length": "32,768 tokens",
763
+ "tool_calling": "native",
764
+ "description": "Model with built-in tool calling capabilities"
 
 
 
 
 
 
 
 
 
 
765
  },
766
  "intent_detector": {
767
  "name": "AshokGakr/intent_detector",
768
+ "classes": 82
 
 
 
 
 
 
 
 
 
769
  },
770
  "emotion_detector": {
771
  "name": "AshokGakr/emotions_detector",
772
  "labels": 28,
773
+ "type": "multi-label classification"
 
 
 
774
  }
775
  }
776
  }
777
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
778
  return info
779
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
780
  # ================== ERROR HANDLERS =================
781
 
782
  @app.exception_handler(HTTPException)
 
789
  @app.exception_handler(Exception)
790
  async def generic_exception_handler(request: Request, exc: Exception):
791
  print(f"❌ Unhandled: {exc}")
 
792
  return JSONResponse(
793
  status_code=500,
794
  content={"error": "Internal server error"}
 
800
  import uvicorn
801
 
802
  print("\n" + "="*50)
803
+ print("🔧 GAKR AI - LFM2.5 WITH TOOL CALLING")
804
  print("="*50)
 
 
805
  print(f"🧠 LFM2.5 Model: {LFM_MODEL_NAME}")
806
  print(f"⚡ Context Window: 32,768 tokens")
807
+ print(f"🛠️ Tool Calling: Native")
808
  print("="*50)
809
 
810
  uvicorn.run(