wuhp commited on
Commit
63f8f90
Β·
verified Β·
1 Parent(s): 615d29a

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +45 -103
app.py CHANGED
@@ -79,7 +79,7 @@ class ExtensionManager:
79
 
80
  def __init__(self):
81
  self.extensions: Dict[str, BaseExtension] = {}
82
- self.extension_interactions: Dict[str, List[str]] = {} # Track which extensions work together
83
  self.load_extensions()
84
 
85
  def load_extensions(self):
@@ -117,21 +117,18 @@ class ExtensionManager:
117
  traceback.print_exc()
118
 
119
  log(f"πŸ“Š Total extensions loaded: {len(self.extensions)}")
120
-
121
- # Detect extension relationships
122
  self._detect_extension_relationships()
123
 
124
  def _detect_extension_relationships(self):
125
  """Detect which extensions can work together based on capabilities"""
126
  log("πŸ”— Detecting extension relationships...")
127
 
128
- # Common patterns of extensions that work well together
129
  relationships = {
130
- 'yfinance': ['visualization', 'deep_research'], # Finance + viz/research
131
- 'deep_research': ['visualization', 'yfinance'], # Research + viz/finance
132
- 'youtube': ['deep_research', 'visualization'], # YouTube + research/viz
133
- 'timer': ['deep_research'], # Timer + research
134
- 'visualization': ['yfinance', 'deep_research', 'youtube'] # Viz works with data sources
135
  }
136
 
137
  for ext_name, compatible in relationships.items():
@@ -162,20 +159,15 @@ class ExtensionManager:
162
  if enabled_exts:
163
  prompt += "\n\n# ENABLED EXTENSIONS\nYou currently have these extensions enabled:\n\n"
164
  for ext in enabled_exts:
165
- # Initialize state for user if needed
166
  ext.initialize_state(user_id)
167
-
168
- # Get extension context
169
  prompt += f"## {ext.display_name}\n{ext.get_system_context()}\n\n"
170
 
171
- # Add current state summary if extension provides it
172
  state = ext.get_state(user_id)
173
  if state and hasattr(ext, 'get_state_summary'):
174
  state_summary = ext.get_state_summary(user_id)
175
  if state_summary:
176
  prompt += f"**Current State:** {state_summary}\n\n"
177
 
178
- # Add multi-extension reasoning guidance
179
  if len(enabled_exts) > 1:
180
  prompt += self._build_multi_extension_guidance(enabled_list)
181
 
@@ -186,7 +178,6 @@ class ExtensionManager:
186
  guidance = "\n\n# MULTI-EXTENSION REASONING\n"
187
  guidance += "You have multiple extensions enabled. You can combine them intelligently:\n\n"
188
 
189
- # Detect enabled combinations
190
  combinations = []
191
 
192
  if 'yfinance' in enabled_list and 'visualization' in enabled_list:
@@ -228,7 +219,6 @@ class ExtensionManager:
228
 
229
  suggestions = []
230
 
231
- # If finance extension returned data with arrays, suggest visualization
232
  if current_extension == 'yfinance' and isinstance(result, dict):
233
  if 'dates' in result and 'close_prices' in result and 'visualization' in compatible:
234
  suggestions.append({
@@ -245,7 +235,6 @@ class ExtensionManager:
245
  'data_available': True
246
  })
247
 
248
- # If research completed, suggest visualization of findings
249
  if current_extension == 'deep_research' and isinstance(result, dict):
250
  if result.get('success') and 'visualization' in compatible:
251
  suggestions.append({
@@ -255,7 +244,6 @@ class ExtensionManager:
255
  'data_available': True
256
  })
257
 
258
- # If YouTube analysis completed, suggest research on topic
259
  if current_extension == 'youtube' and isinstance(result, dict):
260
  if result.get('success') and 'deep_research' in compatible:
261
  suggestions.append({
@@ -291,12 +279,10 @@ class ExtensionManager:
291
 
292
  log(f"πŸ”§ Executing: {function_name}({json.dumps(args, indent=2)[:100]}...)")
293
 
294
- # Find which extension owns this function
295
  handled = False
296
  for ext_name in enabled_list:
297
  ext = self.get_extension(ext_name)
298
  if ext:
299
- # Check if this function is in the extension's tools
300
  for tool in ext.get_tools():
301
  if hasattr(tool, 'function_declarations'):
302
  for func_decl in tool.function_declarations:
@@ -343,13 +329,12 @@ class ImprovedOrchestrator:
343
  self.user_id = user_id
344
  self.enabled_extensions = enabled_extensions
345
  self.search_chat = client.chats.create(model="gemini-2.5-flash")
346
- self.execution_history = [] # Track what's been done
347
 
348
  def analyze_query(self, query: str, file_parts: List = None) -> Dict[str, Any]:
349
  """Analyze query to determine optimal execution strategy"""
350
  log("🧠 Analyzing query to determine strategy...")
351
 
352
- # Check for explicit extension requests first
353
  query_lower = query.lower()
354
  explicit_extensions = []
355
 
@@ -409,14 +394,12 @@ Respond in JSON format:
409
 
410
  if response.candidates and response.candidates[0].content and response.candidates[0].content.parts:
411
  text = response.candidates[0].content.parts[0].text
412
- # Extract JSON from markdown code blocks if present
413
  json_match = re.search(r'```(?:json)?\s*(\{.*?\})\s*```', text, re.DOTALL)
414
  if json_match:
415
  text = json_match.group(1)
416
 
417
  analysis = json.loads(text)
418
 
419
- # Force explicit extensions into the analysis
420
  if explicit_extensions:
421
  analysis['explicit_extensions'] = explicit_extensions
422
  analysis['multi_step'] = True
@@ -427,7 +410,6 @@ Respond in JSON format:
427
  except Exception as e:
428
  log(f"⚠️ Query analysis failed: {e}, using heuristics")
429
 
430
- # Fallback to heuristics with explicit extensions
431
  heuristic = self._heuristic_analysis(query)
432
  if explicit_extensions:
433
  heuristic['explicit_extensions'] = explicit_extensions
@@ -438,19 +420,15 @@ Respond in JSON format:
438
  """Fallback heuristic analysis"""
439
  query_lower = query.lower()
440
 
441
- # Check for search indicators
442
  search_keywords = ['search', 'find', 'look up', 'what is', 'latest', 'current', 'news']
443
  needs_search = "yes" if any(kw in query_lower for kw in search_keywords) else "maybe"
444
 
445
- # Check for multi-step indicators
446
  multi_step_keywords = ['then', 'after that', 'and then', 'research and', 'analyze and']
447
  multi_step = any(kw in query_lower for kw in multi_step_keywords)
448
 
449
- # Check for context references
450
  context_keywords = ['the', 'my', 'that', 'previous', 'earlier']
451
  uses_context = any(kw in query_lower for kw in context_keywords)
452
 
453
- # Determine complexity
454
  word_count = len(query.split())
455
  if word_count < 10 and not multi_step:
456
  complexity = "simple"
@@ -469,33 +447,27 @@ Respond in JSON format:
469
  }
470
 
471
  def execute_with_planning(self, query: str, file_parts: List = None,
472
- reasoning_budget: int = -1) -> Tuple[str, List, List]:
473
  """Execute query with planning and adaptive strategy"""
474
 
475
- # Step 1: Analyze query
476
  analysis = self.analyze_query(query, file_parts)
477
 
478
- # Step 2: Execute based on analysis
479
  search_results = ""
480
  search_citations = None
481
 
482
- # Handle search if needed
483
  if analysis['needs_search'] in ['yes', 'maybe']:
484
  log("πŸ” Executing search phase...")
485
  search_results, search_citations = self.call_search_agent(query, file_parts)
486
 
487
- # Step 3: Execute with tools
488
  log("πŸ› οΈ Executing tool phase...")
489
  tool_results = []
490
  generated_images = []
491
  thoughts = ""
492
 
493
- # Build context-aware prompt
494
  system_prompt = self.extension_manager.build_system_prompt(
495
  self.user_id, self.enabled_extensions
496
  )
497
 
498
- # Enhanced instructions for multi-step tasks
499
  if analysis['multi_step']:
500
  system_prompt += """
501
 
@@ -515,7 +487,6 @@ CONTEXT AWARENESS:
515
  - Reference specific items by their IDs or names from the state
516
  """
517
 
518
- # Add explicit extension requirements
519
  if analysis.get('explicit_extensions'):
520
  explicit_exts = analysis['explicit_extensions']
521
  system_prompt += f"""
@@ -535,7 +506,6 @@ YOU MUST use these extensions in your response:
535
 
536
  system_prompt += "\n\nDo NOT skip these explicitly requested extensions!\n"
537
 
538
- # Multi-turn tool execution
539
  max_rounds = 5 if analysis['complexity'] == 'complex' else 3
540
  current_round = 0
541
 
@@ -556,11 +526,9 @@ YOU MUST use these extensions in your response:
556
  if not function_calls:
557
  log(f"βœ… No more tools needed after round {current_round}")
558
  if text_response:
559
- # Agent provided final answer
560
  return text_response, tool_results, generated_images, thoughts, search_citations
561
  break
562
 
563
- # Execute function calls
564
  results = self.extension_manager.handle_function_calls(
565
  self.user_id, self.enabled_extensions, function_calls
566
  )
@@ -568,17 +536,17 @@ YOU MUST use these extensions in your response:
568
  for (tool_name, result) in results:
569
  tool_results.append((tool_name, result))
570
 
571
- # Extract generated images IMMEDIATELY
572
  if isinstance(result, dict) and 'image_base64' in result:
573
- generated_images.append({
574
  'base64': result['image_base64'],
575
  'title': result.get('message', 'Generated visualization'),
576
  'filepath': result.get('filepath', '')
577
- })
 
578
  log(f"πŸ“Š Captured visualization: {result.get('message', 'Chart')}")
 
579
 
580
- # Check if we should suggest follow-up extension
581
- # Find which extension this tool belongs to
582
  for ext_name in self.enabled_extensions:
583
  ext = self.extension_manager.get_extension(ext_name)
584
  if ext:
@@ -586,7 +554,6 @@ YOU MUST use these extensions in your response:
586
  if hasattr(tool, 'function_declarations'):
587
  for func_decl in tool.function_declarations:
588
  if func_decl.name == tool_name:
589
- # Found the extension, check for suggestions
590
  suggestion = self.extension_manager.suggest_next_extension(
591
  ext_name, result, self.enabled_extensions
592
  )
@@ -594,24 +561,20 @@ YOU MUST use these extensions in your response:
594
  log(f"πŸ’‘ Suggestion: Use {suggestion['extension']}.{suggestion['tool']} - {suggestion['reason']}")
595
  break
596
 
597
- # Prepare for next round
598
  if current_round < max_rounds:
599
  results_summary = self._format_results_for_context(results)
600
 
601
- # Build enhanced context with extension-aware guidance
602
  next_prompt = f"""Previous actions completed:
603
  {results_summary}
604
 
605
  Original query: {query}
606
 
607
  """
608
- # Add suggestions for next steps based on results
609
  suggestions = self._generate_next_step_suggestions(results)
610
  if suggestions:
611
  next_prompt += f"\n**⚠️ REQUIRED NEXT STEPS (DO NOT SKIP):**\n{suggestions}\n\n"
612
  next_prompt += "You MUST complete these steps before finishing. The user explicitly requested these actions.\n\n"
613
 
614
- # Check if explicit extensions still need to be satisfied
615
  if analysis.get('explicit_extensions'):
616
  unsatisfied = self._check_unsatisfied_extensions(
617
  analysis['explicit_extensions'],
@@ -633,22 +596,21 @@ Original query: {query}
633
  next_prompt += "Continue with required steps. If all user requirements are satisfied, provide final answer."
634
 
635
  prompt = next_prompt
636
-
637
- # Don't pass file_parts again in subsequent rounds
638
  file_parts = None
639
 
640
- # Check if we have a final text response from tool agent
641
  if text_response and not function_calls:
642
- # Agent provided final answer without needing synthesis
643
  log(f"βœ… Tool agent provided final answer")
644
  return text_response, tool_results, generated_images, thoughts, search_citations
645
 
646
- # Step 4: Synthesize
 
647
  final_answer, synth_images = self.synthesize_response(
648
  query, search_results, tool_results, search_citations, file_parts
649
  )
650
 
 
651
  generated_images.extend(synth_images)
 
652
 
653
  return final_answer, tool_results, generated_images, thoughts, search_citations
654
 
@@ -657,9 +619,8 @@ Original query: {query}
657
  formatted = []
658
  for tool_name, result in results:
659
  if isinstance(result, dict):
660
- # Clean up result for context
661
  clean_result = dict(result)
662
- clean_result.pop('image_base64', None) # Remove large data
663
  formatted.append(f"- {tool_name}: {json.dumps(clean_result, indent=2)[:300]}")
664
  else:
665
  formatted.append(f"- {tool_name}: {str(result)[:200]}")
@@ -673,36 +634,28 @@ Original query: {query}
673
  if not isinstance(result, dict):
674
  continue
675
 
676
- # Stock history β†’ visualization
677
  if tool_name == 'get_stock_history' and 'visualization' in self.enabled_extensions:
678
  if result.get('success') and 'dates' in result and 'close_prices' in result:
679
  suggestions.append("πŸ“ˆ MUST create a line chart with the stock history data (dates and prices are ready)")
680
 
681
- # Stock comparison β†’ visualization
682
  if tool_name == 'compare_stocks' and 'visualization' in self.enabled_extensions:
683
  if result.get('success') and 'comparison' in result:
684
  suggestions.append("πŸ“Š MUST create a bar chart comparing the stocks (comparison data is ready)")
685
 
686
- # Deep research completed β†’ get stock history if visualization requested
687
  if tool_name == 'conduct_deep_research' and 'yfinance' in self.enabled_extensions:
688
  if result.get('success'):
689
- # Extract company/stock mentioned in research
690
  topic = result.get('topic', '')
691
- # Common stock keywords
692
  if any(word in topic.lower() for word in ['stock', 'nvidia', 'nvda', 'company']):
693
  ticker = 'NVDA' if 'nvidia' in topic.lower() or 'nvda' in topic.lower() else None
694
  if ticker and 'visualization' in self.enabled_extensions:
695
  suggestions.append(f"πŸ“ˆ MUST get {ticker} stock history for past 4 years (user requested graph of growth)")
696
 
697
- # Deep research β†’ visualization
698
  if tool_name == 'conduct_deep_research' and 'visualization' in self.enabled_extensions:
699
  if result.get('success') and result.get('num_searches', 0) > 0:
700
- # Check if there's numerical data to visualize
701
  report = result.get('report', '')
702
  if 'pros' in report.lower() and 'cons' in report.lower():
703
  suggestions.append("πŸ“Š Create a bar chart showing pros vs cons count")
704
 
705
- # YouTube β†’ research
706
  if tool_name == 'analyze_youtube_video' and 'deep_research' in self.enabled_extensions:
707
  if result.get('success'):
708
  suggestions.append("πŸ”¬ Conduct deep research on topics mentioned in the video")
@@ -714,23 +667,20 @@ Original query: {query}
714
  """Check which explicitly requested extensions haven't been used yet"""
715
  used_extensions = set()
716
 
717
- # Map tool calls to extensions
718
  extension_tool_map = {
719
- 'deep_research': ['conduct_deep_research'], # Only actual research counts
720
  'yfinance': ['get_stock_price', 'get_stock_info', 'get_stock_history', 'compare_stocks'],
721
  'visualization': ['create_line_chart', 'create_bar_chart', 'create_scatter_plot', 'create_pie_chart'],
722
  'youtube': ['analyze_youtube_video', 'get_video_chapters'],
723
  'timer': ['set_timer', 'list_timers', 'check_timer', 'cancel_timer']
724
  }
725
 
726
- # Check which extensions have been used
727
  for tool_name, _ in tool_results:
728
  for ext_name, tool_list in extension_tool_map.items():
729
  if tool_name in tool_list:
730
  used_extensions.add(ext_name)
731
  break
732
 
733
- # Return extensions that were requested but not used
734
  unsatisfied = []
735
  for ext in explicit_extensions:
736
  if ext not in used_extensions:
@@ -845,7 +795,7 @@ Original query: {query}
845
  generated_images = []
846
  deep_research_report = None
847
  deep_research_sources = None
848
- has_deep_research = False # Track if deep research was done
849
 
850
  if tool_results:
851
  synthesis_prompt += "[Tool Execution Results]\n"
@@ -856,10 +806,10 @@ Original query: {query}
856
  'title': result.get('message', 'Generated visualization'),
857
  'filepath': result.get('filepath', '')
858
  })
 
859
  result_clean = dict(result)
860
  result_clean.pop('image_base64', None)
861
  synthesis_prompt += f"- {tool_name}: {result_clean.get('message', 'Chart created')}\n"
862
- # Capture deep research report
863
  elif tool_name == 'conduct_deep_research' and isinstance(result, dict):
864
  if result.get('success') and result.get('report'):
865
  has_deep_research = True
@@ -871,7 +821,6 @@ Original query: {query}
871
  result_str = str(result)[:500]
872
  synthesis_prompt += f"- {tool_name}: {result_str}\n"
873
 
874
- # MODIFIED: Different instructions based on whether deep research was done
875
  if has_deep_research:
876
  synthesis_prompt += "\n\nProvide a brief summary of what was accomplished (2-3 sentences). DO NOT include the full research report - it will be displayed separately below your summary."
877
  else:
@@ -880,7 +829,7 @@ Original query: {query}
880
  config = types.GenerateContentConfig(
881
  system_instruction="You are a synthesis specialist. Combine information from multiple sources into coherent, helpful responses. Be concise when deep research reports are available since they will be displayed separately.",
882
  temperature=0.7,
883
- max_output_tokens=2048 # Reduced since we're not including full report
884
  )
885
 
886
  try:
@@ -901,12 +850,10 @@ Original query: {query}
901
  if getattr(part, "text", None):
902
  result_text += part.text
903
 
904
- # MODIFIED: Only append research report if it exists
905
  if has_deep_research and deep_research_report:
906
  result_text += "\n\n---\n\n## πŸ“‹ Complete Research Report\n\n"
907
  result_text += deep_research_report
908
 
909
- # Add sources at the end
910
  if deep_research_sources:
911
  result_text += "\n\n### πŸ“š Research Sources\n\n"
912
  for idx, source in enumerate(deep_research_sources[:30], 1):
@@ -918,6 +865,7 @@ Original query: {query}
918
  if len(deep_research_sources) > 30:
919
  result_text += f"\n*...and {len(deep_research_sources) - 30} more sources*\n"
920
 
 
921
  return result_text, generated_images
922
  except Exception as e:
923
  log(f"⚠️ Synthesis error: {e}")
@@ -968,7 +916,6 @@ def insert_citations_from_grounding(candidates):
968
  return None
969
 
970
 
971
- # Global instances
972
  EXTENSION_MANAGER = ExtensionManager()
973
  CHAT_SESSIONS: Dict[str, Dict[str, Any]] = {}
974
 
@@ -1018,7 +965,6 @@ def chat_with_gemini(api_key, chat_history_msgs, multimodal_input, show_thoughts
1018
  if chat_history_msgs is None:
1019
  chat_history_msgs = []
1020
 
1021
- # Process uploaded files
1022
  file_parts = []
1023
  if uploaded_files:
1024
  log(f"πŸ“Ž Processing {len(uploaded_files)} uploaded file(s)...")
@@ -1034,7 +980,6 @@ def chat_with_gemini(api_key, chat_history_msgs, multimodal_input, show_thoughts
1034
 
1035
  assistant_base_index = len(chat_history_msgs)
1036
 
1037
- # Setup thinking display
1038
  if show_thoughts:
1039
  thought_index = assistant_base_index
1040
  chat_history_msgs.append({"role": "assistant", "content": "<em>πŸ’­ Thinking...</em>"})
@@ -1048,7 +993,6 @@ def chat_with_gemini(api_key, chat_history_msgs, multimodal_input, show_thoughts
1048
  yield chat_history_msgs
1049
 
1050
  try:
1051
- # Check for proactive messages from extensions first
1052
  proactive_msgs = EXTENSION_MANAGER.check_proactive_messages(api_key, enabled_extensions)
1053
  if proactive_msgs:
1054
  for msg in proactive_msgs:
@@ -1056,7 +1000,6 @@ def chat_with_gemini(api_key, chat_history_msgs, multimodal_input, show_thoughts
1056
  answer_index += 1
1057
  yield chat_history_msgs
1058
 
1059
- # Use improved orchestrator
1060
  if enabled_extensions:
1061
  log("🎭 Using improved orchestrator")
1062
  orchestrator = ImprovedOrchestrator(
@@ -1065,11 +1008,14 @@ def chat_with_gemini(api_key, chat_history_msgs, multimodal_input, show_thoughts
1065
 
1066
  budget = reasoning_budget(reasoning_level)
1067
 
1068
- # Execute with planning
1069
  final_answer, tool_results, generated_images, thoughts, search_citations = \
1070
  orchestrator.execute_with_planning(user_text, file_parts, budget)
1071
 
1072
- # Show thoughts
 
 
 
 
1073
  if thoughts and show_thoughts:
1074
  chat_history_msgs[thought_index]["content"] = (
1075
  f"<details open>"
@@ -1081,7 +1027,6 @@ def chat_with_gemini(api_key, chat_history_msgs, multimodal_input, show_thoughts
1081
  )
1082
  yield chat_history_msgs
1083
 
1084
- # Build final content
1085
  final_content = (
1086
  f"<div><strong>πŸ€– Response</strong>"
1087
  f"<div style='white-space:pre-wrap;background:inherit;color:inherit;"
@@ -1089,20 +1034,28 @@ def chat_with_gemini(api_key, chat_history_msgs, multimodal_input, show_thoughts
1089
  f"{final_answer.strip()}</div></div>"
1090
  )
1091
 
1092
- # Add generated images
 
1093
  if generated_images:
1094
- for img_data in generated_images:
 
1095
  final_content += f"\n\n<div style='margin-top:16px;'>"
1096
- final_content += f"<strong>πŸ“Š {img_data['title']}</strong><br/>"
1097
- final_content += f"<img src='data:image/png;base64,{img_data['base64']}' style='max-width:100%;border-radius:8px;box-shadow:0 2px 8px rgba(0,0,0,0.1);'/>"
1098
- if img_data['filepath']:
 
 
 
 
 
 
 
1099
  final_content += f"<br/><small style='color:#666;'>Saved to: {img_data['filepath']}</small>"
1100
  final_content += "</div>"
 
 
 
1101
 
1102
- # REMOVED: The duplicate research report block that was here (lines 1000-1070)
1103
- # Research report is now handled ONLY in synthesize_response() method
1104
-
1105
- # Add citations
1106
  if search_citations:
1107
  final_content += "\n\n" + search_citations
1108
 
@@ -1110,7 +1063,6 @@ def chat_with_gemini(api_key, chat_history_msgs, multimodal_input, show_thoughts
1110
  yield chat_history_msgs
1111
 
1112
  else:
1113
- # No extensions - simple streaming with search
1114
  log("πŸ“Ί Using simple streaming mode")
1115
 
1116
  parts = []
@@ -1178,7 +1130,6 @@ def chat_with_gemini(api_key, chat_history_msgs, multimodal_input, show_thoughts
1178
  )
1179
  yield chat_history_msgs
1180
 
1181
- # Add citations
1182
  if last_chunk:
1183
  citations = insert_citations_from_grounding(last_chunk.candidates)
1184
  if citations:
@@ -1260,7 +1211,6 @@ with gr.Blocks(
1260
  info="Display reasoning process before answers.",
1261
  )
1262
 
1263
- # Build extension checkboxes
1264
  extension_checkboxes = build_extension_ui()
1265
 
1266
  with gr.Column(scale=4):
@@ -1284,14 +1234,12 @@ with gr.Blocks(
1284
  autofocus=True
1285
  )
1286
 
1287
- # Hidden state to track enabled extensions
1288
  enabled_extensions_state = gr.State([])
1289
 
1290
  def clear_box():
1291
  return {"text": "", "files": []}
1292
 
1293
  def handle_chat(api_key_input, chat_history_msgs, multimodal_dict, thinking_flag, reasoning_lvl, *extension_states):
1294
- # Convert extension checkbox states to list of enabled extension names
1295
  enabled = []
1296
  for (ext_name, _), is_enabled in zip(extension_checkboxes, extension_states):
1297
  if is_enabled:
@@ -1309,7 +1257,6 @@ with gr.Blocks(
1309
  if not api_key_input or not enabled_exts:
1310
  return chat_history
1311
 
1312
- # Check all extensions for proactive messages
1313
  proactive_msgs = EXTENSION_MANAGER.check_proactive_messages(api_key_input, enabled_exts)
1314
 
1315
  if proactive_msgs:
@@ -1322,10 +1269,8 @@ with gr.Blocks(
1322
 
1323
  return chat_history
1324
 
1325
- # Get just the checkbox components for inputs
1326
  checkbox_components = [cb for _, cb in extension_checkboxes]
1327
 
1328
- # Main chat submission
1329
  multimodal_msg.submit(
1330
  fn=handle_chat,
1331
  inputs=[api_key, chatbot, multimodal_msg, show_thoughts, reasoning_level] + checkbox_components,
@@ -1333,7 +1278,6 @@ with gr.Blocks(
1333
  queue=True,
1334
  ).then(fn=clear_box, outputs=[multimodal_msg])
1335
 
1336
- # Background timer check - runs every 10 seconds
1337
  timer_check = gr.Timer(value=10, active=True)
1338
 
1339
  def update_enabled_state(*extension_states):
@@ -1343,7 +1287,6 @@ with gr.Blocks(
1343
  enabled.append(ext_name)
1344
  return enabled
1345
 
1346
- # Update enabled extensions state whenever checkboxes change
1347
  for _, cb in extension_checkboxes:
1348
  cb.change(
1349
  fn=update_enabled_state,
@@ -1351,7 +1294,6 @@ with gr.Blocks(
1351
  outputs=[enabled_extensions_state]
1352
  )
1353
 
1354
- # Timer polling for proactive messages
1355
  timer_check.tick(
1356
  fn=check_timers,
1357
  inputs=[api_key, chatbot, enabled_extensions_state],
 
79
 
80
  def __init__(self):
81
  self.extensions: Dict[str, BaseExtension] = {}
82
+ self.extension_interactions: Dict[str, List[str]] = {}
83
  self.load_extensions()
84
 
85
  def load_extensions(self):
 
117
  traceback.print_exc()
118
 
119
  log(f"πŸ“Š Total extensions loaded: {len(self.extensions)}")
 
 
120
  self._detect_extension_relationships()
121
 
122
  def _detect_extension_relationships(self):
123
  """Detect which extensions can work together based on capabilities"""
124
  log("πŸ”— Detecting extension relationships...")
125
 
 
126
  relationships = {
127
+ 'yfinance': ['visualization', 'deep_research'],
128
+ 'deep_research': ['visualization', 'yfinance'],
129
+ 'youtube': ['deep_research', 'visualization'],
130
+ 'timer': ['deep_research'],
131
+ 'visualization': ['yfinance', 'deep_research', 'youtube']
132
  }
133
 
134
  for ext_name, compatible in relationships.items():
 
159
  if enabled_exts:
160
  prompt += "\n\n# ENABLED EXTENSIONS\nYou currently have these extensions enabled:\n\n"
161
  for ext in enabled_exts:
 
162
  ext.initialize_state(user_id)
 
 
163
  prompt += f"## {ext.display_name}\n{ext.get_system_context()}\n\n"
164
 
 
165
  state = ext.get_state(user_id)
166
  if state and hasattr(ext, 'get_state_summary'):
167
  state_summary = ext.get_state_summary(user_id)
168
  if state_summary:
169
  prompt += f"**Current State:** {state_summary}\n\n"
170
 
 
171
  if len(enabled_exts) > 1:
172
  prompt += self._build_multi_extension_guidance(enabled_list)
173
 
 
178
  guidance = "\n\n# MULTI-EXTENSION REASONING\n"
179
  guidance += "You have multiple extensions enabled. You can combine them intelligently:\n\n"
180
 
 
181
  combinations = []
182
 
183
  if 'yfinance' in enabled_list and 'visualization' in enabled_list:
 
219
 
220
  suggestions = []
221
 
 
222
  if current_extension == 'yfinance' and isinstance(result, dict):
223
  if 'dates' in result and 'close_prices' in result and 'visualization' in compatible:
224
  suggestions.append({
 
235
  'data_available': True
236
  })
237
 
 
238
  if current_extension == 'deep_research' and isinstance(result, dict):
239
  if result.get('success') and 'visualization' in compatible:
240
  suggestions.append({
 
244
  'data_available': True
245
  })
246
 
 
247
  if current_extension == 'youtube' and isinstance(result, dict):
248
  if result.get('success') and 'deep_research' in compatible:
249
  suggestions.append({
 
279
 
280
  log(f"πŸ”§ Executing: {function_name}({json.dumps(args, indent=2)[:100]}...)")
281
 
 
282
  handled = False
283
  for ext_name in enabled_list:
284
  ext = self.get_extension(ext_name)
285
  if ext:
 
286
  for tool in ext.get_tools():
287
  if hasattr(tool, 'function_declarations'):
288
  for func_decl in tool.function_declarations:
 
329
  self.user_id = user_id
330
  self.enabled_extensions = enabled_extensions
331
  self.search_chat = client.chats.create(model="gemini-2.5-flash")
332
+ self.execution_history = []
333
 
334
  def analyze_query(self, query: str, file_parts: List = None) -> Dict[str, Any]:
335
  """Analyze query to determine optimal execution strategy"""
336
  log("🧠 Analyzing query to determine strategy...")
337
 
 
338
  query_lower = query.lower()
339
  explicit_extensions = []
340
 
 
394
 
395
  if response.candidates and response.candidates[0].content and response.candidates[0].content.parts:
396
  text = response.candidates[0].content.parts[0].text
 
397
  json_match = re.search(r'```(?:json)?\s*(\{.*?\})\s*```', text, re.DOTALL)
398
  if json_match:
399
  text = json_match.group(1)
400
 
401
  analysis = json.loads(text)
402
 
 
403
  if explicit_extensions:
404
  analysis['explicit_extensions'] = explicit_extensions
405
  analysis['multi_step'] = True
 
410
  except Exception as e:
411
  log(f"⚠️ Query analysis failed: {e}, using heuristics")
412
 
 
413
  heuristic = self._heuristic_analysis(query)
414
  if explicit_extensions:
415
  heuristic['explicit_extensions'] = explicit_extensions
 
420
  """Fallback heuristic analysis"""
421
  query_lower = query.lower()
422
 
 
423
  search_keywords = ['search', 'find', 'look up', 'what is', 'latest', 'current', 'news']
424
  needs_search = "yes" if any(kw in query_lower for kw in search_keywords) else "maybe"
425
 
 
426
  multi_step_keywords = ['then', 'after that', 'and then', 'research and', 'analyze and']
427
  multi_step = any(kw in query_lower for kw in multi_step_keywords)
428
 
 
429
  context_keywords = ['the', 'my', 'that', 'previous', 'earlier']
430
  uses_context = any(kw in query_lower for kw in context_keywords)
431
 
 
432
  word_count = len(query.split())
433
  if word_count < 10 and not multi_step:
434
  complexity = "simple"
 
447
  }
448
 
449
  def execute_with_planning(self, query: str, file_parts: List = None,
450
+ reasoning_budget: int = -1) -> Tuple[str, List, List, str, Optional[str]]:
451
  """Execute query with planning and adaptive strategy"""
452
 
 
453
  analysis = self.analyze_query(query, file_parts)
454
 
 
455
  search_results = ""
456
  search_citations = None
457
 
 
458
  if analysis['needs_search'] in ['yes', 'maybe']:
459
  log("πŸ” Executing search phase...")
460
  search_results, search_citations = self.call_search_agent(query, file_parts)
461
 
 
462
  log("πŸ› οΈ Executing tool phase...")
463
  tool_results = []
464
  generated_images = []
465
  thoughts = ""
466
 
 
467
  system_prompt = self.extension_manager.build_system_prompt(
468
  self.user_id, self.enabled_extensions
469
  )
470
 
 
471
  if analysis['multi_step']:
472
  system_prompt += """
473
 
 
487
  - Reference specific items by their IDs or names from the state
488
  """
489
 
 
490
  if analysis.get('explicit_extensions'):
491
  explicit_exts = analysis['explicit_extensions']
492
  system_prompt += f"""
 
506
 
507
  system_prompt += "\n\nDo NOT skip these explicitly requested extensions!\n"
508
 
 
509
  max_rounds = 5 if analysis['complexity'] == 'complex' else 3
510
  current_round = 0
511
 
 
526
  if not function_calls:
527
  log(f"βœ… No more tools needed after round {current_round}")
528
  if text_response:
 
529
  return text_response, tool_results, generated_images, thoughts, search_citations
530
  break
531
 
 
532
  results = self.extension_manager.handle_function_calls(
533
  self.user_id, self.enabled_extensions, function_calls
534
  )
 
536
  for (tool_name, result) in results:
537
  tool_results.append((tool_name, result))
538
 
539
+ # CRITICAL: Extract generated images IMMEDIATELY
540
  if isinstance(result, dict) and 'image_base64' in result:
541
+ img_data = {
542
  'base64': result['image_base64'],
543
  'title': result.get('message', 'Generated visualization'),
544
  'filepath': result.get('filepath', '')
545
+ }
546
+ generated_images.append(img_data)
547
  log(f"πŸ“Š Captured visualization: {result.get('message', 'Chart')}")
548
+ log(f" πŸ–ΌοΈ Image data length: {len(result['image_base64'])} chars")
549
 
 
 
550
  for ext_name in self.enabled_extensions:
551
  ext = self.extension_manager.get_extension(ext_name)
552
  if ext:
 
554
  if hasattr(tool, 'function_declarations'):
555
  for func_decl in tool.function_declarations:
556
  if func_decl.name == tool_name:
 
557
  suggestion = self.extension_manager.suggest_next_extension(
558
  ext_name, result, self.enabled_extensions
559
  )
 
561
  log(f"πŸ’‘ Suggestion: Use {suggestion['extension']}.{suggestion['tool']} - {suggestion['reason']}")
562
  break
563
 
 
564
  if current_round < max_rounds:
565
  results_summary = self._format_results_for_context(results)
566
 
 
567
  next_prompt = f"""Previous actions completed:
568
  {results_summary}
569
 
570
  Original query: {query}
571
 
572
  """
 
573
  suggestions = self._generate_next_step_suggestions(results)
574
  if suggestions:
575
  next_prompt += f"\n**⚠️ REQUIRED NEXT STEPS (DO NOT SKIP):**\n{suggestions}\n\n"
576
  next_prompt += "You MUST complete these steps before finishing. The user explicitly requested these actions.\n\n"
577
 
 
578
  if analysis.get('explicit_extensions'):
579
  unsatisfied = self._check_unsatisfied_extensions(
580
  analysis['explicit_extensions'],
 
596
  next_prompt += "Continue with required steps. If all user requirements are satisfied, provide final answer."
597
 
598
  prompt = next_prompt
 
 
599
  file_parts = None
600
 
 
601
  if text_response and not function_calls:
 
602
  log(f"βœ… Tool agent provided final answer")
603
  return text_response, tool_results, generated_images, thoughts, search_citations
604
 
605
+ log(f"🎨 Before synthesis: {len(generated_images)} images in list")
606
+
607
  final_answer, synth_images = self.synthesize_response(
608
  query, search_results, tool_results, search_citations, file_parts
609
  )
610
 
611
+ log(f"🎨 After synthesis: {len(synth_images)} new images")
612
  generated_images.extend(synth_images)
613
+ log(f"🎨 Total images to return: {len(generated_images)}")
614
 
615
  return final_answer, tool_results, generated_images, thoughts, search_citations
616
 
 
619
  formatted = []
620
  for tool_name, result in results:
621
  if isinstance(result, dict):
 
622
  clean_result = dict(result)
623
+ clean_result.pop('image_base64', None)
624
  formatted.append(f"- {tool_name}: {json.dumps(clean_result, indent=2)[:300]}")
625
  else:
626
  formatted.append(f"- {tool_name}: {str(result)[:200]}")
 
634
  if not isinstance(result, dict):
635
  continue
636
 
 
637
  if tool_name == 'get_stock_history' and 'visualization' in self.enabled_extensions:
638
  if result.get('success') and 'dates' in result and 'close_prices' in result:
639
  suggestions.append("πŸ“ˆ MUST create a line chart with the stock history data (dates and prices are ready)")
640
 
 
641
  if tool_name == 'compare_stocks' and 'visualization' in self.enabled_extensions:
642
  if result.get('success') and 'comparison' in result:
643
  suggestions.append("πŸ“Š MUST create a bar chart comparing the stocks (comparison data is ready)")
644
 
 
645
  if tool_name == 'conduct_deep_research' and 'yfinance' in self.enabled_extensions:
646
  if result.get('success'):
 
647
  topic = result.get('topic', '')
 
648
  if any(word in topic.lower() for word in ['stock', 'nvidia', 'nvda', 'company']):
649
  ticker = 'NVDA' if 'nvidia' in topic.lower() or 'nvda' in topic.lower() else None
650
  if ticker and 'visualization' in self.enabled_extensions:
651
  suggestions.append(f"πŸ“ˆ MUST get {ticker} stock history for past 4 years (user requested graph of growth)")
652
 
 
653
  if tool_name == 'conduct_deep_research' and 'visualization' in self.enabled_extensions:
654
  if result.get('success') and result.get('num_searches', 0) > 0:
 
655
  report = result.get('report', '')
656
  if 'pros' in report.lower() and 'cons' in report.lower():
657
  suggestions.append("πŸ“Š Create a bar chart showing pros vs cons count")
658
 
 
659
  if tool_name == 'analyze_youtube_video' and 'deep_research' in self.enabled_extensions:
660
  if result.get('success'):
661
  suggestions.append("πŸ”¬ Conduct deep research on topics mentioned in the video")
 
667
  """Check which explicitly requested extensions haven't been used yet"""
668
  used_extensions = set()
669
 
 
670
  extension_tool_map = {
671
+ 'deep_research': ['conduct_deep_research'],
672
  'yfinance': ['get_stock_price', 'get_stock_info', 'get_stock_history', 'compare_stocks'],
673
  'visualization': ['create_line_chart', 'create_bar_chart', 'create_scatter_plot', 'create_pie_chart'],
674
  'youtube': ['analyze_youtube_video', 'get_video_chapters'],
675
  'timer': ['set_timer', 'list_timers', 'check_timer', 'cancel_timer']
676
  }
677
 
 
678
  for tool_name, _ in tool_results:
679
  for ext_name, tool_list in extension_tool_map.items():
680
  if tool_name in tool_list:
681
  used_extensions.add(ext_name)
682
  break
683
 
 
684
  unsatisfied = []
685
  for ext in explicit_extensions:
686
  if ext not in used_extensions:
 
795
  generated_images = []
796
  deep_research_report = None
797
  deep_research_sources = None
798
+ has_deep_research = False
799
 
800
  if tool_results:
801
  synthesis_prompt += "[Tool Execution Results]\n"
 
806
  'title': result.get('message', 'Generated visualization'),
807
  'filepath': result.get('filepath', '')
808
  })
809
+ log(f"πŸ“Š Synthesis: Captured image from {tool_name}")
810
  result_clean = dict(result)
811
  result_clean.pop('image_base64', None)
812
  synthesis_prompt += f"- {tool_name}: {result_clean.get('message', 'Chart created')}\n"
 
813
  elif tool_name == 'conduct_deep_research' and isinstance(result, dict):
814
  if result.get('success') and result.get('report'):
815
  has_deep_research = True
 
821
  result_str = str(result)[:500]
822
  synthesis_prompt += f"- {tool_name}: {result_str}\n"
823
 
 
824
  if has_deep_research:
825
  synthesis_prompt += "\n\nProvide a brief summary of what was accomplished (2-3 sentences). DO NOT include the full research report - it will be displayed separately below your summary."
826
  else:
 
829
  config = types.GenerateContentConfig(
830
  system_instruction="You are a synthesis specialist. Combine information from multiple sources into coherent, helpful responses. Be concise when deep research reports are available since they will be displayed separately.",
831
  temperature=0.7,
832
+ max_output_tokens=2048
833
  )
834
 
835
  try:
 
850
  if getattr(part, "text", None):
851
  result_text += part.text
852
 
 
853
  if has_deep_research and deep_research_report:
854
  result_text += "\n\n---\n\n## πŸ“‹ Complete Research Report\n\n"
855
  result_text += deep_research_report
856
 
 
857
  if deep_research_sources:
858
  result_text += "\n\n### πŸ“š Research Sources\n\n"
859
  for idx, source in enumerate(deep_research_sources[:30], 1):
 
865
  if len(deep_research_sources) > 30:
866
  result_text += f"\n*...and {len(deep_research_sources) - 30} more sources*\n"
867
 
868
+ log(f"🎨 Synthesis returning {len(generated_images)} images")
869
  return result_text, generated_images
870
  except Exception as e:
871
  log(f"⚠️ Synthesis error: {e}")
 
916
  return None
917
 
918
 
 
919
  EXTENSION_MANAGER = ExtensionManager()
920
  CHAT_SESSIONS: Dict[str, Dict[str, Any]] = {}
921
 
 
965
  if chat_history_msgs is None:
966
  chat_history_msgs = []
967
 
 
968
  file_parts = []
969
  if uploaded_files:
970
  log(f"πŸ“Ž Processing {len(uploaded_files)} uploaded file(s)...")
 
980
 
981
  assistant_base_index = len(chat_history_msgs)
982
 
 
983
  if show_thoughts:
984
  thought_index = assistant_base_index
985
  chat_history_msgs.append({"role": "assistant", "content": "<em>πŸ’­ Thinking...</em>"})
 
993
  yield chat_history_msgs
994
 
995
  try:
 
996
  proactive_msgs = EXTENSION_MANAGER.check_proactive_messages(api_key, enabled_extensions)
997
  if proactive_msgs:
998
  for msg in proactive_msgs:
 
1000
  answer_index += 1
1001
  yield chat_history_msgs
1002
 
 
1003
  if enabled_extensions:
1004
  log("🎭 Using improved orchestrator")
1005
  orchestrator = ImprovedOrchestrator(
 
1008
 
1009
  budget = reasoning_budget(reasoning_level)
1010
 
 
1011
  final_answer, tool_results, generated_images, thoughts, search_citations = \
1012
  orchestrator.execute_with_planning(user_text, file_parts, budget)
1013
 
1014
+ # CRITICAL DEBUG: Log what we got back
1015
+ log(f"🎨 Orchestrator returned {len(generated_images)} images")
1016
+ for idx, img in enumerate(generated_images, 1):
1017
+ log(f" Image {idx}: {img.get('title', 'Untitled')} - base64 length: {len(img.get('base64', ''))}")
1018
+
1019
  if thoughts and show_thoughts:
1020
  chat_history_msgs[thought_index]["content"] = (
1021
  f"<details open>"
 
1027
  )
1028
  yield chat_history_msgs
1029
 
 
1030
  final_content = (
1031
  f"<div><strong>πŸ€– Response</strong>"
1032
  f"<div style='white-space:pre-wrap;background:inherit;color:inherit;"
 
1034
  f"{final_answer.strip()}</div></div>"
1035
  )
1036
 
1037
+ # CRITICAL: Add generated images with debug logging
1038
+ log(f"🎨 Attempting to add {len(generated_images)} images to final content")
1039
  if generated_images:
1040
+ for idx, img_data in enumerate(generated_images, 1):
1041
+ log(f" πŸ“Š Adding image {idx}: {img_data.get('title', 'Untitled')}")
1042
  final_content += f"\n\n<div style='margin-top:16px;'>"
1043
+ final_content += f"<strong>πŸ“Š {img_data.get('title', 'Chart')}</strong><br/>"
1044
+
1045
+ base64_data = img_data.get('base64', '')
1046
+ if base64_data:
1047
+ log(f" βœ… Base64 data present: {len(base64_data)} chars")
1048
+ final_content += f"<img src='data:image/png;base64,{base64_data}' style='max-width:100%;border-radius:8px;box-shadow:0 2px 8px rgba(0,0,0,0.1);'/>"
1049
+ else:
1050
+ log(f" ❌ WARNING: No base64 data for image {idx}!")
1051
+
1052
+ if img_data.get('filepath'):
1053
  final_content += f"<br/><small style='color:#666;'>Saved to: {img_data['filepath']}</small>"
1054
  final_content += "</div>"
1055
+ log(f" βœ… All {len(generated_images)} images added to HTML")
1056
+ else:
1057
+ log(f" ⚠️ WARNING: generated_images list is EMPTY!")
1058
 
 
 
 
 
1059
  if search_citations:
1060
  final_content += "\n\n" + search_citations
1061
 
 
1063
  yield chat_history_msgs
1064
 
1065
  else:
 
1066
  log("πŸ“Ί Using simple streaming mode")
1067
 
1068
  parts = []
 
1130
  )
1131
  yield chat_history_msgs
1132
 
 
1133
  if last_chunk:
1134
  citations = insert_citations_from_grounding(last_chunk.candidates)
1135
  if citations:
 
1211
  info="Display reasoning process before answers.",
1212
  )
1213
 
 
1214
  extension_checkboxes = build_extension_ui()
1215
 
1216
  with gr.Column(scale=4):
 
1234
  autofocus=True
1235
  )
1236
 
 
1237
  enabled_extensions_state = gr.State([])
1238
 
1239
  def clear_box():
1240
  return {"text": "", "files": []}
1241
 
1242
  def handle_chat(api_key_input, chat_history_msgs, multimodal_dict, thinking_flag, reasoning_lvl, *extension_states):
 
1243
  enabled = []
1244
  for (ext_name, _), is_enabled in zip(extension_checkboxes, extension_states):
1245
  if is_enabled:
 
1257
  if not api_key_input or not enabled_exts:
1258
  return chat_history
1259
 
 
1260
  proactive_msgs = EXTENSION_MANAGER.check_proactive_messages(api_key_input, enabled_exts)
1261
 
1262
  if proactive_msgs:
 
1269
 
1270
  return chat_history
1271
 
 
1272
  checkbox_components = [cb for _, cb in extension_checkboxes]
1273
 
 
1274
  multimodal_msg.submit(
1275
  fn=handle_chat,
1276
  inputs=[api_key, chatbot, multimodal_msg, show_thoughts, reasoning_level] + checkbox_components,
 
1278
  queue=True,
1279
  ).then(fn=clear_box, outputs=[multimodal_msg])
1280
 
 
1281
  timer_check = gr.Timer(value=10, active=True)
1282
 
1283
  def update_enabled_state(*extension_states):
 
1287
  enabled.append(ext_name)
1288
  return enabled
1289
 
 
1290
  for _, cb in extension_checkboxes:
1291
  cb.change(
1292
  fn=update_enabled_state,
 
1294
  outputs=[enabled_extensions_state]
1295
  )
1296
 
 
1297
  timer_check.tick(
1298
  fn=check_timers,
1299
  inputs=[api_key, chatbot, enabled_extensions_state],