Spaces:
Sleeping
Sleeping
Update app.py
Browse files
app.py
CHANGED
|
@@ -79,7 +79,7 @@ class ExtensionManager:
|
|
| 79 |
|
| 80 |
def __init__(self):
|
| 81 |
self.extensions: Dict[str, BaseExtension] = {}
|
| 82 |
-
self.extension_interactions: Dict[str, List[str]] = {}
|
| 83 |
self.load_extensions()
|
| 84 |
|
| 85 |
def load_extensions(self):
|
|
@@ -117,21 +117,18 @@ class ExtensionManager:
|
|
| 117 |
traceback.print_exc()
|
| 118 |
|
| 119 |
log(f"π Total extensions loaded: {len(self.extensions)}")
|
| 120 |
-
|
| 121 |
-
# Detect extension relationships
|
| 122 |
self._detect_extension_relationships()
|
| 123 |
|
| 124 |
def _detect_extension_relationships(self):
|
| 125 |
"""Detect which extensions can work together based on capabilities"""
|
| 126 |
log("π Detecting extension relationships...")
|
| 127 |
|
| 128 |
-
# Common patterns of extensions that work well together
|
| 129 |
relationships = {
|
| 130 |
-
'yfinance': ['visualization', 'deep_research'],
|
| 131 |
-
'deep_research': ['visualization', 'yfinance'],
|
| 132 |
-
'youtube': ['deep_research', 'visualization'],
|
| 133 |
-
'timer': ['deep_research'],
|
| 134 |
-
'visualization': ['yfinance', 'deep_research', 'youtube']
|
| 135 |
}
|
| 136 |
|
| 137 |
for ext_name, compatible in relationships.items():
|
|
@@ -162,20 +159,15 @@ class ExtensionManager:
|
|
| 162 |
if enabled_exts:
|
| 163 |
prompt += "\n\n# ENABLED EXTENSIONS\nYou currently have these extensions enabled:\n\n"
|
| 164 |
for ext in enabled_exts:
|
| 165 |
-
# Initialize state for user if needed
|
| 166 |
ext.initialize_state(user_id)
|
| 167 |
-
|
| 168 |
-
# Get extension context
|
| 169 |
prompt += f"## {ext.display_name}\n{ext.get_system_context()}\n\n"
|
| 170 |
|
| 171 |
-
# Add current state summary if extension provides it
|
| 172 |
state = ext.get_state(user_id)
|
| 173 |
if state and hasattr(ext, 'get_state_summary'):
|
| 174 |
state_summary = ext.get_state_summary(user_id)
|
| 175 |
if state_summary:
|
| 176 |
prompt += f"**Current State:** {state_summary}\n\n"
|
| 177 |
|
| 178 |
-
# Add multi-extension reasoning guidance
|
| 179 |
if len(enabled_exts) > 1:
|
| 180 |
prompt += self._build_multi_extension_guidance(enabled_list)
|
| 181 |
|
|
@@ -186,7 +178,6 @@ class ExtensionManager:
|
|
| 186 |
guidance = "\n\n# MULTI-EXTENSION REASONING\n"
|
| 187 |
guidance += "You have multiple extensions enabled. You can combine them intelligently:\n\n"
|
| 188 |
|
| 189 |
-
# Detect enabled combinations
|
| 190 |
combinations = []
|
| 191 |
|
| 192 |
if 'yfinance' in enabled_list and 'visualization' in enabled_list:
|
|
@@ -228,7 +219,6 @@ class ExtensionManager:
|
|
| 228 |
|
| 229 |
suggestions = []
|
| 230 |
|
| 231 |
-
# If finance extension returned data with arrays, suggest visualization
|
| 232 |
if current_extension == 'yfinance' and isinstance(result, dict):
|
| 233 |
if 'dates' in result and 'close_prices' in result and 'visualization' in compatible:
|
| 234 |
suggestions.append({
|
|
@@ -245,7 +235,6 @@ class ExtensionManager:
|
|
| 245 |
'data_available': True
|
| 246 |
})
|
| 247 |
|
| 248 |
-
# If research completed, suggest visualization of findings
|
| 249 |
if current_extension == 'deep_research' and isinstance(result, dict):
|
| 250 |
if result.get('success') and 'visualization' in compatible:
|
| 251 |
suggestions.append({
|
|
@@ -255,7 +244,6 @@ class ExtensionManager:
|
|
| 255 |
'data_available': True
|
| 256 |
})
|
| 257 |
|
| 258 |
-
# If YouTube analysis completed, suggest research on topic
|
| 259 |
if current_extension == 'youtube' and isinstance(result, dict):
|
| 260 |
if result.get('success') and 'deep_research' in compatible:
|
| 261 |
suggestions.append({
|
|
@@ -291,12 +279,10 @@ class ExtensionManager:
|
|
| 291 |
|
| 292 |
log(f"π§ Executing: {function_name}({json.dumps(args, indent=2)[:100]}...)")
|
| 293 |
|
| 294 |
-
# Find which extension owns this function
|
| 295 |
handled = False
|
| 296 |
for ext_name in enabled_list:
|
| 297 |
ext = self.get_extension(ext_name)
|
| 298 |
if ext:
|
| 299 |
-
# Check if this function is in the extension's tools
|
| 300 |
for tool in ext.get_tools():
|
| 301 |
if hasattr(tool, 'function_declarations'):
|
| 302 |
for func_decl in tool.function_declarations:
|
|
@@ -343,13 +329,12 @@ class ImprovedOrchestrator:
|
|
| 343 |
self.user_id = user_id
|
| 344 |
self.enabled_extensions = enabled_extensions
|
| 345 |
self.search_chat = client.chats.create(model="gemini-2.5-flash")
|
| 346 |
-
self.execution_history = []
|
| 347 |
|
| 348 |
def analyze_query(self, query: str, file_parts: List = None) -> Dict[str, Any]:
|
| 349 |
"""Analyze query to determine optimal execution strategy"""
|
| 350 |
log("π§ Analyzing query to determine strategy...")
|
| 351 |
|
| 352 |
-
# Check for explicit extension requests first
|
| 353 |
query_lower = query.lower()
|
| 354 |
explicit_extensions = []
|
| 355 |
|
|
@@ -409,14 +394,12 @@ Respond in JSON format:
|
|
| 409 |
|
| 410 |
if response.candidates and response.candidates[0].content and response.candidates[0].content.parts:
|
| 411 |
text = response.candidates[0].content.parts[0].text
|
| 412 |
-
# Extract JSON from markdown code blocks if present
|
| 413 |
json_match = re.search(r'```(?:json)?\s*(\{.*?\})\s*```', text, re.DOTALL)
|
| 414 |
if json_match:
|
| 415 |
text = json_match.group(1)
|
| 416 |
|
| 417 |
analysis = json.loads(text)
|
| 418 |
|
| 419 |
-
# Force explicit extensions into the analysis
|
| 420 |
if explicit_extensions:
|
| 421 |
analysis['explicit_extensions'] = explicit_extensions
|
| 422 |
analysis['multi_step'] = True
|
|
@@ -427,7 +410,6 @@ Respond in JSON format:
|
|
| 427 |
except Exception as e:
|
| 428 |
log(f"β οΈ Query analysis failed: {e}, using heuristics")
|
| 429 |
|
| 430 |
-
# Fallback to heuristics with explicit extensions
|
| 431 |
heuristic = self._heuristic_analysis(query)
|
| 432 |
if explicit_extensions:
|
| 433 |
heuristic['explicit_extensions'] = explicit_extensions
|
|
@@ -438,19 +420,15 @@ Respond in JSON format:
|
|
| 438 |
"""Fallback heuristic analysis"""
|
| 439 |
query_lower = query.lower()
|
| 440 |
|
| 441 |
-
# Check for search indicators
|
| 442 |
search_keywords = ['search', 'find', 'look up', 'what is', 'latest', 'current', 'news']
|
| 443 |
needs_search = "yes" if any(kw in query_lower for kw in search_keywords) else "maybe"
|
| 444 |
|
| 445 |
-
# Check for multi-step indicators
|
| 446 |
multi_step_keywords = ['then', 'after that', 'and then', 'research and', 'analyze and']
|
| 447 |
multi_step = any(kw in query_lower for kw in multi_step_keywords)
|
| 448 |
|
| 449 |
-
# Check for context references
|
| 450 |
context_keywords = ['the', 'my', 'that', 'previous', 'earlier']
|
| 451 |
uses_context = any(kw in query_lower for kw in context_keywords)
|
| 452 |
|
| 453 |
-
# Determine complexity
|
| 454 |
word_count = len(query.split())
|
| 455 |
if word_count < 10 and not multi_step:
|
| 456 |
complexity = "simple"
|
|
@@ -469,33 +447,27 @@ Respond in JSON format:
|
|
| 469 |
}
|
| 470 |
|
| 471 |
def execute_with_planning(self, query: str, file_parts: List = None,
|
| 472 |
-
reasoning_budget: int = -1) -> Tuple[str, List, List]:
|
| 473 |
"""Execute query with planning and adaptive strategy"""
|
| 474 |
|
| 475 |
-
# Step 1: Analyze query
|
| 476 |
analysis = self.analyze_query(query, file_parts)
|
| 477 |
|
| 478 |
-
# Step 2: Execute based on analysis
|
| 479 |
search_results = ""
|
| 480 |
search_citations = None
|
| 481 |
|
| 482 |
-
# Handle search if needed
|
| 483 |
if analysis['needs_search'] in ['yes', 'maybe']:
|
| 484 |
log("π Executing search phase...")
|
| 485 |
search_results, search_citations = self.call_search_agent(query, file_parts)
|
| 486 |
|
| 487 |
-
# Step 3: Execute with tools
|
| 488 |
log("π οΈ Executing tool phase...")
|
| 489 |
tool_results = []
|
| 490 |
generated_images = []
|
| 491 |
thoughts = ""
|
| 492 |
|
| 493 |
-
# Build context-aware prompt
|
| 494 |
system_prompt = self.extension_manager.build_system_prompt(
|
| 495 |
self.user_id, self.enabled_extensions
|
| 496 |
)
|
| 497 |
|
| 498 |
-
# Enhanced instructions for multi-step tasks
|
| 499 |
if analysis['multi_step']:
|
| 500 |
system_prompt += """
|
| 501 |
|
|
@@ -515,7 +487,6 @@ CONTEXT AWARENESS:
|
|
| 515 |
- Reference specific items by their IDs or names from the state
|
| 516 |
"""
|
| 517 |
|
| 518 |
-
# Add explicit extension requirements
|
| 519 |
if analysis.get('explicit_extensions'):
|
| 520 |
explicit_exts = analysis['explicit_extensions']
|
| 521 |
system_prompt += f"""
|
|
@@ -535,7 +506,6 @@ YOU MUST use these extensions in your response:
|
|
| 535 |
|
| 536 |
system_prompt += "\n\nDo NOT skip these explicitly requested extensions!\n"
|
| 537 |
|
| 538 |
-
# Multi-turn tool execution
|
| 539 |
max_rounds = 5 if analysis['complexity'] == 'complex' else 3
|
| 540 |
current_round = 0
|
| 541 |
|
|
@@ -556,11 +526,9 @@ YOU MUST use these extensions in your response:
|
|
| 556 |
if not function_calls:
|
| 557 |
log(f"β
No more tools needed after round {current_round}")
|
| 558 |
if text_response:
|
| 559 |
-
# Agent provided final answer
|
| 560 |
return text_response, tool_results, generated_images, thoughts, search_citations
|
| 561 |
break
|
| 562 |
|
| 563 |
-
# Execute function calls
|
| 564 |
results = self.extension_manager.handle_function_calls(
|
| 565 |
self.user_id, self.enabled_extensions, function_calls
|
| 566 |
)
|
|
@@ -568,17 +536,17 @@ YOU MUST use these extensions in your response:
|
|
| 568 |
for (tool_name, result) in results:
|
| 569 |
tool_results.append((tool_name, result))
|
| 570 |
|
| 571 |
-
# Extract generated images IMMEDIATELY
|
| 572 |
if isinstance(result, dict) and 'image_base64' in result:
|
| 573 |
-
|
| 574 |
'base64': result['image_base64'],
|
| 575 |
'title': result.get('message', 'Generated visualization'),
|
| 576 |
'filepath': result.get('filepath', '')
|
| 577 |
-
}
|
|
|
|
| 578 |
log(f"π Captured visualization: {result.get('message', 'Chart')}")
|
|
|
|
| 579 |
|
| 580 |
-
# Check if we should suggest follow-up extension
|
| 581 |
-
# Find which extension this tool belongs to
|
| 582 |
for ext_name in self.enabled_extensions:
|
| 583 |
ext = self.extension_manager.get_extension(ext_name)
|
| 584 |
if ext:
|
|
@@ -586,7 +554,6 @@ YOU MUST use these extensions in your response:
|
|
| 586 |
if hasattr(tool, 'function_declarations'):
|
| 587 |
for func_decl in tool.function_declarations:
|
| 588 |
if func_decl.name == tool_name:
|
| 589 |
-
# Found the extension, check for suggestions
|
| 590 |
suggestion = self.extension_manager.suggest_next_extension(
|
| 591 |
ext_name, result, self.enabled_extensions
|
| 592 |
)
|
|
@@ -594,24 +561,20 @@ YOU MUST use these extensions in your response:
|
|
| 594 |
log(f"π‘ Suggestion: Use {suggestion['extension']}.{suggestion['tool']} - {suggestion['reason']}")
|
| 595 |
break
|
| 596 |
|
| 597 |
-
# Prepare for next round
|
| 598 |
if current_round < max_rounds:
|
| 599 |
results_summary = self._format_results_for_context(results)
|
| 600 |
|
| 601 |
-
# Build enhanced context with extension-aware guidance
|
| 602 |
next_prompt = f"""Previous actions completed:
|
| 603 |
{results_summary}
|
| 604 |
|
| 605 |
Original query: {query}
|
| 606 |
|
| 607 |
"""
|
| 608 |
-
# Add suggestions for next steps based on results
|
| 609 |
suggestions = self._generate_next_step_suggestions(results)
|
| 610 |
if suggestions:
|
| 611 |
next_prompt += f"\n**β οΈ REQUIRED NEXT STEPS (DO NOT SKIP):**\n{suggestions}\n\n"
|
| 612 |
next_prompt += "You MUST complete these steps before finishing. The user explicitly requested these actions.\n\n"
|
| 613 |
|
| 614 |
-
# Check if explicit extensions still need to be satisfied
|
| 615 |
if analysis.get('explicit_extensions'):
|
| 616 |
unsatisfied = self._check_unsatisfied_extensions(
|
| 617 |
analysis['explicit_extensions'],
|
|
@@ -633,22 +596,21 @@ Original query: {query}
|
|
| 633 |
next_prompt += "Continue with required steps. If all user requirements are satisfied, provide final answer."
|
| 634 |
|
| 635 |
prompt = next_prompt
|
| 636 |
-
|
| 637 |
-
# Don't pass file_parts again in subsequent rounds
|
| 638 |
file_parts = None
|
| 639 |
|
| 640 |
-
# Check if we have a final text response from tool agent
|
| 641 |
if text_response and not function_calls:
|
| 642 |
-
# Agent provided final answer without needing synthesis
|
| 643 |
log(f"β
Tool agent provided final answer")
|
| 644 |
return text_response, tool_results, generated_images, thoughts, search_citations
|
| 645 |
|
| 646 |
-
|
|
|
|
| 647 |
final_answer, synth_images = self.synthesize_response(
|
| 648 |
query, search_results, tool_results, search_citations, file_parts
|
| 649 |
)
|
| 650 |
|
|
|
|
| 651 |
generated_images.extend(synth_images)
|
|
|
|
| 652 |
|
| 653 |
return final_answer, tool_results, generated_images, thoughts, search_citations
|
| 654 |
|
|
@@ -657,9 +619,8 @@ Original query: {query}
|
|
| 657 |
formatted = []
|
| 658 |
for tool_name, result in results:
|
| 659 |
if isinstance(result, dict):
|
| 660 |
-
# Clean up result for context
|
| 661 |
clean_result = dict(result)
|
| 662 |
-
clean_result.pop('image_base64', None)
|
| 663 |
formatted.append(f"- {tool_name}: {json.dumps(clean_result, indent=2)[:300]}")
|
| 664 |
else:
|
| 665 |
formatted.append(f"- {tool_name}: {str(result)[:200]}")
|
|
@@ -673,36 +634,28 @@ Original query: {query}
|
|
| 673 |
if not isinstance(result, dict):
|
| 674 |
continue
|
| 675 |
|
| 676 |
-
# Stock history β visualization
|
| 677 |
if tool_name == 'get_stock_history' and 'visualization' in self.enabled_extensions:
|
| 678 |
if result.get('success') and 'dates' in result and 'close_prices' in result:
|
| 679 |
suggestions.append("π MUST create a line chart with the stock history data (dates and prices are ready)")
|
| 680 |
|
| 681 |
-
# Stock comparison β visualization
|
| 682 |
if tool_name == 'compare_stocks' and 'visualization' in self.enabled_extensions:
|
| 683 |
if result.get('success') and 'comparison' in result:
|
| 684 |
suggestions.append("π MUST create a bar chart comparing the stocks (comparison data is ready)")
|
| 685 |
|
| 686 |
-
# Deep research completed β get stock history if visualization requested
|
| 687 |
if tool_name == 'conduct_deep_research' and 'yfinance' in self.enabled_extensions:
|
| 688 |
if result.get('success'):
|
| 689 |
-
# Extract company/stock mentioned in research
|
| 690 |
topic = result.get('topic', '')
|
| 691 |
-
# Common stock keywords
|
| 692 |
if any(word in topic.lower() for word in ['stock', 'nvidia', 'nvda', 'company']):
|
| 693 |
ticker = 'NVDA' if 'nvidia' in topic.lower() or 'nvda' in topic.lower() else None
|
| 694 |
if ticker and 'visualization' in self.enabled_extensions:
|
| 695 |
suggestions.append(f"π MUST get {ticker} stock history for past 4 years (user requested graph of growth)")
|
| 696 |
|
| 697 |
-
# Deep research β visualization
|
| 698 |
if tool_name == 'conduct_deep_research' and 'visualization' in self.enabled_extensions:
|
| 699 |
if result.get('success') and result.get('num_searches', 0) > 0:
|
| 700 |
-
# Check if there's numerical data to visualize
|
| 701 |
report = result.get('report', '')
|
| 702 |
if 'pros' in report.lower() and 'cons' in report.lower():
|
| 703 |
suggestions.append("π Create a bar chart showing pros vs cons count")
|
| 704 |
|
| 705 |
-
# YouTube β research
|
| 706 |
if tool_name == 'analyze_youtube_video' and 'deep_research' in self.enabled_extensions:
|
| 707 |
if result.get('success'):
|
| 708 |
suggestions.append("π¬ Conduct deep research on topics mentioned in the video")
|
|
@@ -714,23 +667,20 @@ Original query: {query}
|
|
| 714 |
"""Check which explicitly requested extensions haven't been used yet"""
|
| 715 |
used_extensions = set()
|
| 716 |
|
| 717 |
-
# Map tool calls to extensions
|
| 718 |
extension_tool_map = {
|
| 719 |
-
'deep_research': ['conduct_deep_research'],
|
| 720 |
'yfinance': ['get_stock_price', 'get_stock_info', 'get_stock_history', 'compare_stocks'],
|
| 721 |
'visualization': ['create_line_chart', 'create_bar_chart', 'create_scatter_plot', 'create_pie_chart'],
|
| 722 |
'youtube': ['analyze_youtube_video', 'get_video_chapters'],
|
| 723 |
'timer': ['set_timer', 'list_timers', 'check_timer', 'cancel_timer']
|
| 724 |
}
|
| 725 |
|
| 726 |
-
# Check which extensions have been used
|
| 727 |
for tool_name, _ in tool_results:
|
| 728 |
for ext_name, tool_list in extension_tool_map.items():
|
| 729 |
if tool_name in tool_list:
|
| 730 |
used_extensions.add(ext_name)
|
| 731 |
break
|
| 732 |
|
| 733 |
-
# Return extensions that were requested but not used
|
| 734 |
unsatisfied = []
|
| 735 |
for ext in explicit_extensions:
|
| 736 |
if ext not in used_extensions:
|
|
@@ -845,7 +795,7 @@ Original query: {query}
|
|
| 845 |
generated_images = []
|
| 846 |
deep_research_report = None
|
| 847 |
deep_research_sources = None
|
| 848 |
-
has_deep_research = False
|
| 849 |
|
| 850 |
if tool_results:
|
| 851 |
synthesis_prompt += "[Tool Execution Results]\n"
|
|
@@ -856,10 +806,10 @@ Original query: {query}
|
|
| 856 |
'title': result.get('message', 'Generated visualization'),
|
| 857 |
'filepath': result.get('filepath', '')
|
| 858 |
})
|
|
|
|
| 859 |
result_clean = dict(result)
|
| 860 |
result_clean.pop('image_base64', None)
|
| 861 |
synthesis_prompt += f"- {tool_name}: {result_clean.get('message', 'Chart created')}\n"
|
| 862 |
-
# Capture deep research report
|
| 863 |
elif tool_name == 'conduct_deep_research' and isinstance(result, dict):
|
| 864 |
if result.get('success') and result.get('report'):
|
| 865 |
has_deep_research = True
|
|
@@ -871,7 +821,6 @@ Original query: {query}
|
|
| 871 |
result_str = str(result)[:500]
|
| 872 |
synthesis_prompt += f"- {tool_name}: {result_str}\n"
|
| 873 |
|
| 874 |
-
# MODIFIED: Different instructions based on whether deep research was done
|
| 875 |
if has_deep_research:
|
| 876 |
synthesis_prompt += "\n\nProvide a brief summary of what was accomplished (2-3 sentences). DO NOT include the full research report - it will be displayed separately below your summary."
|
| 877 |
else:
|
|
@@ -880,7 +829,7 @@ Original query: {query}
|
|
| 880 |
config = types.GenerateContentConfig(
|
| 881 |
system_instruction="You are a synthesis specialist. Combine information from multiple sources into coherent, helpful responses. Be concise when deep research reports are available since they will be displayed separately.",
|
| 882 |
temperature=0.7,
|
| 883 |
-
max_output_tokens=2048
|
| 884 |
)
|
| 885 |
|
| 886 |
try:
|
|
@@ -901,12 +850,10 @@ Original query: {query}
|
|
| 901 |
if getattr(part, "text", None):
|
| 902 |
result_text += part.text
|
| 903 |
|
| 904 |
-
# MODIFIED: Only append research report if it exists
|
| 905 |
if has_deep_research and deep_research_report:
|
| 906 |
result_text += "\n\n---\n\n## π Complete Research Report\n\n"
|
| 907 |
result_text += deep_research_report
|
| 908 |
|
| 909 |
-
# Add sources at the end
|
| 910 |
if deep_research_sources:
|
| 911 |
result_text += "\n\n### π Research Sources\n\n"
|
| 912 |
for idx, source in enumerate(deep_research_sources[:30], 1):
|
|
@@ -918,6 +865,7 @@ Original query: {query}
|
|
| 918 |
if len(deep_research_sources) > 30:
|
| 919 |
result_text += f"\n*...and {len(deep_research_sources) - 30} more sources*\n"
|
| 920 |
|
|
|
|
| 921 |
return result_text, generated_images
|
| 922 |
except Exception as e:
|
| 923 |
log(f"β οΈ Synthesis error: {e}")
|
|
@@ -968,7 +916,6 @@ def insert_citations_from_grounding(candidates):
|
|
| 968 |
return None
|
| 969 |
|
| 970 |
|
| 971 |
-
# Global instances
|
| 972 |
EXTENSION_MANAGER = ExtensionManager()
|
| 973 |
CHAT_SESSIONS: Dict[str, Dict[str, Any]] = {}
|
| 974 |
|
|
@@ -1018,7 +965,6 @@ def chat_with_gemini(api_key, chat_history_msgs, multimodal_input, show_thoughts
|
|
| 1018 |
if chat_history_msgs is None:
|
| 1019 |
chat_history_msgs = []
|
| 1020 |
|
| 1021 |
-
# Process uploaded files
|
| 1022 |
file_parts = []
|
| 1023 |
if uploaded_files:
|
| 1024 |
log(f"π Processing {len(uploaded_files)} uploaded file(s)...")
|
|
@@ -1034,7 +980,6 @@ def chat_with_gemini(api_key, chat_history_msgs, multimodal_input, show_thoughts
|
|
| 1034 |
|
| 1035 |
assistant_base_index = len(chat_history_msgs)
|
| 1036 |
|
| 1037 |
-
# Setup thinking display
|
| 1038 |
if show_thoughts:
|
| 1039 |
thought_index = assistant_base_index
|
| 1040 |
chat_history_msgs.append({"role": "assistant", "content": "<em>π Thinking...</em>"})
|
|
@@ -1048,7 +993,6 @@ def chat_with_gemini(api_key, chat_history_msgs, multimodal_input, show_thoughts
|
|
| 1048 |
yield chat_history_msgs
|
| 1049 |
|
| 1050 |
try:
|
| 1051 |
-
# Check for proactive messages from extensions first
|
| 1052 |
proactive_msgs = EXTENSION_MANAGER.check_proactive_messages(api_key, enabled_extensions)
|
| 1053 |
if proactive_msgs:
|
| 1054 |
for msg in proactive_msgs:
|
|
@@ -1056,7 +1000,6 @@ def chat_with_gemini(api_key, chat_history_msgs, multimodal_input, show_thoughts
|
|
| 1056 |
answer_index += 1
|
| 1057 |
yield chat_history_msgs
|
| 1058 |
|
| 1059 |
-
# Use improved orchestrator
|
| 1060 |
if enabled_extensions:
|
| 1061 |
log("π Using improved orchestrator")
|
| 1062 |
orchestrator = ImprovedOrchestrator(
|
|
@@ -1065,11 +1008,14 @@ def chat_with_gemini(api_key, chat_history_msgs, multimodal_input, show_thoughts
|
|
| 1065 |
|
| 1066 |
budget = reasoning_budget(reasoning_level)
|
| 1067 |
|
| 1068 |
-
# Execute with planning
|
| 1069 |
final_answer, tool_results, generated_images, thoughts, search_citations = \
|
| 1070 |
orchestrator.execute_with_planning(user_text, file_parts, budget)
|
| 1071 |
|
| 1072 |
-
#
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1073 |
if thoughts and show_thoughts:
|
| 1074 |
chat_history_msgs[thought_index]["content"] = (
|
| 1075 |
f"<details open>"
|
|
@@ -1081,7 +1027,6 @@ def chat_with_gemini(api_key, chat_history_msgs, multimodal_input, show_thoughts
|
|
| 1081 |
)
|
| 1082 |
yield chat_history_msgs
|
| 1083 |
|
| 1084 |
-
# Build final content
|
| 1085 |
final_content = (
|
| 1086 |
f"<div><strong>π€ Response</strong>"
|
| 1087 |
f"<div style='white-space:pre-wrap;background:inherit;color:inherit;"
|
|
@@ -1089,20 +1034,28 @@ def chat_with_gemini(api_key, chat_history_msgs, multimodal_input, show_thoughts
|
|
| 1089 |
f"{final_answer.strip()}</div></div>"
|
| 1090 |
)
|
| 1091 |
|
| 1092 |
-
# Add generated images
|
|
|
|
| 1093 |
if generated_images:
|
| 1094 |
-
for img_data in generated_images:
|
|
|
|
| 1095 |
final_content += f"\n\n<div style='margin-top:16px;'>"
|
| 1096 |
-
final_content += f"<strong>π {img_data
|
| 1097 |
-
|
| 1098 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1099 |
final_content += f"<br/><small style='color:#666;'>Saved to: {img_data['filepath']}</small>"
|
| 1100 |
final_content += "</div>"
|
|
|
|
|
|
|
|
|
|
| 1101 |
|
| 1102 |
-
# REMOVED: The duplicate research report block that was here (lines 1000-1070)
|
| 1103 |
-
# Research report is now handled ONLY in synthesize_response() method
|
| 1104 |
-
|
| 1105 |
-
# Add citations
|
| 1106 |
if search_citations:
|
| 1107 |
final_content += "\n\n" + search_citations
|
| 1108 |
|
|
@@ -1110,7 +1063,6 @@ def chat_with_gemini(api_key, chat_history_msgs, multimodal_input, show_thoughts
|
|
| 1110 |
yield chat_history_msgs
|
| 1111 |
|
| 1112 |
else:
|
| 1113 |
-
# No extensions - simple streaming with search
|
| 1114 |
log("πΊ Using simple streaming mode")
|
| 1115 |
|
| 1116 |
parts = []
|
|
@@ -1178,7 +1130,6 @@ def chat_with_gemini(api_key, chat_history_msgs, multimodal_input, show_thoughts
|
|
| 1178 |
)
|
| 1179 |
yield chat_history_msgs
|
| 1180 |
|
| 1181 |
-
# Add citations
|
| 1182 |
if last_chunk:
|
| 1183 |
citations = insert_citations_from_grounding(last_chunk.candidates)
|
| 1184 |
if citations:
|
|
@@ -1260,7 +1211,6 @@ with gr.Blocks(
|
|
| 1260 |
info="Display reasoning process before answers.",
|
| 1261 |
)
|
| 1262 |
|
| 1263 |
-
# Build extension checkboxes
|
| 1264 |
extension_checkboxes = build_extension_ui()
|
| 1265 |
|
| 1266 |
with gr.Column(scale=4):
|
|
@@ -1284,14 +1234,12 @@ with gr.Blocks(
|
|
| 1284 |
autofocus=True
|
| 1285 |
)
|
| 1286 |
|
| 1287 |
-
# Hidden state to track enabled extensions
|
| 1288 |
enabled_extensions_state = gr.State([])
|
| 1289 |
|
| 1290 |
def clear_box():
|
| 1291 |
return {"text": "", "files": []}
|
| 1292 |
|
| 1293 |
def handle_chat(api_key_input, chat_history_msgs, multimodal_dict, thinking_flag, reasoning_lvl, *extension_states):
|
| 1294 |
-
# Convert extension checkbox states to list of enabled extension names
|
| 1295 |
enabled = []
|
| 1296 |
for (ext_name, _), is_enabled in zip(extension_checkboxes, extension_states):
|
| 1297 |
if is_enabled:
|
|
@@ -1309,7 +1257,6 @@ with gr.Blocks(
|
|
| 1309 |
if not api_key_input or not enabled_exts:
|
| 1310 |
return chat_history
|
| 1311 |
|
| 1312 |
-
# Check all extensions for proactive messages
|
| 1313 |
proactive_msgs = EXTENSION_MANAGER.check_proactive_messages(api_key_input, enabled_exts)
|
| 1314 |
|
| 1315 |
if proactive_msgs:
|
|
@@ -1322,10 +1269,8 @@ with gr.Blocks(
|
|
| 1322 |
|
| 1323 |
return chat_history
|
| 1324 |
|
| 1325 |
-
# Get just the checkbox components for inputs
|
| 1326 |
checkbox_components = [cb for _, cb in extension_checkboxes]
|
| 1327 |
|
| 1328 |
-
# Main chat submission
|
| 1329 |
multimodal_msg.submit(
|
| 1330 |
fn=handle_chat,
|
| 1331 |
inputs=[api_key, chatbot, multimodal_msg, show_thoughts, reasoning_level] + checkbox_components,
|
|
@@ -1333,7 +1278,6 @@ with gr.Blocks(
|
|
| 1333 |
queue=True,
|
| 1334 |
).then(fn=clear_box, outputs=[multimodal_msg])
|
| 1335 |
|
| 1336 |
-
# Background timer check - runs every 10 seconds
|
| 1337 |
timer_check = gr.Timer(value=10, active=True)
|
| 1338 |
|
| 1339 |
def update_enabled_state(*extension_states):
|
|
@@ -1343,7 +1287,6 @@ with gr.Blocks(
|
|
| 1343 |
enabled.append(ext_name)
|
| 1344 |
return enabled
|
| 1345 |
|
| 1346 |
-
# Update enabled extensions state whenever checkboxes change
|
| 1347 |
for _, cb in extension_checkboxes:
|
| 1348 |
cb.change(
|
| 1349 |
fn=update_enabled_state,
|
|
@@ -1351,7 +1294,6 @@ with gr.Blocks(
|
|
| 1351 |
outputs=[enabled_extensions_state]
|
| 1352 |
)
|
| 1353 |
|
| 1354 |
-
# Timer polling for proactive messages
|
| 1355 |
timer_check.tick(
|
| 1356 |
fn=check_timers,
|
| 1357 |
inputs=[api_key, chatbot, enabled_extensions_state],
|
|
|
|
| 79 |
|
| 80 |
def __init__(self):
|
| 81 |
self.extensions: Dict[str, BaseExtension] = {}
|
| 82 |
+
self.extension_interactions: Dict[str, List[str]] = {}
|
| 83 |
self.load_extensions()
|
| 84 |
|
| 85 |
def load_extensions(self):
|
|
|
|
| 117 |
traceback.print_exc()
|
| 118 |
|
| 119 |
log(f"π Total extensions loaded: {len(self.extensions)}")
|
|
|
|
|
|
|
| 120 |
self._detect_extension_relationships()
|
| 121 |
|
| 122 |
def _detect_extension_relationships(self):
|
| 123 |
"""Detect which extensions can work together based on capabilities"""
|
| 124 |
log("π Detecting extension relationships...")
|
| 125 |
|
|
|
|
| 126 |
relationships = {
|
| 127 |
+
'yfinance': ['visualization', 'deep_research'],
|
| 128 |
+
'deep_research': ['visualization', 'yfinance'],
|
| 129 |
+
'youtube': ['deep_research', 'visualization'],
|
| 130 |
+
'timer': ['deep_research'],
|
| 131 |
+
'visualization': ['yfinance', 'deep_research', 'youtube']
|
| 132 |
}
|
| 133 |
|
| 134 |
for ext_name, compatible in relationships.items():
|
|
|
|
| 159 |
if enabled_exts:
|
| 160 |
prompt += "\n\n# ENABLED EXTENSIONS\nYou currently have these extensions enabled:\n\n"
|
| 161 |
for ext in enabled_exts:
|
|
|
|
| 162 |
ext.initialize_state(user_id)
|
|
|
|
|
|
|
| 163 |
prompt += f"## {ext.display_name}\n{ext.get_system_context()}\n\n"
|
| 164 |
|
|
|
|
| 165 |
state = ext.get_state(user_id)
|
| 166 |
if state and hasattr(ext, 'get_state_summary'):
|
| 167 |
state_summary = ext.get_state_summary(user_id)
|
| 168 |
if state_summary:
|
| 169 |
prompt += f"**Current State:** {state_summary}\n\n"
|
| 170 |
|
|
|
|
| 171 |
if len(enabled_exts) > 1:
|
| 172 |
prompt += self._build_multi_extension_guidance(enabled_list)
|
| 173 |
|
|
|
|
| 178 |
guidance = "\n\n# MULTI-EXTENSION REASONING\n"
|
| 179 |
guidance += "You have multiple extensions enabled. You can combine them intelligently:\n\n"
|
| 180 |
|
|
|
|
| 181 |
combinations = []
|
| 182 |
|
| 183 |
if 'yfinance' in enabled_list and 'visualization' in enabled_list:
|
|
|
|
| 219 |
|
| 220 |
suggestions = []
|
| 221 |
|
|
|
|
| 222 |
if current_extension == 'yfinance' and isinstance(result, dict):
|
| 223 |
if 'dates' in result and 'close_prices' in result and 'visualization' in compatible:
|
| 224 |
suggestions.append({
|
|
|
|
| 235 |
'data_available': True
|
| 236 |
})
|
| 237 |
|
|
|
|
| 238 |
if current_extension == 'deep_research' and isinstance(result, dict):
|
| 239 |
if result.get('success') and 'visualization' in compatible:
|
| 240 |
suggestions.append({
|
|
|
|
| 244 |
'data_available': True
|
| 245 |
})
|
| 246 |
|
|
|
|
| 247 |
if current_extension == 'youtube' and isinstance(result, dict):
|
| 248 |
if result.get('success') and 'deep_research' in compatible:
|
| 249 |
suggestions.append({
|
|
|
|
| 279 |
|
| 280 |
log(f"π§ Executing: {function_name}({json.dumps(args, indent=2)[:100]}...)")
|
| 281 |
|
|
|
|
| 282 |
handled = False
|
| 283 |
for ext_name in enabled_list:
|
| 284 |
ext = self.get_extension(ext_name)
|
| 285 |
if ext:
|
|
|
|
| 286 |
for tool in ext.get_tools():
|
| 287 |
if hasattr(tool, 'function_declarations'):
|
| 288 |
for func_decl in tool.function_declarations:
|
|
|
|
| 329 |
self.user_id = user_id
|
| 330 |
self.enabled_extensions = enabled_extensions
|
| 331 |
self.search_chat = client.chats.create(model="gemini-2.5-flash")
|
| 332 |
+
self.execution_history = []
|
| 333 |
|
| 334 |
def analyze_query(self, query: str, file_parts: List = None) -> Dict[str, Any]:
|
| 335 |
"""Analyze query to determine optimal execution strategy"""
|
| 336 |
log("π§ Analyzing query to determine strategy...")
|
| 337 |
|
|
|
|
| 338 |
query_lower = query.lower()
|
| 339 |
explicit_extensions = []
|
| 340 |
|
|
|
|
| 394 |
|
| 395 |
if response.candidates and response.candidates[0].content and response.candidates[0].content.parts:
|
| 396 |
text = response.candidates[0].content.parts[0].text
|
|
|
|
| 397 |
json_match = re.search(r'```(?:json)?\s*(\{.*?\})\s*```', text, re.DOTALL)
|
| 398 |
if json_match:
|
| 399 |
text = json_match.group(1)
|
| 400 |
|
| 401 |
analysis = json.loads(text)
|
| 402 |
|
|
|
|
| 403 |
if explicit_extensions:
|
| 404 |
analysis['explicit_extensions'] = explicit_extensions
|
| 405 |
analysis['multi_step'] = True
|
|
|
|
| 410 |
except Exception as e:
|
| 411 |
log(f"β οΈ Query analysis failed: {e}, using heuristics")
|
| 412 |
|
|
|
|
| 413 |
heuristic = self._heuristic_analysis(query)
|
| 414 |
if explicit_extensions:
|
| 415 |
heuristic['explicit_extensions'] = explicit_extensions
|
|
|
|
| 420 |
"""Fallback heuristic analysis"""
|
| 421 |
query_lower = query.lower()
|
| 422 |
|
|
|
|
| 423 |
search_keywords = ['search', 'find', 'look up', 'what is', 'latest', 'current', 'news']
|
| 424 |
needs_search = "yes" if any(kw in query_lower for kw in search_keywords) else "maybe"
|
| 425 |
|
|
|
|
| 426 |
multi_step_keywords = ['then', 'after that', 'and then', 'research and', 'analyze and']
|
| 427 |
multi_step = any(kw in query_lower for kw in multi_step_keywords)
|
| 428 |
|
|
|
|
| 429 |
context_keywords = ['the', 'my', 'that', 'previous', 'earlier']
|
| 430 |
uses_context = any(kw in query_lower for kw in context_keywords)
|
| 431 |
|
|
|
|
| 432 |
word_count = len(query.split())
|
| 433 |
if word_count < 10 and not multi_step:
|
| 434 |
complexity = "simple"
|
|
|
|
| 447 |
}
|
| 448 |
|
| 449 |
def execute_with_planning(self, query: str, file_parts: List = None,
|
| 450 |
+
reasoning_budget: int = -1) -> Tuple[str, List, List, str, Optional[str]]:
|
| 451 |
"""Execute query with planning and adaptive strategy"""
|
| 452 |
|
|
|
|
| 453 |
analysis = self.analyze_query(query, file_parts)
|
| 454 |
|
|
|
|
| 455 |
search_results = ""
|
| 456 |
search_citations = None
|
| 457 |
|
|
|
|
| 458 |
if analysis['needs_search'] in ['yes', 'maybe']:
|
| 459 |
log("π Executing search phase...")
|
| 460 |
search_results, search_citations = self.call_search_agent(query, file_parts)
|
| 461 |
|
|
|
|
| 462 |
log("π οΈ Executing tool phase...")
|
| 463 |
tool_results = []
|
| 464 |
generated_images = []
|
| 465 |
thoughts = ""
|
| 466 |
|
|
|
|
| 467 |
system_prompt = self.extension_manager.build_system_prompt(
|
| 468 |
self.user_id, self.enabled_extensions
|
| 469 |
)
|
| 470 |
|
|
|
|
| 471 |
if analysis['multi_step']:
|
| 472 |
system_prompt += """
|
| 473 |
|
|
|
|
| 487 |
- Reference specific items by their IDs or names from the state
|
| 488 |
"""
|
| 489 |
|
|
|
|
| 490 |
if analysis.get('explicit_extensions'):
|
| 491 |
explicit_exts = analysis['explicit_extensions']
|
| 492 |
system_prompt += f"""
|
|
|
|
| 506 |
|
| 507 |
system_prompt += "\n\nDo NOT skip these explicitly requested extensions!\n"
|
| 508 |
|
|
|
|
| 509 |
max_rounds = 5 if analysis['complexity'] == 'complex' else 3
|
| 510 |
current_round = 0
|
| 511 |
|
|
|
|
| 526 |
if not function_calls:
|
| 527 |
log(f"β
No more tools needed after round {current_round}")
|
| 528 |
if text_response:
|
|
|
|
| 529 |
return text_response, tool_results, generated_images, thoughts, search_citations
|
| 530 |
break
|
| 531 |
|
|
|
|
| 532 |
results = self.extension_manager.handle_function_calls(
|
| 533 |
self.user_id, self.enabled_extensions, function_calls
|
| 534 |
)
|
|
|
|
| 536 |
for (tool_name, result) in results:
|
| 537 |
tool_results.append((tool_name, result))
|
| 538 |
|
| 539 |
+
# CRITICAL: Extract generated images IMMEDIATELY
|
| 540 |
if isinstance(result, dict) and 'image_base64' in result:
|
| 541 |
+
img_data = {
|
| 542 |
'base64': result['image_base64'],
|
| 543 |
'title': result.get('message', 'Generated visualization'),
|
| 544 |
'filepath': result.get('filepath', '')
|
| 545 |
+
}
|
| 546 |
+
generated_images.append(img_data)
|
| 547 |
log(f"π Captured visualization: {result.get('message', 'Chart')}")
|
| 548 |
+
log(f" πΌοΈ Image data length: {len(result['image_base64'])} chars")
|
| 549 |
|
|
|
|
|
|
|
| 550 |
for ext_name in self.enabled_extensions:
|
| 551 |
ext = self.extension_manager.get_extension(ext_name)
|
| 552 |
if ext:
|
|
|
|
| 554 |
if hasattr(tool, 'function_declarations'):
|
| 555 |
for func_decl in tool.function_declarations:
|
| 556 |
if func_decl.name == tool_name:
|
|
|
|
| 557 |
suggestion = self.extension_manager.suggest_next_extension(
|
| 558 |
ext_name, result, self.enabled_extensions
|
| 559 |
)
|
|
|
|
| 561 |
log(f"π‘ Suggestion: Use {suggestion['extension']}.{suggestion['tool']} - {suggestion['reason']}")
|
| 562 |
break
|
| 563 |
|
|
|
|
| 564 |
if current_round < max_rounds:
|
| 565 |
results_summary = self._format_results_for_context(results)
|
| 566 |
|
|
|
|
| 567 |
next_prompt = f"""Previous actions completed:
|
| 568 |
{results_summary}
|
| 569 |
|
| 570 |
Original query: {query}
|
| 571 |
|
| 572 |
"""
|
|
|
|
| 573 |
suggestions = self._generate_next_step_suggestions(results)
|
| 574 |
if suggestions:
|
| 575 |
next_prompt += f"\n**β οΈ REQUIRED NEXT STEPS (DO NOT SKIP):**\n{suggestions}\n\n"
|
| 576 |
next_prompt += "You MUST complete these steps before finishing. The user explicitly requested these actions.\n\n"
|
| 577 |
|
|
|
|
| 578 |
if analysis.get('explicit_extensions'):
|
| 579 |
unsatisfied = self._check_unsatisfied_extensions(
|
| 580 |
analysis['explicit_extensions'],
|
|
|
|
| 596 |
next_prompt += "Continue with required steps. If all user requirements are satisfied, provide final answer."
|
| 597 |
|
| 598 |
prompt = next_prompt
|
|
|
|
|
|
|
| 599 |
file_parts = None
|
| 600 |
|
|
|
|
| 601 |
if text_response and not function_calls:
|
|
|
|
| 602 |
log(f"β
Tool agent provided final answer")
|
| 603 |
return text_response, tool_results, generated_images, thoughts, search_citations
|
| 604 |
|
| 605 |
+
log(f"π¨ Before synthesis: {len(generated_images)} images in list")
|
| 606 |
+
|
| 607 |
final_answer, synth_images = self.synthesize_response(
|
| 608 |
query, search_results, tool_results, search_citations, file_parts
|
| 609 |
)
|
| 610 |
|
| 611 |
+
log(f"π¨ After synthesis: {len(synth_images)} new images")
|
| 612 |
generated_images.extend(synth_images)
|
| 613 |
+
log(f"π¨ Total images to return: {len(generated_images)}")
|
| 614 |
|
| 615 |
return final_answer, tool_results, generated_images, thoughts, search_citations
|
| 616 |
|
|
|
|
| 619 |
formatted = []
|
| 620 |
for tool_name, result in results:
|
| 621 |
if isinstance(result, dict):
|
|
|
|
| 622 |
clean_result = dict(result)
|
| 623 |
+
clean_result.pop('image_base64', None)
|
| 624 |
formatted.append(f"- {tool_name}: {json.dumps(clean_result, indent=2)[:300]}")
|
| 625 |
else:
|
| 626 |
formatted.append(f"- {tool_name}: {str(result)[:200]}")
|
|
|
|
| 634 |
if not isinstance(result, dict):
|
| 635 |
continue
|
| 636 |
|
|
|
|
| 637 |
if tool_name == 'get_stock_history' and 'visualization' in self.enabled_extensions:
|
| 638 |
if result.get('success') and 'dates' in result and 'close_prices' in result:
|
| 639 |
suggestions.append("π MUST create a line chart with the stock history data (dates and prices are ready)")
|
| 640 |
|
|
|
|
| 641 |
if tool_name == 'compare_stocks' and 'visualization' in self.enabled_extensions:
|
| 642 |
if result.get('success') and 'comparison' in result:
|
| 643 |
suggestions.append("π MUST create a bar chart comparing the stocks (comparison data is ready)")
|
| 644 |
|
|
|
|
| 645 |
if tool_name == 'conduct_deep_research' and 'yfinance' in self.enabled_extensions:
|
| 646 |
if result.get('success'):
|
|
|
|
| 647 |
topic = result.get('topic', '')
|
|
|
|
| 648 |
if any(word in topic.lower() for word in ['stock', 'nvidia', 'nvda', 'company']):
|
| 649 |
ticker = 'NVDA' if 'nvidia' in topic.lower() or 'nvda' in topic.lower() else None
|
| 650 |
if ticker and 'visualization' in self.enabled_extensions:
|
| 651 |
suggestions.append(f"π MUST get {ticker} stock history for past 4 years (user requested graph of growth)")
|
| 652 |
|
|
|
|
| 653 |
if tool_name == 'conduct_deep_research' and 'visualization' in self.enabled_extensions:
|
| 654 |
if result.get('success') and result.get('num_searches', 0) > 0:
|
|
|
|
| 655 |
report = result.get('report', '')
|
| 656 |
if 'pros' in report.lower() and 'cons' in report.lower():
|
| 657 |
suggestions.append("π Create a bar chart showing pros vs cons count")
|
| 658 |
|
|
|
|
| 659 |
if tool_name == 'analyze_youtube_video' and 'deep_research' in self.enabled_extensions:
|
| 660 |
if result.get('success'):
|
| 661 |
suggestions.append("π¬ Conduct deep research on topics mentioned in the video")
|
|
|
|
| 667 |
"""Check which explicitly requested extensions haven't been used yet"""
|
| 668 |
used_extensions = set()
|
| 669 |
|
|
|
|
| 670 |
extension_tool_map = {
|
| 671 |
+
'deep_research': ['conduct_deep_research'],
|
| 672 |
'yfinance': ['get_stock_price', 'get_stock_info', 'get_stock_history', 'compare_stocks'],
|
| 673 |
'visualization': ['create_line_chart', 'create_bar_chart', 'create_scatter_plot', 'create_pie_chart'],
|
| 674 |
'youtube': ['analyze_youtube_video', 'get_video_chapters'],
|
| 675 |
'timer': ['set_timer', 'list_timers', 'check_timer', 'cancel_timer']
|
| 676 |
}
|
| 677 |
|
|
|
|
| 678 |
for tool_name, _ in tool_results:
|
| 679 |
for ext_name, tool_list in extension_tool_map.items():
|
| 680 |
if tool_name in tool_list:
|
| 681 |
used_extensions.add(ext_name)
|
| 682 |
break
|
| 683 |
|
|
|
|
| 684 |
unsatisfied = []
|
| 685 |
for ext in explicit_extensions:
|
| 686 |
if ext not in used_extensions:
|
|
|
|
| 795 |
generated_images = []
|
| 796 |
deep_research_report = None
|
| 797 |
deep_research_sources = None
|
| 798 |
+
has_deep_research = False
|
| 799 |
|
| 800 |
if tool_results:
|
| 801 |
synthesis_prompt += "[Tool Execution Results]\n"
|
|
|
|
| 806 |
'title': result.get('message', 'Generated visualization'),
|
| 807 |
'filepath': result.get('filepath', '')
|
| 808 |
})
|
| 809 |
+
log(f"π Synthesis: Captured image from {tool_name}")
|
| 810 |
result_clean = dict(result)
|
| 811 |
result_clean.pop('image_base64', None)
|
| 812 |
synthesis_prompt += f"- {tool_name}: {result_clean.get('message', 'Chart created')}\n"
|
|
|
|
| 813 |
elif tool_name == 'conduct_deep_research' and isinstance(result, dict):
|
| 814 |
if result.get('success') and result.get('report'):
|
| 815 |
has_deep_research = True
|
|
|
|
| 821 |
result_str = str(result)[:500]
|
| 822 |
synthesis_prompt += f"- {tool_name}: {result_str}\n"
|
| 823 |
|
|
|
|
| 824 |
if has_deep_research:
|
| 825 |
synthesis_prompt += "\n\nProvide a brief summary of what was accomplished (2-3 sentences). DO NOT include the full research report - it will be displayed separately below your summary."
|
| 826 |
else:
|
|
|
|
| 829 |
config = types.GenerateContentConfig(
|
| 830 |
system_instruction="You are a synthesis specialist. Combine information from multiple sources into coherent, helpful responses. Be concise when deep research reports are available since they will be displayed separately.",
|
| 831 |
temperature=0.7,
|
| 832 |
+
max_output_tokens=2048
|
| 833 |
)
|
| 834 |
|
| 835 |
try:
|
|
|
|
| 850 |
if getattr(part, "text", None):
|
| 851 |
result_text += part.text
|
| 852 |
|
|
|
|
| 853 |
if has_deep_research and deep_research_report:
|
| 854 |
result_text += "\n\n---\n\n## π Complete Research Report\n\n"
|
| 855 |
result_text += deep_research_report
|
| 856 |
|
|
|
|
| 857 |
if deep_research_sources:
|
| 858 |
result_text += "\n\n### π Research Sources\n\n"
|
| 859 |
for idx, source in enumerate(deep_research_sources[:30], 1):
|
|
|
|
| 865 |
if len(deep_research_sources) > 30:
|
| 866 |
result_text += f"\n*...and {len(deep_research_sources) - 30} more sources*\n"
|
| 867 |
|
| 868 |
+
log(f"π¨ Synthesis returning {len(generated_images)} images")
|
| 869 |
return result_text, generated_images
|
| 870 |
except Exception as e:
|
| 871 |
log(f"β οΈ Synthesis error: {e}")
|
|
|
|
| 916 |
return None
|
| 917 |
|
| 918 |
|
|
|
|
| 919 |
EXTENSION_MANAGER = ExtensionManager()
|
| 920 |
CHAT_SESSIONS: Dict[str, Dict[str, Any]] = {}
|
| 921 |
|
|
|
|
| 965 |
if chat_history_msgs is None:
|
| 966 |
chat_history_msgs = []
|
| 967 |
|
|
|
|
| 968 |
file_parts = []
|
| 969 |
if uploaded_files:
|
| 970 |
log(f"π Processing {len(uploaded_files)} uploaded file(s)...")
|
|
|
|
| 980 |
|
| 981 |
assistant_base_index = len(chat_history_msgs)
|
| 982 |
|
|
|
|
| 983 |
if show_thoughts:
|
| 984 |
thought_index = assistant_base_index
|
| 985 |
chat_history_msgs.append({"role": "assistant", "content": "<em>π Thinking...</em>"})
|
|
|
|
| 993 |
yield chat_history_msgs
|
| 994 |
|
| 995 |
try:
|
|
|
|
| 996 |
proactive_msgs = EXTENSION_MANAGER.check_proactive_messages(api_key, enabled_extensions)
|
| 997 |
if proactive_msgs:
|
| 998 |
for msg in proactive_msgs:
|
|
|
|
| 1000 |
answer_index += 1
|
| 1001 |
yield chat_history_msgs
|
| 1002 |
|
|
|
|
| 1003 |
if enabled_extensions:
|
| 1004 |
log("π Using improved orchestrator")
|
| 1005 |
orchestrator = ImprovedOrchestrator(
|
|
|
|
| 1008 |
|
| 1009 |
budget = reasoning_budget(reasoning_level)
|
| 1010 |
|
|
|
|
| 1011 |
final_answer, tool_results, generated_images, thoughts, search_citations = \
|
| 1012 |
orchestrator.execute_with_planning(user_text, file_parts, budget)
|
| 1013 |
|
| 1014 |
+
# CRITICAL DEBUG: Log what we got back
|
| 1015 |
+
log(f"π¨ Orchestrator returned {len(generated_images)} images")
|
| 1016 |
+
for idx, img in enumerate(generated_images, 1):
|
| 1017 |
+
log(f" Image {idx}: {img.get('title', 'Untitled')} - base64 length: {len(img.get('base64', ''))}")
|
| 1018 |
+
|
| 1019 |
if thoughts and show_thoughts:
|
| 1020 |
chat_history_msgs[thought_index]["content"] = (
|
| 1021 |
f"<details open>"
|
|
|
|
| 1027 |
)
|
| 1028 |
yield chat_history_msgs
|
| 1029 |
|
|
|
|
| 1030 |
final_content = (
|
| 1031 |
f"<div><strong>π€ Response</strong>"
|
| 1032 |
f"<div style='white-space:pre-wrap;background:inherit;color:inherit;"
|
|
|
|
| 1034 |
f"{final_answer.strip()}</div></div>"
|
| 1035 |
)
|
| 1036 |
|
| 1037 |
+
# CRITICAL: Add generated images with debug logging
|
| 1038 |
+
log(f"π¨ Attempting to add {len(generated_images)} images to final content")
|
| 1039 |
if generated_images:
|
| 1040 |
+
for idx, img_data in enumerate(generated_images, 1):
|
| 1041 |
+
log(f" π Adding image {idx}: {img_data.get('title', 'Untitled')}")
|
| 1042 |
final_content += f"\n\n<div style='margin-top:16px;'>"
|
| 1043 |
+
final_content += f"<strong>π {img_data.get('title', 'Chart')}</strong><br/>"
|
| 1044 |
+
|
| 1045 |
+
base64_data = img_data.get('base64', '')
|
| 1046 |
+
if base64_data:
|
| 1047 |
+
log(f" β
Base64 data present: {len(base64_data)} chars")
|
| 1048 |
+
final_content += f"<img src='data:image/png;base64,{base64_data}' style='max-width:100%;border-radius:8px;box-shadow:0 2px 8px rgba(0,0,0,0.1);'/>"
|
| 1049 |
+
else:
|
| 1050 |
+
log(f" β WARNING: No base64 data for image {idx}!")
|
| 1051 |
+
|
| 1052 |
+
if img_data.get('filepath'):
|
| 1053 |
final_content += f"<br/><small style='color:#666;'>Saved to: {img_data['filepath']}</small>"
|
| 1054 |
final_content += "</div>"
|
| 1055 |
+
log(f" β
All {len(generated_images)} images added to HTML")
|
| 1056 |
+
else:
|
| 1057 |
+
log(f" β οΈ WARNING: generated_images list is EMPTY!")
|
| 1058 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1059 |
if search_citations:
|
| 1060 |
final_content += "\n\n" + search_citations
|
| 1061 |
|
|
|
|
| 1063 |
yield chat_history_msgs
|
| 1064 |
|
| 1065 |
else:
|
|
|
|
| 1066 |
log("πΊ Using simple streaming mode")
|
| 1067 |
|
| 1068 |
parts = []
|
|
|
|
| 1130 |
)
|
| 1131 |
yield chat_history_msgs
|
| 1132 |
|
|
|
|
| 1133 |
if last_chunk:
|
| 1134 |
citations = insert_citations_from_grounding(last_chunk.candidates)
|
| 1135 |
if citations:
|
|
|
|
| 1211 |
info="Display reasoning process before answers.",
|
| 1212 |
)
|
| 1213 |
|
|
|
|
| 1214 |
extension_checkboxes = build_extension_ui()
|
| 1215 |
|
| 1216 |
with gr.Column(scale=4):
|
|
|
|
| 1234 |
autofocus=True
|
| 1235 |
)
|
| 1236 |
|
|
|
|
| 1237 |
enabled_extensions_state = gr.State([])
|
| 1238 |
|
| 1239 |
def clear_box():
|
| 1240 |
return {"text": "", "files": []}
|
| 1241 |
|
| 1242 |
def handle_chat(api_key_input, chat_history_msgs, multimodal_dict, thinking_flag, reasoning_lvl, *extension_states):
|
|
|
|
| 1243 |
enabled = []
|
| 1244 |
for (ext_name, _), is_enabled in zip(extension_checkboxes, extension_states):
|
| 1245 |
if is_enabled:
|
|
|
|
| 1257 |
if not api_key_input or not enabled_exts:
|
| 1258 |
return chat_history
|
| 1259 |
|
|
|
|
| 1260 |
proactive_msgs = EXTENSION_MANAGER.check_proactive_messages(api_key_input, enabled_exts)
|
| 1261 |
|
| 1262 |
if proactive_msgs:
|
|
|
|
| 1269 |
|
| 1270 |
return chat_history
|
| 1271 |
|
|
|
|
| 1272 |
checkbox_components = [cb for _, cb in extension_checkboxes]
|
| 1273 |
|
|
|
|
| 1274 |
multimodal_msg.submit(
|
| 1275 |
fn=handle_chat,
|
| 1276 |
inputs=[api_key, chatbot, multimodal_msg, show_thoughts, reasoning_level] + checkbox_components,
|
|
|
|
| 1278 |
queue=True,
|
| 1279 |
).then(fn=clear_box, outputs=[multimodal_msg])
|
| 1280 |
|
|
|
|
| 1281 |
timer_check = gr.Timer(value=10, active=True)
|
| 1282 |
|
| 1283 |
def update_enabled_state(*extension_states):
|
|
|
|
| 1287 |
enabled.append(ext_name)
|
| 1288 |
return enabled
|
| 1289 |
|
|
|
|
| 1290 |
for _, cb in extension_checkboxes:
|
| 1291 |
cb.change(
|
| 1292 |
fn=update_enabled_state,
|
|
|
|
| 1294 |
outputs=[enabled_extensions_state]
|
| 1295 |
)
|
| 1296 |
|
|
|
|
| 1297 |
timer_check.tick(
|
| 1298 |
fn=check_timers,
|
| 1299 |
inputs=[api_key, chatbot, enabled_extensions_state],
|