Upload 41 files

agent_plugins/search_agent.py

@@ -751,26 +751,57 @@ class ResearchAgent:
             print(f"AOL Async Offload Fail: {e}")
         return []
 
+    def _clean_html(self, html_content: str) -> str:
+        """Removes script, style, navigation, and other non-content tags from HTML."""
+        try:
+            soup = BeautifulSoup(html_content, 'html.parser')
+            for element in soup(["script", "style", "iframe", "noscript", "header", "footer", "nav"]):
+                element.decompose()
+            text = soup.get_text(separator=' ')
+            lines = (line.strip() for line in text.splitlines())
+            chunks = (phrase.strip() for line in lines for phrase in line.split("  "))
+            clean_text = '\n'.join(chunk for chunk in chunks if chunk)
+            return clean_text
+        except Exception:
+            return ""
+
     async def _scrape_page_with_splash(self, url: str) -> str:
-        """Scrapes the full rendered HTML of a web page using a Splash server (BSD 3-Clause)."""
+        """Scrapes the full rendered HTML of a web page using a Splash server (BSD 3-Clause),
+        with an automated direct HTTP client fallback if Splash is offline."""
+        splash_enabled = os.getenv("SPLASH_ENABLED", "true").lower() == "true"
         splash_url = os.getenv("SPLASH_URL", "http://localhost:8050/render.html")
+        
+        # 1. Try Splash Scraper first
+        if splash_enabled:
+            try:
+                async with httpx.AsyncClient() as client:
+                    response = await client.get(
+                        splash_url,
+                        params={"url": url, "timeout": 12, "wait": 1.0},
+                        timeout=15.0
+                    )
+                    if response.status_code == 200:
+                        clean_content = self._clean_html(response.text)
+                        if clean_content and len(clean_content.strip()) > 100:
+                            return clean_content
+            except Exception as e:
+                print(f"[Splash Scrape Fail] Error crawling {url} via Splash: {e}. Trying direct HTTP fallback...")
+
+        # 2. Direct HTTP Fallback Scraper (non-JS render)
         try:
-            async with httpx.AsyncClient() as client:
-                response = await client.get(
-                    splash_url,
-                    params={"url": url, "timeout": 12, "wait": 1.0},
-                    timeout=15.0
-                )
+            headers = {
+                "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+                "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
+                "Accept-Language": "en-US,en;q=0.5"
+            }
+            async with httpx.AsyncClient(follow_redirects=True) as client:
+                response = await client.get(url, headers=headers, timeout=10.0)
                 if response.status_code == 200:
-                    soup = BeautifulSoup(response.text, 'html.parser')
-                    for script in soup(["script", "style"]):
-                        script.decompose()
-                    text = soup.get_text(separator=' ')
-                    lines = (line.strip() for line in text.splitlines())
-                    chunks = (phrase.strip() for line in lines for phrase in line.split("  "))
-                    clean_text = '\n'.join(chunk for chunk in chunks if chunk)
-                    return clean_text
-        except Exception as e:
-            print(f"[Splash Scrape Fail] Error crawling {url} via Splash: {e}")
+                    clean_content = self._clean_html(response.text)
+                    if clean_content and len(clean_content.strip()) > 100:
+                        return clean_content
+        except Exception as direct_err:
+            print(f"[Direct Scrape Fail] Error crawling {url} directly: {direct_err}")
+            
         return ""
 

main.py

@@ -114,6 +114,8 @@ class RealtimeIngestionCluster:
         asyncio.create_task(self._ingestion_loop())
 
     async def _ingestion_loop(self):
+        # Allow startup connection without network congestion
+        await asyncio.sleep(30)
         while self.is_running:
             try:
                 from agent_plugins.search_agent import ResearchAgent
@@ -187,6 +189,8 @@ async def startup_event():
         print("NEURAL BOOT: Initializing Memory and Databases...")
         try:
             Base.metadata.create_all(bind=engine)
+            # Pre-load ChromaDB and SentenceTransformer in startup background thread
+            _ = memory.client
             print("NEURAL BOOT: Systems Synchronized.")
         except Exception as e:
             print(f"BOOT ERROR: {e}")
@@ -200,6 +204,7 @@ import requests
 
 def map_model_for_backend(model: str) -> str:
     openrouter_key = os.environ.get("OPENROUTER_API_KEY")
+    custom_vision_url = os.environ.get("CUSTOM_VISION_API_URL")
     if openrouter_key:
         # OpenRouter Mapping
         if "llama-3.3-70b" in model or model == "llama-3.3-70b-versatile":
@@ -208,13 +213,25 @@ def map_model_for_backend(model: str) -> str:
             return "nousresearch/hermes-3-llama-3.1-70b"
         elif "scout" in model:
             return model  # Let it pass through since meta-llama/llama-4-scout-17b-16e-instruct is valid
+        elif "minicpm" in model:
+            if custom_vision_url:
+                return model  # Let it pass through to custom endpoints/OpenRouter
+            else:
+                # Fallback to a supported high-performance cloud vision model on OpenRouter
+                return "google/gemini-3.5-flash"
         elif "vision" in model:
             return "meta-llama/llama-3.2-11b-vision-instruct"
         return model
     else:
         # GroqCloud Mapping
+        if "minicpm" in model:
+            if custom_vision_url:
+                return model
+            else:
+                # Fallback to a supported vision model on Groq
+                return "meta-llama/llama-4-scout-17b-16e-instruct"
         if "scout" in model or "vision" in model:
-            return "llama-3.2-11b-vision-preview"
+            return "meta-llama/llama-4-scout-17b-16e-instruct"
         if "llama-3.1-8b" in model:
             return "llama-3.1-8b-instant"
         if "llama-3.3-70b" in model:
@@ -320,6 +337,7 @@ class AsyncOpenRouterClient:
                                 if not line:
                                     continue
                                 if line.startswith("data: "):
+                                    print(f"RAW OPENROUTER LINE: {line}")
                                     data_str = line[6:].strip()
                                     if data_str == "[DONE]":
                                         break
@@ -354,6 +372,18 @@ else:
     groq_client = GroqClient(api_key=groq_key) if groq_key else None
     async_groq_client = AsyncGroq(api_key=groq_key) if groq_key else None
 
+custom_vision_url = os.environ.get("CUSTOM_VISION_API_URL")
+custom_vision_key = os.environ.get("CUSTOM_VISION_API_KEY", "no_key_needed")
+custom_vision_client = None
+
+if custom_vision_url:
+    print(f"[SYSTEM] AURA local vision router: Active using endpoint {custom_vision_url}")
+    from openai import AsyncOpenAI
+    custom_vision_client = AsyncOpenAI(
+        base_url=custom_vision_url,
+        api_key=custom_vision_key
+    )
+
 def serialize_to_toon(data: Any, indent: int = 0) -> str:
     """
     Serializes standard dictionary/list objects to Token-Oriented Object Notation (TOON) / YAML-like format.
@@ -414,15 +444,18 @@ class NeuralMemory:
         self._client = None
         self._collection = None
         self._rag = None
+        self._lock = ThreadingLock()
 
     @property
     def client(self):
         if self._client is None:
-            print("NEURAL BOOT: Initializing ChromaDB Memory Vault...")
-            self._client = chromadb.PersistentClient(path=self.path)
-            self._collection = self._client.get_or_create_collection(name="aura_memory_vault")
-            from agent_plugins.rag_advanced import AdvancedRAG
-            self._rag = AdvancedRAG(self._collection)
+            with self._lock:
+                if self._client is None:
+                    print("NEURAL BOOT: Initializing ChromaDB Memory Vault...")
+                    self._client = chromadb.PersistentClient(path=self.path)
+                    self._collection = self._client.get_or_create_collection(name="aura_memory_vault")
+                    from agent_plugins.rag_advanced import AdvancedRAG
+                    self._rag = AdvancedRAG(self._collection)
         return self._client
 
     def retrieve_context(self, query, project_id=None, top_k=3):
@@ -728,38 +761,54 @@ class InferenceEngine:
             print(f"FORCE SEARCH OVERRIDE: Triggered web search for: '{search_query}'")
         # Search intent: main chat always eligible; overlay only in research mode
         elif search_strategy != "local-only" and (not overlay_mode or sandbox.get("assistant_mode") == "research"):
-            intent_prompt = f"""Identify if the user query requires real-time web search or current/temporal information (e.g., live games, scores, current events, weather, stock prices, recent news, software releases/updates).
-Queries NOT requiring search include general coding questions (e.g., how to use Flutter, explain React hooks, what is Python, how to print in C), general math, creative writing, or basic conversation.
+            # Check for temporal keywords override first to prevent LLM classifier from incorrectly skipping
+            p_lower = prompt.lower()
+            tech_skip = ["python", "javascript", "typescript", "java", "c#", "c++", "rust", "html", "css", "flutter", "dart", "react", "angular", "vue", "docker", "git", "sql", "code", "programming", "array", "object", "string", "loop", "regex"]
+            has_tech_skip = any(tk in p_lower for tk in tech_skip)
+            
+            temporal_kws = [
+                "current", "latest", "weather", "news", "score", "match", "today", "yesterday", "tomorrow", 
+                "tonight", "who is", "who won", "ipl", "stock", "price", "update", "now", "live", "results", 
+                "vs", "standing", "points", "ranking", "winner", "election", "chief minister", "prime minister", 
+                "president", "cm of", "pm of", "minister of", "cabinet", "governor of", "ceo of"
+            ]
+            
+            if any(tk in p_lower for tk in temporal_kws) and not has_tech_skip:
+                is_research_needed = True
+                print(f"TEMPORAL KEYWORD OVERRIDE: Triggered web search for: '{prompt}'")
+            else:
+                intent_prompt = f"""Identify if the user query requires real-time web search or current/temporal information (e.g., live games, scores, current events, weather, stock prices, recent news, software releases/updates).
+                Queries NOT requiring search include general coding questions (e.g., how to use Flutter, explain React hooks, what is Python, how to print in C), general math, creative writing, or basic conversation.
 
-User Query: "{prompt}"
+                User Query: "{prompt}"
 
-Respond with ONLY "SEARCH" or "NO_SEARCH" (nothing else, no explanation, no punctuation)."""
-            
-            try:
-                intent_res = await async_groq_client.chat.completions.create(
-                    model="llama-3.3-70b-versatile",
-                    messages=[{"role": "user", "content": intent_prompt}],
-                    max_tokens=5,
-                    temperature=0.0,
-                    timeout=4.0
-                )
-                expanded = intent_res.choices[0].message.content.strip().upper()
-                if "SEARCH" in expanded and "NO_SEARCH" not in expanded:
-                    is_research_needed = True
-                    print(f"NEURAL INTENT CLASSIFICATION: Detected search requirement.")
-                else:
-                    print(f"NEURAL INTENT CLASSIFICATION: Query is static.")
-            except Exception as ex:
-                print(f"LLM Intent Classification failed, falling back to keywords: {ex}")
-                # Fallback keyword-based check
-                is_research_needed = any(k in prompt.lower() for k in [
-                    "score", "news", "today", "weather", "match", "latest", 
-                    "who is", "what is", "how is", "price", "stock", "search", "update",
-                    "current", "now", "live", "results", "scheduled", "vs", "who won",
-                    "standing", "points", "ranking", "winner", "tomorrow", "tonight", "happening"
-                ])
-                if any(k in prompt.lower() for k in ["how to", "code", "function", "install", "react hooks", "flutter", "python"]):
-                    is_research_needed = False
+                Respond with ONLY "SEARCH" or "NO_SEARCH" (nothing else, no explanation, no punctuation)."""
+                
+                try:
+                    intent_res = await async_groq_client.chat.completions.create(
+                        model="llama-3.3-70b-versatile",
+                        messages=[{"role": "user", "content": intent_prompt}],
+                        max_tokens=5,
+                        temperature=0.0,
+                        timeout=4.0
+                    )
+                    expanded = intent_res.choices[0].message.content.strip().upper()
+                    if "SEARCH" in expanded and "NO_SEARCH" not in expanded:
+                        is_research_needed = True
+                        print(f"NEURAL INTENT CLASSIFICATION: Detected search requirement.")
+                    else:
+                        print(f"NEURAL INTENT CLASSIFICATION: Query is static.")
+                except Exception as ex:
+                    print(f"LLM Intent Classification failed, falling back to keywords: {ex}")
+                    # Fallback keyword-based check
+                    is_research_needed = any(k in prompt.lower() for k in [
+                        "score", "news", "today", "weather", "match", "latest", 
+                        "who is", "what is", "how is", "price", "stock", "search", "update",
+                        "current", "now", "live", "results", "scheduled", "vs", "who won",
+                        "standing", "points", "ranking", "winner", "tomorrow", "tonight", "happening"
+                    ])
+                    if any(k in prompt.lower() for k in ["how to", "code", "function", "install", "react hooks", "flutter", "python"]):
+                        is_research_needed = False
 
         # Conversational Query Expansion if needed
         if history and is_research_needed:
@@ -769,6 +818,7 @@ Respond with ONLY "SEARCH" or "NO_SEARCH" (nothing else, no explanation, no punc
                 try:
                     chat_history_str = "\n".join([f"{m.get('role', 'user').upper()}: {m.get('content', '')}" for m in filtered_history[-3:]])
                     expansion_prompt = f"""Given the following chat history and a follow-up query, generate an optimized single search query.
+Note: The current year is 2026. If the query asks for temporal or current events, ensure the search query targets 2026 or the present.
 CHAT HISTORY:
 {chat_history_str}
 
@@ -787,8 +837,19 @@ RESPONSE FORMAT: Output ONLY the optimized search query, nothing else. Do NOT in
                     print(f"NEURAL INTENT: Query expanded to '{search_query}'")
                 except Exception as ex:
                     print(f"Intent expansion failed: {ex}")
+                    search_query = prompt
             else:
                 search_query = prompt
+        else:
+            search_query = prompt
+
+        # Refine search query with current year if it has temporal intent and does not already contain the year
+        current_year = "2026"
+        temporal_kws = ["current", "latest", "news", "today", "weather", "match", "score", "ipl", "stock", "price", "who is", "who won", "election", "cm", "pm", "president", "now"]
+        p_lower = prompt.lower()
+        if is_research_needed and any(tk in p_lower for tk in temporal_kws) and current_year not in search_query:
+            search_query = f"{search_query} {current_year}"
+            print(f"TEMPORAL SEARCH QUERY REFINE: Expanded search query to: '{search_query}'")
 
         research_context = ""
         if is_research_needed:
@@ -823,7 +884,7 @@ RESPONSE FORMAT: Output ONLY the optimized search query, nothing else. Do NOT in
         if overlay_mode:
             memory_context = api_context
         else:
-            memory_context = memory.retrieve_context(prompt, project_id=project_id)
+            memory_context = await asyncio.to_thread(memory.retrieve_context, prompt, project_id)
             if api_context and "No matching free APIs" not in api_context:
                 memory_context += "\n\n" + api_context
 
@@ -1091,17 +1152,32 @@ LIVE DATA RULES:
             
             # Start direct streaming of final response
             try:
-                response_stream = await async_groq_client.chat.completions.create(
-                    model=map_model_for_backend(chosen_model),
-                    messages=messages,
-                    temperature=overlay_params.get("temperature", 0.7),
-                    max_tokens=overlay_params.get("max_tokens", 2048),
-                    stream=True
-                )
+                if base64_image_content and custom_vision_client:
+                    print(f"CUSTOM VISION ROUTING: Sending screenshot to self-hosted endpoint: {custom_vision_url}")
+                    response_stream = await custom_vision_client.chat.completions.create(
+                        model=chosen_model,
+                        messages=messages,
+                        temperature=overlay_params.get("temperature", 0.7),
+                        max_tokens=overlay_params.get("max_tokens", 2048),
+                        stream=True
+                    )
+                else:
+                    response_stream = await async_groq_client.chat.completions.create(
+                        model=map_model_for_backend(chosen_model),
+                        messages=messages,
+                        temperature=overlay_params.get("temperature", 0.7),
+                        max_tokens=overlay_params.get("max_tokens", 2048),
+                        stream=True
+                    )
                 
                 async for chunk in response_stream:
-                    if chunk.choices and chunk.choices[0].delta.content:
-                        yield ("content", chunk.choices[0].delta.content)
+                    if chunk.choices and len(chunk.choices) > 0:
+                        delta = chunk.choices[0].delta
+                        content = getattr(delta, "content", None)
+                        if not content and isinstance(delta, dict):
+                            content = delta.get("content")
+                        if content:
+                            yield ("content", content)
                         
             except Exception as e:
                 cerebras_key = os.environ.get("CEREBRAS_API_KEY")
@@ -1836,7 +1912,7 @@ async def api_chat_fallback(data: dict):
     try:
         if not incognito:
             chat_manager.save_message(user_id, conv_id, project_id, "user", prompt)
-            memory.store_fragment(f"User Question in {project_id}: {prompt}")
+            asyncio.create_task(asyncio.to_thread(memory.store_fragment, f"User Question in {project_id}: {prompt}", project_id))
 
         full_reply = ""
         async for chunk_type, content in inference_core.generate_stream(prompt, data.get("history", []), sandbox=sandbox):
@@ -1846,7 +1922,7 @@ async def api_chat_fallback(data: dict):
         if not incognito:
             chat_manager.save_message(user_id, conv_id, project_id, "assistant", full_reply)
             if len(full_reply) > 50:
-                memory.store_fragment(f"Aura Strategic Advice: {full_reply[:500]}...")
+                asyncio.create_task(asyncio.to_thread(memory.store_fragment, f"Aura Strategic Advice: {full_reply[:500]}...", project_id))
 
         return {"response": strip_robotic_preamble(full_reply)}
     except Exception as e:
@@ -1871,7 +1947,7 @@ async def secured_chat(websocket: WebSocket):
             
             if not incognito:
                 chat_manager.save_message(user_id, conv_id, project_id, "user", prompt)
-                memory.store_fragment(f"User Question in {project_id}: {prompt}")
+                asyncio.create_task(asyncio.to_thread(memory.store_fragment, f"User Question in {project_id}: {prompt}", project_id))
             
             full_reply = ""
             accumulated_thought = ""
@@ -1903,7 +1979,7 @@ async def secured_chat(websocket: WebSocket):
             if not incognito:
                 chat_manager.save_message(user_id, conv_id, project_id, "assistant", full_reply, thought=accumulated_thought)
                 if len(full_reply) > 50:
-                    memory.store_fragment(f"Aura Strategic Advice: {full_reply[:500]}...")
+                    asyncio.create_task(asyncio.to_thread(memory.store_fragment, f"Aura Strategic Advice: {full_reply[:500]}...", project_id))
                 
             await websocket.send_text(json.dumps({"done": True}))
 

overlay_engine.py

@@ -18,7 +18,7 @@ OVERLAY_CONVERSATION_ID = "aura_overlay"
 OVERLAY_SLM_MODEL = "llama-3.1-8b-instant"
 
 # Groq multimodal model (replaces decommissioned llama-3.2-11b-vision-preview)
-GROQ_VISION_MODEL = "meta-llama/llama-4-scout-17b-16e-instruct"
+GROQ_VISION_MODEL = os.environ.get("GROQ_VISION_MODEL", "openbmb/minicpm-v-2.6")
 GROQ_VISION_MAX_B64_BYTES = 3_500_000  # Groq limit is 4MB for base64 images
 
 ASSISTANT_MODES = ("quick", "tutor", "copilot", "research", "focus")
@@ -115,7 +115,8 @@ def get_overlay_inference_params(sandbox: dict, has_screenshot: bool = False) ->
     cfg = MODE_RULES.get(mode, MODE_RULES["copilot"])
     max_tokens = cfg["max_tokens"]
     if has_screenshot:
-        max_tokens = min(max_tokens + 120, 550)
+        # Increase token limit significantly to prevent truncation due to reasoning tokens
+        max_tokens = max(max_tokens, 2048)
     return {
         "max_tokens": max_tokens,
         "temperature": cfg["temperature"],

response_rules.py

@@ -11,6 +11,7 @@ When using realtime / live web data:
 - Silently use current date and time internally.
 - Silently use search results internally.
 - Answer the user directly in natural, conversational language.
+- CRITICAL: Always prioritize the most recent information (e.g., year 2026 facts over older 2021-2025 facts) provided in the LIVE DATA block. Do not rely on your static knowledge cutoff if it contradicts the live web data.
 
 DO NOT:
 - Mention the current date or year unless the user explicitly asked for it.