Update backend.py

backend.py

@@ -1,4 +1,3 @@
-# backend.py — FINAL, HIERARCHICAL MULTI-AGENT SYSTEM v3.4
 import sqlite3
 import os
 import json
@@ -7,7 +6,12 @@ import concurrent.futures
 import traceback
 import zipfile
 import hashlib
-from typing import Optional, Dict, Any
+from typing import Optional, Dict, Any, List
+
+# --- New dependencies for Web Scraping ---
+import requests
+from bs4 import BeautifulSoup
+# ---
 
 import torch
 import psutil
@@ -86,7 +90,7 @@ ROLE_PROMPTS = {
     "ceo": """You are the CEO. Your job is to create a high-level plan and delegate the first task to your Manager.
     Analyze the user's request and provide a list of files to be created.
     Then, create the very first task for the Manager to execute.
-    Respond ONLY with a JSON object with keys "plan" and "initial_task".
+    Respond ONLY with a JSON object.
     Example: {"plan": {"files": ["app.py", "backend.py", "tests/test_app.py"]}, "initial_task": "Start by writing the backend.py file with database functions."}""",
     
     "manager": """You are the Manager. You receive tasks and questions. Your job is to:
@@ -94,39 +98,124 @@ ROLE_PROMPTS = {
     2. After a worker completes a task, assign the NEXT logical task (code another file OR write tests)
     3. Only finish when ALL code and tests are complete
     4. If unsure, ask the CEO
-    Available actions: "delegate_to_worker", "answer_worker", "ask_ceo", "finish_project"
-    Respond ONLY with a JSON object with "action" and "data".
-    Examples:
-    - {"action": "delegate_to_worker", "data": {"worker_type": "coder", "task": "Write the full Python code for app.py"}}
-    - {"action": "delegate_to_worker", "data": {"worker_type": "tester", "task": "Write unit tests for backend.py"}}
-    - {"action": "finish_project", "data": {}}""",
+    Available actions: `delegate_coder`, `delegate_tester`, `answer_worker`, `ask_ceo`, `finish_project`
+    Respond ONLY with a single tool call on a new line.
+    
+    Example Actions:
+    
+    To delegate a task to a Coder:
+    `delegate_coder("Write the full Python code for app.py")`
+    
+    To delegate a task to a Tester:
+    `delegate_tester("Write unit tests for backend.py")`
+    
+    To answer a worker's question:
+    `answer_worker("The main file is main.py, and it should use Pygame.")`
+    
+    To ask a question to the CEO (e.g., if you are confused):
+    `ask_ceo("Should we use a relational database or a NoSQL database for this project?")`
+
+    To finish the entire project:
+    `finish_project()`""",
 
     "worker_coder": """You are a Coder. Your only job is to write code based on the Manager's task.
-    After completing a task, report "task_complete" - DO NOT decide what to do next.
+    If you need more information or documentation to complete a task, you can use the `scrape_web` tool.
+    After completing a task, report back using the `task_complete` tool. DO NOT decide what to do next.
     The Manager will assign the next file to code.
-    Available actions: "write_code", "read_file", "ask_manager", "task_complete"
-    Respond ONLY with a JSON object with "action" and "data".
-    Example: {"action": "write_code", "data": {"file_path": "backend.py", "code": "import sqlite3..."}}""",
+    Available actions: `scrape_web`, `write_code`, `read_file`, `ask_manager`, `task_complete`
+    Respond ONLY with a single tool call on a new line.
+    
+    Example Actions:
+    
+    To scrape a website for information:
+    `scrape_web("https://docs.python.org/3/library/sqlite3.html")`
+    
+    To write code to a file. The second argument is a raw string of the code:
+    `write_code("backend.py", "import sqlite3\\n\\ndef init_db():\\n    # ...")`
+    
+    To read the contents of an existing file:
+    `read_file("main.py")`
+    
+    To ask the Manager a question:
+    `ask_manager("What testing framework should I use for this project?")`
+
+    To report that you have finished your task:
+    `task_complete()`""",
 
     "worker_tester": """You are a Tester. Your only job is to write tests for existing code.
-    After completing test writing, report "task_complete".
+    If you need to look up testing libraries or best practices, use the `scrape_web` tool.
+    After completing test writing, report `task_complete`.
     The Manager will decide if more tests are needed or if the project can finish.
-    Available actions: "write_test", "read_file", "ask_manager", "task_complete"
-    Respond ONLY with a JSON object with "action" and "data".
-    Example: {"action": "write_test", "data": {"file_path": "tests/test_backend.py", "code": "import pytest..."}}"""
+    Available actions: `scrape_web`, `write_test`, `read_file`, `ask_manager`, `task_complete`
+    Respond ONLY with a single tool call on a new line.
+    
+    Example Actions:
+    
+    To scrape a website for information:
+    `scrape_web("https://docs.pytest.org/en/stable/")`
+    
+    To write tests to a file. The second argument is a raw string of the test code:
+    `write_test("tests/test_backend.py", "import pytest\\n\\ndef test_db_connection():\\n    # ...")`
+    
+    To read the contents of an existing file:
+    `read_file("app.py")`
+    
+    To ask the Manager a question:
+    `ask_manager("Are there any specific edge cases I should focus on for the user login tests?")`
+
+    To report that you have finished your task:
+    `task_complete()`"""
 }
+# --------------------------------- NEW PARSER ---------------------------------
+def _extract_tool_call(text: str) -> Optional[Dict[str, Any]]:
+    """
+    Parses a string for a tool-call like `tool_name("arg1", "arg2")` and extracts
+    the function name and its arguments.
+    """
+    text = text.strip().split("\n")[0] # Only consider the first line
+    
+    match = re.search(r'(\w+)\((.*)\)', text, re.DOTALL)
+    if not match:
+        return None
+    
+    tool_name = match.group(1)
+    args_str = match.group(2)
+    
+    # Simple regex to split arguments, handling quoted strings
+    # This is a basic approach and might fail on complex arguments
+    args = []
+    
+    # Handle single string arguments
+    if re.fullmatch(r'\s*"(.*?)"\s*', args_str, re.DOTALL) or re.fullmatch(r"\s*'(.*?)'\s*", args_str, re.DOTALL):
+        args.append(args_str.strip().strip("'\""))
+    else:
+        # Simple split by comma for multiple args
+        for arg in args_str.split(','):
+            args.append(arg.strip().strip("'\""))
+
+    # Convert known numerical args if needed
+    try:
+        if tool_name in ["delegate_coder", "delegate_tester"]:
+            args = [str(arg) for arg in args]
+    except (ValueError, IndexError):
+        return None # Return None if parsing fails
+
+    return {"tool_name": tool_name, "args": args}
 
 # ------------------------------ FILE SYSTEM & AI TOOLS ------------------------------
 def get_project_dir(user_id, project_id):
     path = os.path.join(PROJECT_ROOT, str(user_id), str(project_id)); os.makedirs(path, exist_ok=True); return path
+
 def create_file(project_dir, path, content):
     full_path = os.path.join(project_dir, path); os.makedirs(os.path.dirname(full_path), exist_ok=True)
     with open(full_path, 'w', encoding='utf-8') as f: f.write(content)
+
 def read_file(project_dir, path):
     full_path = os.path.join(project_dir, path)
     try:
         with open(full_path, 'r', encoding='utf-8') as f: return f.read()
     except FileNotFoundError: return "Error: File not found."
+
 def zip_project(project_dir, project_id):
     zip_filename = f"project_{project_id}.zip"; zip_path = os.path.join(os.path.dirname(project_dir), zip_filename)
     with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zf:
@@ -134,37 +223,25 @@ def zip_project(project_dir, project_id):
             for file in files: zf.write(os.path.join(root, file), os.path.relpath(os.path.join(root, file), project_dir))
     return zip_path
 
-def _extract_code(text: str) -> Optional[str]:
-    match = re.search(r"```(?:\w*\n)?([\s\S]*?)```", text)
-    if match: return match.group(1).strip()
-    return text.strip()
-
-# --- THIS IS THE FINAL FIX ---
-def _extract_json(text: str) -> Optional[Dict[str, Any]]:
-    """More robust JSON extractor that handles conversational text after the JSON block."""
-    # First, try to find a markdown block, as it's the most reliable
-    match = re.search(r"```json\s*([\s\S]*?)\s*```", text)
-    if match:
-        json_str = match.group(1)
-        try:
-            return json.loads(json_str)
-        except json.JSONDecodeError:
-            print(f"Failed to decode JSON from markdown block: {json_str[:200]}...")
-            # Fall through to the next method if markdown parsing fails
-            pass
-
-    # If no markdown, find the first '{' and the last '}' to isolate the JSON
+def scrape_web(url: str) -> str:
+    """Scrapes a URL and returns the clean text content, limited to 5000 chars."""
     try:
-        start_index = text.find('{')
-        end_index = text.rfind('}')
-        if start_index != -1 and end_index != -1 and end_index > start_index:
-            json_str = text[start_index : end_index + 1]
-            return json.loads(json_str)
-    except (json.JSONDecodeError, IndexError):
-        # This will be the final failure point if no valid JSON is found
-        print(f"Failed to decode JSON from the entire response: {text[:200]}...")
-        return None
-    return None # Return None if no JSON object is found at all
+        headers = {
+            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
+        }
+        response = requests.get(url, headers=headers, timeout=15)
+        response.raise_for_status()
+        soup = BeautifulSoup(response.content, 'html.parser')
+        for script_or_style in soup(["script", "style"]): script_or_style.decompose()
+        text = soup.get_text()
+        lines = (line.strip() for line in text.splitlines())
+        chunks = (phrase.strip() for line in lines for phrase in line.split("  "))
+        text = '\n'.join(chunk for chunk in chunks if chunk)
+        return text[:5000]
+    except requests.exceptions.RequestException as e:
+        return f"Error: Could not retrieve content from URL. {e}"
+    except Exception as e:
+        return f"Error: An unexpected error occurred during web scraping. {e}"
 
 def generate_with_model(role: str, prompt: str) -> str:
     try:
@@ -175,7 +252,7 @@ def generate_with_model(role: str, prompt: str) -> str:
         outputs = model.generate(**inputs, max_new_tokens=3072, pad_token_id=tokenizer.eos_token_id, use_cache=True)
         return tokenizer.decode(outputs[0][len(inputs.input_ids[0]):], skip_special_tokens=True).strip()
     except Exception as e:
-        print(f"Error during model generation for role {role}: {e}"); return f'{{"action": "error", "data": "{e}"}}'
+        print(f"Error during model generation for role {role}: {e}"); return f"error({e})"
 
 # ------------------------------ HIERARCHICAL AGENT ORCHESTRATOR ------------------------------
 def run_agent_chain(project_id, user_id, initial_prompt):
@@ -197,12 +274,12 @@ def run_agent_chain(project_id, user_id, initial_prompt):
             "current_task": "Start the project.",
             "last_action_result": None,
             "turn": 0,
-            "max_turns": 25, # Safety brake
+            "max_turns": 30,
         }
 
         log_step("ORCHESTRATOR", "Briefing the CEO with the user's request.")
         ceo_response_text = generate_with_model("ceo", initial_prompt)
-        ceo_action = _extract_json(ceo_response_text)
+        ceo_action = json.loads(ceo_response_text) # CEO still uses JSON for its specific output
         if not ceo_action or "plan" not in ceo_action: raise ValueError("CEO failed to provide an initial plan.")
         
         project_context["file_structure"] = ceo_action["plan"].get("files", [])
@@ -223,55 +300,90 @@ def run_agent_chain(project_id, user_id, initial_prompt):
             """
             
             response_text = generate_with_model(next_agent, prompt)
-            agent_response = _extract_json(response_text)
+            tool_call = _extract_tool_call(response_text)
             
-            if not agent_response or "action" not in agent_response:
+            if not tool_call:
                 project_context["last_action_result"] = f"Error: Agent {next_agent} returned invalid response. Retrying."
                 log_step("ORCHESTRATOR", f"Invalid response from {next_agent}. Retrying task.", response_text)
                 project_context["turn"] += 1
                 continue
 
-            action = agent_response["action"]
-            data = agent_response.get("data", {})
-            thought = data.get("thought", f"Decided to perform action: `{action}`")
-
-            if action == "delegate_to_worker":
-                worker_type = data.get("worker_type", "coder")
-                next_agent = f"worker_{worker_type}"
-                project_context["current_task"] = data.get("task")
-                project_context["last_action_result"] = f"Task delegated to {next_agent}."
-                log_step("MANAGER", thought, project_context['last_action_result'])
-
-            elif action == "write_code" or action == "write_test":
-                file_path = data.get("file_path"); raw_code = data.get("code")
-                if not file_path or not raw_code:
+            action = tool_call.get("tool_name")
+            args = tool_call.get("args", [])
+            
+            log_step(next_agent.replace("_", " "), f"Decided to perform action: `{action}` with args: `{args}`")
+
+            if action == "delegate_coder":
+                if not args:
+                    project_context["last_action_result"] = "Error: Missing task for delegate_coder."
+                    next_agent = "manager"
+                    continue
+                next_agent = "worker_coder"
+                project_context["current_task"] = args[0]
+                project_context["last_action_result"] = f"Task delegated to {next_agent.replace('_', ' ')}."
+
+            elif action == "delegate_tester":
+                if not args:
+                    project_context["last_action_result"] = "Error: Missing task for delegate_tester."
+                    next_agent = "manager"
+                    continue
+                next_agent = "worker_tester"
+                project_context["current_task"] = args[0]
+                project_context["last_action_result"] = f"Task delegated to {next_agent.replace('_', ' ')}."
+                
+            elif action == "write_code":
+                if len(args) < 2:
                     project_context["last_action_result"] = "Error: Missing file_path or code in data."
                 else:
-                    code = _extract_code(raw_code)
+                    file_path = args[0]; code = args[1]
                     create_file(project_dir, file_path, code)
                     project_context["last_action_result"] = f"Successfully wrote code to {file_path}."
-                log_step(next_agent.replace("_", " "), thought, project_context['last_action_result'])
+                next_agent = "manager"
+                project_context["current_task"] = f"Worker finished task. What is the next high-level step?"
+
+            elif action == "write_test":
+                if len(args) < 2:
+                    project_context["last_action_result"] = "Error: Missing file_path or code in data."
+                else:
+                    file_path = args[0]; code = args[1]
+                    create_file(project_dir, file_path, code)
+                    project_context["last_action_result"] = f"Successfully wrote tests to {file_path}."
+                next_agent = "manager"
+                project_context["current_task"] = f"Worker finished task. What is the next high-level step?"
+
+            elif action == "scrape_web":
+                if not args:
+                    project_context["last_action_result"] = "Error: Missing URL for scrape_web action."
+                else:
+                    url = args[0]
+                    log_step(next_agent.replace("_", " "), f"Scraping {url}...")
+                    scraped_content = scrape_web(url)
+                    project_context["last_action_result"] = f"Scraping result from {url}:\n\n{scraped_content}"
+                continue # Do not increment turn or change agent yet
 
             elif action == "ask_manager":
+                if not args:
+                    project_context["last_action_result"] = "Error: Missing question for ask_manager."
+                else:
+                    project_context["last_action_result"] = f"A worker has a question."
+                    project_context["current_task"] = args[0]
                 next_agent = "manager"
-                project_context["current_task"] = data.get("question")
-                project_context["last_action_result"] = f"A worker has a question."
-                log_step(next_agent.replace("_", " "), thought, f"Asking Manager: {project_context['current_task']}")
-            
+
             elif action == "answer_worker":
-                next_agent = "manager" 
-                project_context["current_task"] = "The worker's question has been answered. What is the next task for a worker?"
-                project_context["last_action_result"] = f"Manager answered: {data.get('answer')}"
-                log_step("MANAGER", thought, project_context['last_action_result'])
+                if not args:
+                    project_context["last_action_result"] = "Error: Missing answer for answer_worker."
+                else:
+                    project_context["last_action_result"] = f"Manager answered: {args[0]}"
+                    project_context["current_task"] = "The worker's question has been answered. What is the next task for a worker?"
+                next_agent = "manager"
 
             elif action == "task_complete":
-                next_agent = "manager"
-                project_context["current_task"] = f"Worker finished task. What is the next high-level step? (e.g., code next file, write tests, or finish project)"
                 project_context["last_action_result"] = "Worker reported task is complete."
-                log_step(next_agent.replace("_", " "), thought, project_context['last_action_result'])
+                project_context["current_task"] = f"Worker finished task. What is the next high-level step?"
+                next_agent = "manager"
 
             elif action == "finish_project":
-                log_step("MANAGER", thought, "Project is ready for final packaging.")
+                log_step("MANAGER", "Project is ready for final packaging.")
                 break
 
             else: