diff --git a/.gitattributes b/.gitattributes deleted file mode 100644 index c7d9f3332a950355d5a77d85000f05e6f45435ea..0000000000000000000000000000000000000000 --- a/.gitattributes +++ /dev/null @@ -1,34 +0,0 @@ -*.7z filter=lfs diff=lfs merge=lfs -text -*.arrow filter=lfs diff=lfs merge=lfs -text -*.bin filter=lfs diff=lfs merge=lfs -text -*.bz2 filter=lfs diff=lfs merge=lfs -text -*.ckpt filter=lfs diff=lfs merge=lfs -text -*.ftz filter=lfs diff=lfs merge=lfs -text -*.gz filter=lfs diff=lfs merge=lfs -text -*.h5 filter=lfs diff=lfs merge=lfs -text -*.joblib filter=lfs diff=lfs merge=lfs -text -*.lfs.* filter=lfs diff=lfs merge=lfs -text -*.mlmodel filter=lfs diff=lfs merge=lfs -text -*.model filter=lfs diff=lfs merge=lfs -text -*.msgpack filter=lfs diff=lfs merge=lfs -text -*.npy filter=lfs diff=lfs merge=lfs -text -*.npz filter=lfs diff=lfs merge=lfs -text -*.onnx filter=lfs diff=lfs merge=lfs -text -*.ot filter=lfs diff=lfs merge=lfs -text -*.parquet filter=lfs diff=lfs merge=lfs -text -*.pb filter=lfs diff=lfs merge=lfs -text -*.pickle filter=lfs diff=lfs merge=lfs -text -*.pkl filter=lfs diff=lfs merge=lfs -text -*.pt filter=lfs diff=lfs merge=lfs -text -*.pth filter=lfs diff=lfs merge=lfs -text -*.rar filter=lfs diff=lfs merge=lfs -text -*.safetensors filter=lfs diff=lfs merge=lfs -text -saved_model/**/* filter=lfs diff=lfs merge=lfs -text -*.tar.* filter=lfs diff=lfs merge=lfs -text -*.tflite filter=lfs diff=lfs merge=lfs -text -*.tgz filter=lfs diff=lfs merge=lfs -text -*.wasm filter=lfs diff=lfs merge=lfs -text -*.xz filter=lfs diff=lfs merge=lfs -text -*.zip filter=lfs diff=lfs merge=lfs -text -*.zst filter=lfs diff=lfs merge=lfs -text -*tfevents* filter=lfs diff=lfs merge=lfs -text diff --git a/Dockerfile b/Dockerfile index f5561dea4036383209950c2b0d38b8c6724e2f12..c1c88d894a4559758feb5f87f1b540a89324ff8f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,141 +1,48 @@ -FROM nvidia/cuda:11.3.1-base-ubuntu20.04 +FROM python:3.11-slim -ENV DEBIAN_FRONTEND=noninteractive \ - TZ=Aisa/Shanghai \ - LC_CTYPE=C.UTF-8 \ - LANG=C.UTF-8 +# HuggingFace Spaces Dockerfile +# Compatible with free CPU tier -# Remove any third-party apt sources to avoid issues with expiring keys. -# Install some basic utilities -RUN rm -f /etc/apt/sources.list.d/*.list && \ - apt-get update && apt-get install -y \ - curl \ - ca-certificates \ - sudo \ - git \ - git-lfs \ - zip \ - unzip \ - htop \ - bzip2 \ - libx11-6 \ - nginx \ - vim \ - lsof \ - telnet \ - wget \ - build-essential \ - libsndfile-dev \ - software-properties-common \ - && rm -rf /var/lib/apt/lists/* - -ARG BUILD_DATE -ARG VERSION -ARG CODE_RELEASE -RUN \ - echo "**** install openvscode-server runtime dependencies ****" && \ - apt-get update && \ - apt-get install -y \ - jq \ - libatomic1 \ - nano \ - net-tools \ - netcat && \ - echo "**** install openvscode-server ****" && \ - if [ -z ${CODE_RELEASE+x} ]; then \ - CODE_RELEASE=$(curl -sX GET "https://api.github.com/repos/gitpod-io/openvscode-server/releases/latest" \ - | awk '/tag_name/{print $4;exit}' FS='[""]' \ - | sed 's|^openvscode-server-v||'); \ - fi && \ - mkdir -p /app/openvscode-server && \ - curl -o \ - /tmp/openvscode-server.tar.gz -L \ - "https://github.com/gitpod-io/openvscode-server/releases/download/openvscode-server-v${CODE_RELEASE}/openvscode-server-v${CODE_RELEASE}-linux-x64.tar.gz" && \ - tar xf \ - /tmp/openvscode-server.tar.gz -C \ - /app/openvscode-server/ --strip-components=1 && \ - echo "**** clean up ****" && \ - apt-get clean && \ - rm -rf \ - /tmp/* \ - /var/lib/apt/lists/* \ - /var/tmp/* -COPY root/ / - -RUN add-apt-repository ppa:flexiondotorg/nvtop && \ - apt-get upgrade -y && \ - apt-get install -y --no-install-recommends nvtop - -RUN curl -sL https://deb.nodesource.com/setup_18.x | bash - && \ - apt-get install -y nodejs && \ - npm install -g configurable-http-proxy - -# Create a working directory WORKDIR /app -# Create a non-root user and switch to it -RUN adduser --disabled-password --gecos '' --shell /bin/bash user \ - && chown -R user:user /app -RUN echo "user ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/90-user -USER user - -# All users can use /home/user as their home directory -ENV HOME=/home/user -RUN mkdir $HOME/.cache $HOME/.config \ - && chmod -R 777 $HOME - -# Set up the Conda environment -ENV CONDA_AUTO_UPDATE_CONDA=false \ - PATH=$HOME/miniconda/bin:$PATH -RUN curl -sLo ~/miniconda.sh https://repo.continuum.io/miniconda/Miniconda3-py310_23.5.2-0-Linux-x86_64.sh \ - && chmod +x ~/miniconda.sh \ - && ~/miniconda.sh -b -p ~/miniconda \ - && rm ~/miniconda.sh \ - && conda clean -ya - -WORKDIR $HOME/app - -####################################### -# Start root user section -####################################### - -USER root - -# User Debian packages -## Security warning : Potential user code executed as root (build time) -RUN --mount=target=/root/packages.txt,source=packages.txt \ - apt-get update && \ - xargs -r -a /root/packages.txt apt-get install -y --no-install-recommends \ +# System deps +RUN apt-get update && apt-get install -y \ + git curl build-essential \ && rm -rf /var/lib/apt/lists/* -RUN --mount=target=/root/on_startup.sh,source=on_startup.sh,readwrite \ - bash /root/on_startup.sh - -####################################### -# End root user section -####################################### +# Python deps +COPY requirements.txt . +RUN pip install --no-cache-dir --upgrade pip && \ + pip install --no-cache-dir -r requirements.txt -USER user +# App code +COPY . . -# Python packages -RUN --mount=target=requirements.txt,source=requirements.txt \ - pip install --no-cache-dir --upgrade -r requirements.txt +# Setup dirs +RUN mkdir -p /tmp/workspace /tmp/repos /tmp/devin_data -# Copy the current directory contents into the container at $HOME/app setting the owner to the user -COPY --chown=user . $HOME/app +# HF runs as uid 1000 +RUN useradd -m -u 1000 user 2>/dev/null || true +RUN chown -R 1000:1000 /app /tmp/workspace /tmp/repos /tmp/devin_data -WORKDIR $HOME/app +USER 1000 -RUN chmod +x start_server.sh +EXPOSE 7860 -ENV PYTHONUNBUFFERED=1 \ - GRADIO_ALLOW_FLAGGING=never \ - GRADIO_NUM_PORTS=1 \ - GRADIO_SERVER_NAME=0.0.0.0 \ - GRADIO_THEME=huggingface \ - SYSTEM=spaces \ - SHELL=/bin/bash +ENV PORT=7860 +ENV HOST=0.0.0.0 +ENV DB_PATH=/tmp/devin_agent.db +ENV PYTHONUNBUFFERED=1 +ENV PYTHONDONTWRITEBYTECODE=1 -EXPOSE 7860 3000 +# Health check +HEALTHCHECK --interval=30s --timeout=10s --start-period=10s --retries=3 \ + CMD curl -f http://localhost:7860/api/v1/health || exit 1 -CMD ["./start_server.sh"] \ No newline at end of file +CMD ["uvicorn", "main:app", \ + "--host", "0.0.0.0", \ + "--port", "7860", \ + "--workers", "1", \ + "--loop", "asyncio", \ + "--timeout-keep-alive", "75", \ + "--log-level", "info"] diff --git a/README.md b/README.md index a14644b7ff4d18b59edd6a68565cc328f221f7af..8d5b15bc9685fe2c43f49bfbda07c63f91613ef3 100644 --- a/README.md +++ b/README.md @@ -1,24 +1,58 @@ --- -title: Visual Studio Code -emoji: πŸ’»πŸ³ -colorFrom: red -colorTo: blue +title: Devin Agent Platform +emoji: πŸ€– +colorFrom: blue +colorTo: purple sdk: docker -pinned: false -tags: -- vscode -duplicated_from: SpacesExamples/vscode +app_port: 7860 +pinned: true +license: mit +short_description: Production-grade autonomous AI engineering platform --- -Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference -# Autonomous Coding System (Upgraded) +# πŸ€– Devin Agent Platform v2.0 -This system has been upgraded with: -1. **Persistent Memory Engine**: SQLite-backed storage for goals, plans, and execution history. -2. **Full GitHub Automation**: Complete control over repositories, branches, commits, and PRs. -3. **Tool Orchestration Brain**: Autonomous planning and tool routing for complex tasks. +> **Manus/Devin-style Autonomous AI Engineering Platform** +> Real-time WebSocket streaming Β· Autonomous GitHub operations Β· Persistent memory -## New API Endpoints -- `/api/v1/goal`: Submit a high-level goal for autonomous execution. -- `/api/v1/memory/{project_id}`: Retrieve project history. -- `/api/v1/github/*`: Full suite of GitHub automation tools. +## ✨ Features + +- ⚑ **Real-time WebSocket streaming** β€” live token-by-token LLM output +- πŸ—ΊοΈ **Autonomous task planning** β€” goal β†’ plan β†’ execute automatically +- 🧠 **Persistent memory** β€” SQLite-backed conversation + project memory +- πŸ™ **GitHub automation** β€” clone, commit, push, PR, issues autonomously +- πŸ” **Self-healing** β€” auto-retry with exponential backoff +- πŸ“‘ **SSE fallback** β€” Server-Sent Events for streaming compatibility +- 🌐 **REST + WebSocket API** β€” full-featured backend + +## πŸ”Œ API Endpoints + +| Method | Endpoint | Description | +|--------|----------|-------------| +| POST | `/api/v1/tasks/create` | Create autonomous task | +| GET | `/api/v1/tasks/{id}` | Get task details | +| POST | `/api/v1/tasks/{id}/cancel` | Cancel task | +| POST | `/api/v1/tasks/{id}/retry` | Retry failed task | +| GET | `/api/v1/tasks/{id}/stream` | SSE task stream | +| POST | `/api/v1/chat` | Chat with agent | +| POST | `/api/v1/goal` | Submit high-level goal | +| POST | `/api/v1/plan` | Generate execution plan | +| WS | `/ws/tasks/{task_id}` | Live task WebSocket | +| WS | `/ws/logs` | Global log stream | +| WS | `/ws/chat/{session_id}` | Chat WebSocket | +| WS | `/ws/agent/status` | Agent status stream | + +## πŸ”‘ Environment Variables (HF Secrets) + +``` +OPENAI_API_KEY = sk-... (for real AI) +ANTHROPIC_API_KEY = sk-ant-... (alternative) +GITHUB_TOKEN = ghp_... (GitHub ops) +GITHUB_OWNER = your-username (GitHub ops) +``` + +## πŸš€ Quick Start + +Visit `/api/docs` for interactive Swagger UI. + +**Demo mode** works without any API keys β€” set `OPENAI_API_KEY` for real AI. diff --git a/__pycache__/main.cpython-312.pyc b/__pycache__/main.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0a6043f0154f0fc529cbf6cd350edd46c5c0a266 Binary files /dev/null and b/__pycache__/main.cpython-312.pyc differ diff --git a/ngpasswd b/api/__init__.py similarity index 100% rename from ngpasswd rename to api/__init__.py diff --git a/api/__pycache__/__init__.cpython-312.pyc b/api/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..56c0c66ef7122cd03c986d89e42750d26fcb0ab6 Binary files /dev/null and b/api/__pycache__/__init__.cpython-312.pyc differ diff --git a/api/__pycache__/websocket_manager.cpython-312.pyc b/api/__pycache__/websocket_manager.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..39fe470d2d47dfadd2d4e08f78c620aefad89ac6 Binary files /dev/null and b/api/__pycache__/websocket_manager.cpython-312.pyc differ diff --git a/root/etc/s6-overlay/s6-rc.d/init-config-end/dependencies.d/init-openvscode-server b/api/routes/__init__.py similarity index 100% rename from root/etc/s6-overlay/s6-rc.d/init-config-end/dependencies.d/init-openvscode-server rename to api/routes/__init__.py diff --git a/api/routes/__pycache__/__init__.cpython-312.pyc b/api/routes/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5ccc91e45f72dfcd838292572a03de91995f0383 Binary files /dev/null and b/api/routes/__pycache__/__init__.cpython-312.pyc differ diff --git a/api/routes/__pycache__/chat.cpython-312.pyc b/api/routes/__pycache__/chat.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..59bb359f9f22fbfe51c7b297cca6ac42c1c78254 Binary files /dev/null and b/api/routes/__pycache__/chat.cpython-312.pyc differ diff --git a/api/routes/__pycache__/github.cpython-312.pyc b/api/routes/__pycache__/github.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..32bc13856652ed46d33b9497e7325e31674d549a Binary files /dev/null and b/api/routes/__pycache__/github.cpython-312.pyc differ diff --git a/api/routes/__pycache__/health.cpython-312.pyc b/api/routes/__pycache__/health.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7b5fcc886a5431427c10b62efaba98c9c2c17db1 Binary files /dev/null and b/api/routes/__pycache__/health.cpython-312.pyc differ diff --git a/api/routes/__pycache__/memory.cpython-312.pyc b/api/routes/__pycache__/memory.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..82cf4b4d288bba84b1bed52730e51b20b2f738d1 Binary files /dev/null and b/api/routes/__pycache__/memory.cpython-312.pyc differ diff --git a/api/routes/__pycache__/tasks.cpython-312.pyc b/api/routes/__pycache__/tasks.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..93b690aa5eac9b4a354e0d9a727de72c5393c1f9 Binary files /dev/null and b/api/routes/__pycache__/tasks.cpython-312.pyc differ diff --git a/api/routes/chat.py b/api/routes/chat.py new file mode 100644 index 0000000000000000000000000000000000000000..9f46ee598bbdbb3aa29f17ed6ba8147054c9b397 --- /dev/null +++ b/api/routes/chat.py @@ -0,0 +1,214 @@ +""" +Chat + Goal API Routes β€” Real-time streaming responses +""" + +import asyncio +import json +import time +import uuid + +from fastapi import APIRouter, HTTPException, Request +from fastapi.responses import StreamingResponse + +from core.models import ChatRequest, GoalRequest, TaskCreateRequest +from memory.db import save_memory, get_history + +router = APIRouter() + + +def get_engine(request: Request): + return request.app.state.task_engine + + +def get_ws(request: Request): + return request.app.state.ws_manager + + +# ─── Chat (REST + SSE streaming) ─────────────────────────────────────────────── + +@router.post("/chat", summary="Chat with the agent") +async def chat(req: ChatRequest, request: Request): + from core.agent import AgentCore + ws = get_ws(request) + agent = AgentCore(ws) + + messages = [{"role": m.role, "content": m.content} for m in req.messages] + + if req.stream: + async def stream_gen(): + async def _run(): + result = await agent.llm_stream( + messages=messages, + session_id=req.session_id, + model=req.model, + temperature=req.temperature, + max_tokens=req.max_tokens, + ) + await save_memory( + content=result, + memory_type="conversation", + session_id=req.session_id, + project_id=req.project_id, + key="assistant", + ) + # Save user message too + user_msg = next((m["content"] for m in reversed(messages) if m["role"] == "user"), "") + await save_memory( + content=user_msg, + memory_type="conversation", + session_id=req.session_id, + project_id=req.project_id, + key="user", + ) + return result + + room_buffer = [] + original_emit_chat = ws.emit_chat + async def capture_emit(sid, etype, data): + if etype == "llm_chunk": + chunk = data.get("chunk", "") + room_buffer.append(chunk) + yield_data = json.dumps({"type": etype, "data": data, "session_id": sid}) + return yield_data + return None + + # Stream tokens directly + full = "" + from core.agent import AgentCore as _A + import httpx + import os + OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY", "") + ANTHROPIC_API_KEY = os.environ.get("ANTHROPIC_API_KEY", "") + + if OPENAI_API_KEY: + headers = { + "Authorization": f"Bearer {OPENAI_API_KEY}", + "Content-Type": "application/json", + } + payload = { + "model": req.model, + "messages": messages, + "stream": True, + "temperature": req.temperature, + "max_tokens": req.max_tokens, + } + from core.agent import OPENAI_BASE_URL + async with httpx.AsyncClient(timeout=120) as client: + async with client.stream("POST", f"{OPENAI_BASE_URL}/chat/completions", + headers=headers, json=payload) as resp: + async for line in resp.aiter_lines(): + if not line.startswith("data:"): + continue + chunk_str = line[6:].strip() + if chunk_str == "[DONE]": + break + try: + data = json.loads(chunk_str) + delta = data["choices"][0]["delta"].get("content", "") + if delta: + full += delta + yield f"data: {json.dumps({'type': 'llm_chunk', 'data': {'chunk': delta}, 'session_id': req.session_id})}\n\n" + except Exception: + pass + else: + # Demo streaming + demo = ( + f"Hello! I'm your Devin-style AI Agent. I received: '{req.messages[-1].content[:80]}'. " + f"Set OPENAI_API_KEY or ANTHROPIC_API_KEY for real AI responses. " + f"I support real-time streaming, task planning, GitHub automation, and more!" + ) + for word in demo.split(): + chunk = word + " " + full += chunk + await asyncio.sleep(0.04) + yield f"data: {json.dumps({'type': 'llm_chunk', 'data': {'chunk': chunk}, 'session_id': req.session_id})}\n\n" + + yield f"data: {json.dumps({'type': 'stream_end', 'data': {'full_response': full}, 'session_id': req.session_id})}\n\n" + + return StreamingResponse( + stream_gen(), + media_type="text/event-stream", + headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}, + ) + else: + # Non-streaming + agent = AgentCore(get_ws(request)) + result = await agent.llm_stream(messages, session_id=req.session_id) + return { + "response": result, + "session_id": req.session_id, + "model": req.model, + "timestamp": time.time(), + } + + +@router.post("/chat/stream", summary="Explicit streaming chat endpoint") +async def chat_stream(req: ChatRequest, request: Request): + req.stream = True + return await chat(req, request) + + +# ─── Goal API (create task from goal) ───────────────────────────────────────── + +@router.post("/goal", summary="Submit a high-level goal to the agent") +async def submit_goal(req: GoalRequest, request: Request): + engine = get_engine(request) + task_req = TaskCreateRequest( + goal=req.goal, + session_id=req.session_id, + project_id=req.project_id, + stream=req.stream, + metadata={"source": "goal_api", "github_repo": req.github_repo}, + ) + task_id = await engine.submit(task_req) + return { + "task_id": task_id, + "goal": req.goal, + "status": "queued", + "session_id": req.session_id, + "ws_url": f"/ws/tasks/{task_id}", + "stream_url": f"/api/v1/tasks/{task_id}/stream", + } + + +@router.post("/goal/stream", summary="Submit goal with SSE streaming response") +async def submit_goal_stream(req: GoalRequest, request: Request): + req.stream = True + return await submit_goal(req, request) + + +# ─── Execute (direct tool execution) ────────────────────────────────────────── + +@router.post("/execute", summary="Execute a tool directly") +async def execute( + tool: str, + task: str, + request: Request, + session_id: str = "", +): + from tools.executor import ToolExecutor + ws = get_ws(request) + executor = ToolExecutor(ws) + result = await executor.run( + tool=tool, + task=task, + session_id=session_id, + ) + return {"tool": tool, "task": task, "result": result, "session_id": session_id} + + +# ─── Plan (generate plan without executing) ─────────────────────────────────── + +@router.post("/plan", summary="Generate execution plan for a goal") +async def generate_plan(req: GoalRequest, request: Request): + from core.agent import AgentCore + ws = get_ws(request) + agent = AgentCore(ws) + task_id = f"plan_{uuid.uuid4().hex[:8]}" + plan = await agent.plan(goal=req.goal, task_id=task_id, session_id=req.session_id) + return { + "goal": req.goal, + "plan": plan.model_dump(), + "session_id": req.session_id, + "task_id": task_id, + } diff --git a/api/routes/github.py b/api/routes/github.py new file mode 100644 index 0000000000000000000000000000000000000000..d6dc31c778141f8b1633e2ea90254827770b17d3 --- /dev/null +++ b/api/routes/github.py @@ -0,0 +1,336 @@ +""" +GitHub Autonomous Engineering API Routes +Clone, commit, push, PR, issues β€” all autonomous +""" + +import os +import time +import asyncio +import tempfile +import shutil +from typing import Optional + +import httpx +from fastapi import APIRouter, HTTPException, Request + +from core.models import ( + GitHubCloneRequest, GitHubCreateRepoRequest, + GitHubCommitRequest, GitHubPRRequest, GitHubIssueRequest, +) +from memory.db import save_memory + +router = APIRouter() + +GITHUB_TOKEN = os.environ.get("GITHUB_TOKEN", "") +GITHUB_OWNER = os.environ.get("GITHUB_OWNER", "") +GITHUB_API = "https://api.github.com" + + +def gh_headers(): + if not GITHUB_TOKEN: + raise HTTPException(status_code=400, detail="GITHUB_TOKEN not configured") + return { + "Authorization": f"Bearer {GITHUB_TOKEN}", + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + } + + +async def gh_get(path: str) -> dict: + async with httpx.AsyncClient(timeout=30) as client: + r = await client.get(f"{GITHUB_API}{path}", headers=gh_headers()) + r.raise_for_status() + return r.json() + + +async def gh_post(path: str, data: dict) -> dict: + async with httpx.AsyncClient(timeout=30) as client: + r = await client.post(f"{GITHUB_API}{path}", headers=gh_headers(), json=data) + r.raise_for_status() + return r.json() + + +async def gh_put(path: str, data: dict) -> dict: + async with httpx.AsyncClient(timeout=30) as client: + r = await client.put(f"{GITHUB_API}{path}", headers=gh_headers(), json=data) + r.raise_for_status() + return r.json() + + +async def gh_patch(path: str, data: dict) -> dict: + async with httpx.AsyncClient(timeout=30) as client: + r = await client.patch(f"{GITHUB_API}{path}", headers=gh_headers(), json=data) + r.raise_for_status() + return r.json() + + +# ─── Clone ──────────────────────────────────────────────────────────────────── + +@router.post("/clone", summary="Clone a GitHub repository") +async def clone_repo(req: GitHubCloneRequest): + try: + import git + except ImportError: + raise HTTPException(status_code=500, detail="gitpython not installed") + + local_path = req.local_path or f"/tmp/repos/{req.repo_url.split('/')[-1].replace('.git', '')}" + os.makedirs(local_path, exist_ok=True) + + if GITHUB_TOKEN: + url = req.repo_url.replace("https://", f"https://{GITHUB_TOKEN}@") + else: + url = req.repo_url + + try: + if os.path.exists(os.path.join(local_path, ".git")): + repo = git.Repo(local_path) + repo.remotes.origin.pull() + action = "pulled" + else: + repo = git.Repo.clone_from(url, local_path, branch=req.branch, depth=1) + action = "cloned" + + files = [] + for root, dirs, fnames in os.walk(local_path): + dirs[:] = [d for d in dirs if d not in [".git", "node_modules", "__pycache__"]] + for f in fnames[:50]: + files.append(os.path.relpath(os.path.join(root, f), local_path)) + + # Save to memory + await save_memory( + content=f"Repo {req.repo_url} cloned to {local_path}. Files: {', '.join(files[:20])}", + memory_type="repo", + key=req.repo_url, + ) + + return { + "action": action, + "repo_url": req.repo_url, + "local_path": local_path, + "branch": req.branch, + "files_count": len(files), + "files": files[:30], + } + except Exception as e: + raise HTTPException(status_code=500, detail=f"Clone failed: {str(e)}") + + +# ─── Create Repo ────────────────────────────────────────────────────────────── + +@router.post("/create_repo", summary="Create a new GitHub repository") +async def create_repo(req: GitHubCreateRepoRequest): + data = { + "name": req.name, + "description": req.description, + "private": req.private, + "auto_init": req.auto_init, + } + try: + result = await gh_post("/user/repos", data) + return { + "repo": result["full_name"], + "url": result["html_url"], + "clone_url": result["clone_url"], + "default_branch": result.get("default_branch", "main"), + "private": result["private"], + } + except httpx.HTTPStatusError as e: + raise HTTPException(status_code=e.response.status_code, detail=e.response.text) + + +# ─── Commit Files ───────────────────────────────────────────────────────────── + +@router.post("/commit", summary="Commit files to a repository") +async def commit_files(req: GitHubCommitRequest): + import base64 + + owner_repo = req.repo if "/" in req.repo else f"{GITHUB_OWNER}/{req.repo}" + results = [] + + for file_path, content in req.files.items(): + encoded = base64.b64encode(content.encode()).decode() + + # Get current SHA if file exists + sha = None + try: + existing = await gh_get(f"/repos/{owner_repo}/contents/{file_path}?ref={req.branch}") + sha = existing.get("sha") + except Exception: + pass + + payload = { + "message": req.message, + "content": encoded, + "branch": req.branch, + } + if sha: + payload["sha"] = sha + + try: + result = await gh_put(f"/repos/{owner_repo}/contents/{file_path}", payload) + results.append({"file": file_path, "status": "committed", "sha": result["content"]["sha"]}) + except Exception as e: + results.append({"file": file_path, "status": "error", "error": str(e)}) + + return { + "repo": owner_repo, + "branch": req.branch, + "message": req.message, + "files": results, + "committed": sum(1 for r in results if r["status"] == "committed"), + } + + +# ─── Push ───────────────────────────────────────────────────────────────────── + +@router.post("/push", summary="Push local changes to remote") +async def push_changes( + repo_path: str, + branch: str = "main", + message: str = "Auto-commit by Devin Agent", +): + try: + import git + repo = git.Repo(repo_path) + repo.git.add(A=True) + if repo.index.diff("HEAD") or repo.untracked_files: + repo.index.commit(message) + origin = repo.remote("origin") + origin.push(refspec=f"HEAD:{branch}") + return {"status": "pushed", "branch": branch, "message": message} + except Exception as e: + raise HTTPException(status_code=500, detail=f"Push failed: {str(e)}") + + +# ─── Create PR ──────────────────────────────────────────────────────────────── + +@router.post("/pr/create", summary="Create a Pull Request") +async def create_pr(req: GitHubPRRequest): + owner_repo = req.repo if "/" in req.repo else f"{GITHUB_OWNER}/{req.repo}" + data = { + "title": req.title, + "body": req.body, + "head": req.head, + "base": req.base, + "draft": req.draft, + } + try: + result = await gh_post(f"/repos/{owner_repo}/pulls", data) + return { + "pr_number": result["number"], + "title": result["title"], + "url": result["html_url"], + "state": result["state"], + "head": req.head, + "base": req.base, + } + except httpx.HTTPStatusError as e: + raise HTTPException(status_code=e.response.status_code, detail=e.response.text) + + +# ─── Create Issue ───────────────────────────────────────────────────────────── + +@router.post("/issues/create", summary="Create a GitHub Issue") +async def create_issue(req: GitHubIssueRequest): + owner_repo = req.repo if "/" in req.repo else f"{GITHUB_OWNER}/{req.repo}" + data = {"title": req.title, "body": req.body, "labels": req.labels} + try: + result = await gh_post(f"/repos/{owner_repo}/issues", data) + return { + "issue_number": result["number"], + "title": result["title"], + "url": result["html_url"], + "state": result["state"], + } + except httpx.HTTPStatusError as e: + raise HTTPException(status_code=e.response.status_code, detail=e.response.text) + + +# ─── Code Review ────────────────────────────────────────────────────────────── + +@router.post("/review", summary="AI code review for a PR") +async def review_pr(repo: str, pr_number: int, request: Request): + owner_repo = repo if "/" in repo else f"{GITHUB_OWNER}/{repo}" + try: + pr = await gh_get(f"/repos/{owner_repo}/pulls/{pr_number}") + files = await gh_get(f"/repos/{owner_repo}/pulls/{pr_number}/files") + + file_changes = [] + for f in files[:10]: + file_changes.append(f"{f['filename']}: +{f.get('additions',0)}/-{f.get('deletions',0)}") + + ws = request.app.state.ws_manager + from core.agent import AgentCore + agent = AgentCore(ws) + + review_prompt = ( + f"Review this Pull Request:\n" + f"Title: {pr['title']}\n" + f"Description: {pr.get('body', 'No description')}\n" + f"Files changed: {chr(10).join(file_changes)}\n\n" + f"Provide a constructive code review with: summary, potential issues, suggestions, and verdict." + ) + messages = [ + {"role": "system", "content": "You are a senior software engineer doing code review. Be constructive, specific, and helpful."}, + {"role": "user", "content": review_prompt}, + ] + review = await agent.llm_stream(messages) + + # Post review comment + if GITHUB_TOKEN: + await gh_post(f"/repos/{owner_repo}/issues/{pr_number}/comments", {"body": f"πŸ€– **Devin Agent Code Review**\n\n{review}"}) + + return { + "pr_number": pr_number, + "title": pr["title"], + "review": review, + "files_reviewed": len(files), + "posted_to_github": bool(GITHUB_TOKEN), + } + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + + +# ─── Repo Info ──────────────────────────────────────────────────────────────── + +@router.get("/repo/{owner}/{repo}", summary="Get repository info") +async def get_repo_info(owner: str, repo: str): + try: + info = await gh_get(f"/repos/{owner}/{repo}") + return { + "name": info["name"], + "full_name": info["full_name"], + "description": info.get("description"), + "url": info["html_url"], + "default_branch": info["default_branch"], + "language": info.get("language"), + "stars": info["stargazers_count"], + "forks": info["forks_count"], + "open_issues": info["open_issues_count"], + "private": info["private"], + } + except httpx.HTTPStatusError as e: + raise HTTPException(status_code=e.response.status_code, detail=e.response.text) + + +# ─── Status check ───────────────────────────────────────────────────────────── + +@router.get("/status", summary="GitHub integration status") +async def github_status(): + configured = bool(GITHUB_TOKEN) + user = None + if configured: + try: + user_info = await gh_get("/user") + user = user_info.get("login") + except Exception: + configured = False + return { + "configured": configured, + "user": user, + "owner": GITHUB_OWNER or user, + "capabilities": [ + "clone", "create_repo", "commit", "push", + "pr/create", "issues/create", "review" + ], + } diff --git a/api/routes/health.py b/api/routes/health.py new file mode 100644 index 0000000000000000000000000000000000000000..23f54e1467b868ea8deb41de78d6ea1c497a8cd0 --- /dev/null +++ b/api/routes/health.py @@ -0,0 +1,53 @@ +""" +Health + Status Routes +""" + +import time +import os +import psutil +from fastapi import APIRouter, Request + +router = APIRouter() + + +@router.get("/health", summary="Health check") +async def health(request: Request): + ws = request.app.state.ws_manager + engine = request.app.state.task_engine + stats = ws.get_stats() + return { + "status": "healthy", + "version": "2.0.0", + "timestamp": time.time(), + "websocket_connections": stats["total_connections"], + "websocket_rooms": list(stats["rooms"].keys()), + "task_queue_size": engine._queue.qsize(), + "active_tasks": len(engine._active), + "llm": { + "openai": bool(os.environ.get("OPENAI_API_KEY")), + "anthropic": bool(os.environ.get("ANTHROPIC_API_KEY")), + "model": os.environ.get("DEFAULT_MODEL", "gpt-4o"), + }, + "github": bool(os.environ.get("GITHUB_TOKEN")), + } + + +@router.get("/metrics", summary="System metrics") +async def metrics(): + cpu = psutil.cpu_percent(interval=0.1) + mem = psutil.virtual_memory() + disk = psutil.disk_usage("/") + return { + "cpu_percent": cpu, + "memory": { + "total_mb": round(mem.total / 1024 / 1024), + "used_mb": round(mem.used / 1024 / 1024), + "percent": mem.percent, + }, + "disk": { + "total_gb": round(disk.total / 1024 / 1024 / 1024, 1), + "used_gb": round(disk.used / 1024 / 1024 / 1024, 1), + "percent": disk.percent, + }, + "timestamp": time.time(), + } diff --git a/api/routes/memory.py b/api/routes/memory.py new file mode 100644 index 0000000000000000000000000000000000000000..52f485689944fb2bca0f71f4b31172da3176503d --- /dev/null +++ b/api/routes/memory.py @@ -0,0 +1,50 @@ +""" +Memory API Routes β€” Persistent agent memory +""" + +import time +from fastapi import APIRouter, HTTPException, Query +from core.models import MemorySaveRequest, MemorySearchRequest +from memory.db import save_memory, search_memory, get_project_memory, get_history + +router = APIRouter() + + +@router.post("/", summary="Save memory") +async def save(req: MemorySaveRequest): + await save_memory( + content=req.content, + memory_type=req.memory_type.value, + session_id=req.session_id, + project_id=req.project_id, + key=req.key, + metadata=req.metadata, + ) + return {"status": "saved", "memory_type": req.memory_type, "timestamp": time.time()} + + +@router.post("/search", summary="Search memory") +async def search(req: MemorySearchRequest): + results = await search_memory( + query=req.query, + session_id=req.session_id, + project_id=req.project_id, + limit=req.limit, + ) + return {"results": results, "total": len(results), "query": req.query} + + +@router.get("/project/{project_id}", summary="Get project memory") +async def project_memory( + project_id: str, + memory_type: str = Query(default=""), + limit: int = Query(default=100, le=500), +): + results = await get_project_memory(project_id, memory_type=memory_type, limit=limit) + return {"project_id": project_id, "memories": results, "total": len(results)} + + +@router.get("/history/{session_id}", summary="Get conversation history") +async def history(session_id: str, limit: int = Query(default=50, le=200)): + results = await get_history(session_id, limit=limit) + return {"session_id": session_id, "history": results, "total": len(results)} diff --git a/api/routes/tasks.py b/api/routes/tasks.py new file mode 100644 index 0000000000000000000000000000000000000000..82fde607690b1a939594ec278ff6191888af1e63 --- /dev/null +++ b/api/routes/tasks.py @@ -0,0 +1,167 @@ +""" +Task API Routes β€” CRUD + Streaming + WebSocket +""" + +import asyncio +import json +import time +from typing import Optional + +from fastapi import APIRouter, HTTPException, Request, Query +from fastapi.responses import StreamingResponse + +from core.models import ( + TaskCreateRequest, TaskCancelRequest, TaskRetryRequest, TaskResponse, TaskStatus +) +from memory.db import get_task, list_tasks, get_task_events, update_task_status + +router = APIRouter() + + +def get_engine(request: Request): + return request.app.state.task_engine + + +def get_ws(request: Request): + return request.app.state.ws_manager + + +# ─── Create Task ─────────────────────────────────────────────────────────────── + +@router.post("/create", summary="Create & queue a new agent task") +async def create_task(req: TaskCreateRequest, request: Request): + engine = get_engine(request) + task_id = await engine.submit(req) + task = await get_task(task_id) + return { + "task_id": task_id, + "status": "queued", + "goal": req.goal, + "session_id": req.session_id, + "stream_url": f"/api/v1/tasks/{task_id}/stream", + "ws_url": f"/ws/tasks/{task_id}", + "created_at": task["created_at"] if task else time.time(), + } + + +# ─── Get Task ────────────────────────────────────────────────────────────────── + +@router.get("/{task_id}", summary="Get task details") +async def get_task_detail(task_id: str): + task = await get_task(task_id) + if not task: + raise HTTPException(status_code=404, detail=f"Task {task_id} not found") + return task + + +# ─── Get Task Status ─────────────────────────────────────────────────────────── + +@router.get("/{task_id}/status", summary="Get task status only") +async def get_task_status(task_id: str): + task = await get_task(task_id) + if not task: + raise HTTPException(status_code=404, detail=f"Task {task_id} not found") + return { + "task_id": task_id, + "status": task["status"], + "retry_count": task.get("retry_count", 0), + "created_at": task.get("created_at"), + "started_at": task.get("started_at"), + "completed_at": task.get("completed_at"), + } + + +# ─── Cancel Task ─────────────────────────────────────────────────────────────── + +@router.post("/{task_id}/cancel", summary="Cancel a running task") +async def cancel_task(task_id: str, req: TaskCancelRequest, request: Request): + task = await get_task(task_id) + if not task: + raise HTTPException(status_code=404, detail=f"Task {task_id} not found") + if task["status"] in ("completed", "failed", "cancelled"): + raise HTTPException(status_code=400, detail=f"Task already {task['status']}") + engine = get_engine(request) + await engine.cancel(task_id, req.reason) + return {"task_id": task_id, "status": "cancelled", "reason": req.reason} + + +# ─── Retry Task ──────────────────────────────────────────────────────────────── + +@router.post("/{task_id}/retry", summary="Retry a failed task") +async def retry_task(task_id: str, request: Request): + task = await get_task(task_id) + if not task: + raise HTTPException(status_code=404, detail=f"Task {task_id} not found") + if task["status"] not in ("failed", "cancelled"): + raise HTTPException(status_code=400, detail="Only failed/cancelled tasks can be retried") + engine = get_engine(request) + await engine.retry(task_id) + return {"task_id": task_id, "status": "queued", "message": "Task requeued for retry"} + + +# ─── Stream Task Events (SSE) ────────────────────────────────────────────────── + +@router.get("/{task_id}/stream", summary="Stream task events via SSE") +async def stream_task(task_id: str, request: Request): + task = await get_task(task_id) + if not task: + raise HTTPException(status_code=404, detail=f"Task {task_id} not found") + + async def event_generator(): + # First, replay all stored events + events = await get_task_events(task_id) + for ev in events: + data = json.dumps({ + "type": ev["event_type"], + "task_id": task_id, + "timestamp": ev["timestamp"], + "data": json.loads(ev["data"]) if ev.get("data") else {}, + }) + yield f"data: {data}\n\n" + + # Then stream live events via WS manager buffer + ws = get_ws(request) + room = f"task:{task_id}" + last_count = len(events) + + # Poll for new events (for SSE fallback) + for _ in range(600): # max 5 minutes + await asyncio.sleep(0.5) + current_task = await get_task(task_id) + if current_task and current_task["status"] in ("completed", "failed", "cancelled"): + yield f"data: {json.dumps({'type': 'stream_end', 'task_id': task_id, 'status': current_task['status']})}\n\n" + break + # heartbeat + yield f"data: {json.dumps({'type': 'heartbeat', 'timestamp': time.time()})}\n\n" + + return StreamingResponse( + event_generator(), + media_type="text/event-stream", + headers={ + "Cache-Control": "no-cache", + "X-Accel-Buffering": "no", + "Connection": "keep-alive", + }, + ) + + +# ─── List Tasks ──────────────────────────────────────────────────────────────── + +@router.get("/", summary="List tasks") +async def list_all_tasks( + session_id: str = Query(default=""), + limit: int = Query(default=50, le=200), +): + tasks = await list_tasks(session_id=session_id, limit=limit) + return {"tasks": tasks, "total": len(tasks)} + + +# ─── Task Events History ─────────────────────────────────────────────────────── + +@router.get("/{task_id}/events", summary="Get all events for a task") +async def task_events(task_id: str): + task = await get_task(task_id) + if not task: + raise HTTPException(status_code=404, detail=f"Task {task_id} not found") + events = await get_task_events(task_id) + return {"task_id": task_id, "events": events, "total": len(events)} diff --git a/api/websocket_manager.py b/api/websocket_manager.py new file mode 100644 index 0000000000000000000000000000000000000000..68aea8a58e3d2381e6eec51197b6a9f13ad9931d --- /dev/null +++ b/api/websocket_manager.py @@ -0,0 +1,134 @@ +""" +WebSocket Connection Manager β€” Production Grade +Handles rooms, heartbeats, event buffering, reconnect support +""" + +import asyncio +import json +import time +import uuid +from collections import defaultdict +from typing import Dict, List, Optional, Set +import structlog + +log = structlog.get_logger() + + +class WebSocketManager: + def __init__(self): + # room β†’ set of websockets + self._rooms: Dict[str, Set] = defaultdict(set) + # ws β†’ list of rooms + self._ws_rooms: Dict[object, Set[str]] = defaultdict(set) + # Event buffer per room (for replay on reconnect) + self._event_buffer: Dict[str, List] = defaultdict(list) + self._buffer_max = 100 + # Active connection count + self._connection_count = 0 + + async def connect(self, websocket, room: str): + await websocket.accept() + self._rooms[room].add(websocket) + self._ws_rooms[websocket].add(room) + self._connection_count += 1 + log.info("WS connected", room=room, total=self._connection_count) + + # Replay buffered events for this room + buffered = self._event_buffer.get(room, [])[-20:] + for event in buffered: + try: + await websocket.send_json(event) + except Exception: + pass + + await websocket.send_json({ + "type": "connected", + "room": room, + "timestamp": time.time(), + "buffered_events": len(buffered), + }) + + def disconnect(self, websocket, room: Optional[str] = None): + if room: + self._rooms[room].discard(websocket) + self._ws_rooms[websocket].discard(room) + else: + for r in list(self._ws_rooms.get(websocket, [])): + self._rooms[r].discard(websocket) + self._ws_rooms.pop(websocket, None) + self._connection_count = max(0, self._connection_count - 1) + log.info("WS disconnected", room=room, total=self._connection_count) + + async def broadcast(self, room: str, event: dict): + """Broadcast event to all sockets in a room.""" + if "timestamp" not in event: + event["timestamp"] = time.time() + if "id" not in event: + event["id"] = str(uuid.uuid4())[:8] + + # Buffer event + self._event_buffer[room].append(event) + if len(self._event_buffer[room]) > self._buffer_max: + self._event_buffer[room].pop(0) + + dead = set() + for ws in list(self._rooms.get(room, [])): + try: + await ws.send_json(event) + except Exception: + dead.add(ws) + + for ws in dead: + self.disconnect(ws, room) + + async def broadcast_global(self, event: dict): + """Broadcast to ALL connected websockets.""" + for room in list(self._rooms.keys()): + await self.broadcast(room, event) + + async def emit(self, task_id: str, event_type: str, data: dict, session_id: str = ""): + """Emit a structured event to a task room + logs room.""" + event = { + "type": event_type, + "task_id": task_id, + "session_id": session_id, + "timestamp": time.time(), + "data": data, + } + await self.broadcast(f"task:{task_id}", event) + await self.broadcast("logs", event) + await self.broadcast("agent_status", { + "type": "agent_event", + "task_id": task_id, + "event_type": event_type, + "timestamp": time.time(), + }) + + async def emit_chat(self, session_id: str, event_type: str, data: dict): + """Emit event to a chat session room.""" + event = { + "type": event_type, + "session_id": session_id, + "timestamp": time.time(), + "data": data, + } + await self.broadcast(f"chat:{session_id}", event) + + async def heartbeat_loop(self): + """Send heartbeat to all connections every 15s.""" + while True: + await asyncio.sleep(15) + heartbeat = { + "type": "heartbeat", + "timestamp": time.time(), + "connections": self._connection_count, + } + for room in list(self._rooms.keys()): + await self.broadcast(room, heartbeat) + + def get_stats(self) -> dict: + return { + "total_connections": self._connection_count, + "rooms": {r: len(ws) for r, ws in self._rooms.items()}, + "buffered_events": {r: len(e) for r, e in self._event_buffer.items()}, + } diff --git a/api_server.py b/api_server.py deleted file mode 100644 index cf0a968d3eae98ed2e759b7e1266537254a06fbd..0000000000000000000000000000000000000000 --- a/api_server.py +++ /dev/null @@ -1,44 +0,0 @@ -import os -import uvicorn -from fastapi import FastAPI, Depends, HTTPException, status -from fastapi.middleware.cors import CORSMiddleware -from routes import chat, tasks, github, workspace, browser, swarm, webhooks -from auth import get_api_key -import websocket_server - -app = FastAPI(title="Autonomous Coding System API", version="1.0.0") - -# CORS Configuration -app.add_middleware( - CORSMiddleware, - allow_origins=["*"], - allow_credentials=True, - allow_methods=["*"], - allow_headers=["*"], -) - -# Include Routes -app.include_router(chat.router, prefix="/api/v1", tags=["Chat"], dependencies=[Depends(get_api_key)]) -app.include_router(tasks.router, prefix="/api/v1", tags=["Tasks"], dependencies=[Depends(get_api_key)]) -app.include_router(github.router, prefix="/api/v1", tags=["GitHub"], dependencies=[Depends(get_api_key)]) -app.include_router(workspace.router, prefix="/api/v1", tags=["Workspace"], dependencies=[Depends(get_api_key)]) -app.include_router(browser.router, prefix="/api/v1", tags=["Browser"], dependencies=[Depends(get_api_key)]) -app.include_router(swarm.router, prefix="/api/v1", tags=["Swarm"], dependencies=[Depends(get_api_key)]) -app.include_router(webhooks.router, prefix="/api/v1", tags=["Webhooks"]) - -@app.get("/health") -async def health_check(): - return {"status": "healthy", "timestamp": "now"} - -@app.get("/agent/status") -async def agent_status(): - return { - "status": "idle", - "active_tasks": 0, - "memory_usage": "low", - "uptime": "0s" - } - -if __name__ == "__main__": - port = int(os.getenv("API_PORT", 8000)) - uvicorn.run(app, host="0.0.0.0", port=port) diff --git a/auth.py b/auth.py deleted file mode 100644 index 10c7fa5c881897a44ad032cd4395058c85c06d45..0000000000000000000000000000000000000000 --- a/auth.py +++ /dev/null @@ -1,41 +0,0 @@ -import os -from fastapi import Security, HTTPException, status -from fastapi.security.api_key import APIKeyHeader -from jose import JWTError, jwt -from datetime import datetime, timedelta -from typing import Optional - -API_KEY_NAME = "X-API-Key" -api_key_header = APIKeyHeader(name=API_KEY_NAME, auto_error=False) - -SECRET_KEY = os.getenv("JWT_SECRET_KEY", "super-secret-key") -ALGORITHM = "HS256" -ACCESS_TOKEN_EXPIRE_MINUTES = 60 * 24 * 7 # 1 week - -def create_access_token(data: dict, expires_delta: Optional[timedelta] = None): - to_encode = data.copy() - if expires_delta: - expire = datetime.utcnow() + expires_delta - else: - expire = datetime.utcnow() + timedelta(minutes=15) - to_encode.update({"exp": expire}) - encoded_jwt = jwt.encode(to_encode, SECRET_KEY, algorithm=ALGORITHM) - return encoded_jwt - -async def get_api_key(api_key_header: str = Security(api_key_header)): - if api_key_header == os.getenv("API_KEY", "default-api-key"): - return api_key_header - raise HTTPException( - status_code=status.HTTP_403_FORBIDDEN, detail="Could not validate API Key" - ) - -async def verify_token(token: str): - try: - payload = jwt.decode(token, SECRET_KEY, algorithms=[ALGORITHM]) - return payload - except JWTError: - raise HTTPException( - status_code=status.HTTP_401_UNAUTHORIZED, - detail="Could not validate credentials", - headers={"WWW-Authenticate": "Bearer"}, - ) diff --git a/core/__init__.py b/core/__init__.py index 6790eda7a026e5a31dcb78836efdb128d70e580a..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -1 +0,0 @@ -# Core logic package diff --git a/core/__pycache__/__init__.cpython-312.pyc b/core/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..379ed24d70cbb3676abd189e72df51c25c53be05 Binary files /dev/null and b/core/__pycache__/__init__.cpython-312.pyc differ diff --git a/core/__pycache__/agent.cpython-312.pyc b/core/__pycache__/agent.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b5b1ebbbb408d883e9048adb2dd8992582281e2a Binary files /dev/null and b/core/__pycache__/agent.cpython-312.pyc differ diff --git a/core/__pycache__/models.cpython-312.pyc b/core/__pycache__/models.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bb9d95933a1cb106e8eca4cd9eac10505e42bd2e Binary files /dev/null and b/core/__pycache__/models.cpython-312.pyc differ diff --git a/core/__pycache__/task_engine.cpython-312.pyc b/core/__pycache__/task_engine.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..921c872cdb4610246211e471c629395245037bea Binary files /dev/null and b/core/__pycache__/task_engine.cpython-312.pyc differ diff --git a/core/agent.py b/core/agent.py new file mode 100644 index 0000000000000000000000000000000000000000..293d2d0b8356d476b1b4d00fafe7712457de8222 --- /dev/null +++ b/core/agent.py @@ -0,0 +1,392 @@ +""" +Agent Core β€” Planner + Executor + Self-Heal Loop +LLM-powered with OpenAI/Anthropic support, streaming tokens +""" + +import asyncio +import json +import os +import time +from typing import Any, Dict, List, Optional + +import httpx +import structlog + +from core.models import TaskPlan, TaskStep +from api.websocket_manager import WebSocketManager +from memory.db import save_memory, get_history, search_memory + +log = structlog.get_logger() + +OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY", "") +ANTHROPIC_API_KEY = os.environ.get("ANTHROPIC_API_KEY", "") +DEFAULT_MODEL = os.environ.get("DEFAULT_MODEL", "gpt-4o") +OPENAI_BASE_URL = os.environ.get("OPENAI_BASE_URL", "https://api.openai.com/v1") + + +SYSTEM_PROMPT = """You are an elite autonomous AI software engineer β€” like Devin or Manus. +You can plan, code, debug, refactor, test, and deploy software autonomously. +You think step-by-step, write production-quality code, and self-heal on errors. +Always respond in structured JSON when asked for plans or structured output. +""" + +PLANNER_PROMPT = """You are a senior software architect. Given a goal, produce a detailed execution plan. + +Respond ONLY with valid JSON: +{ + "steps": [ + { + "name": "Step name", + "description": "What this step does", + "tool": "code|shell|file|browser|github|memory|search|test|none", + "estimated_seconds": 10 + } + ], + "estimated_duration": 60, + "tools_needed": ["code", "shell"] +} + +Goal: {goal} +Context: {context} +""" + + +class AgentCore: + def __init__(self, ws_manager: WebSocketManager): + self.ws = ws_manager + self.model = DEFAULT_MODEL + + # ─── LLM Call (with streaming) ───────────────────────────────────────────── + + async def llm_stream( + self, + messages: List[Dict], + task_id: str = "", + session_id: str = "", + model: str = "", + temperature: float = 0.7, + max_tokens: int = 4096, + ) -> str: + """Stream LLM tokens, emitting llm_chunk events via WebSocket.""" + model = model or self.model + full_text = "" + + if OPENAI_API_KEY: + full_text = await self._openai_stream( + messages, task_id, session_id, model, temperature, max_tokens + ) + elif ANTHROPIC_API_KEY: + full_text = await self._anthropic_stream( + messages, task_id, session_id, temperature, max_tokens + ) + else: + # Demo mode β€” simulate streaming + full_text = await self._demo_stream(messages, task_id, session_id) + + return full_text + + async def _openai_stream( + self, messages, task_id, session_id, model, temperature, max_tokens + ) -> str: + full_text = "" + headers = { + "Authorization": f"Bearer {OPENAI_API_KEY}", + "Content-Type": "application/json", + } + payload = { + "model": model, + "messages": messages, + "stream": True, + "temperature": temperature, + "max_tokens": max_tokens, + } + async with httpx.AsyncClient(timeout=120) as client: + async with client.stream( + "POST", f"{OPENAI_BASE_URL}/chat/completions", + headers=headers, json=payload + ) as resp: + resp.raise_for_status() + async for line in resp.aiter_lines(): + if not line.startswith("data:"): + continue + chunk = line[6:].strip() + if chunk == "[DONE]": + break + try: + data = json.loads(chunk) + delta = data["choices"][0]["delta"].get("content", "") + if delta: + full_text += delta + if task_id: + await self.ws.emit(task_id, "llm_chunk", { + "chunk": delta, + "accumulated": len(full_text), + }, session_id=session_id) + if session_id and not task_id: + await self.ws.emit_chat(session_id, "llm_chunk", { + "chunk": delta, + }) + except Exception: + pass + return full_text + + async def _anthropic_stream( + self, messages, task_id, session_id, temperature, max_tokens + ) -> str: + full_text = "" + system = "" + filtered = [] + for m in messages: + if m["role"] == "system": + system = m["content"] + else: + filtered.append(m) + headers = { + "x-api-key": ANTHROPIC_API_KEY, + "anthropic-version": "2023-06-01", + "Content-Type": "application/json", + } + payload = { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": max_tokens, + "messages": filtered, + "stream": True, + } + if system: + payload["system"] = system + async with httpx.AsyncClient(timeout=120) as client: + async with client.stream( + "POST", "https://api.anthropic.com/v1/messages", + headers=headers, json=payload + ) as resp: + resp.raise_for_status() + async for line in resp.aiter_lines(): + if not line.startswith("data:"): + continue + try: + data = json.loads(line[5:].strip()) + if data.get("type") == "content_block_delta": + delta = data["delta"].get("text", "") + if delta: + full_text += delta + if task_id: + await self.ws.emit(task_id, "llm_chunk", { + "chunk": delta, + }, session_id=session_id) + if session_id and not task_id: + await self.ws.emit_chat(session_id, "llm_chunk", { + "chunk": delta, + }) + except Exception: + pass + return full_text + + async def _demo_stream(self, messages, task_id, session_id) -> str: + """Demo mode β€” simulate LLM streaming without API key.""" + last_user = next( + (m["content"] for m in reversed(messages) if m["role"] == "user"), "Hello" + ) + response = ( + f"πŸ€– **Devin Agent** (Demo Mode)\n\n" + f"I received your request: *{last_user[:100]}*\n\n" + f"To enable real AI responses, set `OPENAI_API_KEY` or `ANTHROPIC_API_KEY` in your environment.\n\n" + f"**What I can do with a real API key:**\n" + f"- πŸ“‹ Generate detailed execution plans\n" + f"- πŸ’» Write and execute code autonomously\n" + f"- πŸ”§ Debug and self-heal on errors\n" + f"- πŸ™ Manage GitHub repos autonomously\n" + f"- 🧠 Remember long-running project context\n" + f"- πŸš€ Deploy applications automatically\n" + ) + full_text = "" + for word in response.split(): + chunk = word + " " + full_text += chunk + await asyncio.sleep(0.03) + if task_id: + await self.ws.emit(task_id, "llm_chunk", { + "chunk": chunk, + "demo": True, + }, session_id=session_id) + if session_id and not task_id: + await self.ws.emit_chat(session_id, "llm_chunk", { + "chunk": chunk, + "demo": True, + }) + return full_text + + # ─── Planning ────────────────────────────────────────────────────────────── + + async def plan(self, goal: str, task_id: str, session_id: str = "") -> TaskPlan: + """Generate a structured execution plan.""" + # Get context from memory + memories = await search_memory(goal[:50], session_id=session_id) + context = "\n".join([m["content"][:200] for m in memories[:3]]) + + prompt = PLANNER_PROMPT.format(goal=goal, context=context or "No prior context") + + messages = [ + {"role": "system", "content": SYSTEM_PROMPT}, + {"role": "user", "content": prompt}, + ] + + if not OPENAI_API_KEY and not ANTHROPIC_API_KEY: + # Demo plan + return self._demo_plan(goal) + + raw = await self.llm_stream(messages, task_id=task_id, session_id=session_id) + + # Extract JSON from response + try: + # Find JSON block + start = raw.find("{") + end = raw.rfind("}") + 1 + if start >= 0 and end > start: + data = json.loads(raw[start:end]) + else: + data = json.loads(raw) + + steps = [] + for i, s in enumerate(data.get("steps", [])): + steps.append(TaskStep( + name=s.get("name", f"Step {i+1}"), + description=s.get("description", ""), + tool=s.get("tool", "none"), + )) + + return TaskPlan( + goal=goal, + steps=steps if steps else [TaskStep(name="Execute goal", description=goal, tool="code")], + estimated_duration=data.get("estimated_duration", 60), + tools_needed=data.get("tools_needed", []), + ) + except Exception as e: + log.warning("Plan parse failed, using fallback", error=str(e)) + return self._demo_plan(goal) + + def _demo_plan(self, goal: str) -> TaskPlan: + """Fallback plan for demo mode.""" + steps = [ + TaskStep(name="Analyze Requirements", description=f"Analyze: {goal[:60]}", tool="none"), + TaskStep(name="Design Solution", description="Design the solution architecture", tool="none"), + TaskStep(name="Implement", description="Write the implementation code", tool="code"), + TaskStep(name="Test", description="Test the implementation", tool="test"), + TaskStep(name="Document", description="Write documentation", tool="none"), + ] + return TaskPlan( + goal=goal, + steps=steps, + estimated_duration=120, + tools_needed=["code", "test"], + ) + + # ─── Step Execution ──────────────────────────────────────────────────────── + + async def execute_step( + self, + step: TaskStep, + task_id: str, + session_id: str = "", + context: Dict = {}, + ) -> str: + """Execute a single step using the appropriate tool.""" + from tools.executor import ToolExecutor + executor = ToolExecutor(self.ws) + + await self.ws.emit(task_id, "tool_called", { + "tool": step.tool or "none", + "step": step.name, + "description": step.description, + }, session_id=session_id) + + try: + result = await executor.run( + tool=step.tool or "none", + task=step.description, + goal=context.get("goal", ""), + previous=context.get("previous_results", []), + task_id=task_id, + session_id=session_id, + ) + await self.ws.emit(task_id, "tool_result", { + "tool": step.tool, + "step": step.name, + "result": str(result)[:500], + "success": True, + }, session_id=session_id) + return result + except Exception as e: + await self.ws.emit(task_id, "tool_result", { + "tool": step.tool, + "step": step.name, + "error": str(e), + "success": False, + }, session_id=session_id) + return f"Error in {step.name}: {str(e)}" + + # ─── Finalize ────────────────────────────────────────────────────────────── + + async def finalize( + self, + goal: str, + steps: List[TaskStep], + results: List[str], + task_id: str, + session_id: str = "", + ) -> str: + """Compile final result summary.""" + steps_summary = "\n".join([ + f"- {s.name}: {r[:200]}" for s, r in zip(steps, results) + ]) + messages = [ + {"role": "system", "content": SYSTEM_PROMPT}, + {"role": "user", "content": ( + f"Summarize the completion of this goal:\n" + f"Goal: {goal}\n\n" + f"Steps completed:\n{steps_summary}\n\n" + f"Write a concise success summary with key outcomes." + )}, + ] + result = await self.llm_stream(messages, task_id=task_id, session_id=session_id) + return result or f"βœ… Completed: {goal}" + + # ─── Chat ────────────────────────────────────────────────────────────────── + + async def stream_chat(self, session_id: str, user_message: str): + """Stream a conversational chat response.""" + # Save user message to memory + await save_memory( + content=user_message, + memory_type="conversation", + session_id=session_id, + key="user_message", + ) + + # Get conversation history + history = await get_history(session_id, limit=10) + messages = [{"role": "system", "content": SYSTEM_PROMPT}] + for h in reversed(history[-10:]): + messages.append({"role": "user", "content": h["content"]}) + + messages.append({"role": "user", "content": user_message}) + + await self.ws.emit_chat(session_id, "stream_start", { + "status": "generating", + }) + + response = await self.llm_stream(messages, session_id=session_id) + + # Save assistant response to memory + await save_memory( + content=response, + memory_type="conversation", + session_id=session_id, + key="assistant_response", + ) + + await self.ws.emit_chat(session_id, "stream_end", { + "full_response": response, + "status": "complete", + }) + + return response diff --git a/core/database.py b/core/database.py deleted file mode 100644 index 90eff7d35c176046b19c5bc72c3316ec7c8a5c1c..0000000000000000000000000000000000000000 --- a/core/database.py +++ /dev/null @@ -1,132 +0,0 @@ -import sqlite3 -import json -import os -from datetime import datetime -from typing import Any, Dict, List, Optional - -DB_PATH = os.getenv("DB_PATH", "autonomous_coding.db") - -class Database: - def __init__(self): - self.conn = sqlite3.connect(DB_PATH, check_same_thread=False) - self.conn.row_factory = sqlite3.Row - self.create_tables() - - def create_tables(self): - cursor = self.conn.cursor() - - # Tasks table - cursor.execute(''' - CREATE TABLE IF NOT EXISTS tasks ( - id TEXT PRIMARY KEY, - goal TEXT, - type TEXT, - status TEXT, - progress INTEGER, - result TEXT, - error TEXT, - created_at TIMESTAMP, - updated_at TIMESTAMP - ) - ''') - - # Memory table - cursor.execute(''' - CREATE TABLE IF NOT EXISTS memory ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - project_id TEXT, - category TEXT, -- goal, plan, execution, tool, error, file_state - content TEXT, - timestamp TIMESTAMP - ) - ''') - - # Logs table - cursor.execute(''' - CREATE TABLE IF NOT EXISTS logs ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - task_id TEXT, - message TEXT, - timestamp TIMESTAMP, - FOREIGN KEY (task_id) REFERENCES tasks (id) - ) - ''') - - self.conn.commit() - - def save_task(self, task_data: Dict[str, Any]): - cursor = self.conn.cursor() - cursor.execute(''' - INSERT OR REPLACE INTO tasks (id, goal, type, status, progress, result, error, created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) - ''', ( - task_data['id'], - task_data['goal'], - task_data['type'], - task_data['status'], - task_data.get('progress', 0), - json.dumps(task_data.get('result')), - task_data.get('error'), - task_data['created_at'], - task_data['updated_at'] - )) - self.conn.commit() - - def get_task(self, task_id: str) -> Optional[Dict[str, Any]]: - cursor = self.conn.cursor() - cursor.execute('SELECT * FROM tasks WHERE id = ?', (task_id,)) - row = cursor.fetchone() - if row: - data = dict(row) - data['result'] = json.loads(data['result']) if data['result'] else None - return data - return None - - def list_tasks(self) -> List[Dict[str, Any]]: - cursor = self.conn.cursor() - cursor.execute('SELECT * FROM tasks ORDER BY created_at DESC') - rows = cursor.fetchall() - tasks = [] - for row in rows: - data = dict(row) - data['result'] = json.loads(data['result']) if data['result'] else None - tasks.append(data) - return tasks - - def add_memory(self, project_id: str, category: str, content: Any): - cursor = self.conn.cursor() - cursor.execute(''' - INSERT INTO memory (project_id, category, content, timestamp) - VALUES (?, ?, ?, ?) - ''', (project_id, category, json.dumps(content), datetime.utcnow().isoformat())) - self.conn.commit() - - def get_memory(self, project_id: str, category: Optional[str] = None) -> List[Dict[str, Any]]: - cursor = self.conn.cursor() - if category: - cursor.execute('SELECT * FROM memory WHERE project_id = ? AND category = ? ORDER BY timestamp DESC', (project_id, category)) - else: - cursor.execute('SELECT * FROM memory WHERE project_id = ? ORDER BY timestamp DESC', (project_id,)) - - rows = cursor.fetchall() - memories = [] - for row in rows: - data = dict(row) - data['content'] = json.loads(data['content']) - memories.append(data) - return memories - - def add_log(self, task_id: str, message: str): - cursor = self.conn.cursor() - cursor.execute(''' - INSERT INTO logs (task_id, message, timestamp) - VALUES (?, ?, ?) - ''', (task_id, message, datetime.utcnow().isoformat())) - self.conn.commit() - - def get_logs(self, task_id: str) -> List[Dict[str, Any]]: - cursor = self.conn.cursor() - cursor.execute('SELECT * FROM logs WHERE task_id = ? ORDER BY timestamp ASC', (task_id,)) - return [dict(row) for row in cursor.fetchall()] - -db = Database() diff --git a/core/github_engine.py b/core/github_engine.py deleted file mode 100644 index 5a0fae63feb07f76a955742bc6dc49d25e2536ec..0000000000000000000000000000000000000000 --- a/core/github_engine.py +++ /dev/null @@ -1,80 +0,0 @@ -import os -import subprocess -import requests -from typing import Optional, List, Dict, Any - -class GitHubEngine: - def __init__(self, token: Optional[str] = None): - self.token = token or os.getenv("GITHUB_TOKEN") - self.api_base = "https://api.github.com" - self.headers = { - "Authorization": f"token {self.token}", - "Accept": "application/vnd.github.v3+json" - } if self.token else {} - - def _run_git(self, args: List[str], cwd: Optional[str] = None) -> str: - try: - result = subprocess.run( - ["git"] + args, - cwd=cwd, - capture_output=True, - text=True, - check=True - ) - return result.stdout.strip() - except subprocess.CalledProcessError as e: - raise Exception(f"Git command failed: {e.stderr}") - - def clone(self, repo_url: str, dest_path: str) -> str: - if self.token and "github.com" in repo_url: - auth_url = repo_url.replace("https://", f"https://x-access-token:{self.token}@") - else: - auth_url = repo_url - return self._run_git(["clone", auth_url, dest_path]) - - def create_repo(self, name: str, private: bool = True) -> Dict[str, Any]: - resp = requests.post( - f"{self.api_base}/user/repos", - headers=self.headers, - json={"name": name, "private": private} - ) - resp.raise_for_status() - return resp.json() - - def manage_branch(self, repo_path: str, branch_name: str, create: bool = False): - args = ["checkout", "-b" if create else "", branch_name] - args = [a for a in args if a] - return self._run_git(args, cwd=repo_path) - - def commit_and_push(self, repo_path: str, message: str, branch: str = "main"): - self._run_git(["add", "."], cwd=repo_path) - self._run_git(["commit", "-m", message], cwd=repo_path) - self._run_git(["push", "origin", branch], cwd=repo_path) - - def create_pr(self, repo_full_name: str, title: str, body: str, head: str, base: str = "main") -> Dict[str, Any]: - resp = requests.post( - f"{self.api_base}/repos/{repo_full_name}/pulls", - headers=self.headers, - json={"title": title, "body": body, "head": head, "base": base} - ) - resp.raise_for_status() - return resp.json() - - def list_issues(self, repo_full_name: str, state: str = "open") -> List[Dict[str, Any]]: - resp = requests.get( - f"{self.api_base}/repos/{repo_full_name}/issues", - headers=self.headers, - params={"state": state} - ) - resp.raise_for_status() - return resp.json() - - def get_workflow_runs(self, repo_full_name: str) -> List[Dict[str, Any]]: - resp = requests.get( - f"{self.api_base}/repos/{repo_full_name}/actions/runs", - headers=self.headers - ) - resp.raise_for_status() - return resp.json().get("workflow_runs", []) - -github_engine = GitHubEngine() diff --git a/core/llm_router.py b/core/llm_router.py deleted file mode 100644 index a00fa7e660dbed78dc4e86bee5c397b7dffbd79a..0000000000000000000000000000000000000000 --- a/core/llm_router.py +++ /dev/null @@ -1,37 +0,0 @@ -import os -import aiohttp -import json -from typing import List, Dict, Any, Optional - -class LLMRouter: - def __init__(self): - self.gateway_url = "https://gateway.pyaesone-gtckglay.workers.dev/v1/chat/completions" - # API Key is already included in the gateway URL as per user instruction, - # but we'll allow an override via environment variable if needed. - self.api_key = os.getenv("LLM_API_KEY", "") - - async def chat_completion(self, messages: List[Dict[str, str]], model: str = "gpt-4", stream: bool = False) -> Dict[str, Any]: - headers = { - "Content-Type": "application/json" - } - if self.api_key: - headers["Authorization"] = f"Bearer {self.api_key}" - - payload = { - "model": model, - "messages": messages, - "stream": stream - } - - async with aiohttp.ClientSession() as session: - async with session.post(self.gateway_url, headers=headers, json=payload) as response: - if response.status != 200: - error_text = await response.text() - raise Exception(f"LLM Gateway Error ({response.status}): {error_text}") - - if stream: - return response # Return the response object for streaming - - return await response.json() - -llm_router = LLMRouter() diff --git a/core/memory.py b/core/memory.py deleted file mode 100644 index 77f6f5c54f0c65b4e73a0f9ad7ffc9ff8f67e201..0000000000000000000000000000000000000000 --- a/core/memory.py +++ /dev/null @@ -1,44 +0,0 @@ -from typing import Any, Dict, List, Optional -from core.database import db - -class MemoryEngine: - def __init__(self, project_id: str = "default"): - self.project_id = project_id - - def save_goal(self, goal: str): - db.add_memory(self.project_id, "goal", goal) - - def save_plan(self, plan: List[Dict[str, Any]]): - db.add_memory(self.project_id, "plan", plan) - - def save_execution(self, step: str, result: Any): - db.add_memory(self.project_id, "execution", {"step": step, "result": result}) - - def save_tool_usage(self, tool_name: str, args: Dict[str, Any], output: Any): - db.add_memory(self.project_id, "tool", {"tool": tool_name, "args": args, "output": output}) - - def save_error(self, error: str, context: Optional[str] = None): - db.add_memory(self.project_id, "error", {"error": error, "context": context}) - - def save_file_state(self, file_path: str, checksum: str): - db.add_memory(self.project_id, "file_state", {"path": file_path, "checksum": checksum}) - - def get_full_context(self) -> str: - memories = db.get_memory(self.project_id) - if not memories: - return "No previous memory found for this project." - - context_parts = ["### Project Memory Context:"] - for m in reversed(memories): # Oldest to newest - category = m['category'].upper() - content = m['content'] - timestamp = m['timestamp'] - context_parts.append(f"[{timestamp}] {category}: {content}") - - return "\n".join(context_parts) - - def get_recent_memories(self, limit: int = 10) -> List[Dict[str, Any]]: - memories = db.get_memory(self.project_id) - return memories[:limit] - -memory_engine = MemoryEngine() diff --git a/core/models.py b/core/models.py new file mode 100644 index 0000000000000000000000000000000000000000..772070923d8ddf8b5cba203e2d64ddb31c010fe6 --- /dev/null +++ b/core/models.py @@ -0,0 +1,213 @@ +""" +Pydantic Models β€” Task, Chat, Memory, GitHub +""" + +import time +import uuid +from enum import Enum +from typing import Any, Dict, List, Optional +from pydantic import BaseModel, Field + + +def gen_id(prefix: str = "") -> str: + return f"{prefix}{uuid.uuid4().hex[:12]}" + + +# ─── Enums ───────────────────────────────────────────────────────────────────── + +class TaskStatus(str, Enum): + queued = "queued" + initializing = "initializing" + planning = "planning" + executing = "executing" + streaming = "streaming" + waiting_input = "waiting_input" + retrying = "retrying" + finalizing = "finalizing" + completed = "completed" + failed = "failed" + cancelled = "cancelled" + + +class EventType(str, Enum): + task_created = "task_created" + task_queued = "task_queued" + task_started = "task_started" + plan_generated = "plan_generated" + step_started = "step_started" + step_progress = "step_progress" + tool_called = "tool_called" + tool_result = "tool_result" + llm_chunk = "llm_chunk" + memory_updated = "memory_updated" + retry_attempt = "retry_attempt" + step_completed = "step_completed" + warning = "warning" + error = "error" + task_completed = "task_completed" + task_failed = "task_failed" + heartbeat = "heartbeat" + + +class MemoryType(str, Enum): + conversation = "conversation" + task = "task" + project = "project" + execution = "execution" + tool = "tool" + error = "error" + repo = "repo" + planning = "planning" + + +# ─── Task Models ─────────────────────────────────────────────────────────────── + +class TaskCreateRequest(BaseModel): + goal: str = Field(..., min_length=1, max_length=10000, description="What should the agent do?") + session_id: str = Field(default_factory=lambda: gen_id("sess_")) + project_id: str = Field(default="") + stream: bool = True + metadata: Dict[str, Any] = Field(default_factory=dict) + github_repo: Optional[str] = None + auto_commit: bool = False + + +class TaskStep(BaseModel): + id: str = Field(default_factory=lambda: gen_id("step_")) + name: str + description: str = "" + tool: Optional[str] = None + status: str = "pending" + output: Optional[str] = None + error: Optional[str] = None + started_at: Optional[float] = None + completed_at: Optional[float] = None + duration_ms: Optional[float] = None + + +class TaskPlan(BaseModel): + goal: str + steps: List[TaskStep] + estimated_duration: int = 0 + tools_needed: List[str] = [] + created_at: float = Field(default_factory=time.time) + + +class TaskResponse(BaseModel): + id: str + goal: str + status: TaskStatus + session_id: str + project_id: str + plan: Optional[TaskPlan] = None + result: Optional[str] = None + error: Optional[str] = None + created_at: float + started_at: Optional[float] = None + completed_at: Optional[float] = None + retry_count: int = 0 + stream_url: Optional[str] = None + ws_url: Optional[str] = None + + +class TaskCancelRequest(BaseModel): + reason: str = "User cancelled" + + +class TaskRetryRequest(BaseModel): + reset_state: bool = True + + +# ─── Chat Models ─────────────────────────────────────────────────────────────── + +class ChatMessage(BaseModel): + role: str = Field(..., pattern="^(user|assistant|system)$") + content: str + timestamp: float = Field(default_factory=time.time) + + +class ChatRequest(BaseModel): + messages: List[ChatMessage] + session_id: str = Field(default_factory=lambda: gen_id("sess_")) + project_id: str = "" + stream: bool = True + model: str = "gpt-4o" + temperature: float = 0.7 + max_tokens: int = 4096 + system_prompt: Optional[str] = None + + +class GoalRequest(BaseModel): + goal: str = Field(..., min_length=1, max_length=10000) + session_id: str = Field(default_factory=lambda: gen_id("sess_")) + project_id: str = "" + stream: bool = True + auto_execute: bool = True + github_repo: Optional[str] = None + + +# ─── Memory Models ───────────────────────────────────────────────────────────── + +class MemorySaveRequest(BaseModel): + content: str + memory_type: MemoryType + session_id: str = "" + project_id: str = "" + key: str = "" + metadata: Dict[str, Any] = {} + + +class MemorySearchRequest(BaseModel): + query: str + session_id: str = "" + project_id: str = "" + limit: int = 20 + + +# ─── GitHub Models ───────────────────────────────────────────────────────────── + +class GitHubCloneRequest(BaseModel): + repo_url: str + branch: str = "main" + local_path: Optional[str] = None + + +class GitHubCreateRepoRequest(BaseModel): + name: str + description: str = "" + private: bool = False + auto_init: bool = True + + +class GitHubCommitRequest(BaseModel): + repo: str + branch: str = "main" + files: Dict[str, str] # path β†’ content + message: str + create_branch: bool = False + + +class GitHubPRRequest(BaseModel): + repo: str + title: str + body: str = "" + head: str + base: str = "main" + draft: bool = False + + +class GitHubIssueRequest(BaseModel): + repo: str + title: str + body: str = "" + labels: List[str] = [] + + +# ─── Event Schema (unified) ──────────────────────────────────────────────────── + +class StreamEvent(BaseModel): + type: str + task_id: str = "" + session_id: str = "" + timestamp: float = Field(default_factory=time.time) + data: Dict[str, Any] = {} diff --git a/core/orchestrator.py b/core/orchestrator.py deleted file mode 100644 index b37e0aeab673a89684632cec55d5ae7f3c870542..0000000000000000000000000000000000000000 --- a/core/orchestrator.py +++ /dev/null @@ -1,96 +0,0 @@ -import json -import asyncio -from typing import List, Dict, Any, Optional -from core.llm_router import llm_router -from core.memory import memory_engine -from task_manager import task_manager, TaskStatus - -class ToolRegistry: - def __init__(self): - self.tools = { - "github.clone": {"purpose": "Clone a GitHub repository", "params": ["url", "path"]}, - "github.create": {"purpose": "Create a new GitHub repository", "params": ["name", "private"]}, - "github.commit_push": {"purpose": "Commit and push changes", "params": ["path", "message", "branch"]}, - "workspace.read": {"purpose": "Read a file from workspace", "params": ["path"]}, - "workspace.write": {"purpose": "Write content to a file", "params": ["path", "content"]}, - "workspace.list": {"purpose": "List files in a directory", "params": ["path"]}, - "browser.open": {"purpose": "Open a URL in browser", "params": ["url"]}, - } - - def get_metadata(self) -> str: - return json.dumps(self.tools, indent=2) - -class Orchestrator: - def __init__(self): - self.registry = ToolRegistry() - - async def plan(self, goal: str) -> List[Dict[str, Any]]: - context = memory_engine.get_full_context() - tools_info = self.registry.get_metadata() - - prompt = f""" - Goal: {goal} - - Available Tools: - {tools_info} - - Previous Context: - {context} - - Task: Create a step-by-step plan to achieve the goal using the available tools. - Return the plan as a JSON list of objects: [{{"step": 1, "tool": "tool.name", "args": {{...}}, "description": "..."}}] - """ - - messages = [{"role": "system", "content": "You are an expert software engineer planner."}, - {"role": "user", "content": prompt}] - - response = await llm_router.chat_completion(messages) - content = response['choices'][0]['message']['content'] - - # Extract JSON from response - try: - plan = json.loads(content[content.find('['):content.rfind(']')+1]) - return plan - except: - # Fallback or retry logic - return [] - - async def execute_task(self, task_id: str): - task = task_manager.get_task(task_id) - if not task: return - - task.update(status=TaskStatus.RUNNING, progress=10) - memory_engine.save_goal(task.goal) - - plan = await self.plan(task.goal) - if not plan: - task.update(status=TaskStatus.FAILED, error="Failed to generate plan") - return - - memory_engine.save_plan(plan) - task.add_log(f"Plan generated: {len(plan)} steps") - - for i, step in enumerate(plan): - task.add_log(f"Executing step {step['step']}: {step['description']}") - - # In a real system, this would call the actual tool implementation - # For now, we simulate execution and update memory - try: - # Simulate tool call - result = {"status": "success", "step": step['step']} - memory_engine.save_tool_usage(step['tool'], step['args'], result) - memory_engine.save_execution(step['description'], result) - - progress = int(10 + (i + 1) / len(plan) * 80) - task.update(progress=progress) - except Exception as e: - task.add_log(f"Error in step {step['step']}: {str(e)}") - memory_engine.save_error(str(e), f"Step {step['step']}") - # Self-healing: Could re-plan here - task.update(status=TaskStatus.FAILED, error=str(e)) - return - - task.update(status=TaskStatus.COMPLETED, progress=100, result="Goal achieved successfully") - task.add_log("Task completed successfully") - -orchestrator = Orchestrator() diff --git a/core/task_engine.py b/core/task_engine.py new file mode 100644 index 0000000000000000000000000000000000000000..6eeb28d10be4e264994dc2b16beafb7d8b2492b6 --- /dev/null +++ b/core/task_engine.py @@ -0,0 +1,241 @@ +""" +Task Engine β€” Heart of the Autonomous Agent +Manages task lifecycle, planning, execution, self-healing +""" + +import asyncio +import json +import os +import time +import uuid +from typing import Dict, Optional, List + +import structlog + +from core.models import TaskStatus, TaskPlan, TaskStep, TaskCreateRequest +from api.websocket_manager import WebSocketManager +from memory.db import ( + create_task, update_task_status, get_task, save_task_event, + save_memory, get_task_events +) + +log = structlog.get_logger() + +MAX_RETRIES = 3 +MAX_CONCURRENT = 5 + + +class TaskEngine: + def __init__(self, ws_manager: WebSocketManager): + self.ws = ws_manager + self._queue: asyncio.Queue = asyncio.Queue() + self._active: Dict[str, asyncio.Task] = {} + self._running = False + self._workers: List[asyncio.Task] = [] + + async def start(self): + self._running = True + for i in range(MAX_CONCURRENT): + worker = asyncio.create_task(self._worker(i)) + self._workers.append(worker) + log.info("TaskEngine started", workers=MAX_CONCURRENT) + + async def stop(self): + self._running = False + for w in self._workers: + w.cancel() + log.info("TaskEngine stopped") + + # ─── Public API ──────────────────────────────────────────────────────────── + + async def submit(self, req: TaskCreateRequest) -> str: + task_id = f"task_{uuid.uuid4().hex[:10]}" + await create_task( + task_id=task_id, + goal=req.goal, + session_id=req.session_id, + project_id=req.project_id, + metadata={**req.metadata, "github_repo": req.github_repo, "auto_commit": req.auto_commit}, + ) + await self.ws.emit(task_id, "task_created", { + "goal": req.goal, + "session_id": req.session_id, + "stream_url": f"/api/v1/tasks/{task_id}/stream", + "ws_url": f"/ws/tasks/{task_id}", + }, session_id=req.session_id) + await self._queue.put((task_id, req)) + await self.ws.emit(task_id, "task_queued", { + "position": self._queue.qsize(), + }, session_id=req.session_id) + log.info("Task submitted", task_id=task_id, goal=req.goal[:60]) + return task_id + + async def cancel(self, task_id: str, reason: str = "User cancelled"): + if task_id in self._active: + self._active[task_id].cancel() + del self._active[task_id] + await update_task_status(task_id, "cancelled", error=reason) + await self.ws.emit(task_id, "task_failed", {"reason": reason, "status": "cancelled"}) + + async def retry(self, task_id: str): + task = await get_task(task_id) + if not task: + return + req = TaskCreateRequest( + goal=task["goal"], + session_id=task["session_id"] or "", + project_id=task["project_id"] or "", + metadata=task.get("metadata") or {}, + ) + retry_count = (task.get("retry_count") or 0) + 1 + await update_task_status(task_id, "queued", retry_count=retry_count) + await self.ws.emit(task_id, "retry_attempt", {"count": retry_count}) + await self._queue.put((task_id, req)) + + async def handle_chat_message(self, session_id: str, content: str, websocket=None): + """Handle real-time chat message with streaming response.""" + from core.agent import AgentCore + agent = AgentCore(self.ws) + await agent.stream_chat(session_id=session_id, user_message=content) + + # ─── Worker Loop ─────────────────────────────────────────────────────────── + + async def _worker(self, worker_id: int): + log.info(f"Worker {worker_id} started") + while self._running: + try: + task_id, req = await asyncio.wait_for(self._queue.get(), timeout=1.0) + worker_task = asyncio.create_task(self._execute(task_id, req)) + self._active[task_id] = worker_task + await worker_task + self._active.pop(task_id, None) + self._queue.task_done() + except asyncio.TimeoutError: + continue + except asyncio.CancelledError: + break + except Exception as e: + log.error(f"Worker {worker_id} error", error=str(e)) + + async def _execute(self, task_id: str, req: TaskCreateRequest): + """Full task execution lifecycle.""" + from core.agent import AgentCore + agent = AgentCore(self.ws) + + try: + # ── Initializing ──────────────────────────────────────────────── + await update_task_status(task_id, "initializing") + await self.ws.emit(task_id, "task_started", { + "goal": req.goal, + "status": "initializing", + }, session_id=req.session_id) + await save_task_event(task_id, "task_started", {"goal": req.goal}) + + # ── Planning ──────────────────────────────────────────────────── + await update_task_status(task_id, "planning") + await self.ws.emit(task_id, "step_started", { + "step": "Planning", + "status": "planning", + "description": "Generating execution plan...", + }, session_id=req.session_id) + + plan = await agent.plan(goal=req.goal, task_id=task_id, session_id=req.session_id) + + await update_task_status(task_id, "executing", plan=plan.model_dump()) + await self.ws.emit(task_id, "plan_generated", { + "steps": [s.model_dump() for s in plan.steps], + "estimated_duration": plan.estimated_duration, + "tools_needed": plan.tools_needed, + }, session_id=req.session_id) + await save_task_event(task_id, "plan_generated", {"steps_count": len(plan.steps)}) + + # ── Execute Steps ──────────────────────────────────────────────── + results = [] + for i, step in enumerate(plan.steps): + await self.ws.emit(task_id, "step_started", { + "step": step.name, + "step_id": step.id, + "index": i, + "total": len(plan.steps), + "tool": step.tool, + }, session_id=req.session_id) + + step_result = await agent.execute_step( + step=step, + task_id=task_id, + session_id=req.session_id, + context={"goal": req.goal, "previous_results": results}, + ) + results.append(step_result) + + await self.ws.emit(task_id, "step_completed", { + "step": step.name, + "step_id": step.id, + "index": i, + "output": step_result[:500] if isinstance(step_result, str) else str(step_result)[:500], + "status": "completed", + }, session_id=req.session_id) + await save_task_event(task_id, "step_completed", {"step": step.name, "index": i}) + + # ── Finalize ───────────────────────────────────────────────────── + await update_task_status(task_id, "finalizing") + await self.ws.emit(task_id, "step_started", { + "step": "Finalizing", + "description": "Compiling results...", + }, session_id=req.session_id) + + final_result = await agent.finalize( + goal=req.goal, + steps=plan.steps, + results=results, + task_id=task_id, + session_id=req.session_id, + ) + + await update_task_status(task_id, "completed", result=final_result) + await self.ws.emit(task_id, "task_completed", { + "result": final_result, + "steps_completed": len(plan.steps), + "duration": time.time(), + }, session_id=req.session_id) + + # Save to memory + await save_memory( + content=f"Task: {req.goal}\nResult: {final_result}", + memory_type="task", + session_id=req.session_id, + project_id=req.project_id, + key=task_id, + ) + await self.ws.emit(task_id, "memory_updated", { + "type": "task", + "key": task_id, + }, session_id=req.session_id) + + log.info("Task completed", task_id=task_id) + + except asyncio.CancelledError: + await update_task_status(task_id, "cancelled") + await self.ws.emit(task_id, "task_failed", {"reason": "cancelled"}) + except Exception as e: + log.error("Task failed", task_id=task_id, error=str(e)) + task_data = await get_task(task_id) + retry_count = (task_data or {}).get("retry_count", 0) + + await self.ws.emit(task_id, "error", { + "error": str(e), + "retry_count": retry_count, + "will_retry": retry_count < MAX_RETRIES, + }, session_id=req.session_id) + + if retry_count < MAX_RETRIES: + await update_task_status(task_id, "retrying", retry_count=retry_count + 1) + await asyncio.sleep(2 ** retry_count) + await self.ws.emit(task_id, "retry_attempt", {"count": retry_count + 1}) + await self._execute(task_id, req) + else: + await update_task_status(task_id, "failed", error=str(e)) + await self.ws.emit(task_id, "task_failed", { + "error": str(e), + "retry_count": retry_count, + }, session_id=req.session_id) diff --git a/ecosystem.config.cjs b/ecosystem.config.cjs new file mode 100644 index 0000000000000000000000000000000000000000..ddba4e6b870059011d68d1ed7ec67d06c1a86771 --- /dev/null +++ b/ecosystem.config.cjs @@ -0,0 +1,20 @@ +module.exports = { + apps: [ + { + name: 'devin-backend', + script: 'uvicorn', + args: 'main:app --host 0.0.0.0 --port 7860 --loop asyncio --log-level info', + interpreter: 'python3', + cwd: '/home/user/devin-agent/backend', + watch: false, + instances: 1, + exec_mode: 'fork', + env: { + PORT: 7860, + HOST: '0.0.0.0', + DB_PATH: '/tmp/devin_agent.db', + PYTHONUNBUFFERED: '1', + }, + }, + ], +} diff --git a/root/etc/s6-overlay/s6-rc.d/init-openvscode-server/dependencies.d/init-config b/github/__init__.py similarity index 100% rename from root/etc/s6-overlay/s6-rc.d/init-openvscode-server/dependencies.d/init-config rename to github/__init__.py diff --git a/main.py b/main.py new file mode 100644 index 0000000000000000000000000000000000000000..6202e92b7b308de38693263dcb15fb815df68418 --- /dev/null +++ b/main.py @@ -0,0 +1,180 @@ +""" +πŸš€ Devin-Style Autonomous AI Engineering Platform +Production-Grade FastAPI + WebSocket Backend +""" + +import asyncio +import json +import logging +import os +import time +import uuid +from contextlib import asynccontextmanager +from typing import Optional + +import structlog +from fastapi import FastAPI, WebSocket, WebSocketDisconnect, HTTPException, Depends, Request +from fastapi.middleware.cors import CORSMiddleware +from fastapi.middleware.gzip import GZipMiddleware +from fastapi.responses import JSONResponse +from slowapi import Limiter, _rate_limit_exceeded_handler +from slowapi.util import get_remote_address +from slowapi.errors import RateLimitExceeded + +from api.routes import tasks, chat, memory, github, health +from api.websocket_manager import WebSocketManager +from core.task_engine import TaskEngine +from memory.db import init_db + +# ─── Structured Logging ──────────────────────────────────────────────────────── +structlog.configure( + processors=[ + structlog.processors.TimeStamper(fmt="iso"), + structlog.stdlib.add_log_level, + structlog.processors.StackInfoRenderer(), + structlog.dev.ConsoleRenderer(), + ] +) +log = structlog.get_logger() + +# ─── Rate Limiter ────────────────────────────────────────────────────────────── +limiter = Limiter(key_func=get_remote_address) + +# ─── Global Managers (shared state) ─────────────────────────────────────────── +ws_manager = WebSocketManager() +task_engine = TaskEngine(ws_manager) + + +@asynccontextmanager +async def lifespan(app: FastAPI): + """Startup + Shutdown lifecycle.""" + log.info("πŸš€ Starting Devin Agent Platform...") + await init_db() + await task_engine.start() + asyncio.create_task(ws_manager.heartbeat_loop()) + log.info("βœ… Platform ready") + yield + log.info("πŸ›‘ Shutting down...") + await task_engine.stop() + log.info("βœ… Shutdown complete") + + +# ─── FastAPI App ─────────────────────────────────────────────────────────────── +app = FastAPI( + title="πŸ€– Devin Agent Platform", + description="Production-Grade Autonomous AI Engineering Platform", + version="2.0.0", + lifespan=lifespan, + docs_url="/api/docs", + redoc_url="/api/redoc", +) + +app.state.limiter = limiter +app.add_exception_handler(RateLimitExceeded, _rate_limit_exceeded_handler) + +# ─── Middleware ──────────────────────────────────────────────────────────────── +app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) +app.add_middleware(GZipMiddleware, minimum_size=1000) + + +# ─── Request Logging ─────────────────────────────────────────────────────────── +@app.middleware("http") +async def log_requests(request: Request, call_next): + start = time.time() + response = await call_next(request) + duration = round((time.time() - start) * 1000, 2) + log.info("HTTP", method=request.method, path=request.url.path, status=response.status_code, ms=duration) + return response + + +# ─── Inject shared state into routes ────────────────────────────────────────── +app.state.ws_manager = ws_manager +app.state.task_engine = task_engine + + +# ─── REST API Routers ────────────────────────────────────────────────────────── +app.include_router(health.router, prefix="/api/v1", tags=["health"]) +app.include_router(tasks.router, prefix="/api/v1/tasks", tags=["tasks"]) +app.include_router(chat.router, prefix="/api/v1", tags=["chat"]) +app.include_router(memory.router, prefix="/api/v1/memory", tags=["memory"]) +app.include_router(github.router, prefix="/api/v1/github", tags=["github"]) + + +# ─── WebSocket Endpoints ─────────────────────────────────────────────────────── +@app.websocket("/ws/tasks/{task_id}") +async def ws_task(websocket: WebSocket, task_id: str): + """Live streaming for specific task execution.""" + await ws_manager.connect(websocket, room=f"task:{task_id}") + try: + while True: + data = await websocket.receive_text() + msg = json.loads(data) + if msg.get("type") == "ping": + await websocket.send_json({"type": "pong", "timestamp": time.time()}) + except WebSocketDisconnect: + ws_manager.disconnect(websocket, room=f"task:{task_id}") + + +@app.websocket("/ws/logs") +async def ws_logs(websocket: WebSocket): + """Global live log stream.""" + await ws_manager.connect(websocket, room="logs") + try: + while True: + data = await websocket.receive_text() + msg = json.loads(data) + if msg.get("type") == "ping": + await websocket.send_json({"type": "pong", "timestamp": time.time()}) + except WebSocketDisconnect: + ws_manager.disconnect(websocket, room="logs") + + +@app.websocket("/ws/chat/{session_id}") +async def ws_chat(websocket: WebSocket, session_id: str): + """Real-time chat streaming per session.""" + await ws_manager.connect(websocket, room=f"chat:{session_id}") + try: + while True: + data = await websocket.receive_text() + msg = json.loads(data) + if msg.get("type") == "ping": + await websocket.send_json({"type": "pong", "timestamp": time.time()}) + elif msg.get("type") == "chat_message": + # Trigger streaming chat response + asyncio.create_task( + task_engine.handle_chat_message(session_id, msg.get("content", ""), websocket) + ) + except WebSocketDisconnect: + ws_manager.disconnect(websocket, room=f"chat:{session_id}") + + +@app.websocket("/ws/agent/status") +async def ws_agent_status(websocket: WebSocket): + """Global agent status stream.""" + await ws_manager.connect(websocket, room="agent_status") + try: + while True: + data = await websocket.receive_text() + msg = json.loads(data) + if msg.get("type") == "ping": + await websocket.send_json({"type": "pong", "timestamp": time.time()}) + except WebSocketDisconnect: + ws_manager.disconnect(websocket, room="agent_status") + + +# ─── Root ────────────────────────────────────────────────────────────────────── +@app.get("/") +async def root(): + return { + "name": "πŸ€– Devin Agent Platform", + "version": "2.0.0", + "status": "operational", + "docs": "/api/docs", + "websockets": ["/ws/tasks/{task_id}", "/ws/logs", "/ws/chat/{session_id}", "/ws/agent/status"], + } diff --git a/root/etc/s6-overlay/s6-rc.d/svc-openvscode-server/dependencies.d/init-services b/memory/__init__.py similarity index 100% rename from root/etc/s6-overlay/s6-rc.d/svc-openvscode-server/dependencies.d/init-services rename to memory/__init__.py diff --git a/memory/__pycache__/__init__.cpython-312.pyc b/memory/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1952a0ff6a2a22048f4412f42cc0f824a14104de Binary files /dev/null and b/memory/__pycache__/__init__.cpython-312.pyc differ diff --git a/memory/__pycache__/db.cpython-312.pyc b/memory/__pycache__/db.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5592e627062b9e4e02c1ac97ff7f9b8f152e8b15 Binary files /dev/null and b/memory/__pycache__/db.cpython-312.pyc differ diff --git a/memory/db.py b/memory/db.py new file mode 100644 index 0000000000000000000000000000000000000000..68ae8bfba79386be88580dd8e5efcc89714c97d8 --- /dev/null +++ b/memory/db.py @@ -0,0 +1,271 @@ +""" +Production SQLite Database β€” Async via aiosqlite +Handles tasks, memory, sessions, events +""" + +import aiosqlite +import os +import json +import time +from typing import Optional, List, Dict, Any +import structlog + +log = structlog.get_logger() + +DB_PATH = os.environ.get("DB_PATH", "/tmp/devin_agent.db") + + +async def get_db() -> aiosqlite.Connection: + db = await aiosqlite.connect(DB_PATH) + db.row_factory = aiosqlite.Row + await db.execute("PRAGMA journal_mode=WAL") + await db.execute("PRAGMA foreign_keys=ON") + return db + + +async def init_db(): + """Initialize all tables.""" + log.info("Initializing database", path=DB_PATH) + async with aiosqlite.connect(DB_PATH) as db: + await db.execute("PRAGMA journal_mode=WAL") + await db.execute("PRAGMA foreign_keys=ON") + + # Tasks table + await db.execute(""" + CREATE TABLE IF NOT EXISTS tasks ( + id TEXT PRIMARY KEY, + session_id TEXT, + project_id TEXT, + goal TEXT NOT NULL, + status TEXT DEFAULT 'queued', + plan TEXT, + result TEXT, + error TEXT, + metadata TEXT DEFAULT '{}', + created_at REAL, + started_at REAL, + completed_at REAL, + retry_count INTEGER DEFAULT 0 + ) + """) + + # Task events table + await db.execute(""" + CREATE TABLE IF NOT EXISTS task_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + task_id TEXT NOT NULL, + event_type TEXT NOT NULL, + data TEXT DEFAULT '{}', + timestamp REAL, + FOREIGN KEY (task_id) REFERENCES tasks(id) + ) + """) + + # Memory table + await db.execute(""" + CREATE TABLE IF NOT EXISTS memory ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + session_id TEXT, + project_id TEXT, + memory_type TEXT NOT NULL, + key TEXT, + content TEXT NOT NULL, + metadata TEXT DEFAULT '{}', + embedding TEXT, + created_at REAL, + updated_at REAL + ) + """) + + # Sessions table + await db.execute(""" + CREATE TABLE IF NOT EXISTS sessions ( + id TEXT PRIMARY KEY, + project_id TEXT, + user_id TEXT, + metadata TEXT DEFAULT '{}', + created_at REAL, + last_active REAL + ) + """) + + # GitHub operations table + await db.execute(""" + CREATE TABLE IF NOT EXISTS github_ops ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + task_id TEXT, + operation TEXT NOT NULL, + repo TEXT, + branch TEXT, + status TEXT DEFAULT 'pending', + result TEXT, + created_at REAL + ) + """) + + # Indexes + await db.execute("CREATE INDEX IF NOT EXISTS idx_tasks_session ON tasks(session_id)") + await db.execute("CREATE INDEX IF NOT EXISTS idx_tasks_status ON tasks(status)") + await db.execute("CREATE INDEX IF NOT EXISTS idx_events_task ON task_events(task_id)") + await db.execute("CREATE INDEX IF NOT EXISTS idx_memory_session ON memory(session_id)") + await db.execute("CREATE INDEX IF NOT EXISTS idx_memory_project ON memory(project_id)") + await db.execute("CREATE INDEX IF NOT EXISTS idx_memory_type ON memory(memory_type)") + + await db.commit() + log.info("βœ… Database initialized") + + +# ─── Task CRUD ───────────────────────────────────────────────────────────────── + +async def create_task(task_id: str, goal: str, session_id: str = "", project_id: str = "", metadata: dict = {}): + async with aiosqlite.connect(DB_PATH) as db: + await db.execute(""" + INSERT INTO tasks (id, session_id, project_id, goal, status, metadata, created_at) + VALUES (?, ?, ?, ?, 'queued', ?, ?) + """, (task_id, session_id, project_id, goal, json.dumps(metadata), time.time())) + await db.commit() + + +async def update_task_status(task_id: str, status: str, **kwargs): + fields = ["status = ?"] + values = [status] + if status == "executing": + fields.append("started_at = ?") + values.append(time.time()) + if status in ("completed", "failed", "cancelled"): + fields.append("completed_at = ?") + values.append(time.time()) + for k, v in kwargs.items(): + if k in ("plan", "result", "error"): + fields.append(f"{k} = ?") + values.append(v if isinstance(v, str) else json.dumps(v)) + elif k == "retry_count": + fields.append("retry_count = ?") + values.append(v) + values.append(task_id) + async with aiosqlite.connect(DB_PATH) as db: + await db.execute(f"UPDATE tasks SET {', '.join(fields)} WHERE id = ?", values) + await db.commit() + + +async def get_task(task_id: str) -> Optional[Dict]: + async with aiosqlite.connect(DB_PATH) as db: + db.row_factory = aiosqlite.Row + async with db.execute("SELECT * FROM tasks WHERE id = ?", (task_id,)) as cursor: + row = await cursor.fetchone() + if row: + d = dict(row) + d["metadata"] = json.loads(d.get("metadata") or "{}") + d["plan"] = json.loads(d["plan"]) if d.get("plan") else None + return d + return None + + +async def list_tasks(session_id: str = "", limit: int = 50) -> List[Dict]: + async with aiosqlite.connect(DB_PATH) as db: + db.row_factory = aiosqlite.Row + if session_id: + async with db.execute( + "SELECT * FROM tasks WHERE session_id = ? ORDER BY created_at DESC LIMIT ?", + (session_id, limit) + ) as cursor: + rows = await cursor.fetchall() + else: + async with db.execute( + "SELECT * FROM tasks ORDER BY created_at DESC LIMIT ?", (limit,) + ) as cursor: + rows = await cursor.fetchall() + return [dict(r) for r in rows] + + +async def save_task_event(task_id: str, event_type: str, data: dict = {}): + async with aiosqlite.connect(DB_PATH) as db: + await db.execute(""" + INSERT INTO task_events (task_id, event_type, data, timestamp) + VALUES (?, ?, ?, ?) + """, (task_id, event_type, json.dumps(data), time.time())) + await db.commit() + + +async def get_task_events(task_id: str) -> List[Dict]: + async with aiosqlite.connect(DB_PATH) as db: + db.row_factory = aiosqlite.Row + async with db.execute( + "SELECT * FROM task_events WHERE task_id = ? ORDER BY timestamp ASC", (task_id,) + ) as cursor: + rows = await cursor.fetchall() + return [dict(r) for r in rows] + + +# ─── Memory CRUD ─────────────────────────────────────────────────────────────── + +async def save_memory( + content: str, + memory_type: str, + session_id: str = "", + project_id: str = "", + key: str = "", + metadata: dict = {} +): + now = time.time() + async with aiosqlite.connect(DB_PATH) as db: + await db.execute(""" + INSERT INTO memory (session_id, project_id, memory_type, key, content, metadata, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + """, (session_id, project_id, memory_type, key, content, json.dumps(metadata), now, now)) + await db.commit() + + +async def search_memory(query: str, session_id: str = "", project_id: str = "", limit: int = 20) -> List[Dict]: + """Simple keyword search (upgrade to vector search in production).""" + async with aiosqlite.connect(DB_PATH) as db: + db.row_factory = aiosqlite.Row + q = f"%{query}%" + if session_id: + async with db.execute( + "SELECT * FROM memory WHERE session_id = ? AND content LIKE ? ORDER BY updated_at DESC LIMIT ?", + (session_id, q, limit) + ) as cursor: + rows = await cursor.fetchall() + elif project_id: + async with db.execute( + "SELECT * FROM memory WHERE project_id = ? AND content LIKE ? ORDER BY updated_at DESC LIMIT ?", + (project_id, q, limit) + ) as cursor: + rows = await cursor.fetchall() + else: + async with db.execute( + "SELECT * FROM memory WHERE content LIKE ? ORDER BY updated_at DESC LIMIT ?", + (q, limit) + ) as cursor: + rows = await cursor.fetchall() + return [dict(r) for r in rows] + + +async def get_project_memory(project_id: str, memory_type: str = "", limit: int = 100) -> List[Dict]: + async with aiosqlite.connect(DB_PATH) as db: + db.row_factory = aiosqlite.Row + if memory_type: + async with db.execute( + "SELECT * FROM memory WHERE project_id = ? AND memory_type = ? ORDER BY updated_at DESC LIMIT ?", + (project_id, memory_type, limit) + ) as cursor: + rows = await cursor.fetchall() + else: + async with db.execute( + "SELECT * FROM memory WHERE project_id = ? ORDER BY updated_at DESC LIMIT ?", + (project_id, limit) + ) as cursor: + rows = await cursor.fetchall() + return [dict(r) for r in rows] + + +async def get_history(session_id: str, limit: int = 50) -> List[Dict]: + async with aiosqlite.connect(DB_PATH) as db: + db.row_factory = aiosqlite.Row + async with db.execute( + "SELECT * FROM memory WHERE session_id = ? AND memory_type = 'conversation' ORDER BY created_at DESC LIMIT ?", + (session_id, limit) + ) as cursor: + rows = await cursor.fetchall() + return [dict(r) for r in rows] diff --git a/nginx.conf b/nginx.conf deleted file mode 100644 index a1fa501fc1352556f9c2a6c65e3c8ab3b43fcb11..0000000000000000000000000000000000000000 --- a/nginx.conf +++ /dev/null @@ -1,129 +0,0 @@ -error_log /tmp/error.log warn; -worker_processes auto; -pid /tmp/nginx.pid; -include /etc/nginx/modules-enabled/*.conf; - -events { - worker_connections 768; - multi_accept on; -} - -http { - ## - # Basic Settings - ## - - sendfile on; - tcp_nopush on; - tcp_nodelay on; - keepalive_timeout 65; - types_hash_max_size 2048; - proxy_buffering off; - client_max_body_size 800m; - large_client_header_buffers 4 32k; - # server_tokens off; - - # server_names_hash_bucket_size 64; - # server_name_in_redirect off; - - include /etc/nginx/mime.types; - - default_type application/octet-stream; - proxy_temp_path /tmp/proxy_temp; - client_body_temp_path /tmp/client_temp; - fastcgi_temp_path /tmp/fastcgi_temp; - uwsgi_temp_path /tmp/uwsgi_temp; - scgi_temp_path /tmp/scgi_temp; - - ## - # SSL Settings - ## - - ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3; # Dropping SSLv3, ref: POODLE - ssl_prefer_server_ciphers on; - - - ## - # Gzip Settings - ## - - gzip on; - - # gzip_vary on; - # gzip_proxied any; - # gzip_comp_level 6; - # gzip_buffers 16 8k; - # gzip_http_version 1.1; - # gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript; - - ## - # Virtual Host Configs - ## - - #include /etc/nginx/conf.d/*.conf; - #include /etc/nginx/sites-enabled/*; - server { - listen 7860; - - access_log /tmp/access.log; - server_name _; - - root /var/www/; - index index.html; - location /stable-#COMMIT# { - proxy_pass http://127.0.0.1:5050; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "Upgrade"; - proxy_set_header Host $host; - proxy_read_timeout 86400; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } - - location /vscode/ { - auth_basic "Restricted Content"; - auth_basic_user_file /home/user/app/ngpasswd; - proxy_pass http://127.0.0.1:5050/; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "Upgrade"; - proxy_set_header Host $host; - proxy_read_timeout 86400; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } - - location @vscode { - return 302 https://$host/vscode/?folder=/home/user/app; - } - - error_page 502 = @vscode; - location /api/ { - proxy_pass http://127.0.0.1:8000/api/; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "Upgrade"; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } - - location /ws/ { - proxy_pass http://127.0.0.1:8000/ws/; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "Upgrade"; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } - - location / { - proxy_pass http://127.0.0.1:#PORT#; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "Upgrade"; - proxy_set_header Host $host; - proxy_read_timeout 86400; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - #try_files $uri $uri/ =404; - } - } -} \ No newline at end of file diff --git a/on_startup.sh b/on_startup.sh deleted file mode 100644 index b72979be97072eb338a6a97ab4a1e31eabcc8c3a..0000000000000000000000000000000000000000 --- a/on_startup.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -# Write some commands here that will run on root user before startup. -# For example, to clone transformers and install it in dev mode: -# git clone https://github.com/huggingface/transformers.git -# cd transformers && pip install -e ".[dev]" - -npm i -g tsx tslab http-server miniflare@2 -sudo chown -R 1000:1000 "/home/user/" -tslab install - -echo ' -# >>> conda initialize >>> -__conda_setup="$(/home/user/miniconda/bin/conda shell.bash hook 2> /dev/null)" -if [ $? -eq 0 ]; then - eval "$__conda_setup" -else - if [ -f "/home/user/miniconda/etc/profile.d/conda.sh" ]; then - . "/home/user/miniconda/etc/profile.d/conda.sh" - else - export PATH="/home/user/miniconda/bin:$PATH" - fi -fi -unset __conda_setup -# <<< conda initialize <<< -' >> ~/.bashrc - -apt-config dump | grep Sandbox::User -cat < /etc/apt/apt.conf.d/sandbox-disable -APT::Sandbox::User "root"; -EOF -sudo chown -R 1000:1000 "/usr/" \ No newline at end of file diff --git a/packages.txt b/packages.txt deleted file mode 100644 index dc9cbd9e807a5b61cc8c7a841ea3e1849ea78e14..0000000000000000000000000000000000000000 --- a/packages.txt +++ /dev/null @@ -1 +0,0 @@ -tree \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 896effb86ee572fe9f863866b29ec8ebd952ac28..c2dc8344489a23accea6972db25fe35a2feacfa8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,12 +1,28 @@ -# jupyterlab -jupyterlab==3.6.1 -jupyter-server==2.3.0 -fastapi -uvicorn -python-jose[cryptography] -python-multipart -pydantic -websockets -requests -aiohttp[speedups] -brotli +fastapi==0.111.0 +uvicorn[standard]==0.29.0 +websockets==12.0 +pydantic==2.7.1 +pydantic-settings==2.2.1 +python-jose[cryptography]==3.3.0 +python-multipart==0.0.9 +aiohttp==3.9.5 +aiosqlite==0.20.0 +sqlalchemy[asyncio]==2.0.30 +alembic==1.13.1 +httpx==0.27.0 +openai==1.30.1 +anthropic==0.26.1 +gitpython==3.1.43 +pygithub==2.3.0 +python-dotenv==1.0.1 +slowapi==0.1.9 +structlog==24.1.0 +rich==13.7.1 +asyncio-mqtt==0.16.2 +redis==5.0.4 +celery==5.3.6 +passlib[bcrypt]==1.7.4 +cryptography==42.0.7 +typer==0.12.3 +watchfiles==0.21.0 +psutil==5.9.8 diff --git a/root/etc/s6-overlay/s6-rc.d/init-openvscode-server/run b/root/etc/s6-overlay/s6-rc.d/init-openvscode-server/run deleted file mode 100644 index 09d9df46d718a8708cba0b9d4490e1c05fad7249..0000000000000000000000000000000000000000 --- a/root/etc/s6-overlay/s6-rc.d/init-openvscode-server/run +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/with-contenv bash - -mkdir -p /config/{workspace,.ssh} - -if [ -n "${SUDO_PASSWORD}" ] || [ -n "${SUDO_PASSWORD_HASH}" ]; then - echo "setting up sudo access" - if ! grep -q 'abc' /etc/sudoers; then - echo "adding abc to sudoers" - echo "abc ALL=(ALL:ALL) ALL" >> /etc/sudoers - fi - if [ -n "${SUDO_PASSWORD_HASH}" ]; then - echo "setting sudo password using sudo password hash" - sed -i "s|^abc:\!:|abc:${SUDO_PASSWORD_HASH}:|" /etc/shadow - else - echo "setting sudo password using SUDO_PASSWORD env var" - echo -e "${SUDO_PASSWORD}\n${SUDO_PASSWORD}" | passwd abc - fi -fi - -[[ ! -f /config/.bashrc ]] && \ - cp /root/.bashrc /config/.bashrc -[[ ! -f /config/.profile ]] && \ - cp /root/.profile /config/.profile - -# fix permissions (ignore contents of /config/workspace) -echo "setting permissions::config" -find /config -path /config/workspace -prune -o -exec chown abc:abc {} + -chown abc:abc /config/workspace -echo "setting permissions::app" -chown -R abc:abc /app/openvscode-server - -chmod 700 /config/.ssh -if [ -n "$(ls -A /config/.ssh)" ]; then - chmod 600 /config/.ssh/* -fi diff --git a/root/etc/s6-overlay/s6-rc.d/init-openvscode-server/type b/root/etc/s6-overlay/s6-rc.d/init-openvscode-server/type deleted file mode 100644 index 3d92b15f2d56c7753feb51fd035d8c490de86bd7..0000000000000000000000000000000000000000 --- a/root/etc/s6-overlay/s6-rc.d/init-openvscode-server/type +++ /dev/null @@ -1 +0,0 @@ -oneshot \ No newline at end of file diff --git a/root/etc/s6-overlay/s6-rc.d/init-openvscode-server/up b/root/etc/s6-overlay/s6-rc.d/init-openvscode-server/up deleted file mode 100644 index 823fdbd957141ff29d6bed36476522449c4364e9..0000000000000000000000000000000000000000 --- a/root/etc/s6-overlay/s6-rc.d/init-openvscode-server/up +++ /dev/null @@ -1 +0,0 @@ -/etc/s6-overlay/s6-rc.d/init-openvscode-server/run \ No newline at end of file diff --git a/root/etc/s6-overlay/s6-rc.d/svc-openvscode-server/notification-fd b/root/etc/s6-overlay/s6-rc.d/svc-openvscode-server/notification-fd deleted file mode 100644 index e440e5c842586965a7fb77deda2eca68612b1f53..0000000000000000000000000000000000000000 --- a/root/etc/s6-overlay/s6-rc.d/svc-openvscode-server/notification-fd +++ /dev/null @@ -1 +0,0 @@ -3 \ No newline at end of file diff --git a/root/etc/s6-overlay/s6-rc.d/svc-openvscode-server/run b/root/etc/s6-overlay/s6-rc.d/svc-openvscode-server/run deleted file mode 100644 index 25c33ba2955939139c1a4171432bf69b80e4d54a..0000000000000000000000000000000000000000 --- a/root/etc/s6-overlay/s6-rc.d/svc-openvscode-server/run +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/with-contenv bash - -if [ -n "$CONNECTION_SECRET" ]; then - CODE_ARGS="${CODE_ARGS} --connection-secret ${CONNECTION_SECRET}" - echo "Using connection secret from ${CONNECTION_SECRET}" -elif [ -n "$CONNECTION_TOKEN" ]; then - CODE_ARGS="${CODE_ARGS} --connection-token ${CONNECTION_TOKEN}" - echo "Using connection token ${CONNECTION_TOKEN}" -else - CODE_ARGS="${CODE_ARGS} --without-connection-token" - echo "**** No connection token is set ****" -fi - -exec \ - s6-notifyoncheck -d -n 300 -w 1000 -c "nc -z 127.0.0.1 3000" \ - cd /app/openvscode-server s6-setuidgid abc \ - /app/openvscode-server/bin/openvscode-server \ - --host 0.0.0.0 \ - --port 3000 \ - --disable-telemetry \ - ${CODE_ARGS} diff --git a/root/etc/s6-overlay/s6-rc.d/svc-openvscode-server/type b/root/etc/s6-overlay/s6-rc.d/svc-openvscode-server/type deleted file mode 100644 index 1780f9f44efd7a9a5240468e2d3d851ae5b7a471..0000000000000000000000000000000000000000 --- a/root/etc/s6-overlay/s6-rc.d/svc-openvscode-server/type +++ /dev/null @@ -1 +0,0 @@ -longrun \ No newline at end of file diff --git a/root/etc/s6-overlay/s6-rc.d/user/contents.d/svc-openvscode-server b/root/etc/s6-overlay/s6-rc.d/user/contents.d/svc-openvscode-server deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/root/usr/local/bin/install-extension b/root/usr/local/bin/install-extension deleted file mode 100644 index a95290be5801afbec981402d51affc8059371b95..0000000000000000000000000000000000000000 --- a/root/usr/local/bin/install-extension +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/with-contenv bash -# shellcheck shell=bash - -_install=(/app/openvscode-server/bin/openvscode-server "--install-extension") - -if [ "$(whoami)" == "abc" ]; then - "${_install[@]}" "$@" -else - s6-setuidgid abc "${_install[@]}" "$@" -fi diff --git a/routes/__init__.py b/routes/__init__.py deleted file mode 100644 index d212dab603147209ac855bcc695bf0a3e23636c1..0000000000000000000000000000000000000000 --- a/routes/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# Routes package diff --git a/routes/browser.py b/routes/browser.py deleted file mode 100644 index f91b42097adf7759919ae601ce73a9d4b0c54f1c..0000000000000000000000000000000000000000 --- a/routes/browser.py +++ /dev/null @@ -1,20 +0,0 @@ -from fastapi import APIRouter, HTTPException -from pydantic import BaseModel - -router = APIRouter() - -@router.post("/browser/open") -async def open_browser(url: str): - return {"status": "opened", "url": url} - -@router.post("/browser/click") -async def click_element(selector: str): - return {"status": "clicked", "selector": selector} - -@router.post("/browser/type") -async def type_text(selector: str, text: str): - return {"status": "typed", "selector": selector} - -@router.get("/browser/screenshot") -async def take_screenshot(): - return {"screenshot_url": "http://example.com/screenshot.png"} diff --git a/routes/chat.py b/routes/chat.py deleted file mode 100644 index 34d3c176635b6773b65aa70b73d8177e8c9ff9c7..0000000000000000000000000000000000000000 --- a/routes/chat.py +++ /dev/null @@ -1,55 +0,0 @@ -from fastapi import APIRouter, HTTPException, BackgroundTasks -from pydantic import BaseModel -from typing import List, Optional, Dict, Any -from core.llm_router import llm_router -from core.memory import memory_engine -from core.orchestrator import orchestrator -from task_manager import task_manager - -router = APIRouter() - -class ChatMessage(BaseModel): - role: str - content: str - -class ChatRequest(BaseModel): - messages: List[ChatMessage] - model: str = "gpt-4" - stream: bool = False - project_id: str = "default" - -class GoalRequest(BaseModel): - goal: str - project_id: str = "default" - -@router.post("/chat") -async def chat(request: ChatRequest): - # Load memory context - memory_engine.project_id = request.project_id - context = memory_engine.get_full_context() - - # Inject memory into messages - messages = [m.dict() for m in request.messages] - if context: - messages.insert(0, {"role": "system", "content": f"Previous memory context:\n{context}"}) - - try: - response = await llm_router.chat_completion(messages, model=request.model, stream=request.stream) - return response - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) - -@router.post("/goal") -async def receive_goal(request: GoalRequest, background_tasks: BackgroundTasks): - task = task_manager.create_task(request.goal) - memory_engine.project_id = request.project_id - - # Run orchestration in background - background_tasks.add_task(orchestrator.execute_task, task.id) - - return {"status": "goal_received", "task_id": task.id} - -@router.get("/memory/{project_id}") -async def get_memory_endpoint(project_id: str): - memory_engine.project_id = project_id - return {"project_id": project_id, "memories": memory_engine.get_recent_memories()} diff --git a/routes/github.py b/routes/github.py deleted file mode 100644 index c6fbda38409d19dd74ebe0937620b1900483bcec..0000000000000000000000000000000000000000 --- a/routes/github.py +++ /dev/null @@ -1,67 +0,0 @@ -from fastapi import APIRouter, HTTPException, Depends -from pydantic import BaseModel -from typing import Optional, List -from core.github_engine import github_engine -from auth import get_api_key - -router = APIRouter() - -class RepoClone(BaseModel): - url: str - path: str - -class RepoCreate(BaseModel): - name: str - private: bool = True - -class CommitPush(BaseModel): - path: str - message: str - branch: str = "main" - -class PRCreate(BaseModel): - repo_full_name: str - title: str - body: str - head: str - base: str = "main" - -@router.post("/github/clone") -async def clone_repo(repo: RepoClone): - try: - github_engine.clone(repo.url, repo.path) - return {"status": "success", "message": f"Cloned to {repo.path}"} - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) - -@router.post("/github/create") -async def create_repo(repo: RepoCreate): - try: - result = github_engine.create_repo(repo.name, repo.private) - return {"status": "success", "data": result} - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) - -@router.post("/github/commit-push") -async def commit_push(data: CommitPush): - try: - github_engine.commit_and_push(data.path, data.message, data.branch) - return {"status": "success"} - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) - -@router.post("/github/pr/create") -async def create_pull_request(pr: PRCreate): - try: - result = github_engine.create_pr(pr.repo_full_name, pr.title, pr.body, pr.head, pr.base) - return {"status": "success", "data": result} - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) - -@router.get("/github/issues/{owner}/{repo}") -async def list_issues(owner: str, repo: str): - try: - issues = github_engine.list_issues(f"{owner}/{repo}") - return {"status": "success", "data": issues} - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) diff --git a/routes/swarm.py b/routes/swarm.py deleted file mode 100644 index e28daf04a7cbc9c2fa896fd384c80ae3cba9561f..0000000000000000000000000000000000000000 --- a/routes/swarm.py +++ /dev/null @@ -1,19 +0,0 @@ -from fastapi import APIRouter, HTTPException - -router = APIRouter() - -@router.post("/swarm/run") -async def run_swarm(goal: str): - return {"status": "swarm_started", "goal": goal} - -@router.post("/planner/run") -async def run_planner(goal: str): - return {"status": "planner_started"} - -@router.post("/reviewer/run") -async def run_reviewer(code: str): - return {"status": "reviewer_started"} - -@router.post("/debugger/run") -async def run_debugger(error: str): - return {"status": "debugger_started"} diff --git a/routes/tasks.py b/routes/tasks.py deleted file mode 100644 index 1d01ac87527db984f2e7f0c4c81dd47bdf54936f..0000000000000000000000000000000000000000 --- a/routes/tasks.py +++ /dev/null @@ -1,43 +0,0 @@ -from fastapi import APIRouter, HTTPException -from pydantic import BaseModel -from typing import List, Optional -from task_manager import task_manager, TaskStatus - -router = APIRouter() - -class TaskCreate(BaseModel): - goal: str - task_type: Optional[str] = "general" - -@router.post("/tasks/create") -async def create_task(task_in: TaskCreate): - task = task_manager.create_task(task_in.goal, task_in.task_type) - return task - -@router.get("/tasks/{task_id}") -async def get_task(task_id: str): - task = task_manager.get_task(task_id) - if not task: - raise HTTPException(status_code=404, detail="Task not found") - return task - -@router.post("/tasks/{task_id}/cancel") -async def cancel_task(task_id: str): - success = task_manager.cancel_task(task_id) - if not success: - raise HTTPException(status_code=404, detail="Task not found or cannot be cancelled") - return {"status": "cancelled"} - -@router.post("/tasks/{task_id}/retry") -async def retry_task(task_id: str): - # Logic to retry task - return {"status": "retrying"} - -@router.get("/execute") -async def execute_command(command: str): - # Logic to execute a single command - return {"output": f"Executed: {command}"} - -@router.post("/plan") -async def create_plan(goal: str): - return {"plan": ["Step 1", "Step 2"]} diff --git a/routes/webhooks.py b/routes/webhooks.py deleted file mode 100644 index 5285ad8e2b7f7328cfd58ca0e746ad2639d79759..0000000000000000000000000000000000000000 --- a/routes/webhooks.py +++ /dev/null @@ -1,18 +0,0 @@ -from fastapi import APIRouter, Request - -router = APIRouter() - -@router.post("/webhook/github") -async def github_webhook(request: Request): - payload = await request.json() - return {"status": "received", "source": "github"} - -@router.post("/webhook/n8n") -async def n8n_webhook(request: Request): - payload = await request.json() - return {"status": "received", "source": "n8n"} - -@router.post("/webhook/custom") -async def custom_webhook(request: Request): - payload = await request.json() - return {"status": "received", "source": "custom"} diff --git a/routes/workspace.py b/routes/workspace.py deleted file mode 100644 index 025064640b7cd76e61feb867703868962f44e000..0000000000000000000000000000000000000000 --- a/routes/workspace.py +++ /dev/null @@ -1,21 +0,0 @@ -from fastapi import APIRouter, HTTPException -from pydantic import BaseModel -import os - -router = APIRouter() - -@router.get("/workspace/files") -async def list_files(path: str = "."): - return {"files": []} - -@router.get("/workspace/read") -async def read_file(path: str): - return {"content": "file content"} - -@router.post("/workspace/write") -async def write_file(path: str, content: str): - return {"status": "written", "path": path} - -@router.delete("/workspace/delete") -async def delete_file(path: str): - return {"status": "deleted", "path": path} diff --git a/start_server.sh b/start_server.sh deleted file mode 100644 index 2e0546c40ac3391ee3d7ae2a958b87d28ffc1e45..0000000000000000000000000000000000000000 --- a/start_server.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash - -NGX_NAME="${NGX_NAME:-admin}" -NGX_PASS="${NGX_PASS:-admin}" -CRYPTPASS=`openssl passwd -apr1 ${NGX_PASS}` -PORT="${PORT:-8080}" - -echo "USERNAME:" $NGX_NAME -echo "PASSWORD:" $NGX_PASS - -echo "${NGX_NAME}:${CRYPTPASS}" > ngpasswd - -COMMIT=$(cat /app/openvscode-server/product.json | awk '/commit/{print $4;exit}' FS='[""]') -sed -i "s/#COMMIT#/$COMMIT/" nginx.conf -sed -i "s/#PORT#/$PORT/" nginx.conf -nginx -c $PWD/nginx.conf - -# Start the API server in the background -export API_PORT=8000 -python3 api_server.py & - - -set +e -if [[ ! -z "$REPO" ]]; then - dir=$(basename "$REPO" .git) - echo start to clone initial repo $REPO into $dir - git clone --progress $REPO $dir - cd $dir - git config --global user.name "$(git log -1 --pretty=format:'%an')" - git config --global user.email "$(git log -1 --pretty=format:'%ae')" - cd .. -else - git config --global user.name "$SPACE_AUTHOR_NAME" - git config --global user.email "$SPACE_AUTHOR_NAME@hf.co" -fi - -git config --global http.postBuffer 524288000 - -echo "Starting VSCode Server..." -vscode=/app/openvscode-server/bin/openvscode-server -vscode_cli=/app/openvscode-server/bin/remote-cli/openvscode-server -$vscode --install-extension ms-toolsai.jupyter -$vscode --install-extension ms-python.python -ln -s $vscode_cli $(dirname $vscode_cli)/code -set -e -exec $vscode --host 0.0.0.0 --port 5050 --without-connection-token \"${@}\" -- diff --git a/task_manager.py b/task_manager.py deleted file mode 100644 index 0441b2b6fe5edc84e254380c51f48937f589b19c..0000000000000000000000000000000000000000 --- a/task_manager.py +++ /dev/null @@ -1,87 +0,0 @@ -import uuid -from enum import Enum -from typing import Dict, List, Optional, Any -from datetime import datetime -from core.database import db - -class TaskStatus(str, Enum): - PENDING = "pending" - RUNNING = "running" - COMPLETED = "completed" - FAILED = "failed" - CANCELLED = "cancelled" - -class Task: - def __init__(self, goal: str, task_type: str = "general", task_id: str = None): - self.id = task_id or str(uuid.uuid4()) - self.goal = goal - self.type = task_type - self.status = TaskStatus.PENDING - self.created_at = datetime.utcnow().isoformat() - self.updated_at = datetime.utcnow().isoformat() - self.result = None - self.error = None - self.progress = 0 - self.logs = [] - - @classmethod - def from_dict(cls, data: Dict[str, Any]): - task = cls(goal=data['goal'], task_type=data['type'], task_id=data['id']) - task.status = TaskStatus(data['status']) - task.progress = data['progress'] - task.result = data['result'] - task.error = data['error'] - task.created_at = data['created_at'] - task.updated_at = data['updated_at'] - return task - - def to_dict(self): - return { - "id": self.id, - "goal": self.goal, - "type": self.type, - "status": self.status, - "progress": self.progress, - "result": self.result, - "error": self.error, - "created_at": self.created_at, - "updated_at": self.updated_at - } - - def update(self, status: TaskStatus = None, progress: int = None, result: Any = None, error: str = None): - if status: self.status = status - if progress is not None: self.progress = progress - if result is not None: self.result = result - if error is not None: self.error = error - self.updated_at = datetime.utcnow().isoformat() - db.save_task(self.to_dict()) - - def add_log(self, message: str): - db.add_log(self.id, message) - -class TaskManager: - def create_task(self, goal: str, task_type: str = "general") -> Task: - task = Task(goal, task_type) - db.save_task(task.to_dict()) - return task - - def get_task(self, task_id: str) -> Optional[Task]: - data = db.get_task(task_id) - if data: - task = Task.from_dict(data) - task.logs = db.get_logs(task_id) - return task - return None - - def list_tasks(self) -> List[Task]: - task_dicts = db.list_tasks() - return [Task.from_dict(d) for d in task_dicts] - - def cancel_task(self, task_id: str) -> bool: - task = self.get_task(task_id) - if task and task.status in [TaskStatus.PENDING, TaskStatus.RUNNING]: - task.update(status=TaskStatus.CANCELLED) - return True - return False - -task_manager = TaskManager() diff --git a/root/etc/s6-overlay/s6-rc.d/user/contents.d/init-openvscode-server b/tools/__init__.py similarity index 100% rename from root/etc/s6-overlay/s6-rc.d/user/contents.d/init-openvscode-server rename to tools/__init__.py diff --git a/tools/executor.py b/tools/executor.py new file mode 100644 index 0000000000000000000000000000000000000000..2ae0b80afbd92030c28334a35653c342c2599eb4 --- /dev/null +++ b/tools/executor.py @@ -0,0 +1,176 @@ +""" +Tool Executor β€” Routes tool calls to the right implementation +Supports: code, shell, file, browser, github, memory, search, test, none +""" + +import asyncio +import os +import subprocess +import tempfile +import time +from typing import Any, List, Optional + +import structlog + +from api.websocket_manager import WebSocketManager + +log = structlog.get_logger() + + +class ToolExecutor: + def __init__(self, ws_manager: WebSocketManager): + self.ws = ws_manager + + async def run( + self, + tool: str, + task: str, + goal: str = "", + previous: List = [], + task_id: str = "", + session_id: str = "", + ) -> str: + tool = (tool or "none").lower().strip() + + dispatch = { + "code": self._tool_code, + "shell": self._tool_shell, + "file": self._tool_file, + "github": self._tool_github, + "memory": self._tool_memory, + "search": self._tool_search, + "test": self._tool_test, + "browser": self._tool_browser, + "none": self._tool_none, + } + + fn = dispatch.get(tool, self._tool_none) + return await fn(task=task, goal=goal, previous=previous, task_id=task_id, session_id=session_id) + + # ─── Code Tool ───────────────────────────────────────────────────────────── + async def _tool_code(self, task, goal, previous, task_id, session_id) -> str: + """Generate code using LLM.""" + from core.agent import AgentCore + agent = AgentCore(self.ws) + messages = [ + {"role": "system", "content": "You are an expert software engineer. Write clean, production-quality code. Return only the code with minimal explanation."}, + {"role": "user", "content": f"Task: {task}\nGoal: {goal}\n\nWrite the code to accomplish this."}, + ] + result = await agent.llm_stream(messages, task_id=task_id, session_id=session_id) + return result or f"# Code for: {task}" + + # ─── Shell Tool ──────────────────────────────────────────────────────────── + async def _tool_shell(self, task, goal, previous, task_id, session_id) -> str: + """Execute shell commands safely in a temp workspace.""" + # Extract command from task description + from core.agent import AgentCore + agent = AgentCore(self.ws) + messages = [ + {"role": "system", "content": "Extract the shell command to run. Return ONLY the command, nothing else."}, + {"role": "user", "content": f"Task: {task}"}, + ] + cmd = await agent.llm_stream(messages, task_id=task_id, session_id=session_id) + cmd = cmd.strip().strip("`").strip() + + # Safety: block dangerous commands + blocked = ["rm -rf /", ":(){ :|:& };:", "mkfs", "dd if=", "shutdown", "reboot", "halt"] + for b in blocked: + if b in cmd: + return f"❌ Blocked dangerous command: {cmd}" + + try: + await self.ws.emit(task_id, "step_progress", { + "action": "shell_exec", + "command": cmd[:200], + }, session_id=session_id) + proc = await asyncio.create_subprocess_shell( + cmd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + cwd="/tmp", + ) + stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=30) + output = stdout.decode()[:2000] + (stderr.decode()[:500] if stderr else "") + return output or "Command executed (no output)" + except asyncio.TimeoutError: + return "⚠️ Command timed out after 30s" + except Exception as e: + return f"❌ Shell error: {str(e)}" + + # ─── File Tool ───────────────────────────────────────────────────────────── + async def _tool_file(self, task, goal, previous, task_id, session_id) -> str: + """Create or modify files.""" + from core.agent import AgentCore + agent = AgentCore(self.ws) + messages = [ + {"role": "system", "content": "Generate file content. Respond with JSON: {\"filename\": \"...\", \"content\": \"...\"}"}, + {"role": "user", "content": f"Task: {task}\nGoal: {goal}"}, + ] + raw = await agent.llm_stream(messages, task_id=task_id, session_id=session_id) + try: + import json + start = raw.find("{") + end = raw.rfind("}") + 1 + data = json.loads(raw[start:end]) + filename = data.get("filename", "output.txt") + content = data.get("content", raw) + path = f"/tmp/workspace/{filename}" + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w") as f: + f.write(content) + await self.ws.emit(task_id, "step_progress", { + "action": "file_written", + "filename": filename, + "size": len(content), + }, session_id=session_id) + return f"βœ… File written: {filename} ({len(content)} chars)" + except Exception as e: + return f"File task result: {raw[:500]}" + + # ─── GitHub Tool ─────────────────────────────────────────────────────────── + async def _tool_github(self, task, goal, previous, task_id, session_id) -> str: + """Perform GitHub operations.""" + return f"GitHub: {task}\n(Set GITHUB_TOKEN to enable real GitHub operations)" + + # ─── Memory Tool ─────────────────────────────────────────────────────────── + async def _tool_memory(self, task, goal, previous, task_id, session_id) -> str: + """Save/retrieve from memory.""" + from memory.db import save_memory, search_memory + results = await search_memory(task[:50], session_id=session_id) + if results: + return "\n".join([r["content"][:300] for r in results[:3]]) + return "No relevant memories found" + + # ─── Search Tool ─────────────────────────────────────────────────────────── + async def _tool_search(self, task, goal, previous, task_id, session_id) -> str: + """Web search using available APIs.""" + return f"Search result for: {task}\n(Integrate search API for real results)" + + # ─── Test Tool ───────────────────────────────────────────────────────────── + async def _tool_test(self, task, goal, previous, task_id, session_id) -> str: + """Generate and run tests.""" + from core.agent import AgentCore + agent = AgentCore(self.ws) + messages = [ + {"role": "system", "content": "Write test cases for the given task. Use pytest format."}, + {"role": "user", "content": f"Write tests for: {task}\nContext: {goal}"}, + ] + result = await agent.llm_stream(messages, task_id=task_id, session_id=session_id) + return result or f"# Tests for: {task}" + + # ─── Browser Tool ────────────────────────────────────────────────────────── + async def _tool_browser(self, task, goal, previous, task_id, session_id) -> str: + """Browser automation (stub β€” extend with playwright).""" + return f"Browser task: {task}\n(Install playwright for real browser automation)" + + # ─── None Tool ───────────────────────────────────────────────────────────── + async def _tool_none(self, task, goal, previous, task_id, session_id) -> str: + """Use LLM directly without tools.""" + from core.agent import AgentCore + agent = AgentCore(self.ws) + messages = [ + {"role": "system", "content": "You are an expert engineer. Complete the task thoroughly."}, + {"role": "user", "content": f"Task: {task}\nGoal context: {goal}"}, + ] + result = await agent.llm_stream(messages, task_id=task_id, session_id=session_id) + return result or f"Completed: {task}" diff --git a/websocket_server.py b/websocket_server.py deleted file mode 100644 index 5ea2b89889c8e794f1381ab7018581f9b8092e5e..0000000000000000000000000000000000000000 --- a/websocket_server.py +++ /dev/null @@ -1,48 +0,0 @@ -import asyncio -import json -from fastapi import APIRouter, WebSocket, WebSocketDisconnect -from typing import List - -router = APIRouter() - -class ConnectionManager: - def __init__(self): - self.active_connections: List[WebSocket] = [] - - async def connect(self, websocket: WebSocket): - await websocket.accept() - self.active_connections.append(websocket) - - def disconnect(self, websocket: WebSocket): - self.active_connections.remove(websocket) - - async def send_personal_message(self, message: str, websocket: WebSocket): - await websocket.send_text(message) - - async def broadcast(self, message: str): - for connection in self.active_connections: - await connection.send_text(message) - -manager = ConnectionManager() - -@router.websocket("/ws/logs") -async def websocket_logs(websocket: WebSocket): - await manager.connect(websocket) - try: - while True: - data = await websocket.receive_text() - # Echo for now, will be used for live logs - await manager.send_personal_message(f"Log received: {data}", websocket) - except WebSocketDisconnect: - manager.disconnect(websocket) - -@router.websocket("/ws/tasks/{task_id}") -async def websocket_task_updates(websocket: WebSocket, task_id: str): - await manager.connect(websocket) - try: - while True: - # In a real scenario, this would listen to task updates - await asyncio.sleep(10) - await manager.send_personal_message(json.dumps({"task_id": task_id, "status": "running"}), websocket) - except WebSocketDisconnect: - manager.disconnect(websocket)