Spaces:
Sleeping
Sleeping
Commit
·
c16e1c9
1
Parent(s):
20a1017
Add RAG MCP Server with Supabase vector search
Browse files- .gitignore +2 -1
- backend/api/mcp_clients/admin_client.py +36 -0
- backend/api/mcp_clients/rag_client.py +39 -0
- backend/api/mcp_clients/web_client.py +38 -0
- backend/api/routes/admin.py +81 -0
- backend/api/routes/agent.py +30 -0
- backend/api/routes/analytics.py +103 -0
- backend/api/routes/rag.py +28 -0
- backend/api/routes/web.py +28 -0
- backend/api/services/agent_orchestrator.py +82 -0
- backend/api/services/llm_client.py +7 -2
- backend/api/utils/text_extractor.py +41 -0
- backend/mcp_servers/database.py +203 -0
- backend/mcp_servers/embeddings.py +17 -0
- backend/mcp_servers/main.py +172 -0
- backend/tests/test_agent_orchestrator.py +21 -0
- requirements.txt +6 -1
.gitignore
CHANGED
|
@@ -1 +1,2 @@
|
|
| 1 |
-
venv/
|
|
|
|
|
|
| 1 |
+
venv/
|
| 2 |
+
.env
|
backend/api/mcp_clients/admin_client.py
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import httpx
|
| 3 |
+
from dotenv import load_dotenv
|
| 4 |
+
|
| 5 |
+
load_dotenv()
|
| 6 |
+
|
| 7 |
+
class AdminClient:
|
| 8 |
+
"""
|
| 9 |
+
Communicates with the Admin MCP governance server.
|
| 10 |
+
"""
|
| 11 |
+
|
| 12 |
+
def __init__(self):
|
| 13 |
+
self.base_url = os.getenv("ADMIN_MCP_URL")
|
| 14 |
+
self.alert_endpoint = f"{self.base_url}/alert"
|
| 15 |
+
|
| 16 |
+
async def alert(self, tenant_id: str, message: str, redflag: dict):
|
| 17 |
+
"""
|
| 18 |
+
Sends redflag violation to admin server.
|
| 19 |
+
"""
|
| 20 |
+
|
| 21 |
+
try:
|
| 22 |
+
async with httpx.AsyncClient() as client:
|
| 23 |
+
response = await client.post(
|
| 24 |
+
self.alert_endpoint,
|
| 25 |
+
json={
|
| 26 |
+
"tenant_id": tenant_id,
|
| 27 |
+
"message": message,
|
| 28 |
+
"violations": redflag.get("matches", []),
|
| 29 |
+
}
|
| 30 |
+
)
|
| 31 |
+
|
| 32 |
+
return response.status_code == 200
|
| 33 |
+
|
| 34 |
+
except Exception as e:
|
| 35 |
+
print("Admin Client Error:", e)
|
| 36 |
+
return False
|
backend/api/mcp_clients/rag_client.py
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import httpx
|
| 3 |
+
from dotenv import load_dotenv
|
| 4 |
+
|
| 5 |
+
load_dotenv()
|
| 6 |
+
|
| 7 |
+
class RAGClient:
|
| 8 |
+
"""
|
| 9 |
+
Communicates with the RAG MCP server.
|
| 10 |
+
"""
|
| 11 |
+
|
| 12 |
+
def __init__(self):
|
| 13 |
+
self.base_url = os.getenv("RAG_MCP_URL")
|
| 14 |
+
self.search_endpoint = f"{self.base_url}/search"
|
| 15 |
+
|
| 16 |
+
async def search(self, query: str, tenant_id: str):
|
| 17 |
+
"""
|
| 18 |
+
Sends the query to the RAG server and returns document chunks.
|
| 19 |
+
"""
|
| 20 |
+
|
| 21 |
+
try:
|
| 22 |
+
async with httpx.AsyncClient() as client:
|
| 23 |
+
response = await client.post(
|
| 24 |
+
self.search_endpoint,
|
| 25 |
+
json={
|
| 26 |
+
"query": query,
|
| 27 |
+
"tenant_id": tenant_id
|
| 28 |
+
}
|
| 29 |
+
)
|
| 30 |
+
|
| 31 |
+
if response.status_code != 200:
|
| 32 |
+
return []
|
| 33 |
+
|
| 34 |
+
data = response.json()
|
| 35 |
+
return data.get("results", [])
|
| 36 |
+
|
| 37 |
+
except Exception as e:
|
| 38 |
+
print("RAG Client Error:", e)
|
| 39 |
+
return []
|
backend/api/mcp_clients/web_client.py
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import httpx
|
| 3 |
+
from dotenv import load_dotenv
|
| 4 |
+
|
| 5 |
+
load_dotenv()
|
| 6 |
+
|
| 7 |
+
class WebClient:
|
| 8 |
+
"""
|
| 9 |
+
Communicates with the Web Search MCP server.
|
| 10 |
+
"""
|
| 11 |
+
|
| 12 |
+
def __init__(self):
|
| 13 |
+
self.base_url = os.getenv("WEB_MCP_URL")
|
| 14 |
+
self.search_endpoint = f"{self.base_url}/search"
|
| 15 |
+
|
| 16 |
+
async def search(self, query: str):
|
| 17 |
+
"""
|
| 18 |
+
Sends the query to the Web Search server and returns search results.
|
| 19 |
+
"""
|
| 20 |
+
|
| 21 |
+
try:
|
| 22 |
+
async with httpx.AsyncClient() as client:
|
| 23 |
+
response = await client.post(
|
| 24 |
+
self.search_endpoint,
|
| 25 |
+
json={
|
| 26 |
+
"query": query
|
| 27 |
+
}
|
| 28 |
+
)
|
| 29 |
+
|
| 30 |
+
if response.status_code != 200:
|
| 31 |
+
return []
|
| 32 |
+
|
| 33 |
+
data = response.json()
|
| 34 |
+
return data.get("results", [])
|
| 35 |
+
|
| 36 |
+
except Exception as e:
|
| 37 |
+
print("Web Client Error:", e)
|
| 38 |
+
return []
|
backend/api/routes/admin.py
ADDED
|
@@ -0,0 +1,81 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from fastapi import APIRouter, Header, HTTPException
|
| 2 |
+
from typing import List
|
| 3 |
+
|
| 4 |
+
router = APIRouter()
|
| 5 |
+
|
| 6 |
+
# Temporary in-memory store (replace with Supabase later)
|
| 7 |
+
TENANT_RULES = {}
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
def get_rules_for_tenant(tenant_id: str) -> List[str]:
|
| 11 |
+
return TENANT_RULES.get(tenant_id, [])
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
@router.get("/admin/rules")
|
| 15 |
+
async def get_redflag_rules(
|
| 16 |
+
x_tenant_id: str = Header(None)
|
| 17 |
+
):
|
| 18 |
+
"""
|
| 19 |
+
Returns all red-flag rules for this tenant.
|
| 20 |
+
"""
|
| 21 |
+
|
| 22 |
+
if not x_tenant_id:
|
| 23 |
+
raise HTTPException(status_code=400, detail="Missing tenant ID")
|
| 24 |
+
|
| 25 |
+
return {
|
| 26 |
+
"tenant_id": x_tenant_id,
|
| 27 |
+
"rules": get_rules_for_tenant(x_tenant_id)
|
| 28 |
+
}
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
@router.post("/admin/rules")
|
| 32 |
+
async def add_redflag_rule(
|
| 33 |
+
rule: str,
|
| 34 |
+
x_tenant_id: str = Header(None)
|
| 35 |
+
):
|
| 36 |
+
"""
|
| 37 |
+
Adds a new red-flag rule to this tenant.
|
| 38 |
+
"""
|
| 39 |
+
|
| 40 |
+
if not x_tenant_id:
|
| 41 |
+
raise HTTPException(status_code=400, detail="Missing tenant ID")
|
| 42 |
+
|
| 43 |
+
rules = get_rules_for_tenant(x_tenant_id)
|
| 44 |
+
|
| 45 |
+
if rule not in rules:
|
| 46 |
+
rules.append(rule)
|
| 47 |
+
|
| 48 |
+
TENANT_RULES[x_tenant_id] = rules
|
| 49 |
+
|
| 50 |
+
return {
|
| 51 |
+
"tenant_id": x_tenant_id,
|
| 52 |
+
"added_rule": rule,
|
| 53 |
+
"rules": rules
|
| 54 |
+
}
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
@router.delete("/admin/rules/{rule}")
|
| 58 |
+
async def delete_redflag_rule(
|
| 59 |
+
rule: str,
|
| 60 |
+
x_tenant_id: str = Header(None)
|
| 61 |
+
):
|
| 62 |
+
"""
|
| 63 |
+
Deletes a red-flag rule for this tenant.
|
| 64 |
+
"""
|
| 65 |
+
|
| 66 |
+
if not x_tenant_id:
|
| 67 |
+
raise HTTPException(status_code=400, detail="Missing tenant ID")
|
| 68 |
+
|
| 69 |
+
rules = get_rules_for_tenant(x_tenant_id)
|
| 70 |
+
|
| 71 |
+
if rule not in rules:
|
| 72 |
+
raise HTTPException(status_code=404, detail="Rule not found")
|
| 73 |
+
|
| 74 |
+
rules.remove(rule)
|
| 75 |
+
TENANT_RULES[x_tenant_id] = rules
|
| 76 |
+
|
| 77 |
+
return {
|
| 78 |
+
"tenant_id": x_tenant_id,
|
| 79 |
+
"deleted_rule": rule,
|
| 80 |
+
"rules": rules
|
| 81 |
+
}
|
backend/api/routes/agent.py
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from fastapi import APIRouter, Header, HTTPException
|
| 2 |
+
from api.services.agent_orchestrator import AgentOrchestrator
|
| 3 |
+
|
| 4 |
+
router = APIRouter()
|
| 5 |
+
agent = AgentOrchestrator()
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
@router.post("/agent")
|
| 9 |
+
async def agent_chat(
|
| 10 |
+
message: str,
|
| 11 |
+
x_tenant_id: str = Header(None)
|
| 12 |
+
):
|
| 13 |
+
"""
|
| 14 |
+
Main chat endpoint.
|
| 15 |
+
Frontend will call this to talk with the AI agent.
|
| 16 |
+
"""
|
| 17 |
+
|
| 18 |
+
if not x_tenant_id:
|
| 19 |
+
raise HTTPException(status_code=400, detail="Missing tenant ID")
|
| 20 |
+
|
| 21 |
+
result = await agent.process_message(message, x_tenant_id)
|
| 22 |
+
|
| 23 |
+
return {
|
| 24 |
+
"response": result["response"],
|
| 25 |
+
"intent": result["intent"],
|
| 26 |
+
"tool": result["tool"],
|
| 27 |
+
"redflag": result["redflag"],
|
| 28 |
+
"rag_results": result["rag_results"],
|
| 29 |
+
"web_results": result["web_results"]
|
| 30 |
+
}
|
backend/api/routes/analytics.py
ADDED
|
@@ -0,0 +1,103 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from fastapi import APIRouter, Header, HTTPException
|
| 2 |
+
|
| 3 |
+
router = APIRouter()
|
| 4 |
+
|
| 5 |
+
# Mock in-memory analytics (replace with Supabase later)
|
| 6 |
+
ANALYTICS_DATA = {
|
| 7 |
+
"tool_usage": {
|
| 8 |
+
"rag": 12,
|
| 9 |
+
"web": 8,
|
| 10 |
+
"admin": 3
|
| 11 |
+
},
|
| 12 |
+
"redflags": [
|
| 13 |
+
{
|
| 14 |
+
"tenant": "tenant123",
|
| 15 |
+
"match": "salary",
|
| 16 |
+
"message": "get salary data now",
|
| 17 |
+
"timestamp": "2025-01-14T10:22:00Z"
|
| 18 |
+
}
|
| 19 |
+
],
|
| 20 |
+
"activity": {
|
| 21 |
+
"total_queries": 23,
|
| 22 |
+
"active_users": 4,
|
| 23 |
+
"last_query": "2025-01-14T10:24:31Z"
|
| 24 |
+
}
|
| 25 |
+
}
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
@router.get("/analytics/overview")
|
| 29 |
+
async def analytics_overview(
|
| 30 |
+
x_tenant_id: str = Header(None)
|
| 31 |
+
):
|
| 32 |
+
"""
|
| 33 |
+
Returns an overview of analytics for the dashboard.
|
| 34 |
+
"""
|
| 35 |
+
|
| 36 |
+
if not x_tenant_id:
|
| 37 |
+
raise HTTPException(status_code=400, detail="Missing tenant ID")
|
| 38 |
+
|
| 39 |
+
return {
|
| 40 |
+
"tenant_id": x_tenant_id,
|
| 41 |
+
"overview": {
|
| 42 |
+
"total_queries": ANALYTICS_DATA["activity"]["total_queries"],
|
| 43 |
+
"tool_usage": ANALYTICS_DATA["tool_usage"],
|
| 44 |
+
"redflag_count": len(ANALYTICS_DATA["redflags"]),
|
| 45 |
+
"active_users": ANALYTICS_DATA["activity"]["active_users"]
|
| 46 |
+
}
|
| 47 |
+
}
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
@router.get("/analytics/tool-usage")
|
| 51 |
+
async def analytics_tool_usage(
|
| 52 |
+
x_tenant_id: str = Header(None)
|
| 53 |
+
):
|
| 54 |
+
"""
|
| 55 |
+
Returns how often each tool (RAG, Web, Admin) was used.
|
| 56 |
+
"""
|
| 57 |
+
|
| 58 |
+
if not x_tenant_id:
|
| 59 |
+
raise HTTPException(status_code=400, detail="Missing tenant ID")
|
| 60 |
+
|
| 61 |
+
return {
|
| 62 |
+
"tenant_id": x_tenant_id,
|
| 63 |
+
"tool_usage": ANALYTICS_DATA["tool_usage"]
|
| 64 |
+
}
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
@router.get("/analytics/redflags")
|
| 68 |
+
async def analytics_redflags(
|
| 69 |
+
x_tenant_id: str = Header(None)
|
| 70 |
+
):
|
| 71 |
+
"""
|
| 72 |
+
Returns red-flag violations for this tenant.
|
| 73 |
+
"""
|
| 74 |
+
|
| 75 |
+
if not x_tenant_id:
|
| 76 |
+
raise HTTPException(status_code=400, detail="Missing tenant ID")
|
| 77 |
+
|
| 78 |
+
redflags = [
|
| 79 |
+
r for r in ANALYTICS_DATA["redflags"]
|
| 80 |
+
if r["tenant"] == x_tenant_id
|
| 81 |
+
]
|
| 82 |
+
|
| 83 |
+
return {
|
| 84 |
+
"tenant_id": x_tenant_id,
|
| 85 |
+
"redflags": redflags
|
| 86 |
+
}
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
@router.get("/analytics/activity")
|
| 90 |
+
async def analytics_activity(
|
| 91 |
+
x_tenant_id: str = Header(None)
|
| 92 |
+
):
|
| 93 |
+
"""
|
| 94 |
+
Returns general tenant activity statistics.
|
| 95 |
+
"""
|
| 96 |
+
|
| 97 |
+
if not x_tenant_id:
|
| 98 |
+
raise HTTPException(status_code=400, detail="Missing tenant ID")
|
| 99 |
+
|
| 100 |
+
return {
|
| 101 |
+
"tenant_id": x_tenant_id,
|
| 102 |
+
"activity": ANALYTICS_DATA["activity"]
|
| 103 |
+
}
|
backend/api/routes/rag.py
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from fastapi import APIRouter, Header, HTTPException
|
| 2 |
+
from api.mcp_clients.rag_client import RAGClient
|
| 3 |
+
|
| 4 |
+
router = APIRouter()
|
| 5 |
+
rag_client = RAGClient()
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
@router.post("/rag/search")
|
| 9 |
+
async def rag_search(
|
| 10 |
+
query: str,
|
| 11 |
+
x_tenant_id: str = Header(None)
|
| 12 |
+
):
|
| 13 |
+
"""
|
| 14 |
+
Search tenant knowledge base using the RAG MCP server.
|
| 15 |
+
"""
|
| 16 |
+
|
| 17 |
+
if not x_tenant_id:
|
| 18 |
+
raise HTTPException(status_code=400, detail="Missing tenant ID")
|
| 19 |
+
|
| 20 |
+
try:
|
| 21 |
+
results = await rag_client.search(query, x_tenant_id)
|
| 22 |
+
return {
|
| 23 |
+
"tenant_id": x_tenant_id,
|
| 24 |
+
"query": query,
|
| 25 |
+
"results": results
|
| 26 |
+
}
|
| 27 |
+
except Exception as e:
|
| 28 |
+
raise HTTPException(status_code=500, detail=str(e))
|
backend/api/routes/web.py
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from fastapi import APIRouter, Header, HTTPException
|
| 2 |
+
from api.mcp_clients.web_client import WebClient
|
| 3 |
+
|
| 4 |
+
router = APIRouter()
|
| 5 |
+
web_client = WebClient()
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
@router.post("/web/search")
|
| 9 |
+
async def web_search(
|
| 10 |
+
query: str,
|
| 11 |
+
x_tenant_id: str = Header(None)
|
| 12 |
+
):
|
| 13 |
+
"""
|
| 14 |
+
Perform a live internet search using the Web MCP server.
|
| 15 |
+
"""
|
| 16 |
+
|
| 17 |
+
if not x_tenant_id:
|
| 18 |
+
raise HTTPException(status_code=400, detail="Missing tenant ID")
|
| 19 |
+
|
| 20 |
+
try:
|
| 21 |
+
results = await web_client.search(query)
|
| 22 |
+
return {
|
| 23 |
+
"tenant_id": x_tenant_id,
|
| 24 |
+
"query": query,
|
| 25 |
+
"results": results
|
| 26 |
+
}
|
| 27 |
+
except Exception as e:
|
| 28 |
+
raise HTTPException(status_code=500, detail=str(e))
|
backend/api/services/agent_orchestrator.py
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from api.services.intent_classifier import IntentClassifier
|
| 2 |
+
from api.services.redflag_detector import RedFlagDetector
|
| 3 |
+
from api.services.tool_selector import ToolSelector
|
| 4 |
+
from api.services.prompt_builder import PromptBuilder
|
| 5 |
+
from api.services.llm_client import LLMClient
|
| 6 |
+
|
| 7 |
+
from api.mcp_clients.rag_client import RAGClient
|
| 8 |
+
from api.mcp_clients.web_client import WebClient
|
| 9 |
+
from api.mcp_clients.admin_client import AdminClient
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
class AgentOrchestrator:
|
| 13 |
+
|
| 14 |
+
def __init__(self):
|
| 15 |
+
# Services
|
| 16 |
+
self.intent_classifier = IntentClassifier()
|
| 17 |
+
self.redflag_detector = RedFlagDetector()
|
| 18 |
+
self.tool_selector = ToolSelector()
|
| 19 |
+
self.prompt_builder = PromptBuilder()
|
| 20 |
+
self.llm_client = LLMClient()
|
| 21 |
+
|
| 22 |
+
# MCP Tool Clients
|
| 23 |
+
self.rag_client = RAGClient()
|
| 24 |
+
self.web_client = WebClient()
|
| 25 |
+
self.admin_client = AdminClient()
|
| 26 |
+
|
| 27 |
+
async def process_message(self, user_message: str, tenant_id: str):
|
| 28 |
+
"""
|
| 29 |
+
Main agent workflow for processing user input.
|
| 30 |
+
"""
|
| 31 |
+
|
| 32 |
+
# 1. Intent
|
| 33 |
+
intent = self.intent_classifier.classify(user_message)
|
| 34 |
+
|
| 35 |
+
# 2. Red-Flag check
|
| 36 |
+
tenant_rules = [] # TODO: pull from database later
|
| 37 |
+
redflag = self.redflag_detector.check(user_message, tenant_rules)
|
| 38 |
+
|
| 39 |
+
# 3. Tool selection
|
| 40 |
+
tool = self.tool_selector.select_tool(intent, redflag)
|
| 41 |
+
|
| 42 |
+
# Tool outputs
|
| 43 |
+
rag_results = None
|
| 44 |
+
web_results = None
|
| 45 |
+
|
| 46 |
+
# 4. Execute selected tool (if any)
|
| 47 |
+
if tool == "rag":
|
| 48 |
+
rag_results = await self.rag_client.search(user_message, tenant_id)
|
| 49 |
+
|
| 50 |
+
elif tool == "web":
|
| 51 |
+
web_results = await self.web_client.search(user_message)
|
| 52 |
+
|
| 53 |
+
elif tool == "admin":
|
| 54 |
+
# Optional: notify admins
|
| 55 |
+
try:
|
| 56 |
+
await self.admin_client.alert(tenant_id, user_message, redflag)
|
| 57 |
+
except:
|
| 58 |
+
pass
|
| 59 |
+
|
| 60 |
+
# 5. Build prompt for LLM
|
| 61 |
+
prompt = self.prompt_builder.build(
|
| 62 |
+
user_message=user_message,
|
| 63 |
+
tool=tool,
|
| 64 |
+
rag_results=rag_results,
|
| 65 |
+
web_results=web_results,
|
| 66 |
+
redflag_info=redflag,
|
| 67 |
+
tenant_id=tenant_id
|
| 68 |
+
)
|
| 69 |
+
|
| 70 |
+
# 6. Call LLM
|
| 71 |
+
ai_response = self.llm_client.generate(prompt)
|
| 72 |
+
|
| 73 |
+
# 7. Return combined output
|
| 74 |
+
return {
|
| 75 |
+
"intent": intent,
|
| 76 |
+
"tool": tool,
|
| 77 |
+
"redflag": redflag,
|
| 78 |
+
"rag_results": rag_results,
|
| 79 |
+
"web_results": web_results,
|
| 80 |
+
"response": ai_response,
|
| 81 |
+
"prompt_used": prompt
|
| 82 |
+
}
|
backend/api/services/llm_client.py
CHANGED
|
@@ -1,13 +1,18 @@
|
|
|
|
|
| 1 |
import requests
|
|
|
|
|
|
|
|
|
|
| 2 |
|
| 3 |
class LLMClient:
|
| 4 |
"""
|
| 5 |
Uses a LOCAL Llama model via Ollama.
|
| 6 |
"""
|
| 7 |
|
| 8 |
-
def __init__(self, model="
|
| 9 |
self.model = model
|
| 10 |
-
|
|
|
|
| 11 |
|
| 12 |
def generate(self, prompt: str) -> str:
|
| 13 |
payload = {
|
|
|
|
| 1 |
+
import os
|
| 2 |
import requests
|
| 3 |
+
from dotenv import load_dotenv
|
| 4 |
+
|
| 5 |
+
load_dotenv()
|
| 6 |
|
| 7 |
class LLMClient:
|
| 8 |
"""
|
| 9 |
Uses a LOCAL Llama model via Ollama.
|
| 10 |
"""
|
| 11 |
|
| 12 |
+
def __init__(self, model=os.getenv("OLLAMA_MODEL")):
|
| 13 |
self.model = model
|
| 14 |
+
ollama_url = os.getenv("OLLAMA_URL")
|
| 15 |
+
self.url = f"{ollama_url}/api/generate"
|
| 16 |
|
| 17 |
def generate(self, prompt: str) -> str:
|
| 18 |
payload = {
|
backend/api/utils/text_extractor.py
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import re
|
| 2 |
+
|
| 3 |
+
def extract_text(text: str, max_words: int = 300):
|
| 4 |
+
"""
|
| 5 |
+
Split raw text into chunks of ~300 words.
|
| 6 |
+
Suitable for document ingestion before embeddings.
|
| 7 |
+
|
| 8 |
+
Args:
|
| 9 |
+
text (str): Raw text input
|
| 10 |
+
max_words (int): Max words per chunk (default 300)
|
| 11 |
+
|
| 12 |
+
Returns:
|
| 13 |
+
List[str]: List of chunked text segments
|
| 14 |
+
"""
|
| 15 |
+
|
| 16 |
+
# Normalize whitespace
|
| 17 |
+
clean = re.sub(r'\s+', ' ', text).strip()
|
| 18 |
+
|
| 19 |
+
if not clean:
|
| 20 |
+
return []
|
| 21 |
+
|
| 22 |
+
words = clean.split(" ")
|
| 23 |
+
chunks = []
|
| 24 |
+
|
| 25 |
+
current = []
|
| 26 |
+
count = 0
|
| 27 |
+
|
| 28 |
+
for word in words:
|
| 29 |
+
current.append(word)
|
| 30 |
+
count += 1
|
| 31 |
+
|
| 32 |
+
if count >= max_words:
|
| 33 |
+
chunks.append(" ".join(current))
|
| 34 |
+
current = []
|
| 35 |
+
count = 0
|
| 36 |
+
|
| 37 |
+
# Add final chunk
|
| 38 |
+
if current:
|
| 39 |
+
chunks.append(" ".join(current))
|
| 40 |
+
|
| 41 |
+
return chunks
|
backend/mcp_servers/database.py
ADDED
|
@@ -0,0 +1,203 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Supabase database connection and utilities for MCP servers.
|
| 3 |
+
|
| 4 |
+
This module provides both:
|
| 5 |
+
1. Direct PostgreSQL connections (via psycopg2) for pgvector operations
|
| 6 |
+
2. Supabase client for REST API operations
|
| 7 |
+
"""
|
| 8 |
+
|
| 9 |
+
import os
|
| 10 |
+
from typing import Optional, List, Dict, Any
|
| 11 |
+
import psycopg2
|
| 12 |
+
import psycopg2.extras
|
| 13 |
+
from supabase import create_client, Client
|
| 14 |
+
from dotenv import load_dotenv
|
| 15 |
+
|
| 16 |
+
# Load environment variables
|
| 17 |
+
load_dotenv()
|
| 18 |
+
|
| 19 |
+
# -----------------------------------
|
| 20 |
+
# Environment variables
|
| 21 |
+
# -----------------------------------
|
| 22 |
+
|
| 23 |
+
DATABASE_URL = os.getenv("POSTGRESQL_URL") # Direct PostgreSQL connection
|
| 24 |
+
SUPABASE_URL = os.getenv("SUPABASE_URL")
|
| 25 |
+
SUPABASE_KEY = os.getenv("SUPABASE_SERVICE_KEY") # MUST be service role key
|
| 26 |
+
|
| 27 |
+
# Global Supabase client instance
|
| 28 |
+
_supabase_client: Optional[Client] = None
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
# -----------------------------------
|
| 32 |
+
# PostgreSQL Connection (for pgvector)
|
| 33 |
+
# -----------------------------------
|
| 34 |
+
|
| 35 |
+
def get_connection():
|
| 36 |
+
"""
|
| 37 |
+
Establish a direct PostgreSQL connection for pgvector operations.
|
| 38 |
+
"""
|
| 39 |
+
if not DATABASE_URL:
|
| 40 |
+
raise ValueError(
|
| 41 |
+
"PostgreSQL connection string not configured. "
|
| 42 |
+
"Set POSTGRESQL_URL in your .env file."
|
| 43 |
+
)
|
| 44 |
+
|
| 45 |
+
return psycopg2.connect(DATABASE_URL)
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
# -----------------------------------
|
| 49 |
+
# Database Schema Initialization
|
| 50 |
+
# -----------------------------------
|
| 51 |
+
|
| 52 |
+
def initialize_database():
|
| 53 |
+
"""
|
| 54 |
+
Initialize the database schema:
|
| 55 |
+
- Enable pgvector extension
|
| 56 |
+
- Create documents table with vector support
|
| 57 |
+
"""
|
| 58 |
+
try:
|
| 59 |
+
conn = get_connection()
|
| 60 |
+
cur = conn.cursor()
|
| 61 |
+
|
| 62 |
+
# Enable pgvector extension
|
| 63 |
+
cur.execute("CREATE EXTENSION IF NOT EXISTS vector;")
|
| 64 |
+
print("✅ pgvector extension enabled")
|
| 65 |
+
|
| 66 |
+
# Create documents table
|
| 67 |
+
cur.execute("""
|
| 68 |
+
CREATE TABLE IF NOT EXISTS documents (
|
| 69 |
+
id BIGSERIAL PRIMARY KEY,
|
| 70 |
+
tenant_id TEXT NOT NULL,
|
| 71 |
+
chunk_text TEXT NOT NULL,
|
| 72 |
+
embedding vector(384) NOT NULL,
|
| 73 |
+
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
|
| 74 |
+
);
|
| 75 |
+
""")
|
| 76 |
+
print("✅ documents table created")
|
| 77 |
+
|
| 78 |
+
# Create index for vector similarity search
|
| 79 |
+
cur.execute("""
|
| 80 |
+
CREATE INDEX IF NOT EXISTS documents_embedding_idx
|
| 81 |
+
ON documents
|
| 82 |
+
USING ivfflat (embedding vector_cosine_ops)
|
| 83 |
+
WITH (lists = 100);
|
| 84 |
+
""")
|
| 85 |
+
print("✅ vector index created")
|
| 86 |
+
|
| 87 |
+
# Create index for tenant_id for faster filtering
|
| 88 |
+
cur.execute("""
|
| 89 |
+
CREATE INDEX IF NOT EXISTS documents_tenant_id_idx
|
| 90 |
+
ON documents (tenant_id);
|
| 91 |
+
""")
|
| 92 |
+
print("✅ tenant_id index created")
|
| 93 |
+
|
| 94 |
+
conn.commit()
|
| 95 |
+
cur.close()
|
| 96 |
+
conn.close()
|
| 97 |
+
print("✅ Database schema initialized successfully")
|
| 98 |
+
|
| 99 |
+
except Exception as e:
|
| 100 |
+
print(f"❌ Database initialization error: {e}")
|
| 101 |
+
# Don't raise - allow the app to continue even if table exists
|
| 102 |
+
if "already exists" not in str(e).lower():
|
| 103 |
+
raise
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
# -----------------------------------
|
| 107 |
+
# Document + Embedding Operations
|
| 108 |
+
# -----------------------------------
|
| 109 |
+
|
| 110 |
+
def insert_document_chunks(tenant_id: str, text: str, embedding: list):
|
| 111 |
+
"""
|
| 112 |
+
Insert document chunk + embedding.
|
| 113 |
+
"""
|
| 114 |
+
try:
|
| 115 |
+
conn = get_connection()
|
| 116 |
+
cur = conn.cursor()
|
| 117 |
+
|
| 118 |
+
cur.execute(
|
| 119 |
+
"""
|
| 120 |
+
INSERT INTO documents (tenant_id, chunk_text, embedding)
|
| 121 |
+
VALUES (%s, %s, %s);
|
| 122 |
+
""",
|
| 123 |
+
(tenant_id, text, embedding)
|
| 124 |
+
)
|
| 125 |
+
|
| 126 |
+
conn.commit()
|
| 127 |
+
cur.close()
|
| 128 |
+
conn.close()
|
| 129 |
+
|
| 130 |
+
except Exception as e:
|
| 131 |
+
print("DB INSERT ERROR:", e)
|
| 132 |
+
raise
|
| 133 |
+
|
| 134 |
+
|
| 135 |
+
def search_vectors(tenant_id: str, vector: list, limit: int = 5) -> List[str]:
|
| 136 |
+
"""
|
| 137 |
+
Perform semantic vector search using pgvector.
|
| 138 |
+
"""
|
| 139 |
+
try:
|
| 140 |
+
conn = get_connection()
|
| 141 |
+
cur = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
|
| 142 |
+
|
| 143 |
+
cur.execute(
|
| 144 |
+
"""
|
| 145 |
+
SELECT
|
| 146 |
+
chunk_text,
|
| 147 |
+
1 - (embedding <=> %s::vector(384)) AS similarity
|
| 148 |
+
FROM documents
|
| 149 |
+
WHERE tenant_id = %s
|
| 150 |
+
ORDER BY embedding <=> %s::vector(384)
|
| 151 |
+
LIMIT %s;
|
| 152 |
+
""",
|
| 153 |
+
(vector, tenant_id, vector, limit)
|
| 154 |
+
)
|
| 155 |
+
|
| 156 |
+
rows = cur.fetchall()
|
| 157 |
+
|
| 158 |
+
cur.close()
|
| 159 |
+
conn.close()
|
| 160 |
+
|
| 161 |
+
return [row["chunk_text"] for row in rows]
|
| 162 |
+
|
| 163 |
+
except Exception as e:
|
| 164 |
+
print("DB SEARCH ERROR:", e)
|
| 165 |
+
return []
|
| 166 |
+
|
| 167 |
+
|
| 168 |
+
# -----------------------------------
|
| 169 |
+
# Supabase Client (for REST operations)
|
| 170 |
+
# -----------------------------------
|
| 171 |
+
|
| 172 |
+
def get_supabase_client() -> Client:
|
| 173 |
+
"""
|
| 174 |
+
Get or create Supabase client.
|
| 175 |
+
"""
|
| 176 |
+
global _supabase_client
|
| 177 |
+
|
| 178 |
+
if _supabase_client is None:
|
| 179 |
+
if not SUPABASE_URL or not SUPABASE_KEY:
|
| 180 |
+
raise ValueError(
|
| 181 |
+
"Supabase credentials missing. "
|
| 182 |
+
"Set SUPABASE_URL and SUPABASE_SERVICE_KEY."
|
| 183 |
+
)
|
| 184 |
+
|
| 185 |
+
_supabase_client = create_client(SUPABASE_URL, SUPABASE_KEY)
|
| 186 |
+
|
| 187 |
+
return _supabase_client
|
| 188 |
+
|
| 189 |
+
|
| 190 |
+
def reset_client():
|
| 191 |
+
global _supabase_client
|
| 192 |
+
_supabase_client = None
|
| 193 |
+
|
| 194 |
+
|
| 195 |
+
# Table names
|
| 196 |
+
TABLES = {
|
| 197 |
+
"tenants": "tenants",
|
| 198 |
+
"documents": "documents",
|
| 199 |
+
"embeddings": "tenant_embeddings",
|
| 200 |
+
"redflag_rules": "redflag_rules",
|
| 201 |
+
"analytics": "analytics_events",
|
| 202 |
+
"tool_usage": "tool_usage_stats",
|
| 203 |
+
}
|
backend/mcp_servers/embeddings.py
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from sentence_transformers import SentenceTransformer
|
| 2 |
+
|
| 3 |
+
# Load MiniLM model (384-dimensional embeddings)
|
| 4 |
+
model = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2")
|
| 5 |
+
|
| 6 |
+
def embed_text(text: str):
|
| 7 |
+
"""
|
| 8 |
+
Generate sentence embedding for use with pgvector.
|
| 9 |
+
|
| 10 |
+
Args:
|
| 11 |
+
text (str): Input text
|
| 12 |
+
|
| 13 |
+
Returns:
|
| 14 |
+
List[float]: 384-dimensional embedding vector
|
| 15 |
+
"""
|
| 16 |
+
vector = model.encode(text)
|
| 17 |
+
return vector.tolist()
|
backend/mcp_servers/main.py
ADDED
|
@@ -0,0 +1,172 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from fastapi import FastAPI, HTTPException
|
| 2 |
+
from pydantic import BaseModel
|
| 3 |
+
from fastapi.middleware.cors import CORSMiddleware
|
| 4 |
+
from dotenv import load_dotenv
|
| 5 |
+
import sys
|
| 6 |
+
import os
|
| 7 |
+
|
| 8 |
+
# --------------------------------------------------------
|
| 9 |
+
# Fix Python module paths
|
| 10 |
+
# --------------------------------------------------------
|
| 11 |
+
|
| 12 |
+
current_dir = os.path.dirname(__file__)
|
| 13 |
+
parent_dir = os.path.dirname(current_dir)
|
| 14 |
+
|
| 15 |
+
sys.path.insert(0, current_dir) # For embeddings + database
|
| 16 |
+
sys.path.insert(0, os.path.join(parent_dir, "api")) # For utils
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
# --------------------------------------------------------
|
| 20 |
+
# Imports AFTER adjusting paths
|
| 21 |
+
# --------------------------------------------------------
|
| 22 |
+
|
| 23 |
+
from embeddings import embed_text
|
| 24 |
+
from database import insert_document_chunks, search_vectors, initialize_database
|
| 25 |
+
from utils.text_extractor import extract_text
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
# --------------------------------------------------------
|
| 29 |
+
# Load environment variables
|
| 30 |
+
# --------------------------------------------------------
|
| 31 |
+
|
| 32 |
+
load_dotenv()
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
# --------------------------------------------------------
|
| 36 |
+
# FastAPI App
|
| 37 |
+
# --------------------------------------------------------
|
| 38 |
+
|
| 39 |
+
app = FastAPI(
|
| 40 |
+
title="RAG MCP Server",
|
| 41 |
+
description="Provides semantic search + ingestion for tenant knowledge bases",
|
| 42 |
+
version="1.0.0"
|
| 43 |
+
)
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
# --------------------------------------------------------
|
| 47 |
+
# Enable CORS
|
| 48 |
+
# --------------------------------------------------------
|
| 49 |
+
|
| 50 |
+
app.add_middleware(
|
| 51 |
+
CORSMiddleware,
|
| 52 |
+
allow_origins=["*"],
|
| 53 |
+
allow_credentials=True,
|
| 54 |
+
allow_methods=["*"],
|
| 55 |
+
allow_headers=["*"],
|
| 56 |
+
)
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
# --------------------------------------------------------
|
| 60 |
+
# Startup Event - Initialize Database
|
| 61 |
+
# --------------------------------------------------------
|
| 62 |
+
|
| 63 |
+
@app.on_event("startup")
|
| 64 |
+
async def startup_event():
|
| 65 |
+
"""Initialize database schema on server startup."""
|
| 66 |
+
try:
|
| 67 |
+
print("Initializing database schema...")
|
| 68 |
+
initialize_database()
|
| 69 |
+
except Exception as e:
|
| 70 |
+
print(f"Warning: Database initialization failed: {e}")
|
| 71 |
+
print("Server will continue, but database operations may fail.")
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
# --------------------------------------------------------
|
| 75 |
+
# Request Models
|
| 76 |
+
# --------------------------------------------------------
|
| 77 |
+
|
| 78 |
+
class IngestPayload(BaseModel):
|
| 79 |
+
tenant_id: str
|
| 80 |
+
content: str
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
class SearchPayload(BaseModel):
|
| 84 |
+
query: str
|
| 85 |
+
tenant_id: str
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
# --------------------------------------------------------
|
| 89 |
+
# Health Check
|
| 90 |
+
# --------------------------------------------------------
|
| 91 |
+
|
| 92 |
+
@app.get("/")
|
| 93 |
+
def root():
|
| 94 |
+
return {"status": "RAG MCP SERVER RUNNING"}
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
# --------------------------------------------------------
|
| 98 |
+
# Ingest Route
|
| 99 |
+
# --------------------------------------------------------
|
| 100 |
+
|
| 101 |
+
@app.post("/ingest")
|
| 102 |
+
def ingest(payload: IngestPayload):
|
| 103 |
+
"""
|
| 104 |
+
Ingest raw text:
|
| 105 |
+
- Chunk text
|
| 106 |
+
- Embed chunks
|
| 107 |
+
- Store in Postgres
|
| 108 |
+
"""
|
| 109 |
+
try:
|
| 110 |
+
chunks = extract_text(payload.content)
|
| 111 |
+
|
| 112 |
+
if not chunks:
|
| 113 |
+
raise HTTPException(400, "No text found to ingest.")
|
| 114 |
+
|
| 115 |
+
inserted = 0
|
| 116 |
+
|
| 117 |
+
for chunk in chunks:
|
| 118 |
+
embedding = embed_text(chunk)
|
| 119 |
+
insert_document_chunks(payload.tenant_id, chunk, embedding)
|
| 120 |
+
inserted += 1
|
| 121 |
+
|
| 122 |
+
return {
|
| 123 |
+
"status": "ok",
|
| 124 |
+
"tenant_id": payload.tenant_id,
|
| 125 |
+
"chunks_stored": inserted
|
| 126 |
+
}
|
| 127 |
+
|
| 128 |
+
except Exception as e:
|
| 129 |
+
raise HTTPException(status_code=500, detail=str(e))
|
| 130 |
+
|
| 131 |
+
|
| 132 |
+
# --------------------------------------------------------
|
| 133 |
+
# Search Route
|
| 134 |
+
# --------------------------------------------------------
|
| 135 |
+
|
| 136 |
+
@app.post("/search")
|
| 137 |
+
def search(payload: SearchPayload):
|
| 138 |
+
"""
|
| 139 |
+
Semantic search using pgvector + MiniLM embeddings.
|
| 140 |
+
"""
|
| 141 |
+
try:
|
| 142 |
+
query_embedding = embed_text(payload.query)
|
| 143 |
+
results = search_vectors(payload.tenant_id, query_embedding, limit=5)
|
| 144 |
+
|
| 145 |
+
return {
|
| 146 |
+
"tenant_id": payload.tenant_id,
|
| 147 |
+
"query": payload.query,
|
| 148 |
+
"results": results
|
| 149 |
+
}
|
| 150 |
+
|
| 151 |
+
except Exception as e:
|
| 152 |
+
raise HTTPException(status_code=500, detail=str(e))
|
| 153 |
+
|
| 154 |
+
|
| 155 |
+
# --------------------------------------------------------
|
| 156 |
+
# Allow "python main.py" to start server
|
| 157 |
+
# --------------------------------------------------------
|
| 158 |
+
|
| 159 |
+
if __name__ == "__main__":
|
| 160 |
+
import uvicorn
|
| 161 |
+
|
| 162 |
+
print("Starting RAG MCP Server on http://0.0.0.0:8001")
|
| 163 |
+
print("API Documentation: http://localhost:8001/docs")
|
| 164 |
+
print("Note: Reload mode disabled when running directly")
|
| 165 |
+
|
| 166 |
+
# Run the app directly (reload doesn't work with app object)
|
| 167 |
+
uvicorn.run(
|
| 168 |
+
app, # Pass the app object directly
|
| 169 |
+
host="0.0.0.0",
|
| 170 |
+
port=8001,
|
| 171 |
+
reload=False # Reload requires module path, not app object
|
| 172 |
+
)
|
backend/tests/test_agent_orchestrator.py
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import sys
|
| 2 |
+
from pathlib import Path
|
| 3 |
+
|
| 4 |
+
# Add backend directory to Python path
|
| 5 |
+
backend_dir = Path(__file__).parent.parent
|
| 6 |
+
sys.path.insert(0, str(backend_dir))
|
| 7 |
+
|
| 8 |
+
import asyncio
|
| 9 |
+
from api.services.agent_orchestrator import AgentOrchestrator
|
| 10 |
+
|
| 11 |
+
agent = AgentOrchestrator()
|
| 12 |
+
|
| 13 |
+
async def run():
|
| 14 |
+
result = await agent.process_message(
|
| 15 |
+
"summarize our internal policy",
|
| 16 |
+
tenant_id="tenant123"
|
| 17 |
+
)
|
| 18 |
+
print(result["response"])
|
| 19 |
+
print("Tool used:", result["tool"])
|
| 20 |
+
|
| 21 |
+
asyncio.run(run())
|
requirements.txt
CHANGED
|
@@ -1,3 +1,8 @@
|
|
| 1 |
fastapi
|
| 2 |
uvicorn
|
| 3 |
-
requests
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
fastapi
|
| 2 |
uvicorn
|
| 3 |
+
requests
|
| 4 |
+
httpx
|
| 5 |
+
python-dotenv
|
| 6 |
+
psycopg2
|
| 7 |
+
supabase
|
| 8 |
+
sentence-transformers
|