Spaces:
Sleeping
Sleeping
Commit ·
f775c99
0
Parent(s):
Axiom V4.6: Production-Locked. Multilingual Neural Core Operational.
Browse files- .env.example +4 -0
- .gitignore +5 -0
- .gitkeep +0 -0
- Dockerfile +49 -0
- README.md +20 -0
- app/__init__.py +1 -0
- app/agents/graph.py +85 -0
- app/agents/nodes.py +172 -0
- app/agents/state.py +36 -0
- app/api/__init__.py +3 -0
- app/api/history.py +101 -0
- app/api/ingest.py +240 -0
- app/api/keys.py +84 -0
- app/api/run.py +156 -0
- app/api/vault.py +68 -0
- app/core/auth.py +139 -0
- app/core/chunking.py +92 -0
- app/core/database.py +41 -0
- app/core/embeddings.py +90 -0
- app/core/evaluator.py +90 -0
- app/core/monitor.py +41 -0
- app/core/reranker.py +59 -0
- app/core/retriever.py +110 -0
- app/main.py +83 -0
- app/mcp/__init__.py +0 -0
- app/mcp/server.py +115 -0
- app/prompts/__init__.py +1 -0
- app/prompts/templates.py +137 -0
- app/skills/__init__.py +1 -0
- app/skills/database.py +138 -0
- app/skills/github.py +82 -0
- migrations/001_init_vault.sql +171 -0
- requirements.txt +54 -0
.env.example
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
SUPABASE_URL=https://your-project.supabase.co
|
| 2 |
+
SUPABASE_SERVICE_KEY=your-service-role-key
|
| 3 |
+
GROQ_API_KEY=gsk_...
|
| 4 |
+
EMBEDDING_MODE=local
|
.gitignore
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
node_modules/
|
| 2 |
+
.env
|
| 3 |
+
.next/
|
| 4 |
+
__pycache__/
|
| 5 |
+
*.pyc
|
.gitkeep
ADDED
|
File without changes
|
Dockerfile
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# 1. SOTA Base Image (Lean & Stable)
|
| 2 |
+
FROM python:3.11-slim
|
| 3 |
+
|
| 4 |
+
# 2. Environment Hardening
|
| 5 |
+
ENV PYTHONUNBUFFERED=1 \
|
| 6 |
+
PYTHONDONTWRITEBYTECODE=1 \
|
| 7 |
+
PIP_NO_CACHE_DIR=1 \
|
| 8 |
+
PIP_DISABLE_PIP_VERSION_CHECK=1 \
|
| 9 |
+
HOME=/home/user \
|
| 10 |
+
PATH=/home/user/.local/bin:$PATH \
|
| 11 |
+
HF_HUB_OFFLINE=0
|
| 12 |
+
|
| 13 |
+
# 3. System Intelligence (Hardened for Docling V2 & Python-Magic)
|
| 14 |
+
RUN apt-get update && apt-get install -y --no-install-recommends \
|
| 15 |
+
build-essential \
|
| 16 |
+
libmagic1 \
|
| 17 |
+
libmagic-dev \
|
| 18 |
+
libgomp1 \
|
| 19 |
+
poppler-utils \
|
| 20 |
+
tesseract-ocr \
|
| 21 |
+
libgl1 \
|
| 22 |
+
libglib2.0-0 \
|
| 23 |
+
libxml2-dev \
|
| 24 |
+
libxslt-dev \
|
| 25 |
+
curl \
|
| 26 |
+
&& rm -rf /var/lib/apt/lists/*
|
| 27 |
+
|
| 28 |
+
# 4. Secure User Architecture (Hugging Face Standard)
|
| 29 |
+
RUN useradd -m -u 1000 user
|
| 30 |
+
USER user
|
| 31 |
+
WORKDIR $HOME/app
|
| 32 |
+
|
| 33 |
+
# 5. Ingestion Buffer Setup
|
| 34 |
+
# Ensure the temp directory for Docling exists and is writable
|
| 35 |
+
RUN mkdir -p /tmp/axiom_ingest && chmod 777 /tmp/axiom_ingest
|
| 36 |
+
|
| 37 |
+
# 6. Dependency Hydration
|
| 38 |
+
COPY --chown=user requirements.txt .
|
| 39 |
+
RUN pip install --no-cache-dir --user -r requirements.txt
|
| 40 |
+
|
| 41 |
+
# 7. Application Ingestion
|
| 42 |
+
COPY --chown=user . .
|
| 43 |
+
|
| 44 |
+
# 8. Port Specification
|
| 45 |
+
EXPOSE 7860
|
| 46 |
+
|
| 47 |
+
# 9. Start Engine (SOTA Event Loop Optimization)
|
| 48 |
+
# --loop asyncio is mandatory for RAGAS 0.2/nest_asyncio compatibility
|
| 49 |
+
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "7860", "--loop", "asyncio", "--proxy-headers", "--forwarded-allow-ips", "*"]
|
README.md
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
title: Axiom Engine API
|
| 3 |
+
emoji: 🛡️
|
| 4 |
+
colorFrom: green
|
| 5 |
+
colorTo: blue
|
| 6 |
+
sdk: docker
|
| 7 |
+
app_port: 7860
|
| 8 |
+
pinned: false
|
| 9 |
+
---
|
| 10 |
+
|
| 11 |
+
# Axiom Engine — Reasoning Core
|
| 12 |
+
Enterprise Evidence-Gated RAG API.
|
| 13 |
+
|
| 14 |
+
## Technical Specifications
|
| 15 |
+
- **Runtime:** Python 3.11 FastAPI
|
| 16 |
+
- **Intelligence:** LangGraph + Llama 3.3
|
| 17 |
+
- **Vector Engine:** Supabase 1024-dim (Local)
|
| 18 |
+
- **Deployment:** Docker (Hugging Face Spaces)
|
| 19 |
+
|
| 20 |
+
*Strictly for internal Lexpertz AI organizational use.*
|
app/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
|
app/agents/graph.py
ADDED
|
@@ -0,0 +1,85 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from langgraph.graph import StateGraph, END, START
|
| 2 |
+
from app.agents.state import AgentState
|
| 3 |
+
from app.agents.nodes import (
|
| 4 |
+
retrieve_node,
|
| 5 |
+
distill_node,
|
| 6 |
+
generate_node,
|
| 7 |
+
grade_generation_node,
|
| 8 |
+
strategist_node
|
| 9 |
+
)
|
| 10 |
+
|
| 11 |
+
# --- 1. ROUTING LOGIC ---
|
| 12 |
+
|
| 13 |
+
def route_post_retrieval(state: AgentState):
|
| 14 |
+
"""
|
| 15 |
+
Determines path based on evidence status, document count, and AXM-CLI commands.
|
| 16 |
+
"""
|
| 17 |
+
if state.get("status") == "no_evidence":
|
| 18 |
+
return "end"
|
| 19 |
+
|
| 20 |
+
command = state.get("command") or ""
|
| 21 |
+
filenames = state.get("filenames",[])
|
| 22 |
+
|
| 23 |
+
# SOTA: Multi-Flag Check. If "-c" is anywhere in the command string.
|
| 24 |
+
if "-c" in command or len(filenames) > 1:
|
| 25 |
+
return "strategist"
|
| 26 |
+
|
| 27 |
+
return "distill"
|
| 28 |
+
|
| 29 |
+
def route_post_grading(state: AgentState):
|
| 30 |
+
"""
|
| 31 |
+
The Adversarial Gatekeeper. Routes to Architect for retry.
|
| 32 |
+
"""
|
| 33 |
+
if state.get("status") == "verified":
|
| 34 |
+
return "end"
|
| 35 |
+
|
| 36 |
+
# Safety: Limit retry recursion to prevent token burn
|
| 37 |
+
current_retries = state.get("retry_count", 0)
|
| 38 |
+
if current_retries < 2:
|
| 39 |
+
return "retry"
|
| 40 |
+
|
| 41 |
+
return "end"
|
| 42 |
+
|
| 43 |
+
# --- 2. THE CIRCUIT DESIGN ---
|
| 44 |
+
workflow = StateGraph(AgentState)
|
| 45 |
+
|
| 46 |
+
# Register Professional Agent Nodes
|
| 47 |
+
workflow.add_node("Librarian", retrieve_node)
|
| 48 |
+
workflow.add_node("Editor", distill_node)
|
| 49 |
+
workflow.add_node("Strategist", strategist_node)
|
| 50 |
+
workflow.add_node("Architect", generate_node)
|
| 51 |
+
workflow.add_node("Prosecutor", grade_generation_node)
|
| 52 |
+
|
| 53 |
+
# --- 3. THE WIRING ---
|
| 54 |
+
|
| 55 |
+
# Start Node
|
| 56 |
+
workflow.add_edge(START, "Librarian")
|
| 57 |
+
|
| 58 |
+
# A. Gate 1: Post-Retrieval Routing
|
| 59 |
+
workflow.add_conditional_edges(
|
| 60 |
+
"Librarian",
|
| 61 |
+
route_post_retrieval,
|
| 62 |
+
{
|
| 63 |
+
"end": END,
|
| 64 |
+
"strategist": "Strategist",
|
| 65 |
+
"distill": "Editor"
|
| 66 |
+
}
|
| 67 |
+
)
|
| 68 |
+
|
| 69 |
+
# B. Reasoning & Comparison Processing
|
| 70 |
+
workflow.add_edge("Editor", "Architect")
|
| 71 |
+
workflow.add_edge("Strategist", "Architect")
|
| 72 |
+
workflow.add_edge("Architect", "Prosecutor")
|
| 73 |
+
|
| 74 |
+
# C. Gate 2: The Adversarial/Retry Loop
|
| 75 |
+
workflow.add_conditional_edges(
|
| 76 |
+
"Prosecutor",
|
| 77 |
+
route_post_grading,
|
| 78 |
+
{
|
| 79 |
+
"end": END,
|
| 80 |
+
"retry": "Architect"
|
| 81 |
+
}
|
| 82 |
+
)
|
| 83 |
+
|
| 84 |
+
# Compile the Sovereign Brain
|
| 85 |
+
app_graph = workflow.compile()
|
app/agents/nodes.py
ADDED
|
@@ -0,0 +1,172 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import time
|
| 3 |
+
import re
|
| 4 |
+
from typing import cast, List, Dict, Any, Union, Optional
|
| 5 |
+
|
| 6 |
+
from langchain_core.caches import BaseCache
|
| 7 |
+
from langchain_core.callbacks import Callbacks
|
| 8 |
+
from langchain_core.outputs import ChatResult
|
| 9 |
+
from pydantic import SecretStr
|
| 10 |
+
|
| 11 |
+
from langchain_groq import ChatGroq
|
| 12 |
+
from langchain_nvidia_ai_endpoints import ChatNVIDIA
|
| 13 |
+
|
| 14 |
+
from app.agents.state import AgentState
|
| 15 |
+
from app.core.retriever import hybrid_search
|
| 16 |
+
from app.core.reranker import get_reranked_scores
|
| 17 |
+
from app.core.monitor import monitor
|
| 18 |
+
from app.core.evaluator import axiom_evaluator
|
| 19 |
+
|
| 20 |
+
# IMPORTING THE ENTERPRISE REGISTRY
|
| 21 |
+
from app.prompts.templates import (
|
| 22 |
+
VERIFICATION_PROMPT,
|
| 23 |
+
DISTILLATION_PROMPT,
|
| 24 |
+
STRATEGIST_COMPARATIVE_PROMPT,
|
| 25 |
+
GRADING_PROMPT,
|
| 26 |
+
distill_parser,
|
| 27 |
+
grade_parser
|
| 28 |
+
)
|
| 29 |
+
|
| 30 |
+
try:
|
| 31 |
+
ChatGroq.model_rebuild()
|
| 32 |
+
ChatNVIDIA.model_rebuild()
|
| 33 |
+
print("AXIOM-CORE: Neural Registry Stabilized.")
|
| 34 |
+
except Exception as e:
|
| 35 |
+
print(f"AXIOM-CORE: Model rebuild notice: {e}")
|
| 36 |
+
|
| 37 |
+
_nv_key = os.getenv("NVIDIA_API_KEY")
|
| 38 |
+
_groq_key = os.getenv("GROQ_API_KEY")
|
| 39 |
+
|
| 40 |
+
base_llm: Any
|
| 41 |
+
editor_llm_core: Any
|
| 42 |
+
prosecutor_llm_core: Any
|
| 43 |
+
|
| 44 |
+
if _nv_key:
|
| 45 |
+
try:
|
| 46 |
+
base_llm = ChatNVIDIA(model="meta/llama-3.3-70b-instruct", nvidia_api_key=_nv_key, temperature=0, max_tokens=2048)
|
| 47 |
+
editor_llm_core = ChatNVIDIA(model="nvidia/nvidia-nemotron-nano-9b-v2", nvidia_api_key=_nv_key, temperature=0.1, max_tokens=1024)
|
| 48 |
+
prosecutor_llm_core = ChatNVIDIA(model="meta/llama-3.1-405b-instruct", nvidia_api_key=_nv_key, temperature=0, max_tokens=512)
|
| 49 |
+
except: _nv_key = None
|
| 50 |
+
|
| 51 |
+
if not _nv_key:
|
| 52 |
+
base_llm = ChatGroq(temperature=0, model="llama-3.3-70b-versatile", api_key=SecretStr(_groq_key) if _groq_key else None) # type: ignore
|
| 53 |
+
editor_llm_core = ChatGroq(temperature=0, model="llama-3.1-8b-instant", api_key=SecretStr(_groq_key) if _groq_key else None) # type: ignore
|
| 54 |
+
prosecutor_llm_core = base_llm
|
| 55 |
+
|
| 56 |
+
simple_llm = base_llm
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
async def retrieve_node(state: AgentState):
|
| 60 |
+
raw_question = state["question"].strip()
|
| 61 |
+
command = None
|
| 62 |
+
clean_question = raw_question
|
| 63 |
+
|
| 64 |
+
# SOTA: Multi-Flag Parser (Captures '-a -t -v' as a single block)
|
| 65 |
+
cmd_match = re.match(r'^/axm\s+((?:-[a-z]+\s*|\.\.\s*)+)(.*)', raw_question, re.IGNORECASE | re.DOTALL)
|
| 66 |
+
if cmd_match:
|
| 67 |
+
command = cmd_match.group(1).strip().lower()
|
| 68 |
+
clean_question = cmd_match.group(2).strip()
|
| 69 |
+
print(f"AXM-CLI: Detected Commands [{command}]")
|
| 70 |
+
|
| 71 |
+
filenames = state.get("filenames",[])
|
| 72 |
+
is_vault_mode = "vault" in filenames or len(filenames) == 0
|
| 73 |
+
search_input = None if is_vault_mode else filenames
|
| 74 |
+
|
| 75 |
+
# SOTA: Use 'in' to check for specific flags inside the chained command
|
| 76 |
+
is_deep_audit = command and "-a" in command
|
| 77 |
+
search_limit = 60 if is_deep_audit else 30
|
| 78 |
+
top_k = 20 if is_deep_audit else 12
|
| 79 |
+
|
| 80 |
+
initial_chunks = await hybrid_search(query=clean_question, user_id=state["user_id"], filename=search_input, limit=search_limit)
|
| 81 |
+
if not initial_chunks:
|
| 82 |
+
return {"documents":[], "generation": "Insufficient Evidence.", "status": "no_evidence", "command": command, "question": clean_question}
|
| 83 |
+
|
| 84 |
+
gold_chunks = await get_reranked_scores(query=clean_question, documents=initial_chunks, top_k=top_k)
|
| 85 |
+
return {"documents": gold_chunks, "status": "thinking", "active_node": "Librarian", "command": command, "question": clean_question }
|
| 86 |
+
|
| 87 |
+
async def distill_node(state: AgentState):
|
| 88 |
+
context_text = monitor.guard_context(state["documents"])
|
| 89 |
+
if not context_text.strip():
|
| 90 |
+
return {"generation": "NO RELEVANT EVIDENCE", "status": "thinking"}
|
| 91 |
+
|
| 92 |
+
chain = DISTILLATION_PROMPT | base_llm | distill_parser
|
| 93 |
+
|
| 94 |
+
try:
|
| 95 |
+
raw_response = await chain.ainvoke({"context": context_text, "question": state["question"]})
|
| 96 |
+
|
| 97 |
+
brief_content = raw_response.brief
|
| 98 |
+
preambles_to_strip =["Here is the synthesized evidence brief:", "Based on the provided snippets:", "Synthesized Evidence Brief:", "Here is the brief:"]
|
| 99 |
+
for preamble in preambles_to_strip:
|
| 100 |
+
brief_content = brief_content.replace(preamble, "")
|
| 101 |
+
|
| 102 |
+
return {"generation": brief_content.strip() if raw_response.has_relevant_evidence else "NO RELEVANT EVIDENCE", "status": "thinking", "active_node": "Editor"}
|
| 103 |
+
except Exception as e:
|
| 104 |
+
print(f"⚠️ EDITOR JSON FAILSAFE TRIGGERED: {e}")
|
| 105 |
+
cleaned_context = re.sub(r'--- EXHIBIT_(START|END)_ID_\w+ ---', '', context_text)
|
| 106 |
+
return {"generation": cleaned_context[:6000], "status": "thinking", "active_node": "Editor"}
|
| 107 |
+
|
| 108 |
+
async def strategist_node(state: AgentState):
|
| 109 |
+
context_text = monitor.guard_context(state["documents"])
|
| 110 |
+
chain = STRATEGIST_COMPARATIVE_PROMPT | simple_llm
|
| 111 |
+
response = await chain.ainvoke({"context": context_text, "question": state["question"]})
|
| 112 |
+
return {"generation": str(response.content), "status": "thinking", "active_node": "Strategist"}
|
| 113 |
+
|
| 114 |
+
async def generate_node(state: AgentState):
|
| 115 |
+
distilled_brief = state["generation"]
|
| 116 |
+
command = state.get("command")
|
| 117 |
+
history = state.get("history",[])
|
| 118 |
+
|
| 119 |
+
if "NO RELEVANT EVIDENCE" in distilled_brief:
|
| 120 |
+
return {"generation": "No direct evidence found in the vault.", "status": "verifying"}
|
| 121 |
+
|
| 122 |
+
history_context = ""
|
| 123 |
+
# SOTA Check: Is the reset flag present?
|
| 124 |
+
if history and (not command or ".." not in command):
|
| 125 |
+
history_context = "\n\n### PREVIOUS AUDIT CONTEXT:\n"
|
| 126 |
+
for turn in history[-3:]:
|
| 127 |
+
history_context += f"{turn['role'].upper()}: {turn['content']}\n"
|
| 128 |
+
|
| 129 |
+
# SOTA Check: Is the table flag present?
|
| 130 |
+
formatting_directive = "\n\nCRITICAL: You are in TABLE MODE. Output strictly as a Markdown Data Grid." if command and "-t" in command else ""
|
| 131 |
+
|
| 132 |
+
chain = VERIFICATION_PROMPT | simple_llm
|
| 133 |
+
response = await chain.ainvoke({"context": f"{history_context}\n\nEVIDENCE:\n{distilled_brief}{formatting_directive}", "question": state["question"]})
|
| 134 |
+
return {"generation": str(response.content), "status": "verifying", "active_node": "Architect"}
|
| 135 |
+
|
| 136 |
+
async def grade_generation_node(state: AgentState):
|
| 137 |
+
generation = state.get("generation", "")
|
| 138 |
+
if "No direct evidence found" in generation or not generation.strip():
|
| 139 |
+
return {"hallucination_score": 1.0, "metrics": {"faithfulness": 1.0, "precision": 1.0, "relevance": 1.0}, "status": "verified", "active_node": "Prosecutor"}
|
| 140 |
+
|
| 141 |
+
command = state.get("command")
|
| 142 |
+
# SOTA Check: Is the intense verification flag present?
|
| 143 |
+
intensify = command is not None and "-v" in command
|
| 144 |
+
|
| 145 |
+
context_list = state["documents"]
|
| 146 |
+
context_str = "\n\n".join(context_list)
|
| 147 |
+
|
| 148 |
+
try:
|
| 149 |
+
chain = GRADING_PROMPT | prosecutor_llm_core | grade_parser
|
| 150 |
+
grade = await chain.ainvoke({"context": context_str, "generation": generation})
|
| 151 |
+
|
| 152 |
+
if str(grade.is_hallucinating).strip().lower() == "true":
|
| 153 |
+
print(f"LOGIC BREACH (NIM): {grade.explanation}")
|
| 154 |
+
return {"hallucination_score": 0.0, "status": "thinking", "retry_count": state.get("retry_count", 0) + 1, "active_node": "Prosecutor"}
|
| 155 |
+
except Exception as e:
|
| 156 |
+
print(f"⚠️ PROSECUTOR JSON FAILSAFE: {e}")
|
| 157 |
+
pass
|
| 158 |
+
|
| 159 |
+
try:
|
| 160 |
+
print("--- AXIOM: EXECUTING RAGAS MATHEMATICAL AUDIT ---")
|
| 161 |
+
scores = await axiom_evaluator.score_response(state["question"], generation, context_list)
|
| 162 |
+
faith = scores.get('faithfulness', 0.0)
|
| 163 |
+
threshold = 0.9 if intensify else 0.7
|
| 164 |
+
|
| 165 |
+
if faith < threshold:
|
| 166 |
+
print(f"FAITHFULNESS BREACH: {faith} (Threshold: {threshold})")
|
| 167 |
+
return {"hallucination_score": faith, "metrics": scores, "status": "thinking", "retry_count": state.get("retry_count", 0) + 1, "active_node": "Prosecutor"}
|
| 168 |
+
|
| 169 |
+
return {"hallucination_score": faith, "metrics": scores, "status": "verified", "active_node": "Prosecutor"}
|
| 170 |
+
except Exception as e:
|
| 171 |
+
print(f"⚠️ PROSECUTOR SYSTEM FAILSAFE: {e}")
|
| 172 |
+
return {"hallucination_score": 0.5, "status": "verified", "active_node": "Prosecutor", "metrics": {"faithfulness": 0.5, "precision": 1.0, "relevance": 1.0}}
|
app/agents/state.py
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import TypedDict, List, Dict, Any, Optional
|
| 2 |
+
|
| 3 |
+
class AgentState(TypedDict):
|
| 4 |
+
"""
|
| 5 |
+
The Axiom Sovereign State Machine V4.6.
|
| 6 |
+
Synchronized for LangGraph 0.2.x and Command-Aware Auditing.
|
| 7 |
+
"""
|
| 8 |
+
# --- Input Context ---
|
| 9 |
+
question: str
|
| 10 |
+
user_id: str
|
| 11 |
+
filenames: List[str]
|
| 12 |
+
|
| 13 |
+
# --- V4.6 Persistent Memory ---
|
| 14 |
+
# history: Stores last 5 turns: [{"role": "user", "content": "..."}, ...]
|
| 15 |
+
history: List[Dict[str, str]]
|
| 16 |
+
|
| 17 |
+
# --- V4.6 Command Register ---
|
| 18 |
+
# command: Stores the specific /axm shorthand (e.g., "-a", "-t", "..")
|
| 19 |
+
command: Optional[str]
|
| 20 |
+
|
| 21 |
+
# --- Working Memory ---
|
| 22 |
+
# comparison_map: Intermediate JSON extraction for multi-doc audits
|
| 23 |
+
comparison_map: Dict[str, Any]
|
| 24 |
+
documents: List[str]
|
| 25 |
+
|
| 26 |
+
# --- Output State ---
|
| 27 |
+
generation: str
|
| 28 |
+
status: str
|
| 29 |
+
|
| 30 |
+
# --- Telemetry & Logic Control ---
|
| 31 |
+
hallucination_score: float
|
| 32 |
+
metrics: Dict[str, float]
|
| 33 |
+
retry_count: int
|
| 34 |
+
|
| 35 |
+
# SSE Observability (Step 0, Step 1, etc.)
|
| 36 |
+
active_node: Optional[str]
|
app/api/__init__.py
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Expose routers for cleaner imports in main.py,
|
| 2 |
+
from . import ingest
|
| 3 |
+
from . import run
|
app/api/history.py
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import asyncio
|
| 2 |
+
from fastapi import APIRouter, Depends, HTTPException, Query, Path
|
| 3 |
+
from pydantic import BaseModel, Field
|
| 4 |
+
from typing import List, Dict, Any, cast, Optional
|
| 5 |
+
from app.core.database import db
|
| 6 |
+
from app.core.auth import get_current_user
|
| 7 |
+
|
| 8 |
+
router = APIRouter()
|
| 9 |
+
|
| 10 |
+
# --- 1. RESPONSE MODELS (Strict Serialization) ---
|
| 11 |
+
class DocumentItem(BaseModel):
|
| 12 |
+
filename: str
|
| 13 |
+
status: str
|
| 14 |
+
created_at: str
|
| 15 |
+
|
| 16 |
+
class ChatMessageItem(BaseModel):
|
| 17 |
+
id: int
|
| 18 |
+
role: str
|
| 19 |
+
content: str
|
| 20 |
+
metrics: Optional[Dict[str, float]] = None
|
| 21 |
+
created_at: str
|
| 22 |
+
|
| 23 |
+
# --- 2. ENDPOINTS ---
|
| 24 |
+
|
| 25 |
+
@router.get("/documents", response_model=List[DocumentItem])
|
| 26 |
+
async def get_user_documents(
|
| 27 |
+
# Bounded Limit: Max 100 documents at a time to prevent RAM exhaustion
|
| 28 |
+
limit: int = Query(100, ge=1, le=500, description="Pagination limit"),
|
| 29 |
+
user_id: str = Depends(get_current_user)
|
| 30 |
+
):
|
| 31 |
+
"""
|
| 32 |
+
SOTA Document Fetcher:
|
| 33 |
+
Retrieves the latest documents asynchronously.
|
| 34 |
+
"""
|
| 35 |
+
if not db:
|
| 36 |
+
raise HTTPException(status_code=503, detail="Vault DB Offline")
|
| 37 |
+
|
| 38 |
+
try:
|
| 39 |
+
# Non-blocking Database Call
|
| 40 |
+
res = await asyncio.to_thread(
|
| 41 |
+
lambda: db.table("documents")
|
| 42 |
+
.select("filename, status, created_at")
|
| 43 |
+
.eq("user_id", user_id)
|
| 44 |
+
.order("created_at", desc=True)
|
| 45 |
+
.limit(limit)
|
| 46 |
+
.execute()
|
| 47 |
+
)
|
| 48 |
+
|
| 49 |
+
return cast(List[Dict[str, Any]], res.data)
|
| 50 |
+
except Exception as e:
|
| 51 |
+
print(f"❌ DOCUMENT FETCH ERROR: {e}")
|
| 52 |
+
raise HTTPException(status_code=500, detail="Failed to retrieve vault ledger")
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
@router.get("/chat/{filename}", response_model=List[ChatMessageItem])
|
| 56 |
+
async def get_document_chat_history(
|
| 57 |
+
# Path Validation: Protects against directory traversal and malformed URLs
|
| 58 |
+
filename: str = Path(..., min_length=1, max_length=255),
|
| 59 |
+
# Chat Limit: Fetch only the most recent 50 messages to keep UI snappy
|
| 60 |
+
limit: int = Query(50, ge=1, le=200),
|
| 61 |
+
user_id: str = Depends(get_current_user)
|
| 62 |
+
):
|
| 63 |
+
"""
|
| 64 |
+
V4.6 SOTA History Hydrator:
|
| 65 |
+
Fetches the latest conversation efficiently, offloaded to a background thread.
|
| 66 |
+
"""
|
| 67 |
+
if not db:
|
| 68 |
+
raise HTTPException(status_code=503, detail="Vault DB Offline")
|
| 69 |
+
|
| 70 |
+
try:
|
| 71 |
+
# 1. Non-blocking Document ID Fetch
|
| 72 |
+
doc_res = await asyncio.to_thread(
|
| 73 |
+
lambda: db.table("documents").select("id").eq("filename", filename).eq("user_id", user_id).execute()
|
| 74 |
+
)
|
| 75 |
+
doc_data = cast(List[Dict[str, Any]], doc_res.data)
|
| 76 |
+
|
| 77 |
+
if not doc_data:
|
| 78 |
+
return [] # Cleanly return empty history if document doesn't exist
|
| 79 |
+
|
| 80 |
+
doc_id = doc_data[0]['id']
|
| 81 |
+
|
| 82 |
+
# 2. Non-blocking Messages Fetch (Smart Chronology)
|
| 83 |
+
# We fetch the LATEST 'limit' messages first (desc=True)
|
| 84 |
+
msg_res = await asyncio.to_thread(
|
| 85 |
+
lambda: db.table("chat_messages")
|
| 86 |
+
.select("id, role, content, metrics, created_at")
|
| 87 |
+
.eq("document_id", doc_id)
|
| 88 |
+
.order("created_at", desc=True) # Get newest first
|
| 89 |
+
.limit(limit)
|
| 90 |
+
.execute()
|
| 91 |
+
)
|
| 92 |
+
|
| 93 |
+
messages = cast(List[Dict[str, Any]], msg_res.data)
|
| 94 |
+
|
| 95 |
+
# 3. Reverse the array in Python so the UI displays Oldest -> Newest
|
| 96 |
+
return messages[::-1]
|
| 97 |
+
|
| 98 |
+
except Exception as e:
|
| 99 |
+
print(f"❌ HISTORY FETCH ERROR: {str(e)}")
|
| 100 |
+
# Don't return an empty array on DB crash; tell the UI something went wrong
|
| 101 |
+
raise HTTPException(status_code=500, detail="Failed to retrieve audit history")
|
app/api/ingest.py
ADDED
|
@@ -0,0 +1,240 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import uuid
|
| 3 |
+
import math
|
| 4 |
+
import asyncio
|
| 5 |
+
from typing import Optional, List, Any, Dict, cast
|
| 6 |
+
from fastapi import APIRouter, UploadFile, File, HTTPException, BackgroundTasks, Depends, Path
|
| 7 |
+
from pydantic import BaseModel
|
| 8 |
+
|
| 9 |
+
from app.core.database import db
|
| 10 |
+
from app.core.chunking import chunker
|
| 11 |
+
from app.core.embeddings import get_embedding
|
| 12 |
+
from app.core.auth import get_current_user
|
| 13 |
+
|
| 14 |
+
# Lazy Initialization for Docling
|
| 15 |
+
_converter = None
|
| 16 |
+
|
| 17 |
+
def get_converter():
|
| 18 |
+
global _converter
|
| 19 |
+
if _converter is None:
|
| 20 |
+
print("AXIOM-CORE: Waking up Docling V2 Intelligence...")
|
| 21 |
+
from docling.document_converter import DocumentConverter # type: ignore
|
| 22 |
+
_converter = DocumentConverter()
|
| 23 |
+
return _converter
|
| 24 |
+
|
| 25 |
+
router = APIRouter()
|
| 26 |
+
TEMP_DIR = "/tmp/axiom_ingest"
|
| 27 |
+
|
| 28 |
+
# --- HELPER: CONCURRENT VECTORIZATION ---
|
| 29 |
+
async def fetch_embedding_concurrently(chunk_text: str, semaphore: asyncio.Semaphore) -> List[float]:
|
| 30 |
+
"""Bounds concurrent requests to NVIDIA to prevent 429 Rate Limits."""
|
| 31 |
+
async with semaphore:
|
| 32 |
+
return await asyncio.to_thread(get_embedding, chunk_text, input_type="document")
|
| 33 |
+
|
| 34 |
+
# --- THE SOTA BACKGROUND ENGINE ---
|
| 35 |
+
async def process_document(file_path: str, filename: str, user_id: str) -> None:
|
| 36 |
+
try:
|
| 37 |
+
print(f"AXIOM-CORE: Parsing {filename} (Async Mode)")
|
| 38 |
+
|
| 39 |
+
# 1. CPU-Bound Task Offloading (Prevents GIL Freeze)
|
| 40 |
+
converter = get_converter()
|
| 41 |
+
conv_result = await asyncio.to_thread(converter.convert, file_path)
|
| 42 |
+
markdown_content = conv_result.document.export_to_markdown()
|
| 43 |
+
|
| 44 |
+
# 2. Register Document (Non-blocking DB)
|
| 45 |
+
document_id: Optional[int] = None
|
| 46 |
+
if db:
|
| 47 |
+
doc_res = await asyncio.to_thread(
|
| 48 |
+
lambda: db.table("documents").insert({
|
| 49 |
+
"filename": filename, "user_id": user_id, "status": "processing", "is_permanent": False
|
| 50 |
+
}).execute()
|
| 51 |
+
)
|
| 52 |
+
data = cast(List[Dict[str, Any]], doc_res.data)
|
| 53 |
+
if data: document_id = data[0].get('id')
|
| 54 |
+
|
| 55 |
+
if not document_id: raise RuntimeError("DB Insert Failed")
|
| 56 |
+
|
| 57 |
+
# 3. Chunking
|
| 58 |
+
chunks = chunker.split_text(markdown_content)
|
| 59 |
+
print(f"AXIOM-CORE: Vectorizing {len(chunks)} chunks concurrently...")
|
| 60 |
+
|
| 61 |
+
# 4. SOTA: Concurrent Batch Vectorization (Massive Speedup)
|
| 62 |
+
# Limit to 5 parallel connections to respect NVIDIA NIM rate limits
|
| 63 |
+
semaphore = asyncio.Semaphore(5)
|
| 64 |
+
tasks =[fetch_embedding_concurrently(chunk, semaphore) for chunk in chunks]
|
| 65 |
+
vectors = await asyncio.gather(*tasks)
|
| 66 |
+
|
| 67 |
+
# 5. Assemble Payload
|
| 68 |
+
data_payload: List[Dict[str, Any]] =[]
|
| 69 |
+
for i, (chunk_text, vector) in enumerate(zip(chunks, vectors)):
|
| 70 |
+
data_payload.append({
|
| 71 |
+
"document_id": document_id, "user_id": user_id, "content": chunk_text,
|
| 72 |
+
"embedding": vector, "metadata": {"index": i, "source": filename, "engine": "docling-v2-nim"}
|
| 73 |
+
})
|
| 74 |
+
|
| 75 |
+
# 6. Non-Blocking Batch DB Insertion
|
| 76 |
+
if db:
|
| 77 |
+
# Strictly typed helper function to satisfy Mypy
|
| 78 |
+
def insert_batch(batch_data: List[Dict[str, Any]]) -> None:
|
| 79 |
+
db.table("document_chunks").insert(cast(Any, batch_data)).execute()
|
| 80 |
+
|
| 81 |
+
BATCH_SIZE = 50
|
| 82 |
+
for j in range(0, len(data_payload), BATCH_SIZE):
|
| 83 |
+
batch = data_payload[j : j + BATCH_SIZE]
|
| 84 |
+
# Pass the function and its arguments natively to to_thread
|
| 85 |
+
await asyncio.to_thread(insert_batch, batch)
|
| 86 |
+
|
| 87 |
+
# Helper for the status update to avoid another lambda
|
| 88 |
+
def update_status() -> None:
|
| 89 |
+
db.table("documents").update({"status": "indexed"}).eq("id", document_id).execute()
|
| 90 |
+
|
| 91 |
+
await asyncio.to_thread(update_status)
|
| 92 |
+
|
| 93 |
+
print(f"COMPLETE: {filename} indexed successfully.")
|
| 94 |
+
|
| 95 |
+
except Exception as e:
|
| 96 |
+
print(f"❌ INGESTION FAILED: {str(e)}")
|
| 97 |
+
if db:
|
| 98 |
+
await asyncio.to_thread(
|
| 99 |
+
lambda: db.table("documents").update({"status": "error"}).eq("filename", filename).execute()
|
| 100 |
+
)
|
| 101 |
+
finally:
|
| 102 |
+
if os.path.exists(file_path): os.remove(file_path)
|
| 103 |
+
|
| 104 |
+
# --- ROUTES ---
|
| 105 |
+
|
| 106 |
+
@router.post("/upload")
|
| 107 |
+
async def ingest_document(
|
| 108 |
+
background_tasks: BackgroundTasks,
|
| 109 |
+
file: UploadFile = File(...),
|
| 110 |
+
user_id: str = Depends(get_current_user)
|
| 111 |
+
):
|
| 112 |
+
"""File Upload Handler with Path Traversal Protection"""
|
| 113 |
+
if not file.filename or not file.filename.lower().endswith(".pdf"):
|
| 114 |
+
raise HTTPException(status_code=400, detail="Protocol Violation: PDF Document Required.")
|
| 115 |
+
|
| 116 |
+
try:
|
| 117 |
+
os.makedirs(TEMP_DIR, exist_ok=True)
|
| 118 |
+
# Prevent Directory Traversal by stripping paths
|
| 119 |
+
safe_filename = os.path.basename(file.filename)
|
| 120 |
+
file_path = f"{TEMP_DIR}/{uuid.uuid4()}_{safe_filename}"
|
| 121 |
+
|
| 122 |
+
# Async file read/write
|
| 123 |
+
content = await file.read()
|
| 124 |
+
await asyncio.to_thread(lambda: open(file_path, "wb").write(content))
|
| 125 |
+
|
| 126 |
+
# FastAPI Native Async Background Task
|
| 127 |
+
background_tasks.add_task(process_document, file_path, safe_filename, user_id)
|
| 128 |
+
|
| 129 |
+
return {"status": "queued", "filename": safe_filename}
|
| 130 |
+
except Exception as e:
|
| 131 |
+
raise HTTPException(status_code=500, detail=str(e))
|
| 132 |
+
|
| 133 |
+
@router.get("/status/{filename}")
|
| 134 |
+
async def get_ingestion_status(filename: str = Path(...), user_id: str = Depends(get_current_user)):
|
| 135 |
+
if not db: return {"status": "error", "message": "DB Offline"}
|
| 136 |
+
res = await asyncio.to_thread(
|
| 137 |
+
lambda: db.table("documents").select("status").eq("filename", filename).eq("user_id", user_id).order("created_at", desc=True).limit(1).execute()
|
| 138 |
+
)
|
| 139 |
+
data = cast(List[Dict[str, Any]], res.data)
|
| 140 |
+
return {"status": data[0].get('status', 'unknown')} if data else {"status": "not_found"}
|
| 141 |
+
|
| 142 |
+
@router.get("/latest")
|
| 143 |
+
async def get_latest_document(user_id: str = Depends(get_current_user)):
|
| 144 |
+
if not db: return {"status": "error"}
|
| 145 |
+
res = await asyncio.to_thread(
|
| 146 |
+
lambda: db.table("documents").select("filename, status").eq("user_id", user_id).order("created_at", desc=True).limit(1).execute()
|
| 147 |
+
)
|
| 148 |
+
data = cast(List[Dict[str, Any]], res.data)
|
| 149 |
+
return {"status": "success", "filename": data[0].get("filename"), "doc_status": data[0].get("status")} if data else {"status": "none"}
|
| 150 |
+
|
| 151 |
+
@router.get("/metadata/{filename}")
|
| 152 |
+
async def get_document_metadata(filename: str = Path(...), user_id: str = Depends(get_current_user)):
|
| 153 |
+
if not db: return {"status": "error"}
|
| 154 |
+
res = await asyncio.to_thread(
|
| 155 |
+
lambda: db.table("documents").select("*").eq("filename", filename).eq("user_id", user_id).order("created_at", desc=True).limit(1).execute()
|
| 156 |
+
)
|
| 157 |
+
data_list = cast(List[Dict[str, Any]], res.data)
|
| 158 |
+
if not data_list: return {"status": "not_found"}
|
| 159 |
+
doc_data = data_list[0]
|
| 160 |
+
|
| 161 |
+
chunks = await asyncio.to_thread(
|
| 162 |
+
lambda: db.table("document_chunks").select("id", count=cast(Any, "exact")).eq("document_id", doc_data['id']).execute()
|
| 163 |
+
)
|
| 164 |
+
return {
|
| 165 |
+
"filename": filename,
|
| 166 |
+
"status": doc_data.get('status'),
|
| 167 |
+
"created_at": doc_data.get('created_at'),
|
| 168 |
+
"chunk_count": chunks.count if chunks.count else 0,
|
| 169 |
+
"is_permanent": doc_data.get('is_permanent', False)
|
| 170 |
+
}
|
| 171 |
+
|
| 172 |
+
class SaveRequest(BaseModel):
|
| 173 |
+
filename: str
|
| 174 |
+
|
| 175 |
+
@router.post("/save")
|
| 176 |
+
async def save_document_to_vault(req: SaveRequest, user_id: str = Depends(get_current_user)):
|
| 177 |
+
if db:
|
| 178 |
+
await asyncio.to_thread(
|
| 179 |
+
lambda: db.table("documents").update({"is_permanent": True}).eq("filename", req.filename).eq("user_id", user_id).execute()
|
| 180 |
+
)
|
| 181 |
+
return {"status": "persisted"}
|
| 182 |
+
return {"status": "error"}
|
| 183 |
+
|
| 184 |
+
@router.delete("/documents/{filename}")
|
| 185 |
+
async def delete_document(filename: str = Path(...), user_id: str = Depends(get_current_user)):
|
| 186 |
+
if not db: raise HTTPException(status_code=500, detail="Vault DB Offline")
|
| 187 |
+
await asyncio.to_thread(
|
| 188 |
+
lambda: db.table("documents").delete().eq("filename", filename).eq("user_id", user_id).execute()
|
| 189 |
+
)
|
| 190 |
+
return {"status": "purged", "filename": filename}
|
| 191 |
+
|
| 192 |
+
def sanitize_float(val: Any) -> float:
|
| 193 |
+
try:
|
| 194 |
+
f_val = float(val)
|
| 195 |
+
return f_val if math.isfinite(f_val) else 0.0
|
| 196 |
+
except (TypeError, ValueError):
|
| 197 |
+
return 0.0
|
| 198 |
+
|
| 199 |
+
@router.get("/telemetry")
|
| 200 |
+
async def get_system_telemetry(user_id: str = Depends(get_current_user)):
|
| 201 |
+
"""Async offloaded telemetry aggregation."""
|
| 202 |
+
if not db: return {"chunks": "--", "persistence": "--", "blocked": "--", "latency": "--"}
|
| 203 |
+
|
| 204 |
+
try:
|
| 205 |
+
docs_res = await asyncio.to_thread(lambda: db.table("documents").select("id, is_permanent").eq("user_id", user_id).execute())
|
| 206 |
+
docs = cast(List[Dict[str, Any]], docs_res.data)
|
| 207 |
+
|
| 208 |
+
total_docs = len(docs)
|
| 209 |
+
persisted_docs = sum(1 for d in docs if d.get("is_permanent", False))
|
| 210 |
+
persistence_rate = f"{int((persisted_docs / total_docs) * 100)}%" if total_docs > 0 else "0%"
|
| 211 |
+
|
| 212 |
+
chunks_res = await asyncio.to_thread(lambda: db.table("document_chunks").select("id", count=cast(Any, "exact")).eq("user_id", user_id).execute())
|
| 213 |
+
total_chunks = chunks_res.count if chunks_res.count else 0
|
| 214 |
+
|
| 215 |
+
logs_res = await asyncio.to_thread(lambda: db.table("audit_logs").select("faithfulness, precision, relevance, latency").eq("user_id", user_id).order("created_at", desc=True).limit(50).execute())
|
| 216 |
+
logs = cast(List[Dict[str, Any]], logs_res.data)
|
| 217 |
+
|
| 218 |
+
avg_faith, avg_prec, avg_rel, avg_latency, blocked = 0.0, 0.0, 0.0, 0.0, 0
|
| 219 |
+
|
| 220 |
+
if logs:
|
| 221 |
+
total_logs = len(logs)
|
| 222 |
+
avg_faith = sum(sanitize_float(l.get("faithfulness", 0.0)) for l in logs) / total_logs
|
| 223 |
+
avg_prec = sum(sanitize_float(l.get("precision", 0.0)) for l in logs) / total_logs
|
| 224 |
+
avg_rel = sum(sanitize_float(l.get("relevance", 0.0)) for l in logs) / total_logs
|
| 225 |
+
blocked = sum(1 for l in logs if sanitize_float(l.get("faithfulness", 0.0)) < 0.8)
|
| 226 |
+
|
| 227 |
+
valid_latencies =[sanitize_float(l.get("latency", 0.0)) for l in logs if sanitize_float(l.get("latency", 0.0)) > 0]
|
| 228 |
+
if valid_latencies:
|
| 229 |
+
avg_latency = sum(valid_latencies) / len(valid_latencies)
|
| 230 |
+
|
| 231 |
+
return {
|
| 232 |
+
"chunks": str(total_chunks),
|
| 233 |
+
"persistence": persistence_rate,
|
| 234 |
+
"blocked": str(blocked),
|
| 235 |
+
"latency": f"{sanitize_float(avg_latency):.1f}s",
|
| 236 |
+
"ragas": {"faithfulness": sanitize_float(avg_faith), "precision": sanitize_float(avg_prec), "relevance": sanitize_float(avg_rel)}
|
| 237 |
+
}
|
| 238 |
+
except Exception as e:
|
| 239 |
+
print(f"❌ TELEMETRY ERROR: {e}")
|
| 240 |
+
return {"chunks": "--", "persistence": "--", "blocked": "--", "latency": "--"}
|
app/api/keys.py
ADDED
|
@@ -0,0 +1,84 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import secrets
|
| 2 |
+
import hashlib
|
| 3 |
+
import asyncio
|
| 4 |
+
from fastapi import APIRouter, Depends, HTTPException, Path
|
| 5 |
+
from pydantic import BaseModel, Field
|
| 6 |
+
from typing import List, Dict, Any, cast
|
| 7 |
+
from app.core.auth import get_current_user
|
| 8 |
+
from app.core.database import db
|
| 9 |
+
|
| 10 |
+
router = APIRouter()
|
| 11 |
+
|
| 12 |
+
# --- 1. STRICT SCHEMAS ---
|
| 13 |
+
class CreateKeyRequest(BaseModel):
|
| 14 |
+
# Firewall: Prevent DB Overflow by capping the name length
|
| 15 |
+
name: str = Field(..., min_length=1, max_length=64, description="Name identifier for the API key")
|
| 16 |
+
|
| 17 |
+
def generate_secure_key():
|
| 18 |
+
"""
|
| 19 |
+
Generates a production-grade API key.
|
| 20 |
+
Returns: (raw_key_for_user, hashed_key_for_db, key_hint_for_ui)
|
| 21 |
+
"""
|
| 22 |
+
raw_secret = secrets.token_hex(24)
|
| 23 |
+
full_key = f"axm_live_{raw_secret}"
|
| 24 |
+
key_hash = hashlib.sha256(full_key.encode()).hexdigest()
|
| 25 |
+
key_hint = f"{full_key[:12]}...{full_key[-4:]}"
|
| 26 |
+
return full_key, key_hash, key_hint
|
| 27 |
+
|
| 28 |
+
# --- 2. ASYNC ENDPOINTS ---
|
| 29 |
+
|
| 30 |
+
@router.post("/")
|
| 31 |
+
async def create_api_key(req: CreateKeyRequest, user_id: str = Depends(get_current_user)):
|
| 32 |
+
"""Generates a new API key. The raw key is returned ONLY ONCE."""
|
| 33 |
+
if not db: raise HTTPException(503, "DB Offline")
|
| 34 |
+
|
| 35 |
+
full_key, key_hash, key_hint = generate_secure_key()
|
| 36 |
+
|
| 37 |
+
# Non-blocking DB Insert
|
| 38 |
+
await asyncio.to_thread(
|
| 39 |
+
lambda: db.table("api_keys").insert({
|
| 40 |
+
"user_id": user_id,
|
| 41 |
+
"name": req.name,
|
| 42 |
+
"key_value": key_hash,
|
| 43 |
+
"key_hint": key_hint
|
| 44 |
+
}).execute()
|
| 45 |
+
)
|
| 46 |
+
|
| 47 |
+
return {
|
| 48 |
+
"status": "success",
|
| 49 |
+
"name": req.name,
|
| 50 |
+
"key_value": full_key,
|
| 51 |
+
"message": "Please copy this key now. You will not be able to see it again."
|
| 52 |
+
}
|
| 53 |
+
|
| 54 |
+
@router.get("/")
|
| 55 |
+
async def list_api_keys(user_id: str = Depends(get_current_user)):
|
| 56 |
+
"""Lists all active API keys using their secure hints asynchronously."""
|
| 57 |
+
if not db: raise HTTPException(503, "DB Offline")
|
| 58 |
+
|
| 59 |
+
res = await asyncio.to_thread(
|
| 60 |
+
lambda: db.table("api_keys")
|
| 61 |
+
.select("id, name, created_at, last_used_at, is_active, key_hint")
|
| 62 |
+
.eq("user_id", user_id)
|
| 63 |
+
.order("created_at", desc=True)
|
| 64 |
+
.execute()
|
| 65 |
+
)
|
| 66 |
+
|
| 67 |
+
return {"keys": cast(List[Dict[str, Any]], res.data)}
|
| 68 |
+
|
| 69 |
+
@router.delete("/{key_id}")
|
| 70 |
+
async def revoke_api_key(
|
| 71 |
+
key_id: str = Path(..., description="ID of the key to revoke"),
|
| 72 |
+
user_id: str = Depends(get_current_user)
|
| 73 |
+
):
|
| 74 |
+
"""Instantly revokes an API key (Sets is_active to False) without freezing the UI."""
|
| 75 |
+
if not db: raise HTTPException(503, "DB Offline")
|
| 76 |
+
|
| 77 |
+
await asyncio.to_thread(
|
| 78 |
+
lambda: db.table("api_keys")
|
| 79 |
+
.update({"is_active": False})
|
| 80 |
+
.eq("id", key_id)
|
| 81 |
+
.eq("user_id", user_id)
|
| 82 |
+
.execute()
|
| 83 |
+
)
|
| 84 |
+
return {"status": "revoked", "id": key_id}
|
app/api/run.py
ADDED
|
@@ -0,0 +1,156 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import time
|
| 2 |
+
import math
|
| 3 |
+
import json
|
| 4 |
+
import asyncio
|
| 5 |
+
from fastapi import APIRouter, HTTPException, Depends, Request
|
| 6 |
+
from sse_starlette.sse import EventSourceResponse
|
| 7 |
+
from pydantic import BaseModel
|
| 8 |
+
from app.agents.graph import app_graph
|
| 9 |
+
from app.agents.state import AgentState
|
| 10 |
+
from app.core.auth import get_current_user
|
| 11 |
+
from app.core.database import db
|
| 12 |
+
from typing import Dict, Any, cast, List, AsyncGenerator
|
| 13 |
+
|
| 14 |
+
router = APIRouter()
|
| 15 |
+
|
| 16 |
+
# --- 1. SCHEMAS ---
|
| 17 |
+
class VerificationRequest(BaseModel):
|
| 18 |
+
question: str
|
| 19 |
+
filenames: List[str]
|
| 20 |
+
|
| 21 |
+
class VerificationResponse(BaseModel):
|
| 22 |
+
answer: str
|
| 23 |
+
status: str
|
| 24 |
+
evidence_count: int
|
| 25 |
+
metrics: Dict[str, float]
|
| 26 |
+
|
| 27 |
+
def sanitize_float(val: Any) -> float:
|
| 28 |
+
try:
|
| 29 |
+
f_val = float(val)
|
| 30 |
+
return f_val if math.isfinite(f_val) else 0.0
|
| 31 |
+
except (TypeError, ValueError):
|
| 32 |
+
return 0.0
|
| 33 |
+
|
| 34 |
+
# --- 2. STREAMING ENDPOINT ---
|
| 35 |
+
@router.post("/verify")
|
| 36 |
+
async def run_verification(
|
| 37 |
+
payload: VerificationRequest,
|
| 38 |
+
user_id: str = Depends(get_current_user)
|
| 39 |
+
):
|
| 40 |
+
async def event_generator() -> AsyncGenerator[Dict[str, Any], None]:
|
| 41 |
+
try:
|
| 42 |
+
start_time = time.time()
|
| 43 |
+
print(f"--- STREAM STARTED FOR: {payload.question[:30]}... ---")
|
| 44 |
+
|
| 45 |
+
history_buffer: List[Dict[str, str]] = []
|
| 46 |
+
is_root_reset = payload.question.strip().startswith("/axm ..")
|
| 47 |
+
|
| 48 |
+
if db and not is_root_reset:
|
| 49 |
+
try:
|
| 50 |
+
primary_file = payload.filenames[0] if payload.filenames else "vault"
|
| 51 |
+
doc_res = db.table("documents").select("id").eq("filename", primary_file).eq("user_id", user_id).execute()
|
| 52 |
+
|
| 53 |
+
# FIX: Explicit cast to allow indexing
|
| 54 |
+
doc_rows = cast(List[Dict[str, Any]], doc_res.data)
|
| 55 |
+
if doc_rows:
|
| 56 |
+
doc_id = doc_rows[0]['id']
|
| 57 |
+
hist_res = db.table("chat_messages").select("role, content").eq("document_id", doc_id).eq("user_id", user_id).order("created_at", desc=True).limit(5).execute()
|
| 58 |
+
|
| 59 |
+
# FIX: Cast to match AgentState history requirements
|
| 60 |
+
raw_hist = cast(List[Dict[str, str]], hist_res.data)
|
| 61 |
+
history_buffer = raw_hist[::-1]
|
| 62 |
+
except Exception as e:
|
| 63 |
+
print(f"AXM-MEM: History hydration failed: {e}")
|
| 64 |
+
|
| 65 |
+
initial_state: AgentState = {
|
| 66 |
+
"question": payload.question,
|
| 67 |
+
"user_id": user_id,
|
| 68 |
+
"filenames": payload.filenames,
|
| 69 |
+
"history": history_buffer,
|
| 70 |
+
"command": None,
|
| 71 |
+
"comparison_map": {},
|
| 72 |
+
"documents":[],
|
| 73 |
+
"generation": "",
|
| 74 |
+
"hallucination_score": 0.0,
|
| 75 |
+
"metrics": {},
|
| 76 |
+
"status": "thinking",
|
| 77 |
+
"retry_count": 0,
|
| 78 |
+
"active_node": None
|
| 79 |
+
}
|
| 80 |
+
|
| 81 |
+
full_generation = ""
|
| 82 |
+
final_metrics: Dict[str, float] = {}
|
| 83 |
+
current_active_node = "System"
|
| 84 |
+
|
| 85 |
+
ui_node_map = {
|
| 86 |
+
"retrieve_node": "Librarian", "Librarian": "Librarian",
|
| 87 |
+
"distill_node": "Editor", "Editor": "Editor",
|
| 88 |
+
"strategist_node": "Strategist", "Strategist": "Strategist",
|
| 89 |
+
"generate_node": "Architect", "Architect": "Architect",
|
| 90 |
+
"grade_generation_node": "Prosecutor", "Prosecutor": "Prosecutor"
|
| 91 |
+
}
|
| 92 |
+
|
| 93 |
+
async for event in app_graph.astream_events(initial_state, version="v1"):
|
| 94 |
+
kind = event["event"]
|
| 95 |
+
name = event["name"]
|
| 96 |
+
|
| 97 |
+
if kind == "on_chain_start" and name in ui_node_map:
|
| 98 |
+
current_active_node = ui_node_map[name]
|
| 99 |
+
yield {"event": "node_update", "data": json.dumps({"node": current_active_node, "status": "active"})}
|
| 100 |
+
|
| 101 |
+
elif kind == "on_chat_model_stream":
|
| 102 |
+
if current_active_node in ["Architect", "Strategist"]:
|
| 103 |
+
chunk = event["data"].get("chunk")
|
| 104 |
+
content = ""
|
| 105 |
+
if chunk:
|
| 106 |
+
if hasattr(chunk, "content"): content = str(chunk.content)
|
| 107 |
+
elif isinstance(chunk, dict) and "content" in chunk: content = str(chunk["content"])
|
| 108 |
+
|
| 109 |
+
if content:
|
| 110 |
+
full_generation += content
|
| 111 |
+
yield {"event": "token", "data": json.dumps({"text": content})}
|
| 112 |
+
|
| 113 |
+
elif kind == "on_chain_end" and name in ["generate_node", "Architect"]:
|
| 114 |
+
# FIX: Explicit type annotation for Mypy
|
| 115 |
+
node_output: Dict[str, Any] = event["data"].get("output", {})
|
| 116 |
+
if not full_generation and "generation" in node_output:
|
| 117 |
+
full_generation = str(node_output["generation"])
|
| 118 |
+
yield {"event": "token", "data": json.dumps({"text": full_generation})}
|
| 119 |
+
|
| 120 |
+
elif kind == "on_chain_end" and name in ["grade_generation_node", "Prosecutor"]:
|
| 121 |
+
# FIX: Explicit type annotation for Mypy
|
| 122 |
+
eval_output: Dict[str, Any] = event["data"].get("output", {})
|
| 123 |
+
final_metrics = eval_output.get("metrics", {})
|
| 124 |
+
|
| 125 |
+
if not full_generation.strip():
|
| 126 |
+
full_generation = "Verification Failed: Audit logic rejected the draft."
|
| 127 |
+
yield {"event": "token", "data": json.dumps({"text": full_generation})}
|
| 128 |
+
|
| 129 |
+
actual_latency = round(time.time() - start_time, 2)
|
| 130 |
+
safe_metrics = {k: sanitize_float(v) for k, v in final_metrics.items()}
|
| 131 |
+
|
| 132 |
+
if db:
|
| 133 |
+
try:
|
| 134 |
+
primary_file = payload.filenames[0] if payload.filenames else "vault"
|
| 135 |
+
doc_res = db.table("documents").select("id").eq("filename", primary_file).eq("user_id", user_id).execute()
|
| 136 |
+
doc_data = cast(List[Dict[str, Any]], doc_res.data)
|
| 137 |
+
|
| 138 |
+
if doc_data:
|
| 139 |
+
doc_id = doc_data[0]['id']
|
| 140 |
+
db.table("chat_messages").insert({"document_id": doc_id, "user_id": user_id, "role": "user", "content": payload.question}).execute()
|
| 141 |
+
db.table("chat_messages").insert({"document_id": doc_id, "user_id": user_id, "role": "assistant", "content": full_generation, "metrics": safe_metrics}).execute()
|
| 142 |
+
db.table("audit_logs").insert({"user_id": user_id, "question": payload.question, "faithfulness": safe_metrics.get("faithfulness", 0.0), "latency": actual_latency}).execute()
|
| 143 |
+
except Exception as log_err:
|
| 144 |
+
print(f"SSE DB ERROR: {log_err}")
|
| 145 |
+
|
| 146 |
+
yield {
|
| 147 |
+
"event": "audit_complete",
|
| 148 |
+
"data": json.dumps({"answer": full_generation, "metrics": safe_metrics})
|
| 149 |
+
}
|
| 150 |
+
|
| 151 |
+
except Exception as e:
|
| 152 |
+
error_msg = str(e)
|
| 153 |
+
print(f"❌ MASTER STREAM CRASH: {error_msg}")
|
| 154 |
+
yield {"event": "error", "data": json.dumps({"detail": f"Backend Engine Disconnected: {error_msg}"})}
|
| 155 |
+
|
| 156 |
+
return EventSourceResponse(event_generator())
|
app/api/vault.py
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import asyncio
|
| 2 |
+
from fastapi import APIRouter, HTTPException, Depends
|
| 3 |
+
from pydantic import BaseModel, Field
|
| 4 |
+
from typing import List, Dict, Any, cast
|
| 5 |
+
from app.core.database import db
|
| 6 |
+
from app.core.embeddings import get_embedding
|
| 7 |
+
from app.core.auth import get_current_user
|
| 8 |
+
|
| 9 |
+
router = APIRouter()
|
| 10 |
+
|
| 11 |
+
# --- 1. HARDENED SCHEMAS (Security) ---
|
| 12 |
+
class VaultSearchRequest(BaseModel):
|
| 13 |
+
# Bound the query to ~400 words to prevent NVIDIA NIM Token Limit Crashes (HTTP 413)
|
| 14 |
+
query: str = Field(..., min_length=2, max_length=2000, description="The audit query")
|
| 15 |
+
# Cap the limit to prevent DB Compute DoS attacks
|
| 16 |
+
limit: int = Field(default=5, ge=1, le=50, description="Max results to return")
|
| 17 |
+
|
| 18 |
+
class VaultSearchResult(BaseModel):
|
| 19 |
+
id: int
|
| 20 |
+
filename: str
|
| 21 |
+
content: str
|
| 22 |
+
similarity: float
|
| 23 |
+
fts_rank: float
|
| 24 |
+
|
| 25 |
+
# --- 2. ASYNC OPTIMIZED ENDPOINT (Speed) ---
|
| 26 |
+
@router.post("/search", response_model=List[VaultSearchResult])
|
| 27 |
+
async def search_vault(
|
| 28 |
+
req: VaultSearchRequest,
|
| 29 |
+
user_id: str = Depends(get_current_user)
|
| 30 |
+
):
|
| 31 |
+
"""
|
| 32 |
+
SOTA Hybrid Interrogator:
|
| 33 |
+
Executes a parallel Vector + Keyword search across the user's entire vault.
|
| 34 |
+
Now utilizes asyncio thread-pooling for non-blocking execution.
|
| 35 |
+
"""
|
| 36 |
+
if not db:
|
| 37 |
+
raise HTTPException(status_code=503, detail="Vault Engine Offline")
|
| 38 |
+
|
| 39 |
+
try:
|
| 40 |
+
# 1. Non-Blocking Embedding Generation
|
| 41 |
+
# Offload the synchronous NVIDIA network call to a background thread
|
| 42 |
+
query_vector = await asyncio.to_thread(
|
| 43 |
+
get_embedding,
|
| 44 |
+
text=req.query,
|
| 45 |
+
input_type="query"
|
| 46 |
+
)
|
| 47 |
+
|
| 48 |
+
# 2. Prepare RPC Parameters
|
| 49 |
+
rpc_params = {
|
| 50 |
+
"query_text": req.query,
|
| 51 |
+
"query_embedding": query_vector,
|
| 52 |
+
"match_count": req.limit,
|
| 53 |
+
"target_user_id": user_id
|
| 54 |
+
}
|
| 55 |
+
|
| 56 |
+
# 3. Non-Blocking Database Execution
|
| 57 |
+
# Offload the synchronous Supabase HTTP request to prevent event loop freezing
|
| 58 |
+
res = await asyncio.to_thread(
|
| 59 |
+
lambda: db.rpc("hybrid_vault_search", rpc_params).execute()
|
| 60 |
+
)
|
| 61 |
+
|
| 62 |
+
# 4. Strict Type Casting & Return
|
| 63 |
+
return cast(List[Dict[str, Any]], res.data)
|
| 64 |
+
|
| 65 |
+
except Exception as e:
|
| 66 |
+
print(f"❌ VAULT SEARCH ERROR: {str(e)}")
|
| 67 |
+
# Generic 500 prevents leaking exact database schemas/errors to the client
|
| 68 |
+
raise HTTPException(status_code=500, detail="Retrieval Protocol Failure")
|
app/core/auth.py
ADDED
|
@@ -0,0 +1,139 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import requests
|
| 3 |
+
import hashlib
|
| 4 |
+
import asyncio
|
| 5 |
+
from datetime import datetime
|
| 6 |
+
from fastapi import Depends, HTTPException
|
| 7 |
+
from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
|
| 8 |
+
from jose import jwt, jwk
|
| 9 |
+
from typing import Optional, Dict, Any, List, cast
|
| 10 |
+
from app.core.database import db
|
| 11 |
+
|
| 12 |
+
# --- Security Configuration ---
|
| 13 |
+
security = HTTPBearer()
|
| 14 |
+
|
| 15 |
+
# SOTA: Resilient Clerk Public Key Manager
|
| 16 |
+
class ClerkKeyManager:
|
| 17 |
+
_instance = None
|
| 18 |
+
_jwks: Optional[Dict[str, Any]] = None
|
| 19 |
+
|
| 20 |
+
def __new__(cls):
|
| 21 |
+
if cls._instance is None:
|
| 22 |
+
cls._instance = super(ClerkKeyManager, cls).__new__(cls)
|
| 23 |
+
return cls._instance
|
| 24 |
+
|
| 25 |
+
def get_jwks(self, force_refresh: bool = False) -> Dict[str, Any]:
|
| 26 |
+
"""
|
| 27 |
+
Fetches Clerk's public keys.
|
| 28 |
+
Supports forced cache invalidation to survive automatic Key Rotations.
|
| 29 |
+
"""
|
| 30 |
+
if self._jwks is None or force_refresh:
|
| 31 |
+
jwks_url = os.getenv("CLERK_JWKS_URL")
|
| 32 |
+
if not jwks_url:
|
| 33 |
+
print("⚠️ SECURITY ALERT: CLERK_JWKS_URL missing.")
|
| 34 |
+
return {}
|
| 35 |
+
|
| 36 |
+
try:
|
| 37 |
+
# Synchronous request, but we will wrap this in to_thread when calling
|
| 38 |
+
response = requests.get(jwks_url, timeout=10)
|
| 39 |
+
response.raise_for_status()
|
| 40 |
+
self._jwks = response.json()
|
| 41 |
+
if force_refresh:
|
| 42 |
+
print("AXIOM-AUTH: Clerk JWKS Cache Successfully Rotated.")
|
| 43 |
+
except Exception as e:
|
| 44 |
+
print(f"❌ AUTH ERROR: Failed to fetch JWKS: {e}")
|
| 45 |
+
return self._jwks or {} # Fallback to stale cache if network is down
|
| 46 |
+
return self._jwks
|
| 47 |
+
|
| 48 |
+
key_manager = ClerkKeyManager()
|
| 49 |
+
|
| 50 |
+
async def get_current_user(auth: HTTPAuthorizationCredentials = Depends(security)) -> str:
|
| 51 |
+
"""
|
| 52 |
+
V4.6 Enterprise Dual-Auth Guard:
|
| 53 |
+
Fully async, non-blocking, and resilient to cryptographic key rotation.
|
| 54 |
+
"""
|
| 55 |
+
token = auth.credentials
|
| 56 |
+
|
| 57 |
+
# ==========================================
|
| 58 |
+
# PATH A: AXIOM API KEY (MCP / IDE / CLI)
|
| 59 |
+
# ==========================================
|
| 60 |
+
if token.startswith("axm_live_") or token.startswith("axm_test_"):
|
| 61 |
+
if not db:
|
| 62 |
+
raise HTTPException(status_code=503, detail="Auth Database Offline")
|
| 63 |
+
|
| 64 |
+
token_hash = hashlib.sha256(token.encode()).hexdigest()
|
| 65 |
+
|
| 66 |
+
# 1. Non-Blocking High-speed lookup
|
| 67 |
+
res = await asyncio.to_thread(
|
| 68 |
+
lambda: db.table("api_keys").select("user_id, is_active").eq("key_value", token_hash).execute()
|
| 69 |
+
)
|
| 70 |
+
key_data = cast(List[Dict[str, Any]], res.data)
|
| 71 |
+
|
| 72 |
+
# 2. Reject if invalid or revoked
|
| 73 |
+
if not key_data or not key_data[0].get("is_active"):
|
| 74 |
+
raise HTTPException(status_code=401, detail="Invalid or Revoked Axiom API Key.")
|
| 75 |
+
|
| 76 |
+
# 3. SOTA: Fire-and-Forget Timestamp Update (Zero added latency for the user)
|
| 77 |
+
def update_timestamp():
|
| 78 |
+
try:
|
| 79 |
+
db.table("api_keys").update({"last_used_at": datetime.utcnow().isoformat()}).eq("key_value", token_hash).execute()
|
| 80 |
+
except Exception as e:
|
| 81 |
+
print(f"⚠️ Timestamp Update Failed (Non-fatal): {e}")
|
| 82 |
+
|
| 83 |
+
asyncio.create_task(asyncio.to_thread(update_timestamp))
|
| 84 |
+
|
| 85 |
+
return str(key_data[0]["user_id"])
|
| 86 |
+
|
| 87 |
+
# ==========================================
|
| 88 |
+
# PATH B: CLERK JWT (Web Browser Dashboard)
|
| 89 |
+
# ==========================================
|
| 90 |
+
|
| 91 |
+
# 1. Async fetch of the JWKS cache
|
| 92 |
+
jwks = await asyncio.to_thread(key_manager.get_jwks)
|
| 93 |
+
|
| 94 |
+
if not jwks:
|
| 95 |
+
if os.getenv("ENV") == "development":
|
| 96 |
+
payload = jwt.get_unverified_claims(token)
|
| 97 |
+
return str(payload.get("sub"))
|
| 98 |
+
raise HTTPException(status_code=503, detail="Auth Engine Misconfigured")
|
| 99 |
+
|
| 100 |
+
try:
|
| 101 |
+
header = jwt.get_unverified_header(token)
|
| 102 |
+
kid = header.get("kid")
|
| 103 |
+
|
| 104 |
+
# 2. Smart Cache Invalidation (The Key Rotation Fix)
|
| 105 |
+
# If the 'kid' in the JWT isn't in our cache, Clerk likely rotated their keys.
|
| 106 |
+
# We force a refresh of the JWKS cache and try one more time.
|
| 107 |
+
if kid not in[k.get("kid") for k in jwks.get("keys", [])]:
|
| 108 |
+
print(f"AXIOM-AUTH: Unknown Key ID '{kid}' detected. Forcing JWKS rotation...")
|
| 109 |
+
jwks = await asyncio.to_thread(key_manager.get_jwks, force_refresh=True)
|
| 110 |
+
|
| 111 |
+
public_key = None
|
| 112 |
+
for key in jwks.get("keys", []):
|
| 113 |
+
if key["kid"] == kid:
|
| 114 |
+
public_key = jwk.construct(key)
|
| 115 |
+
break
|
| 116 |
+
|
| 117 |
+
if not public_key:
|
| 118 |
+
raise HTTPException(status_code=401, detail="Invalid Security Key ID. Issuer may have revoked the key.")
|
| 119 |
+
|
| 120 |
+
# 3. VERIFY SIGNATURE, ISSUER, AND EXPIRATION
|
| 121 |
+
# (This is CPU bound, but extremely fast; safe to run synchronously here)
|
| 122 |
+
payload = jwt.decode(
|
| 123 |
+
token,
|
| 124 |
+
public_key,
|
| 125 |
+
algorithms=["RS256"],
|
| 126 |
+
options={"verify_aud": False}
|
| 127 |
+
)
|
| 128 |
+
|
| 129 |
+
user_id = payload.get("sub")
|
| 130 |
+
if not user_id:
|
| 131 |
+
raise HTTPException(status_code=401, detail="Identity Subject Missing")
|
| 132 |
+
|
| 133 |
+
return str(user_id)
|
| 134 |
+
|
| 135 |
+
except jwt.ExpiredSignatureError:
|
| 136 |
+
raise HTTPException(status_code=401, detail="Identity Session Expired")
|
| 137 |
+
except Exception as e:
|
| 138 |
+
print(f"❌ AUTH BREACH ATTEMPT: {str(e)}")
|
| 139 |
+
raise HTTPException(status_code=401, detail="Identity Handshake Denied")
|
app/core/chunking.py
ADDED
|
@@ -0,0 +1,92 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import re
|
| 2 |
+
import asyncio
|
| 3 |
+
from typing import List, Optional
|
| 4 |
+
import tiktoken
|
| 5 |
+
from langchain_text_splitters import RecursiveCharacterTextSplitter
|
| 6 |
+
|
| 7 |
+
class AxiomChunker:
|
| 8 |
+
"""
|
| 9 |
+
SOTA Token-Aware Chunker (V4.6 Enterprise).
|
| 10 |
+
Preserves structural Markdown (Tables/Headers) for Financial/Legal accuracy.
|
| 11 |
+
Includes async offloading to prevent GIL freezes on massive PDFs.
|
| 12 |
+
"""
|
| 13 |
+
def __init__(self, chunk_size: int = 400, chunk_overlap: int = 50):
|
| 14 |
+
self.chunk_size = chunk_size
|
| 15 |
+
self.chunk_overlap = chunk_overlap
|
| 16 |
+
self.tokenizer: Optional[tiktoken.Encoding] = None
|
| 17 |
+
self.splitter: Optional[RecursiveCharacterTextSplitter] = None
|
| 18 |
+
|
| 19 |
+
def _lazy_init(self) -> None:
|
| 20 |
+
"""Fires only when the first document needs to be chunked."""
|
| 21 |
+
if self.tokenizer is None or self.splitter is None:
|
| 22 |
+
print("AXIOM-CORE: Initializing Tiktoken & Semantic Splitters...")
|
| 23 |
+
self.tokenizer = tiktoken.get_encoding("cl100k_base")
|
| 24 |
+
|
| 25 |
+
# SOTA Splitter: Respects structural boundaries
|
| 26 |
+
self.splitter = RecursiveCharacterTextSplitter(
|
| 27 |
+
chunk_size=self.chunk_size,
|
| 28 |
+
chunk_overlap=self.chunk_overlap,
|
| 29 |
+
length_function=self._count_tokens,
|
| 30 |
+
# Prioritize splitting on double newlines to keep table rows intact
|
| 31 |
+
separators=["\n\n", "\n", ". ", " ", ""]
|
| 32 |
+
)
|
| 33 |
+
|
| 34 |
+
def _count_tokens(self, text: str) -> int:
|
| 35 |
+
if self.tokenizer is None:
|
| 36 |
+
self.tokenizer = tiktoken.get_encoding("cl100k_base")
|
| 37 |
+
return len(self.tokenizer.encode(text))
|
| 38 |
+
|
| 39 |
+
def _sanitize_markdown(self, text: str) -> str:
|
| 40 |
+
"""
|
| 41 |
+
SOTA Preservative Sanitization:
|
| 42 |
+
Strips invisible/junk characters but STRICTLY PRESERVES tables,
|
| 43 |
+
lists, and headers which are vital for vector semantics.
|
| 44 |
+
"""
|
| 45 |
+
# Remove massive blocks of empty newlines (e.g., page breaks)
|
| 46 |
+
text = re.sub(r'\n{3,}', '\n\n', text)
|
| 47 |
+
# Remove zero-width spaces and weird unicode artifacts
|
| 48 |
+
text = text.replace('\u200b', '').replace('\ufeff', '')
|
| 49 |
+
# Clean trailing whitespace on lines without destroying the line itself
|
| 50 |
+
text = re.sub(r'[ \t]+$', '', text, flags=re.MULTILINE)
|
| 51 |
+
return text.strip()
|
| 52 |
+
|
| 53 |
+
def split_text(self, text: str) -> List[str]:
|
| 54 |
+
"""Synchronous split (Internal use)"""
|
| 55 |
+
self._lazy_init()
|
| 56 |
+
|
| 57 |
+
# PHASE 1: PRESERVATIVE SANITIZATION
|
| 58 |
+
clean_text = self._sanitize_markdown(text)
|
| 59 |
+
|
| 60 |
+
# PHASE 2: SMART SPLITTING
|
| 61 |
+
if not self.splitter:
|
| 62 |
+
raise RuntimeError("Splitter failed to initialize")
|
| 63 |
+
|
| 64 |
+
raw_chunks = self.splitter.split_text(clean_text)
|
| 65 |
+
|
| 66 |
+
# PHASE 3: GUARDRAILS (NVIDIA NIM 512-Token Limit Compliance)
|
| 67 |
+
valid_chunks =[]
|
| 68 |
+
for c in raw_chunks:
|
| 69 |
+
# 510 allows 2 tokens overhead for embedding engine system prompts
|
| 70 |
+
if self._count_tokens(c) <= 510:
|
| 71 |
+
valid_chunks.append(c)
|
| 72 |
+
|
| 73 |
+
return valid_chunks
|
| 74 |
+
|
| 75 |
+
async def asplit_text(self, text: str) -> List[str]:
|
| 76 |
+
"""
|
| 77 |
+
Async wrapper: Offloads heavy CPU tokenization to a background thread.
|
| 78 |
+
Prevents FastAPI event loop starvation when parsing 100+ page PDFs.
|
| 79 |
+
"""
|
| 80 |
+
return await asyncio.to_thread(self.split_text, text)
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
# Singleton Instance
|
| 84 |
+
chunker = AxiomChunker()
|
| 85 |
+
|
| 86 |
+
def get_chunks(text: str) -> List[str]:
|
| 87 |
+
"""Universal synchronous interface."""
|
| 88 |
+
return chunker.split_text(text)
|
| 89 |
+
|
| 90 |
+
async def aget_chunks(text: str) -> List[str]:
|
| 91 |
+
"""Universal async interface for the Ingestion Port."""
|
| 92 |
+
return await chunker.asplit_text(text)
|
app/core/database.py
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import threading
|
| 3 |
+
from typing import Optional
|
| 4 |
+
from supabase import create_client, Client
|
| 5 |
+
|
| 6 |
+
class Database:
|
| 7 |
+
"""
|
| 8 |
+
SOTA Thread-Safe Singleton for Supabase.
|
| 9 |
+
Optimized for high-concurrency async background tasks.
|
| 10 |
+
"""
|
| 11 |
+
_instance: Optional['Database'] = None
|
| 12 |
+
_lock = threading.Lock()
|
| 13 |
+
client: Optional[Client] = None
|
| 14 |
+
|
| 15 |
+
def __new__(cls) -> 'Database':
|
| 16 |
+
if cls._instance is None:
|
| 17 |
+
with cls._lock:
|
| 18 |
+
if cls._instance is None:
|
| 19 |
+
cls._instance = super(Database, cls).__new__(cls)
|
| 20 |
+
cls._instance._init_client()
|
| 21 |
+
return cls._instance
|
| 22 |
+
|
| 23 |
+
def _init_client(self) -> None:
|
| 24 |
+
url = os.environ.get("SUPABASE_URL")
|
| 25 |
+
key = os.environ.get("SUPABASE_SERVICE_KEY")
|
| 26 |
+
|
| 27 |
+
if not url or not key:
|
| 28 |
+
print("⚠️ CRITICAL: Supabase credentials missing. Vault features will be disabled.")
|
| 29 |
+
return
|
| 30 |
+
|
| 31 |
+
try:
|
| 32 |
+
# SOTA: Initializing the Master Service Client
|
| 33 |
+
self.client = create_client(url, key)
|
| 34 |
+
print("AXIOM-CORE: Vault Database Connection Established.")
|
| 35 |
+
except Exception as e:
|
| 36 |
+
print(f"❌ DATABASE INIT ERROR: {e}")
|
| 37 |
+
|
| 38 |
+
# Global Accessor
|
| 39 |
+
# Initialized once at module load, but shielded by the Singleton logic.
|
| 40 |
+
_db_manager = Database()
|
| 41 |
+
db: Optional[Client] = _db_manager.client
|
app/core/embeddings.py
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import threading
|
| 3 |
+
import numpy as np # type: ignore
|
| 4 |
+
from typing import List, Any, Optional
|
| 5 |
+
from openai import OpenAI
|
| 6 |
+
|
| 7 |
+
class EmbeddingAdapter:
|
| 8 |
+
"""
|
| 9 |
+
SOTA Multilingual Inference Adapter (V4.6 Thread-Safe).
|
| 10 |
+
Upgraded to Llama-Nemotron-Embed-1B-v2 for global sovereign audits.
|
| 11 |
+
Native Integration via 0.3.7 Update.
|
| 12 |
+
"""
|
| 13 |
+
_instance: Optional['EmbeddingAdapter'] = None
|
| 14 |
+
_client: Optional[OpenAI] = None
|
| 15 |
+
_model_name: str = "nvidia/llama-nemotron-embed-1b-v2"
|
| 16 |
+
|
| 17 |
+
# THE SHIELD: Thread lock for concurrent batching in ingest.py
|
| 18 |
+
_lock: threading.Lock = threading.Lock()
|
| 19 |
+
|
| 20 |
+
def __new__(cls) -> 'EmbeddingAdapter':
|
| 21 |
+
if cls._instance is None:
|
| 22 |
+
with cls._lock:
|
| 23 |
+
if cls._instance is None:
|
| 24 |
+
cls._instance = super(EmbeddingAdapter, cls).__new__(cls)
|
| 25 |
+
return cls._instance
|
| 26 |
+
|
| 27 |
+
def _lazy_init(self) -> None:
|
| 28 |
+
"""Thread-safe initialization of the NVIDIA client."""
|
| 29 |
+
if self._client is not None:
|
| 30 |
+
return
|
| 31 |
+
|
| 32 |
+
with self._lock:
|
| 33 |
+
if self._client is not None:
|
| 34 |
+
return
|
| 35 |
+
|
| 36 |
+
# SOTA: Fetch key at runtime, not import time
|
| 37 |
+
api_key = os.getenv("NVIDIA_API_KEY")
|
| 38 |
+
if not api_key:
|
| 39 |
+
raise RuntimeError("CRITICAL: NVIDIA_API_KEY missing.")
|
| 40 |
+
|
| 41 |
+
print(f"AXIOM-CORE: Multilingual Link Established via {self._model_name} (Native)")
|
| 42 |
+
|
| 43 |
+
self._client = OpenAI(
|
| 44 |
+
base_url="https://integrate.api.nvidia.com/v1",
|
| 45 |
+
api_key=api_key.strip(), # Strip removes hidden newlines!
|
| 46 |
+
max_retries=5,
|
| 47 |
+
timeout=60.0
|
| 48 |
+
)
|
| 49 |
+
|
| 50 |
+
def _normalize(self, vector: List[float]) -> List[float]:
|
| 51 |
+
"""Mathematically enforces Unit Length (L2 Norm) for pgvector speed."""
|
| 52 |
+
arr = np.array(vector)
|
| 53 |
+
norm = np.linalg.norm(arr)
|
| 54 |
+
if norm == 0:
|
| 55 |
+
return vector
|
| 56 |
+
return (arr / norm).tolist()
|
| 57 |
+
|
| 58 |
+
def embed_text(self, text: str, is_query: bool = False) -> List[float]:
|
| 59 |
+
"""Transmits text to NVIDIA grid and returns a normalized 1024-D vector."""
|
| 60 |
+
self._lazy_init()
|
| 61 |
+
|
| 62 |
+
if self._client is None:
|
| 63 |
+
return[0.0] * 1024
|
| 64 |
+
|
| 65 |
+
try:
|
| 66 |
+
target_type = "query" if is_query else "passage"
|
| 67 |
+
|
| 68 |
+
response = self._client.embeddings.create(
|
| 69 |
+
input=[text],
|
| 70 |
+
model=self._model_name,
|
| 71 |
+
extra_body={
|
| 72 |
+
"input_type": target_type,
|
| 73 |
+
"truncate": "END",
|
| 74 |
+
"dimensions": 1024 # Forces compatibility with Supabase schema
|
| 75 |
+
}
|
| 76 |
+
)
|
| 77 |
+
raw_vector = response.data[0].embedding
|
| 78 |
+
return self._normalize(raw_vector[:1024])
|
| 79 |
+
|
| 80 |
+
except Exception as e:
|
| 81 |
+
print(f"⚠️ NEURAL LINK FAILURE: {str(e)}")
|
| 82 |
+
return[0.0] * 1024
|
| 83 |
+
|
| 84 |
+
# Singleton Instance
|
| 85 |
+
_engine = EmbeddingAdapter()
|
| 86 |
+
|
| 87 |
+
def get_embedding(text: str, input_type: str = "query") -> List[float]:
|
| 88 |
+
"""Universal thread-safe interface for the Axiom Engine."""
|
| 89 |
+
is_query = True if input_type == "query" else False
|
| 90 |
+
return _engine.embed_text(text, is_query=is_query)
|
app/core/evaluator.py
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import math
|
| 3 |
+
import asyncio
|
| 4 |
+
from typing import Dict, List, Optional, Any
|
| 5 |
+
|
| 6 |
+
# SOTA: Migration to RAGAS 0.4.x API
|
| 7 |
+
from ragas import evaluate, SingleTurnSample, EvaluationDataset
|
| 8 |
+
from ragas.metrics import Faithfulness
|
| 9 |
+
from ragas.llms import LangchainLLMWrapper
|
| 10 |
+
from langchain_openai import ChatOpenAI
|
| 11 |
+
from pydantic import SecretStr
|
| 12 |
+
|
| 13 |
+
# CRITICAL: Fix for Pydantic V2 "BaseCache" error
|
| 14 |
+
from langchain_core.caches import BaseCache
|
| 15 |
+
from langchain_core.callbacks import Callbacks
|
| 16 |
+
from langchain_core.language_models.chat_models import BaseChatModel
|
| 17 |
+
|
| 18 |
+
class AxiomEvaluator:
|
| 19 |
+
"""
|
| 20 |
+
The Lite Auditor (V4.6 Production Refactor)
|
| 21 |
+
Stabilized for LangChain 1.x Parent-Child Type Resolution & Concurrency.
|
| 22 |
+
"""
|
| 23 |
+
def __init__(self) -> None:
|
| 24 |
+
# 1. MYPY FIX: Use 'Any' because RAGAS 0.4.x wrappers are dynamically typed
|
| 25 |
+
self.evaluator_llm: Any = None
|
| 26 |
+
self.faithfulness_metric: Optional[Faithfulness] = None
|
| 27 |
+
|
| 28 |
+
def _lazy_init(self) -> None:
|
| 29 |
+
"""Initializes RAGAS V2 and stabilizes the Pydantic Registry."""
|
| 30 |
+
if self.evaluator_llm is None:
|
| 31 |
+
try:
|
| 32 |
+
BaseChatModel.model_rebuild()
|
| 33 |
+
ChatOpenAI.model_rebuild()
|
| 34 |
+
print("AXIOM-CORE: Evaluator Registry Synchronized.")
|
| 35 |
+
except Exception as e:
|
| 36 |
+
print(f"AXIOM-CORE: Registry notice (Non-fatal): {e}")
|
| 37 |
+
|
| 38 |
+
print("AXIOM-CORE: Materializing RAGAS V2 Auditor (NVIDIA NIM)...")
|
| 39 |
+
raw_key = os.environ.get("NVIDIA_API_KEY")
|
| 40 |
+
|
| 41 |
+
llm = ChatOpenAI(
|
| 42 |
+
model="meta/llama-3.3-70b-instruct",
|
| 43 |
+
temperature=0,
|
| 44 |
+
api_key=SecretStr(raw_key) if raw_key else None,
|
| 45 |
+
base_url="https://integrate.api.nvidia.com/v1",
|
| 46 |
+
max_completion_tokens=2048
|
| 47 |
+
)
|
| 48 |
+
|
| 49 |
+
self.evaluator_llm = LangchainLLMWrapper(llm)
|
| 50 |
+
self.faithfulness_metric = Faithfulness(llm=self.evaluator_llm)
|
| 51 |
+
|
| 52 |
+
async def score_response(self, question: str, answer: str, contexts: List[str]) -> Dict[str, float]:
|
| 53 |
+
self._lazy_init()
|
| 54 |
+
try:
|
| 55 |
+
# Prepare RAGAS V2 Sample
|
| 56 |
+
sample = SingleTurnSample(
|
| 57 |
+
user_input=question,
|
| 58 |
+
response=answer,
|
| 59 |
+
retrieved_contexts=contexts
|
| 60 |
+
)
|
| 61 |
+
dataset = EvaluationDataset(samples=[sample])
|
| 62 |
+
|
| 63 |
+
# 2. SOTA THREAD POOLING
|
| 64 |
+
def run_ragas() -> Any:
|
| 65 |
+
return evaluate(dataset=dataset, metrics=[self.faithfulness_metric]) # type: ignore
|
| 66 |
+
|
| 67 |
+
result = await asyncio.to_thread(run_ragas)
|
| 68 |
+
|
| 69 |
+
# 3. Offload Pandas DataFrame operations
|
| 70 |
+
def extract_score() -> float:
|
| 71 |
+
scores_df = result.to_pandas()
|
| 72 |
+
return float(scores_df["faithfulness"].iloc[0])
|
| 73 |
+
|
| 74 |
+
raw_val = await asyncio.to_thread(extract_score)
|
| 75 |
+
|
| 76 |
+
# 4. Sanitize for JSON compliance
|
| 77 |
+
faithfulness_score = raw_val if math.isfinite(raw_val) else 0.0
|
| 78 |
+
print(f"AXIOM-AUDIT: Faithfulness Score Verified at {faithfulness_score * 100}%")
|
| 79 |
+
|
| 80 |
+
return {
|
| 81 |
+
"faithfulness": faithfulness_score,
|
| 82 |
+
"relevance": 1.0,
|
| 83 |
+
"precision": 1.0
|
| 84 |
+
}
|
| 85 |
+
|
| 86 |
+
except Exception as e:
|
| 87 |
+
print(f"RAGAS V2 EVAL ERROR: {e}")
|
| 88 |
+
return {"faithfulness": 0.0, "relevance": 1.0, "precision": 1.0}
|
| 89 |
+
|
| 90 |
+
axiom_evaluator = AxiomEvaluator()
|
app/core/monitor.py
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import tiktoken
|
| 2 |
+
from typing import List, Optional
|
| 3 |
+
|
| 4 |
+
class ContextMonitor:
|
| 5 |
+
"""SOTA Token Sentry V4.6. Enterprise-grade context truncation."""
|
| 6 |
+
_instance: Optional["ContextMonitor"] = None
|
| 7 |
+
encoder: Optional[tiktoken.Encoding] = None
|
| 8 |
+
LIMIT: int = 100000
|
| 9 |
+
|
| 10 |
+
def __new__(cls) -> "ContextMonitor":
|
| 11 |
+
if cls._instance is None:
|
| 12 |
+
cls._instance = super(ContextMonitor, cls).__new__(cls)
|
| 13 |
+
return cls._instance
|
| 14 |
+
|
| 15 |
+
def _lazy_init(self) -> None:
|
| 16 |
+
if self.encoder is None:
|
| 17 |
+
print("AXIOM-CORE: Materializing Token Sentry...")
|
| 18 |
+
self.encoder = tiktoken.get_encoding("cl100k_base")
|
| 19 |
+
|
| 20 |
+
def count_tokens(self, text: str) -> int:
|
| 21 |
+
if not text: return 0
|
| 22 |
+
self._lazy_init()
|
| 23 |
+
return len(self.encoder.encode(text)) # type: ignore
|
| 24 |
+
|
| 25 |
+
def guard_context(self, context_list: List[str]) -> str:
|
| 26 |
+
self._lazy_init()
|
| 27 |
+
current_parts: List[str] = []
|
| 28 |
+
total_tokens = 0
|
| 29 |
+
|
| 30 |
+
for chunk in context_list:
|
| 31 |
+
tokens = self.count_tokens(chunk) + 4
|
| 32 |
+
if total_tokens + tokens > self.LIMIT:
|
| 33 |
+
break
|
| 34 |
+
current_parts.append(chunk)
|
| 35 |
+
total_tokens += tokens
|
| 36 |
+
|
| 37 |
+
pressure = (total_tokens / self.LIMIT) * 100
|
| 38 |
+
print(f"CONTEXT_PRESSURE: {pressure:.1f}% ({total_tokens} tokens)")
|
| 39 |
+
return "\n\n".join(current_parts)
|
| 40 |
+
|
| 41 |
+
monitor = ContextMonitor()
|
app/core/reranker.py
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import asyncio
|
| 3 |
+
from typing import List, Optional
|
| 4 |
+
from langchain_nvidia_ai_endpoints import NVIDIARerank
|
| 5 |
+
from langchain_core.documents import Document
|
| 6 |
+
|
| 7 |
+
class AxiomReranker:
|
| 8 |
+
"""
|
| 9 |
+
SOTA Cloud-Lean Reranker Delegate (V4.6.1).
|
| 10 |
+
Engineered for Nemotron-Multilingual Synergy.
|
| 11 |
+
"""
|
| 12 |
+
_instance = None
|
| 13 |
+
_client: Optional[NVIDIARerank] = None
|
| 14 |
+
# THE VERIFIED SLUG:
|
| 15 |
+
_model_name: str = "nvidia/llama-nemotron-rerank-1b-v2"
|
| 16 |
+
|
| 17 |
+
def __new__(cls) -> 'AxiomReranker':
|
| 18 |
+
if cls._instance is None:
|
| 19 |
+
cls._instance = super(AxiomReranker, cls).__new__(cls)
|
| 20 |
+
return cls._instance
|
| 21 |
+
|
| 22 |
+
def _lazy_init(self, top_k: int) -> None:
|
| 23 |
+
if self._client is None:
|
| 24 |
+
api_key = os.getenv("NVIDIA_API_KEY")
|
| 25 |
+
if not api_key:
|
| 26 |
+
raise RuntimeError("CRITICAL: NVIDIA_API_KEY missing.")
|
| 27 |
+
|
| 28 |
+
print(f"AXIOM-CORE: Materializing {self._model_name}...")
|
| 29 |
+
|
| 30 |
+
self._client = NVIDIARerank(
|
| 31 |
+
model=self._model_name,
|
| 32 |
+
api_key=api_key, # type: ignore
|
| 33 |
+
top_n=top_k
|
| 34 |
+
)
|
| 35 |
+
else:
|
| 36 |
+
self._client.top_n = top_k
|
| 37 |
+
|
| 38 |
+
async def rerank(self, query: str, documents: List[str], top_k: int = 10) -> List[str]:
|
| 39 |
+
if not documents: return []
|
| 40 |
+
if len(documents) <= top_k: return documents
|
| 41 |
+
|
| 42 |
+
self._lazy_init(top_k=top_k)
|
| 43 |
+
|
| 44 |
+
def perform_rerank() -> List[str]:
|
| 45 |
+
if not self._client: return documents[:top_k]
|
| 46 |
+
lc_docs = [Document(page_content=txt) for txt in documents]
|
| 47 |
+
compressed_docs = self._client.compress_documents(query=query, documents=lc_docs)
|
| 48 |
+
return [doc.page_content for doc in compressed_docs]
|
| 49 |
+
|
| 50 |
+
try:
|
| 51 |
+
return await asyncio.to_thread(perform_rerank)
|
| 52 |
+
except Exception as e:
|
| 53 |
+
print(f"⚠️ RERANKER FAILSAFE: {e}")
|
| 54 |
+
return documents[:top_k]
|
| 55 |
+
|
| 56 |
+
_reranker_instance = AxiomReranker()
|
| 57 |
+
|
| 58 |
+
async def get_reranked_scores(query: str, documents: List[str], top_k: int = 10) -> List[str]:
|
| 59 |
+
return await _reranker_instance.rerank(query, documents, top_k=top_k)
|
app/core/retriever.py
ADDED
|
@@ -0,0 +1,110 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import asyncio
|
| 2 |
+
from typing import List, Dict, Any, Optional, cast, Union
|
| 3 |
+
from app.core.database import db
|
| 4 |
+
from app.core.embeddings import get_embedding
|
| 5 |
+
|
| 6 |
+
async def hybrid_search(
|
| 7 |
+
query: str,
|
| 8 |
+
user_id: str,
|
| 9 |
+
filename: Optional[Union[str, List[str]]] = None,
|
| 10 |
+
limit: int = 20
|
| 11 |
+
) -> List[str]:
|
| 12 |
+
"""
|
| 13 |
+
SOTA Retrieval Engine V4.6.
|
| 14 |
+
Fully Asynchronous. Concurrent Multi-Doc Fetching.
|
| 15 |
+
Injects 'Exhibit-ID' metadata envelopes to force granular citations.
|
| 16 |
+
"""
|
| 17 |
+
if not db:
|
| 18 |
+
return[]
|
| 19 |
+
|
| 20 |
+
try:
|
| 21 |
+
# 1. Non-Blocking NVIDIA Embedding Generation
|
| 22 |
+
vector = await asyncio.to_thread(get_embedding, query, "query")
|
| 23 |
+
|
| 24 |
+
is_vault_mode = not filename or filename == "vault" or filename ==["vault"]
|
| 25 |
+
|
| 26 |
+
# =========================================================
|
| 27 |
+
# PATH A: GLOBAL VAULT SEARCH (Multi-file hybrid search)
|
| 28 |
+
# =========================================================
|
| 29 |
+
if is_vault_mode:
|
| 30 |
+
def run_vault_rpc() -> Any:
|
| 31 |
+
return db.rpc("hybrid_vault_search", {
|
| 32 |
+
"query_text": query,
|
| 33 |
+
"query_embedding": vector,
|
| 34 |
+
"match_count": limit,
|
| 35 |
+
"target_user_id": user_id
|
| 36 |
+
}).execute()
|
| 37 |
+
|
| 38 |
+
# Non-blocking RPC Call
|
| 39 |
+
res = await asyncio.to_thread(run_vault_rpc)
|
| 40 |
+
rows = cast(List[Dict[str, Any]], res.data)
|
| 41 |
+
|
| 42 |
+
return[
|
| 43 |
+
f"--- EXHIBIT_START_ID_{i+1} ---\n"
|
| 44 |
+
f"FILE_SOURCE: {row['filename']}\n"
|
| 45 |
+
f"DATA_CONTENT: {row['content']}\n"
|
| 46 |
+
f"--- EXHIBIT_END_ID_{i+1} ---"
|
| 47 |
+
for i, row in enumerate(rows)
|
| 48 |
+
]
|
| 49 |
+
|
| 50 |
+
# =========================================================
|
| 51 |
+
# PATH B: TARGETED DOCUMENT SEARCH (Multi-doc Synthesis)
|
| 52 |
+
# =========================================================
|
| 53 |
+
target_files: List[str] =[]
|
| 54 |
+
if isinstance(filename, str):
|
| 55 |
+
target_files = [filename]
|
| 56 |
+
elif isinstance(filename, list):
|
| 57 |
+
target_files = filename
|
| 58 |
+
|
| 59 |
+
def fetch_docs() -> Any:
|
| 60 |
+
return db.table("documents").select("id, filename").in_("filename", target_files).eq("user_id", user_id).execute()
|
| 61 |
+
|
| 62 |
+
doc_res = await asyncio.to_thread(fetch_docs)
|
| 63 |
+
doc_data = cast(List[Dict[str, Any]], doc_res.data)
|
| 64 |
+
|
| 65 |
+
if not doc_data:
|
| 66 |
+
print(f"RETRIEVER: Context {target_files} missing from vault.")
|
| 67 |
+
return []
|
| 68 |
+
|
| 69 |
+
doc_ids = [d['id'] for d in doc_data]
|
| 70 |
+
id_to_name = {d['id']: d['filename'] for d in doc_data}
|
| 71 |
+
limit_per_doc = max(1, limit // len(doc_ids))
|
| 72 |
+
|
| 73 |
+
# SOTA OPTIMIZATION: Concurrent RPC execution
|
| 74 |
+
# Instead of querying documents sequentially, we query them simultaneously!
|
| 75 |
+
async def fetch_chunks(d_id: int) -> List[Dict[str, Any]]:
|
| 76 |
+
def run_chunk_rpc() -> Any:
|
| 77 |
+
return db.rpc("match_document_chunks", {
|
| 78 |
+
"query_embedding": vector,
|
| 79 |
+
"match_limit": limit_per_doc,
|
| 80 |
+
"target_document_id": d_id,
|
| 81 |
+
"target_user_id": user_id
|
| 82 |
+
}).execute()
|
| 83 |
+
|
| 84 |
+
chunk_res = await asyncio.to_thread(run_chunk_rpc)
|
| 85 |
+
chunk_rows = cast(List[Dict[str, Any]], chunk_res.data)
|
| 86 |
+
|
| 87 |
+
# Tag each chunk with its exact filename
|
| 88 |
+
for r in chunk_rows:
|
| 89 |
+
r['filename'] = id_to_name[d_id]
|
| 90 |
+
return chunk_rows
|
| 91 |
+
|
| 92 |
+
# 3. Fire all document queries to Supabase AT THE SAME TIME
|
| 93 |
+
tasks =[fetch_chunks(d_id) for d_id in doc_ids]
|
| 94 |
+
results_nested = await asyncio.gather(*tasks)
|
| 95 |
+
|
| 96 |
+
# Flatten the nested results array
|
| 97 |
+
all_rows = [row for sublist in results_nested for row in sublist]
|
| 98 |
+
|
| 99 |
+
# SOTA ENVELOPE INJECTION
|
| 100 |
+
return[
|
| 101 |
+
f"--- EXHIBIT_START_ID_{i+1} ---\n"
|
| 102 |
+
f"FILE_SOURCE: {row['filename']}\n"
|
| 103 |
+
f"DATA_CONTENT: {row['content']}\n"
|
| 104 |
+
f"--- EXHIBIT_END_ID_{i+1} ---"
|
| 105 |
+
for i, row in enumerate(all_rows)
|
| 106 |
+
]
|
| 107 |
+
|
| 108 |
+
except Exception as e:
|
| 109 |
+
print(f"❌ RETRIEVER CRITICAL ERROR: {e}")
|
| 110 |
+
return[]
|
app/main.py
ADDED
|
@@ -0,0 +1,83 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
from dotenv import load_dotenv
|
| 3 |
+
|
| 4 |
+
# SOTA: Load environment variables BEFORE any AI components initialize.
|
| 5 |
+
# This guarantees LangSmith telemetry hooks attach correctly.
|
| 6 |
+
load_dotenv()
|
| 7 |
+
|
| 8 |
+
import nest_asyncio
|
| 9 |
+
nest_asyncio.apply()
|
| 10 |
+
|
| 11 |
+
import time
|
| 12 |
+
from contextlib import asynccontextmanager
|
| 13 |
+
from fastapi import FastAPI, Request
|
| 14 |
+
from fastapi.middleware.cors import CORSMiddleware
|
| 15 |
+
from fastapi.middleware.gzip import GZipMiddleware
|
| 16 |
+
|
| 17 |
+
# Axiom Core Imports
|
| 18 |
+
from app.api import ingest, run, history, vault, keys
|
| 19 |
+
from app.core.database import db
|
| 20 |
+
|
| 21 |
+
# --- SOTA: Lifespan Management ---
|
| 22 |
+
@asynccontextmanager
|
| 23 |
+
async def lifespan(app: FastAPI):
|
| 24 |
+
print("AXIOM_CORE: Logic Core Initialized. Dependencies Warm.")
|
| 25 |
+
print("AXIOM_CORE: LangSmith Telemetry Active." if os.getenv("LANGCHAIN_TRACING_V2") == "true" else "AXIOM_CORE: Telemetry Offline.")
|
| 26 |
+
yield
|
| 27 |
+
print("AXIOM_CORE: System Offboarding Complete.")
|
| 28 |
+
|
| 29 |
+
app = FastAPI(
|
| 30 |
+
title="Axiom Engine API",
|
| 31 |
+
description="V4.6 Sovereign Evidence-Gated Intelligence (Multilingual)",
|
| 32 |
+
version="4.6.0",
|
| 33 |
+
lifespan=lifespan
|
| 34 |
+
)
|
| 35 |
+
|
| 36 |
+
# --- SOTA: Performance Middleware ---
|
| 37 |
+
app.add_middleware(GZipMiddleware, minimum_size=500)
|
| 38 |
+
|
| 39 |
+
# --- SOTA: Strict CORS Security ---
|
| 40 |
+
origins =[
|
| 41 |
+
"http://localhost:3000",
|
| 42 |
+
"https://axiom-engine-six.vercel.app",
|
| 43 |
+
"https://lexpertz-axiom-engine.vercel.app",
|
| 44 |
+
"https://huggingface.co",
|
| 45 |
+
]
|
| 46 |
+
|
| 47 |
+
app.add_middleware(
|
| 48 |
+
CORSMiddleware,
|
| 49 |
+
allow_origins=origins,
|
| 50 |
+
allow_origin_regex=r"https://.*-lexpertzai-projects\.vercel\.app",
|
| 51 |
+
allow_credentials=True,
|
| 52 |
+
allow_methods=["GET", "POST", "PUT", "DELETE", "OPTIONS"],
|
| 53 |
+
allow_headers=["*"],
|
| 54 |
+
)
|
| 55 |
+
|
| 56 |
+
# --- SOTA: Telemetry Middleware ---
|
| 57 |
+
@app.middleware("http")
|
| 58 |
+
async def add_process_time_header(request: Request, call_next):
|
| 59 |
+
start_time = time.time()
|
| 60 |
+
response = await call_next(request)
|
| 61 |
+
process_time = time.time() - start_time
|
| 62 |
+
response.headers["X-Process-Time"] = str(round(process_time, 4))
|
| 63 |
+
return response
|
| 64 |
+
|
| 65 |
+
# --- Router Registration ---
|
| 66 |
+
app.include_router(ingest.router, prefix="/api/v1", tags=["Ingestion"])
|
| 67 |
+
app.include_router(run.router, prefix="/api/v1", tags=["Reasoning"])
|
| 68 |
+
app.include_router(history.router, prefix="/api/v1", tags=["History"])
|
| 69 |
+
app.include_router(vault.router, prefix="/api/v1/vault", tags=["Vault"])
|
| 70 |
+
app.include_router(keys.router, prefix="/api/v1/keys", tags=["API Keys"])
|
| 71 |
+
|
| 72 |
+
# --- System Health Monitoring ---
|
| 73 |
+
@app.get("/health")
|
| 74 |
+
async def health_check():
|
| 75 |
+
db_status = "online" if db else "offline"
|
| 76 |
+
return {
|
| 77 |
+
"status": "operational",
|
| 78 |
+
"version": "4.6.0",
|
| 79 |
+
"vault_link": db_status,
|
| 80 |
+
"engine": "Axiom Sovereign V4.6",
|
| 81 |
+
"architect": "meta/llama-3.3-70b-instruct",
|
| 82 |
+
"vector_core": "nvidia/llama-nemotron-embed-1b-v2"
|
| 83 |
+
}
|
app/mcp/__init__.py
ADDED
|
File without changes
|
app/mcp/server.py
ADDED
|
@@ -0,0 +1,115 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import sys
|
| 3 |
+
import asyncio
|
| 4 |
+
from typing import List, Optional
|
| 5 |
+
from dotenv import load_dotenv
|
| 6 |
+
from mcp.server.fastmcp import FastMCP
|
| 7 |
+
from pydantic import Field
|
| 8 |
+
|
| 9 |
+
# 1. PATH RESOLUTION
|
| 10 |
+
BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
|
| 11 |
+
ENV_PATH = os.path.join(BASE_DIR, ".env")
|
| 12 |
+
load_dotenv(ENV_PATH)
|
| 13 |
+
|
| 14 |
+
# Axiom Intelligence Imports
|
| 15 |
+
from app.agents.graph import app_graph
|
| 16 |
+
from app.agents.state import AgentState # CRITICAL: For MyPy type safety
|
| 17 |
+
from app.core.retriever import hybrid_search
|
| 18 |
+
from app.skills.github import execute_github_audit
|
| 19 |
+
from app.skills.database import upload_local_csv_to_vault, execute_dataset_audit
|
| 20 |
+
|
| 21 |
+
# Initialize FastMCP
|
| 22 |
+
mcp = FastMCP("Axiom-Sovereign-Gateway")
|
| 23 |
+
SYSTEM_USER = os.getenv("MCP_SYSTEM_USER", "svc-axiom-core")
|
| 24 |
+
|
| 25 |
+
# --- 2. THE SOVEREIGN TOOLSET ---
|
| 26 |
+
|
| 27 |
+
@mcp.tool()
|
| 28 |
+
async def run_axiom_audit(
|
| 29 |
+
question: str = Field(..., description="The high-level audit query"),
|
| 30 |
+
filenames: List[str] = Field(..., description="The PDF filenames to target")
|
| 31 |
+
) -> str:
|
| 32 |
+
"""Executes a formal evidence-gated audit using the Sovereign Architect."""
|
| 33 |
+
|
| 34 |
+
# SOTA: Strictly typed AgentState initialization
|
| 35 |
+
initial_state: AgentState = {
|
| 36 |
+
"question": question,
|
| 37 |
+
"user_id": SYSTEM_USER,
|
| 38 |
+
"filenames": filenames,
|
| 39 |
+
"history": [],
|
| 40 |
+
"command": None,
|
| 41 |
+
"comparison_map": {},
|
| 42 |
+
"documents": [],
|
| 43 |
+
"generation": "",
|
| 44 |
+
"hallucination_score": 0.0,
|
| 45 |
+
"metrics": {},
|
| 46 |
+
"status": "thinking",
|
| 47 |
+
"retry_count": 0,
|
| 48 |
+
"active_node": None
|
| 49 |
+
}
|
| 50 |
+
|
| 51 |
+
try:
|
| 52 |
+
# V1.x: Explicitly providing version='v1' ensures compatibility with SSE
|
| 53 |
+
final_state = await app_graph.ainvoke(initial_state, version="v1")
|
| 54 |
+
return str(final_state.get("generation", "Audit yielded no results."))
|
| 55 |
+
except Exception as e:
|
| 56 |
+
return f"Axiom Core Error: {str(e)}"
|
| 57 |
+
|
| 58 |
+
@mcp.tool()
|
| 59 |
+
async def search_axiom_vault(
|
| 60 |
+
query: str = Field(..., description="Search query"),
|
| 61 |
+
filenames: List[str] = Field(default=[], description="Specific files to search")
|
| 62 |
+
) -> str:
|
| 63 |
+
"""Performs a high-speed hybrid vector search."""
|
| 64 |
+
try:
|
| 65 |
+
results = await hybrid_search(query=query, user_id=SYSTEM_USER, filename=filenames)
|
| 66 |
+
return "\n\n".join(results) if results else "No evidence found."
|
| 67 |
+
except Exception as e:
|
| 68 |
+
return f"Retrieval Error: {str(e)}"
|
| 69 |
+
|
| 70 |
+
@mcp.tool()
|
| 71 |
+
async def audit_code_implementation(
|
| 72 |
+
file_path: str = Field(..., description="Path to code file"),
|
| 73 |
+
file_content: str = Field(..., description="Raw source code"),
|
| 74 |
+
audit_query: str = Field(..., description="Audit rule"),
|
| 75 |
+
vault_filenames: List[str] = Field(default=[], description="Policy docs")
|
| 76 |
+
) -> str:
|
| 77 |
+
"""Verifies if local code logic matches policy."""
|
| 78 |
+
return await execute_github_audit(
|
| 79 |
+
file_path=file_path, file_content=file_content,
|
| 80 |
+
audit_query=audit_query, vault_filenames=vault_filenames,
|
| 81 |
+
system_user=SYSTEM_USER
|
| 82 |
+
)
|
| 83 |
+
|
| 84 |
+
@mcp.tool()
|
| 85 |
+
async def upload_csv_dataset(
|
| 86 |
+
file_path: str = Field(..., description="CSV path"),
|
| 87 |
+
dataset_name: str = Field(..., description="Dataset ID")
|
| 88 |
+
) -> str:
|
| 89 |
+
"""Uploads spreadsheet to Vault."""
|
| 90 |
+
return await upload_local_csv_to_vault(
|
| 91 |
+
file_path=file_path, dataset_name=dataset_name, system_user=SYSTEM_USER
|
| 92 |
+
)
|
| 93 |
+
|
| 94 |
+
@mcp.tool()
|
| 95 |
+
async def audit_live_dataset(
|
| 96 |
+
dataset_name: str = Field(..., description="Dataset name"),
|
| 97 |
+
audit_query: str = Field(..., description="Reconciliation instructions"),
|
| 98 |
+
vault_filenames: List[str] = Field(default=[], description="PDF evidence")
|
| 99 |
+
) -> str:
|
| 100 |
+
"""Reconciles PDF evidence against a LIVE dataset."""
|
| 101 |
+
return await execute_dataset_audit(
|
| 102 |
+
dataset_name=dataset_name, audit_query=audit_query,
|
| 103 |
+
vault_filenames=vault_filenames, system_user=SYSTEM_USER
|
| 104 |
+
)
|
| 105 |
+
|
| 106 |
+
# --- 3. TRANSPORT EXECUTION ---
|
| 107 |
+
|
| 108 |
+
if __name__ == "__main__":
|
| 109 |
+
transport_mode = os.getenv("MCP_TRANSPORT", "stdio")
|
| 110 |
+
|
| 111 |
+
if transport_mode == "sse":
|
| 112 |
+
print("Axiom MCP Gateway Launching in SSE Mode")
|
| 113 |
+
mcp.run(transport="sse")
|
| 114 |
+
else:
|
| 115 |
+
mcp.run(transport="stdio")
|
app/prompts/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
|
app/prompts/templates.py
ADDED
|
@@ -0,0 +1,137 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from langchain_core.prompts import ChatPromptTemplate
|
| 2 |
+
from langchain_core.output_parsers import PydanticOutputParser
|
| 3 |
+
from pydantic import BaseModel, Field
|
| 4 |
+
|
| 5 |
+
# -----------------------------------------------------------------------------
|
| 6 |
+
# 1. ENTERPRISE SCHEMA REGISTRY (SOTA: Hidden Chain-of-Thought)
|
| 7 |
+
# -----------------------------------------------------------------------------
|
| 8 |
+
# By placing 'scratchpad' first, we force the LLM to reason step-by-step
|
| 9 |
+
# before committing to a final boolean or brief. This drastically reduces hallucinations.
|
| 10 |
+
|
| 11 |
+
class DistilledContext(BaseModel):
|
| 12 |
+
scratchpad: str = Field(description="Step-by-step reasoning: analyze what the user wants, and identify if the snippets contain it.")
|
| 13 |
+
has_relevant_evidence: bool = Field(description="True ONLY if the snippets contain facts directly answering the query.")
|
| 14 |
+
brief: str = Field(description="The synthesized evidence. Preserve exact markers (e.g., --- EXHIBIT_START_ID_1 ---) and code blocks.")
|
| 15 |
+
|
| 16 |
+
class HallucinationGrade(BaseModel):
|
| 17 |
+
scratchpad: str = Field(description="Step-by-step logic: compare the DRAFT REPORT against the RAW EVIDENCE. Look for missing citations or fabricated facts.")
|
| 18 |
+
is_hallucinating: str = Field(description="Must be 'true' or 'false'.")
|
| 19 |
+
explanation: str = Field(description="Final summary of the grade logic.")
|
| 20 |
+
|
| 21 |
+
distill_parser = PydanticOutputParser(pydantic_object=DistilledContext)
|
| 22 |
+
grade_parser = PydanticOutputParser(pydantic_object=HallucinationGrade)
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
# -----------------------------------------------------------------------------
|
| 26 |
+
# 2. PROMPT REGISTRY (SOTA: XML Boundaries & Dynamic Priming)
|
| 27 |
+
# -----------------------------------------------------------------------------
|
| 28 |
+
|
| 29 |
+
# --- AXIOM SOVEREIGN ARCHITECT IDENTITY ---
|
| 30 |
+
AXIOM_SYSTEM_INSTRUCTION = """<role>
|
| 31 |
+
You are the Axiom Sovereign Architect, an elite Enterprise AI Auditor.
|
| 32 |
+
Your mandate is to perform high-fidelity, evidence-gated audits across Financial, Legal, Code, and Database domains.
|
| 33 |
+
</role>
|
| 34 |
+
|
| 35 |
+
<core_directives>
|
| 36 |
+
1. NO CHATTER: Never use conversational filler (e.g., "Here is the report").
|
| 37 |
+
2. MARKDOWN ONLY: Use standard Markdown (`###`, `**text**`).
|
| 38 |
+
3. HTML BAN: NEVER use HTML tags (`<font>`, `<b>`, etc.). It will crash the system.
|
| 39 |
+
4. DYNAMIC HEADER: Start your response with a Markdown H3 header dynamically generated based on the User's Query topic. (e.g., `### [Insert Topic] AUDIT REPORT`).
|
| 40 |
+
5. NO INTERNAL RECAP: Do not mention "The Editor Node" or "Synthesized Brief". Speak as the final authority.
|
| 41 |
+
</core_directives>
|
| 42 |
+
|
| 43 |
+
<domain_protocols>
|
| 44 |
+
- FINANCIAL: Verify column headers before extracting numbers to prevent "Column Drift". State units explicitly.
|
| 45 |
+
- LEGAL: Distinguish between obligations ("shall") vs. rights ("may"). Call out omissions.
|
| 46 |
+
- CODE: Identify the PDF clause and point to the specific line/block of code.
|
| 47 |
+
- DATABASE: Treat "Live Axiom Database" JSON records as the absolute source of truth.
|
| 48 |
+
</domain_protocols>
|
| 49 |
+
|
| 50 |
+
<citation_protocol>
|
| 51 |
+
1. Granular Footnotes: Map every specific claim to its unique Exhibit ID using academic markers: [1], [2].
|
| 52 |
+
2. Source References Section: Conclude your report with a `### Source References` section.
|
| 53 |
+
3. List Format Requirement: Follow the exact structure shown in the example below. Do not use block quotes or paragraphs for the references.
|
| 54 |
+
|
| 55 |
+
<example_format>
|
| 56 |
+
### Source References
|
| 57 |
+
* **[1]** SOURCE: `Filename.pdf` | LOCATION: `Section/Header`
|
| 58 |
+
* **[2]** SOURCE: `Vault_Database` | LOCATION: `Row 42`
|
| 59 |
+
</example_format>
|
| 60 |
+
</citation_protocol>
|
| 61 |
+
|
| 62 |
+
<rejection_protocol>
|
| 63 |
+
If the evidence vault does not contain the answer, output EXACTLY AND ONLY: "No direct evidence found in the vault."
|
| 64 |
+
</rejection_protocol>
|
| 65 |
+
"""
|
| 66 |
+
|
| 67 |
+
VERIFICATION_PROMPT = ChatPromptTemplate.from_messages([
|
| 68 |
+
("system", AXIOM_SYSTEM_INSTRUCTION),
|
| 69 |
+
("human", "<audit_query>\n{question}\n</audit_query>\n\n<evidence_vault>\n{context}\n</evidence_vault>\n\nGenerate the Final Verified Audit Report:"),
|
| 70 |
+
])
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
# --- THE DISTILLATION PROMPT (The Editor Node) ---
|
| 74 |
+
DISTILLATION_PROMPT = ChatPromptTemplate.from_messages([
|
| 75 |
+
("system", """<role>
|
| 76 |
+
You are the Axiom Context Editor. Your goal is to clean and structure messy RAG snippets for downstream reasoning.
|
| 77 |
+
</role>
|
| 78 |
+
|
| 79 |
+
<editorial_mandate>
|
| 80 |
+
1. Noise Extraction: Strip away redundant metadata, UI artifacts, and filler.
|
| 81 |
+
2. Syntax Preservation: PRESERVE exact syntax and structure for Code and JSON.
|
| 82 |
+
3. Marker Preservation: You MUST preserve all `--- EXHIBIT_START_ID_N ---` boundary markers exactly.
|
| 83 |
+
4. No Summarization: Provide raw, cleaned facts in a high-density format.
|
| 84 |
+
</editorial_mandate>
|
| 85 |
+
|
| 86 |
+
<critical_instruction>
|
| 87 |
+
You MUST output ONLY a valid JSON object matching the exact schema below. No markdown wrappers (` ```json `).
|
| 88 |
+
{format_instructions}
|
| 89 |
+
</critical_instruction>"""),
|
| 90 |
+
("human", "<user_query>\n{question}\n</user_query>\n\n<raw_database_snippets>\n{context}\n</raw_database_snippets>"),
|
| 91 |
+
]).partial(format_instructions=distill_parser.get_format_instructions())
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
# --- THE STRATEGIST PROMPT (The Reduce Node) ---
|
| 95 |
+
STRATEGIST_COMPARATIVE_PROMPT = ChatPromptTemplate.from_messages([
|
| 96 |
+
("system", """<role>
|
| 97 |
+
You are the Axiom Strategist. Your mission is a Comparative Cross-Domain Audit.
|
| 98 |
+
</role>
|
| 99 |
+
|
| 100 |
+
<mandate>
|
| 101 |
+
Analyze the excerpts and identify the exact delta (differences). Look for Contradictions, Reconciliation Failures, and Implementation Gaps.
|
| 102 |
+
</mandate>
|
| 103 |
+
|
| 104 |
+
<output_protocol>
|
| 105 |
+
1. Dynamic Header: Start with `### [Topic] COMPARATIVE MATRIX`.
|
| 106 |
+
2. Comparative Matrix: Create a Markdown table mapping specific deviations (Columns: Feature/Clause | Source 1 | Source 2 | Risk Delta).
|
| 107 |
+
3. Synthesis Summary: Write a brief executive summary below the table.
|
| 108 |
+
4. Strict Citations: Use [1],[2] footnotes and conclude with a `### Source References` bulleted list.
|
| 109 |
+
5. NO HTML: Never use `<font>`, `<b>`, or any HTML tags.
|
| 110 |
+
</output_protocol>
|
| 111 |
+
|
| 112 |
+
<rejection_protocol>
|
| 113 |
+
If no comparative divergence is found, state ONLY: "No significant comparative divergence detected between the exhibits."
|
| 114 |
+
</rejection_protocol>"""),
|
| 115 |
+
("human", "<audit_query>\n{question}\n</audit_query>\n\n<exhibits>\n{context}\n</exhibits>\n\nGenerate the Comparative Audit Report:"),
|
| 116 |
+
])
|
| 117 |
+
|
| 118 |
+
|
| 119 |
+
# --- THE ADVERSARIAL GRADER (The Prosecutor Node) ---
|
| 120 |
+
GRADING_PROMPT = ChatPromptTemplate.from_messages([
|
| 121 |
+
("system", """<role>
|
| 122 |
+
You are the Axiom Prosecutor. Your role is purely adversarial.
|
| 123 |
+
You are grading the Architect's 'DRAFT REPORT' against the 'RAW EVIDENCE'.
|
| 124 |
+
</role>
|
| 125 |
+
|
| 126 |
+
<grading_criteria>
|
| 127 |
+
1. The Ghost Check: Does the report mention facts NOT found in the raw evidence? (Hallucination)
|
| 128 |
+
2. The Column-Drift Check: Did the Architect extract a number from the wrong column?
|
| 129 |
+
3. The Citation Check: Did the Architect fail to include footnotes?
|
| 130 |
+
</grading_criteria>
|
| 131 |
+
|
| 132 |
+
<critical_instruction>
|
| 133 |
+
You MUST output ONLY a valid JSON object matching the exact schema below. No markdown wrappers.
|
| 134 |
+
{format_instructions}
|
| 135 |
+
</critical_instruction>"""),
|
| 136 |
+
("human", "<raw_evidence>\n{context}\n</raw_evidence>\n\n<draft_report>\n{generation}\n</draft_report>"),
|
| 137 |
+
]).partial(format_instructions=grade_parser.get_format_instructions())
|
app/skills/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
|
app/skills/database.py
ADDED
|
@@ -0,0 +1,138 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import json
|
| 3 |
+
import asyncio
|
| 4 |
+
import pandas as pd
|
| 5 |
+
from typing import List, Dict, Any, cast
|
| 6 |
+
|
| 7 |
+
from app.core.database import db
|
| 8 |
+
from app.agents.graph import app_graph
|
| 9 |
+
from app.agents.state import AgentState
|
| 10 |
+
from app.core.retriever import hybrid_search
|
| 11 |
+
|
| 12 |
+
# ==========================================
|
| 13 |
+
# 1. SKILL-SPECIFIC PROMPT ADAPTERS
|
| 14 |
+
# ==========================================
|
| 15 |
+
DATASET_QUERY_WRAPPER = """Audit the static document claims against the LIVE structured dataset below.
|
| 16 |
+
|
| 17 |
+
### LIVE DATASET ({dataset_name}):
|
| 18 |
+
{dataset_content}
|
| 19 |
+
|
| 20 |
+
### AUDIT QUERY:
|
| 21 |
+
{audit_query}
|
| 22 |
+
"""
|
| 23 |
+
|
| 24 |
+
DATASET_EXHIBIT_WRAPPER = """--- EXHIBIT_START_ID_DATABASE ---
|
| 25 |
+
FILE_SOURCE: Live Axiom Database | Dataset: {dataset_name}
|
| 26 |
+
DATA_CONTENT: {dataset_content}
|
| 27 |
+
--- EXHIBIT_END_ID_DATABASE ---"""
|
| 28 |
+
|
| 29 |
+
# ==========================================
|
| 30 |
+
# 2. SKILL EXECUTION LOGIC
|
| 31 |
+
# ==========================================
|
| 32 |
+
|
| 33 |
+
async def upload_local_csv_to_vault(
|
| 34 |
+
file_path: str,
|
| 35 |
+
dataset_name: str,
|
| 36 |
+
system_user: str
|
| 37 |
+
) -> str:
|
| 38 |
+
"""Reads a local CSV and securely uploads it to the user's JSONB vault."""
|
| 39 |
+
if not db:
|
| 40 |
+
return "CRITICAL ERROR: Database offline."
|
| 41 |
+
|
| 42 |
+
if not os.path.exists(file_path):
|
| 43 |
+
return f"Error: File not found at {file_path}."
|
| 44 |
+
|
| 45 |
+
try:
|
| 46 |
+
# 1. Thread-safe parsing
|
| 47 |
+
def parse_csv():
|
| 48 |
+
df = pd.read_csv(file_path)
|
| 49 |
+
return df.fillna("").columns.tolist(), df.to_dict(orient="records")
|
| 50 |
+
|
| 51 |
+
columns, records = await asyncio.to_thread(parse_csv)
|
| 52 |
+
|
| 53 |
+
# 2. Non-blocking Database ops
|
| 54 |
+
await asyncio.to_thread(
|
| 55 |
+
lambda: db.table("user_datasets")
|
| 56 |
+
.delete()
|
| 57 |
+
.eq("user_id", system_user)
|
| 58 |
+
.eq("dataset_name", dataset_name)
|
| 59 |
+
.execute()
|
| 60 |
+
)
|
| 61 |
+
|
| 62 |
+
await asyncio.to_thread(
|
| 63 |
+
lambda: db.table("user_datasets").insert({
|
| 64 |
+
"user_id": system_user,
|
| 65 |
+
"dataset_name": dataset_name,
|
| 66 |
+
"columns": columns,
|
| 67 |
+
"data": records
|
| 68 |
+
}).execute()
|
| 69 |
+
)
|
| 70 |
+
|
| 71 |
+
return f"SUCCESS: Ingested {len(records)} rows into dataset '{dataset_name}'."
|
| 72 |
+
|
| 73 |
+
except Exception as e:
|
| 74 |
+
return f"Failed to parse or upload CSV: {str(e)}"
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
async def execute_dataset_audit(
|
| 78 |
+
dataset_name: str,
|
| 79 |
+
audit_query: str,
|
| 80 |
+
vault_filenames: List[str],
|
| 81 |
+
system_user: str
|
| 82 |
+
) -> str:
|
| 83 |
+
"""Pulls live JSONB data and cross-references it with PDF context."""
|
| 84 |
+
if not db:
|
| 85 |
+
return "CRITICAL ERROR: Database offline."
|
| 86 |
+
|
| 87 |
+
try:
|
| 88 |
+
# 1. Non-blocking fetch
|
| 89 |
+
res = await asyncio.to_thread(
|
| 90 |
+
lambda: db.table("user_datasets")
|
| 91 |
+
.select("columns, data")
|
| 92 |
+
.eq("user_id", system_user)
|
| 93 |
+
.eq("dataset_name", dataset_name)
|
| 94 |
+
.execute()
|
| 95 |
+
)
|
| 96 |
+
rows = cast(List[Dict[str, Any]], res.data)
|
| 97 |
+
|
| 98 |
+
if not rows:
|
| 99 |
+
return f"Dataset '{dataset_name}' not found. Upload it first."
|
| 100 |
+
|
| 101 |
+
dataset_content = json.dumps(rows[0]["data"][:500], indent=2)
|
| 102 |
+
|
| 103 |
+
# 2. Formatting
|
| 104 |
+
formatted_query = DATASET_QUERY_WRAPPER.format(
|
| 105 |
+
dataset_name=dataset_name, dataset_content=dataset_content, audit_query=audit_query
|
| 106 |
+
)
|
| 107 |
+
|
| 108 |
+
db_exhibit = DATASET_EXHIBIT_WRAPPER.format(
|
| 109 |
+
dataset_name=dataset_name, dataset_content=dataset_content
|
| 110 |
+
)
|
| 111 |
+
|
| 112 |
+
# 3. PDF Retrieval
|
| 113 |
+
pdf_context = await hybrid_search(query=audit_query, user_id=system_user, filename=vault_filenames) if vault_filenames else []
|
| 114 |
+
all_context = pdf_context + [db_exhibit]
|
| 115 |
+
|
| 116 |
+
# 4. Strictly Typed State
|
| 117 |
+
initial_state: AgentState = {
|
| 118 |
+
"question": formatted_query,
|
| 119 |
+
"user_id": system_user,
|
| 120 |
+
"filenames": vault_filenames,
|
| 121 |
+
"history": [],
|
| 122 |
+
"command": None,
|
| 123 |
+
"comparison_map": {},
|
| 124 |
+
"documents": all_context,
|
| 125 |
+
"generation": "",
|
| 126 |
+
"hallucination_score": 0.0,
|
| 127 |
+
"metrics": {},
|
| 128 |
+
"status": "thinking",
|
| 129 |
+
"retry_count": 0,
|
| 130 |
+
"active_node": None
|
| 131 |
+
}
|
| 132 |
+
|
| 133 |
+
# 5. Invoke circuit with SSE versioning
|
| 134 |
+
final_state = await app_graph.ainvoke(initial_state, version="v1")
|
| 135 |
+
return str(final_state.get("generation", "Audit complete."))
|
| 136 |
+
|
| 137 |
+
except Exception as e:
|
| 138 |
+
return f"Database Audit Error: {str(e)}"
|
app/skills/github.py
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
from typing import List
|
| 3 |
+
from app.agents.graph import app_graph
|
| 4 |
+
from app.core.retriever import hybrid_search
|
| 5 |
+
from app.agents.state import AgentState
|
| 6 |
+
|
| 7 |
+
# --- PROMPT ADAPTERS ---
|
| 8 |
+
GITHUB_QUERY_WRAPPER = """Audit this codebase implementation against the provided vault evidence.
|
| 9 |
+
|
| 10 |
+
### GITHUB CODE ({file_path}):
|
| 11 |
+
{file_content}
|
| 12 |
+
|
| 13 |
+
### AUDIT QUERY:
|
| 14 |
+
{audit_query}
|
| 15 |
+
"""
|
| 16 |
+
|
| 17 |
+
GITHUB_EXHIBIT_WRAPPER = """--- EXHIBIT_START_ID_CODE ---
|
| 18 |
+
FILE_SOURCE: GitHub Evidence | File: {file_path}
|
| 19 |
+
DATA_CONTENT: {file_content}
|
| 20 |
+
--- EXHIBIT_END_ID_CODE ---"""
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
# --- SKILL EXECUTION LOGIC ---
|
| 24 |
+
async def execute_github_audit(
|
| 25 |
+
file_path: str,
|
| 26 |
+
file_content: str,
|
| 27 |
+
audit_query: str,
|
| 28 |
+
vault_filenames: List[str],
|
| 29 |
+
system_user: str
|
| 30 |
+
) -> str:
|
| 31 |
+
"""
|
| 32 |
+
V4.6 SOTA: Receives local code content and cross-references it with Cloud PDF Vault.
|
| 33 |
+
Upgraded for LangGraph 1.x Strict State Typing.
|
| 34 |
+
"""
|
| 35 |
+
try:
|
| 36 |
+
# 1. Format the data using the decoupled wrappers
|
| 37 |
+
formatted_query = GITHUB_QUERY_WRAPPER.format(
|
| 38 |
+
file_path=file_path,
|
| 39 |
+
file_content=file_content,
|
| 40 |
+
audit_query=audit_query
|
| 41 |
+
)
|
| 42 |
+
|
| 43 |
+
code_exhibit = GITHUB_EXHIBIT_WRAPPER.format(
|
| 44 |
+
file_path=file_path,
|
| 45 |
+
file_content=file_content
|
| 46 |
+
)
|
| 47 |
+
|
| 48 |
+
# 2. Pull secondary context from the Supabase PDF Vault
|
| 49 |
+
pdf_context = await hybrid_search(
|
| 50 |
+
query=audit_query,
|
| 51 |
+
user_id=system_user,
|
| 52 |
+
filename=vault_filenames
|
| 53 |
+
) if vault_filenames else []
|
| 54 |
+
|
| 55 |
+
# 3. Merge Local Code with Cloud PDFs into the context stream
|
| 56 |
+
all_context = pdf_context + [code_exhibit]
|
| 57 |
+
|
| 58 |
+
# 4. Initialize State for V4.6 Graph (Strictly Typed)
|
| 59 |
+
initial_state: AgentState = {
|
| 60 |
+
"question": formatted_query,
|
| 61 |
+
"user_id": system_user,
|
| 62 |
+
"filenames": vault_filenames,
|
| 63 |
+
"history": [],
|
| 64 |
+
"command": None,
|
| 65 |
+
"comparison_map": {},
|
| 66 |
+
"documents": all_context,
|
| 67 |
+
"generation": "",
|
| 68 |
+
"hallucination_score": 0.0,
|
| 69 |
+
"metrics": {},
|
| 70 |
+
"status": "thinking",
|
| 71 |
+
"retry_count": 0,
|
| 72 |
+
"active_node": None
|
| 73 |
+
}
|
| 74 |
+
|
| 75 |
+
# 5. Invoke the Sovereign reasoning circuit
|
| 76 |
+
# Using version='v1' to maintain SSE stream compatibility
|
| 77 |
+
final_state = await app_graph.ainvoke(initial_state, version="v1")
|
| 78 |
+
return str(final_state.get("generation", "Audit yielded no results."))
|
| 79 |
+
|
| 80 |
+
except Exception as e:
|
| 81 |
+
print(f"❌ Sovereign Audit Error: {str(e)}")
|
| 82 |
+
return f"Sovereign Audit Error: {str(e)}"
|
migrations/001_init_vault.sql
ADDED
|
@@ -0,0 +1,171 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
-- AXIOM VAULT MASTER SCHEMA V2.7-STABLE
|
| 2 |
+
-- 1. Enable AI Extensions
|
| 3 |
+
create extension if not exists vector;
|
| 4 |
+
|
| 5 |
+
-- 2. Parent Documents Table (The Context Hub)
|
| 6 |
+
create table documents (
|
| 7 |
+
id bigserial primary key,
|
| 8 |
+
filename text not null,
|
| 9 |
+
user_id text not null, -- Clerk Identity
|
| 10 |
+
status text default 'processing',
|
| 11 |
+
is_permanent boolean default false, -- Persistence Logic
|
| 12 |
+
created_at timestamptz default now()
|
| 13 |
+
);
|
| 14 |
+
|
| 15 |
+
-- 3. Evidence Chunks Table (The Vector Store)
|
| 16 |
+
create table document_chunks (
|
| 17 |
+
id bigserial primary key,
|
| 18 |
+
document_id bigint references documents(id) on delete cascade,
|
| 19 |
+
user_id text not null, -- Clerk Identity
|
| 20 |
+
content text not null,
|
| 21 |
+
embedding vector(1024), -- NVIDIA NIM E5-v5 Standard
|
| 22 |
+
metadata jsonb,
|
| 23 |
+
created_at timestamptz default now(),
|
| 24 |
+
-- V2.7: Full-Text Search Vector (Calculated for Keyword matching)
|
| 25 |
+
fts_content tsvector generated always as (to_tsvector('english', content)) stored
|
| 26 |
+
);
|
| 27 |
+
|
| 28 |
+
-- 4. Audit Logs Table (Security Telemetry)
|
| 29 |
+
create table audit_logs (
|
| 30 |
+
id bigserial primary key,
|
| 31 |
+
user_id text not null,
|
| 32 |
+
document_id text,
|
| 33 |
+
question text not null,
|
| 34 |
+
faithfulness float default 0,
|
| 35 |
+
precision float default 0,
|
| 36 |
+
relevance float default 0,
|
| 37 |
+
latency float default 0,
|
| 38 |
+
created_at timestamptz default now()
|
| 39 |
+
);
|
| 40 |
+
|
| 41 |
+
-- 5. Security Framework (Row Level Security)
|
| 42 |
+
alter table documents enable row level security;
|
| 43 |
+
alter table document_chunks enable row level security;
|
| 44 |
+
alter table audit_logs enable row level security;
|
| 45 |
+
|
| 46 |
+
-- Document Policies
|
| 47 |
+
create policy "Users can only view their own documents" on documents for select using (user_id = auth.jwt() ->> 'sub');
|
| 48 |
+
create policy "Users can only insert their own documents" on documents for insert with check (user_id = auth.jwt() ->> 'sub');
|
| 49 |
+
create policy "Users can only delete their own documents" on documents for delete using (user_id = auth.jwt() ->> 'sub');
|
| 50 |
+
|
| 51 |
+
-- Chunk Policies
|
| 52 |
+
create policy "Users can only view their own chunks" on document_chunks for select using (user_id = auth.jwt() ->> 'sub');
|
| 53 |
+
|
| 54 |
+
-- Audit Log Policies
|
| 55 |
+
create policy "Users can only view their own logs" on audit_logs for select using (user_id = auth.jwt() ->> 'sub');
|
| 56 |
+
create policy "Users can insert own logs" on audit_logs for insert with check (user_id = auth.jwt() ->> 'sub');
|
| 57 |
+
|
| 58 |
+
-- 6. High-Performance Multi-Index Strategy
|
| 59 |
+
-- Semantic Search Index (Meaning)
|
| 60 |
+
create index on document_chunks using hnsw (embedding vector_cosine_ops);
|
| 61 |
+
-- Keyword Search Index (Exact matches)
|
| 62 |
+
create index idx_fts_content on document_chunks using gin (fts_content);
|
| 63 |
+
|
| 64 |
+
-- V2.7-PATCH: Optimized Hybrid Vault Search
|
| 65 |
+
CREATE OR REPLACE FUNCTION hybrid_vault_search(
|
| 66 |
+
query_text TEXT,
|
| 67 |
+
query_embedding VECTOR(1024),
|
| 68 |
+
match_count INT,
|
| 69 |
+
target_user_id TEXT
|
| 70 |
+
) RETURNS TABLE (
|
| 71 |
+
id BIGINT,
|
| 72 |
+
document_id BIGINT,
|
| 73 |
+
filename TEXT,
|
| 74 |
+
content TEXT,
|
| 75 |
+
similarity FLOAT,
|
| 76 |
+
fts_rank REAL
|
| 77 |
+
) LANGUAGE plpgsql AS $$
|
| 78 |
+
BEGIN
|
| 79 |
+
RETURN QUERY
|
| 80 |
+
SELECT
|
| 81 |
+
c.id,
|
| 82 |
+
c.document_id,
|
| 83 |
+
d.filename,
|
| 84 |
+
c.content,
|
| 85 |
+
1 - (c.embedding <=> query_embedding) AS similarity,
|
| 86 |
+
-- FIX 1: websearch_to_tsquery for natural language resilience
|
| 87 |
+
ts_rank_cd(c.fts_content, websearch_to_tsquery('english', query_text)) AS fts_rank
|
| 88 |
+
FROM document_chunks c
|
| 89 |
+
JOIN documents d ON c.document_id = d.id
|
| 90 |
+
WHERE c.user_id = target_user_id
|
| 91 |
+
-- FIX 2: Enterprise Hybrid Weighting (0.7 Vector + 0.3 Keyword)
|
| 92 |
+
ORDER BY (0.7 * (1 - (c.embedding <=> query_embedding)) + 0.3 * ts_rank_cd(c.fts_content, websearch_to_tsquery('english', query_text))) DESC
|
| 93 |
+
LIMIT match_count;
|
| 94 |
+
END;
|
| 95 |
+
$$;
|
| 96 |
+
-- 8. THE DOCUMENT SCOPE (Fixes Context Bleed)
|
| 97 |
+
-- Searches ONLY within a specific document ID.
|
| 98 |
+
create or replace function match_document_chunks(
|
| 99 |
+
query_embedding vector(1024),
|
| 100 |
+
match_limit int,
|
| 101 |
+
target_document_id bigint,
|
| 102 |
+
target_user_id text
|
| 103 |
+
) returns table (
|
| 104 |
+
content text,
|
| 105 |
+
similarity float
|
| 106 |
+
) language plpgsql as $$
|
| 107 |
+
begin
|
| 108 |
+
return query
|
| 109 |
+
select
|
| 110 |
+
document_chunks.content,
|
| 111 |
+
1 - (document_chunks.embedding <=> query_embedding) as similarity
|
| 112 |
+
from document_chunks
|
| 113 |
+
where document_id = target_document_id and user_id = target_user_id
|
| 114 |
+
order by document_chunks.embedding <=> query_embedding
|
| 115 |
+
limit match_limit;
|
| 116 |
+
end;
|
| 117 |
+
$$;
|
| 118 |
+
-- V2.9: Chat Persistence Layer
|
| 119 |
+
create table chat_messages (
|
| 120 |
+
id bigserial primary key,
|
| 121 |
+
document_id bigint references documents(id) on delete cascade,
|
| 122 |
+
user_id text not null,
|
| 123 |
+
role text not null check (role in ('user', 'assistant')),
|
| 124 |
+
content text not null,
|
| 125 |
+
-- We store the RAGAS metrics JSON here so the history keeps the scores!
|
| 126 |
+
metrics jsonb,
|
| 127 |
+
created_at timestamptz default now()
|
| 128 |
+
);
|
| 129 |
+
|
| 130 |
+
-- Enable Security
|
| 131 |
+
alter table chat_messages enable row level security;
|
| 132 |
+
|
| 133 |
+
-- Policies (Strict User Isolation)
|
| 134 |
+
create policy "Users can only view their own chat history"
|
| 135 |
+
on chat_messages for select using (user_id = auth.jwt() ->> 'sub');
|
| 136 |
+
|
| 137 |
+
create policy "Users can insert their own chat messages"
|
| 138 |
+
on chat_messages for insert with check (user_id = auth.jwt() ->> 'sub');
|
| 139 |
+
|
| 140 |
+
-- Index for fast loading of long histories
|
| 141 |
+
create index idx_chat_history on chat_messages(document_id, created_at);
|
| 142 |
+
|
| 143 |
+
CREATE TABLE IF NOT EXISTS api_keys (
|
| 144 |
+
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
| 145 |
+
user_id TEXT NOT NULL, -- Links to their Clerk ID
|
| 146 |
+
name TEXT NOT NULL, -- e.g., "MacBook Claude Desktop"
|
| 147 |
+
key_value TEXT NOT NULL UNIQUE, -- The actual axm_live_... token
|
| 148 |
+
last_used_at TIMESTAMPTZ,
|
| 149 |
+
created_at TIMESTAMPTZ DEFAULT NOW(),
|
| 150 |
+
is_active BOOLEAN DEFAULT TRUE
|
| 151 |
+
);
|
| 152 |
+
|
| 153 |
+
-- Index for ultra-fast auth lookups during API calls
|
| 154 |
+
CREATE INDEX IF NOT EXISTS idx_api_keys_value ON api_keys(key_value);
|
| 155 |
+
CREATE INDEX IF NOT EXISTS idx_api_keys_user ON api_keys(user_id);
|
| 156 |
+
|
| 157 |
+
ALTER TABLE api_keys
|
| 158 |
+
ADD COLUMN key_hint TEXT;
|
| 159 |
+
|
| 160 |
+
CREATE TABLE IF NOT EXISTS user_datasets (
|
| 161 |
+
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
| 162 |
+
user_id TEXT NOT NULL, -- Links to their Clerk Auth / MCP Token
|
| 163 |
+
dataset_name TEXT NOT NULL, -- e.g., "Q1_Financial_Ledger"
|
| 164 |
+
columns TEXT[] NOT NULL, -- e.g., ["Date", "Category", "Amount"]
|
| 165 |
+
data JSONB NOT NULL, -- The actual rows of the CSV/Excel
|
| 166 |
+
created_at TIMESTAMPTZ DEFAULT NOW()
|
| 167 |
+
);
|
| 168 |
+
|
| 169 |
+
-- Indexes for ultra-fast JSONB querying and user isolation
|
| 170 |
+
CREATE INDEX IF NOT EXISTS idx_user_datasets_user_id ON user_datasets(user_id);
|
| 171 |
+
CREATE INDEX IF NOT EXISTS idx_user_datasets_name ON user_datasets(user_id, dataset_name);
|
requirements.txt
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# --- Infrastructure (Non-AI Stack) ---
|
| 2 |
+
fastapi==0.135.3
|
| 3 |
+
uvicorn[standard]==0.44.0
|
| 4 |
+
python-dotenv==1.2.2
|
| 5 |
+
python-multipart==0.0.26
|
| 6 |
+
python-jose[cryptography]==3.5.0
|
| 7 |
+
pycryptodome==3.23.0
|
| 8 |
+
tiktoken==0.12.0
|
| 9 |
+
numpy==2.4.4
|
| 10 |
+
httpx[http2]==0.28.1
|
| 11 |
+
sse-starlette==3.3.4
|
| 12 |
+
nest_asyncio==1.6.0
|
| 13 |
+
|
| 14 |
+
# --- LangChain 1.x Core Ecosystem ---
|
| 15 |
+
langchain-core==1.2.28
|
| 16 |
+
langchain==1.2.15
|
| 17 |
+
langchain-community==0.4.1
|
| 18 |
+
langchain-text-splitters==1.1.1
|
| 19 |
+
langchain-groq==1.1.2
|
| 20 |
+
langchain-openai==1.1.12
|
| 21 |
+
langchain-experimental==0.4.1
|
| 22 |
+
langchain-nvidia-ai-endpoints==1.2.1
|
| 23 |
+
|
| 24 |
+
# --- LangGraph (1.x Series) ---
|
| 25 |
+
langgraph==1.1.6
|
| 26 |
+
|
| 27 |
+
# --- Evaluation Framework ---
|
| 28 |
+
ragas==0.4.3
|
| 29 |
+
|
| 30 |
+
# --- AI Provider SDK ---
|
| 31 |
+
openai==2.31.0
|
| 32 |
+
|
| 33 |
+
# --- Document Digestion & Data ---
|
| 34 |
+
docling==2.88.0
|
| 35 |
+
pdf2image==1.17.0
|
| 36 |
+
pypdf==6.10.0
|
| 37 |
+
pandas==3.0.2
|
| 38 |
+
pydantic==2.13.0
|
| 39 |
+
pydantic-settings==2.13.1
|
| 40 |
+
supabase==2.28.3
|
| 41 |
+
python-magic==0.4.27
|
| 42 |
+
|
| 43 |
+
# --- Model Context Protocol & Tools ---
|
| 44 |
+
mcp==1.27.0
|
| 45 |
+
PyGithub==2.9.1
|
| 46 |
+
|
| 47 |
+
# --- Transitive Safety Pins ---
|
| 48 |
+
langsmith==0.7.30
|
| 49 |
+
tenacity==9.1.4
|
| 50 |
+
jsonpatch==1.33
|
| 51 |
+
setuptools==75.8.0
|
| 52 |
+
|
| 53 |
+
# --- FORCE CPU-ONLY TORCH ---
|
| 54 |
+
torch==2.2.2 --index-url https://download.pytorch.org/whl/cpu
|