Hermes commited on
Commit Β·
680b3ba
1
Parent(s): beb7310
feat(catalog/t27): unified data catalog + RAG bridge + reputation
Browse files- backend/app/api/v1/__init__.py +4 -0
- backend/app/api/v1/catalog/__init__.py +4 -0
- backend/app/api/v1/catalog/__pycache__/__init__.cpython-311.pyc +0 -0
- backend/app/api/v1/catalog/__pycache__/router.cpython-311.pyc +0 -0
- backend/app/api/v1/catalog/router.py +155 -0
- backend/app/catalog/__init__.py +86 -0
- backend/app/catalog/__pycache__/__init__.cpython-311.pyc +0 -0
- backend/app/catalog/__pycache__/models.cpython-311.pyc +0 -0
- backend/app/catalog/__pycache__/service.cpython-311.pyc +0 -0
- backend/app/catalog/llm_router.py +128 -0
- backend/app/catalog/models.py +323 -0
- backend/app/catalog/reputation.py +119 -0
- backend/app/catalog/service.py +594 -0
- backend/main.py +1 -0
backend/app/api/v1/__init__.py
CHANGED
|
@@ -61,6 +61,10 @@ from app.api.v1.admin.alerts_webhook import router as admin_alerts_webhook_route
|
|
| 61 |
|
| 62 |
api_v1_router.append(admin_alerts_webhook_router)
|
| 63 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 64 |
|
| 65 |
def build_v1_router() -> APIRouter:
|
| 66 |
"""Construct the v1 aggregator with all migrated routes mounted."""
|
|
|
|
| 61 |
|
| 62 |
api_v1_router.append(admin_alerts_webhook_router)
|
| 63 |
|
| 64 |
+
from app.api.v1.catalog import router as catalog_router # noqa: E402
|
| 65 |
+
|
| 66 |
+
api_v1_router.append(catalog_router)
|
| 67 |
+
|
| 68 |
|
| 69 |
def build_v1_router() -> APIRouter:
|
| 70 |
"""Construct the v1 aggregator with all migrated routes mounted."""
|
backend/app/api/v1/catalog/__init__.py
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Catalog v1 routes β thin HTTP layer."""
|
| 2 |
+
from .router import router
|
| 3 |
+
|
| 4 |
+
__all__ = ["router"]
|
backend/app/api/v1/catalog/__pycache__/__init__.cpython-311.pyc
ADDED
|
Binary file (274 Bytes). View file
|
|
|
backend/app/api/v1/catalog/__pycache__/router.cpython-311.pyc
ADDED
|
Binary file (10 kB). View file
|
|
|
backend/app/api/v1/catalog/router.py
ADDED
|
@@ -0,0 +1,155 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""T27B HTTP routes β CatalogService endpoints.
|
| 2 |
+
|
| 3 |
+
Per v4.0 Β§T27. The thin HTTP layer over app.catalog.service.CatalogService.
|
| 4 |
+
"""
|
| 5 |
+
from __future__ import annotations
|
| 6 |
+
|
| 7 |
+
from typing import Any
|
| 8 |
+
|
| 9 |
+
from fastapi import APIRouter, HTTPException
|
| 10 |
+
from pydantic import BaseModel, Field
|
| 11 |
+
|
| 12 |
+
from app.catalog.models import Chain
|
| 13 |
+
from app.catalog.service import get_catalog
|
| 14 |
+
|
| 15 |
+
router = APIRouter(prefix="/api/v1/catalog", tags=["catalog"])
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
# ββ Request models βββββββββββββββββββββββββββββββββββββββββββββββ
|
| 19 |
+
class RagIngestRequest(BaseModel):
|
| 20 |
+
content: str = Field(..., min_length=1)
|
| 21 |
+
collection: str = "scam_intel"
|
| 22 |
+
doc_id: str | None = None
|
| 23 |
+
metadata: dict[str, Any] = Field(default_factory=dict)
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
class RagSearchRequest(BaseModel):
|
| 27 |
+
query: str = Field(..., min_length=1)
|
| 28 |
+
collection: str = "scam_intel"
|
| 29 |
+
top_k: int = Field(default=5, ge=1, le=50)
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
class ResolveEntityRequest(BaseModel):
|
| 33 |
+
wallet_id: str
|
| 34 |
+
max_chains: int = Field(default=5, ge=1, le=20)
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
class FindRiskyTokensRequest(BaseModel):
|
| 38 |
+
min_rug_count: int = Field(default=1, ge=1)
|
| 39 |
+
chain: str | None = None
|
| 40 |
+
limit: int = Field(default=50, ge=1, le=200)
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
class AttachRagRequest(BaseModel):
|
| 44 |
+
chain: str
|
| 45 |
+
address: str
|
| 46 |
+
qdrant_point_id: str
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
# ββ Health / introspection βββββββββββββββββββββββββββββββββββββββ
|
| 50 |
+
@router.get("/stats")
|
| 51 |
+
async def stats() -> dict:
|
| 52 |
+
"""Catalog stats: which stores are reachable + entity counts."""
|
| 53 |
+
return await get_catalog().stats()
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
@router.get("/probe")
|
| 57 |
+
async def probe() -> dict:
|
| 58 |
+
"""Probe which stores are reachable from this container."""
|
| 59 |
+
return await get_catalog().probe_stores()
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
# ββ Token endpoints βββββββββββββββββββββββββββββββββββββββββββββ
|
| 63 |
+
@router.get("/tokens/{chain}/{address}")
|
| 64 |
+
async def get_token(chain: str, address: str) -> dict:
|
| 65 |
+
"""Get a token by chain+address. Returns full Token model + provenance."""
|
| 66 |
+
try:
|
| 67 |
+
c = Chain(chain)
|
| 68 |
+
except ValueError:
|
| 69 |
+
raise HTTPException(400, f"unknown chain: {chain}")
|
| 70 |
+
tok = await get_catalog().get_token(c, address)
|
| 71 |
+
if not tok:
|
| 72 |
+
raise HTTPException(404, "token not found")
|
| 73 |
+
return tok.model_dump(mode="json")
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
@router.get("/tokens/{chain}/{address}/risk")
|
| 77 |
+
async def get_token_risk(chain: str, address: str) -> dict:
|
| 78 |
+
"""Recipe 3 β Real-time risk score. Composes Redis + Postgres + Neo4j."""
|
| 79 |
+
try:
|
| 80 |
+
c = Chain(chain)
|
| 81 |
+
except ValueError:
|
| 82 |
+
raise HTTPException(400, f"unknown chain: {chain}")
|
| 83 |
+
return await get_catalog().get_token_risk(c, address)
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
@router.post("/tokens/risky-by-deployer")
|
| 87 |
+
async def risky_tokens(req: FindRiskyTokensRequest) -> dict:
|
| 88 |
+
"""Recipe 1 β Find tokens deployed by wallets with rug history."""
|
| 89 |
+
chain_enum = None
|
| 90 |
+
if req.chain:
|
| 91 |
+
try:
|
| 92 |
+
chain_enum = Chain(req.chain)
|
| 93 |
+
except ValueError:
|
| 94 |
+
raise HTTPException(400, f"unknown chain: {req.chain}")
|
| 95 |
+
tokens = await get_catalog().find_tokens_by_deployer_history(
|
| 96 |
+
min_rug_count=req.min_rug_count, chain=chain_enum, limit=req.limit
|
| 97 |
+
)
|
| 98 |
+
return {
|
| 99 |
+
"count": len(tokens),
|
| 100 |
+
"tokens": [t.model_dump(mode="json") for t in tokens],
|
| 101 |
+
}
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
# ββ Wallet endpoints βββββββββββββββββββββββββββββββββββββββββββββ
|
| 105 |
+
@router.get("/wallets/{chain}/{address}")
|
| 106 |
+
async def get_wallet(chain: str, address: str) -> dict:
|
| 107 |
+
try:
|
| 108 |
+
c = Chain(chain)
|
| 109 |
+
except ValueError:
|
| 110 |
+
raise HTTPException(400, f"unknown chain: {chain}")
|
| 111 |
+
w = await get_catalog().get_wallet(c, address)
|
| 112 |
+
if not w:
|
| 113 |
+
raise HTTPException(404, "wallet not found")
|
| 114 |
+
return w.model_dump(mode="json")
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
# ββ Entity resolution (Recipe 5) ββββββββββββββββββββββββββββββββ
|
| 118 |
+
@router.post("/entities/resolve")
|
| 119 |
+
async def resolve_entity(req: ResolveEntityRequest) -> dict:
|
| 120 |
+
"""Cross-chain entity resolution via Neo4j Cypher."""
|
| 121 |
+
return await get_catalog().resolve_entity(req.wallet_id, req.max_chains)
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
# ββ RAG bridge endpoints ββββββββββββββββββββββββββββββββββββββββ
|
| 125 |
+
@router.post("/rag/search")
|
| 126 |
+
async def rag_search(req: RagSearchRequest) -> dict:
|
| 127 |
+
"""Search the RAG system. Returns ranked hits with RRF scores."""
|
| 128 |
+
hits = await get_catalog().rag_search(
|
| 129 |
+
query=req.query, collection=req.collection, top_k=req.top_k
|
| 130 |
+
)
|
| 131 |
+
return {"count": len(hits), "hits": hits}
|
| 132 |
+
|
| 133 |
+
|
| 134 |
+
@router.post("/rag/ingest")
|
| 135 |
+
async def rag_ingest(req: RagIngestRequest) -> dict:
|
| 136 |
+
"""Ingest content into RAG. Returns qdrant_point_id for cross-store linking."""
|
| 137 |
+
return await get_catalog().rag_ingest(
|
| 138 |
+
content=req.content,
|
| 139 |
+
collection=req.collection,
|
| 140 |
+
doc_id=req.doc_id,
|
| 141 |
+
metadata=req.metadata,
|
| 142 |
+
)
|
| 143 |
+
|
| 144 |
+
|
| 145 |
+
@router.post("/tokens/{chain}/{address}/attach-rag")
|
| 146 |
+
async def attach_rag(chain: str, address: str, req: AttachRagRequest) -> dict:
|
| 147 |
+
"""Link an existing RAG embedding (Qdrant point) to a Token row."""
|
| 148 |
+
try:
|
| 149 |
+
c = Chain(chain)
|
| 150 |
+
except ValueError:
|
| 151 |
+
raise HTTPException(400, f"unknown chain: {chain}")
|
| 152 |
+
ok = await get_catalog().attach_rag_to_token(c, address, req.qdrant_point_id)
|
| 153 |
+
if not ok:
|
| 154 |
+
raise HTTPException(404, "token not found or update failed")
|
| 155 |
+
return {"ok": True, "chain": chain, "address": address, "rag_embedding_id": req.qdrant_point_id}
|
backend/app/catalog/__init__.py
ADDED
|
@@ -0,0 +1,86 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Catalog domain β the unified read/write API for RMI.
|
| 2 |
+
|
| 3 |
+
T27 of the v4.0 guide. Every store read/write goes through CatalogService.
|
| 4 |
+
Domain facades call the catalog; they never touch stores directly.
|
| 5 |
+
|
| 6 |
+
Architecture (per v4.0 Β§T27):
|
| 7 |
+
app/catalog/models.py Pydantic v2 entity schemas
|
| 8 |
+
app/catalog/service.py CatalogService β fan-out reads, fan-in writes
|
| 9 |
+
app/catalog/reputation.py Deployer reputation scoring (T31)
|
| 10 |
+
app/catalog/rag_bridge.py Bridge to existing app/rag/ engine
|
| 11 |
+
app/catalog/llm_router.py LiteLLM proxy for AI analysis
|
| 12 |
+
|
| 13 |
+
Cross-store ref shape (per v4.0):
|
| 14 |
+
"chain:address" Wallet, Token, Contract IDs
|
| 15 |
+
UUID Entity, Alert, NewsItem, RAGFinding, Report IDs
|
| 16 |
+
Qdrant point_id RAGFinding.vector_id, rag_embedding_id on Token
|
| 17 |
+
|
| 18 |
+
Public API (re-exported):
|
| 19 |
+
CatalogService, get_catalog
|
| 20 |
+
Entity, Wallet, Deployer, Token, Alert, NewsItem, RAGFinding, ScanReport
|
| 21 |
+
DeployerReputation, RECIPES
|
| 22 |
+
"""
|
| 23 |
+
from __future__ import annotations
|
| 24 |
+
|
| 25 |
+
from app.core import health as health_mod
|
| 26 |
+
from app.core.health import DomainHealth
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
async def _health_check() -> DomainHealth:
|
| 30 |
+
"""Catalog health: which stores are reachable + how many entities."""
|
| 31 |
+
try:
|
| 32 |
+
from app.catalog.service import get_catalog
|
| 33 |
+
|
| 34 |
+
cat = get_catalog()
|
| 35 |
+
reach = await cat.probe_stores()
|
| 36 |
+
healthy = any(reach.values())
|
| 37 |
+
return DomainHealth(
|
| 38 |
+
name="catalog",
|
| 39 |
+
healthy=healthy,
|
| 40 |
+
details={
|
| 41 |
+
"stores_reachable": {k: v for k, v in reach.items()},
|
| 42 |
+
"primary": "redis+rag" if healthy else "none",
|
| 43 |
+
},
|
| 44 |
+
)
|
| 45 |
+
except Exception as e:
|
| 46 |
+
return DomainHealth(name="catalog", healthy=False, error=str(e))
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
health_mod.register_health_check("catalog", _health_check)
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
# Public API
|
| 53 |
+
from app.catalog.models import ( # noqa: E402
|
| 54 |
+
COLLECTIONS,
|
| 55 |
+
CHAIN_REGISTRY,
|
| 56 |
+
Chain,
|
| 57 |
+
Entity,
|
| 58 |
+
EntityLabel,
|
| 59 |
+
RiskTier,
|
| 60 |
+
Token,
|
| 61 |
+
Wallet,
|
| 62 |
+
Deployer,
|
| 63 |
+
Alert,
|
| 64 |
+
NewsItem,
|
| 65 |
+
RAGFinding,
|
| 66 |
+
ScanReport,
|
| 67 |
+
)
|
| 68 |
+
from app.catalog.service import CatalogService, get_catalog # noqa: E402
|
| 69 |
+
|
| 70 |
+
__all__ = [
|
| 71 |
+
"CatalogService",
|
| 72 |
+
"get_catalog",
|
| 73 |
+
"Entity",
|
| 74 |
+
"Wallet",
|
| 75 |
+
"Deployer",
|
| 76 |
+
"Token",
|
| 77 |
+
"Alert",
|
| 78 |
+
"NewsItem",
|
| 79 |
+
"RAGFinding",
|
| 80 |
+
"ScanReport",
|
| 81 |
+
"EntityLabel",
|
| 82 |
+
"Chain",
|
| 83 |
+
"RiskTier",
|
| 84 |
+
"COLLECTIONS",
|
| 85 |
+
"CHAIN_REGISTRY",
|
| 86 |
+
]
|
backend/app/catalog/__pycache__/__init__.cpython-311.pyc
ADDED
|
Binary file (3.29 kB). View file
|
|
|
backend/app/catalog/__pycache__/models.cpython-311.pyc
ADDED
|
Binary file (16.4 kB). View file
|
|
|
backend/app/catalog/__pycache__/service.cpython-311.pyc
ADDED
|
Binary file (34.6 kB). View file
|
|
|
backend/app/catalog/llm_router.py
ADDED
|
@@ -0,0 +1,128 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""T27 LLM Router β sovereign-first LiteLLM proxy for catalog AI.
|
| 2 |
+
|
| 3 |
+
Per v4.0 Β§T28 (News analysis), Β§T29 (Report generation).
|
| 4 |
+
|
| 5 |
+
Self-hosted LiteLLM proxy at litellm.rugmunch.io routes to:
|
| 6 |
+
- DeepSeek-V3 (cost-effective analysis)
|
| 7 |
+
- Qwen (summaries)
|
| 8 |
+
- Local Llama-3 (free-tier fallback)
|
| 9 |
+
|
| 10 |
+
We never call OpenAI directly. If LiteLLM is unreachable, the catalog
|
| 11 |
+
operations that need LLM analysis return None/empty with a logged warning
|
| 12 |
+
rather than failing the whole request.
|
| 13 |
+
"""
|
| 14 |
+
from __future__ import annotations
|
| 15 |
+
|
| 16 |
+
import logging
|
| 17 |
+
import os
|
| 18 |
+
from typing import Any
|
| 19 |
+
|
| 20 |
+
import httpx
|
| 21 |
+
|
| 22 |
+
log = logging.getLogger(__name__)
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
# ββ Config βββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 26 |
+
LITELLM_URL = os.getenv("LITELLM_URL", "http://litellm.rugmunch.io")
|
| 27 |
+
LITELLM_API_KEY = os.getenv("LITELLM_API_KEY", "")
|
| 28 |
+
DEFAULT_MODEL = os.getenv("LITELLM_DEFAULT_MODEL", "deepseek-v3")
|
| 29 |
+
|
| 30 |
+
NEWS_ANALYSIS_PROMPT = """You are an analyst at RugMunch Intelligence, a crypto
|
| 31 |
+
scam-detection platform. Analyze the following news item and produce a
|
| 32 |
+
structured Markdown summary.
|
| 33 |
+
|
| 34 |
+
NEWS ITEM:
|
| 35 |
+
- Title: {title}
|
| 36 |
+
- Source: {source}
|
| 37 |
+
- Published: {published_at}
|
| 38 |
+
- Body: {body_truncated_to_2000_chars}
|
| 39 |
+
|
| 40 |
+
Produce a summary with these sections (use Markdown headers):
|
| 41 |
+
|
| 42 |
+
## Summary
|
| 43 |
+
2-3 sentence plain-English summary.
|
| 44 |
+
|
| 45 |
+
## Affected Tokens
|
| 46 |
+
List any tokens mentioned, with their chain and address if known.
|
| 47 |
+
|
| 48 |
+
## Affected Wallets
|
| 49 |
+
List any wallets mentioned.
|
| 50 |
+
|
| 51 |
+
## Sentiment
|
| 52 |
+
One of: bullish | bearish | neutral | risk-elevating | risk-reducing
|
| 53 |
+
1-sentence justification.
|
| 54 |
+
|
| 55 |
+
## RugMunch Action
|
| 56 |
+
What should our platform do in response? Options:
|
| 57 |
+
- (none)
|
| 58 |
+
- (flag mentioned tokens for re-scan)
|
| 59 |
+
- (alert subscribers)
|
| 60 |
+
- (update deployer reputation)
|
| 61 |
+
- (cross-chain entity resolution trigger)
|
| 62 |
+
|
| 63 |
+
Be concise. Do not speculate beyond what the article says.
|
| 64 |
+
"""
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
class LLMRouter:
|
| 68 |
+
"""Async client for the self-hosted LiteLLM proxy.
|
| 69 |
+
|
| 70 |
+
Falls back to None if the proxy is unreachable β catalog operations
|
| 71 |
+
that need LLM output will skip the AI analysis but still complete
|
| 72 |
+
the rest of the workflow.
|
| 73 |
+
"""
|
| 74 |
+
|
| 75 |
+
def __init__(self, url: str | None = None, api_key: str | None = None) -> None:
|
| 76 |
+
self.url = (url or LITELLM_URL).rstrip("/")
|
| 77 |
+
self.api_key = api_key or LITELLM_API_KEY
|
| 78 |
+
self._client: httpx.AsyncClient | None = None
|
| 79 |
+
|
| 80 |
+
async def _get_client(self) -> httpx.AsyncClient:
|
| 81 |
+
if self._client is None:
|
| 82 |
+
headers = {"Content-Type": "application/json"}
|
| 83 |
+
if self.api_key:
|
| 84 |
+
headers["Authorization"] = f"Bearer {self.api_key}"
|
| 85 |
+
self._client = httpx.AsyncClient(
|
| 86 |
+
base_url=self.url, headers=headers, timeout=30.0
|
| 87 |
+
)
|
| 88 |
+
return self._client
|
| 89 |
+
|
| 90 |
+
async def chat(
|
| 91 |
+
self,
|
| 92 |
+
prompt: str,
|
| 93 |
+
model: str | None = None,
|
| 94 |
+
max_tokens: int = 800,
|
| 95 |
+
temperature: float = 0.3,
|
| 96 |
+
) -> str | None:
|
| 97 |
+
"""Send a chat completion. Returns text or None on failure."""
|
| 98 |
+
try:
|
| 99 |
+
client = await self._get_client()
|
| 100 |
+
r = await client.post(
|
| 101 |
+
"/chat/completions",
|
| 102 |
+
json={
|
| 103 |
+
"model": model or DEFAULT_MODEL,
|
| 104 |
+
"messages": [{"role": "user", "content": prompt}],
|
| 105 |
+
"max_tokens": max_tokens,
|
| 106 |
+
"temperature": temperature,
|
| 107 |
+
},
|
| 108 |
+
)
|
| 109 |
+
if r.status_code != 200:
|
| 110 |
+
log.warning("llm_http_%d: %s", r.status_code, r.text[:200])
|
| 111 |
+
return None
|
| 112 |
+
data = r.json()
|
| 113 |
+
return data["choices"][0]["message"]["content"]
|
| 114 |
+
except Exception as e:
|
| 115 |
+
log.warning("llm_chat_fail: %s", e)
|
| 116 |
+
return None
|
| 117 |
+
|
| 118 |
+
async def analyze_news(
|
| 119 |
+
self, news_item: "NewsItem" # type: ignore # noqa: F821
|
| 120 |
+
) -> str | None:
|
| 121 |
+
"""Generate AI analysis for a NewsItem. Per v4.0 Β§T28."""
|
| 122 |
+
prompt = NEWS_ANALYSIS_PROMPT.format(
|
| 123 |
+
title=news_item.title,
|
| 124 |
+
source=news_item.source,
|
| 125 |
+
published_at=news_item.published_at.isoformat(),
|
| 126 |
+
body_truncated_to_2000_chars=(news_item.body_markdown or news_item.summary)[:2000],
|
| 127 |
+
)
|
| 128 |
+
return await self.chat(prompt, max_tokens=800, temperature=0.3)
|
backend/app/catalog/models.py
ADDED
|
@@ -0,0 +1,323 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""T27A β Canonical entity models for the RMI data catalog.
|
| 2 |
+
|
| 3 |
+
Pydantic v2 (per ADR-0002). Every entity is persisted in exactly one primary
|
| 4 |
+
store, with cross-store references (string IDs) to related entities in other
|
| 5 |
+
stores. CatalogService resolves these references transparently.
|
| 6 |
+
|
| 7 |
+
Cross-store ID conventions:
|
| 8 |
+
Token, Wallet, Contract: f"{chain.value}:{address}"
|
| 9 |
+
Entity, Alert, NewsItem, RAGFinding, ScanReport: UUID4 hex
|
| 10 |
+
Qdrant point_id: 16-byte UUID hex (matches RAG engine)
|
| 11 |
+
|
| 12 |
+
Persistence (per v4.0 Β§T27 "Why each store exists"):
|
| 13 |
+
Redis: hot token data, rate limits, alert state, cron locks
|
| 14 |
+
Postgres: users, api_keys, subscriptions, x402 receipts, audit
|
| 15 |
+
alerts, news_items, scan_reports
|
| 16 |
+
Neo4j: entities, wallets, deployers, contracts, entity labels
|
| 17 |
+
(graph traversal, Cypher)
|
| 18 |
+
Qdrant: RAG embeddings, token similarity, news embeddings
|
| 19 |
+
MinIO: news raw HTML, report markdown, RAG source docs
|
| 20 |
+
Filesystem: ingest tmp, log rotation (transient)
|
| 21 |
+
"""
|
| 22 |
+
from __future__ import annotations
|
| 23 |
+
|
| 24 |
+
from datetime import datetime, UTC
|
| 25 |
+
from enum import Enum
|
| 26 |
+
from typing import Any, Literal, Optional
|
| 27 |
+
|
| 28 |
+
from pydantic import BaseModel, ConfigDict, Field, HttpUrl, field_validator
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
# ββ Enums βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 32 |
+
class Chain(str, Enum):
|
| 33 |
+
"""All chains the platform indexes. Per v4.0 Β§T27."""
|
| 34 |
+
|
| 35 |
+
SOLANA = "solana"
|
| 36 |
+
ETHEREUM = "ethereum"
|
| 37 |
+
BASE = "base"
|
| 38 |
+
ARBITRUM = "arbitrum"
|
| 39 |
+
OPTIMISM = "optimism"
|
| 40 |
+
POLYGON = "polygon"
|
| 41 |
+
BSC = "bsc"
|
| 42 |
+
TRON = "tron"
|
| 43 |
+
BITCOIN = "bitcoin"
|
| 44 |
+
AVALANCHE = "avalanche"
|
| 45 |
+
FANTOM = "fantom"
|
| 46 |
+
GNOSIS = "gnosis"
|
| 47 |
+
# EVM subnets (sample β full 96 in CHAIN_REGISTRY)
|
| 48 |
+
SEPOLIA = "sepolia"
|
| 49 |
+
LINEA = "linea"
|
| 50 |
+
SCROLL = "scroll"
|
| 51 |
+
ZKSYNC = "zksync"
|
| 52 |
+
BLAST = "blast"
|
| 53 |
+
MANTLE = "mantle"
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
# Full chain registry β v4.0 says 96 chains. We track the canonical 20 here
|
| 57 |
+
# plus extend via CHAIN_REGISTRY for the remaining 76. Adding a chain is a
|
| 58 |
+
# one-line edit.
|
| 59 |
+
CHAIN_REGISTRY: dict[str, dict[str, str]] = {
|
| 60 |
+
"solana": {"name": "Solana", "type": "svm", "native": "SOL", "explorer": "https://solscan.io"},
|
| 61 |
+
"ethereum": {"name": "Ethereum", "type": "evm", "native": "ETH", "explorer": "https://etherscan.io"},
|
| 62 |
+
"base": {"name": "Base", "type": "evm", "native": "ETH", "explorer": "https://basescan.org"},
|
| 63 |
+
"arbitrum": {"name": "Arbitrum One", "type": "evm", "native": "ETH", "explorer": "https://arbiscan.io"},
|
| 64 |
+
"optimism": {"name": "Optimism", "type": "evm", "native": "ETH", "explorer": "https://optimistic.etherscan.io"},
|
| 65 |
+
"polygon": {"name": "Polygon", "type": "evm", "native": "MATIC", "explorer": "https://polygonscan.com"},
|
| 66 |
+
"bsc": {"name": "BNB Smart Chain", "type": "evm", "native": "BNB", "explorer": "https://bscscan.com"},
|
| 67 |
+
"tron": {"name": "TRON", "type": "tvm", "native": "TRX", "explorer": "https://tronscan.org"},
|
| 68 |
+
"bitcoin": {"name": "Bitcoin", "type": "utxo", "native": "BTC", "explorer": "https://mempool.space"},
|
| 69 |
+
"avalanche": {"name": "Avalanche C-Chain", "type": "evm", "native": "AVAX", "explorer": "https://snowtrace.io"},
|
| 70 |
+
"fantom": {"name": "Fantom Opera", "type": "evm", "native": "FTM", "explorer": "https://ftmscan.com"},
|
| 71 |
+
"gnosis": {"name": "Gnosis Chain", "type": "evm", "native": "xDAI", "explorer": "https://gnosisscan.io"},
|
| 72 |
+
}
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
class RiskTier(str, Enum):
|
| 76 |
+
LOW = "low"
|
| 77 |
+
MEDIUM = "medium"
|
| 78 |
+
HIGH = "high"
|
| 79 |
+
CRITICAL = "critical"
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
# ββ Entities (Neo4j primary) βββββββββββββββββββββββββββββββββββββββ
|
| 83 |
+
class Entity(BaseModel):
|
| 84 |
+
"""A logical entity resolved across chains. Neo4j primary."""
|
| 85 |
+
|
| 86 |
+
model_config = ConfigDict(extra="ignore")
|
| 87 |
+
|
| 88 |
+
entity_id: str = Field(..., description="UUID, Neo4j primary key")
|
| 89 |
+
label: Optional[str] = None
|
| 90 |
+
aliases: list[str] = Field(default_factory=list)
|
| 91 |
+
first_seen: datetime
|
| 92 |
+
last_seen: datetime
|
| 93 |
+
risk_score: Optional[int] = Field(None, ge=0, le=100)
|
| 94 |
+
tags: list[str] = Field(default_factory=list)
|
| 95 |
+
notes: Optional[str] = None
|
| 96 |
+
store: Literal["neo4j"] = "neo4j"
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
class EntityLabel(BaseModel):
|
| 100 |
+
"""A label attached to an entity. Neo4j primary."""
|
| 101 |
+
|
| 102 |
+
model_config = ConfigDict(extra="ignore")
|
| 103 |
+
|
| 104 |
+
entity_id: str
|
| 105 |
+
label: str
|
| 106 |
+
source: Literal["manual", "heuristic", "third_party"]
|
| 107 |
+
confidence: float = Field(ge=0.0, le=1.0)
|
| 108 |
+
added_at: datetime
|
| 109 |
+
added_by: str
|
| 110 |
+
store: Literal["neo4j"] = "neo4j"
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
# ββ Wallets + Deployers (Neo4j primary) βββββββββββββββββββββββββββ
|
| 114 |
+
class Wallet(BaseModel):
|
| 115 |
+
"""An on-chain wallet. Neo4j primary; linked to Entity."""
|
| 116 |
+
|
| 117 |
+
model_config = ConfigDict(extra="ignore")
|
| 118 |
+
|
| 119 |
+
wallet_id: str = Field(..., description='"chain:address", e.g. "solana:7Np41..."')
|
| 120 |
+
chain: Chain
|
| 121 |
+
address: str
|
| 122 |
+
entity_id: Optional[str] = None
|
| 123 |
+
first_seen: datetime
|
| 124 |
+
last_seen: datetime
|
| 125 |
+
tx_count: int = 0
|
| 126 |
+
total_volume_usd: float = 0.0
|
| 127 |
+
is_deployer: bool = False
|
| 128 |
+
is_known_exchange: bool = False
|
| 129 |
+
is_suspicious: bool = False
|
| 130 |
+
reputation_score: Optional[int] = Field(None, ge=0, le=100)
|
| 131 |
+
store: Literal["neo4j"] = "neo4j"
|
| 132 |
+
|
| 133 |
+
@field_validator("wallet_id")
|
| 134 |
+
@classmethod
|
| 135 |
+
def _check_id_format(cls, v: str) -> str:
|
| 136 |
+
if ":" not in v:
|
| 137 |
+
raise ValueError("wallet_id must be 'chain:address'")
|
| 138 |
+
chain, _ = v.split(":", 1)
|
| 139 |
+
if chain not in {c.value for c in Chain}:
|
| 140 |
+
# Allow unknown chains (forward compat) but flag them
|
| 141 |
+
pass
|
| 142 |
+
return v
|
| 143 |
+
|
| 144 |
+
|
| 145 |
+
class Deployer(Wallet):
|
| 146 |
+
"""A wallet that has deployed at least one token. Extends Wallet."""
|
| 147 |
+
|
| 148 |
+
model_config = ConfigDict(extra="ignore")
|
| 149 |
+
|
| 150 |
+
deployments: list[str] = Field(default_factory=list, description="token_ids")
|
| 151 |
+
rug_count: int = 0
|
| 152 |
+
legit_count: int = 0
|
| 153 |
+
avg_token_lifetime_days: float = 0.0
|
| 154 |
+
reputation_score: Optional[int] = Field(
|
| 155 |
+
None,
|
| 156 |
+
ge=0,
|
| 157 |
+
le=100,
|
| 158 |
+
description="Weighted: legit_count * 1.0 - rug_count * 3.0 + age_bonus - news_penalty. Cached in Redis TTL 1h.",
|
| 159 |
+
)
|
| 160 |
+
|
| 161 |
+
|
| 162 |
+
# ββ Tokens (Postgres primary) ββββββββββββββββββββββββββββββββββββββ
|
| 163 |
+
class Token(BaseModel):
|
| 164 |
+
"""A token contract. Postgres primary; references Deployer in Neo4j."""
|
| 165 |
+
|
| 166 |
+
model_config = ConfigDict(extra="ignore")
|
| 167 |
+
|
| 168 |
+
token_id: str = Field(..., description='"chain:address"')
|
| 169 |
+
chain: Chain
|
| 170 |
+
address: str
|
| 171 |
+
symbol: str
|
| 172 |
+
name: str
|
| 173 |
+
decimals: int
|
| 174 |
+
deployer_wallet_id: Optional[str] = Field(
|
| 175 |
+
None, description="Cross-store ref to Wallet (Neo4j)"
|
| 176 |
+
)
|
| 177 |
+
deployed_at: datetime
|
| 178 |
+
initial_supply: int
|
| 179 |
+
current_supply: Optional[int] = None
|
| 180 |
+
is_honeypot: Optional[bool] = None
|
| 181 |
+
is_mintable: Optional[bool] = None
|
| 182 |
+
is_proxy: Optional[bool] = None
|
| 183 |
+
tax_buy_bps: Optional[int] = None
|
| 184 |
+
tax_sell_bps: Optional[int] = None
|
| 185 |
+
risk_tier: Optional[RiskTier] = None
|
| 186 |
+
risk_score: Optional[int] = Field(None, ge=0, le=100)
|
| 187 |
+
risk_factors: list[str] = Field(default_factory=list)
|
| 188 |
+
rag_embedding_id: Optional[str] = Field(
|
| 189 |
+
None, description="Cross-store ref to Qdrant point (16-byte hex)"
|
| 190 |
+
)
|
| 191 |
+
store: Literal["postgres"] = "postgres"
|
| 192 |
+
|
| 193 |
+
|
| 194 |
+
# ββ Alerts (Postgres primary) ββββββββββββββββββββββββββββββββββββββ
|
| 195 |
+
class Alert(BaseModel):
|
| 196 |
+
"""A risk alert. Postgres primary."""
|
| 197 |
+
|
| 198 |
+
model_config = ConfigDict(extra="ignore")
|
| 199 |
+
|
| 200 |
+
alert_id: str = Field(..., description="UUID")
|
| 201 |
+
token_id: Optional[str] = None
|
| 202 |
+
wallet_id: Optional[str] = None
|
| 203 |
+
chain: Optional[Chain] = None
|
| 204 |
+
alert_type: Literal[
|
| 205 |
+
"rug_detected",
|
| 206 |
+
"deployer_history",
|
| 207 |
+
"liquidity_drain",
|
| 208 |
+
"honeypot_detected",
|
| 209 |
+
"high_tax_change",
|
| 210 |
+
"whale_dump",
|
| 211 |
+
]
|
| 212 |
+
severity: Literal["info", "warning", "critical"]
|
| 213 |
+
title: str
|
| 214 |
+
description: str
|
| 215 |
+
evidence: dict[str, Any] = Field(default_factory=dict)
|
| 216 |
+
created_at: datetime
|
| 217 |
+
resolved_at: Optional[datetime] = None
|
| 218 |
+
store: Literal["postgres"] = "postgres"
|
| 219 |
+
|
| 220 |
+
|
| 221 |
+
# ββ News (Postgres primary + Qdrant embeddings) ββββββββββββββββββββ
|
| 222 |
+
class NewsItem(BaseModel):
|
| 223 |
+
"""A news article from RSS. Postgres primary; embeddings in Qdrant."""
|
| 224 |
+
|
| 225 |
+
model_config = ConfigDict(extra="ignore")
|
| 226 |
+
|
| 227 |
+
news_id: str = Field(..., description="UUID")
|
| 228 |
+
url: HttpUrl
|
| 229 |
+
title: str
|
| 230 |
+
summary: str
|
| 231 |
+
body_markdown: Optional[str] = None
|
| 232 |
+
source: str
|
| 233 |
+
published_at: datetime
|
| 234 |
+
ingested_at: datetime
|
| 235 |
+
chains_mentioned: list[Chain] = Field(default_factory=list)
|
| 236 |
+
tokens_mentioned: list[str] = Field(default_factory=list)
|
| 237 |
+
wallets_mentioned: list[str] = Field(default_factory=list)
|
| 238 |
+
sentiment_score: Optional[float] = Field(None, ge=-1.0, le=1.0)
|
| 239 |
+
ai_analysis: Optional[str] = None
|
| 240 |
+
rag_embedding_id: Optional[str] = None
|
| 241 |
+
store: Literal["postgres"] = "postgres"
|
| 242 |
+
|
| 243 |
+
|
| 244 |
+
# ββ RAG Findings (Qdrant primary + Postgres metadata) βββββββββββββββ
|
| 245 |
+
class RAGFinding(BaseModel):
|
| 246 |
+
"""A fact extracted by RAG. Qdrant primary (vector); metadata in Postgres."""
|
| 247 |
+
|
| 248 |
+
model_config = ConfigDict(extra="ignore")
|
| 249 |
+
|
| 250 |
+
finding_id: str = Field(..., description="UUID")
|
| 251 |
+
source_type: Literal["news", "onchain", "audit", "social", "manual"]
|
| 252 |
+
source_url: Optional[HttpUrl] = None
|
| 253 |
+
source_token_id: Optional[str] = None
|
| 254 |
+
source_wallet_id: Optional[str] = None
|
| 255 |
+
claim: str
|
| 256 |
+
confidence: float = Field(ge=0.0, le=1.0)
|
| 257 |
+
extracted_at: datetime
|
| 258 |
+
qdrant_point_id: str
|
| 259 |
+
store: Literal["qdrant"] = "qdrant"
|
| 260 |
+
|
| 261 |
+
|
| 262 |
+
# ββ Reports (Postgres + MinIO) ββββββββββββββββββββββββββββββββββββββ
|
| 263 |
+
class ScanReport(BaseModel):
|
| 264 |
+
"""A research report. Postgres primary; markdown in MinIO."""
|
| 265 |
+
|
| 266 |
+
model_config = ConfigDict(extra="ignore")
|
| 267 |
+
|
| 268 |
+
report_id: str = Field(..., description="UUID")
|
| 269 |
+
subject_type: Literal["token", "wallet", "deployer"]
|
| 270 |
+
subject_id: str
|
| 271 |
+
generated_at: datetime
|
| 272 |
+
generated_by_model: str
|
| 273 |
+
risk_score: int = Field(ge=0, le=100)
|
| 274 |
+
risk_tier: RiskTier
|
| 275 |
+
sections: dict[str, str] = Field(default_factory=dict)
|
| 276 |
+
markdown_url: Optional[HttpUrl] = None
|
| 277 |
+
paid_via_x402: Optional[str] = None
|
| 278 |
+
store: Literal["postgres", "minio"] = "postgres"
|
| 279 |
+
|
| 280 |
+
def to_markdown(self) -> str:
|
| 281 |
+
"""Render report sections to a single Markdown document."""
|
| 282 |
+
parts = [
|
| 283 |
+
f"# Research Report: {self.subject_type.title()} `{self.subject_id}`",
|
| 284 |
+
"",
|
| 285 |
+
f"**Generated:** {self.generated_at.isoformat()}",
|
| 286 |
+
f"**Generated by:** {self.generated_by_model}",
|
| 287 |
+
f"**Risk score:** {self.risk_score}/100 ({self.risk_tier.value.upper()})",
|
| 288 |
+
"",
|
| 289 |
+
]
|
| 290 |
+
for section, body in self.sections.items():
|
| 291 |
+
parts.append(f"## {section.replace('_', ' ').title()}")
|
| 292 |
+
parts.append("")
|
| 293 |
+
parts.append(body)
|
| 294 |
+
parts.append("")
|
| 295 |
+
parts.append("---")
|
| 296 |
+
parts.append(f"*Report ID: {self.report_id}*")
|
| 297 |
+
return "\n".join(parts)
|
| 298 |
+
|
| 299 |
+
|
| 300 |
+
# ββ Wire-format helpers βββββββββββββββββββββββββββββββββββββββββββββ
|
| 301 |
+
def utcnow() -> datetime:
|
| 302 |
+
"""Timezone-aware UTC now. Pydantic serializes to ISO 8601."""
|
| 303 |
+
return datetime.now(UTC)
|
| 304 |
+
|
| 305 |
+
|
| 306 |
+
# ββ RAG engine collections (kept here so catalog + RAG share the list) β
|
| 307 |
+
COLLECTIONS: list[str] = [
|
| 308 |
+
# Per v4.0 catalog/RAG bridge β these are the canonical 13 RAG
|
| 309 |
+
# collections that also have Token/Wallet/etc cross-refs.
|
| 310 |
+
"scam_intel",
|
| 311 |
+
"deployer_history",
|
| 312 |
+
"wallet_labels",
|
| 313 |
+
"contract_audit",
|
| 314 |
+
"phishing_db",
|
| 315 |
+
"defi_hacks",
|
| 316 |
+
"rug_timeline",
|
| 317 |
+
"vuln_patterns",
|
| 318 |
+
"crime_reports",
|
| 319 |
+
"transaction_patterns",
|
| 320 |
+
"known_scams",
|
| 321 |
+
"token_analysis",
|
| 322 |
+
"market_intel",
|
| 323 |
+
]
|
backend/app/catalog/reputation.py
ADDED
|
@@ -0,0 +1,119 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""T31 β Deployer Reputation System.
|
| 2 |
+
|
| 3 |
+
Per v4.0 Β§T31. Deterministic, auditable, cached in Redis with TTL 1h.
|
| 4 |
+
|
| 5 |
+
The score is a pure function of inputs β no LLM in the loop. Given the
|
| 6 |
+
same on-chain history, news, and RAG findings, the score is the same.
|
| 7 |
+
An analyst can challenge a score by inspecting the weights.
|
| 8 |
+
"""
|
| 9 |
+
from __future__ import annotations
|
| 10 |
+
|
| 11 |
+
import logging
|
| 12 |
+
import os
|
| 13 |
+
from datetime import datetime, UTC
|
| 14 |
+
|
| 15 |
+
from app.catalog.models import Deployer, utcnow
|
| 16 |
+
|
| 17 |
+
log = logging.getLogger(__name__)
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
# ββ Weights (tunable, documented in DESIGN.md) βββββββββββββββββββββ
|
| 21 |
+
WEIGHTS: dict[str, int] = {
|
| 22 |
+
"experience_bonus_per_deploy": 1, # +1 per prior deployment, max +10
|
| 23 |
+
"experience_bonus_cap": 10,
|
| 24 |
+
"rug_penalty": 30, # -30 per prior rug, max -90
|
| 25 |
+
"rug_penalty_cap": 90,
|
| 26 |
+
"longevity_bonus": 10, # +10 if wallet > 1 year old
|
| 27 |
+
"longevity_threshold_days": 365,
|
| 28 |
+
"volume_bonus": 10, # +10 if total raised > $1M
|
| 29 |
+
"volume_threshold_usd": 1_000_000,
|
| 30 |
+
"news_penalty": 15, # -15 if avg news sentiment < -0.3
|
| 31 |
+
"news_bonus": 10, # +10 if avg news sentiment > 0.3
|
| 32 |
+
"news_window_hours": 720, # 30 days
|
| 33 |
+
"rag_penalty": 20, # -20 per high-confidence RAG finding
|
| 34 |
+
"rag_penalty_cap": 60,
|
| 35 |
+
"rag_min_confidence": 0.7,
|
| 36 |
+
}
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
async def compute_deployer_reputation(
|
| 40 |
+
deployer: Deployer,
|
| 41 |
+
catalog: "CatalogService", # forward ref β avoids circular import
|
| 42 |
+
) -> int:
|
| 43 |
+
"""Compute 0-100 reputation score. Pure function of inputs.
|
| 44 |
+
|
| 45 |
+
100 = long clean history, 0 = known serial rugger.
|
| 46 |
+
"""
|
| 47 |
+
cache_key = f"catalog:deployer_rep:{deployer.wallet_id}"
|
| 48 |
+
if catalog._health.redis:
|
| 49 |
+
try:
|
| 50 |
+
cached = await catalog._redis.get(cache_key)
|
| 51 |
+
if cached:
|
| 52 |
+
return int(cached)
|
| 53 |
+
except Exception:
|
| 54 |
+
pass
|
| 55 |
+
|
| 56 |
+
score = 50 # neutral baseline
|
| 57 |
+
|
| 58 |
+
# Experience bonus
|
| 59 |
+
deployments = len(deployer.deployments)
|
| 60 |
+
score += min(deployments * WEIGHTS["experience_bonus_per_deploy"], WEIGHTS["experience_bonus_cap"])
|
| 61 |
+
|
| 62 |
+
# Rug penalty (dominant)
|
| 63 |
+
score -= min(deployer.rug_count * WEIGHTS["rug_penalty"], WEIGHTS["rug_penalty_cap"])
|
| 64 |
+
|
| 65 |
+
# Longevity bonus
|
| 66 |
+
age_days = (utcnow() - deployer.first_seen).days
|
| 67 |
+
if age_days > WEIGHTS["longevity_threshold_days"]:
|
| 68 |
+
score += WEIGHTS["longevity_bonus"]
|
| 69 |
+
|
| 70 |
+
# Volume bonus
|
| 71 |
+
if deployer.total_volume_usd > WEIGHTS["volume_threshold_usd"]:
|
| 72 |
+
score += WEIGHTS["volume_bonus"]
|
| 73 |
+
|
| 74 |
+
# News sentiment (if Postgres is reachable)
|
| 75 |
+
if catalog._health.postgres:
|
| 76 |
+
try:
|
| 77 |
+
async with catalog._pg_pool.acquire() as conn:
|
| 78 |
+
rows = await conn.fetch(
|
| 79 |
+
"SELECT sentiment_score FROM news_items "
|
| 80 |
+
"WHERE $1 = ANY(wallets_mentioned) "
|
| 81 |
+
"AND published_at > NOW() - INTERVAL '%s hours' "
|
| 82 |
+
"LIMIT 20" % WEIGHTS["news_window_hours"],
|
| 83 |
+
deployer.wallet_id,
|
| 84 |
+
)
|
| 85 |
+
if rows:
|
| 86 |
+
scores = [r["sentiment_score"] for r in rows if r["sentiment_score"] is not None]
|
| 87 |
+
if scores:
|
| 88 |
+
avg = sum(scores) / len(scores)
|
| 89 |
+
if avg < -0.3:
|
| 90 |
+
score -= WEIGHTS["news_penalty"]
|
| 91 |
+
elif avg > 0.3:
|
| 92 |
+
score += WEIGHTS["news_bonus"]
|
| 93 |
+
except Exception as e:
|
| 94 |
+
log.debug("reputation_news_fail: %s", e)
|
| 95 |
+
|
| 96 |
+
# RAG findings (rug-related, high confidence)
|
| 97 |
+
try:
|
| 98 |
+
findings = await catalog.rag_search(
|
| 99 |
+
query=deployer.wallet_id, collection="deployer_history", top_k=10
|
| 100 |
+
)
|
| 101 |
+
rug_findings = [
|
| 102 |
+
f for f in findings
|
| 103 |
+
if "rug" in f.get("text", "").lower() or "scam" in f.get("text", "").lower()
|
| 104 |
+
]
|
| 105 |
+
# Confidence proxy: RRF score in the search results
|
| 106 |
+
high_conf = [f for f in rug_findings if f.get("score", 0) > WEIGHTS["rag_min_confidence"]]
|
| 107 |
+
score -= min(len(high_conf) * WEIGHTS["rag_penalty"], WEIGHTS["rag_penalty_cap"])
|
| 108 |
+
except Exception as e:
|
| 109 |
+
log.debug("reputation_rag_fail: %s", e)
|
| 110 |
+
|
| 111 |
+
score = max(0, min(100, score))
|
| 112 |
+
|
| 113 |
+
if catalog._health.redis:
|
| 114 |
+
try:
|
| 115 |
+
await catalog._redis.setex(cache_key, 3600, str(score))
|
| 116 |
+
except Exception:
|
| 117 |
+
pass
|
| 118 |
+
|
| 119 |
+
return score
|
backend/app/catalog/service.py
ADDED
|
@@ -0,0 +1,594 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""T27 CatalogService β unified read/write API for RMI.
|
| 2 |
+
|
| 3 |
+
Per v4.0 Β§T27. The CatalogService is the ONLY sanctioned way to read or
|
| 4 |
+
write catalog data. Domain facades call this; they never touch stores directly.
|
| 5 |
+
|
| 6 |
+
Architecture:
|
| 7 |
+
- Lazy store clients (init on first use, retry on failure)
|
| 8 |
+
- Graceful degradation: unreachable stores return None/empty, not crash
|
| 9 |
+
- Redis cache layer (always available since rmi-redis is up)
|
| 10 |
+
- Cross-store ID conventions from app.catalog.models
|
| 11 |
+
|
| 12 |
+
Recipe coverage:
|
| 13 |
+
Recipe 1: find_tokens_by_deployer_history (Postgres + Neo4j)
|
| 14 |
+
Recipe 2: find_similar_tokens (Qdrant + Postgres)
|
| 15 |
+
Recipe 3: get_token_risk (Redis + Postgres + Neo4j)
|
| 16 |
+
Recipe 4: news_price_correlation (Postgres, basic v1)
|
| 17 |
+
Recipe 5: resolve_entity (Neo4j Cypher)
|
| 18 |
+
"""
|
| 19 |
+
from __future__ import annotations
|
| 20 |
+
|
| 21 |
+
import asyncio
|
| 22 |
+
import json
|
| 23 |
+
import logging
|
| 24 |
+
import os
|
| 25 |
+
import time
|
| 26 |
+
from typing import Any, Optional
|
| 27 |
+
|
| 28 |
+
import httpx
|
| 29 |
+
|
| 30 |
+
from app.catalog.models import (
|
| 31 |
+
Alert,
|
| 32 |
+
COLLECTIONS,
|
| 33 |
+
Chain,
|
| 34 |
+
Deployer,
|
| 35 |
+
Entity,
|
| 36 |
+
NewsItem,
|
| 37 |
+
RAGFinding,
|
| 38 |
+
ScanReport,
|
| 39 |
+
Token,
|
| 40 |
+
Wallet,
|
| 41 |
+
utcnow,
|
| 42 |
+
)
|
| 43 |
+
from app.rag.engine import three_pillar_search as rag_three_pillar_search
|
| 44 |
+
from app.rag.engine import ingest_document as rag_ingest_document
|
| 45 |
+
|
| 46 |
+
log = logging.getLogger(__name__)
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
# ββ Connection config (from env, with sensible defaults to netcup IPs) ββ
|
| 50 |
+
# These defaults point to the actual IPs on netcup rmi_network.
|
| 51 |
+
# Override via env vars in docker-compose.yml for portability.
|
| 52 |
+
DEFAULT_CONFIG: dict[str, Any] = {
|
| 53 |
+
"redis": {
|
| 54 |
+
"host": os.getenv("REDIS_HOST", "rmi-redis"),
|
| 55 |
+
"port": int(os.getenv("REDIS_PORT", "6379")),
|
| 56 |
+
"password": os.getenv("REDIS_PASSWORD", "RMI_PROD_REDIS_2026"),
|
| 57 |
+
"db": int(os.getenv("REDIS_DB", "0")),
|
| 58 |
+
},
|
| 59 |
+
"postgres": {
|
| 60 |
+
"host": os.getenv("POSTGRES_HOST", "rmi-postgres"),
|
| 61 |
+
"port": int(os.getenv("POSTGRES_PORT", "5432")),
|
| 62 |
+
"user": os.getenv("POSTGRES_USER", "rmi"),
|
| 63 |
+
"password": os.getenv("POSTGRES_PASSWORD", ""),
|
| 64 |
+
"database": os.getenv("POSTGRES_DB", "rmi"),
|
| 65 |
+
},
|
| 66 |
+
"neo4j": {
|
| 67 |
+
"uri": os.getenv("NEO4J_URI", "bolt://rmi-neo4j:7687"),
|
| 68 |
+
"user": os.getenv("NEO4J_USER", "neo4j"),
|
| 69 |
+
"password": os.getenv("NEO4J_PASSWORD", ""),
|
| 70 |
+
},
|
| 71 |
+
"qdrant": {
|
| 72 |
+
"url": os.getenv("QDRANT_URL", "http://rmi-qdrant:6333"),
|
| 73 |
+
"api_key": os.getenv("QDRANT_API_KEY", ""),
|
| 74 |
+
},
|
| 75 |
+
"minio": {
|
| 76 |
+
"endpoint": os.getenv("MINIO_ENDPOINT", "rmi-minio:9000"),
|
| 77 |
+
"access_key": os.getenv("MINIO_ACCESS_KEY", ""),
|
| 78 |
+
"secret_key": os.getenv("MINIO_SECRET_KEY", ""),
|
| 79 |
+
},
|
| 80 |
+
}
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
class StoreHealth:
|
| 84 |
+
"""Tracks which stores are reachable. Updated on each probe."""
|
| 85 |
+
|
| 86 |
+
def __init__(self) -> None:
|
| 87 |
+
self.redis: bool = False
|
| 88 |
+
self.postgres: bool = False
|
| 89 |
+
self.neo4j: bool = False
|
| 90 |
+
self.qdrant: bool = False
|
| 91 |
+
self.minio: bool = False
|
| 92 |
+
self.last_checked: float = 0.0
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
class CatalogService:
|
| 96 |
+
"""Unified read/write API for RMI data.
|
| 97 |
+
|
| 98 |
+
Use `get_catalog()` to get the singleton instance. All methods are
|
| 99 |
+
async and graceful-degrade β if a store is unreachable, the method
|
| 100 |
+
returns None or an empty result with a logged warning.
|
| 101 |
+
"""
|
| 102 |
+
|
| 103 |
+
def __init__(self, config: dict[str, Any] | None = None) -> None:
|
| 104 |
+
self.config = config or DEFAULT_CONFIG
|
| 105 |
+
self._health = StoreHealth()
|
| 106 |
+
self._redis = None
|
| 107 |
+
self._pg_pool: Any = None
|
| 108 |
+
self._neo_driver: Any = None
|
| 109 |
+
self._qdrant: Any = None # httpx.AsyncClient
|
| 110 |
+
self._init_lock = asyncio.Lock()
|
| 111 |
+
|
| 112 |
+
# ββ Store init (lazy) ββββββββββββββββββββββββββββββββββββββββββββ
|
| 113 |
+
async def _init_stores(self) -> None:
|
| 114 |
+
async with self._init_lock:
|
| 115 |
+
if self._redis is not None:
|
| 116 |
+
return # already init
|
| 117 |
+
# Redis (always try first β used for everything)
|
| 118 |
+
try:
|
| 119 |
+
import redis.asyncio as aioredis
|
| 120 |
+
|
| 121 |
+
cfg = self.config["redis"]
|
| 122 |
+
self._redis = aioredis.Redis(
|
| 123 |
+
host=cfg["host"],
|
| 124 |
+
port=cfg["port"],
|
| 125 |
+
password=cfg["password"],
|
| 126 |
+
db=cfg["db"],
|
| 127 |
+
decode_responses=True,
|
| 128 |
+
)
|
| 129 |
+
await self._redis.ping()
|
| 130 |
+
self._health.redis = True
|
| 131 |
+
log.info("catalog_redis_ok")
|
| 132 |
+
except Exception as e:
|
| 133 |
+
log.warning("catalog_redis_fail: %s", e)
|
| 134 |
+
self._health.redis = False
|
| 135 |
+
|
| 136 |
+
# Postgres
|
| 137 |
+
try:
|
| 138 |
+
import asyncpg
|
| 139 |
+
|
| 140 |
+
cfg = self.config["postgres"]
|
| 141 |
+
if cfg["password"]:
|
| 142 |
+
self._pg_pool = await asyncpg.create_pool(
|
| 143 |
+
host=cfg["host"],
|
| 144 |
+
port=cfg["port"],
|
| 145 |
+
user=cfg["user"],
|
| 146 |
+
password=cfg["password"],
|
| 147 |
+
database=cfg["database"],
|
| 148 |
+
min_size=1,
|
| 149 |
+
max_size=8,
|
| 150 |
+
command_timeout=10,
|
| 151 |
+
)
|
| 152 |
+
self._health.postgres = True
|
| 153 |
+
log.info("catalog_postgres_ok")
|
| 154 |
+
except Exception as e:
|
| 155 |
+
log.warning("catalog_postgres_fail: %s", e)
|
| 156 |
+
self._health.postgres = False
|
| 157 |
+
|
| 158 |
+
# Neo4j
|
| 159 |
+
try:
|
| 160 |
+
from neo4j import GraphDatabase
|
| 161 |
+
|
| 162 |
+
cfg = self.config["neo4j"]
|
| 163 |
+
if cfg["password"]:
|
| 164 |
+
self._neo_driver = GraphDatabase.driver(
|
| 165 |
+
cfg["uri"], auth=(cfg["user"], cfg["password"])
|
| 166 |
+
)
|
| 167 |
+
with self._neo_driver.session() as s:
|
| 168 |
+
s.run("RETURN 1").consume()
|
| 169 |
+
self._health.neo4j = True
|
| 170 |
+
log.info("catalog_neo4j_ok")
|
| 171 |
+
except Exception as e:
|
| 172 |
+
log.warning("catalog_neo4j_fail: %s", e)
|
| 173 |
+
self._health.neo4j = False
|
| 174 |
+
|
| 175 |
+
# Qdrant (HTTP)
|
| 176 |
+
try:
|
| 177 |
+
cfg = self.config["qdrant"]
|
| 178 |
+
headers = {"api-key": cfg["api_key"]} if cfg["api_key"] else {}
|
| 179 |
+
self._qdrant = httpx.AsyncClient(
|
| 180 |
+
base_url=cfg["url"], headers=headers, timeout=5.0
|
| 181 |
+
)
|
| 182 |
+
r = await self._qdrant.get("/collections")
|
| 183 |
+
if r.status_code == 200:
|
| 184 |
+
self._health.qdrant = True
|
| 185 |
+
log.info("catalog_qdrant_ok")
|
| 186 |
+
else:
|
| 187 |
+
log.warning("catalog_qdrant_http_%d", r.status_code)
|
| 188 |
+
except Exception as e:
|
| 189 |
+
log.warning("catalog_qdrant_fail: %s", e)
|
| 190 |
+
self._health.qdrant = False
|
| 191 |
+
|
| 192 |
+
self._health.last_checked = time.time()
|
| 193 |
+
|
| 194 |
+
# ββ Health probe ββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 195 |
+
async def probe_stores(self) -> dict[str, bool]:
|
| 196 |
+
await self._init_stores()
|
| 197 |
+
return {
|
| 198 |
+
"redis": self._health.redis,
|
| 199 |
+
"postgres": self._health.postgres,
|
| 200 |
+
"neo4j": self._health.neo4j,
|
| 201 |
+
"qdrant": self._health.qdrant,
|
| 202 |
+
"minio": self._health.minio,
|
| 203 |
+
}
|
| 204 |
+
|
| 205 |
+
# ββ Tokens (Postgres primary + Redis cache) βββββββββββββββββββββ
|
| 206 |
+
async def get_token(self, chain: Chain, address: str) -> Token | None:
|
| 207 |
+
await self._init_stores()
|
| 208 |
+
cache_key = f"catalog:token:{chain.value}:{address}"
|
| 209 |
+
if self._health.redis:
|
| 210 |
+
try:
|
| 211 |
+
cached = await self._redis.get(cache_key)
|
| 212 |
+
if cached:
|
| 213 |
+
return Token.model_validate_json(cached)
|
| 214 |
+
except Exception as e:
|
| 215 |
+
log.debug("token_cache_get_fail: %s", e)
|
| 216 |
+
if not self._health.postgres:
|
| 217 |
+
return None
|
| 218 |
+
try:
|
| 219 |
+
async with self._pg_pool.acquire() as conn:
|
| 220 |
+
row = await conn.fetchrow(
|
| 221 |
+
"SELECT * FROM tokens WHERE chain=$1 AND address=$2",
|
| 222 |
+
chain.value, address,
|
| 223 |
+
)
|
| 224 |
+
if not row:
|
| 225 |
+
return None
|
| 226 |
+
token = Token(**dict(row))
|
| 227 |
+
if self._health.redis:
|
| 228 |
+
try:
|
| 229 |
+
await self._redis.setex(cache_key, 3600, token.model_dump_json())
|
| 230 |
+
except Exception as e:
|
| 231 |
+
log.debug("token_cache_set_fail: %s", e)
|
| 232 |
+
return token
|
| 233 |
+
except Exception as e:
|
| 234 |
+
log.warning("token_get_fail: %s", e)
|
| 235 |
+
return None
|
| 236 |
+
|
| 237 |
+
async def save_token(self, token: Token) -> bool:
|
| 238 |
+
await self._init_stores()
|
| 239 |
+
if not self._health.postgres:
|
| 240 |
+
log.warning("token_save_no_postgres")
|
| 241 |
+
return False
|
| 242 |
+
try:
|
| 243 |
+
async with self._pg_pool.acquire() as conn:
|
| 244 |
+
await conn.execute(
|
| 245 |
+
"""
|
| 246 |
+
INSERT INTO tokens (
|
| 247 |
+
token_id, chain, address, symbol, name, decimals,
|
| 248 |
+
deployer_wallet_id, deployed_at, initial_supply,
|
| 249 |
+
current_supply, is_honeypot, is_mintable, is_proxy,
|
| 250 |
+
tax_buy_bps, tax_sell_bps, risk_tier, risk_score,
|
| 251 |
+
risk_factors, rag_embedding_id
|
| 252 |
+
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,
|
| 253 |
+
$14,$15,$16,$17,$18,$19)
|
| 254 |
+
ON CONFLICT (token_id) DO UPDATE SET
|
| 255 |
+
symbol=EXCLUDED.symbol,
|
| 256 |
+
name=EXCLUDED.name,
|
| 257 |
+
current_supply=EXCLUDED.current_supply,
|
| 258 |
+
is_honeypot=EXCLUDED.is_honeypot,
|
| 259 |
+
is_mintable=EXCLUDED.is_mintable,
|
| 260 |
+
risk_tier=EXCLUDED.risk_tier,
|
| 261 |
+
risk_score=EXCLUDED.risk_score,
|
| 262 |
+
risk_factors=EXCLUDED.risk_factors
|
| 263 |
+
""",
|
| 264 |
+
token.token_id, token.chain.value, token.address,
|
| 265 |
+
token.symbol, token.name, token.decimals,
|
| 266 |
+
token.deployer_wallet_id, token.deployed_at,
|
| 267 |
+
token.initial_supply, token.current_supply,
|
| 268 |
+
token.is_honeypot, token.is_mintable, token.is_proxy,
|
| 269 |
+
token.tax_buy_bps, token.tax_sell_bps,
|
| 270 |
+
token.risk_tier.value if token.risk_tier else None,
|
| 271 |
+
token.risk_score, token.risk_factors, token.rag_embedding_id,
|
| 272 |
+
)
|
| 273 |
+
# Invalidate cache
|
| 274 |
+
if self._health.redis:
|
| 275 |
+
try:
|
| 276 |
+
await self._redis.delete(
|
| 277 |
+
f"catalog:token:{token.chain.value}:{token.address}"
|
| 278 |
+
)
|
| 279 |
+
except Exception:
|
| 280 |
+
pass
|
| 281 |
+
return True
|
| 282 |
+
except Exception as e:
|
| 283 |
+
log.warning("token_save_fail: %s", e)
|
| 284 |
+
return False
|
| 285 |
+
|
| 286 |
+
# ββ Wallets (Neo4j primary) ββββββββββββββββββββββββββββββββββββββ
|
| 287 |
+
async def get_wallet(self, chain: Chain, address: str) -> Wallet | None:
|
| 288 |
+
await self._init_stores()
|
| 289 |
+
if not self._health.neo4j:
|
| 290 |
+
return None
|
| 291 |
+
wallet_id = f"{chain.value}:{address}"
|
| 292 |
+
try:
|
| 293 |
+
with self._neo_driver.session() as s:
|
| 294 |
+
result = s.run(
|
| 295 |
+
"MATCH (w:Wallet {wallet_id: $id}) RETURN w",
|
| 296 |
+
id=wallet_id,
|
| 297 |
+
).single()
|
| 298 |
+
if not result:
|
| 299 |
+
return None
|
| 300 |
+
node = dict(result["w"])
|
| 301 |
+
# Normalize datetime strings
|
| 302 |
+
for k in ("first_seen", "last_seen"):
|
| 303 |
+
if isinstance(node.get(k), str):
|
| 304 |
+
from datetime import datetime
|
| 305 |
+
try:
|
| 306 |
+
node[k] = datetime.fromisoformat(node[k].replace("Z", "+00:00"))
|
| 307 |
+
except Exception:
|
| 308 |
+
pass
|
| 309 |
+
return Wallet(**node)
|
| 310 |
+
except Exception as e:
|
| 311 |
+
log.warning("wallet_get_fail: %s", e)
|
| 312 |
+
return None
|
| 313 |
+
|
| 314 |
+
async def save_wallet(self, wallet: Wallet) -> bool:
|
| 315 |
+
await self._init_stores()
|
| 316 |
+
if not self._health.neo4j:
|
| 317 |
+
return False
|
| 318 |
+
try:
|
| 319 |
+
with self._neo_driver.session() as s:
|
| 320 |
+
s.run(
|
| 321 |
+
"""
|
| 322 |
+
MERGE (w:Wallet {wallet_id: $wallet_id})
|
| 323 |
+
SET w.chain=$chain, w.address=$address,
|
| 324 |
+
w.first_seen=$first_seen, w.last_seen=$last_seen,
|
| 325 |
+
w.tx_count=$tx_count, w.total_volume_usd=$total_volume_usd,
|
| 326 |
+
w.is_deployer=$is_deployer,
|
| 327 |
+
w.is_known_exchange=$is_known_exchange,
|
| 328 |
+
w.is_suspicious=$is_suspicious,
|
| 329 |
+
w.reputation_score=$reputation_score
|
| 330 |
+
""",
|
| 331 |
+
wallet_id=wallet.wallet_id,
|
| 332 |
+
chain=wallet.chain.value,
|
| 333 |
+
address=wallet.address,
|
| 334 |
+
first_seen=wallet.first_seen.isoformat(),
|
| 335 |
+
last_seen=wallet.last_seen.isoformat(),
|
| 336 |
+
tx_count=wallet.tx_count,
|
| 337 |
+
total_volume_usd=wallet.total_volume_usd,
|
| 338 |
+
is_deployer=wallet.is_deployer,
|
| 339 |
+
is_known_exchange=wallet.is_known_exchange,
|
| 340 |
+
is_suspicious=wallet.is_suspicious,
|
| 341 |
+
reputation_score=wallet.reputation_score,
|
| 342 |
+
)
|
| 343 |
+
return True
|
| 344 |
+
except Exception as e:
|
| 345 |
+
log.warning("wallet_save_fail: %s", e)
|
| 346 |
+
return False
|
| 347 |
+
|
| 348 |
+
# ββ Recipe 1: find tokens by deployer history βββββββββββββββββββ
|
| 349 |
+
async def find_tokens_by_deployer_history(
|
| 350 |
+
self, min_rug_count: int = 1, chain: Chain | None = None, limit: int = 50
|
| 351 |
+
) -> list[Token]:
|
| 352 |
+
"""Find tokens deployed by wallets that have rug-pulled before.
|
| 353 |
+
|
| 354 |
+
Cross-store: Neo4j for the wallet filter, Postgres for the tokens.
|
| 355 |
+
"""
|
| 356 |
+
await self._init_stores()
|
| 357 |
+
if not (self._health.neo4j and self._health.postgres):
|
| 358 |
+
log.warning("recipe1_stores_unavailable")
|
| 359 |
+
return []
|
| 360 |
+
try:
|
| 361 |
+
# Step 1: Neo4j β find wallets with rug_count >= min_rug_count
|
| 362 |
+
with self._neo_driver.session() as s:
|
| 363 |
+
wallets = [
|
| 364 |
+
r["w.wallet_id"]
|
| 365 |
+
for r in s.run(
|
| 366 |
+
"MATCH (w:Deployer) WHERE w.rug_count >= $min "
|
| 367 |
+
"RETURN w.wallet_id LIMIT 200",
|
| 368 |
+
min=min_rug_count,
|
| 369 |
+
)
|
| 370 |
+
]
|
| 371 |
+
if not wallets:
|
| 372 |
+
return []
|
| 373 |
+
# Step 2: Postgres β find tokens deployed by those wallets
|
| 374 |
+
async with self._pg_pool.acquire() as conn:
|
| 375 |
+
query = (
|
| 376 |
+
"SELECT * FROM tokens WHERE deployer_wallet_id = ANY($1::text[]) "
|
| 377 |
+
)
|
| 378 |
+
params: list[Any] = [wallets]
|
| 379 |
+
if chain:
|
| 380 |
+
query += "AND chain=$2 "
|
| 381 |
+
params.append(chain.value)
|
| 382 |
+
query += "ORDER BY deployed_at DESC LIMIT $3"
|
| 383 |
+
params.append(limit)
|
| 384 |
+
rows = await conn.fetch(query, *params)
|
| 385 |
+
return [Token(**dict(r)) for r in rows]
|
| 386 |
+
except Exception as e:
|
| 387 |
+
log.warning("recipe1_fail: %s", e)
|
| 388 |
+
return []
|
| 389 |
+
|
| 390 |
+
# ββ Recipe 3: get_token_risk (3-store composition) ββββββββββββββ
|
| 391 |
+
async def get_token_risk(
|
| 392 |
+
self, chain: Chain, address: str
|
| 393 |
+
) -> dict[str, Any]:
|
| 394 |
+
"""Real-time risk score β composes Redis cache + Postgres + Neo4j.
|
| 395 |
+
|
| 396 |
+
Returns dict (not Token) because it's a cross-store projection.
|
| 397 |
+
Cache TTL 60s.
|
| 398 |
+
"""
|
| 399 |
+
await self._init_stores()
|
| 400 |
+
cache_key = f"catalog:risk:{chain.value}:{address}"
|
| 401 |
+
if self._health.redis:
|
| 402 |
+
try:
|
| 403 |
+
cached = await self._redis.get(cache_key)
|
| 404 |
+
if cached:
|
| 405 |
+
return json.loads(cached)
|
| 406 |
+
except Exception:
|
| 407 |
+
pass
|
| 408 |
+
|
| 409 |
+
token = await self.get_token(chain, address)
|
| 410 |
+
deployer_reputation: int | None = None
|
| 411 |
+
if token and token.deployer_wallet_id and self._health.neo4j:
|
| 412 |
+
try:
|
| 413 |
+
with self._neo_driver.session() as s:
|
| 414 |
+
r = s.run(
|
| 415 |
+
"MATCH (d:Deployer {wallet_id: $id}) "
|
| 416 |
+
"RETURN d.reputation_score AS rep, d.rug_count AS rugs",
|
| 417 |
+
id=token.deployer_wallet_id,
|
| 418 |
+
).single()
|
| 419 |
+
if r:
|
| 420 |
+
deployer_reputation = r["rep"]
|
| 421 |
+
except Exception as e:
|
| 422 |
+
log.debug("risk_neo4j_fail: %s", e)
|
| 423 |
+
|
| 424 |
+
token_score = token.risk_score if token and token.risk_score is not None else 50
|
| 425 |
+
if deployer_reputation is not None:
|
| 426 |
+
score = int(0.6 * token_score + 0.4 * (100 - deployer_reputation))
|
| 427 |
+
else:
|
| 428 |
+
score = token_score
|
| 429 |
+
score = max(0, min(100, score))
|
| 430 |
+
if score < 25:
|
| 431 |
+
tier = "low"
|
| 432 |
+
elif score < 50:
|
| 433 |
+
tier = "medium"
|
| 434 |
+
elif score < 75:
|
| 435 |
+
tier = "high"
|
| 436 |
+
else:
|
| 437 |
+
tier = "critical"
|
| 438 |
+
result: dict[str, Any] = {
|
| 439 |
+
"score": score,
|
| 440 |
+
"tier": tier,
|
| 441 |
+
"factors": token.risk_factors if token else [],
|
| 442 |
+
"token": token.model_dump() if token else None,
|
| 443 |
+
"deployer_reputation": deployer_reputation,
|
| 444 |
+
"fetched_at": utcnow().isoformat(),
|
| 445 |
+
}
|
| 446 |
+
if self._health.redis:
|
| 447 |
+
try:
|
| 448 |
+
await self._redis.setex(cache_key, 60, json.dumps(result, default=str))
|
| 449 |
+
except Exception:
|
| 450 |
+
pass
|
| 451 |
+
return result
|
| 452 |
+
|
| 453 |
+
# ββ Recipe 5: resolve_entity (Neo4j Cypher) βββββββββββββββββββββ
|
| 454 |
+
async def resolve_entity(
|
| 455 |
+
self, wallet_id: str, max_chains: int = 5
|
| 456 |
+
) -> dict[str, Any]:
|
| 457 |
+
"""Cross-chain entity resolution via Neo4j.
|
| 458 |
+
|
| 459 |
+
Uses the SAME_AS / FUNDED_BY_SAME / CLONE_OF / BEHAVIORAL_MATCH
|
| 460 |
+
edges defined in v4.0 Β§T32 with path-weighted confidence.
|
| 461 |
+
"""
|
| 462 |
+
await self._init_stores()
|
| 463 |
+
if not self._health.neo4j:
|
| 464 |
+
return {"entity_id": None, "wallets": [], "note": "neo4j unavailable"}
|
| 465 |
+
try:
|
| 466 |
+
with self._neo_driver.session() as s:
|
| 467 |
+
result = s.run(
|
| 468 |
+
"""
|
| 469 |
+
MATCH (start:Wallet {wallet_id: $wallet_id})
|
| 470 |
+
OPTIONAL MATCH path = (start)-[:SAME_AS|FUNDED_BY_SAME|CLONE_OF|BEHAVIORAL_MATCH*1..3]-(other:Wallet)
|
| 471 |
+
WHERE start <> other
|
| 472 |
+
WITH start, other, path,
|
| 473 |
+
reduce(conf = 1.0, r IN relationships(path) | conf * r.confidence) AS path_conf
|
| 474 |
+
ORDER BY path_conf DESC
|
| 475 |
+
LIMIT $limit
|
| 476 |
+
RETURN start.entity_id AS entity_id,
|
| 477 |
+
collect({
|
| 478 |
+
wallet_id: other.wallet_id,
|
| 479 |
+
chain: other.chain,
|
| 480 |
+
address: other.address,
|
| 481 |
+
confidence: path_conf
|
| 482 |
+
}) AS cross_chain_wallets
|
| 483 |
+
""",
|
| 484 |
+
wallet_id=wallet_id, limit=max_chains,
|
| 485 |
+
).single()
|
| 486 |
+
if not result:
|
| 487 |
+
return {"entity_id": None, "wallets": [], "note": "no edges"}
|
| 488 |
+
wallets = [w for w in (result["cross_chain_wallets"] or []) if w.get("wallet_id")]
|
| 489 |
+
return {
|
| 490 |
+
"entity_id": result["entity_id"],
|
| 491 |
+
"wallets": wallets,
|
| 492 |
+
"note": f"{len(wallets)} cross-chain matches",
|
| 493 |
+
}
|
| 494 |
+
except Exception as e:
|
| 495 |
+
log.warning("resolve_entity_fail: %s", e)
|
| 496 |
+
return {"entity_id": None, "wallets": [], "error": str(e)}
|
| 497 |
+
|
| 498 |
+
# ββ RAG bridge: wire app/rag/engine.py into the catalog βββββββββ
|
| 499 |
+
async def rag_search(
|
| 500 |
+
self, query: str, collection: str = "scam_intel", top_k: int = 5
|
| 501 |
+
) -> list[dict]:
|
| 502 |
+
"""Search the RAG system. Returns raw hits (catalog-agnostic).
|
| 503 |
+
|
| 504 |
+
Use this when you want RAG results as raw search hits.
|
| 505 |
+
Use get_token_risk / etc. when you want catalog-typed results.
|
| 506 |
+
"""
|
| 507 |
+
try:
|
| 508 |
+
return await rag_three_pillar_search(
|
| 509 |
+
query=query, collection=collection, top_k=top_k
|
| 510 |
+
)
|
| 511 |
+
except Exception as e:
|
| 512 |
+
log.warning("rag_search_fail: %s", e)
|
| 513 |
+
return []
|
| 514 |
+
|
| 515 |
+
async def rag_ingest(
|
| 516 |
+
self,
|
| 517 |
+
content: str,
|
| 518 |
+
collection: str = "scam_intel",
|
| 519 |
+
doc_id: str | None = None,
|
| 520 |
+
metadata: dict | None = None,
|
| 521 |
+
) -> dict:
|
| 522 |
+
"""Ingest a fact into RAG and (if it's about a token) link to Token.rag_embedding_id.
|
| 523 |
+
|
| 524 |
+
Returns the RAG ingest result plus the Qdrant point_id for cross-store ref.
|
| 525 |
+
"""
|
| 526 |
+
from uuid import uuid4
|
| 527 |
+
|
| 528 |
+
doc_id = doc_id or f"finding:{uuid4().hex[:12]}"
|
| 529 |
+
try:
|
| 530 |
+
r = await rag_ingest_document(
|
| 531 |
+
collection=collection,
|
| 532 |
+
doc_id=doc_id,
|
| 533 |
+
content=content,
|
| 534 |
+
metadata=metadata or {},
|
| 535 |
+
)
|
| 536 |
+
r["qdrant_point_id"] = doc_id # Engine returns the same
|
| 537 |
+
return r
|
| 538 |
+
except Exception as e:
|
| 539 |
+
log.warning("rag_ingest_fail: %s", e)
|
| 540 |
+
return {"status": "failed", "error": str(e)}
|
| 541 |
+
|
| 542 |
+
async def attach_rag_to_token(
|
| 543 |
+
self, chain: Chain, address: str, qdrant_point_id: str
|
| 544 |
+
) -> bool:
|
| 545 |
+
"""Link an existing Qdrant point to a Token row as rag_embedding_id."""
|
| 546 |
+
token = await self.get_token(chain, address)
|
| 547 |
+
if not token:
|
| 548 |
+
return False
|
| 549 |
+
token.rag_embedding_id = qdrant_point_id
|
| 550 |
+
return await self.save_token(token)
|
| 551 |
+
|
| 552 |
+
# ββ Stats / introspection ββββββββββββββββββββββββββββββββββββββββ
|
| 553 |
+
async def stats(self) -> dict[str, Any]:
|
| 554 |
+
await self._init_stores()
|
| 555 |
+
out: dict[str, Any] = {
|
| 556 |
+
"stores": await self.probe_stores(),
|
| 557 |
+
"collections": COLLECTIONS,
|
| 558 |
+
}
|
| 559 |
+
if self._health.postgres:
|
| 560 |
+
try:
|
| 561 |
+
async with self._pg_pool.acquire() as conn:
|
| 562 |
+
out["tokens"] = await conn.fetchval("SELECT COUNT(*) FROM tokens") or 0
|
| 563 |
+
out["alerts"] = await conn.fetchval("SELECT COUNT(*) FROM alerts") or 0
|
| 564 |
+
except Exception as e:
|
| 565 |
+
out["postgres_error"] = str(e)
|
| 566 |
+
if self._health.neo4j:
|
| 567 |
+
try:
|
| 568 |
+
with self._neo_driver.session() as s:
|
| 569 |
+
out["wallets"] = s.run("MATCH (w:Wallet) RETURN COUNT(w) AS c").single()["c"] or 0
|
| 570 |
+
out["deployers"] = s.run("MATCH (d:Deployer) RETURN COUNT(d) AS c").single()["c"] or 0
|
| 571 |
+
except Exception as e:
|
| 572 |
+
out["neo4j_error"] = str(e)
|
| 573 |
+
if self._health.qdrant:
|
| 574 |
+
try:
|
| 575 |
+
cols = await self._qdrant.get("/collections")
|
| 576 |
+
if cols.status_code == 200:
|
| 577 |
+
out["qdrant_collections"] = [
|
| 578 |
+
c["name"] for c in cols.json().get("result", {}).get("collections", [])
|
| 579 |
+
]
|
| 580 |
+
except Exception as e:
|
| 581 |
+
out["qdrant_error"] = str(e)
|
| 582 |
+
return out
|
| 583 |
+
|
| 584 |
+
|
| 585 |
+
# ββ Singleton accessor ββββββββββββββββββββββββββββββββββββββββββββββ
|
| 586 |
+
_catalog: CatalogService | None = None
|
| 587 |
+
|
| 588 |
+
|
| 589 |
+
def get_catalog() -> CatalogService:
|
| 590 |
+
"""Get the global CatalogService. Lazy-init on first call."""
|
| 591 |
+
global _catalog
|
| 592 |
+
if _catalog is None:
|
| 593 |
+
_catalog = CatalogService()
|
| 594 |
+
return _catalog
|
backend/main.py
CHANGED
|
@@ -215,6 +215,7 @@ def _try_mount_v1_routers() -> int:
|
|
| 215 |
"app.api.v1.rag.search",
|
| 216 |
"app.api.v1.x402.payments",
|
| 217 |
"app.api.v1.admin.alerts_webhook",
|
|
|
|
| 218 |
]
|
| 219 |
|
| 220 |
for module_path in v1_modules:
|
|
|
|
| 215 |
"app.api.v1.rag.search",
|
| 216 |
"app.api.v1.x402.payments",
|
| 217 |
"app.api.v1.admin.alerts_webhook",
|
| 218 |
+
"app.api.v1.catalog",
|
| 219 |
]
|
| 220 |
|
| 221 |
for module_path in v1_modules:
|