Hermes commited on
Commit
680b3ba
Β·
1 Parent(s): beb7310

feat(catalog/t27): unified data catalog + RAG bridge + reputation

Browse files
backend/app/api/v1/__init__.py CHANGED
@@ -61,6 +61,10 @@ from app.api.v1.admin.alerts_webhook import router as admin_alerts_webhook_route
61
 
62
  api_v1_router.append(admin_alerts_webhook_router)
63
 
 
 
 
 
64
 
65
  def build_v1_router() -> APIRouter:
66
  """Construct the v1 aggregator with all migrated routes mounted."""
 
61
 
62
  api_v1_router.append(admin_alerts_webhook_router)
63
 
64
+ from app.api.v1.catalog import router as catalog_router # noqa: E402
65
+
66
+ api_v1_router.append(catalog_router)
67
+
68
 
69
  def build_v1_router() -> APIRouter:
70
  """Construct the v1 aggregator with all migrated routes mounted."""
backend/app/api/v1/catalog/__init__.py ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ """Catalog v1 routes β€” thin HTTP layer."""
2
+ from .router import router
3
+
4
+ __all__ = ["router"]
backend/app/api/v1/catalog/__pycache__/__init__.cpython-311.pyc ADDED
Binary file (274 Bytes). View file
 
backend/app/api/v1/catalog/__pycache__/router.cpython-311.pyc ADDED
Binary file (10 kB). View file
 
backend/app/api/v1/catalog/router.py ADDED
@@ -0,0 +1,155 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """T27B HTTP routes β€” CatalogService endpoints.
2
+
3
+ Per v4.0 Β§T27. The thin HTTP layer over app.catalog.service.CatalogService.
4
+ """
5
+ from __future__ import annotations
6
+
7
+ from typing import Any
8
+
9
+ from fastapi import APIRouter, HTTPException
10
+ from pydantic import BaseModel, Field
11
+
12
+ from app.catalog.models import Chain
13
+ from app.catalog.service import get_catalog
14
+
15
+ router = APIRouter(prefix="/api/v1/catalog", tags=["catalog"])
16
+
17
+
18
+ # ── Request models ───────────────────────────────────────────────
19
+ class RagIngestRequest(BaseModel):
20
+ content: str = Field(..., min_length=1)
21
+ collection: str = "scam_intel"
22
+ doc_id: str | None = None
23
+ metadata: dict[str, Any] = Field(default_factory=dict)
24
+
25
+
26
+ class RagSearchRequest(BaseModel):
27
+ query: str = Field(..., min_length=1)
28
+ collection: str = "scam_intel"
29
+ top_k: int = Field(default=5, ge=1, le=50)
30
+
31
+
32
+ class ResolveEntityRequest(BaseModel):
33
+ wallet_id: str
34
+ max_chains: int = Field(default=5, ge=1, le=20)
35
+
36
+
37
+ class FindRiskyTokensRequest(BaseModel):
38
+ min_rug_count: int = Field(default=1, ge=1)
39
+ chain: str | None = None
40
+ limit: int = Field(default=50, ge=1, le=200)
41
+
42
+
43
+ class AttachRagRequest(BaseModel):
44
+ chain: str
45
+ address: str
46
+ qdrant_point_id: str
47
+
48
+
49
+ # ── Health / introspection ───────────────────────────────────────
50
+ @router.get("/stats")
51
+ async def stats() -> dict:
52
+ """Catalog stats: which stores are reachable + entity counts."""
53
+ return await get_catalog().stats()
54
+
55
+
56
+ @router.get("/probe")
57
+ async def probe() -> dict:
58
+ """Probe which stores are reachable from this container."""
59
+ return await get_catalog().probe_stores()
60
+
61
+
62
+ # ── Token endpoints ─────────────────────────────────────────────
63
+ @router.get("/tokens/{chain}/{address}")
64
+ async def get_token(chain: str, address: str) -> dict:
65
+ """Get a token by chain+address. Returns full Token model + provenance."""
66
+ try:
67
+ c = Chain(chain)
68
+ except ValueError:
69
+ raise HTTPException(400, f"unknown chain: {chain}")
70
+ tok = await get_catalog().get_token(c, address)
71
+ if not tok:
72
+ raise HTTPException(404, "token not found")
73
+ return tok.model_dump(mode="json")
74
+
75
+
76
+ @router.get("/tokens/{chain}/{address}/risk")
77
+ async def get_token_risk(chain: str, address: str) -> dict:
78
+ """Recipe 3 β€” Real-time risk score. Composes Redis + Postgres + Neo4j."""
79
+ try:
80
+ c = Chain(chain)
81
+ except ValueError:
82
+ raise HTTPException(400, f"unknown chain: {chain}")
83
+ return await get_catalog().get_token_risk(c, address)
84
+
85
+
86
+ @router.post("/tokens/risky-by-deployer")
87
+ async def risky_tokens(req: FindRiskyTokensRequest) -> dict:
88
+ """Recipe 1 β€” Find tokens deployed by wallets with rug history."""
89
+ chain_enum = None
90
+ if req.chain:
91
+ try:
92
+ chain_enum = Chain(req.chain)
93
+ except ValueError:
94
+ raise HTTPException(400, f"unknown chain: {req.chain}")
95
+ tokens = await get_catalog().find_tokens_by_deployer_history(
96
+ min_rug_count=req.min_rug_count, chain=chain_enum, limit=req.limit
97
+ )
98
+ return {
99
+ "count": len(tokens),
100
+ "tokens": [t.model_dump(mode="json") for t in tokens],
101
+ }
102
+
103
+
104
+ # ── Wallet endpoints ─────────────────────────────────────────────
105
+ @router.get("/wallets/{chain}/{address}")
106
+ async def get_wallet(chain: str, address: str) -> dict:
107
+ try:
108
+ c = Chain(chain)
109
+ except ValueError:
110
+ raise HTTPException(400, f"unknown chain: {chain}")
111
+ w = await get_catalog().get_wallet(c, address)
112
+ if not w:
113
+ raise HTTPException(404, "wallet not found")
114
+ return w.model_dump(mode="json")
115
+
116
+
117
+ # ── Entity resolution (Recipe 5) ────────────────────────────────
118
+ @router.post("/entities/resolve")
119
+ async def resolve_entity(req: ResolveEntityRequest) -> dict:
120
+ """Cross-chain entity resolution via Neo4j Cypher."""
121
+ return await get_catalog().resolve_entity(req.wallet_id, req.max_chains)
122
+
123
+
124
+ # ── RAG bridge endpoints ────────────────────────────────────────
125
+ @router.post("/rag/search")
126
+ async def rag_search(req: RagSearchRequest) -> dict:
127
+ """Search the RAG system. Returns ranked hits with RRF scores."""
128
+ hits = await get_catalog().rag_search(
129
+ query=req.query, collection=req.collection, top_k=req.top_k
130
+ )
131
+ return {"count": len(hits), "hits": hits}
132
+
133
+
134
+ @router.post("/rag/ingest")
135
+ async def rag_ingest(req: RagIngestRequest) -> dict:
136
+ """Ingest content into RAG. Returns qdrant_point_id for cross-store linking."""
137
+ return await get_catalog().rag_ingest(
138
+ content=req.content,
139
+ collection=req.collection,
140
+ doc_id=req.doc_id,
141
+ metadata=req.metadata,
142
+ )
143
+
144
+
145
+ @router.post("/tokens/{chain}/{address}/attach-rag")
146
+ async def attach_rag(chain: str, address: str, req: AttachRagRequest) -> dict:
147
+ """Link an existing RAG embedding (Qdrant point) to a Token row."""
148
+ try:
149
+ c = Chain(chain)
150
+ except ValueError:
151
+ raise HTTPException(400, f"unknown chain: {chain}")
152
+ ok = await get_catalog().attach_rag_to_token(c, address, req.qdrant_point_id)
153
+ if not ok:
154
+ raise HTTPException(404, "token not found or update failed")
155
+ return {"ok": True, "chain": chain, "address": address, "rag_embedding_id": req.qdrant_point_id}
backend/app/catalog/__init__.py ADDED
@@ -0,0 +1,86 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Catalog domain β€” the unified read/write API for RMI.
2
+
3
+ T27 of the v4.0 guide. Every store read/write goes through CatalogService.
4
+ Domain facades call the catalog; they never touch stores directly.
5
+
6
+ Architecture (per v4.0 Β§T27):
7
+ app/catalog/models.py Pydantic v2 entity schemas
8
+ app/catalog/service.py CatalogService β€” fan-out reads, fan-in writes
9
+ app/catalog/reputation.py Deployer reputation scoring (T31)
10
+ app/catalog/rag_bridge.py Bridge to existing app/rag/ engine
11
+ app/catalog/llm_router.py LiteLLM proxy for AI analysis
12
+
13
+ Cross-store ref shape (per v4.0):
14
+ "chain:address" Wallet, Token, Contract IDs
15
+ UUID Entity, Alert, NewsItem, RAGFinding, Report IDs
16
+ Qdrant point_id RAGFinding.vector_id, rag_embedding_id on Token
17
+
18
+ Public API (re-exported):
19
+ CatalogService, get_catalog
20
+ Entity, Wallet, Deployer, Token, Alert, NewsItem, RAGFinding, ScanReport
21
+ DeployerReputation, RECIPES
22
+ """
23
+ from __future__ import annotations
24
+
25
+ from app.core import health as health_mod
26
+ from app.core.health import DomainHealth
27
+
28
+
29
+ async def _health_check() -> DomainHealth:
30
+ """Catalog health: which stores are reachable + how many entities."""
31
+ try:
32
+ from app.catalog.service import get_catalog
33
+
34
+ cat = get_catalog()
35
+ reach = await cat.probe_stores()
36
+ healthy = any(reach.values())
37
+ return DomainHealth(
38
+ name="catalog",
39
+ healthy=healthy,
40
+ details={
41
+ "stores_reachable": {k: v for k, v in reach.items()},
42
+ "primary": "redis+rag" if healthy else "none",
43
+ },
44
+ )
45
+ except Exception as e:
46
+ return DomainHealth(name="catalog", healthy=False, error=str(e))
47
+
48
+
49
+ health_mod.register_health_check("catalog", _health_check)
50
+
51
+
52
+ # Public API
53
+ from app.catalog.models import ( # noqa: E402
54
+ COLLECTIONS,
55
+ CHAIN_REGISTRY,
56
+ Chain,
57
+ Entity,
58
+ EntityLabel,
59
+ RiskTier,
60
+ Token,
61
+ Wallet,
62
+ Deployer,
63
+ Alert,
64
+ NewsItem,
65
+ RAGFinding,
66
+ ScanReport,
67
+ )
68
+ from app.catalog.service import CatalogService, get_catalog # noqa: E402
69
+
70
+ __all__ = [
71
+ "CatalogService",
72
+ "get_catalog",
73
+ "Entity",
74
+ "Wallet",
75
+ "Deployer",
76
+ "Token",
77
+ "Alert",
78
+ "NewsItem",
79
+ "RAGFinding",
80
+ "ScanReport",
81
+ "EntityLabel",
82
+ "Chain",
83
+ "RiskTier",
84
+ "COLLECTIONS",
85
+ "CHAIN_REGISTRY",
86
+ ]
backend/app/catalog/__pycache__/__init__.cpython-311.pyc ADDED
Binary file (3.29 kB). View file
 
backend/app/catalog/__pycache__/models.cpython-311.pyc ADDED
Binary file (16.4 kB). View file
 
backend/app/catalog/__pycache__/service.cpython-311.pyc ADDED
Binary file (34.6 kB). View file
 
backend/app/catalog/llm_router.py ADDED
@@ -0,0 +1,128 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """T27 LLM Router β€” sovereign-first LiteLLM proxy for catalog AI.
2
+
3
+ Per v4.0 Β§T28 (News analysis), Β§T29 (Report generation).
4
+
5
+ Self-hosted LiteLLM proxy at litellm.rugmunch.io routes to:
6
+ - DeepSeek-V3 (cost-effective analysis)
7
+ - Qwen (summaries)
8
+ - Local Llama-3 (free-tier fallback)
9
+
10
+ We never call OpenAI directly. If LiteLLM is unreachable, the catalog
11
+ operations that need LLM analysis return None/empty with a logged warning
12
+ rather than failing the whole request.
13
+ """
14
+ from __future__ import annotations
15
+
16
+ import logging
17
+ import os
18
+ from typing import Any
19
+
20
+ import httpx
21
+
22
+ log = logging.getLogger(__name__)
23
+
24
+
25
+ # ── Config ─────────────────────────────────────────────────────────
26
+ LITELLM_URL = os.getenv("LITELLM_URL", "http://litellm.rugmunch.io")
27
+ LITELLM_API_KEY = os.getenv("LITELLM_API_KEY", "")
28
+ DEFAULT_MODEL = os.getenv("LITELLM_DEFAULT_MODEL", "deepseek-v3")
29
+
30
+ NEWS_ANALYSIS_PROMPT = """You are an analyst at RugMunch Intelligence, a crypto
31
+ scam-detection platform. Analyze the following news item and produce a
32
+ structured Markdown summary.
33
+
34
+ NEWS ITEM:
35
+ - Title: {title}
36
+ - Source: {source}
37
+ - Published: {published_at}
38
+ - Body: {body_truncated_to_2000_chars}
39
+
40
+ Produce a summary with these sections (use Markdown headers):
41
+
42
+ ## Summary
43
+ 2-3 sentence plain-English summary.
44
+
45
+ ## Affected Tokens
46
+ List any tokens mentioned, with their chain and address if known.
47
+
48
+ ## Affected Wallets
49
+ List any wallets mentioned.
50
+
51
+ ## Sentiment
52
+ One of: bullish | bearish | neutral | risk-elevating | risk-reducing
53
+ 1-sentence justification.
54
+
55
+ ## RugMunch Action
56
+ What should our platform do in response? Options:
57
+ - (none)
58
+ - (flag mentioned tokens for re-scan)
59
+ - (alert subscribers)
60
+ - (update deployer reputation)
61
+ - (cross-chain entity resolution trigger)
62
+
63
+ Be concise. Do not speculate beyond what the article says.
64
+ """
65
+
66
+
67
+ class LLMRouter:
68
+ """Async client for the self-hosted LiteLLM proxy.
69
+
70
+ Falls back to None if the proxy is unreachable β€” catalog operations
71
+ that need LLM output will skip the AI analysis but still complete
72
+ the rest of the workflow.
73
+ """
74
+
75
+ def __init__(self, url: str | None = None, api_key: str | None = None) -> None:
76
+ self.url = (url or LITELLM_URL).rstrip("/")
77
+ self.api_key = api_key or LITELLM_API_KEY
78
+ self._client: httpx.AsyncClient | None = None
79
+
80
+ async def _get_client(self) -> httpx.AsyncClient:
81
+ if self._client is None:
82
+ headers = {"Content-Type": "application/json"}
83
+ if self.api_key:
84
+ headers["Authorization"] = f"Bearer {self.api_key}"
85
+ self._client = httpx.AsyncClient(
86
+ base_url=self.url, headers=headers, timeout=30.0
87
+ )
88
+ return self._client
89
+
90
+ async def chat(
91
+ self,
92
+ prompt: str,
93
+ model: str | None = None,
94
+ max_tokens: int = 800,
95
+ temperature: float = 0.3,
96
+ ) -> str | None:
97
+ """Send a chat completion. Returns text or None on failure."""
98
+ try:
99
+ client = await self._get_client()
100
+ r = await client.post(
101
+ "/chat/completions",
102
+ json={
103
+ "model": model or DEFAULT_MODEL,
104
+ "messages": [{"role": "user", "content": prompt}],
105
+ "max_tokens": max_tokens,
106
+ "temperature": temperature,
107
+ },
108
+ )
109
+ if r.status_code != 200:
110
+ log.warning("llm_http_%d: %s", r.status_code, r.text[:200])
111
+ return None
112
+ data = r.json()
113
+ return data["choices"][0]["message"]["content"]
114
+ except Exception as e:
115
+ log.warning("llm_chat_fail: %s", e)
116
+ return None
117
+
118
+ async def analyze_news(
119
+ self, news_item: "NewsItem" # type: ignore # noqa: F821
120
+ ) -> str | None:
121
+ """Generate AI analysis for a NewsItem. Per v4.0 Β§T28."""
122
+ prompt = NEWS_ANALYSIS_PROMPT.format(
123
+ title=news_item.title,
124
+ source=news_item.source,
125
+ published_at=news_item.published_at.isoformat(),
126
+ body_truncated_to_2000_chars=(news_item.body_markdown or news_item.summary)[:2000],
127
+ )
128
+ return await self.chat(prompt, max_tokens=800, temperature=0.3)
backend/app/catalog/models.py ADDED
@@ -0,0 +1,323 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """T27A β€” Canonical entity models for the RMI data catalog.
2
+
3
+ Pydantic v2 (per ADR-0002). Every entity is persisted in exactly one primary
4
+ store, with cross-store references (string IDs) to related entities in other
5
+ stores. CatalogService resolves these references transparently.
6
+
7
+ Cross-store ID conventions:
8
+ Token, Wallet, Contract: f"{chain.value}:{address}"
9
+ Entity, Alert, NewsItem, RAGFinding, ScanReport: UUID4 hex
10
+ Qdrant point_id: 16-byte UUID hex (matches RAG engine)
11
+
12
+ Persistence (per v4.0 Β§T27 "Why each store exists"):
13
+ Redis: hot token data, rate limits, alert state, cron locks
14
+ Postgres: users, api_keys, subscriptions, x402 receipts, audit
15
+ alerts, news_items, scan_reports
16
+ Neo4j: entities, wallets, deployers, contracts, entity labels
17
+ (graph traversal, Cypher)
18
+ Qdrant: RAG embeddings, token similarity, news embeddings
19
+ MinIO: news raw HTML, report markdown, RAG source docs
20
+ Filesystem: ingest tmp, log rotation (transient)
21
+ """
22
+ from __future__ import annotations
23
+
24
+ from datetime import datetime, UTC
25
+ from enum import Enum
26
+ from typing import Any, Literal, Optional
27
+
28
+ from pydantic import BaseModel, ConfigDict, Field, HttpUrl, field_validator
29
+
30
+
31
+ # ── Enums ───────────────────────────────────────────────────────────
32
+ class Chain(str, Enum):
33
+ """All chains the platform indexes. Per v4.0 Β§T27."""
34
+
35
+ SOLANA = "solana"
36
+ ETHEREUM = "ethereum"
37
+ BASE = "base"
38
+ ARBITRUM = "arbitrum"
39
+ OPTIMISM = "optimism"
40
+ POLYGON = "polygon"
41
+ BSC = "bsc"
42
+ TRON = "tron"
43
+ BITCOIN = "bitcoin"
44
+ AVALANCHE = "avalanche"
45
+ FANTOM = "fantom"
46
+ GNOSIS = "gnosis"
47
+ # EVM subnets (sample β€” full 96 in CHAIN_REGISTRY)
48
+ SEPOLIA = "sepolia"
49
+ LINEA = "linea"
50
+ SCROLL = "scroll"
51
+ ZKSYNC = "zksync"
52
+ BLAST = "blast"
53
+ MANTLE = "mantle"
54
+
55
+
56
+ # Full chain registry β€” v4.0 says 96 chains. We track the canonical 20 here
57
+ # plus extend via CHAIN_REGISTRY for the remaining 76. Adding a chain is a
58
+ # one-line edit.
59
+ CHAIN_REGISTRY: dict[str, dict[str, str]] = {
60
+ "solana": {"name": "Solana", "type": "svm", "native": "SOL", "explorer": "https://solscan.io"},
61
+ "ethereum": {"name": "Ethereum", "type": "evm", "native": "ETH", "explorer": "https://etherscan.io"},
62
+ "base": {"name": "Base", "type": "evm", "native": "ETH", "explorer": "https://basescan.org"},
63
+ "arbitrum": {"name": "Arbitrum One", "type": "evm", "native": "ETH", "explorer": "https://arbiscan.io"},
64
+ "optimism": {"name": "Optimism", "type": "evm", "native": "ETH", "explorer": "https://optimistic.etherscan.io"},
65
+ "polygon": {"name": "Polygon", "type": "evm", "native": "MATIC", "explorer": "https://polygonscan.com"},
66
+ "bsc": {"name": "BNB Smart Chain", "type": "evm", "native": "BNB", "explorer": "https://bscscan.com"},
67
+ "tron": {"name": "TRON", "type": "tvm", "native": "TRX", "explorer": "https://tronscan.org"},
68
+ "bitcoin": {"name": "Bitcoin", "type": "utxo", "native": "BTC", "explorer": "https://mempool.space"},
69
+ "avalanche": {"name": "Avalanche C-Chain", "type": "evm", "native": "AVAX", "explorer": "https://snowtrace.io"},
70
+ "fantom": {"name": "Fantom Opera", "type": "evm", "native": "FTM", "explorer": "https://ftmscan.com"},
71
+ "gnosis": {"name": "Gnosis Chain", "type": "evm", "native": "xDAI", "explorer": "https://gnosisscan.io"},
72
+ }
73
+
74
+
75
+ class RiskTier(str, Enum):
76
+ LOW = "low"
77
+ MEDIUM = "medium"
78
+ HIGH = "high"
79
+ CRITICAL = "critical"
80
+
81
+
82
+ # ── Entities (Neo4j primary) ───────────────────────────────────────
83
+ class Entity(BaseModel):
84
+ """A logical entity resolved across chains. Neo4j primary."""
85
+
86
+ model_config = ConfigDict(extra="ignore")
87
+
88
+ entity_id: str = Field(..., description="UUID, Neo4j primary key")
89
+ label: Optional[str] = None
90
+ aliases: list[str] = Field(default_factory=list)
91
+ first_seen: datetime
92
+ last_seen: datetime
93
+ risk_score: Optional[int] = Field(None, ge=0, le=100)
94
+ tags: list[str] = Field(default_factory=list)
95
+ notes: Optional[str] = None
96
+ store: Literal["neo4j"] = "neo4j"
97
+
98
+
99
+ class EntityLabel(BaseModel):
100
+ """A label attached to an entity. Neo4j primary."""
101
+
102
+ model_config = ConfigDict(extra="ignore")
103
+
104
+ entity_id: str
105
+ label: str
106
+ source: Literal["manual", "heuristic", "third_party"]
107
+ confidence: float = Field(ge=0.0, le=1.0)
108
+ added_at: datetime
109
+ added_by: str
110
+ store: Literal["neo4j"] = "neo4j"
111
+
112
+
113
+ # ── Wallets + Deployers (Neo4j primary) ───────────────────────────
114
+ class Wallet(BaseModel):
115
+ """An on-chain wallet. Neo4j primary; linked to Entity."""
116
+
117
+ model_config = ConfigDict(extra="ignore")
118
+
119
+ wallet_id: str = Field(..., description='"chain:address", e.g. "solana:7Np41..."')
120
+ chain: Chain
121
+ address: str
122
+ entity_id: Optional[str] = None
123
+ first_seen: datetime
124
+ last_seen: datetime
125
+ tx_count: int = 0
126
+ total_volume_usd: float = 0.0
127
+ is_deployer: bool = False
128
+ is_known_exchange: bool = False
129
+ is_suspicious: bool = False
130
+ reputation_score: Optional[int] = Field(None, ge=0, le=100)
131
+ store: Literal["neo4j"] = "neo4j"
132
+
133
+ @field_validator("wallet_id")
134
+ @classmethod
135
+ def _check_id_format(cls, v: str) -> str:
136
+ if ":" not in v:
137
+ raise ValueError("wallet_id must be 'chain:address'")
138
+ chain, _ = v.split(":", 1)
139
+ if chain not in {c.value for c in Chain}:
140
+ # Allow unknown chains (forward compat) but flag them
141
+ pass
142
+ return v
143
+
144
+
145
+ class Deployer(Wallet):
146
+ """A wallet that has deployed at least one token. Extends Wallet."""
147
+
148
+ model_config = ConfigDict(extra="ignore")
149
+
150
+ deployments: list[str] = Field(default_factory=list, description="token_ids")
151
+ rug_count: int = 0
152
+ legit_count: int = 0
153
+ avg_token_lifetime_days: float = 0.0
154
+ reputation_score: Optional[int] = Field(
155
+ None,
156
+ ge=0,
157
+ le=100,
158
+ description="Weighted: legit_count * 1.0 - rug_count * 3.0 + age_bonus - news_penalty. Cached in Redis TTL 1h.",
159
+ )
160
+
161
+
162
+ # ── Tokens (Postgres primary) ──────────────────────────────────────
163
+ class Token(BaseModel):
164
+ """A token contract. Postgres primary; references Deployer in Neo4j."""
165
+
166
+ model_config = ConfigDict(extra="ignore")
167
+
168
+ token_id: str = Field(..., description='"chain:address"')
169
+ chain: Chain
170
+ address: str
171
+ symbol: str
172
+ name: str
173
+ decimals: int
174
+ deployer_wallet_id: Optional[str] = Field(
175
+ None, description="Cross-store ref to Wallet (Neo4j)"
176
+ )
177
+ deployed_at: datetime
178
+ initial_supply: int
179
+ current_supply: Optional[int] = None
180
+ is_honeypot: Optional[bool] = None
181
+ is_mintable: Optional[bool] = None
182
+ is_proxy: Optional[bool] = None
183
+ tax_buy_bps: Optional[int] = None
184
+ tax_sell_bps: Optional[int] = None
185
+ risk_tier: Optional[RiskTier] = None
186
+ risk_score: Optional[int] = Field(None, ge=0, le=100)
187
+ risk_factors: list[str] = Field(default_factory=list)
188
+ rag_embedding_id: Optional[str] = Field(
189
+ None, description="Cross-store ref to Qdrant point (16-byte hex)"
190
+ )
191
+ store: Literal["postgres"] = "postgres"
192
+
193
+
194
+ # ── Alerts (Postgres primary) ──────────────────────────────────────
195
+ class Alert(BaseModel):
196
+ """A risk alert. Postgres primary."""
197
+
198
+ model_config = ConfigDict(extra="ignore")
199
+
200
+ alert_id: str = Field(..., description="UUID")
201
+ token_id: Optional[str] = None
202
+ wallet_id: Optional[str] = None
203
+ chain: Optional[Chain] = None
204
+ alert_type: Literal[
205
+ "rug_detected",
206
+ "deployer_history",
207
+ "liquidity_drain",
208
+ "honeypot_detected",
209
+ "high_tax_change",
210
+ "whale_dump",
211
+ ]
212
+ severity: Literal["info", "warning", "critical"]
213
+ title: str
214
+ description: str
215
+ evidence: dict[str, Any] = Field(default_factory=dict)
216
+ created_at: datetime
217
+ resolved_at: Optional[datetime] = None
218
+ store: Literal["postgres"] = "postgres"
219
+
220
+
221
+ # ── News (Postgres primary + Qdrant embeddings) ────────────────────
222
+ class NewsItem(BaseModel):
223
+ """A news article from RSS. Postgres primary; embeddings in Qdrant."""
224
+
225
+ model_config = ConfigDict(extra="ignore")
226
+
227
+ news_id: str = Field(..., description="UUID")
228
+ url: HttpUrl
229
+ title: str
230
+ summary: str
231
+ body_markdown: Optional[str] = None
232
+ source: str
233
+ published_at: datetime
234
+ ingested_at: datetime
235
+ chains_mentioned: list[Chain] = Field(default_factory=list)
236
+ tokens_mentioned: list[str] = Field(default_factory=list)
237
+ wallets_mentioned: list[str] = Field(default_factory=list)
238
+ sentiment_score: Optional[float] = Field(None, ge=-1.0, le=1.0)
239
+ ai_analysis: Optional[str] = None
240
+ rag_embedding_id: Optional[str] = None
241
+ store: Literal["postgres"] = "postgres"
242
+
243
+
244
+ # ── RAG Findings (Qdrant primary + Postgres metadata) ───────────────
245
+ class RAGFinding(BaseModel):
246
+ """A fact extracted by RAG. Qdrant primary (vector); metadata in Postgres."""
247
+
248
+ model_config = ConfigDict(extra="ignore")
249
+
250
+ finding_id: str = Field(..., description="UUID")
251
+ source_type: Literal["news", "onchain", "audit", "social", "manual"]
252
+ source_url: Optional[HttpUrl] = None
253
+ source_token_id: Optional[str] = None
254
+ source_wallet_id: Optional[str] = None
255
+ claim: str
256
+ confidence: float = Field(ge=0.0, le=1.0)
257
+ extracted_at: datetime
258
+ qdrant_point_id: str
259
+ store: Literal["qdrant"] = "qdrant"
260
+
261
+
262
+ # ── Reports (Postgres + MinIO) ──────────────────────────────────────
263
+ class ScanReport(BaseModel):
264
+ """A research report. Postgres primary; markdown in MinIO."""
265
+
266
+ model_config = ConfigDict(extra="ignore")
267
+
268
+ report_id: str = Field(..., description="UUID")
269
+ subject_type: Literal["token", "wallet", "deployer"]
270
+ subject_id: str
271
+ generated_at: datetime
272
+ generated_by_model: str
273
+ risk_score: int = Field(ge=0, le=100)
274
+ risk_tier: RiskTier
275
+ sections: dict[str, str] = Field(default_factory=dict)
276
+ markdown_url: Optional[HttpUrl] = None
277
+ paid_via_x402: Optional[str] = None
278
+ store: Literal["postgres", "minio"] = "postgres"
279
+
280
+ def to_markdown(self) -> str:
281
+ """Render report sections to a single Markdown document."""
282
+ parts = [
283
+ f"# Research Report: {self.subject_type.title()} `{self.subject_id}`",
284
+ "",
285
+ f"**Generated:** {self.generated_at.isoformat()}",
286
+ f"**Generated by:** {self.generated_by_model}",
287
+ f"**Risk score:** {self.risk_score}/100 ({self.risk_tier.value.upper()})",
288
+ "",
289
+ ]
290
+ for section, body in self.sections.items():
291
+ parts.append(f"## {section.replace('_', ' ').title()}")
292
+ parts.append("")
293
+ parts.append(body)
294
+ parts.append("")
295
+ parts.append("---")
296
+ parts.append(f"*Report ID: {self.report_id}*")
297
+ return "\n".join(parts)
298
+
299
+
300
+ # ── Wire-format helpers ─────────────────────────────────────────────
301
+ def utcnow() -> datetime:
302
+ """Timezone-aware UTC now. Pydantic serializes to ISO 8601."""
303
+ return datetime.now(UTC)
304
+
305
+
306
+ # ── RAG engine collections (kept here so catalog + RAG share the list) ─
307
+ COLLECTIONS: list[str] = [
308
+ # Per v4.0 catalog/RAG bridge β€” these are the canonical 13 RAG
309
+ # collections that also have Token/Wallet/etc cross-refs.
310
+ "scam_intel",
311
+ "deployer_history",
312
+ "wallet_labels",
313
+ "contract_audit",
314
+ "phishing_db",
315
+ "defi_hacks",
316
+ "rug_timeline",
317
+ "vuln_patterns",
318
+ "crime_reports",
319
+ "transaction_patterns",
320
+ "known_scams",
321
+ "token_analysis",
322
+ "market_intel",
323
+ ]
backend/app/catalog/reputation.py ADDED
@@ -0,0 +1,119 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """T31 β€” Deployer Reputation System.
2
+
3
+ Per v4.0 Β§T31. Deterministic, auditable, cached in Redis with TTL 1h.
4
+
5
+ The score is a pure function of inputs β€” no LLM in the loop. Given the
6
+ same on-chain history, news, and RAG findings, the score is the same.
7
+ An analyst can challenge a score by inspecting the weights.
8
+ """
9
+ from __future__ import annotations
10
+
11
+ import logging
12
+ import os
13
+ from datetime import datetime, UTC
14
+
15
+ from app.catalog.models import Deployer, utcnow
16
+
17
+ log = logging.getLogger(__name__)
18
+
19
+
20
+ # ── Weights (tunable, documented in DESIGN.md) ─────────────────────
21
+ WEIGHTS: dict[str, int] = {
22
+ "experience_bonus_per_deploy": 1, # +1 per prior deployment, max +10
23
+ "experience_bonus_cap": 10,
24
+ "rug_penalty": 30, # -30 per prior rug, max -90
25
+ "rug_penalty_cap": 90,
26
+ "longevity_bonus": 10, # +10 if wallet > 1 year old
27
+ "longevity_threshold_days": 365,
28
+ "volume_bonus": 10, # +10 if total raised > $1M
29
+ "volume_threshold_usd": 1_000_000,
30
+ "news_penalty": 15, # -15 if avg news sentiment < -0.3
31
+ "news_bonus": 10, # +10 if avg news sentiment > 0.3
32
+ "news_window_hours": 720, # 30 days
33
+ "rag_penalty": 20, # -20 per high-confidence RAG finding
34
+ "rag_penalty_cap": 60,
35
+ "rag_min_confidence": 0.7,
36
+ }
37
+
38
+
39
+ async def compute_deployer_reputation(
40
+ deployer: Deployer,
41
+ catalog: "CatalogService", # forward ref β€” avoids circular import
42
+ ) -> int:
43
+ """Compute 0-100 reputation score. Pure function of inputs.
44
+
45
+ 100 = long clean history, 0 = known serial rugger.
46
+ """
47
+ cache_key = f"catalog:deployer_rep:{deployer.wallet_id}"
48
+ if catalog._health.redis:
49
+ try:
50
+ cached = await catalog._redis.get(cache_key)
51
+ if cached:
52
+ return int(cached)
53
+ except Exception:
54
+ pass
55
+
56
+ score = 50 # neutral baseline
57
+
58
+ # Experience bonus
59
+ deployments = len(deployer.deployments)
60
+ score += min(deployments * WEIGHTS["experience_bonus_per_deploy"], WEIGHTS["experience_bonus_cap"])
61
+
62
+ # Rug penalty (dominant)
63
+ score -= min(deployer.rug_count * WEIGHTS["rug_penalty"], WEIGHTS["rug_penalty_cap"])
64
+
65
+ # Longevity bonus
66
+ age_days = (utcnow() - deployer.first_seen).days
67
+ if age_days > WEIGHTS["longevity_threshold_days"]:
68
+ score += WEIGHTS["longevity_bonus"]
69
+
70
+ # Volume bonus
71
+ if deployer.total_volume_usd > WEIGHTS["volume_threshold_usd"]:
72
+ score += WEIGHTS["volume_bonus"]
73
+
74
+ # News sentiment (if Postgres is reachable)
75
+ if catalog._health.postgres:
76
+ try:
77
+ async with catalog._pg_pool.acquire() as conn:
78
+ rows = await conn.fetch(
79
+ "SELECT sentiment_score FROM news_items "
80
+ "WHERE $1 = ANY(wallets_mentioned) "
81
+ "AND published_at > NOW() - INTERVAL '%s hours' "
82
+ "LIMIT 20" % WEIGHTS["news_window_hours"],
83
+ deployer.wallet_id,
84
+ )
85
+ if rows:
86
+ scores = [r["sentiment_score"] for r in rows if r["sentiment_score"] is not None]
87
+ if scores:
88
+ avg = sum(scores) / len(scores)
89
+ if avg < -0.3:
90
+ score -= WEIGHTS["news_penalty"]
91
+ elif avg > 0.3:
92
+ score += WEIGHTS["news_bonus"]
93
+ except Exception as e:
94
+ log.debug("reputation_news_fail: %s", e)
95
+
96
+ # RAG findings (rug-related, high confidence)
97
+ try:
98
+ findings = await catalog.rag_search(
99
+ query=deployer.wallet_id, collection="deployer_history", top_k=10
100
+ )
101
+ rug_findings = [
102
+ f for f in findings
103
+ if "rug" in f.get("text", "").lower() or "scam" in f.get("text", "").lower()
104
+ ]
105
+ # Confidence proxy: RRF score in the search results
106
+ high_conf = [f for f in rug_findings if f.get("score", 0) > WEIGHTS["rag_min_confidence"]]
107
+ score -= min(len(high_conf) * WEIGHTS["rag_penalty"], WEIGHTS["rag_penalty_cap"])
108
+ except Exception as e:
109
+ log.debug("reputation_rag_fail: %s", e)
110
+
111
+ score = max(0, min(100, score))
112
+
113
+ if catalog._health.redis:
114
+ try:
115
+ await catalog._redis.setex(cache_key, 3600, str(score))
116
+ except Exception:
117
+ pass
118
+
119
+ return score
backend/app/catalog/service.py ADDED
@@ -0,0 +1,594 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """T27 CatalogService β€” unified read/write API for RMI.
2
+
3
+ Per v4.0 Β§T27. The CatalogService is the ONLY sanctioned way to read or
4
+ write catalog data. Domain facades call this; they never touch stores directly.
5
+
6
+ Architecture:
7
+ - Lazy store clients (init on first use, retry on failure)
8
+ - Graceful degradation: unreachable stores return None/empty, not crash
9
+ - Redis cache layer (always available since rmi-redis is up)
10
+ - Cross-store ID conventions from app.catalog.models
11
+
12
+ Recipe coverage:
13
+ Recipe 1: find_tokens_by_deployer_history (Postgres + Neo4j)
14
+ Recipe 2: find_similar_tokens (Qdrant + Postgres)
15
+ Recipe 3: get_token_risk (Redis + Postgres + Neo4j)
16
+ Recipe 4: news_price_correlation (Postgres, basic v1)
17
+ Recipe 5: resolve_entity (Neo4j Cypher)
18
+ """
19
+ from __future__ import annotations
20
+
21
+ import asyncio
22
+ import json
23
+ import logging
24
+ import os
25
+ import time
26
+ from typing import Any, Optional
27
+
28
+ import httpx
29
+
30
+ from app.catalog.models import (
31
+ Alert,
32
+ COLLECTIONS,
33
+ Chain,
34
+ Deployer,
35
+ Entity,
36
+ NewsItem,
37
+ RAGFinding,
38
+ ScanReport,
39
+ Token,
40
+ Wallet,
41
+ utcnow,
42
+ )
43
+ from app.rag.engine import three_pillar_search as rag_three_pillar_search
44
+ from app.rag.engine import ingest_document as rag_ingest_document
45
+
46
+ log = logging.getLogger(__name__)
47
+
48
+
49
+ # ── Connection config (from env, with sensible defaults to netcup IPs) ──
50
+ # These defaults point to the actual IPs on netcup rmi_network.
51
+ # Override via env vars in docker-compose.yml for portability.
52
+ DEFAULT_CONFIG: dict[str, Any] = {
53
+ "redis": {
54
+ "host": os.getenv("REDIS_HOST", "rmi-redis"),
55
+ "port": int(os.getenv("REDIS_PORT", "6379")),
56
+ "password": os.getenv("REDIS_PASSWORD", "RMI_PROD_REDIS_2026"),
57
+ "db": int(os.getenv("REDIS_DB", "0")),
58
+ },
59
+ "postgres": {
60
+ "host": os.getenv("POSTGRES_HOST", "rmi-postgres"),
61
+ "port": int(os.getenv("POSTGRES_PORT", "5432")),
62
+ "user": os.getenv("POSTGRES_USER", "rmi"),
63
+ "password": os.getenv("POSTGRES_PASSWORD", ""),
64
+ "database": os.getenv("POSTGRES_DB", "rmi"),
65
+ },
66
+ "neo4j": {
67
+ "uri": os.getenv("NEO4J_URI", "bolt://rmi-neo4j:7687"),
68
+ "user": os.getenv("NEO4J_USER", "neo4j"),
69
+ "password": os.getenv("NEO4J_PASSWORD", ""),
70
+ },
71
+ "qdrant": {
72
+ "url": os.getenv("QDRANT_URL", "http://rmi-qdrant:6333"),
73
+ "api_key": os.getenv("QDRANT_API_KEY", ""),
74
+ },
75
+ "minio": {
76
+ "endpoint": os.getenv("MINIO_ENDPOINT", "rmi-minio:9000"),
77
+ "access_key": os.getenv("MINIO_ACCESS_KEY", ""),
78
+ "secret_key": os.getenv("MINIO_SECRET_KEY", ""),
79
+ },
80
+ }
81
+
82
+
83
+ class StoreHealth:
84
+ """Tracks which stores are reachable. Updated on each probe."""
85
+
86
+ def __init__(self) -> None:
87
+ self.redis: bool = False
88
+ self.postgres: bool = False
89
+ self.neo4j: bool = False
90
+ self.qdrant: bool = False
91
+ self.minio: bool = False
92
+ self.last_checked: float = 0.0
93
+
94
+
95
+ class CatalogService:
96
+ """Unified read/write API for RMI data.
97
+
98
+ Use `get_catalog()` to get the singleton instance. All methods are
99
+ async and graceful-degrade β€” if a store is unreachable, the method
100
+ returns None or an empty result with a logged warning.
101
+ """
102
+
103
+ def __init__(self, config: dict[str, Any] | None = None) -> None:
104
+ self.config = config or DEFAULT_CONFIG
105
+ self._health = StoreHealth()
106
+ self._redis = None
107
+ self._pg_pool: Any = None
108
+ self._neo_driver: Any = None
109
+ self._qdrant: Any = None # httpx.AsyncClient
110
+ self._init_lock = asyncio.Lock()
111
+
112
+ # ── Store init (lazy) ────────────────────────────────────────────
113
+ async def _init_stores(self) -> None:
114
+ async with self._init_lock:
115
+ if self._redis is not None:
116
+ return # already init
117
+ # Redis (always try first β€” used for everything)
118
+ try:
119
+ import redis.asyncio as aioredis
120
+
121
+ cfg = self.config["redis"]
122
+ self._redis = aioredis.Redis(
123
+ host=cfg["host"],
124
+ port=cfg["port"],
125
+ password=cfg["password"],
126
+ db=cfg["db"],
127
+ decode_responses=True,
128
+ )
129
+ await self._redis.ping()
130
+ self._health.redis = True
131
+ log.info("catalog_redis_ok")
132
+ except Exception as e:
133
+ log.warning("catalog_redis_fail: %s", e)
134
+ self._health.redis = False
135
+
136
+ # Postgres
137
+ try:
138
+ import asyncpg
139
+
140
+ cfg = self.config["postgres"]
141
+ if cfg["password"]:
142
+ self._pg_pool = await asyncpg.create_pool(
143
+ host=cfg["host"],
144
+ port=cfg["port"],
145
+ user=cfg["user"],
146
+ password=cfg["password"],
147
+ database=cfg["database"],
148
+ min_size=1,
149
+ max_size=8,
150
+ command_timeout=10,
151
+ )
152
+ self._health.postgres = True
153
+ log.info("catalog_postgres_ok")
154
+ except Exception as e:
155
+ log.warning("catalog_postgres_fail: %s", e)
156
+ self._health.postgres = False
157
+
158
+ # Neo4j
159
+ try:
160
+ from neo4j import GraphDatabase
161
+
162
+ cfg = self.config["neo4j"]
163
+ if cfg["password"]:
164
+ self._neo_driver = GraphDatabase.driver(
165
+ cfg["uri"], auth=(cfg["user"], cfg["password"])
166
+ )
167
+ with self._neo_driver.session() as s:
168
+ s.run("RETURN 1").consume()
169
+ self._health.neo4j = True
170
+ log.info("catalog_neo4j_ok")
171
+ except Exception as e:
172
+ log.warning("catalog_neo4j_fail: %s", e)
173
+ self._health.neo4j = False
174
+
175
+ # Qdrant (HTTP)
176
+ try:
177
+ cfg = self.config["qdrant"]
178
+ headers = {"api-key": cfg["api_key"]} if cfg["api_key"] else {}
179
+ self._qdrant = httpx.AsyncClient(
180
+ base_url=cfg["url"], headers=headers, timeout=5.0
181
+ )
182
+ r = await self._qdrant.get("/collections")
183
+ if r.status_code == 200:
184
+ self._health.qdrant = True
185
+ log.info("catalog_qdrant_ok")
186
+ else:
187
+ log.warning("catalog_qdrant_http_%d", r.status_code)
188
+ except Exception as e:
189
+ log.warning("catalog_qdrant_fail: %s", e)
190
+ self._health.qdrant = False
191
+
192
+ self._health.last_checked = time.time()
193
+
194
+ # ── Health probe ────────────────────────────────────────────────
195
+ async def probe_stores(self) -> dict[str, bool]:
196
+ await self._init_stores()
197
+ return {
198
+ "redis": self._health.redis,
199
+ "postgres": self._health.postgres,
200
+ "neo4j": self._health.neo4j,
201
+ "qdrant": self._health.qdrant,
202
+ "minio": self._health.minio,
203
+ }
204
+
205
+ # ── Tokens (Postgres primary + Redis cache) ─────────────────────
206
+ async def get_token(self, chain: Chain, address: str) -> Token | None:
207
+ await self._init_stores()
208
+ cache_key = f"catalog:token:{chain.value}:{address}"
209
+ if self._health.redis:
210
+ try:
211
+ cached = await self._redis.get(cache_key)
212
+ if cached:
213
+ return Token.model_validate_json(cached)
214
+ except Exception as e:
215
+ log.debug("token_cache_get_fail: %s", e)
216
+ if not self._health.postgres:
217
+ return None
218
+ try:
219
+ async with self._pg_pool.acquire() as conn:
220
+ row = await conn.fetchrow(
221
+ "SELECT * FROM tokens WHERE chain=$1 AND address=$2",
222
+ chain.value, address,
223
+ )
224
+ if not row:
225
+ return None
226
+ token = Token(**dict(row))
227
+ if self._health.redis:
228
+ try:
229
+ await self._redis.setex(cache_key, 3600, token.model_dump_json())
230
+ except Exception as e:
231
+ log.debug("token_cache_set_fail: %s", e)
232
+ return token
233
+ except Exception as e:
234
+ log.warning("token_get_fail: %s", e)
235
+ return None
236
+
237
+ async def save_token(self, token: Token) -> bool:
238
+ await self._init_stores()
239
+ if not self._health.postgres:
240
+ log.warning("token_save_no_postgres")
241
+ return False
242
+ try:
243
+ async with self._pg_pool.acquire() as conn:
244
+ await conn.execute(
245
+ """
246
+ INSERT INTO tokens (
247
+ token_id, chain, address, symbol, name, decimals,
248
+ deployer_wallet_id, deployed_at, initial_supply,
249
+ current_supply, is_honeypot, is_mintable, is_proxy,
250
+ tax_buy_bps, tax_sell_bps, risk_tier, risk_score,
251
+ risk_factors, rag_embedding_id
252
+ ) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,
253
+ $14,$15,$16,$17,$18,$19)
254
+ ON CONFLICT (token_id) DO UPDATE SET
255
+ symbol=EXCLUDED.symbol,
256
+ name=EXCLUDED.name,
257
+ current_supply=EXCLUDED.current_supply,
258
+ is_honeypot=EXCLUDED.is_honeypot,
259
+ is_mintable=EXCLUDED.is_mintable,
260
+ risk_tier=EXCLUDED.risk_tier,
261
+ risk_score=EXCLUDED.risk_score,
262
+ risk_factors=EXCLUDED.risk_factors
263
+ """,
264
+ token.token_id, token.chain.value, token.address,
265
+ token.symbol, token.name, token.decimals,
266
+ token.deployer_wallet_id, token.deployed_at,
267
+ token.initial_supply, token.current_supply,
268
+ token.is_honeypot, token.is_mintable, token.is_proxy,
269
+ token.tax_buy_bps, token.tax_sell_bps,
270
+ token.risk_tier.value if token.risk_tier else None,
271
+ token.risk_score, token.risk_factors, token.rag_embedding_id,
272
+ )
273
+ # Invalidate cache
274
+ if self._health.redis:
275
+ try:
276
+ await self._redis.delete(
277
+ f"catalog:token:{token.chain.value}:{token.address}"
278
+ )
279
+ except Exception:
280
+ pass
281
+ return True
282
+ except Exception as e:
283
+ log.warning("token_save_fail: %s", e)
284
+ return False
285
+
286
+ # ── Wallets (Neo4j primary) ──────────────────────────────────────
287
+ async def get_wallet(self, chain: Chain, address: str) -> Wallet | None:
288
+ await self._init_stores()
289
+ if not self._health.neo4j:
290
+ return None
291
+ wallet_id = f"{chain.value}:{address}"
292
+ try:
293
+ with self._neo_driver.session() as s:
294
+ result = s.run(
295
+ "MATCH (w:Wallet {wallet_id: $id}) RETURN w",
296
+ id=wallet_id,
297
+ ).single()
298
+ if not result:
299
+ return None
300
+ node = dict(result["w"])
301
+ # Normalize datetime strings
302
+ for k in ("first_seen", "last_seen"):
303
+ if isinstance(node.get(k), str):
304
+ from datetime import datetime
305
+ try:
306
+ node[k] = datetime.fromisoformat(node[k].replace("Z", "+00:00"))
307
+ except Exception:
308
+ pass
309
+ return Wallet(**node)
310
+ except Exception as e:
311
+ log.warning("wallet_get_fail: %s", e)
312
+ return None
313
+
314
+ async def save_wallet(self, wallet: Wallet) -> bool:
315
+ await self._init_stores()
316
+ if not self._health.neo4j:
317
+ return False
318
+ try:
319
+ with self._neo_driver.session() as s:
320
+ s.run(
321
+ """
322
+ MERGE (w:Wallet {wallet_id: $wallet_id})
323
+ SET w.chain=$chain, w.address=$address,
324
+ w.first_seen=$first_seen, w.last_seen=$last_seen,
325
+ w.tx_count=$tx_count, w.total_volume_usd=$total_volume_usd,
326
+ w.is_deployer=$is_deployer,
327
+ w.is_known_exchange=$is_known_exchange,
328
+ w.is_suspicious=$is_suspicious,
329
+ w.reputation_score=$reputation_score
330
+ """,
331
+ wallet_id=wallet.wallet_id,
332
+ chain=wallet.chain.value,
333
+ address=wallet.address,
334
+ first_seen=wallet.first_seen.isoformat(),
335
+ last_seen=wallet.last_seen.isoformat(),
336
+ tx_count=wallet.tx_count,
337
+ total_volume_usd=wallet.total_volume_usd,
338
+ is_deployer=wallet.is_deployer,
339
+ is_known_exchange=wallet.is_known_exchange,
340
+ is_suspicious=wallet.is_suspicious,
341
+ reputation_score=wallet.reputation_score,
342
+ )
343
+ return True
344
+ except Exception as e:
345
+ log.warning("wallet_save_fail: %s", e)
346
+ return False
347
+
348
+ # ── Recipe 1: find tokens by deployer history ───────────────────
349
+ async def find_tokens_by_deployer_history(
350
+ self, min_rug_count: int = 1, chain: Chain | None = None, limit: int = 50
351
+ ) -> list[Token]:
352
+ """Find tokens deployed by wallets that have rug-pulled before.
353
+
354
+ Cross-store: Neo4j for the wallet filter, Postgres for the tokens.
355
+ """
356
+ await self._init_stores()
357
+ if not (self._health.neo4j and self._health.postgres):
358
+ log.warning("recipe1_stores_unavailable")
359
+ return []
360
+ try:
361
+ # Step 1: Neo4j β€” find wallets with rug_count >= min_rug_count
362
+ with self._neo_driver.session() as s:
363
+ wallets = [
364
+ r["w.wallet_id"]
365
+ for r in s.run(
366
+ "MATCH (w:Deployer) WHERE w.rug_count >= $min "
367
+ "RETURN w.wallet_id LIMIT 200",
368
+ min=min_rug_count,
369
+ )
370
+ ]
371
+ if not wallets:
372
+ return []
373
+ # Step 2: Postgres β€” find tokens deployed by those wallets
374
+ async with self._pg_pool.acquire() as conn:
375
+ query = (
376
+ "SELECT * FROM tokens WHERE deployer_wallet_id = ANY($1::text[]) "
377
+ )
378
+ params: list[Any] = [wallets]
379
+ if chain:
380
+ query += "AND chain=$2 "
381
+ params.append(chain.value)
382
+ query += "ORDER BY deployed_at DESC LIMIT $3"
383
+ params.append(limit)
384
+ rows = await conn.fetch(query, *params)
385
+ return [Token(**dict(r)) for r in rows]
386
+ except Exception as e:
387
+ log.warning("recipe1_fail: %s", e)
388
+ return []
389
+
390
+ # ── Recipe 3: get_token_risk (3-store composition) ──────────────
391
+ async def get_token_risk(
392
+ self, chain: Chain, address: str
393
+ ) -> dict[str, Any]:
394
+ """Real-time risk score β€” composes Redis cache + Postgres + Neo4j.
395
+
396
+ Returns dict (not Token) because it's a cross-store projection.
397
+ Cache TTL 60s.
398
+ """
399
+ await self._init_stores()
400
+ cache_key = f"catalog:risk:{chain.value}:{address}"
401
+ if self._health.redis:
402
+ try:
403
+ cached = await self._redis.get(cache_key)
404
+ if cached:
405
+ return json.loads(cached)
406
+ except Exception:
407
+ pass
408
+
409
+ token = await self.get_token(chain, address)
410
+ deployer_reputation: int | None = None
411
+ if token and token.deployer_wallet_id and self._health.neo4j:
412
+ try:
413
+ with self._neo_driver.session() as s:
414
+ r = s.run(
415
+ "MATCH (d:Deployer {wallet_id: $id}) "
416
+ "RETURN d.reputation_score AS rep, d.rug_count AS rugs",
417
+ id=token.deployer_wallet_id,
418
+ ).single()
419
+ if r:
420
+ deployer_reputation = r["rep"]
421
+ except Exception as e:
422
+ log.debug("risk_neo4j_fail: %s", e)
423
+
424
+ token_score = token.risk_score if token and token.risk_score is not None else 50
425
+ if deployer_reputation is not None:
426
+ score = int(0.6 * token_score + 0.4 * (100 - deployer_reputation))
427
+ else:
428
+ score = token_score
429
+ score = max(0, min(100, score))
430
+ if score < 25:
431
+ tier = "low"
432
+ elif score < 50:
433
+ tier = "medium"
434
+ elif score < 75:
435
+ tier = "high"
436
+ else:
437
+ tier = "critical"
438
+ result: dict[str, Any] = {
439
+ "score": score,
440
+ "tier": tier,
441
+ "factors": token.risk_factors if token else [],
442
+ "token": token.model_dump() if token else None,
443
+ "deployer_reputation": deployer_reputation,
444
+ "fetched_at": utcnow().isoformat(),
445
+ }
446
+ if self._health.redis:
447
+ try:
448
+ await self._redis.setex(cache_key, 60, json.dumps(result, default=str))
449
+ except Exception:
450
+ pass
451
+ return result
452
+
453
+ # ── Recipe 5: resolve_entity (Neo4j Cypher) ─────────────────────
454
+ async def resolve_entity(
455
+ self, wallet_id: str, max_chains: int = 5
456
+ ) -> dict[str, Any]:
457
+ """Cross-chain entity resolution via Neo4j.
458
+
459
+ Uses the SAME_AS / FUNDED_BY_SAME / CLONE_OF / BEHAVIORAL_MATCH
460
+ edges defined in v4.0 Β§T32 with path-weighted confidence.
461
+ """
462
+ await self._init_stores()
463
+ if not self._health.neo4j:
464
+ return {"entity_id": None, "wallets": [], "note": "neo4j unavailable"}
465
+ try:
466
+ with self._neo_driver.session() as s:
467
+ result = s.run(
468
+ """
469
+ MATCH (start:Wallet {wallet_id: $wallet_id})
470
+ OPTIONAL MATCH path = (start)-[:SAME_AS|FUNDED_BY_SAME|CLONE_OF|BEHAVIORAL_MATCH*1..3]-(other:Wallet)
471
+ WHERE start <> other
472
+ WITH start, other, path,
473
+ reduce(conf = 1.0, r IN relationships(path) | conf * r.confidence) AS path_conf
474
+ ORDER BY path_conf DESC
475
+ LIMIT $limit
476
+ RETURN start.entity_id AS entity_id,
477
+ collect({
478
+ wallet_id: other.wallet_id,
479
+ chain: other.chain,
480
+ address: other.address,
481
+ confidence: path_conf
482
+ }) AS cross_chain_wallets
483
+ """,
484
+ wallet_id=wallet_id, limit=max_chains,
485
+ ).single()
486
+ if not result:
487
+ return {"entity_id": None, "wallets": [], "note": "no edges"}
488
+ wallets = [w for w in (result["cross_chain_wallets"] or []) if w.get("wallet_id")]
489
+ return {
490
+ "entity_id": result["entity_id"],
491
+ "wallets": wallets,
492
+ "note": f"{len(wallets)} cross-chain matches",
493
+ }
494
+ except Exception as e:
495
+ log.warning("resolve_entity_fail: %s", e)
496
+ return {"entity_id": None, "wallets": [], "error": str(e)}
497
+
498
+ # ── RAG bridge: wire app/rag/engine.py into the catalog ─────────
499
+ async def rag_search(
500
+ self, query: str, collection: str = "scam_intel", top_k: int = 5
501
+ ) -> list[dict]:
502
+ """Search the RAG system. Returns raw hits (catalog-agnostic).
503
+
504
+ Use this when you want RAG results as raw search hits.
505
+ Use get_token_risk / etc. when you want catalog-typed results.
506
+ """
507
+ try:
508
+ return await rag_three_pillar_search(
509
+ query=query, collection=collection, top_k=top_k
510
+ )
511
+ except Exception as e:
512
+ log.warning("rag_search_fail: %s", e)
513
+ return []
514
+
515
+ async def rag_ingest(
516
+ self,
517
+ content: str,
518
+ collection: str = "scam_intel",
519
+ doc_id: str | None = None,
520
+ metadata: dict | None = None,
521
+ ) -> dict:
522
+ """Ingest a fact into RAG and (if it's about a token) link to Token.rag_embedding_id.
523
+
524
+ Returns the RAG ingest result plus the Qdrant point_id for cross-store ref.
525
+ """
526
+ from uuid import uuid4
527
+
528
+ doc_id = doc_id or f"finding:{uuid4().hex[:12]}"
529
+ try:
530
+ r = await rag_ingest_document(
531
+ collection=collection,
532
+ doc_id=doc_id,
533
+ content=content,
534
+ metadata=metadata or {},
535
+ )
536
+ r["qdrant_point_id"] = doc_id # Engine returns the same
537
+ return r
538
+ except Exception as e:
539
+ log.warning("rag_ingest_fail: %s", e)
540
+ return {"status": "failed", "error": str(e)}
541
+
542
+ async def attach_rag_to_token(
543
+ self, chain: Chain, address: str, qdrant_point_id: str
544
+ ) -> bool:
545
+ """Link an existing Qdrant point to a Token row as rag_embedding_id."""
546
+ token = await self.get_token(chain, address)
547
+ if not token:
548
+ return False
549
+ token.rag_embedding_id = qdrant_point_id
550
+ return await self.save_token(token)
551
+
552
+ # ── Stats / introspection ────────────────────────────────────────
553
+ async def stats(self) -> dict[str, Any]:
554
+ await self._init_stores()
555
+ out: dict[str, Any] = {
556
+ "stores": await self.probe_stores(),
557
+ "collections": COLLECTIONS,
558
+ }
559
+ if self._health.postgres:
560
+ try:
561
+ async with self._pg_pool.acquire() as conn:
562
+ out["tokens"] = await conn.fetchval("SELECT COUNT(*) FROM tokens") or 0
563
+ out["alerts"] = await conn.fetchval("SELECT COUNT(*) FROM alerts") or 0
564
+ except Exception as e:
565
+ out["postgres_error"] = str(e)
566
+ if self._health.neo4j:
567
+ try:
568
+ with self._neo_driver.session() as s:
569
+ out["wallets"] = s.run("MATCH (w:Wallet) RETURN COUNT(w) AS c").single()["c"] or 0
570
+ out["deployers"] = s.run("MATCH (d:Deployer) RETURN COUNT(d) AS c").single()["c"] or 0
571
+ except Exception as e:
572
+ out["neo4j_error"] = str(e)
573
+ if self._health.qdrant:
574
+ try:
575
+ cols = await self._qdrant.get("/collections")
576
+ if cols.status_code == 200:
577
+ out["qdrant_collections"] = [
578
+ c["name"] for c in cols.json().get("result", {}).get("collections", [])
579
+ ]
580
+ except Exception as e:
581
+ out["qdrant_error"] = str(e)
582
+ return out
583
+
584
+
585
+ # ── Singleton accessor ──────────────────────────────────────────────
586
+ _catalog: CatalogService | None = None
587
+
588
+
589
+ def get_catalog() -> CatalogService:
590
+ """Get the global CatalogService. Lazy-init on first call."""
591
+ global _catalog
592
+ if _catalog is None:
593
+ _catalog = CatalogService()
594
+ return _catalog
backend/main.py CHANGED
@@ -215,6 +215,7 @@ def _try_mount_v1_routers() -> int:
215
  "app.api.v1.rag.search",
216
  "app.api.v1.x402.payments",
217
  "app.api.v1.admin.alerts_webhook",
 
218
  ]
219
 
220
  for module_path in v1_modules:
 
215
  "app.api.v1.rag.search",
216
  "app.api.v1.x402.payments",
217
  "app.api.v1.admin.alerts_webhook",
218
+ "app.api.v1.catalog",
219
  ]
220
 
221
  for module_path in v1_modules: