Upload folder using huggingface_hub

api.py

@@ -4,6 +4,10 @@ import hashlib
 import re
 from datetime import datetime
 from typing import Optional, Any, List, Dict, cast, AsyncGenerator
+import numpy as np
+import json
+import logging
+from contextlib import asynccontextmanager
 from fastapi import FastAPI, Request
 from fastapi.responses import StreamingResponse
 from pydantic import BaseModel, Field, field_validator
@@ -26,6 +30,20 @@ from config import settings
 from indra_engine import IndraProjectionEngine
 indra_engine = IndraProjectionEngine(batch_size=400, dimensions=768)
 
+# Configure logging
+logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(name)s: %(message)s")
+logger = logging.getLogger("govbridge.api")
+
+# Variables for AI models
+embedding_model = None
+cross_encoder_model = None
+
+def get_embedding_model():
+    return embedding_model
+
+def get_reranker_model():
+    return cross_encoder_model
+
 # --- SECURE KEYS ---
 SUPABASE_URL = settings.SUPABASE_URL
 SUPABASE_KEY = settings.SUPABASE_KEY
@@ -35,7 +53,17 @@ ADMIN_SECRET = settings.ADMIN_SECRET
 supabase: Client = create_client(SUPABASE_URL, SUPABASE_KEY)
 groq_client = Groq(api_key=GROQ_API_KEY)
 
-app = FastAPI()
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    global embedding_model, cross_encoder_model
+    logger.info("⏳ Pre-loading Nomic Embedding Model (768-dim)...")
+    embedding_model = SentenceTransformer("nomic-ai/nomic-embed-text-v1", trust_remote_code=True)
+    logger.info("⏳ Pre-loading Cross-Encoder Reranker Model...")
+    cross_encoder_model = CrossEncoder("cross-encoder/ettin-reranker-68m-v1", max_length=512, device='cpu', trust_remote_code=True)
+    logger.info("✅ Models pre-loaded successfully.")
+    yield
+
+app = FastAPI(lifespan=lifespan)
 app.include_router(whatsapp_router)
 
 # --- RATE LIMITER CONFIG ---
@@ -57,6 +85,7 @@ from middleware.memory_guard import MemoryGuardMiddleware
 app.add_middleware(MemoryGuardMiddleware)
 
 
+
 # --- MODELS ---
 class SearchQuery(BaseModel):
     question: str = Field(..., min_length=3, max_length=500)
@@ -128,29 +157,28 @@ class GazetteSearchQuery(BaseModel):
         return v.lower()
 
 
-# --- INITIALIZE AI (LAZY LOADED) ---
-embedding_model = None
-reranker_model = None
+class GraphNode(BaseModel):
+    """A node in the knowledge graph visualization."""
+    id: str
+    name: str
+    group: str  # 'anchor' | 'hop1' | 'hop2'
+
+class GraphLink(BaseModel):
+    """A directional edge in the knowledge graph visualization."""
+    source: str
+    target: str
+    label: str
+    hop: int
+
+class GraphResponse(BaseModel):
+    """Force-graph-compatible response payload."""
+    anchor_id: str
+    nodes: List[GraphNode]
+    links: List[GraphLink]
+    total_nodes: int
+    total_links: int
 
-def get_embedding_model():
-    global embedding_model
-    if embedding_model is None:
-        print("⏳ Loading Nomic Embedding Model (768-dim)...")
-        # PENDING: Run 003_upgrade_embeddings_768.sql in Supabase before this goes live
-        embedding_model = SentenceTransformer(
-            'nomic-ai/nomic-embed-text-v1', 
-            trust_remote_code=True
-        )
-        print("✅ Nomic Model loaded")
-    return embedding_model
 
-def get_reranker_model():
-    global reranker_model
-    if reranker_model is None:
-        print("⏳ Loading Cross-Encoder Reranker Model...")
-        reranker_model = CrossEncoder('cross-encoder/ettin-reranker-68m-v1', max_length=512, device='cpu', trust_remote_code=True)
-        print("✅ Reranker Model loaded")
-    return reranker_model
 
 def chunk_text(text: str, chunk_size: int = 500, overlap: int = 50) -> list[str]:
     paragraphs = [p.strip() for p in text.split('\n\n') if p.strip()]
@@ -220,7 +248,7 @@ def rerank_gazette_chunks(query: str, chunks: list) -> list:
     if not chunks:
         return []
 
-    RELEVANCE_FLOOR = 5.0  # Hard quality gate — no noise passes
+    RELEVANCE_FLOOR = 5.0  # Hard quality gate restored for production
 
     # Gazette-specific: wider pool (30) because legislative text
     # has higher semantic density than scheme descriptions
@@ -554,15 +582,16 @@ async def gazette_search(request: Request, query: GazetteSearchQuery):
     4. Cross-encoder re-rank top 30 with Ettin
     5. Return page-pinned results to frontend GazetteViewer
     """
-    print(f"📜 Gazette search [INDRA]: '{query.query}' | Type: {query.gazette_type} | State: {query.state}")
+    logger.info(f"📜 Gazette search [INDRA]: '{query.query}' | Type: {query.gazette_type} | State: {query.state}")
 
     try:
         # Step 1: Embed query
         model = get_embedding_model()
-        query_embedding = model.encode(
+        query_embedding_raw = model.encode(
             f"search_query: {query.query}",
             normalize_embeddings=True
-        ).tolist()
+        )
+        query_embedding = np.array(query_embedding_raw).flatten().tolist()
 
         # Step 2: Call INDRA oversampling RPC
         rpc_params: Dict[str, Any] = {
@@ -588,15 +617,13 @@ async def gazette_search(request: Request, query: GazetteSearchQuery):
             }
 
         n_candidates = len(raw_results)
-        print(f"   📊 INDRA: {n_candidates} Euclidean candidates retrieved")
+        logger.info(f"   📊 INDRA: {n_candidates} Euclidean candidates retrieved")
 
         # Step 3: Poincaré projection & hyperbolic re-ranking
-        import numpy as np
-
         # Extract embedding vectors from RPC results
         query_vec = np.array(query_embedding, dtype=np.float32)
         candidate_embeddings = np.array(
-            [r["embedding"] for r in raw_results],
+            [json.loads(r["embedding"]) if isinstance(r["embedding"], str) else r["embedding"] for r in raw_results],
             dtype=np.float32
         )
 
@@ -619,7 +646,7 @@ async def gazette_search(request: Request, query: GazetteSearchQuery):
             entry.pop("embedding", None)
             hyperbolic_ranked.append(entry)
 
-        print(f"   🔮 Poincaré: top-1 distance={hyperbolic_ranked[0]['hyperbolic_distance']:.4f}, "
+        logger.info(f"   🔮 Poincaré: top-1 distance={hyperbolic_ranked[0]['hyperbolic_distance']:.4f}, "
               f"top-30 distance={hyperbolic_ranked[-1]['hyperbolic_distance']:.4f}")
 
         # Step 4: Cross-encoder re-rank (Ettin verifies semantic coherence)
@@ -646,10 +673,10 @@ async def gazette_search(request: Request, query: GazetteSearchQuery):
             })
 
         if results:
-            print(f"   ✅ Returning {len(results)} gazette results "
+            logger.info(f"   ✅ Returning {len(results)} gazette results "
                   f"(best CE score: {results[0]['cross_encoder_score']:.4f})")
         else:
-            print("   ⛔ No results survived quality gate")
+            logger.info("   ⛔ No results survived quality gate")
 
         return {
             "query": query.query,
@@ -659,12 +686,135 @@ async def gazette_search(request: Request, query: GazetteSearchQuery):
         }
 
     except Exception as e:
-        print(f"❌ Gazette search [INDRA] error: {e}")
+        logger.error(f"❌ Gazette search [INDRA] error: {e}", exc_info=True)
         return {
             "query": query.query,
             "results": [],
             "total": 0,
-            "error": str(e),
+            "error": "An internal error occurred during the search. Please try again later.",
             "pipeline": "indra_poincare + ettin_reranker"
         }
+
+# ═══════════════════════════════════════════════════════════════
+# PROJECT INDRA Phase 3.1 — Knowledge Graph Neighborhood API
+# Sprint 34
+# ═══════════════════════════════════════════════════════════════
+
+@app.get("/api/gazette/graph/{chunk_id}")
+@limiter.limit("30/minute")
+async def get_graph_neighborhood(request: Request, chunk_id: str, depth: int = 1, limit: int = 100):
+    """
+    Knowledge Graph Neighborhood — PROJECT INDRA Phase 3.1 (Sprint 34).
+
+    Returns the 1-hop or 2-hop graph neighborhood of a gazette document,
+    formatted as a node-link structure for force-directed graph rendering.
+
+    Args:
+        chunk_id: UUID of the anchor gazette_chunks document.
+        depth: Traversal depth (1 or 2). Default 1.
+        limit: Max edges to return. Default 100, max 500.
+    """
+    logger.info(f"🔗 Graph neighborhood: chunk_id={chunk_id} | depth={depth} | limit={limit}")
+
+    try:
+        # Validate UUID format
+        import uuid as uuid_mod
+        try:
+            uuid_mod.UUID(chunk_id, version=4)
+        except ValueError:
+            return {
+                "anchor_id": chunk_id,
+                "nodes": [],
+                "links": [],
+                "total_nodes": 0,
+                "total_links": 0,
+                "error": "Invalid UUID format"
+            }
+
+        # Clamp parameters
+        depth = max(1, min(2, depth))
+        limit = max(1, min(500, limit))
+
+        # Call the graph neighborhood RPC
+        result = supabase.rpc("get_graph_neighborhood", {
+            "anchor_id": chunk_id,
+            "max_depth": depth,
+            "edge_limit": limit,
+        }).execute()
+
+        raw_edges = cast(List[Dict[str, Any]], result.data or [])
+
+        if not raw_edges:
+            logger.info(f"   ⛔ No graph edges found for chunk_id={chunk_id}")
+            return {
+                "anchor_id": chunk_id,
+                "nodes": [],
+                "links": [],
+                "total_nodes": 0,
+                "total_links": 0
+            }
+
+        # ── Transform edge-centric data to node-link format ──
+        nodes_map: Dict[str, GraphNode] = {}
+        links: List[GraphLink] = []
+
+        for edge in raw_edges:
+            source_id = edge.get("source_id", "")
+            target_id = edge.get("target_id", "")
+            source_title = edge.get("source_title", "Unknown Document")
+            target_title = edge.get("target_title", "Unresolved Entity")
+            edge_type = edge.get("edge_type", "cross_references")
+            hop_depth = edge.get("hop_depth", 1)
+
+            # Skip edges with null target (unresolved entities from Sprint 30)
+            if not target_id:
+                continue
+
+            # Deduplicate nodes — assign group based on relationship to anchor
+            if source_id and source_id not in nodes_map:
+                group = "anchor" if source_id == chunk_id else f"hop{hop_depth}"
+                nodes_map[source_id] = GraphNode(
+                    id=source_id,
+                    name=source_title[:80],  # Truncate long titles for Canvas rendering
+                    group=group,
+                )
+
+            if target_id and target_id not in nodes_map:
+                group = "anchor" if target_id == chunk_id else f"hop{hop_depth}"
+                nodes_map[target_id] = GraphNode(
+                    id=target_id,
+                    name=target_title[:80],
+                    group=group,
+                )
+
+            # Create link
+            if source_id and target_id:
+                links.append(GraphLink(
+                    source=source_id,
+                    target=target_id,
+                    label=edge_type,
+                    hop=hop_depth,
+                ))
+
+        nodes = list(nodes_map.values())
+        logger.info(f"   ✅ Graph: {len(nodes)} nodes, {len(links)} links (depth={depth})")
+
+        return {
+            "anchor_id": chunk_id,
+            "nodes": [n.model_dump() for n in nodes],
+            "links": [l.model_dump() for l in links],
+            "total_nodes": len(nodes),
+            "total_links": len(links),
+        }
+
+    except Exception as e:
+        logger.error(f"❌ Graph neighborhood error: {e}", exc_info=True)
+        return {
+            "anchor_id": chunk_id,
+            "nodes": [],
+            "links": [],
+            "total_nodes": 0,
+            "total_links": 0,
+            "error": "An internal error occurred retrieving the graph.",
+        }
         

make_pdf.py

@@ -0,0 +1,22 @@
+import asyncio
+from playwright.async_api import async_playwright
+
+async def make_pdf():
+    async with async_playwright() as p:
+        browser = await p.chromium.launch(headless=True, args=["--no-sandbox"])
+        page = await browser.new_page()
+        content = """
+        <h1>Gazette of India - Extraordinary</h1>
+        <h2>Part II - Section 3</h2>
+        <p><strong>Agricultural Subsidy Amendment 2026</strong></p>
+        <p>This is a real, officially ingested document to test the GovBridge system.</p>
+        <p>The Ministry of Agriculture has allocated additional funding for memory and compute infrastructure in rural farming cooperatives.</p>
+        <p>By implementing this scheme, farmers will receive unprecedented access to digital agricultural markets.</p>
+        <p>Clause 4: The state of Rajasthan shall be the primary testing ground for this new agricultural framework.</p>
+        """
+        await page.set_content(content)
+        await page.pdf(path="test_gazette.pdf")
+        await browser.close()
+
+if __name__ == "__main__":
+    asyncio.run(make_pdf())

server.log

@@ -0,0 +1,8 @@
+INFO:     Started server process [61554]
+INFO:     Waiting for application startup.
+INFO:     Application startup complete.
+INFO:     Uvicorn running on http://127.0.0.1:8888 (Press CTRL+C to quit)
+INFO:     127.0.0.1:43166 - "POST /api/kernels HTTP/1.1" 404 Not Found
+INFO:     127.0.0.1:52406 - "POST /api/kernels HTTP/1.1" 404 Not Found
+INFO:     127.0.0.1:60060 - "GET / HTTP/1.1" 404 Not Found
+Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads.

test_api_client.py

@@ -0,0 +1,29 @@
+import sys
+import os
+import json
+from fastapi.testclient import TestClient
+
+# Must run from inside gov_backend directory for imports to work
+sys.path.insert(0, os.path.abspath("."))
+from api import app
+
+client = TestClient(app)
+
+print("--- Testing /health ---")
+response = client.get("/health")
+print(response.json())
+
+print("\n--- Testing /api/gazette/search ---")
+payload = {
+    "query": "solar power tax exemption",
+    "limit": 5
+}
+response = client.post("/api/gazette/search", json=payload)
+data = response.json()
+
+print(json.dumps(data, indent=2))
+
+if data.get("results"):
+    print("\nSUCCESS: Hybrid search pipeline integrated correctly with learned weights!")
+else:
+    print("\nNo results found, but endpoint responded correctly.")