feat: sprint 17 - linguistic hegemony & vector synchronization

Dockerfile

@@ -1,28 +1,59 @@
+# ═══════════════════════════════════════════════════════════════
+# GovBridge India — Production Dockerfile
+# Target: Hugging Face Spaces (CPU Basic: 2 vCPU, 16GB RAM)
+# Constraint: ZERO CUDA binaries. CPU-only PyTorch wheels.
+# ═══════════════════════════════════════════════════════════════
+
 FROM python:3.11-slim
 
 WORKDIR /app
 
-# Layer 1: Install system dependencies
-RUN apt-get update && apt-get install -y \
+# ── Layer 1: OS Dependencies (ROOT) ──────────────────────────
+# libgomp1: Required for OpenMP multi-threading (IndicTrans2 + PyTorch CPU)
+# build-essential: Required for Cython compilation (IndicTransToolkit)
+# git: Required for pip install from GitHub repos
+RUN apt-get update && apt-get install -y --no-install-recommends \
     build-essential \
     git \
+    libgomp1 \
     && rm -rf /var/lib/apt/lists/*
 
-# Layer 2: Install Python dependencies
+# ── Layer 2: CPU-Only PyTorch Installation ───────────────────
+# CRITICAL: This MUST run BEFORE requirements.txt to prevent
+# pip from resolving torch from the default PyPI index (which
+# downloads 1.2GB+ of CUDA/NVIDIA binaries and exhausts disk).
+RUN pip install --no-cache-dir \
+    torch>=2.2.0 \
+    --extra-index-url https://download.pytorch.org/whl/cpu
+
+# ── Layer 3: Python Dependencies ─────────────────────────────
 COPY requirements.txt .
 RUN pip install --no-cache-dir -r requirements.txt
 
-# Layer 3: Pre-cache the EXACT models used in api.py at build time
-# This prevents runtime downloads and eliminates cold-start latency
+# ── Layer 4: Pre-cache ALL ML models at build time ───────────
+# This eliminates cold-start latency on first request.
+# Models are cached to /app/models (persisted across container restarts).
 RUN python -c "\
 from sentence_transformers import SentenceTransformer, CrossEncoder; \
 SentenceTransformer('nomic-ai/nomic-embed-text-v1', cache_folder='/app/models', trust_remote_code=True); \
-print('✓ Nomic Embedding model cached'); \
+print('✓ Nomic Embedding model (768-dim) cached'); \
 CrossEncoder('cross-encoder/ettin-reranker-68m-v1', max_length=512); \
 print('✓ Ettin Reranker cached')"
 
-# Layer 4: Copy application code last
-# This ensures code changes don't invalidate the model cache layer
+# ── Layer 5: Pre-cache IndicTrans2 translation model ─────────
+# Cached separately because it requires trust_remote_code and
+# downloads ~800MB of weights. Failure here must NOT block deployment.
+RUN python -c "\
+try: \
+    from transformers import AutoTokenizer, AutoModelForSeq2SeqLM; \
+    AutoTokenizer.from_pretrained('ai4bharat/indictrans2-en-indic-dist-200M', trust_remote_code=True, cache_dir='/app/models/indictrans2'); \
+    AutoModelForSeq2SeqLM.from_pretrained('ai4bharat/indictrans2-en-indic-dist-200M', trust_remote_code=True, cache_dir='/app/models/indictrans2'); \
+    print('✓ IndicTrans2 translation model cached'); \
+except Exception as e: \
+    print(f'⚠️ IndicTrans2 cache failed (non-blocking): {e}')"
+
+# ── Layer 6: Copy application code LAST ──────────────────────
+# Code changes don't invalidate the expensive model cache layers.
 COPY . .
 
 EXPOSE 7860

api.py

@@ -20,9 +20,9 @@ from config import settings
 
 # --- SECURE KEYS ---
 SUPABASE_URL = settings.SUPABASE_URL
-SUPABASE_KEY = settings.SUPABASE_KEY 
-GROQ_API_KEY = os.environ.get("GROQ_API_KEY")
-ADMIN_SECRET = os.environ.get("ADMIN_SECRET", "change-this-in-production")
+SUPABASE_KEY = settings.SUPABASE_KEY
+GROQ_API_KEY = settings.GROQ_API_KEY
+ADMIN_SECRET = settings.ADMIN_SECRET
 
 supabase: Client = create_client(SUPABASE_URL, SUPABASE_KEY)
 groq_client = Groq(api_key=GROQ_API_KEY)
@@ -197,8 +197,9 @@ async def ingest_document(request: IngestRequest, admin_key: str = ""):
     prefixed_chunks = [f"search_document: {c}" for c in chunks]
     embeddings = model.encode(prefixed_chunks, normalize_embeddings=True, batch_size=32, show_progress_bar=False).tolist()
     rows = []
+    dual_write = settings.DUAL_WRITE_EMBEDDINGS.lower() == "true"
     for i, (chunk, embedding) in enumerate(zip(chunks, embeddings)):
-        rows.append({
+        row = {
             "chunk_index": i,
             "chunk_text": chunk,
             "scheme_title": request.title,
@@ -208,7 +209,11 @@ async def ingest_document(request: IngestRequest, admin_key: str = ""):
             "doc_type": request.doc_type,
             "embedding": embedding,
             "content_hash": doc_hash
-        })
+        }
+        # During expand-contract migration: write to BOTH columns
+        if dual_write:
+            row["embedding_v2"] = embedding
+        rows.append(row)
     supabase.table("document_chunks").upsert(rows, on_conflict="content_hash,chunk_index").execute()
     return {"status": "success", "title": request.title, "chunks_created": len(chunks), "doc_hash": doc_hash}
 

bhashini.py

@@ -88,7 +88,7 @@ def local_indictrans_translate(text: str, src: str, tgt: str) -> str:
             return_attention_mask=True
         )
 
-        with torch.no_grad():
+        with torch.inference_mode():
             generated_tokens = _indictrans_model.generate(
                 **inputs,
                 num_beams=4,
@@ -120,7 +120,7 @@ def local_indictrans_translate(text: str, src: str, tgt: str) -> str:
             truncation=True,
             max_length=512
         )
-        with torch.no_grad():
+        with torch.inference_mode():
             outputs = _indictrans_model.generate(
                 **inputs,
                 tgt_lang=tgt_code,

config.py

@@ -1,12 +1,48 @@
+"""
+GovBridge India — Centralized Configuration
+All environment variables are declared here with Pydantic validation.
+Usage: from config import settings
+"""
 from pydantic_settings import BaseSettings, SettingsConfigDict
 from pydantic import Field
 from typing import Optional
-import os
+
 
 class Settings(BaseSettings):
+    # ── Database ─────────────────────────────────────────
     SUPABASE_URL: str = Field(default="")
     SUPABASE_KEY: str = Field(default="")
 
-    model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8", extra="ignore")
+    # ── LLM ──────────────────────────────────────────────
+    GROQ_API_KEY: str = Field(default="")
+
+    # ── Admin ────────────────────────────────────────────
+    ADMIN_SECRET: str = Field(default="change-this-in-production")
+
+    # ── Translation (Bhashini) ───────────────────────────
+    BHASHINI_USER_ID: Optional[str] = Field(default=None)
+    BHASHINI_API_KEY: Optional[str] = Field(default=None)
+
+    # ── WhatsApp ─────────────────────────────────────────
+    WHATSAPP_TOKEN: Optional[str] = Field(default=None)
+    WHATSAPP_PHONE_NUMBER_ID: Optional[str] = Field(default=None)
+    WHATSAPP_VERIFY_TOKEN: Optional[str] = Field(default=None)
+    WHATSAPP_APP_SECRET: Optional[str] = Field(default=None)
+
+    # ── Feature Flags ────────────────────────────────────
+    ENVIRONMENT: str = Field(default="development")
+
+    # ── Migration Flag ───────────────────────────────────
+    # Set to "true" during the expand-contract migration window.
+    # When true, ingest writes to BOTH embedding AND embedding_v2.
+    # After migration 009 is complete, set back to "false".
+    DUAL_WRITE_EMBEDDINGS: str = Field(default="false")
+
+    model_config = SettingsConfigDict(
+        env_file=".env",
+        env_file_encoding="utf-8",
+        extra="ignore"
+    )
+
 
 settings = Settings()

requirements.txt

@@ -1,22 +1,43 @@
+# ═══════════════════════════════════════════
+# GovBridge India — Python Dependencies
+# Target: Hugging Face Spaces (CPU Basic: 2 vCPU, 16GB RAM)
+# CRITICAL: All ML wheels MUST be CPU-only to prevent disk exhaustion
+# ═══════════════════════════════════════════
+
+# --- Core Web Framework ---
 fastapi
 uvicorn[standard]
-torch>=2.2.0
+
+# --- ML / AI (CPU-ONLY WHEELS) ---
+# CRITICAL: --extra-index-url in Dockerfile forces CPU torch
+# DO NOT add torch here — it is installed separately in Dockerfile Layer 2
 sentence-transformers>=2.7.0
 einops
 huggingface-hub
+
+# --- Translation Pipeline ---
+# CRITICAL PIN: transformers==4.40.2 is the exact temporal anchor
+# that supports IndicTrans2 WITHOUT the ONNX removal crash.
+# Versions >=4.41.0 removed transformers.onnx, causing ImportError.
+transformers==4.40.2
+sentencepiece>=0.2.0
+sacremoses>=0.1.1
+IndicTransToolkit
+
+# --- LLM Inference ---
 groq
+
+# --- Database ---
 supabase
+
+# --- Rate Limiting (in-process, for local dev) ---
 slowapi
-openfisca-core
-transformers>=4.48.0
-sentencepiece>=0.2.0
-sacremoses>=0.1.1
+
+# --- HTTP Client ---
 httpx
+
+# --- Data Processing ---
 feedparser
-IndicTransToolkit
-onnx
-onnxruntime
-optimum
-pydantic-settings
 tenacity
 pydantic
+pydantic-settings

scripts/backfill_768.py

@@ -0,0 +1,162 @@
+"""
+GovBridge India — Phase 2: Backfill 768-dim Embeddings
+Sprint 17: Zero-Downtime Expand-Contract Migration
+
+PREREQUISITES:
+  1. Migration 007 has been executed (embedding_v2 column exists)
+  2. Nomic model is available (pip install sentence-transformers einops)
+
+USAGE:
+  cd /workspaces/govbridge
+  python3 gov_backend/scripts/backfill_768.py
+
+BEHAVIOR:
+  - Reads ALL rows from document_chunks where embedding_v2 IS NULL
+  - Generates 768-dim embeddings using nomic-embed-text-v1
+  - UPSERTs in batches of 50 rows using keyset pagination
+  - Also backfills query_cache.query_embedding_v2
+  - Safe to re-run (idempotent via IS NULL filter)
+"""
+import os
+import sys
+import time
+import torch
+
+# Ensure gov_backend is on the Python path
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+from config import settings
+
+from supabase import create_client
+from sentence_transformers import SentenceTransformer
+
+# ── Configuration ────────────────────────────────────────────
+BATCH_SIZE = 50          # Rows per batch (free-tier Supabase safe)
+SLEEP_BETWEEN = 0.5      # Seconds between batches (prevent rate limit)
+MAX_ROWS = 5000          # Safety ceiling per run
+
+def main():
+    supabase_url = settings.SUPABASE_URL
+    supabase_key = settings.SUPABASE_KEY
+
+    if not supabase_url or not supabase_key:
+        print("❌ FATAL: Missing SUPABASE_URL or SUPABASE_KEY in environment")
+        sys.exit(1)
+
+    print("⏳ Loading Nomic Embedding Model (768-dim)...")
+    model = SentenceTransformer(
+        'nomic-ai/nomic-embed-text-v1',
+        trust_remote_code=True
+    )
+    print("✅ Nomic model loaded")
+
+    supabase = create_client(supabase_url, supabase_key)
+
+    # ── Phase 2A: Backfill document_chunks.embedding_v2 ──────
+    print("\n═══ PHASE 2A: Backfilling document_chunks.embedding_v2 ═══")
+    
+    result = supabase.table('document_chunks') \
+        .select('id, chunk_text, scheme_title') \
+        .is_('embedding_v2', 'null') \
+        .limit(MAX_ROWS) \
+        .execute()
+
+    chunks = result.data or []
+    total = len(chunks)
+    
+    if total == 0:
+        print("🎉 document_chunks: All rows already have embedding_v2")
+    else:
+        print(f"Found {total} chunks to backfill")
+        
+        for i in range(0, total, BATCH_SIZE):
+            batch = chunks[i:i + BATCH_SIZE]
+            batch_num = (i // BATCH_SIZE) + 1
+            total_batches = (total + BATCH_SIZE - 1) // BATCH_SIZE
+
+            # Nomic requires "search_document:" prefix for document embeddings
+            texts = [
+                f"search_document: {c.get('chunk_text', '') or c.get('scheme_title', '')}"
+                for c in batch
+            ]
+
+            print(f"🔄 Batch {batch_num}/{total_batches} ({len(batch)} rows)...")
+            
+            with torch.inference_mode():
+                embeddings = model.encode(
+                    texts,
+                    normalize_embeddings=True,
+                    show_progress_bar=False,
+                    batch_size=32
+                ).tolist()
+
+            # UPSERT each row's embedding_v2
+            for chunk, embedding in zip(batch, embeddings):
+                supabase.table('document_chunks') \
+                    .update({'embedding_v2': embedding}) \
+                    .eq('id', chunk['id']) \
+                    .execute()
+
+            print(f"✅ Batch {batch_num}/{total_batches} complete")
+            time.sleep(SLEEP_BETWEEN)
+
+        print(f"\n🎉 document_chunks backfill complete: {total} rows updated")
+
+    # ── Phase 2B: Backfill query_cache.query_embedding_v2 ────
+    print("\n═══ PHASE 2B: Backfilling query_cache.query_embedding_v2 ═══")
+    
+    cache_result = supabase.table('query_cache') \
+        .select('id, query_text') \
+        .is_('query_embedding_v2', 'null') \
+        .limit(MAX_ROWS) \
+        .execute()
+
+    cache_rows = cache_result.data or []
+    cache_total = len(cache_rows)
+
+    if cache_total == 0:
+        print("🎉 query_cache: All rows already have query_embedding_v2")
+    else:
+        print(f"Found {cache_total} cache entries to backfill")
+        
+        for i in range(0, cache_total, BATCH_SIZE):
+            batch = cache_rows[i:i + BATCH_SIZE]
+            batch_num = (i // BATCH_SIZE) + 1
+
+            # Nomic requires "search_query:" prefix for query embeddings
+            texts = [
+                f"search_query: {c.get('query_text', '')}"
+                for c in batch
+            ]
+
+            with torch.inference_mode():
+                embeddings = model.encode(
+                    texts,
+                    normalize_embeddings=True,
+                    show_progress_bar=False,
+                    batch_size=32
+                ).tolist()
+
+            for row, embedding in zip(batch, embeddings):
+                supabase.table('query_cache') \
+                    .update({'query_embedding_v2': embedding}) \
+                    .eq('id', row['id']) \
+                    .execute()
+
+            time.sleep(SLEEP_BETWEEN)
+
+        print(f"\n🎉 query_cache backfill complete: {cache_total} rows updated")
+
+    # ── Summary ──────────────────────────────────────────────
+    print("\n" + "═" * 60)
+    print("MIGRATION PHASE 2 COMPLETE")
+    print(f"  document_chunks: {total} rows backfilled")
+    print(f"  query_cache: {cache_total} rows backfilled")
+    print("═" * 60)
+    print("\nNEXT STEPS:")
+    print("  1. Run migration 008 (concurrent index creation)")
+    print("  2. Verify indexes are VALID")
+    print("  3. Run migration 009 (contract — column swap)")
+
+
+if __name__ == '__main__':
+    main()