feat: add /embed endpoint v1.4.0

app.py

@@ -4,6 +4,7 @@ from transformers import (
     pipeline,
     AutoTokenizer,
 )
+from sentence_transformers import SentenceTransformer
 import torch
 import re
 import hashlib
@@ -19,7 +20,7 @@ logger = logging.getLogger(__name__)
 app = FastAPI(
     title="CivicPulse NLP API",
     description="NLP microservice for Civic Pulse Engine — Municipality of Pulilan, Bulacan",
-    version="1.3.0",
+    version="1.4.0",
 )
 
 # ═══════════════════════════════════════════════════════════════════════════
@@ -61,6 +62,11 @@ logger.info("Loading spaCy NER model: en_core_web_sm...")
 nlp_spacy = spacy.load("en_core_web_sm")
 logger.info("spaCy NER model loaded.")
 
+# ── 5. Embedding Model (for RAG pipeline) ──────────────────────────────────
+logger.info("Loading embedding model: meedan/paraphrase-filipino-mpnet-base-v2...")
+embedding_model = SentenceTransformer("meedan/paraphrase-filipino-mpnet-base-v2")
+logger.info("Embedding model loaded.")
+
 # ═══════════════════════════════════════════════════════════════════════════
 # CONSTANTS
 # ═══════════════════════════════════════════════════════════════════════════
@@ -188,6 +194,13 @@ class PreprocessResponse(BaseModel):
     language: str
     is_spam: bool
 
+class EmbedRequest(BaseModel):
+    text: str
+
+class EmbedResponse(BaseModel):
+    embedding: list[float]
+    dimensions: int
+
 class SentimentRequest(BaseModel):
     text: str
     comment_id: str | None = None
@@ -242,9 +255,9 @@ class TopicResponse(BaseModel):
 def root():
     return {
         "service": "CivicPulse NLP API",
-        "version": "1.3.0",
+        "version": "1.4.0",
         "status": "running",
-        "endpoints": ["/preprocess", "/sentiment", "/claim-detection", "/topic-classification", "/health"],
+        "endpoints": ["/preprocess", "/embed", "/sentiment", "/claim-detection", "/topic-classification", "/health"],
     }
 
 @app.get("/health")
@@ -252,7 +265,7 @@ def health():
     """Keep-alive endpoint. GitHub Actions pings this every 25 min."""
     return {
         "status": "ok",
-        "models_loaded": ["sentiment", "claim-tokenizer", "topic-classification", "spacy-ner"],
+        "models_loaded": ["sentiment", "claim-tokenizer", "topic-classification", "spacy-ner", "embedding"],
         "claim_detection_mode": CLAIM_DETECTION_MODE,
     }
 
@@ -290,6 +303,25 @@ def preprocess(request: PreprocessRequest):
         is_spam=spam,
     )
 
+# ── Embed ──────────────────────────────────────────────────────────────────
+@app.post("/embed", response_model=EmbedResponse)
+def embed(request: EmbedRequest):
+    """
+    Generate a 768-dimension dense vector embedding for a text string.
+    Used for: (1) embedding lgu_documents into pgvector, and
+              (2) embedding flagged claims for cosine similarity search.
+    Model: meedan/paraphrase-filipino-mpnet-base-v2
+    """
+    text = request.text.strip()
+    if not text:
+        raise HTTPException(status_code=422, detail="text field cannot be empty.")
+    try:
+        vector = embedding_model.encode(text, normalize_embeddings=True).tolist()
+        return EmbedResponse(embedding=vector, dimensions=len(vector))
+    except Exception as e:
+        logger.error(f"Embedding error: {e}")
+        raise HTTPException(status_code=500, detail=f"Embedding error: {str(e)}")
+
 # ── Sentiment ──────────────────────────────────────────────────────────────
 @app.post("/sentiment", response_model=SentimentResponse)
 def analyze_sentiment(request: SentimentRequest):

requirements.txt

@@ -6,4 +6,5 @@ pydantic==2.7.0
 sentencepiece==0.2.0
 scipy==1.13.0
 spacy==3.8.11
-langdetect==1.0.9
+langdetect==1.0.9
+sentence-transformers==3.0.1