Update: backend v3.2 — privacy pipeline, consensus, new scripts

app.py

@@ -1,98 +1,114 @@
-"""
-MediRAG Backend - FastAPI only (No Gradio)
-React frontend on Vercel, this is just the API backend
-"""
-
-import os
-import sys
-import subprocess
-import logging
-import requests
-
-# Configure logging
-logging.basicConfig(level=logging.INFO)
-logger = logging.getLogger(__name__)
-
-# Set cache directories for Hugging Face
-os.environ["TRANSFORMERS_CACHE"] = "/tmp/transformers_cache"
-os.environ["HF_HOME"] = "/tmp/hf_home"
-os.environ["TORCH_HOME"] = "/tmp/torch_cache"
-
-# Add src to path
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), "src"))
-
-# Install spaCy model if not present (optional — server starts without it)
-try:
-    import spacy
-    try:
-        spacy.load("en_core_sci_lg")
-        logger.info("spaCy model en_core_sci_lg loaded.")
-    except OSError:
-        # Try installing the model at runtime
-        try:
-            logger.info("Attempting to install scispacy model en_core_sci_lg...")
-            subprocess.run([
-                sys.executable, "-m", "pip", "install", "--quiet",
-                "https://s3-us-west-2.amazonaws.com/ai2-s2-scispacy/releases/v0.5.4/en_core_sci_lg-0.5.4.tar.gz"
-            ], check=True, timeout=300)
-            spacy.load("en_core_sci_lg")
-            logger.info("spaCy model installed and loaded.")
-        except Exception as model_err:
-            logger.warning(f"Could not install spaCy model: {model_err}. NER features will be limited.")
-except ImportError:
-    logger.warning("spacy/scispacy not installed. NER features will be limited but server will still start.")
-
-# Download datasets using huggingface_hub
-from huggingface_hub import hf_hub_download
-
-# Check and download index and data files
-data_dir = os.path.join(os.path.dirname(__file__), "data")
-index_dir = os.path.join(data_dir, "index")
-os.makedirs(index_dir, exist_ok=True)
-
-faiss_path = os.path.join(index_dir, "faiss.index")
-metadata_path = os.path.join(index_dir, "metadata_store.pkl")
-bm25_path = os.path.join(index_dir, "bm25_cache.pkl")
-vocab_path = os.path.join(data_dir, "drugbank vocabulary.csv")
-rxnorm_path = os.path.join(data_dir, "rxnorm_cache.csv")
-
-def download_dataset_files():
-    """Download FAISS index and other core data from Hugging Face Dataset"""
-    repo_id = "joytheslothh/MediRAG-Index-Data"
-    token = os.environ.get("HF_TOKEN")
-    if not token:
-        logger.warning("HF_TOKEN environment variable is not set. Dataset download might fail if repo is private.")
-
-    try:
-        if not os.path.exists(faiss_path):
-            logger.info("Downloading faiss.index from HF dataset...")
-            hf_hub_download(repo_id=repo_id, filename="index/faiss.index", local_dir=data_dir, repo_type="dataset", token=token)
-        if not os.path.exists(metadata_path):
-            logger.info("Downloading metadata_store.pkl from HF dataset...")
-            hf_hub_download(repo_id=repo_id, filename="index/metadata_store.pkl", local_dir=data_dir, repo_type="dataset", token=token)
-        if not os.path.exists(bm25_path):
-            logger.info("Downloading bm25_cache.pkl from HF dataset...")
-            hf_hub_download(repo_id=repo_id, filename="index/bm25_cache.pkl", local_dir=data_dir, repo_type="dataset", token=token)
-        if not os.path.exists(vocab_path):
-            logger.info("Downloading drugbank vocabulary.csv from HF dataset...")
-            hf_hub_download(repo_id=repo_id, filename="drugbank vocabulary.csv", local_dir=data_dir, repo_type="dataset", token=token)
-        if not os.path.exists(rxnorm_path):
-            logger.info("Downloading rxnorm_cache.csv from HF dataset...")
-            hf_hub_download(repo_id=repo_id, filename="rxnorm_cache.csv", local_dir=data_dir, repo_type="dataset", token=token)
-    except Exception as e:
-        logger.error(f"Failed to download dataset files: {e}")
-        logger.warning("Backend may not start correctly or queries may fail.")
-
-# Trigger download at startup
-download_dataset_files()
-
-# Import FastAPI app - this is the main backend for React frontend
-from src.api.main import app
-
-if __name__ == "__main__":
-    import uvicorn
-    # Get port from environment (Hugging Face uses 7860)
-    port = int(os.environ.get("PORT", 7860))
-    
-    logger.info("Starting FastAPI backend on port {}".format(port))
-    uvicorn.run(app, host="0.0.0.0", port=port)
+"""
+MediRAG Backend - FastAPI only (No Gradio)
+React frontend on Vercel, this is just the API backend
+"""
+
+import os
+import sys
+import subprocess
+import logging
+import requests
+
+# Configure logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# Set cache directories for Hugging Face
+os.environ["TRANSFORMERS_CACHE"] = "/tmp/transformers_cache"
+os.environ["HF_HOME"] = "/tmp/hf_home"
+os.environ["TORCH_HOME"] = "/tmp/torch_cache"
+
+# Add src to path
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "src"))
+
+# Install spaCy model if not present (optional — server starts without it)
+try:
+    import spacy
+    try:
+        spacy.load("en_core_sci_lg")
+        logger.info("spaCy model en_core_sci_lg loaded.")
+    except OSError:
+        # Try installing the model at runtime
+        try:
+            logger.info("Attempting to install scispacy model en_core_sci_lg...")
+            subprocess.run([
+                sys.executable, "-m", "pip", "install", "--quiet",
+                "https://s3-us-west-2.amazonaws.com/ai2-s2-scispacy/releases/v0.5.4/en_core_sci_lg-0.5.4.tar.gz"
+            ], check=True, timeout=300)
+            spacy.load("en_core_sci_lg")
+            logger.info("spaCy model installed and loaded.")
+        except Exception as model_err:
+            logger.warning(f"Could not install spaCy model: {model_err}. NER features will be limited.")
+except ImportError:
+    logger.warning("spacy/scispacy not installed. NER features will be limited but server will still start.")
+
+# Download datasets using huggingface_hub
+from huggingface_hub import hf_hub_download
+import yaml
+from pathlib import Path
+
+# Check if config_local.yaml exists or USE_LOCAL_DATASET is set to skip HF downloads
+config_path = os.environ.get("MEDIRAG_CONFIG", "config_local.yaml" if Path("config_local.yaml").exists() else "config.yaml")
+try:
+    with open(config_path, "r", encoding="utf-8") as f:
+        config_data = yaml.safe_load(f)
+except Exception:
+    config_data = {}
+
+use_local_dataset = config_data.get("retrieval", {}).get("use_local_dataset", False) or os.environ.get("USE_LOCAL_DATASET", "false").lower() == "true"
+
+# Check and download index and data files
+data_dir = os.path.join(os.path.dirname(__file__), "data")
+index_dir = os.path.join(data_dir, "index")
+os.makedirs(index_dir, exist_ok=True)
+
+faiss_path = os.path.join(index_dir, "faiss.index")
+metadata_path = os.path.join(index_dir, "metadata_store.pkl")
+bm25_path = os.path.join(index_dir, "bm25_cache.pkl")
+vocab_path = os.path.join(data_dir, "drugbank vocabulary.csv")
+rxnorm_path = os.path.join(data_dir, "rxnorm_cache.csv")
+
+def download_dataset_files():
+    """Download FAISS index and other core data from Hugging Face Dataset"""
+    if use_local_dataset:
+        logger.info("[LOCAL MODE] Bypassing Hugging Face repository download. Relying on local datasets in data/index/.")
+        return
+
+    repo_id = "joytheslothh/MediRAG-Index-Data"
+    token = os.environ.get("HF_TOKEN")
+    if not token:
+        logger.warning("HF_TOKEN environment variable is not set. Dataset download might fail if repo is private.")
+
+    try:
+        if not os.path.exists(faiss_path):
+            logger.info("Downloading faiss.index from HF dataset...")
+            hf_hub_download(repo_id=repo_id, filename="index/faiss.index", local_dir=data_dir, repo_type="dataset", token=token)
+        if not os.path.exists(metadata_path):
+            logger.info("Downloading metadata_store.pkl from HF dataset...")
+            hf_hub_download(repo_id=repo_id, filename="index/metadata_store.pkl", local_dir=data_dir, repo_type="dataset", token=token)
+        if not os.path.exists(bm25_path):
+            logger.info("Downloading bm25_cache.pkl from HF dataset...")
+            hf_hub_download(repo_id=repo_id, filename="index/bm25_cache.pkl", local_dir=data_dir, repo_type="dataset", token=token)
+        if not os.path.exists(vocab_path):
+            logger.info("Downloading drugbank vocabulary.csv from HF dataset...")
+            hf_hub_download(repo_id=repo_id, filename="drugbank vocabulary.csv", local_dir=data_dir, repo_type="dataset", token=token)
+        if not os.path.exists(rxnorm_path):
+            logger.info("Downloading rxnorm_cache.csv from HF dataset...")
+            hf_hub_download(repo_id=repo_id, filename="rxnorm_cache.csv", local_dir=data_dir, repo_type="dataset", token=token)
+    except Exception as e:
+        logger.error(f"Failed to download dataset files: {e}")
+        logger.warning("Backend may not start correctly or queries may fail.")
+
+# Trigger download at startup
+download_dataset_files()
+
+# Import FastAPI app - this is the main backend for React frontend
+from src.api.main import app
+
+if __name__ == "__main__":
+    import uvicorn
+    # Get port from environment (Hugging Face uses 7860)
+    port = int(os.environ.get("PORT", 7860))
+    
+    logger.info("Starting FastAPI backend on port {}".format(port))
+    uvicorn.run(app, host="0.0.0.0", port=port)

src/__init__.py

@@ -15,10 +15,11 @@ def _setup_logging() -> None:
     log_format = "%(asctime)s [%(levelname)s] %(name)s: %(message)s"
     log_file = "logs/medirag.log"
 
-    # Try to load level from config.yaml
+    # Try to load level from config_local.yaml or config.yaml
     try:
         import yaml
-        with open("config.yaml", "r") as f:
+        config_path = os.environ.get("MEDIRAG_CONFIG", "config_local.yaml" if os.path.exists("config_local.yaml") else "config.yaml")
+        with open(config_path, "r", encoding="utf-8") as f:
             cfg = yaml.safe_load(f)
         level_str = cfg.get("logging", {}).get("level", "INFO")
         log_level = getattr(logging, level_str.upper(), logging.INFO)

src/api/main.py

@@ -32,6 +32,7 @@ from datetime import datetime
 from fastapi import FastAPI, HTTPException, File, UploadFile
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import RedirectResponse
+from pydantic import BaseModel
 
 import threading
 from src.api.schemas import (
@@ -54,7 +55,8 @@ from src.pipeline.retriever import Retriever
 # Logging
 # ---------------------------------------------------------------------------
 try:
-    _cfg = yaml.safe_load(Path("config.yaml").read_text())
+    _config_path = os.environ.get("MEDIRAG_CONFIG", "config_local.yaml" if Path("config_local.yaml").exists() else "config.yaml")
+    _cfg = yaml.safe_load(Path(_config_path).read_text())
     _log_level = _cfg.get("logging", {}).get("level", "INFO")
     _ollama_base = _cfg.get("llm", {}).get("base_url", "http://localhost:11434")
     _api_cfg = _cfg.get("api", {})
@@ -354,6 +356,40 @@ def evaluate(req: EvaluateRequest) -> EvaluateResponse:
     )
 
 
+# ---------------------------------------------------------------------------
+# POST /translate  — lightweight Hinglish to English translation route
+# ---------------------------------------------------------------------------
+class TranslateRequest(BaseModel):
+    text: str
+    llm_provider: Optional[str] = None
+    llm_api_key: Optional[str] = None
+    llm_model: Optional[str] = None
+    ollama_url: Optional[str] = None
+
+class TranslateResponse(BaseModel):
+    translated_text: str
+
+@app.post("/translate", response_model=TranslateResponse, tags=["translation"])
+def translate(req: TranslateRequest) -> TranslateResponse:
+    llm_overrides = {}
+    if req.llm_provider:
+        llm_overrides["provider"] = req.llm_provider
+    if req.llm_api_key:
+        llm_overrides["api_key"] = req.llm_api_key
+    if req.llm_model:
+        llm_overrides["model"] = req.llm_model
+    if req.ollama_url:
+        llm_overrides["ollama_url"] = req.ollama_url
+
+    try:
+        from src.pipeline.generator import translate_hinglish_to_english
+        translated = translate_hinglish_to_english(req.text, _cfg, overrides=llm_overrides)
+        return TranslateResponse(translated_text=translated)
+    except Exception as exc:
+        logger.exception("Translation endpoint failed: %s", exc)
+        raise HTTPException(status_code=500, detail=str(exc))
+
+
 # ---------------------------------------------------------------------------
 # POST /query  — end-to-end: question → retrieve → generate → evaluate
 # ---------------------------------------------------------------------------
@@ -373,20 +409,120 @@ def query(req: QueryRequest) -> QueryResponse:
     import time as _time
     t_total = _time.perf_counter()
 
-    logger.info("POST /query — question=%r, top_k=%d", req.question[:80], req.top_k)
+    # Extract request overrides into a dict for translator + generator
+    llm_overrides = {}
+    if req.llm_provider:
+        llm_overrides["provider"] = req.llm_provider
+    if req.llm_api_key:
+        llm_overrides["api_key"] = req.llm_api_key
+    if req.llm_model:
+        llm_overrides["model"] = req.llm_model
+    if req.ollama_url:
+        llm_overrides["ollama_url"] = req.ollama_url
+    if req.system_prompt:
+        llm_overrides["system_prompt"] = req.system_prompt
+    if req.persona:
+        llm_overrides["persona"] = req.persona
+
+    original_hinglish = None
+    question_to_use = req.question
 
-    # Step 1: Retrieve
+    if req.translate_hinglish:
+        if req.original_hinglish_query:
+            original_hinglish = req.original_hinglish_query
+            question_to_use = req.question
+            logger.info("PRE-TRANSLATED AUDIT SUBMISSION: %r -> %r", original_hinglish, question_to_use)
+        else:
+            try:
+                from src.pipeline.generator import translate_hinglish_to_english
+                translated_q = translate_hinglish_to_english(req.question, _cfg, overrides=llm_overrides)
+                if translated_q.strip().lower() != req.question.strip().lower():
+                    original_hinglish = req.question
+                    question_to_use = translated_q
+                    logger.info("AUTO-TRANSLATED HINGLISH QUERY: %r -> %r", original_hinglish, question_to_use)
+            except Exception as exc:
+                logger.error("Hinglish translation module failed: %s", exc)
+
+    logger.info("POST /query — question=%r, processed_question=%r, top_k=%d", 
+                req.question[:50], question_to_use[:50], req.top_k)
+
+    # Safe Semantic Cache lookup
+    q_vec = None
+    from src.pipeline.semantic_cache import SafeSemanticCache
+    semantic_cache = SafeSemanticCache()
+    
+    # Check if retriever can encode
     retriever: Optional[Retriever] = getattr(app.state, "retriever", None)
     if retriever is None:
-        # Fallback: instantiate now (slower first call)
         try:
             retriever = Retriever(_cfg)
+            app.state.retriever = retriever
         except Exception as exc:
             raise HTTPException(status_code=503,
                 detail=f"Retriever unavailable: {exc}") from exc
 
     try:
-        raw_results = retriever.search(req.question, top_k=req.top_k)
+        retriever._load_model()
+        if retriever._model:
+            q_vec = retriever._model.encode(
+                [question_to_use.strip()],
+                normalize_embeddings=True,
+                convert_to_numpy=True,
+            )[0].astype(np.float32)
+            
+            cache_hit = semantic_cache.get(
+                query_emb=q_vec,
+                patient_allergies=req.patient_allergies or [],
+                department=req.department or "default",
+                overrides=llm_overrides
+            )
+            if cache_hit:
+                logger.info("SEMANTIC CACHE HIT: Returning safe cached response instantly.")
+                retrieved_chunks = [
+                    RetrievedChunk(
+                        chunk_id=c.get("chunk_id"),
+                        text=c.get("text"),
+                        source=c.get("source", ""),
+                        pub_type=c.get("pub_type", ""),
+                        pub_year=c.get("pub_year"),
+                        title=c.get("title", ""),
+                        similarity_score=c.get("similarity_score", 0.0)
+                    ) for c in cache_hit.get("retrieved_chunks", [])
+                ]
+                mr_dict = cache_hit.get("module_results", {})
+                return QueryResponse(
+                    question=cache_hit.get("question", question_to_use),
+                    generated_answer=cache_hit.get("generated_answer"),
+                    retrieved_chunks=retrieved_chunks,
+                    composite_score=cache_hit.get("composite_score", 1.0),
+                    hrs=cache_hit.get("hrs", 0),
+                    confidence_level=cache_hit.get("confidence_level", "UNKNOWN"),
+                    risk_band=cache_hit.get("risk_band", "UNKNOWN"),
+                    module_results=ModuleResults(
+                        faithfulness=_module_score(mr_dict, "faithfulness"),
+                        entity_verifier=_module_score(mr_dict, "entity_verifier"),
+                        source_credibility=_module_score(mr_dict, "source_credibility"),
+                        contradiction=_module_score(mr_dict, "contradiction"),
+                        ragas=_module_score(mr_dict, "ragas"),
+                    ),
+                    total_pipeline_ms=0,
+                    intervention_applied=cache_hit.get("intervention_applied", False),
+                    intervention_reason=cache_hit.get("intervention_reason"),
+                    original_answer=cache_hit.get("original_answer"),
+                    intervention_details=cache_hit.get("intervention_details"),
+                    consensus_results=cache_hit.get("consensus_results"),
+                    privacy_applied=cache_hit.get("privacy_applied", False),
+                    privacy_details=cache_hit.get("privacy_details"),
+                    original_hinglish_query=cache_hit.get("original_hinglish_query"),
+                    coverage_gap=cache_hit.get("coverage_gap", False),
+                    coverage_gap_details=cache_hit.get("coverage_gap_details"),
+                )
+    except Exception as exc:
+        logger.error("Failed semantic cache retrieval lookup: %s", exc)
+
+    # Step 1: Retrieve
+    try:
+        raw_results = retriever.search(question_to_use, top_k=req.top_k)
     except FileNotFoundError as exc:
         raise HTTPException(status_code=503,
             detail=f"FAISS index not found: {exc}") from exc
@@ -429,32 +565,16 @@ def query(req: QueryRequest) -> QueryResponse:
     top_faiss_cosine = (
         raw_results[0][1].get("_top_faiss_cosine", 0.0) if raw_results else 0.0
     )
-
-    # Convert request overrides into a dict for generator
-    llm_overrides = {}
-    if req.llm_provider:
-        llm_overrides["provider"] = req.llm_provider
-    if req.llm_api_key:
-        llm_overrides["api_key"] = req.llm_api_key
-    if req.llm_model:
-        llm_overrides["model"] = req.llm_model
-    if req.ollama_url:
-        llm_overrides["ollama_url"] = req.ollama_url
-    if req.system_prompt:
-        llm_overrides["system_prompt"] = req.system_prompt
-    if req.persona:
-        llm_overrides["persona"] = req.persona
-
     # =========================================================================
     # Step 2a: PRIVACY SHIELD — MediRAG redacts PHI (Option 1)
     # =========================================================================
     p_mapping = {}
     privacy_applied = False
-    question_to_gen = req.question
+    question_to_gen = question_to_use
     
     if req.use_privacy_shield:
         from src.pipeline.privacy import shield
-        question_to_gen, p_mapping = shield.redact(req.question)
+        question_to_gen, p_mapping = shield.redact(question_to_use)
         if p_mapping:
             privacy_applied = True
             logger.info("PRIVACY INTERVENTION: Redacted %d items from question.", len(p_mapping))
@@ -489,7 +609,7 @@ def query(req: QueryRequest) -> QueryResponse:
             providers.append("ollama") # fallback to local if no second key
 
         logger.info("Running Consensus Layer with %s", providers)
-        consensus_results = run_consensus_check(req.question, context_chunks, _cfg, providers=providers)
+        consensus_results = run_consensus_check(question_to_use, context_chunks, _cfg, providers=providers)
         
         # If consensus finds a safer merged answer, we promote it
         # and update the primary answer for the evaluation loop
@@ -505,7 +625,7 @@ def query(req: QueryRequest) -> QueryResponse:
     # Step 3: Evaluate
     try:
         eval_result = run_evaluation(
-            question=req.question,
+            question=question_to_use,
             answer=answer,
             context_chunks=context_chunks,
             run_ragas=req.run_ragas,
@@ -514,7 +634,7 @@ def query(req: QueryRequest) -> QueryResponse:
     except Exception as exc:
         logger.exception("Evaluation failed: %s", exc)
         try:
-            log_audit("query", req.question, answer, 100, "EVAL_ERROR", 0.0,
+            log_audit("query", question_to_use, answer, 100, "EVAL_ERROR", 0.0,
                       int((_time.perf_counter() - t_total) * 1000),
                       False, {"error": str(exc), "error_type": "evaluation_failure"})
         except Exception:
@@ -538,6 +658,50 @@ def query(req: QueryRequest) -> QueryResponse:
     original_answer = None
     intervention_details = None
 
+    # Dynamic Department-specific Safety Policies & Patient Allergy Gates
+    hrs_block_threshold = 86
+    hrs_retry_threshold = 61
+    
+    if req.department:
+        dept_lower = req.department.lower()
+        if "pediatric" in dept_lower:
+            hrs_block_threshold = 50
+            hrs_retry_threshold = 20
+            logger.info("DEPARTMENT SAFETY TUNING (Pediatrics): Block >= 50, Regenerate >= 20")
+        elif "oncology" in dept_lower:
+            hrs_block_threshold = 55
+            hrs_retry_threshold = 25
+            logger.info("DEPARTMENT SAFETY TUNING (Oncology): Block >= 55, Regenerate >= 25")
+        elif "cardiology" in dept_lower:
+            hrs_block_threshold = 60
+            hrs_retry_threshold = 30
+            logger.info("DEPARTMENT SAFETY TUNING (Cardiology): Block >= 60, Regenerate >= 30")
+        elif "emergency" in dept_lower or "er" in dept_lower:
+            hrs_block_threshold = 70
+            hrs_retry_threshold = 50
+            logger.info("DEPARTMENT SAFETY TUNING (ER): Block >= 70, Regenerate >= 50")
+        elif "opd" in dept_lower:
+            hrs_block_threshold = 80
+            hrs_retry_threshold = 60
+            logger.info("DEPARTMENT SAFETY TUNING (OPD): Block >= 80, Regenerate >= 60")
+
+    # Custom Admin Console Override
+    if req.custom_hrs_limit is not None:
+        hrs_retry_threshold = req.custom_hrs_limit
+        hrs_block_threshold = min(95, req.custom_hrs_limit + (30 if req.custom_hrs_limit <= 30 else 20))
+        logger.info(f"HOSPITAL CONSOLE OVERRIDE: Block >= {hrs_block_threshold}, Regenerate >= {hrs_retry_threshold}")
+
+    # Patient Allergy Safety Interception
+    allergy_intercepted = False
+    allergen_matched = None
+    if req.patient_allergies:
+        text_to_scan = (question_to_use + " " + answer).lower()
+        for allergen in req.patient_allergies:
+            if allergen.strip().lower() in text_to_scan:
+                allergy_intercepted = True
+                allergen_matched = allergen.strip().capitalize()
+                break
+
     faith_score = (mod_results.get("faithfulness") or {}).get("score", 1.0)
 
     # Source-credibility-aware faith threshold: high-credibility sources get more tolerance
@@ -570,7 +734,7 @@ def query(req: QueryRequest) -> QueryResponse:
     # FDA direct lookup can still retrieve the right data even when initial FAISS
     # retrieval missed it. Don't label those as coverage gaps — let intervention run.
     _ev_entities = (mod_results.get("entity_verifier") or {}).get("details", {}).get("entities", [])
-    _q_lower_cg = req.question.lower()
+    _q_lower_cg = question_to_use.lower()
     _drug_in_question = any(
         e.get("rxcui") and e.get("entity", "").lower() in _q_lower_cg
         for e in _ev_entities
@@ -606,17 +770,33 @@ def query(req: QueryRequest) -> QueryResponse:
             is_refusal_answer, top_faiss_cosine, faith_score,
         )
 
-    # Tier 1: CRITICAL BLOCK (HRS ≥ 86) — response is too dangerous to show
+    # Tier 1: CRITICAL BLOCK (HRS ≥ hrs_block_threshold) — response is too dangerous to show
     # Coverage gap: skip both tiers — regenerating from an empty DB won't help
-    if coverage_gap:
+    if allergy_intercepted:
+        original_answer = answer
+        answer = (
+            "⛔ PATIENT SAFETY SHIELD — ALLERGY CONTRADICTION BLOCKED\n\n"
+            f"Prescribing or recommending {allergen_matched} is STRICTLY CONTRAINDICATED "
+            f"because this patient is flagged as severely ALLERGIC to: {allergen_matched}.\n\n"
+            "Immediate Action: Cancel drug order and consult guidelines for safe alternative therapies (e.g. Paracetamol instead of NSAIDs)."
+        )
+        hrs = 100
+        intervention_applied = True
+        intervention_reason = "CRITICAL_ALLERGY_BLOCKED"
+        intervention_details = {
+            "hrs_original": 100,
+            "message": f"Response blocked: Patient has an active chart allergy to {allergen_matched}.",
+        }
+        logger.warning("INTERVENTION: CRITICAL_ALLERGY_BLOCKED — allergen=%s", allergen_matched)
+    elif coverage_gap:
         logger.info("COVERAGE_GAP — skipping intervention (regeneration cannot add missing data).")
-    elif hrs >= 86:
+    elif hrs >= hrs_block_threshold:
         original_answer = answer
         answer = (
             "⛔ UNSAFE RESPONSE BLOCKED by MediRAG Safety Gate.\n\n"
             "The generated answer was flagged as CRITICAL risk "
-            f"(Health Risk Score: {hrs}/100). "
-            "It showed signs of hallucination or contradiction with the retrieved evidence. "
+            f"(Health Risk Score: {hrs}/100, Ward Limit: {hrs_block_threshold}%).\n\n"
+            "It showed signs of clinical hallucination or contradiction with the retrieved evidence. "
             "Please consult a qualified medical professional or rephrase your question."
         )
         intervention_applied = True
@@ -624,12 +804,12 @@ def query(req: QueryRequest) -> QueryResponse:
         intervention_details = {
             "hrs_original": hrs,
             "faithfulness": faith_score,
-            "message": "Response blocked: HRS ≥ 86 (CRITICAL risk band).",
+            "message": f"Response blocked: HRS >= {hrs_block_threshold} (Ward limit exceeded).",
         }
-        logger.warning("INTERVENTION: CRITICAL_BLOCKED — HRS=%d", hrs)
+        logger.warning("INTERVENTION: CRITICAL_BLOCKED — HRS=%d (limit=%d)", hrs, hrs_block_threshold)
 
     # Tier 2: HIGH RISK REGENERATION
-    elif hrs >= 61 or faith_score < faith_threshold:
+    elif hrs >= hrs_retry_threshold or faith_score < faith_threshold:
         original_answer = answer
         original_hrs = hrs
         logger.warning(
@@ -651,7 +831,7 @@ def query(req: QueryRequest) -> QueryResponse:
                         e["entity"] for e in ev_details.get("entities", [])
                         if e.get("status") == "VERIFIED" and e.get("rxcui")
                     ]
-                    q_lower = req.question.lower()
+                    q_lower = question_to_use.lower()
                     for drug in verified_drugs:
                         if drug.lower() in q_lower:
                             fda_direct += app.state.retriever.get_fda_chunks(drug)
@@ -668,7 +848,7 @@ def query(req: QueryRequest) -> QueryResponse:
                 guideline_direct: list[dict] = []
                 if top_faiss_cosine < 0.85:
                     try:
-                        guideline_direct = app.state.retriever.get_guideline_chunks(req.question)
+                        guideline_direct = app.state.retriever.get_guideline_chunks(question_to_use)
                         if guideline_direct:
                             logger.info("Direct guideline lookup found %d chunks", len(guideline_direct))
                     except Exception as gl_exc:
@@ -682,11 +862,11 @@ def query(req: QueryRequest) -> QueryResponse:
                 # For drug/clinical questions, expand query toward authoritative sources
                 _drug_terms = ("contraindication", "dosage", "dose", "interaction",
                                "warning", "adverse", "side effect", "mechanism")
-                _q_lower = req.question.lower()
+                _q_lower = question_to_use.lower()
                 retry_query = (
-                    f"FDA drug label clinical guideline {req.question}"
+                    f"FDA drug label clinical guideline {question_to_use}"
                     if any(t in _q_lower for t in _drug_terms)
-                    else req.question
+                    else question_to_use
                 )
                 fresh_results = app.state.retriever.search(retry_query, top_k=req.top_k)
                 fresh_chunks: list[dict] = []
@@ -705,10 +885,10 @@ def query(req: QueryRequest) -> QueryResponse:
             except Exception:
                 retry_chunks = context_chunks
 
-            answer = generate_strict_answer(req.question, retry_chunks, _cfg, overrides=llm_overrides)
+            answer = generate_strict_answer(question_to_use, retry_chunks, _cfg, overrides=llm_overrides)
             # Re-evaluate the corrected answer
             eval_result = run_evaluation(
-                question=req.question,
+                question=question_to_use,
                 answer=answer,
                 context_chunks=retry_chunks,
                 run_ragas=False,  # skip RAGAS on retry to reduce latency
@@ -740,15 +920,59 @@ def query(req: QueryRequest) -> QueryResponse:
     logger.info("POST /query → HRS=%d (%s) intervention=%s in %d ms total",
                 hrs, details.get("risk_band", "?"), intervention_reason or "none", total_ms)
 
-    log_audit("query", req.question, answer, hrs, details.get("risk_band", "UNKNOWN"), composite, total_ms, intervention_applied, {
+    log_audit("query", question_to_use, answer, hrs, details.get("risk_band", "UNKNOWN"), composite, total_ms, intervention_applied, {
         "module_results": mod_results,
         "confidence_level": details.get("confidence_level", "UNKNOWN"),
         "intervention_reason": intervention_reason,
         "original_answer": original_answer,
     })
 
+    # Save successful evaluation to Safe Semantic Cache
+    if q_vec is not None and not coverage_gap:
+        try:
+            response_dict = {
+                "question": question_to_use,
+                "generated_answer": answer,
+                "retrieved_chunks": [
+                    {
+                        "chunk_id": c.chunk_id,
+                        "text": c.text,
+                        "source": c.source,
+                        "pub_type": c.pub_type,
+                        "pub_year": c.pub_year,
+                        "title": c.title,
+                        "similarity_score": c.similarity_score
+                    } for c in retrieved_chunks_out
+                ],
+                "composite_score": composite,
+                "hrs": hrs,
+                "confidence_level": details.get("confidence_level", "UNKNOWN"),
+                "risk_band": details.get("risk_band", "UNKNOWN"),
+                "module_results": mod_results,
+                "intervention_applied": intervention_applied,
+                "intervention_reason": intervention_reason,
+                "original_answer": original_answer,
+                "intervention_details": intervention_details,
+                "consensus_results": consensus_results,
+                "privacy_applied": privacy_applied,
+                "privacy_details": {"redacted_count": len(p_mapping)} if privacy_applied else None,
+                "original_hinglish_query": original_hinglish,
+                "coverage_gap": coverage_gap,
+                "coverage_gap_details": coverage_gap_details,
+            }
+            semantic_cache.store(
+                query_text=question_to_use,
+                query_emb=q_vec,
+                response=response_dict,
+                patient_allergies=req.patient_allergies or [],
+                department=req.department or "default",
+                overrides=llm_overrides
+            )
+        except Exception as exc:
+            logger.error("Failed to store in semantic cache: %s", exc)
+
     return QueryResponse(
-        question=req.question,
+        question=question_to_use,
         generated_answer=answer,
         retrieved_chunks=retrieved_chunks_out,
         composite_score=composite,
@@ -770,6 +994,7 @@ def query(req: QueryRequest) -> QueryResponse:
         consensus_results=consensus_results,
         privacy_applied=privacy_applied,
         privacy_details={"redacted_count": len(p_mapping)} if privacy_applied else None,
+        original_hinglish_query=original_hinglish,
         coverage_gap=coverage_gap,
         coverage_gap_details=coverage_gap_details,
     )

src/api/schemas.py

@@ -206,6 +206,14 @@ class QueryRequest(BaseModel):
         default=False,
         description="Automatically redact PHI/PII (names, IDs) before external API calls.",
     )
+    translate_hinglish: bool = Field(
+        default=False,
+        description="Translate query from Hinglish to English before processing",
+    )
+    original_hinglish_query: Optional[str] = Field(
+        default=None,
+        description="[OPTIONAL] Pre-translated original Hinglish text from the front-end audit gate."
+    )
     system_prompt: Optional[str] = Field(
         default=None,
         description="Custom system prompt to override the default clinical persona."
@@ -214,6 +222,22 @@ class QueryRequest(BaseModel):
         default="physician",
         description="The target audience for the response: 'physician' or 'patient'."
     )
+    department: Optional[str] = Field(
+        default=None,
+        description="[OPTIONAL] Active department (oncology, cardiology, pediatrics, opd, etc.) to trigger custom safety thresholds."
+    )
+    patient_allergies: Optional[list[str]] = Field(
+        default=None,
+        description="[OPTIONAL] List of patient drug allergies to scan against recommended medications."
+    )
+    custom_hrs_limit: Optional[int] = Field(
+        default=None,
+        description="[OPTIONAL] Custom HRS risk tolerance percentage (0-100) set by the hospital console."
+    )
+    custom_latency_limit: Optional[int] = Field(
+        default=None,
+        description="[OPTIONAL] Custom max allowed latency in ms."
+    )
 
 
 class RetrievedChunk(BaseModel):
@@ -264,6 +288,7 @@ class QueryResponse(BaseModel):
     # Privacy Shield fields
     privacy_applied: bool = Field(default=False)
     privacy_details: Optional[Dict[str, Any]] = Field(default=None)
+    original_hinglish_query: Optional[str] = Field(default=None, description="The original Hinglish query before translation")
     # Coverage gap gate — distinguishes missing DB coverage from hallucination
     coverage_gap: bool = Field(
         default=False,

src/modules/entity_verifier.py

@@ -172,6 +172,56 @@ def _lookup_rxnorm_api(drug_name: str, timeout: int = 4) -> Optional[str]:
     return None
 
 
+@lru_cache(maxsize=1024)
+def _cached_drug_interactions(rxcuis_tuple: tuple[str, ...], timeout: int) -> list[dict]:
+    """
+    Synchronous cached NIH REST request to resolve drug interactions.
+    """
+    rxcuis_str = "+".join(rxcuis_tuple)
+    url = f"https://rxnav.nlm.nih.gov/REST/interaction/list.json?rxcuis={rxcuis_str}"
+    try:
+        resp = requests.get(url, timeout=timeout)
+        if resp.status_code != 200:
+            return []
+        
+        data = resp.json()
+        interactions = []
+        
+        # ONCHigh returns fullInteractionTypeGroup
+        groups = data.get("fullInteractionTypeGroup", [])
+        for group in groups:
+            for fit in group.get("fullInteractionType", []):
+                for pair in fit.get("interactionPair", []):
+                    # Extract concepts
+                    concepts = pair.get("interactionConcept", [])
+                    drugs_involved = [c.get("minConcept", {}).get("name", "Unknown") for c in concepts]
+                    severity = pair.get("severity", "high").lower() # default to high since it's from ONCHigh
+                    description = pair.get("description", "")
+                    interactions.append({
+                        "drugs": drugs_involved,
+                        "severity": severity,
+                        "description": description
+                    })
+        return interactions
+    except Exception as e:
+        logger.error("Failed to fetch drug interactions from RxNav: %s", e)
+        return []
+
+
+def check_drug_interactions(rxcuis: list[str], timeout: int = 5) -> list[dict]:
+    """
+    Query RxNav API for drug-drug interactions between a list of RxCUIs.
+    Uses sorted tuple transformation to enable efficient order-independent caching.
+    """
+    if len(rxcuis) < 2:
+        return []
+    
+    # Sort RxCUIs to ensure cache consistency regardless of list order
+    rxcuis_tuple = tuple(sorted(rxcuis))
+    return _cached_drug_interactions(rxcuis_tuple, timeout)
+
+
+
 # ---------------------------------------------------------------------------
 # Public API
 # ---------------------------------------------------------------------------
@@ -306,12 +356,29 @@ def verify_entities(
 
         entity_results.append(result)
 
+    # --- Drug-Drug Interaction Check (DDI) -----------------------------------
+    # Gather standard RxCUIs for the verified drugs
+    rxcuis = [ent["rxcui"] for ent in entity_results if ent.get("rxcui")]
+    unique_rxcuis = list(set(rxcuis))
+    interactions = []
+    
+    if len(unique_rxcuis) >= 2:
+        logger.info("Multiple drugs detected in answer (%s) — checking for interactions...", unique_rxcuis)
+        interactions = check_drug_interactions(unique_rxcuis)
+        if interactions:
+            logger.warning("DDI Check: Found %d drug interactions!", len(interactions))
+            drug_flagged += len(interactions)
+
     # --- Score ---------------------------------------------------------------
     # Score is based on drug entities only (per SRS Section 6.2)
     if drug_total == 0:
         score = 0.5  # neutral — no drug entities to verify
     else:
-        score = drug_verified / drug_total
+        # Base score is drug_verified / drug_total
+        base_score = drug_verified / drug_total
+        # Deduct score for multi-drug interactions (0.2 deduction per interaction, cap at 0.0)
+        interaction_deduction = len(interactions) * 0.20
+        score = max(0.0, base_score - interaction_deduction)
 
     details = {
         "total_entities": len(raw_entities),
@@ -319,12 +386,13 @@ def verify_entities(
         "verified_count": drug_verified,
         "flagged_count": drug_flagged,
         "entities": entity_results,
+        "interactions": interactions,
     }
 
     latency_ms = int((time.perf_counter() - t0) * 1000)
     logger.info(
-        "Entity verification: %.3f (%d/%d drugs verified) in %d ms",
-        score, drug_verified, drug_total, latency_ms,
+        "Entity verification: %.3f (%d/%d drugs verified, %d DDI found) in %d ms",
+        score, drug_verified, drug_total, len(interactions), latency_ms,
     )
     return EvalResult(
         module_name="entity_verifier",

src/pipeline/generator.py

@@ -50,7 +50,8 @@ _load_env()
 
 def _load_config() -> dict:
     try:
-        return yaml.safe_load(Path("config.yaml").read_text())
+        config_path = os.environ.get("MEDIRAG_CONFIG", "config_local.yaml" if Path("config_local.yaml").exists() else "config.yaml")
+        return yaml.safe_load(Path(config_path).read_text())
     except Exception:
         return {}
 
@@ -582,3 +583,78 @@ def generate_strict_answer(
         return _generate_groq(prompt, effective_config)
     else:
         raise RuntimeError(f"Unknown LLM provider '{provider}'.")
+
+
+def generate_simple_prompt(
+    prompt: str,
+    config: Optional[dict] = None,
+    overrides: Optional[dict] = None,
+) -> str:
+    """Execute a simple prompt on the active LLM provider without context formatting."""
+    if config is None:
+        config = _load_config()
+
+    effective_llm = dict(config.get("llm", {}))
+    if overrides:
+        if overrides.get("provider"):
+            effective_llm["provider"] = overrides["provider"]
+        if overrides.get("api_key"):
+            pk = (overrides.get("provider") or "gemini").lower()
+            key_map = {
+                "gemini": "gemini_api_key",
+                "openai": "openai_api_key",
+                "mistral": "mistral_api_key",
+                "groq": "groq_api_key",
+            }
+            effective_llm[key_map.get(pk, "gemini_api_key")] = overrides["api_key"]
+        if overrides.get("model"):
+            pk = (overrides.get("provider") or "gemini").lower()
+            model_map = {
+                "gemini": "gemini_model",
+                "openai": "openai_model",
+                "mistral": "model",
+                "groq": "groq_model",
+            }
+            effective_llm[model_map.get(pk, "gemini_model")] = overrides["model"]
+        if overrides.get("ollama_url"):
+            effective_llm["base_url"] = overrides["ollama_url"]
+
+    effective_config = {**config, "llm": effective_llm}
+    provider = effective_llm.get("provider", "gemini").lower()
+
+    if provider == "gemini":
+        return _generate_gemini(prompt, effective_config)
+    elif provider == "openai":
+        return _generate_openai(prompt, effective_config)
+    elif provider == "ollama":
+        return _generate_ollama(prompt, effective_config)
+    elif provider == "mistral":
+        return _generate_mistral(prompt, effective_config)
+    elif provider == "groq":
+        return _generate_groq(prompt, effective_config)
+    else:
+        raise RuntimeError(f"Unknown LLM provider '{provider}'.")
+
+
+def translate_hinglish_to_english(
+    question: str,
+    config: Optional[dict] = None,
+    overrides: Optional[dict] = None,
+) -> str:
+    """Translate clinical query from Hinglish or standard Hindi to professional English."""
+    prompt = (
+        "You are an expert bilingual clinical query translator. You will receive a medical question "
+        "written in Hinglish (a mixture of Hindi and English written in the Latin alphabet) or standard Hindi. "
+        "Convert the Hinglish/Hindi question into a clear, professional, grammatically correct English clinical query. "
+        "If the input query is already completely in English, return it exactly as it is with no edits. "
+        "Do NOT add any conversational preamble, greetings, explanation, or formatting. Only return the translated English query.\n\n"
+        f"Query: {question}\n"
+        "English Translation:"
+    )
+    try:
+        translated = generate_simple_prompt(prompt, config=config, overrides=overrides)
+        return translated.strip().strip('"').strip("'")
+    except Exception as exc:
+        logger.warning("Hinglish translation failed: %s. Using original query.", exc)
+        return question
+

src/pipeline/retriever.py

@@ -433,7 +433,9 @@ class Retriever:
 # ---------------------------------------------------------------------------
 
 def _load_config() -> dict:
-    with open("config.yaml", "r", encoding="utf-8") as f:
+    import os
+    config_path = os.environ.get("MEDIRAG_CONFIG", "config_local.yaml" if os.path.exists("config_local.yaml") else "config.yaml")
+    with open(config_path, "r", encoding="utf-8") as f:
         return yaml.safe_load(f)
 
 

src/pipeline/semantic_cache.py

@@ -0,0 +1,112 @@
+import sqlite3
+import json
+import hashlib
+import numpy as np
+import logging
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+class SafeSemanticCache:
+    def __init__(self, db_path="data/cache.db", threshold=0.97):
+        self.db_path = db_path
+        self.threshold = threshold
+        self._init_db()
+
+    def _init_db(self):
+        # Ensure containing directory exists
+        Path(self.db_path).parent.mkdir(parents=True, exist_ok=True)
+        conn = sqlite3.connect(self.db_path)
+        conn.execute("""
+            CREATE TABLE IF NOT EXISTS semantic_cache (
+                id INTEGER PRIMARY KEY,
+                query_text TEXT,
+                embedding BLOB,
+                cache_hash TEXT UNIQUE,
+                response_json TEXT,
+                timestamp DATETIME DEFAULT CURRENT_TIMESTAMP
+            )
+        """)
+        conn.commit()
+        conn.close()
+
+    def _generate_hash(self, query_emb: np.ndarray, patient_allergies: list[str], department: str, overrides: dict) -> str:
+        # Create a deterministic representation of the safety environment
+        allergies_str = ",".join(sorted([a.lower().strip() for a in patient_allergies]))
+        dept_str = department.lower().strip()
+        overrides_str = json.dumps(overrides, sort_keys=True)
+        
+        # Round embedding to 4 decimals to ensure stability against float discrepancies
+        emb_str = np.round(query_emb, 4).tobytes()
+        
+        hasher = hashlib.sha256()
+        hasher.update(emb_str)
+        hasher.update(allergies_str.encode('utf-8'))
+        hasher.update(dept_str.encode('utf-8'))
+        hasher.update(overrides_str.encode('utf-8'))
+        return hasher.hexdigest()
+
+    def get(self, query_emb: np.ndarray, patient_allergies: list[str], department: str, overrides: dict) -> dict | None:
+        target_hash = self._generate_hash(query_emb, patient_allergies, department, overrides)
+        
+        conn = sqlite3.connect(self.db_path)
+        cursor = conn.cursor()
+        # Direct hash lookup first (O(1) fast path)
+        cursor.execute("SELECT response_json FROM semantic_cache WHERE cache_hash = ?", (target_hash,))
+        row = cursor.fetchone()
+        
+        if row:
+            conn.close()
+            logger.info("Semantic Cache: Direct hash hit! Returning safe response.")
+            try:
+                return json.loads(row[0])
+            except Exception as e:
+                logger.error(f"Failed to parse cached JSON: {e}")
+                return None
+
+        # Fuzzy lookup (Cosine similarity fallback under identical safety settings)
+        cursor.execute("SELECT query_text, embedding, response_json, cache_hash FROM semantic_cache")
+        rows = cursor.fetchall()
+        conn.close()
+
+        for query, emb_bytes, response_json, cached_hash in rows:
+            saved_emb = np.frombuffer(emb_bytes, dtype=np.float32)
+            # Compute cosine similarity
+            norm_product = np.linalg.norm(query_emb) * np.linalg.norm(saved_emb)
+            if norm_product == 0:
+                continue
+            cosine = np.dot(query_emb, saved_emb) / norm_product
+            
+            if cosine >= self.threshold:
+                # Re-verify that the safety hash matches (no allergy difference)
+                # To prevent cross-contamination, fuzzy match requires identical allergies/department config
+                candidate_hash = self._generate_hash(saved_emb, patient_allergies, department, overrides)
+                if candidate_hash == cached_hash:
+                    logger.info(f"Semantic Cache: Fuzzy similarity hit! ({cosine:.4f})")
+                    try:
+                        return json.loads(response_json)
+                    except Exception as e:
+                        logger.error(f"Failed to parse cached JSON in fuzzy match: {e}")
+                        continue
+                    
+        return None
+
+    def store(self, query_text: str, query_emb: np.ndarray, response: dict, patient_allergies: list[str], department: str, overrides: dict):
+        target_hash = self._generate_hash(query_emb, patient_allergies, department, overrides)
+        conn = sqlite3.connect(self.db_path)
+        try:
+            conn.execute("""
+                INSERT OR REPLACE INTO semantic_cache (query_text, embedding, cache_hash, response_json)
+                VALUES (?, ?, ?, ?)
+            """, (
+                query_text,
+                query_emb.tobytes(),
+                target_hash,
+                json.dumps(response)
+            ))
+            conn.commit()
+            logger.info("Saved successful evaluation to Semantic Cache.")
+        except Exception as e:
+            logger.error(f"Failed to store in cache: {e}")
+        finally:
+            conn.close()

tests/test_modules.py

@@ -64,3 +64,27 @@ def test_aggregator_logic():
     assert abs(res.score - 0.9) < 0.01
     assert res.details["hrs"] == 10
     assert res.details["risk_band"] == "LOW"
+
+
+def test_drug_interactions_and_entity_verifier():
+    from src.modules.entity_verifier import check_drug_interactions, verify_entities
+    
+    # 1. Test DDI check directly with known interactive drugs (Warfarin: 11289, Ibuprofen: 5640)
+    interactions = check_drug_interactions(["11289", "5640"])
+    
+    # Verify interactions structure is a valid list of dicts
+    assert isinstance(interactions, list)
+    if interactions:
+        assert "drugs" in interactions[0]
+        assert "severity" in interactions[0]
+        assert "description" in interactions[0]
+        
+    # 2. Verify fallback & interface safety of verify_entities
+    res = verify_entities(
+        answer="Patient is taking Metformin and Lisinopril.",
+        question="What medications is the patient on?",
+        context_docs=["The patient is prescribed Metformin 500mg and Lisinopril 10mg."]
+    )
+    assert res.score is not None
+    assert isinstance(res.details, dict)
+