Upload 5 files

faq_routes.py

@@ -0,0 +1,239 @@
+#Basic Packages
+import io
+import pandas as pd
+import uuid
+import traceback
+from collections import defaultdict
+import time
+import os
+import json
+
+#API Packages
+from fastapi import APIRouter, UploadFile, File, HTTPException, Body, Path, Request
+
+#FAQ CSV Validator Package
+from pydantic import BaseModel
+
+#Calling Functions from other py files
+from faq_services import gemini_model, db, load_faqs, add_faq_to_csv, faq_path
+from chatbot_prompt import generate_prompt
+from ircc_updater import manual_ircc_update_with_result
+
+router = APIRouter()
+
+user_question_count = defaultdict(int)
+QUESTION_LIMIT = 3
+WHATSAPP_LINK = "https://wa.me/1234567890"
+GREETING_KEYWORDS = {"hi", "hello", "hey", "good morning", "good evening", "good afternoon", "greetings"}
+QUESTION_LOG_FILE = "question_limit_log.json"
+
+# Data validation classes
+class QuestionRequest(BaseModel):
+    query: str
+
+class FAQItem(BaseModel):
+    question: str
+    answer: str
+
+def is_greeting(text: str) -> bool:
+    lower = text.lower().strip()
+    return any(greet in lower for greet in GREETING_KEYWORDS) or len(lower) <= 12
+
+# Load or initialize question count data
+def load_question_log():
+    if not os.path.exists(QUESTION_LOG_FILE):
+        return {}
+    with open(QUESTION_LOG_FILE, "r") as f:
+        return json.load(f)
+
+def save_question_log(log_data):
+    with open(QUESTION_LOG_FILE, "w") as f:
+        json.dump(log_data, f)
+
+# In-memory cache loaded on startup
+user_question_count = load_question_log()
+
+# Chat endpoint API
+@router.post("/ask")
+async def ask_faq(request: QuestionRequest, http_request: Request):
+    ip = http_request.client.host
+    query = request.query.strip()
+
+    count = user_question_count.get(ip, 0)
+
+    if count >= QUESTION_LIMIT:
+        return {
+            "message": f"You've reached the free question limit. Please contact us on WhatsApp: {WHATSAPP_LINK}"
+        }
+
+    if not is_greeting(query):
+        user_question_count[ip] = count + 1
+        save_question_log(user_question_count)
+
+        if user_question_count[ip] >= QUESTION_LIMIT:
+            return {
+                "message": f"You've reached the free question limit. Please contact us on WhatsApp: {WHATSAPP_LINK}"
+            }
+
+    # Run search & generate response
+    results = db.similarity_search(query, k=3)
+    context = "\n\n".join([doc.page_content for doc in results])
+    prompt = generate_prompt(context, query)
+
+    try:
+        response = gemini_model.generate_content(prompt)
+        return {"answer": response.text.strip()}
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+# Add Single FAQ API
+@router.post("/add_faq")
+async def add_faq(faq: FAQItem):
+    try:
+        df = pd.read_csv(faq_path, encoding="utf-8")
+        if ((df["prompt"] == faq.question) & (df["response"] == faq.answer)).any():
+            raise HTTPException(status_code=400, detail="FAQ already exists.")
+        new_df = pd.DataFrame([{"id": str(uuid.uuid4()), "prompt": faq.question, "response": faq.answer}])
+        updated_df = pd.concat([df, new_df], ignore_index=True)
+        updated_df.to_csv(faq_path, index=False, encoding="utf-8")
+        global db
+        db = load_faqs()
+        return {"message": "FAQ added successfully."}
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+# Upload CSV API
+@router.post("/upload_faqs_csv")
+async def upload_faqs_csv(file: UploadFile = File(...)):
+    if not file.filename.endswith(".csv"):
+        return {
+            "status": "error",
+            "message": "Invalid file type",
+            "error": "Only CSV files are supported."
+        }
+
+    try:
+        contents = await file.read()
+        df = pd.read_csv(io.BytesIO(contents))
+
+        if "question" not in df.columns or "answer" not in df.columns:
+            return {
+                "status": "error",
+                "message": "Invalid CSV structure",
+                "error": "CSV must contain 'question' and 'answer' columns."
+            }
+
+        for _, row in df.iterrows():
+            question = str(row["question"]).strip()
+            answer = str(row["answer"]).strip()
+            if question and answer:
+                add_faq_to_csv(question, answer)
+
+        global db
+        db = load_faqs()
+
+        return {
+            "status": "success",
+            "message": "FAQs uploaded and added successfully."
+        }
+
+    except Exception as e:
+        traceback.print_exc()
+        return {
+            "status": "error",
+            "message": "Failed to process CSV",
+            "error": str(e)
+        }
+
+# Delete Single FAQ API
+@router.delete("/delete_faq")
+async def delete_faq(faq: FAQItem = Body(...)):
+    try:
+        df = pd.read_csv(faq_path, encoding="utf-8")
+        filtered_df = df[~((df["prompt"] == faq.question) & (df["response"] == faq.answer))]
+        if len(df) == len(filtered_df):
+            raise HTTPException(status_code=404, detail="FAQ not found.")
+        filtered_df.to_csv(faq_path, index=False, encoding="utf-8")
+        global db
+        db = load_faqs()
+        return {"message": "FAQ deleted successfully."}
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+@router.delete("/deleted/{faq_id}")
+async def delete_faq_by_id(faq_id: str = Path(...)):
+    try:
+        df = pd.read_csv(faq_path, encoding="utf-8")
+        if "id" not in df.columns:
+            raise HTTPException(status_code=500, detail="CSV does not contain 'id' column.")
+
+        filtered_df = df[df["id"] != faq_id]
+        if len(filtered_df) == len(df):
+            raise HTTPException(status_code=404, detail="FAQ with given ID not found.")
+
+        filtered_df.to_csv(faq_path, index=False, encoding="utf-8")
+        global db
+        db = load_faqs()
+        return {"message": f"FAQ with ID {faq_id} deleted successfully."}
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+# Delete All FAQs API
+@router.delete("/delete/destroyall")
+async def delete_all_faqs():
+    try:
+        pd.DataFrame(columns=["id", "prompt", "response"]).to_csv(faq_path, index=False, encoding="utf-8")
+        global db
+        db = load_faqs()
+        return {"message": "All FAQs deleted successfully."}
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+# Show All FAQs API
+@router.get("/get_faqs")
+async def get_faqs():
+    try:
+        df = pd.read_csv(faq_path, encoding="utf-8")
+        df = df.astype(str)
+        result = df.rename(columns={"prompt": "question", "response": "answer"}).to_dict(orient="records")
+        return result
+    except FileNotFoundError:
+        raise HTTPException(status_code=404, detail="FAQ CSV file not found.")
+    except pd.errors.ParserError as e:
+        raise HTTPException(status_code=500, detail=f"CSV Parsing Error: {str(e)}")
+    except UnicodeDecodeError as e:
+        raise HTTPException(status_code=500, detail=f"Encoding Error: {str(e)}")
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Unexpected Error: {str(e)}")
+
+# Retrain DB
+@router.post("/retrain")
+async def retrain_db():
+    try:
+        global db
+        db = load_faqs()
+        return {"message": "Chatbot retrained successfully."}
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+@router.get("/update_ircc_faqs")
+async def update_ircc_faqs():
+    try:
+        added = manual_ircc_update_with_result()
+
+        # fallback if None
+        if not added:
+            return {
+                "message": "IRCC FAQs updated manually.",
+                "added_count": 0,
+                "entries": []
+            }
+
+        return {
+            "message": "IRCC FAQs updated manually.",
+            "added_count": len(added),
+            "entries": added
+        }
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+

faq_services.py

@@ -0,0 +1,70 @@
+# faq_services.py
+import os
+import pandas as pd
+import uuid
+from dotenv import load_dotenv
+
+# LangChain / Vector DB
+from langchain.vectorstores import Milvus
+from langchain.embeddings import SentenceTransformerEmbeddings
+from langchain.docstore.document import Document
+from langchain.document_loaders.csv_loader import CSVLoader
+
+# Google Gemini
+import google.generativeai as genai
+
+# ---------------------- Environment Setup ----------------------
+
+os.environ["HF_HOME"] = "/tmp/hf_cache"
+load_dotenv()
+
+api_key = os.getenv("GOOGLE_API_KEY")
+genai.configure(api_key=api_key)
+
+gemini_model = genai.GenerativeModel(
+    model_name="gemini-2.0-flash",
+    generation_config={"temperature": 0.5}
+)
+
+# ---------------------- File & Model Config ----------------------
+
+faq_path = "faqs.csv"
+embedding_model = SentenceTransformerEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2")
+
+# Zilliz (Milvus) Cloud Config
+milvus_uri = os.getenv("ZILLIZ_URI")
+milvus_token = os.getenv("ZILLIZ_TOKEN")
+collection_name = os.getenv("ZILLIZ_COLLECTION", "visaverse_faqs")
+
+# ---------------------- Load FAQ Vector DB ----------------------
+
+def load_faqs():
+    if not os.path.exists(faq_path):
+        pd.DataFrame(columns=["id", "prompt", "response"]).to_csv(faq_path, index=False, encoding="utf-8")
+
+    loader = CSVLoader(faq_path, encoding="utf-8")
+    docs = loader.load()
+
+    if not docs:
+        docs = [Document(page_content="This is a placeholder FAQ")]
+
+    return Milvus.from_documents(
+        documents=docs,
+        embedding=embedding_model,
+        connection_args={
+            "uri": milvus_uri,
+            "token": milvus_token,
+        },
+        collection_name=collection_name,
+    )
+
+db = load_faqs()
+
+# ---------------------- Append New FAQ to CSV ----------------------
+
+def add_faq_to_csv(question: str, answer: str):
+    df = pd.read_csv(faq_path, encoding="utf-8")
+    if not ((df["prompt"] == question) & (df["response"] == answer)).any():
+        new_row = pd.DataFrame([{"id": str(uuid.uuid4()), "prompt": question, "response": answer}])
+        df = pd.concat([df, new_row], ignore_index=True)
+        df.to_csv(faq_path, index=False, encoding="utf-8")

ircc_updater.py

@@ -0,0 +1,66 @@
+import requests
+from bs4 import BeautifulSoup
+from langchain.embeddings import SentenceTransformerEmbeddings
+from faq_services import db
+from apscheduler.schedulers.background import BackgroundScheduler
+from langchain.schema import Document
+
+# Config
+embedding_model = SentenceTransformerEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2")
+ircc_urls = [
+    "https://www.canada.ca/en/immigration-refugees-citizenship.html",
+    "https://www.canadavisa.com/ircc.html",
+    "https://www.canada.ca/en/immigration-refugees-citizenship/services/immigrate-canada.html"
+]
+
+# Scrape IRCC
+def scrape_ircc_content():
+    results = []
+    for url in ircc_urls:
+        try:
+            resp = requests.get(url, timeout=10)
+            soup = BeautifulSoup(resp.text, "html.parser")
+            for tag in soup.find_all(["h2", "h3", "p"]):
+                text = tag.get_text(strip=True)
+                if len(text) > 50:
+                    results.append(text)
+        except Exception as e:
+            print(f"[IRCC] Failed to fetch from {url}: {e}")
+    return list(set(results))
+
+# Update Vector DB
+def update_ircc_embeddings():
+    print("[IRCC] Checking for new IRCC content...")
+    texts = scrape_ircc_content()
+    new_texts = []
+
+    for t in texts:
+        results = db.similarity_search(t, k=1)
+        if not results or results[0].page_content.strip().lower() != t.strip().lower():
+            new_texts.append(t)
+
+    if new_texts:
+        # Convert to Document objects
+        documents = [Document(page_content=text) for text in new_texts]
+        embeddings = embedding_model.embed_documents([doc.page_content for doc in documents])
+
+        # Add to DB
+        db.add_documents(documents=documents, embeddings=embeddings)
+        print(f"[IRCC] Added {len(new_texts)} new entries.")
+        return new_texts
+    else:
+        print("[IRCC] No new entries found.")
+        return new_texts
+
+def manual_ircc_update():
+    update_ircc_embeddings()
+
+def manual_ircc_update_with_result():
+    return update_ircc_embeddings()
+
+# Scheduler
+def start_ircc_scheduler():
+    scheduler = BackgroundScheduler()
+    scheduler.add_job(update_ircc_embeddings, 'interval', days=7)
+    scheduler.start()
+    print("[IRCC] Scheduler started: checks every 7 days.")

main.py

@@ -0,0 +1,25 @@
+#API Packages
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+
+#Route Calling From faq_routes.py
+from faq_routes import router as faq_router
+from ircc_updater import start_ircc_scheduler
+
+app = FastAPI()
+
+# Enable CORS for frontend
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+# Include FAQ API routes
+app.include_router(faq_router)
+
+@app.on_event("startup")
+async def startup_event():
+    start_ircc_scheduler()

requirements.txt

@@ -0,0 +1,13 @@
+chromadb==1.0.9
+fastapi==0.115.9
+uvicorn==0.34.2
+google-generativeai==0.8.5
+langchain==0.3.25
+langchain-community==0.3.24
+python-dotenv==1.1.0
+pandas==2.2.3
+python-multipart==0.0.20
+sentence-transformers==4.1.0
+pymilvus
+BeautifulSoup4
+APScheduler