Added: Free Embedding

.env

@@ -2,4 +2,4 @@ GOOGLE_API_KEY=AIzaSyBA4RZNkKsqwkRrMCPbio7zrQ4xo12XpHI
 ZILLIZ_URI=https://in03-1da78c9c65effba.serverless.aws-eu-central-1.cloud.zilliz.com
 ZILLIZ_TOKEN=02a96317810ef222f0752a53bd9a3a2d0e46740ccf85f8ec4d18a4427bface8e82d57735f54fb0592802ed883573aeee40e9f529
 ZILLIZ_COLLECTION=visaverse_faqs
-OPENAI_API_KEY=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
+OPENAI_API_KEY=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

faq_services.py

@@ -14,6 +14,7 @@ from langchain.schema import HumanMessage
 from langchain.docstore.document import Document
 from langchain_community.document_loaders import CSVLoader
 from langchain.schema import SystemMessage, HumanMessage
+from langchain_community.embeddings import HuggingFaceEmbeddings
 import difflib
 from pymilvus import connections, utility, Collection
 from pymilvus.orm.schema import FieldSchema
@@ -26,10 +27,18 @@ os.environ["HF_HOME"] = "/tmp/hf_cache"  # Optional cleanup
 # ---------------------- File & Model Config ----------------------
 
 faq_path = "faqs.csv"
-embedding_model = OpenAIEmbeddings(
-    model="text-embedding-3-small",
-    openai_api_key=os.getenv("OPENAI_API_KEY")
+
+# 💰 FREE Embeddings - Sentence Transformers (saves ~$18/day!)
+print("🚀 Loading FREE embedding model (all-mpnet-base-v2)...")
+embedding_model = HuggingFaceEmbeddings(
+    model_name="sentence-transformers/all-mpnet-base-v2",
+    model_kwargs={'device': 'cpu'},
+    encode_kwargs={'normalize_embeddings': True}
 )
+print("✅ FREE embedding model loaded successfully!")
+
+# NOTE: Dimension changed from 1536 (OpenAI) to 768 (mpnet)
+EMBEDDING_DIM = 768
 
 IMPORTANT_KEYWORDS = [
     "visa", "permanent residency", "PR", "study permit", "work permit", "immigration",
@@ -94,16 +103,16 @@ def ensure_collection_matches_schema(expected_dim: int, collection_name: str, ur
                     if field.params and "dim" in field.params:
                         actual_dim = int(field.params["dim"])
                         if actual_dim != expected_dim:
-                            print(f"Collection '{collection_name}' has dim {actual_dim}, expected {expected_dim}. Dropping it.")
+                            print(f"⚠️ Collection dim mismatch: {actual_dim} vs {expected_dim}. Dropping old collection.")
                             utility.drop_collection(collection_name)
                             return
                         else:
-                            print(f"Collection '{collection_name}' has correct dimension: {expected_dim}.")
+                            print(f"✅ Collection '{collection_name}' has correct dimension: {expected_dim}.")
                             return
-            print(f"Could not find vector field in collection '{collection_name}'. Dropping for safety.")
+            print(f"⚠️ Could not find vector field in collection '{collection_name}'. Dropping for safety.")
             utility.drop_collection(collection_name)
         else:
-            print(f"Collection '{collection_name}' does not exist. It will be created.")
+            print(f"📝 Collection '{collection_name}' does not exist. It will be created.")
     except Exception as e:
         print(f"Failed to validate or drop collection: {e}")
 
@@ -113,9 +122,9 @@ def load_faqs():
     if not os.path.exists(faq_path):
         pd.DataFrame(columns=["id", "prompt", "response"]).to_csv(faq_path, index=False, encoding="utf-8")
 
-    # Check collection schema
+    # Check collection schema (768 dims for mpnet vs 1536 for OpenAI)
     ensure_collection_matches_schema(
-        expected_dim=1536,
+        expected_dim=768,  # FREE model dimension
         collection_name=collection_name,
         uri=milvus_uri,
         token=milvus_token

ircc_updater.py

@@ -1,18 +1,13 @@
 # updated ircc_updater.py
 import requests
 from bs4 import BeautifulSoup
-from langchain_openai import OpenAIEmbeddings
 from langchain.schema import Document
-from faq_services import db
+from faq_services import db, embedding_model
 from apscheduler.schedulers.background import BackgroundScheduler
 import os
 from datetime import datetime
 
-# Config
-embedding_model = OpenAIEmbeddings(
-    model="text-embedding-3-small",
-    openai_api_key=os.getenv("OPENAI_API_KEY")
-)
+# Note: Using FREE embeddings from faq_services (all-mpnet-base-v2)
 
 
 # Main IRCC pages to crawl for links

requirements.txt

@@ -10,6 +10,10 @@ langchain-openai
 openai
 tiktoken
 
+# FREE Embeddings (Sentence Transformers)
+sentence-transformers
+torch
+
 # Vector DB (Milvus/Zilliz only)
 pymilvus