Upload faiss_vdb_script.py

vdb_script/faiss_vdb_script.py

@@ -1,18 +1,33 @@
 import os
+from dotenv import load_dotenv
 from langchain.document_loaders import PyPDFLoader, DirectoryLoader
 from langchain.text_splitter import RecursiveCharacterTextSplitter
-from langchain.embeddings.openai import OpenAIEmbeddings
+from langchain.agents import Tool, AgentExecutor
+from langchain.tools.retriever import create_retriever_tool
+from langchain.prompts import ChatPromptTemplate, MessagesPlaceholder
 from langchain_community.vectorstores import FAISS
-from langchain.chains import RetrievalQA
-from langchain.llms import OpenAI
-from dotenv import load_dotenv
-
-# Load environment variable for OpenAI key
+from langchain_community.embeddings import AzureOpenAIEmbeddings
+from langchain_community.chat_models import AzureChatOpenAI
+from openai import AzureOpenAI
+import warnings
+ 
+# Load environment variables
 load_dotenv()
+AZURE_OPENAI_API_KEY = os.getenv("AZURE_OPENAI_API_KEY")
+AZURE_OPENAI_ENDPOINT = os.getenv("AZURE_OPENAI_ENDPOINT")
+AZURE_OPENAI_LLM_DEPLOYMENT = os.getenv("AZURE_OPENAI_LLM_DEPLOYMENT")
+AZURE_OPENAI_EMBEDDING_DEPLOYMENT = os.getenv("AZURE_OPENAI_EMBEDDING_DEPLOYMENT")
+ 
+if not all([AZURE_OPENAI_API_KEY, AZURE_OPENAI_ENDPOINT, AZURE_OPENAI_LLM_DEPLOYMENT, AZURE_OPENAI_EMBEDDING_DEPLOYMENT]):
+    raise ValueError("Missing one or more Azure OpenAI environment variables.")
+ 
+warnings.filterwarnings("ignore")
+
+AZURE_OPENAI_API_KEY = os.getenv("AZURE_OPENAI_API_KEY")
+if not AZURE_OPENAI_API_KEY:
+    raise ValueError("Missing AZURE_OPENAI_API_KEY in environment variables.")
 
-OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
-if not OPENAI_API_KEY:
-    raise ValueError("Missing OPENAI_API_KEY in environment variables.")
+chunk_size = 500
 
 # Extract Data from the PDFs
 def load_pdf_file(data_path):
@@ -22,15 +37,27 @@ def load_pdf_file(data_path):
 
 # Split the data into chunks
 def text_split(docs):
-    splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=20)
+    splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=20)
     return splitter.split_documents(docs)
 
 # Set up LLM and Embedding
-llm = OpenAI(model_name="gpt-4o-mini", temperature=0.5, openai_api_key=OPENAI_API_KEY)
-embeddings = OpenAIEmbeddings(openai_api_key=OPENAI_API_KEY)
+llm = AzureChatOpenAI(
+    deployment_name=AZURE_OPENAI_LLM_DEPLOYMENT,
+    azure_endpoint=AZURE_OPENAI_ENDPOINT,
+    openai_api_key=AZURE_OPENAI_API_KEY,
+    openai_api_version="2023-12-01-preview"  # or your supported version
+    # temperature=0.5  # Only if supported by your deployment
+)
+embeddings = AzureOpenAIEmbeddings(
+    azure_deployment=AZURE_OPENAI_EMBEDDING_DEPLOYMENT,
+    azure_endpoint=AZURE_OPENAI_ENDPOINT,
+    openai_api_key=AZURE_OPENAI_API_KEY,
+    openai_api_version="2023-12-01-preview",
+    chunk_size=chunk_size  # or another value up to 2048
+)
 
 # Load PDF, chunk it, embed it, and store in FAISS
-pdf_docs = load_pdf_file("/kaggle/input/rag-test")  # Update this to your PDF folder
+pdf_docs = load_pdf_file("Dataset/")  # Update this to your PDF folder
 chunks = text_split(pdf_docs)
 
 vectorstore = FAISS.from_documents(chunks, embeddings)