Update summary_extractor.py

summary_extractor.py

@@ -1,15 +1,14 @@
-# import openai
+import openai
 import json
 from typing import Dict
 import os
 from typing import List
-from langchain.chat_models import ChatOpenAI
-from langchain.document_loaders import PyPDFLoader
+from langchain_openai import AzureChatOpenAI
 from langchain.chains.mapreduce import MapReduceChain
 from langchain.text_splitter import CharacterTextSplitter
 from langchain.chains.summarize import load_summarize_chain
 from langchain.prompts import PromptTemplate
-
+from langchain_community.document_loaders import UnstructuredFileLoader
 
 class Extractor:
 
@@ -20,13 +19,16 @@ class Extractor:
         config (dict): Configuration settings loaded from a JSON file.
         pdf_file_path (str): Path to the input PDF file.
     """
+
     def __init__(self):
         """
         Initialize the Extractor class.
         """
 
-        # Set OpenAI API key
-        # os.environ["OPENAI_API_KEY"] = ""
+        openai.api_type = os.getenv['api_type']
+        os.environ["AZURE_OPENAI_API_KEY"] = os.getenv['api_key']
+        os.environ["AZURE_OPENAI_ENDPOINT"] = os.getenv['api_base']
+        os.environ["OPENAI_API_VERSION"] = os.getenv['api_version']
 
     def _document_loader(self,pdf_file_path) -> List[str]:
         """
@@ -36,7 +38,7 @@ class Extractor:
             List[str]: List of text content from each page.
         """
         try:
-            loader = PyPDFLoader(pdf_file_path.name)
+            loader = UnstructuredFileLoader(pdf_file_path)
             pages = loader.load_and_split()
             return pages
 
@@ -54,15 +56,15 @@ class Extractor:
         try:
             # Load the document texts
             docs = self._document_loader(pdf_file_path)
-
+    
             # Initialize the text splitter with specified chunk size and overlap
             text_splitter = CharacterTextSplitter.from_tiktoken_encoder(
                 chunk_size=1000, chunk_overlap=200
             )
-
+    
             # Split the documents into chunks
             split_docs = text_splitter.split_documents(docs)
-
+    
             # Return the list of split document chunks
             return split_docs
 
@@ -70,13 +72,15 @@ class Extractor:
             print(f"Error while splitting document text: {str(e)}")
 
 
-    def _refine_summary(self,pdf_file_path) -> str:
+    def refine_summary(self,pdf_file_path) -> str:
+
         """
         Generate a refined summary of the document using language models.
 
         Returns:
             str: Refined summary text.
         """
+
         try:
             # Split documents into chunks for efficient processing
             split_docs = self._document_text_spilliter(pdf_file_path)
@@ -103,7 +107,7 @@ class Extractor:
 
             # Load the summarization chain using the ChatOpenAI language model
             chain = load_summarize_chain(
-                llm = ChatOpenAI(temperature=0),
+                llm = AzureChatOpenAI(azure_deployment = "ChatGPT"),
                 chain_type="refine",
                 question_prompt=prompt,
                 refine_prompt=refine_prompt,