Spaces:
Sleeping
Sleeping
File size: 6,420 Bytes
ec7ddd1 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 |
import streamlit as st
import os
from openai import OpenAI
import tempfile
from langchain.chains import ConversationalRetrievalChain
from langchain_openai import ChatOpenAI, OpenAIEmbeddings
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain_community.vectorstores import Chroma
from langchain_community.document_loaders import (
PyPDFLoader,
TextLoader,
CSVLoader
)
from datetime import datetime
import pytz
# DocumentRAG class with environment variable support for API Key
class DocumentRAG:
def __init__(self):
self.document_store = None
self.qa_chain = None
self.document_summary = ""
self.chat_history = []
self.last_processed_time = None
self.api_key = os.getenv("OPENAI_API_KEY") # Fetch the API key from environment variable
self.init_time = datetime.now(pytz.UTC)
if not self.api_key:
raise ValueError("API Key not found. Make sure to set the 'OPENAI_API_KEY' environment variable.")
def process_documents(self, file_paths):
if not self.api_key:
return "Please set the OpenAI API key in the environment variables."
if not file_paths:
return "Please upload documents first."
try:
documents = []
for file_path in file_paths:
if file_path.name.endswith('.pdf'):
loader = PyPDFLoader(file_path.name)
elif file_path.name.endswith('.txt'):
loader = TextLoader(file_path.name)
elif file_path.name.endswith('.csv'):
loader = CSVLoader(file_path.name)
else:
continue
try:
documents.extend(loader.load())
except Exception as e:
print(f"Error loading {file_path.name}: {str(e)}")
continue
if not documents:
return "No valid documents were processed. Please check your files."
text_splitter = RecursiveCharacterTextSplitter(
chunk_size=1000,
chunk_overlap=200,
length_function=len
)
documents = text_splitter.split_documents(documents)
combined_text = " ".join([doc.page_content for doc in documents])
self.document_summary = self.generate_summary(combined_text)
embeddings = OpenAIEmbeddings(api_key=self.api_key)
self.document_store = Chroma.from_documents(documents, embeddings)
self.qa_chain = ConversationalRetrievalChain.from_llm(
ChatOpenAI(temperature=0, model_name='gpt-4', api_key=self.api_key),
self.document_store.as_retriever(search_kwargs={'k': 6}),
return_source_documents=True,
verbose=False
)
self.last_processed_time = datetime.now(pytz.UTC)
return "Documents processed successfully!"
except Exception as e:
return f"Error processing documents: {str(e)}"
def generate_summary(self, text):
"""Generate a summary of the uploaded documents."""
if not self.api_key:
return "API Key not set. Please set it in the environment variables."
try:
client = OpenAI(api_key=self.api_key)
response = client.chat.completions.create(
model="gpt-4",
messages=[
{"role": "system", "content": "Summarize the document content concisely and provide 3-5 key points for discussion."},
{"role": "user", "content": text[:4000]}
],
temperature=0.3
)
return response.choices[0].message.content
except Exception as e:
return f"Error generating summary: {str(e)}"
def handle_query(self, question, history):
if not self.qa_chain:
return history + [("System", "Please process the documents first.")]
try:
preface = """
Instruction: Respond in English. Be professional and concise, keeping the response under 300 words.
If you cannot provide an answer, say: "I am not sure about this question. Please try asking something else."
"""
query = f"{preface}\nQuery: {question}"
result = self.qa_chain({
"question": query,
"chat_history": [(q, a) for q, a in history]
})
if "answer" not in result:
return history + [("System", "Sorry, an error occurred.")]
history.append((question, result["answer"]))
return history
except Exception as e:
return history + [("System", f"Error: {str(e)}")]
# Streamlit UI
st.title("Document Analyzer and Podcast Generator")
# Fetch the API key status
if "OPENAI_API_KEY" not in os.environ or not os.getenv("OPENAI_API_KEY"):
st.error("The 'OPENAI_API_KEY' environment variable is not set. Please configure it in your hosting environment.")
else:
st.success("API Key successfully loaded from environment variable.")
# Initialize RAG system
try:
rag_system = DocumentRAG()
except ValueError as e:
st.error(str(e))
st.stop()
# File upload
st.subheader("Step 1: Upload Documents")
uploaded_files = st.file_uploader("Upload files (PDF, TXT, CSV)", accept_multiple_files=True)
if st.button("Process Documents"):
if uploaded_files:
file_paths = [tempfile.NamedTemporaryFile(delete=False, suffix=os.path.splitext(file.name)[1]).name for file in uploaded_files]
for file, temp_path in zip(uploaded_files, file_paths):
with open(temp_path, 'wb') as temp_file:
temp_file.write(file.read())
st.success(rag_system.process_documents(file_paths))
else:
st.warning("No files uploaded.")
# Document Q&A
st.subheader("Step 2: Ask Questions")
if rag_system.qa_chain:
history = []
user_question = st.text_input("Ask a question:")
if st.button("Submit Question"):
history = rag_system.handle_query(user_question, history)
for question, answer in history:
st.chat_message("user").write(question)
st.chat_message("assistant").write(answer)
else:
st.info("Please process documents before asking questions.")
|