Update app.py

app.py

@@ -1,28 +1,29 @@
 import streamlit as st
-from dotenv import load_dotenv
+import os
+import time
+import logging
+import io
+import requests
+from bs4 import BeautifulSoup
 from PyPDF2 import PdfReader
+import pdfplumber
+import docx
 from langchain.text_splitter import CharacterTextSplitter
 from langchain.embeddings.openai import OpenAIEmbeddings
 from langchain.vectorstores import FAISS
 from langchain.chains.question_answering import load_qa_chain
 from langchain.llms import OpenAI
-import time
-import logging
-import pdfplumber
-import os
-import requests
-from bs4 import BeautifulSoup
-import docx  # Importing docx for Word document processing
 
 
 def fetch_and_process_pdf(url):
-    response = requests.get(url)
-    if response.status_code == 200:
+    try:
+        response = requests.get(url)
+        response.raise_for_status()
         pdf_file = io.BytesIO(response.content)
         text = process_pdf(pdf_file)
         return text
-    else:
-        logging.error(f"Failed to fetch PDF from {url}. Status Code: {response.status_code}")
+    except requests.HTTPError as e:
+        logging.error(f"Failed to fetch PDF from {url}. Error: {e}")
         return ""
 
 def process_pdf(pdf):
@@ -78,7 +79,17 @@ def read_documents_from_directory(directory):
             combined_text += read_word(file_path)
     return combined_text
 	
-	
+def get_pdf_links_from_dataset(url):
+    try:
+        response = requests.get(url)
+        response.raise_for_status()
+        soup = BeautifulSoup(response.text, 'html.parser')
+        pdf_links = [link.get('href') for link in soup.find_all('a') if '.pdf' in link.get('href')]
+        return pdf_links
+    except requests.HTTPError as e:
+        logging.error(f"Failed to get PDF links from dataset. Error: {e}")
+        return []
+        
 #train_directory = r'C:\Users\writa\Downloads\Crypto'
 url = "https://huggingface.co/datasets/Writo/realestate_data/tree/main"
 
@@ -90,15 +101,18 @@ def main():
     # Ensure train_directory is accessible in Hugging Face Space
     #text = read_documents_from_directory(train_directory)
 
-    def get_pdf_links_from_dataset(url):
-        response = requests.get(url)
-        soup = BeautifulSoup(response.text, 'html.parser')
-        pdf_links = [link.get('href') for link in soup.find_all('a') if link.get('href').endswith('.pdf')]
-        return pdf_links
-
     dataset_url = 'https://huggingface.co/datasets/Writo/realestate_data/tree/main'
     pdf_links = get_pdf_links_from_dataset(dataset_url)
-    print(pdf_links)
+
+    if pdf_links:
+        with st.spinner("Processing PDFs, please wait..."):
+            text = ""
+            for link in pdf_links:
+                text += fetch_and_process_pdf(link)
+
+    
+		
+	text = read_documents_from_directory(train_directory)
 
     # Processing text and setting up the AI model
     char_text_splitter = CharacterTextSplitter(separator="\n", chunk_size=1000,