Manav0603 commited on
Commit
7f510d0
·
verified ·
1 Parent(s): ef208ea

Update main.py

Browse files
Files changed (1) hide show
  1. main.py +44 -5
main.py CHANGED
@@ -1,10 +1,12 @@
1
- from langchain_community.document_loaders import PyPDFLoader
 
2
  from langchain_community.embeddings import HuggingFaceEmbeddings
3
  from langchain_community.vectorstores import Chroma
4
  from langchain.text_splitter import RecursiveCharacterTextSplitter
5
  from langchain.chains import RetrievalQA
6
  from langchain_community.llms import HuggingFaceHub
7
  import os
 
8
 
9
  # Initialize qa as None; will be updated after PDF upload
10
  qa = None
@@ -14,7 +16,7 @@ def initialize_llm():
14
  """Initialize LLM using HuggingFace API"""
15
  try:
16
  # Get HF token from environment
17
- hf_token = os.getenv("token2")
18
 
19
  if not hf_token:
20
  print("Warning: No HuggingFace token found")
@@ -57,33 +59,70 @@ def process_pdf(pdf_file_path):
57
  if not documents:
58
  return "Error: No content found in PDF"
59
 
 
 
 
 
 
 
 
 
 
 
 
 
60
  # 2️⃣ Split text
61
  text_splitter = RecursiveCharacterTextSplitter(
62
  chunk_size=1000,
63
- chunk_overlap=200
 
64
  )
65
  docs = text_splitter.split_documents(documents)
66
 
 
 
 
 
 
 
 
 
 
 
 
67
  # 3️⃣ Create embeddings
68
  embeddings = HuggingFaceEmbeddings(
69
  model_name="sentence-transformers/all-MiniLM-L6-v2",
70
  model_kwargs={'device': 'cpu'}
71
  )
72
 
 
 
73
  # 4️⃣ Store embeddings in ChromaDB (temp folder)
 
 
 
 
 
74
  vectorstore = Chroma.from_documents(
75
  docs,
76
  embeddings,
77
  persist_directory="./chroma_cache"
78
  )
79
 
 
 
80
  # 5️⃣ Setup RetrievalQA with Mistral model via API
81
  qa = RetrievalQA.from_chain_type(
82
  llm=llm,
83
- retriever=vectorstore.as_retriever(search_kwargs={"k": 3})
 
 
84
  )
85
 
86
- return "PDF processed and QA ready!"
 
 
87
 
88
  except Exception as e:
89
  return f"Error processing PDF: {str(e)}"
 
1
+ # Alternative: if you want to use PyPDF2 instead
2
+ # from langchain_community.document_loaders import PyPDFLoader
3
  from langchain_community.embeddings import HuggingFaceEmbeddings
4
  from langchain_community.vectorstores import Chroma
5
  from langchain.text_splitter import RecursiveCharacterTextSplitter
6
  from langchain.chains import RetrievalQA
7
  from langchain_community.llms import HuggingFaceHub
8
  import os
9
+ import shutil
10
 
11
  # Initialize qa as None; will be updated after PDF upload
12
  qa = None
 
16
  """Initialize LLM using HuggingFace API"""
17
  try:
18
  # Get HF token from environment
19
+ hf_token = os.getenv("HUGGINGFACE_TOKEN")
20
 
21
  if not hf_token:
22
  print("Warning: No HuggingFace token found")
 
59
  if not documents:
60
  return "Error: No content found in PDF"
61
 
62
+ print(f"Loaded {len(documents)} pages from PDF")
63
+
64
+ # Check if documents have content
65
+ total_content = ""
66
+ for doc in documents:
67
+ total_content += doc.page_content
68
+
69
+ if not total_content.strip():
70
+ return "Error: PDF appears to be empty or contains no extractable text"
71
+
72
+ print(f"Total content length: {len(total_content)} characters")
73
+
74
  # 2️⃣ Split text
75
  text_splitter = RecursiveCharacterTextSplitter(
76
  chunk_size=1000,
77
+ chunk_overlap=200,
78
+ length_function=len,
79
  )
80
  docs = text_splitter.split_documents(documents)
81
 
82
+ if not docs:
83
+ return "Error: No text chunks created from PDF"
84
+
85
+ # Filter out empty chunks
86
+ docs = [doc for doc in docs if doc.page_content.strip()]
87
+
88
+ if not docs:
89
+ return "Error: All text chunks are empty after processing"
90
+
91
+ print(f"Created {len(docs)} text chunks")
92
+
93
  # 3️⃣ Create embeddings
94
  embeddings = HuggingFaceEmbeddings(
95
  model_name="sentence-transformers/all-MiniLM-L6-v2",
96
  model_kwargs={'device': 'cpu'}
97
  )
98
 
99
+ print("Embeddings model loaded")
100
+
101
  # 4️⃣ Store embeddings in ChromaDB (temp folder)
102
+ # Clear existing chroma cache first
103
+ import shutil
104
+ if os.path.exists("./chroma_cache"):
105
+ shutil.rmtree("./chroma_cache")
106
+
107
  vectorstore = Chroma.from_documents(
108
  docs,
109
  embeddings,
110
  persist_directory="./chroma_cache"
111
  )
112
 
113
+ print("Vector store created")
114
+
115
  # 5️⃣ Setup RetrievalQA with Mistral model via API
116
  qa = RetrievalQA.from_chain_type(
117
  llm=llm,
118
+ chain_type="stuff",
119
+ retriever=vectorstore.as_retriever(search_kwargs={"k": 3}),
120
+ return_source_documents=False
121
  )
122
 
123
+ print("QA system ready")
124
+
125
+ return f"✅ PDF processed successfully! Found {len(docs)} text chunks ready for questions."
126
 
127
  except Exception as e:
128
  return f"Error processing PDF: {str(e)}"