Rahbarnisa commited on
Commit
6613385
·
verified ·
1 Parent(s): b8b8a14

Update src/loader.py

Browse files
Files changed (1) hide show
  1. src/loader.py +27 -12
src/loader.py CHANGED
@@ -1,12 +1,27 @@
1
- from langchain_community.document_loaders import PyPDFLoader
2
- import os
3
-
4
- def load_all_pdfs(folder_path="data"):
5
- documents = []
6
-
7
- for file in os.listdir(folder_path):
8
- if file.endswith(".pdf"):
9
- loader = PyPDFLoader(os.path.join(folder_path, file))
10
- documents.extend(loader.load())
11
-
12
- return documents
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from langchain_community.document_loaders import PyPDFLoader
2
+ import os
3
+
4
+ def load_all_pdfs(folder_path="data"):
5
+ documents = []
6
+
7
+ for file in os.listdir(folder_path):
8
+ if file.endswith(".pdf"):
9
+ loader = PyPDFLoader(os.path.join(folder_path, file))
10
+ documents.extend(loader.load())
11
+
12
+ return documents
13
+
14
+ from datasets import load_dataset
15
+
16
+ dataset = load_dataset("Rahbarnisa/RAG_docs", split="train")
17
+
18
+ import os
19
+ from datasets import load_dataset
20
+
21
+ dataset = load_dataset("Rahbarnisa/RAG_docs", split="train")
22
+
23
+ pdf_files = []
24
+
25
+ for item in dataset:
26
+ # depends on structure, but usually:
27
+ pdf_files.append(item["file"])