anupamdas commited on
Commit
7399ef0
·
verified ·
1 Parent(s): 80b2a07

Update local_knowledge_base.py

Browse files
Files changed (1) hide show
  1. local_knowledge_base.py +49 -49
local_knowledge_base.py CHANGED
@@ -1,50 +1,50 @@
1
- from phi.vectordb.lancedb import LanceDb
2
- from phi.knowledge.pdf import PDFKnowledgeBase, PDFReader
3
- from phi.embedder.google import GeminiEmbedder
4
- from phi.vectordb.search import SearchType
5
- from phi.utils.log import logger
6
- import os
7
-
8
- def load_knowledge_base():
9
- """
10
- Loads or creates a knowledge base from PDF documents in the 'knowledge' folder
11
- using ChromaDB for storage.
12
-
13
- This version includes a manual loop to process one file at a time,
14
- which is a robust workaround for a bug in the library's multi-file handling.
15
- """
16
- knowledge_dir = "./knowledge"
17
- db_dir = "./lance_db"
18
- table_name = "local_pdf_knowledge"
19
-
20
- if not os.path.exists(knowledge_dir) or not os.listdir(knowledge_dir):
21
- logger.warning(f"The '{knowledge_dir}' directory is empty or does not exist. No local knowledge base will be loaded.")
22
- return None
23
-
24
- # Get a list of all PDF files to process
25
- pdf_files = [f for f in os.listdir(knowledge_dir) if f.lower().endswith(".pdf")]
26
- if not pdf_files:
27
- logger.warning(f"No PDF files found in the '{knowledge_dir}' directory.")
28
- return None
29
-
30
- logger.info("Loading Knowledge Base using LanceDb with manual file processing...")
31
-
32
- try:
33
- knowledge_base = PDFKnowledgeBase(
34
- path=knowledge_dir,
35
- vector_db=LanceDb(
36
- table_name=table_name,
37
- uri=db_dir,
38
- embedder=GeminiEmbedder(model="models/text-embedding-004"),
39
- search_type=SearchType.keyword,
40
- ),
41
- reader=PDFReader(chunk=True)
42
- )
43
-
44
- logger.info("All files processed successfully.")
45
-
46
- return knowledge_base
47
-
48
- except Exception as e:
49
- logger.error(f"An unexpected error occurred during manual file loading: {e}")
50
  return None
 
1
+ from phi.vectordb.lancedb import LanceDb
2
+ from phi.knowledge.pdf import PDFKnowledgeBase, PDFReader
3
+ from phi.embedder.google import GeminiEmbedder
4
+ from phi.vectordb.search import SearchType
5
+ from phi.utils.log import logger
6
+ import os
7
+
8
+ def load_knowledge_base():
9
+ """
10
+ Loads or creates a knowledge base from PDF documents in the 'knowledge' folder
11
+ using LanceDB for storage.
12
+
13
+ This version includes a manual loop to process one file at a time,
14
+ which is a robust workaround for a bug in the library's multi-file handling.
15
+ """
16
+ knowledge_dir = "./knowledge"
17
+ db_dir = "./lance_db"
18
+ table_name = "local_pdf_knowledge"
19
+
20
+ if not os.path.exists(knowledge_dir) or not os.listdir(knowledge_dir):
21
+ logger.warning(f"The '{knowledge_dir}' directory is empty or does not exist. No local knowledge base will be loaded.")
22
+ return None
23
+
24
+ # Get a list of all PDF files to process
25
+ pdf_files = [f for f in os.listdir(knowledge_dir) if f.lower().endswith(".pdf")]
26
+ if not pdf_files:
27
+ logger.warning(f"No PDF files found in the '{knowledge_dir}' directory.")
28
+ return None
29
+
30
+ logger.info("Loading Knowledge Base using LanceDb with manual file processing...")
31
+
32
+ try:
33
+ knowledge_base = PDFKnowledgeBase(
34
+ path=knowledge_dir,
35
+ vector_db=LanceDb(
36
+ table_name=table_name,
37
+ uri=db_dir,
38
+ embedder=GeminiEmbedder(model="models/text-embedding-004"),
39
+ search_type=SearchType.keyword,
40
+ ),
41
+ reader=PDFReader(chunk=True)
42
+ )
43
+
44
+ logger.info("All files processed successfully.")
45
+
46
+ return knowledge_base
47
+
48
+ except Exception as e:
49
+ logger.error(f"An unexpected error occurred during manual file loading: {e}")
50
  return None