Update local_knowledge_base.py
Browse files- local_knowledge_base.py +49 -49
local_knowledge_base.py
CHANGED
|
@@ -1,50 +1,50 @@
|
|
| 1 |
-
from phi.vectordb.lancedb import LanceDb
|
| 2 |
-
from phi.knowledge.pdf import PDFKnowledgeBase, PDFReader
|
| 3 |
-
from phi.embedder.google import GeminiEmbedder
|
| 4 |
-
from phi.vectordb.search import SearchType
|
| 5 |
-
from phi.utils.log import logger
|
| 6 |
-
import os
|
| 7 |
-
|
| 8 |
-
def load_knowledge_base():
|
| 9 |
-
"""
|
| 10 |
-
Loads or creates a knowledge base from PDF documents in the 'knowledge' folder
|
| 11 |
-
using
|
| 12 |
-
|
| 13 |
-
This version includes a manual loop to process one file at a time,
|
| 14 |
-
which is a robust workaround for a bug in the library's multi-file handling.
|
| 15 |
-
"""
|
| 16 |
-
knowledge_dir = "./knowledge"
|
| 17 |
-
db_dir = "./lance_db"
|
| 18 |
-
table_name = "local_pdf_knowledge"
|
| 19 |
-
|
| 20 |
-
if not os.path.exists(knowledge_dir) or not os.listdir(knowledge_dir):
|
| 21 |
-
logger.warning(f"The '{knowledge_dir}' directory is empty or does not exist. No local knowledge base will be loaded.")
|
| 22 |
-
return None
|
| 23 |
-
|
| 24 |
-
# Get a list of all PDF files to process
|
| 25 |
-
pdf_files = [f for f in os.listdir(knowledge_dir) if f.lower().endswith(".pdf")]
|
| 26 |
-
if not pdf_files:
|
| 27 |
-
logger.warning(f"No PDF files found in the '{knowledge_dir}' directory.")
|
| 28 |
-
return None
|
| 29 |
-
|
| 30 |
-
logger.info("Loading Knowledge Base using LanceDb with manual file processing...")
|
| 31 |
-
|
| 32 |
-
try:
|
| 33 |
-
knowledge_base = PDFKnowledgeBase(
|
| 34 |
-
path=knowledge_dir,
|
| 35 |
-
vector_db=LanceDb(
|
| 36 |
-
table_name=table_name,
|
| 37 |
-
uri=db_dir,
|
| 38 |
-
embedder=GeminiEmbedder(model="models/text-embedding-004"),
|
| 39 |
-
search_type=SearchType.keyword,
|
| 40 |
-
),
|
| 41 |
-
reader=PDFReader(chunk=True)
|
| 42 |
-
)
|
| 43 |
-
|
| 44 |
-
logger.info("All files processed successfully.")
|
| 45 |
-
|
| 46 |
-
return knowledge_base
|
| 47 |
-
|
| 48 |
-
except Exception as e:
|
| 49 |
-
logger.error(f"An unexpected error occurred during manual file loading: {e}")
|
| 50 |
return None
|
|
|
|
| 1 |
+
from phi.vectordb.lancedb import LanceDb
|
| 2 |
+
from phi.knowledge.pdf import PDFKnowledgeBase, PDFReader
|
| 3 |
+
from phi.embedder.google import GeminiEmbedder
|
| 4 |
+
from phi.vectordb.search import SearchType
|
| 5 |
+
from phi.utils.log import logger
|
| 6 |
+
import os
|
| 7 |
+
|
| 8 |
+
def load_knowledge_base():
|
| 9 |
+
"""
|
| 10 |
+
Loads or creates a knowledge base from PDF documents in the 'knowledge' folder
|
| 11 |
+
using LanceDB for storage.
|
| 12 |
+
|
| 13 |
+
This version includes a manual loop to process one file at a time,
|
| 14 |
+
which is a robust workaround for a bug in the library's multi-file handling.
|
| 15 |
+
"""
|
| 16 |
+
knowledge_dir = "./knowledge"
|
| 17 |
+
db_dir = "./lance_db"
|
| 18 |
+
table_name = "local_pdf_knowledge"
|
| 19 |
+
|
| 20 |
+
if not os.path.exists(knowledge_dir) or not os.listdir(knowledge_dir):
|
| 21 |
+
logger.warning(f"The '{knowledge_dir}' directory is empty or does not exist. No local knowledge base will be loaded.")
|
| 22 |
+
return None
|
| 23 |
+
|
| 24 |
+
# Get a list of all PDF files to process
|
| 25 |
+
pdf_files = [f for f in os.listdir(knowledge_dir) if f.lower().endswith(".pdf")]
|
| 26 |
+
if not pdf_files:
|
| 27 |
+
logger.warning(f"No PDF files found in the '{knowledge_dir}' directory.")
|
| 28 |
+
return None
|
| 29 |
+
|
| 30 |
+
logger.info("Loading Knowledge Base using LanceDb with manual file processing...")
|
| 31 |
+
|
| 32 |
+
try:
|
| 33 |
+
knowledge_base = PDFKnowledgeBase(
|
| 34 |
+
path=knowledge_dir,
|
| 35 |
+
vector_db=LanceDb(
|
| 36 |
+
table_name=table_name,
|
| 37 |
+
uri=db_dir,
|
| 38 |
+
embedder=GeminiEmbedder(model="models/text-embedding-004"),
|
| 39 |
+
search_type=SearchType.keyword,
|
| 40 |
+
),
|
| 41 |
+
reader=PDFReader(chunk=True)
|
| 42 |
+
)
|
| 43 |
+
|
| 44 |
+
logger.info("All files processed successfully.")
|
| 45 |
+
|
| 46 |
+
return knowledge_base
|
| 47 |
+
|
| 48 |
+
except Exception as e:
|
| 49 |
+
logger.error(f"An unexpected error occurred during manual file loading: {e}")
|
| 50 |
return None
|