Spaces:
Build error
Build error
Update utils/database.py
Browse files- utils/database.py +20 -7
utils/database.py
CHANGED
|
@@ -3,11 +3,14 @@ import streamlit as st
|
|
| 3 |
import sqlite3
|
| 4 |
from sqlite3 import Error
|
| 5 |
from datetime import datetime
|
| 6 |
-
from langchain.chains.conversational_retrieval.base import
|
|
|
|
|
|
|
| 7 |
from langchain.memory import ConversationBufferMemory
|
| 8 |
from langchain.chat_models import ChatOpenAI
|
| 9 |
import os
|
| 10 |
|
|
|
|
| 11 |
def create_connection(db_file):
|
| 12 |
try:
|
| 13 |
conn = sqlite3.connect(db_file)
|
|
@@ -16,6 +19,7 @@ def create_connection(db_file):
|
|
| 16 |
st.error(f"Error: {e}")
|
| 17 |
return None
|
| 18 |
|
|
|
|
| 19 |
def create_tables(conn):
|
| 20 |
try:
|
| 21 |
sql_create_documents_table = """
|
|
@@ -56,16 +60,20 @@ def create_tables(conn):
|
|
| 56 |
except Error as e:
|
| 57 |
st.error(f"Error: {e}")
|
| 58 |
|
|
|
|
| 59 |
def get_documents(conn):
|
| 60 |
"""Retrieve documents from database"""
|
| 61 |
try:
|
| 62 |
cursor = conn.cursor()
|
| 63 |
-
cursor.execute(
|
|
|
|
|
|
|
| 64 |
return cursor.fetchall()
|
| 65 |
except Exception as e:
|
| 66 |
st.error(f"Error retrieving documents: {e}")
|
| 67 |
return []
|
| 68 |
|
|
|
|
| 69 |
def insert_document(conn, doc_name, doc_content):
|
| 70 |
"""Insert a document into database"""
|
| 71 |
try:
|
|
@@ -74,13 +82,14 @@ def insert_document(conn, doc_name, doc_content):
|
|
| 74 |
if not cursor.fetchone():
|
| 75 |
conn.execute(
|
| 76 |
"INSERT INTO documents (name, content) VALUES (?, ?)",
|
| 77 |
-
(doc_name, doc_content)
|
| 78 |
)
|
| 79 |
return True
|
| 80 |
except Exception as e:
|
| 81 |
st.error(f"Error inserting document: {e}")
|
| 82 |
return False
|
| 83 |
|
|
|
|
| 84 |
def initialize_qa_system(vector_store):
|
| 85 |
"""Initialize QA system with proper chat handling"""
|
| 86 |
try:
|
|
@@ -92,7 +101,7 @@ def initialize_qa_system(vector_store):
|
|
| 92 |
|
| 93 |
memory = ConversationBufferMemory(
|
| 94 |
memory_key="chat_history",
|
| 95 |
-
return_messages=True
|
| 96 |
)
|
| 97 |
|
| 98 |
qa_chain = ConversationalRetrievalChain.from_llm(
|
|
@@ -100,7 +109,7 @@ def initialize_qa_system(vector_store):
|
|
| 100 |
retriever=vector_store.as_retriever(search_kwargs={"k": 2}),
|
| 101 |
memory=memory,
|
| 102 |
return_source_documents=True,
|
| 103 |
-
verbose=True
|
| 104 |
)
|
| 105 |
|
| 106 |
return qa_chain
|
|
@@ -109,25 +118,29 @@ def initialize_qa_system(vector_store):
|
|
| 109 |
st.error(f"Error initializing QA system: {e}")
|
| 110 |
return None
|
| 111 |
|
|
|
|
| 112 |
def initialize_faiss(embeddings, documents, document_names):
|
| 113 |
"""Initialize FAISS vector store"""
|
| 114 |
try:
|
| 115 |
from langchain.vectorstores import FAISS
|
|
|
|
| 116 |
vector_store = FAISS.from_texts(
|
| 117 |
documents,
|
| 118 |
embeddings,
|
| 119 |
-
metadatas=[{"source": name} for name in document_names]
|
| 120 |
)
|
| 121 |
return vector_store
|
| 122 |
except Exception as e:
|
| 123 |
st.error(f"Error initializing FAISS: {e}")
|
| 124 |
return None
|
| 125 |
|
| 126 |
-
|
|
|
|
| 127 |
def get_embeddings_model():
|
| 128 |
"""Get the embeddings model"""
|
| 129 |
try:
|
| 130 |
from langchain.embeddings import HuggingFaceEmbeddings
|
|
|
|
| 131 |
model_name = "sentence-transformers/all-MiniLM-L6-v2"
|
| 132 |
embeddings = HuggingFaceEmbeddings(model_name=model_name)
|
| 133 |
return embeddings
|
|
|
|
| 3 |
import sqlite3
|
| 4 |
from sqlite3 import Error
|
| 5 |
from datetime import datetime
|
| 6 |
+
from langchain.chains.conversational_retrieval.base import (
|
| 7 |
+
ConversationalRetrievalChain,
|
| 8 |
+
)
|
| 9 |
from langchain.memory import ConversationBufferMemory
|
| 10 |
from langchain.chat_models import ChatOpenAI
|
| 11 |
import os
|
| 12 |
|
| 13 |
+
|
| 14 |
def create_connection(db_file):
|
| 15 |
try:
|
| 16 |
conn = sqlite3.connect(db_file)
|
|
|
|
| 19 |
st.error(f"Error: {e}")
|
| 20 |
return None
|
| 21 |
|
| 22 |
+
|
| 23 |
def create_tables(conn):
|
| 24 |
try:
|
| 25 |
sql_create_documents_table = """
|
|
|
|
| 60 |
except Error as e:
|
| 61 |
st.error(f"Error: {e}")
|
| 62 |
|
| 63 |
+
|
| 64 |
def get_documents(conn):
|
| 65 |
"""Retrieve documents from database"""
|
| 66 |
try:
|
| 67 |
cursor = conn.cursor()
|
| 68 |
+
cursor.execute(
|
| 69 |
+
"SELECT id, name, upload_date FROM documents ORDER BY upload_date DESC"
|
| 70 |
+
)
|
| 71 |
return cursor.fetchall()
|
| 72 |
except Exception as e:
|
| 73 |
st.error(f"Error retrieving documents: {e}")
|
| 74 |
return []
|
| 75 |
|
| 76 |
+
|
| 77 |
def insert_document(conn, doc_name, doc_content):
|
| 78 |
"""Insert a document into database"""
|
| 79 |
try:
|
|
|
|
| 82 |
if not cursor.fetchone():
|
| 83 |
conn.execute(
|
| 84 |
"INSERT INTO documents (name, content) VALUES (?, ?)",
|
| 85 |
+
(doc_name, doc_content),
|
| 86 |
)
|
| 87 |
return True
|
| 88 |
except Exception as e:
|
| 89 |
st.error(f"Error inserting document: {e}")
|
| 90 |
return False
|
| 91 |
|
| 92 |
+
|
| 93 |
def initialize_qa_system(vector_store):
|
| 94 |
"""Initialize QA system with proper chat handling"""
|
| 95 |
try:
|
|
|
|
| 101 |
|
| 102 |
memory = ConversationBufferMemory(
|
| 103 |
memory_key="chat_history",
|
| 104 |
+
return_messages=True,
|
| 105 |
)
|
| 106 |
|
| 107 |
qa_chain = ConversationalRetrievalChain.from_llm(
|
|
|
|
| 109 |
retriever=vector_store.as_retriever(search_kwargs={"k": 2}),
|
| 110 |
memory=memory,
|
| 111 |
return_source_documents=True,
|
| 112 |
+
verbose=True,
|
| 113 |
)
|
| 114 |
|
| 115 |
return qa_chain
|
|
|
|
| 118 |
st.error(f"Error initializing QA system: {e}")
|
| 119 |
return None
|
| 120 |
|
| 121 |
+
|
| 122 |
def initialize_faiss(embeddings, documents, document_names):
|
| 123 |
"""Initialize FAISS vector store"""
|
| 124 |
try:
|
| 125 |
from langchain.vectorstores import FAISS
|
| 126 |
+
|
| 127 |
vector_store = FAISS.from_texts(
|
| 128 |
documents,
|
| 129 |
embeddings,
|
| 130 |
+
metadatas=[{"source": name} for name in document_names],
|
| 131 |
)
|
| 132 |
return vector_store
|
| 133 |
except Exception as e:
|
| 134 |
st.error(f"Error initializing FAISS: {e}")
|
| 135 |
return None
|
| 136 |
|
| 137 |
+
|
| 138 |
+
@st.cache_resource
|
| 139 |
def get_embeddings_model():
|
| 140 |
"""Get the embeddings model"""
|
| 141 |
try:
|
| 142 |
from langchain.embeddings import HuggingFaceEmbeddings
|
| 143 |
+
|
| 144 |
model_name = "sentence-transformers/all-MiniLM-L6-v2"
|
| 145 |
embeddings = HuggingFaceEmbeddings(model_name=model_name)
|
| 146 |
return embeddings
|