from langchain_community.vectorstores import FAISS import os from langchain.text_splitter import CharacterTextSplitter import json import os import random import re from concurrent.futures import ThreadPoolExecutor, as_completed import google.generativeai as genai import nltk import pandas as pd from groq import Groq from langchain.chains.summarize import load_summarize_chain from langchain.docstore.document import Document from langchain.prompts import PromptTemplate from langchain_community.retrievers import BM25Retriever from langchain.retrievers import EnsembleRetriever from langchain.retrievers.contextual_compression import ContextualCompressionRetriever from langchain.text_splitter import CharacterTextSplitter from langchain.text_splitter import RecursiveCharacterTextSplitter from langchain_cohere import CohereRerank from langchain_community.document_loaders import Docx2txtLoader from langchain_community.document_loaders import TextLoader from langchain_community.document_loaders import UnstructuredCSVLoader from langchain_community.document_loaders import UnstructuredExcelLoader from langchain_community.document_loaders import UnstructuredHTMLLoader from langchain_community.document_loaders import UnstructuredMarkdownLoader from langchain_community.document_loaders import UnstructuredPDFLoader from langchain_community.document_loaders import UnstructuredPowerPointLoader from langchain_community.document_loaders import UnstructuredXMLLoader from langchain_community.document_loaders.csv_loader import CSVLoader from langchain_community.llms import Cohere from langchain_community.vectorstores import Chroma from langchain_core.output_parsers.openai_tools import PydanticToolsParser from langchain_core.prompts import ChatPromptTemplate from langchain_core.pydantic_v1 import BaseModel, Field from langchain_core.runnables import RunnablePassthrough from langchain_openai import ChatOpenAI from typing import List nltk.download('punkt') from dotenv import load_dotenv load_dotenv() GROQ_API_KEY = os.getenv("GROQ_API_KEY") COHERE_API_KEY = os.getenv("COHERE_API_KEY") OPENAI_API_KEY = os.getenv("OPENAI_API_KEY") GOOGLE_API_KEY1= os.getenv("GOOGLE_API_KEY_1") GOOGLE_API_KEY= os.getenv("GOOGLE_API_KEY") client = Groq( api_key= GROQ_API_KEY, ) genai.configure(api_key=GOOGLE_API_KEY) os.environ["GOOGLE_API_KEY"] = GOOGLE_API_KEY from langchain_google_genai import GoogleGenerativeAIEmbeddings, ChatGoogleGenerativeAI from langchain_google_vertexai import VertexAIEmbeddings from langchain_huggingface import HuggingFaceEmbeddings embeddings = GoogleGenerativeAIEmbeddings(model="models/embedding-001", task_type="retrieval_document") llm = ChatGoogleGenerativeAI(model='gemini-2.0-flash-thinking-exp-01-21',temperature=0.6) def check_persist_directory(id, file_name): directory_path = f"./vector_database/{file_name}" check = os.path.exists(directory_path) return check def check_path_exists(path): return os.path.exists(path) def load_file(loader): return loader.load() from langchain_community.document_loaders import PyPDFLoader def extract_data2(): documents = [] base_dir = os.path.dirname(os.path.abspath(__file__)) directory_path = os.path.join(base_dir, "..", "data") # Chuẩn hóa đường dẫn directory_path = os.path.abspath(directory_path) if not os.path.exists(directory_path) or not any( os.path.isfile(os.path.join(directory_path, f)) for f in os.listdir(directory_path)): return False tasks = [] with ThreadPoolExecutor() as executor: for file in os.listdir(directory_path): if file.endswith(".pdf"): pdf_path = os.path.join(directory_path, file) loader = PyPDFLoader(pdf_path) tasks.append(executor.submit(load_file, loader)) elif file.endswith('.docx') or file.endswith('.doc'): doc_path = os.path.join(directory_path, file) loader = Docx2txtLoader(doc_path) tasks.append(executor.submit(load_file, loader)) elif file.endswith('.txt'): txt_path = os.path.join(directory_path, file) loader = TextLoader(txt_path, encoding="utf8") tasks.append(executor.submit(load_file, loader)) elif file.endswith('.pptx'): ppt_path = os.path.join(directory_path, file) loader = UnstructuredPowerPointLoader(ppt_path) tasks.append(executor.submit(load_file, loader)) elif file.endswith('.csv'): csv_path = os.path.join(directory_path, file) loader = UnstructuredCSVLoader(csv_path) tasks.append(executor.submit(load_file, loader)) elif file.endswith('.xlsx'): excel_path = os.path.join(directory_path, file) loader = UnstructuredExcelLoader(excel_path) tasks.append(executor.submit(load_file, loader)) elif file.endswith('.json'): json_path = os.path.join(directory_path, file) loader = TextLoader(json_path) tasks.append(executor.submit(load_file, loader)) elif file.endswith('.md'): md_path = os.path.join(directory_path, file) loader = UnstructuredMarkdownLoader(md_path) tasks.append(executor.submit(load_file, loader)) for future in as_completed(tasks): result = future.result() documents.extend(result) text_splitter = CharacterTextSplitter(chunk_size=1500, chunk_overlap=500) texts = text_splitter.split_documents(documents) Chroma.from_documents(documents=texts, embedding=embeddings, persist_directory=f"./vector_database") return texts class Search(BaseModel): queries: List[str] = Field( ..., description="Truy vấn riêng biệt để tìm kiếm, giữ nguyên ý chính câu hỏi riêng biệt", ) def query_analyzer(query): output_parser = PydanticToolsParser(tools=[Search]) system = """Bạn có khả năng đưa ra các truy vấn tìm kiếm chính xác để lấy thông tin giúp trả lời các yêu cầu của người dùng. Các truy vấn của bạn phải chính xác, không được bỏ ngắn rút gọn. Nếu bạn cần tra cứu hai hoặc nhiều thông tin riêng biệt, bạn có thể làm điều đó!. Trả lời câu hỏi bằng tiếng Việt(Vietnamese), không được dùng ngôn ngữ khác. Bạn chỉ cần tách câu hỏi khi cần thiết hoặc giữ nguyên câu hỏi vui lòng là câu hỏi không phải tên của người dùng""" prompt = ChatPromptTemplate.from_messages( [ ("system", system), ("human", "{question}"), ] ) llm = ChatOpenAI(model="gpt-4o-mini", temperature=0.0) structured_llm = llm.with_structured_output(Search) query_analyzer = {"question": RunnablePassthrough()} | prompt | structured_llm text = query_analyzer.invoke(query) return text def chat_llama3(prompt_query): try: chat_completion = client.chat.completions.create( messages=[ { "role": "system", "content": "Bạn là một trợ lý trung thưc, trả lời dựa trên nội dung tài liệu được cung cấp. Chỉ trả lời liên quan đến câu hỏi một cách đầy đủ chính xác, không bỏ sót thông tin." }, { "role": "user", "content": f"{prompt_query}", } ], model="llama3-70b-8192", temperature=0.0, max_tokens=9000, stop=None, stream=False, ) return chat_completion.choices[0].message.content except Exception as error: return False import os import os def extract_multi_metadata_content(texts, tests): extracted_content = [] for idx, test in enumerate(tests): test_filename = os.path.basename(test).lower() temp_content = [] for x in texts: source_path = x.metadata.get('source', '') source_filename = os.path.basename(source_path).lower() if source_filename == test_filename: temp_content.append(x.page_content) if not temp_content: print(f"[!] Không tìm thấy nội dung cho file {test_filename}") if idx == 0: extracted_content.append(f"Dữ liệu của {test}:\n{''.join(temp_content)}") else: extracted_content.append(''.join(temp_content)) return '\n'.join(extracted_content) def find_matching_files_in_docs_12_id(text, id): base_dir = os.path.dirname(os.path.abspath(__file__)) directory_path = os.path.join(base_dir, "..", "data") # Chuẩn hóa đường dẫn directory_path = os.path.abspath(directory_path) folder_path = directory_path search_terms = [] search_terms_old = [] matching_index = [] search_origin = re.findall(r'\b\w+\.\w+\b|\b\w+\b', text) search_terms_origin = [] for word in search_origin: if '.' in word: search_terms_origin.append(word) else: search_terms_origin.extend(re.findall(r'\b\w+\b', word)) file_names_with_extension = re.findall(r'\b\w+\.\w+\b|\b\w+\b', text.lower()) file_names_with_extension_old = re.findall(r'\b(\w+\.\w+)\b', text) for file_name in search_terms_origin: if "." in file_name: term_position = search_terms_origin.index(file_name) search_terms_old.append(file_name) for file_name in file_names_with_extension_old: if "." in file_name: search_terms_old.append(file_name) for file_name in file_names_with_extension: search_terms.append(file_name) clean_text_old = text clean_text = text.lower() search_terms_old1 = list(set(search_terms_old)) for term in search_terms_old: clean_text_old = clean_text_old.replace(term, '') for term in search_terms: clean_text = clean_text.replace(term, '') words_old = re.findall(r'\b\w+\b', clean_text_old) search_terms_old.extend(words_old) matching_files = set() matching_files_old = set() for root, dirs, files in os.walk(folder_path): for file in files: for term in search_terms: if term.lower() in file.lower(): term_position = search_terms.index(term) term_value = search_terms_origin[term_position] matching_files.add(file) matching_index.append(term_position) break matching_files_old1 = [] matching_index.sort() for x in matching_index: matching_files_old1.append(search_terms_origin[x]) return matching_files, matching_files_old1 def convert_xlsx_to_csv(xlsx_file_path, csv_file_path): df = pd.read_excel(xlsx_file_path) df.to_csv(csv_file_path, index=False) def save_list_CSV_id(file_list, id): text = "" for x in file_list: if x.endswith('.xlsx'): old = f"./user_file/{id}/{x}" new = old.replace(".xlsx", ".csv") convert_xlsx_to_csv(old, new) x = x.replace(".xlsx", ".csv") loader1 = CSVLoader(f"./user_file/{id}/{x}") docs1 = loader1.load() text += f"Dữ liệu file {x}:\n" for z in docs1: text += z.page_content + "\n" return text def merge_files(file_set, file_list): """Hàm này ghép lại các tên file dựa trên điều kiện đã cho.""" merged_files = {} for file_name in file_list: name = file_name.split('.')[0] for f in file_set: if name in f: merged_files[name] = f break return merged_files def replace_keys_with_values(original_dict, replacement_dict): new_dict = {} for key, value in original_dict.items(): if key in replacement_dict: new_key = replacement_dict[key] new_dict[new_key] = value else: new_dict[key] = value return new_dict def aws1_csv_id(new_dict_csv, id): text = "" query_all = "" keyword = [] for key, value in new_dict_csv.items(): query_all += value keyword.append(key) test = save_list_CSV_id(keyword, id) text += test sources = ",".join(keyword) return text, query_all, sources def chat_gemini(prompt): generation_config = { "temperature": 0.0, "top_p": 0.0, "top_k": 0, "max_output_tokens": 8192, } safety_settings = [ { "category": "HARM_CATEGORY_HARASSMENT", "threshold": "BLOCK_MEDIUM_AND_ABOVE" }, { "category": "HARM_CATEGORY_HATE_SPEECH", "threshold": "BLOCK_MEDIUM_AND_ABOVE" }, { "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", "threshold": "BLOCK_MEDIUM_AND_ABOVE" }, { "category": "HARM_CATEGORY_DANGEROUS_CONTENT", "threshold": "BLOCK_MEDIUM_AND_ABOVE" }, ] model = genai.GenerativeModel(model_name="gemini-2.0-flash", generation_config=generation_config, safety_settings=safety_settings) convo = model.start_chat(history=[]) convo.send_message(prompt) return convo.last.text def question_answer(question): # Hàm sinh ra câu trả lời try: answer = chat_gemini(question) return answer except: completion = chat_llama3(question) if completion: return completion #bước 5,6 def aws1_all_id(new_dict, text_alls, id, thread_id): answer = "" COHERE_API_KEY1 = os.getenv("COHERE_API_KEY_2") os.environ["COHERE_API_KEY"] = COHERE_API_KEY1 answer_relevant = "" directory = "" for key, value in new_dict.items(): query = value keyword, keyword2 = find_matching_files_in_docs_12_id(query, id) # Tìm kiếm có từ khóa file trong câu hay không data = extract_multi_metadata_content(text_alls, keyword) # lấy tất cả data liên quan đến keyword if keyword: # Extraction -> Spliting -> Embedding file_name = next(iter(keyword)) text_splitter = CharacterTextSplitter(chunk_size=3200, chunk_overlap=1500) texts_data = text_splitter.split_text(data) # bước 5: retrieal <-> retriever file if check_persist_directory(id, file_name): vectordb_query = Chroma(persist_directory=f"./vector_database/{file_name}", embedding_function=embeddings) else: vectordb_query = Chroma.from_texts(texts_data, embedding=embeddings, persist_directory=f"./vector_database/{file_name}") # bước 6 # fustion retriever + bm25 k_1 = len(texts_data) retriever = vectordb_query.as_retriever(search_kwargs={f"k": k_1}) # Truy vấn từ vectordatabase # Thiết lập BM25 bm25_retriever = BM25Retriever.from_texts(texts_data) # Truy vấn từ đoạn text trích xuất đước bm25_retriever.k = k_1 # Thiết lập số đoạn #Kết hợp cả 2 loại retriever ensemble_retriever = EnsembleRetriever(retrievers=[bm25_retriever, retriever], weights=[0.7, 0.4]) # Lấy kết quả truy vấn docs = ensemble_retriever.get_relevant_documents(f"{query}") # storage and retriever query # Bước 6 --> Lưu vào FAISS path = f"./vector_database/FAISS/{file_name}" if check_path_exists(path): docsearch = FAISS.load_local(path, embeddings, allow_dangerous_deserialization=True) else: docsearch = FAISS.from_documents(docs, embeddings) docsearch.save_local(f"./vector_database/FAISS/{file_name}") docsearch = FAISS.load_local(path, embeddings, allow_dangerous_deserialization=True) k_2 = len(docs) #bước 7 DÙNG cohere để xếp hạng lại các đoạn liên quan compressor = CohereRerank(top_n=10,model = "rerank-multilingual-v3.0") retrieve3 = docsearch.as_retriever(search_kwargs={f"k": k_2}) compression_retriever = ContextualCompressionRetriever( base_compressor=compressor, base_retriever=retrieve3 ) compressed_docs = compression_retriever.get_relevant_documents(f"{query}") # Lấy các đoạn liên quan (10 đoạn) # bước 8 if compressed_docs: data = compressed_docs[0].page_content text = ''.join(map(lambda x: x.page_content, compressed_docs)) # Tổng hợp 10 đoạn liên quan nhất để bổ sung ngữ nghĩa prompt_document = f"Dựa vào nội dung sau:{text}. Hãy trả lời câu hỏi sau đây: {query}. Mà không thay đổi nội dung mà mình đã cung cấp" answer_for = question_answer(prompt_document) # Dùng gemini để response answer += answer_for + "\n" answer_relevant = data directory = file_name return answer, answer_relevant, directory def extract_content_between_keywords(query, keywords): contents = {} num_keywords = len(keywords) keyword_positions = [] for i in range(num_keywords): keyword = keywords[i] keyword_position = query.find(keyword) keyword_positions.append(keyword_position) if keyword_position == -1: continue next_keyword_position = len(query) for j in range(i + 1, num_keywords): next_keyword = keywords[j] next_keyword_position = query.find(next_keyword) if next_keyword_position != -1: break if i == 0: content_before = query[:keyword_position].strip() else: content_before = query[keyword_positions[i - 1] + len(keywords[i - 1]):keyword_position].strip() if i == num_keywords - 1: content_after = query[keyword_position + len(keyword):].strip() else: content_after = query[keyword_position + len(keyword):next_keyword_position].strip() content = f"{content_before} {keyword} {content_after}" contents[keyword] = content return contents def handle_query(question, text_all, compression_retriever, id, thread_id): COHERE_API_KEY_3 = os.environ["COHERE_API_KEY_3"] os.environ["COHERE_API_KEY"] = COHERE_API_KEY_3 query = question x = query # Tìm kiếm keyword liên quan keyword, key_words_old = find_matching_files_in_docs_12_id(query, id) file_list = keyword #bước 4 --> Nhánh có key word if file_list: list_keywords2 = list(key_words_old) contents1 = extract_content_between_keywords(query, list_keywords2) merged_result = merge_files(keyword, list_keywords2) original_dict = contents1 replacement_dict = merged_result new_dict = replace_keys_with_values(original_dict, replacement_dict) files_to_remove = [filename for filename in new_dict.keys() if filename.endswith('.xlsx') or filename.endswith('.csv')] removed_files = {} for filename in files_to_remove: removed_files[filename] = new_dict[filename] for filename in files_to_remove: new_dict.pop(filename) test_csv = "" text_csv, query_csv, source = aws1_csv_id(removed_files, id) prompt_csv = "" answer_csv = "" if test_csv: prompt_csv = f"Dựa vào nội dung sau: {text_csv}. Hãy trả lời câu hỏi sau đây: {query_csv}. Bằng tiếng Việt" answer_csv = question_answer(prompt_csv) answer_document, data_relevant, source = aws1_all_id(new_dict, text_all, id, thread_id) answer_all1 = answer_document + answer_csv return answer_all1, data_relevant, source #bước 4 --> Nhánh không có keyword else: compressed_docs = compression_retriever.get_relevant_documents(f"{query}") relevance_score_float = float(compressed_docs[0].metadata['relevance_score']) # Xử lý khi điểm số thấp if relevance_score_float <= 0: documents1 = [] for file in os.listdir(f"./data"): if file.endswith('.csv'): csv_path = f"./data" + file loader = UnstructuredCSVLoader(csv_path) documents1.extend(loader.load()) elif file.endswith('.xlsx'): excel_path = f"./data" + file loader = UnstructuredExcelLoader(excel_path) documents1.extend(loader.load()) text_splitter_csv = CharacterTextSplitter.from_tiktoken_encoder(chunk_size=2200, chunk_overlap=1500) texts_csv = text_splitter_csv.split_documents(documents1) vectordb_csv = Chroma.from_documents(documents=texts_csv, embedding=embeddings, persist_directory=f'./vector_database/csv/{thread_id}') k = len(texts_csv) retriever_csv = vectordb_csv.as_retriever(search_kwargs={"k": k}) llm = Cohere(temperature=0) compressor_csv = CohereRerank(top_n=3, model="rerank-multilingual-v3.0") compression_retriever_csv = ContextualCompressionRetriever( base_compressor=compressor_csv, base_retriever=retriever_csv ) compressed_docs_csv = compression_retriever_csv.get_relevant_documents(f"{query}") file_path = compressed_docs_csv[0].metadata['source'] if file_path.endswith('.xlsx'): new = file_path.replace(".xlsx", ".csv") convert_xlsx_to_csv(file_path, new) loader1 = CSVLoader(new) else: loader1 = CSVLoader(file_path) docs1 = loader1.load() text = " " for z in docs1: text += z.page_content + "\n" prompt_csv = f"Dựa vào nội dung sau: {text}. Hãy trả lời câu hỏi sau đây: {query}. Bằng tiếng Việt" answer_csv = question_answer(prompt_csv) return answer_csv else: #bước 4 - trích xuất thông tin file liên quan nhất file_path = compressed_docs[0].metadata['source'] file_path = file_path.replace('\\', '/') if file_path.endswith(".pdf"): loader = PyPDFLoader(file_path) elif file_path.endswith('.docx') or file_path.endswith('doc'): loader = Docx2txtLoader(file_path) elif file_path.endswith('.txt'): loader = TextLoader(file_path, encoding="utf8") elif file_path.endswith('.pptx'): loader = UnstructuredPowerPointLoader(file_path) elif file_path.endswith('.xml'): loader = UnstructuredXMLLoader(file_path) elif file_path.endswith('.html'): loader = UnstructuredHTMLLoader(file_path) elif file_path.endswith('.json'): loader = TextLoader(file_path) elif file_path.endswith('.md'): loader = UnstructuredMarkdownLoader(file_path) elif file_path.endswith('.xlsx'): file_path_new = file_path.replace(".xlsx", ".csv") convert_xlsx_to_csv(file_path, file_path_new) loader = CSVLoader(file_path_new) elif file_path.endswith('.csv'): loader = CSVLoader(file_path) # Extraction -> Spliting -> Embedding(phân tách chia file và nhúng vào Chroma) text_splitter = CharacterTextSplitter(chunk_size=3200, chunk_overlap=1500) texts = text_splitter.split_documents(loader.load()) k_1 = len(texts) # Bước 5 (Lưu vào trong Chroma) file_name = os.path.basename(file_path) if check_persist_directory(id, file_name): vectordb_file = Chroma(persist_directory=f"./vector_database/{file_name}", embedding_function=embeddings) else: vectordb_file = Chroma.from_documents(texts, embedding=embeddings, persist_directory=f"./vector_database/{file_name}") # set up bm25 (Bước 6) retriever_file = vectordb_file.as_retriever(search_kwargs={f"k": k_1}) # Truy vấn từ nội dung được lưu trong vectordb Chroma bm25_retriever = BM25Retriever.from_documents(texts) # Truy vấn từ tài liệu extract được trong bước Extraction -> Spliting -> Embedding bm25_retriever.k = k_1 #Kết hợp truy vấn ensemble_retriever = EnsembleRetriever(retrievers=[bm25_retriever, retriever_file], weights=[0.6, 0.4]) docs = ensemble_retriever.get_relevant_documents(f"{query}") import hashlib file_name = os.path.basename(file_path) hash_name = hashlib.md5(file_name.encode('utf-8')).hexdigest() path = f"./vector_database/FAISS/{hash_name}" # Kiểm tra thư mục tồn tại if os.path.exists(path): docsearch = FAISS.load_local(path, embeddings, allow_dangerous_deserialization=True) else: docsearch = FAISS.from_documents(docs, embeddings) docsearch.save_local(path) # Dùng biến path luôn cho thống nhất docsearch = FAISS.load_local(path, embeddings, allow_dangerous_deserialization=True) k_2 = len(docs) #Dùng cohere truy vấn rerank xếp hạng lại một lần nx retrieve3 = docsearch.as_retriever(search_kwargs={f"k": k_2}) compressor_file = CohereRerank(top_n=10, model="rerank-multilingual-v3.0") # Sử dụng cohere để rerank compression_retriever_file = ContextualCompressionRetriever( base_compressor=compressor_file, base_retriever=retrieve3 ) compressed_docs_file = compression_retriever_file.get_relevant_documents(f"{x}") query = question text = ''.join(map(lambda x: x.page_content, compressed_docs_file)) # Tổng hợp lại các đoạn liên quan để trả lời prompt = f"Dựa vào nội dung sau:{text}. Hãy trả lời câu hỏi sau đây: {query}. Mà không thay đổi, chỉnh sửa nội dung mà mình đã cung cấp" answer = question_answer(prompt) # Tạo ra câu trả lời list_relevant = compressed_docs_file[0].page_content source = file_name return answer, list_relevant, source def handle_query_upgrade_keyword_old(query_all, text_all, id): COHERE_API_KEY_2 = os.environ["COHERE_API_KEY_2"] #bước 3 os.environ["COHERE_API_KEY"] = COHERE_API_KEY_2 # phân tách câu hỏi ng dùng test = query_analyzer(query_all) #nhận câu truy vấn được phân tích thành nhiều ý nhỏ test_string = str(test) #lấy list câu hỏi matches = re.findall(r"'([^']*)'", test_string) vectordb = Chroma(persist_directory=f"./vector_database", embedding_function=embeddings) k = len(text_all) retriever = vectordb.as_retriever(search_kwargs={"k": k}) compressor = CohereRerank(top_n=5, model="rerank-multilingual-v3.0") compression_retriever = ContextualCompressionRetriever(base_compressor=compressor, base_retriever= retriever) with ThreadPoolExecutor() as executor: futures = {executor.submit(handle_query, query, text_all, compression_retriever, id, i): query for i, query in enumerate(matches)} results = [] data_relevant = [] sources = [] for future in as_completed(futures): try: result, list_data, list_source = future.result() results.append(result) data_relevant.append(list_data) sources.append(list_source) except Exception as e: print(f'An error occurred: {e}') answer_all = ''.join(results) prompt1 = f"Dựa vào nội dung sau:{answer_all}. Hãy trả lời câu hỏi sau đây: {query_all}. Mà không thay đổi, chỉnh sửa nội dung mà mình đã cung cấp" answer1 = question_answer(prompt1) return answer1, data_relevant, sources # text_all1 = extract_data2() # data = handle_query_upgrade_keyword_old("Tên người làm cùng khóa luận tốt nghiệp với Võ Như Ý trong file KLTN_20133118_20133080_Tuy_chinh_chatbot ",text_all1,"hello") # print(data[0])