PBThuong96 commited on
Commit
927ef83
·
verified ·
1 Parent(s): 1ac8620

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +267 -458
app.py CHANGED
@@ -1,6 +1,8 @@
1
- # app.py - DeepMed AI - Fixed Version
2
- import os
3
  import sys
 
 
 
4
  import logging
5
  import traceback
6
  import gradio as gr
@@ -9,513 +11,320 @@ import docx2txt
9
  import chromadb
10
  from chromadb.config import Settings
11
  from shutil import rmtree
12
- import gc
13
-
14
- # Fix SQLite for Hugging Face
15
- try:
16
- __import__('pysqlite3')
17
- sys.modules['sqlite3'] = sys.modules.pop('pysqlite3')
18
- except ImportError:
19
- logging.warning("pysqlite3 not found, using default sqlite3")
20
 
21
- # LangChain imports
22
- try:
23
- from langchain_google_genai import ChatGoogleGenerativeAI
24
- from langchain_chroma import Chroma
25
- from langchain_community.document_loaders import PyPDFLoader
26
- from langchain_text_splitters import RecursiveCharacterTextSplitter
27
- from langchain_community.retrievers import BM25Retriever
28
- from langchain.retrievers.ensemble import EnsembleRetriever
29
- from langchain.chains import create_retrieval_chain, create_history_aware_retriever
30
- from langchain.chains.combine_documents import create_stuff_documents_chain
31
- from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder
32
- from langchain_core.messages import HumanMessage, AIMessage
33
- from langchain_core.documents import Document
34
- from langchain_huggingface import HuggingFaceEmbeddings
35
- logging.info("✅ All LangChain imports successful")
36
- except ImportError as e:
37
- logging.error(f"❌ Import error: {e}")
38
- sys.exit(1)
39
 
40
- # Configuration
41
  GOOGLE_API_KEY = os.getenv("GOOGLE_API_KEY")
42
  DATA_PATH = "medical_data"
43
  DB_PATH = "chroma_db"
44
- MAX_HISTORY_TURNS = 5 # Reduced for stability
45
  FORCE_REBUILD_DB = False
46
 
47
- # Setup logging
48
- logging.basicConfig(
49
- level=logging.INFO,
50
- format="%(asctime)s [%(levelname)s] %(message)s",
51
- handlers=[
52
- logging.StreamHandler(),
53
- logging.FileHandler("deepmed.log", encoding='utf-8')
54
- ]
55
- )
56
 
57
- def safe_process_excel(file_path: str, filename: str) -> list:
58
- """Safely process Excel files with error handling"""
 
 
 
59
  docs = []
60
  try:
61
- logging.info(f"Processing Excel file: {filename}")
62
-
63
  if file_path.endswith(".csv"):
64
- df = pd.read_csv(file_path, encoding='utf-8', on_bad_lines='skip')
65
  else:
66
  df = pd.read_excel(file_path)
67
-
68
- # Clean dataframe
69
- df = df.dropna(how='all')
70
- df = df.fillna("Không có thông tin")
71
-
72
  for idx, row in df.iterrows():
73
- try:
74
- content_parts = []
75
- for col in df.columns:
76
- if pd.notna(row[col]):
77
- value = str(row[col]).strip()
78
- if value and value.lower() not in ['nan', 'none', '']:
79
- content_parts.append(f"{col}: {value}")
80
-
81
- if content_parts:
82
- page_content = f"Tài liệu: {filename} (Dòng {idx+1}):\n" + "\n".join(content_parts)
83
- metadata = {
84
- "source": filename,
85
- "row": idx+1,
86
- "type": "excel",
87
- "doc_id": f"{filename}_row_{idx+1}"
88
- }
89
- docs.append(Document(page_content=page_content, metadata=metadata))
90
-
91
- except Exception as e:
92
- logging.warning(f"Error processing row {idx+1} in {filename}: {e}")
93
- continue
94
 
95
  except Exception as e:
96
- logging.error(f"Failed to process Excel {filename}: {e}")
97
 
98
  return docs
99
 
100
- def load_documents_safely() -> list:
101
- """Load documents with comprehensive error handling"""
102
- documents = []
103
-
104
- # Create data directory if not exists
105
- if not os.path.exists(DATA_PATH):
106
- os.makedirs(DATA_PATH, exist_ok=True)
107
- logging.info(f"Created data directory: {DATA_PATH}")
108
- return documents
109
-
110
- # Get all files
111
- all_files = []
112
- for root, _, files in os.walk(DATA_PATH):
113
- for file in files:
114
- if file.lower().endswith(('.pdf', '.docx', '.xlsx', '.xls', '.csv', '.txt', '.md')):
115
- all_files.append(os.path.join(root, file))
116
-
117
- if not all_files:
118
- logging.warning(f"No documents found in {DATA_PATH}")
119
- return documents
120
-
121
- logging.info(f"Found {len(all_files)} files to process")
122
-
123
- # Process each file
124
- for file_path in all_files:
125
- filename = os.path.basename(file_path)
126
- file_ext = os.path.splitext(filename)[1].lower()
127
 
128
- try:
129
- if file_ext == '.pdf':
130
- loader = PyPDFLoader(file_path)
131
- docs = loader.load()
132
- for doc in docs:
133
- doc.metadata.update({
134
- "source": filename,
135
- "file_type": "pdf"
136
- })
137
- documents.extend(docs)
138
- logging.info(f"✓ Loaded PDF: {filename} ({len(docs)} pages)")
139
 
140
- elif file_ext == '.docx':
141
- text = docx2txt.process(file_path)
142
- if text.strip():
143
- doc = Document(
144
- page_content=text,
145
- metadata={"source": filename, "file_type": "docx"}
146
- )
147
- documents.append(doc)
148
- logging.info(f"✓ Loaded DOCX: {filename}")
149
-
150
- elif file_ext in ['.xlsx', '.xls', '.csv']:
151
- excel_docs = safe_process_excel(file_path, filename)
152
- documents.extend(excel_docs)
153
- logging.info(f"✓ Loaded Excel: {filename} ({len(excel_docs)} rows)")
154
 
155
- elif file_ext in ['.txt', '.md']:
156
- with open(file_path, 'r', encoding='utf-8', errors='ignore') as f:
157
- text = f.read()
158
- if text.strip():
159
- doc = Document(
160
- page_content=text,
161
- metadata={"source": filename, "file_type": "txt"}
162
- )
163
- documents.append(doc)
164
- logging.info(f"✓ Loaded TXT: {filename}")
165
 
166
- except Exception as e:
167
- logging.error(f" Failed to load {filename}: {e}")
168
- continue
169
-
170
- logging.info(f"Total documents loaded: {len(documents)}")
 
 
 
 
171
  return documents
172
 
173
- def create_simple_retriever():
174
- """Create a simplified retriever for Hugging Face"""
175
- try:
176
- logging.info("Initializing embedding model...")
177
-
178
- # Use lightweight model
179
- embedding_model = HuggingFaceEmbeddings(
180
- model_name="sentence-transformers/all-MiniLM-L6-v2",
181
- model_kwargs={'device': 'cpu'},
182
- encode_kwargs={'normalize_embeddings': True}
183
- )
184
-
185
- # Check if DB exists
186
- if os.path.exists(DB_PATH) and not FORCE_REBUILD_DB:
187
- try:
188
- logging.info(f"Loading existing vector store from {DB_PATH}")
189
- vectorstore = Chroma(
190
- persist_directory=DB_PATH,
191
- embedding_function=embedding_model,
192
- client_settings=Settings(anonymized_telemetry=False)
193
- )
194
-
195
- # Test if vectorstore works
196
- test_results = vectorstore.similarity_search("test", k=1)
197
- logging.info(f"Vector store loaded successfully with {vectorstore._collection.count()} documents")
198
- return vectorstore.as_retriever(search_kwargs={"k": 8})
199
-
200
- except Exception as e:
201
- logging.warning(f"Failed to load existing DB: {e}, creating new...")
202
- if os.path.exists(DB_PATH):
203
- rmtree(DB_PATH, ignore_errors=True)
204
-
205
- # Create new vector store
206
- logging.info("Creating new vector store...")
207
- documents = load_documents_safely()
208
-
209
- if not documents:
210
- logging.warning("No documents to index")
 
211
  return None
 
 
 
212
 
213
- # Simple text splitting
214
- text_splitter = RecursiveCharacterTextSplitter(
215
- chunk_size=600,
216
- chunk_overlap=100,
217
- length_function=len,
218
- separators=["\n\n", "\n", " ", ""]
219
- )
220
-
221
- splits = text_splitter.split_documents(documents)
222
- logging.info(f"Split into {len(splits)} chunks")
223
-
224
- # Create vector store
225
  vectorstore = Chroma.from_documents(
226
- documents=splits,
227
- embedding=embedding_model,
228
  persist_directory=DB_PATH,
229
- client_settings=Settings(anonymized_telemetry=False)
230
  )
231
-
232
- logging.info(f"Created vector store with {len(splits)} chunks")
233
- return vectorstore.as_retriever(search_kwargs={"k": 8})
234
-
235
- except Exception as e:
236
- logging.error(f"Failed to create retriever: {e}")
237
- return None
238
 
239
- class MedicalAssistant:
240
- def __init__(self):
241
- self.llm = None
242
- self.retriever = None
243
- self.chain = None
244
- self.initialized = False
245
-
246
- def initialize(self):
247
- """Initialize the assistant"""
248
- try:
249
- # Check API key
250
- if not GOOGLE_API_KEY:
251
- logging.error("GOOGLE_API_KEY environment variable is not set")
252
- self.llm = self.create_fallback_llm()
253
- else:
254
- self.llm = ChatGoogleGenerativeAI(
255
- model="gemini-2.5-flash",
256
- temperature=0.1,
257
- google_api_key=GOOGLE_API_KEY,
258
- max_output_tokens=1000,
259
- timeout=30
260
- )
261
- logging.info("✅ Gemini LLM initialized")
262
-
263
- # Create retriever
264
- self.retriever = create_simple_retriever()
265
-
266
- # Build chain
267
- self._build_chain()
268
-
269
- self.initialized = True
270
- logging.info("✅ Medical Assistant initialized successfully")
271
- return True
272
-
273
- except Exception as e:
274
- logging.error(f"❌ Failed to initialize: {e}")
275
- self.llm = self.create_fallback_llm()
276
- self._build_chain()
277
- return False
278
 
279
- def create_fallback_llm(self):
280
- """Create a fallback LLM when Gemini fails"""
281
- from langchain.llms.fake import FakeListLLM
282
- responses = [
283
- "Xin lỗi, tôi đang gặp sự cố kết nối với hệ thống AI. Vui lòng thử lại sau.",
284
- "Hệ thống tạm thời không khả dụng. Vui lòng kiểm tra kết nối internet.",
285
- "Tôi không thể xử lý yêu cầu của bạn ngay lúc này."
286
- ]
287
- return FakeListLLM(responses=responses)
 
 
 
 
288
 
289
- def _build_chain(self):
290
- """Build the RAG chain"""
291
- try:
292
- # System prompt
293
- system_prompt = """Bạn là DeepMed AI, trợ lý y tế thông minh.
294
- Trả lời câu hỏi dựa trên thông tin được cung cấp.
295
- Nếu không có thông tin, hãy nói rõ.
296
- Luôn trả lời bằng tiếng Việt.
297
-
298
- Context: {context}
299
-
300
- Câu hỏi: {input}"""
301
-
302
- prompt = ChatPromptTemplate.from_messages([
303
- ("system", system_prompt),
304
- MessagesPlaceholder("chat_history"),
305
- ("human", "{input}"),
306
- ])
307
-
308
- if self.retriever and self.llm:
309
- # Create RAG chain
310
- question_answer_chain = create_stuff_documents_chain(self.llm, prompt)
311
- self.chain = create_retrieval_chain(self.retriever, question_answer_chain)
312
- logging.info("✅ RAG chain built with retriever")
313
- else:
314
- # Simple chain without retrieval
315
- self.chain = prompt | self.llm
316
- logging.info("✅ Simple LLM chain built")
317
-
318
- except Exception as e:
319
- logging.error(f"Failed to build chain: {e}")
320
- # Create a minimal working chain
321
- self.chain = lambda x: {"answer": "Xin lỗi, hệ thống đang bảo trì."}
322
 
323
- def chat(self, message: str, history: list):
324
- """Process chat message"""
325
- if not self.initialized:
326
- if not self.initialize():
327
- yield "❌ Hệ thống chưa thể khởi động. Vui lòng thử lại sau."
328
- return
329
 
 
 
 
 
330
  try:
331
- # Prepare chat history
332
- chat_history = []
333
- for user_msg, bot_msg in history[-MAX_HISTORY_TURNS:]:
334
- chat_history.append(HumanMessage(content=str(user_msg)))
335
- chat_history.append(AIMessage(content=str(bot_msg)))
336
 
337
- # Create input
338
- inputs = {
339
- "input": message,
340
- "chat_history": chat_history
341
- }
342
-
343
- # Get response
344
- if hasattr(self.chain, 'invoke'):
345
- response = self.chain.invoke(inputs)
346
-
347
- if isinstance(response, dict) and "answer" in response:
348
- answer = response["answer"]
349
- elif hasattr(response, 'content'):
350
- answer = response.content
351
- else:
352
- answer = str(response)
353
-
354
- yield answer
355
- else:
356
- yield "Xin chào! Tôi là DeepMed AI. Tôi có thể giúp gì cho bạn về y tế?"
357
-
358
  except Exception as e:
359
- logging.error(f"Chat error: {e}")
360
- yield f"⚠️ Đã xảy ra lỗi: {str(e)[:100]}"
361
 
362
- # Create assistant instance
363
- assistant = MedicalAssistant()
364
-
365
- # Initialize on startup (but don't block)
366
- import threading
367
- def init_in_background():
368
- assistant.initialize()
369
-
370
- threading.Thread(target=init_in_background, daemon=True).start()
371
-
372
- # Gradio Interface
373
- def gradio_chat(message, history):
374
- """Wrapper for Gradio chat"""
375
- for response in assistant.chat(message, history):
376
- yield response
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
377
 
378
- # Custom CSS
379
- css = """
380
- .gradio-container {
381
- font-family: 'Arial', sans-serif;
382
- max-width: 800px;
383
- margin: 0 auto;
384
- }
385
 
386
- #chatbot {
387
- border: 1px solid #e0e0e0;
388
- border-radius: 10px;
389
- padding: 15px;
390
- min-height: 400px;
391
- max-height: 500px;
392
- overflow-y: auto;
393
- background: #f9f9f9;
394
- }
395
 
396
- .user, .assistant {
397
- padding: 10px 15px;
398
- margin: 8px 0;
399
- border-radius: 15px;
400
- max-width: 80%;
401
- }
 
 
 
 
 
 
 
 
 
 
402
 
403
- .user {
404
- background: #e3f2fd;
405
- margin-left: auto;
406
- border-bottom-right-radius: 5px;
407
- }
 
 
408
 
409
- .assistant {
410
- background: #f5f5f5;
411
- margin-right: auto;
412
- border-bottom-left-radius: 5px;
413
- }
 
 
 
 
 
414
 
415
- input {
416
- border-radius: 20px;
417
- padding: 12px 20px;
418
- border: 2px solid #4a90e2;
419
- }
 
 
 
 
 
 
 
 
 
 
 
420
 
421
- button {
422
- border-radius: 20px;
423
- padding: 10px 20px;
424
- background: #4a90e2;
425
- color: white;
426
- border: none;
427
- }
428
 
429
- button:hover {
430
- background: #357abd;
431
- }
432
 
433
- @media (max-width: 768px) {
434
- .gradio-container {
435
- padding: 10px;
436
- }
437
-
438
- #chatbot {
439
- min-height: 300px;
440
- max-height: 400px;
441
- }
442
-
443
- .user, .assistant {
444
- max-width: 90%;
445
- }
446
- }
447
  """
448
 
449
- # Create Gradio interface
450
- with gr.Blocks(css=css, title="DeepMed AI - Trợ lý Y tế") as demo:
451
- gr.Markdown("# 🏥 DeepMed AI - Trợ Y tế Thông minh")
452
- gr.Markdown("Hỏi đáp về thuốc, bệnh lý và hướng dẫn y tế")
453
-
454
- chatbot = gr.Chatbot(
455
- height=400,
456
- label="Hội thoại",
457
- placeholder="Xin chào! Tôi có thể giúp gì cho bạn?"
458
- )
459
-
460
- with gr.Row():
461
- msg = gr.Textbox(
462
- label="Câu hỏi của bạn",
463
- placeholder="Nhập câu hỏi về y tế...",
464
- scale=4
465
- )
466
- submit_btn = gr.Button("Gửi", variant="primary", scale=1)
467
- clear_btn = gr.Button("Xóa", variant="secondary", scale=1)
468
-
469
- # Footer
470
- gr.Markdown("---")
471
- gr.Markdown("⚠️ **Lưu ý:** Thông tin chỉ mang tính tham khảo. Vui lòng tham khảo ý kiến bác sĩ trước khi áp dụng.")
472
-
473
- # Event handlers
474
- def clear_chat():
475
- return None
476
 
477
- # Submit function
478
- def respond(message, chat_history):
479
- chat_history.append((message, ""))
480
- yield chat_history
481
-
482
- response = ""
483
- for chunk in gradio_chat(message, chat_history[:-1]):
484
- response = chunk
485
- chat_history[-1] = (message, response)
486
- yield chat_history
487
-
488
- # Connect events
489
- msg.submit(
490
- respond,
491
- [msg, chatbot],
492
- [chatbot]
493
- ).then(lambda: "", outputs=[msg])
494
-
495
- submit_btn.click(
496
- respond,
497
- [msg, chatbot],
498
- [chatbot]
499
- ).then(lambda: "", outputs=[msg])
500
-
501
- clear_btn.click(
502
- clear_chat,
503
- outputs=[chatbot]
504
  )
505
 
506
- # Launch with error handling
507
  if __name__ == "__main__":
508
- try:
509
- logging.info("🚀 Starting DeepMed AI...")
510
- demo.queue(max_size=10)
511
- demo.launch(
512
- server_name="0.0.0.0",
513
- server_port=7860,
514
- show_error=True,
515
- debug=False,
516
- share=False
517
- )
518
- except Exception as e:
519
- logging.error(f"Failed to launch app: {e}")
520
- print(f"Error: {e}")
521
- sys.exit(1)
 
1
+ __import__("pysqlite3")
 
2
  import sys
3
+ sys.modules["sqlite3"] = sys.modules.pop("pysqlite3")
4
+
5
+ import os
6
  import logging
7
  import traceback
8
  import gradio as gr
 
11
  import chromadb
12
  from chromadb.config import Settings
13
  from shutil import rmtree
 
 
 
 
 
 
 
 
14
 
15
+ from langchain_google_genai import ChatGoogleGenerativeAI
16
+ from langchain_chroma import Chroma
17
+ from langchain_community.document_loaders import PyPDFLoader
18
+ from langchain_text_splitters import RecursiveCharacterTextSplitter
19
+ from langchain_community.retrievers import BM25Retriever
20
+ from langchain.retrievers.ensemble import EnsembleRetriever
21
+ from langchain.chains import create_retrieval_chain, create_history_aware_retriever
22
+ from langchain.chains.combine_documents import create_stuff_documents_chain
23
+ from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder
24
+ from langchain_core.messages import HumanMessage, AIMessage
25
+ from langchain_core.documents import Document
26
+ from langchain_huggingface import HuggingFaceEmbeddings
27
+ from langchain.retrievers import ContextualCompressionRetriever
28
+ from langchain.retrievers.document_compressors import CrossEncoderReranker
29
+ from langchain_community.cross_encoders import HuggingFaceCrossEncoder
 
 
 
30
 
 
31
  GOOGLE_API_KEY = os.getenv("GOOGLE_API_KEY")
32
  DATA_PATH = "medical_data"
33
  DB_PATH = "chroma_db"
34
+ MAX_HISTORY_TURNS = 6
35
  FORCE_REBUILD_DB = False
36
 
37
+ logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
 
 
 
 
 
 
 
 
38
 
39
+ def process_excel_file(file_path: str, filename: str) -> list[Document]:
40
+ """
41
+ Xử lý Excel thông minh: Biến mỗi dòng thành một Document riêng biệt
42
+ giúp tìm kiếm chính xác từng bản ghi thuốc/bệnh nhân.
43
+ """
44
  docs = []
45
  try:
 
 
46
  if file_path.endswith(".csv"):
47
+ df = pd.read_csv(file_path)
48
  else:
49
  df = pd.read_excel(file_path)
50
+
51
+ df.dropna(how='all', inplace=True)
52
+ df.fillna("Không có thông tin", inplace=True)
53
+
 
54
  for idx, row in df.iterrows():
55
+ content_parts = []
56
+ for col_name, val in row.items():
57
+ clean_val = str(val).strip()
58
+ if clean_val and clean_val.lower() != "nan":
59
+ content_parts.append(f"{col_name}: {clean_val}")
60
+
61
+ if content_parts:
62
+ page_content = f"Dữ liệu từ file {filename} (Dòng {idx+1}):\n" + "\n".join(content_parts)
63
+ metadata = {"source": filename, "row": idx+1, "type": "excel_record"}
64
+ docs.append(Document(page_content=page_content, metadata=metadata))
 
 
 
 
 
 
 
 
 
 
 
65
 
66
  except Exception as e:
67
+ logging.error(f"Lỗi xử Excel {filename}: {e}")
68
 
69
  return docs
70
 
71
+ def load_documents_from_folder(folder_path: str) -> list[Document]:
72
+ logging.info(f"--- Bắt đầu quét thư mục: {folder_path} ---")
73
+ documents: list[Document] = []
74
+ if not os.path.exists(folder_path):
75
+ os.makedirs(folder_path, exist_ok=True)
76
+ return []
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
77
 
78
+ for root, _, files in os.walk(folder_path):
79
+ for filename in files:
80
+ file_path = os.path.join(root, filename)
81
+ filename_lower = filename.lower()
82
+ try:
83
+ if filename_lower.endswith(".pdf"):
84
+ loader = PyPDFLoader(file_path)
85
+ docs = loader.load()
86
+ for d in docs: d.metadata["source"] = filename
87
+ documents.extend(docs)
 
88
 
89
+ elif filename_lower.endswith(".docx"):
90
+ text = docx2txt.process(file_path)
91
+ if text.strip():
92
+ documents.append(Document(page_content=text, metadata={"source": filename}))
 
 
 
 
 
 
 
 
 
 
93
 
94
+ elif filename_lower.endswith((".xlsx", ".xls", ".csv")):
95
+ excel_docs = process_excel_file(file_path, filename)
96
+ documents.extend(excel_docs)
 
 
 
 
 
 
 
97
 
98
+ elif filename_lower.endswith((".txt", ".md")):
99
+ with open(file_path, "r", encoding="utf-8") as f: text = f.read()
100
+ if text.strip():
101
+ documents.append(Document(page_content=text, metadata={"source": filename}))
102
+
103
+ except Exception as e:
104
+ logging.error(f"Lỗi đọc file {filename}: {e}")
105
+
106
+ logging.info(f"Tổng cộng đã load: {len(documents)} tài liệu gốc.")
107
  return documents
108
 
109
+ def get_retriever_chain():
110
+ logging.info("--- Tải Embedding Model ---")
111
+ embedding_model = HuggingFaceEmbeddings(model_name="sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2")
112
+
113
+ vectorstore = None
114
+ splits = []
115
+
116
+ chroma_settings = Settings(anonymized_telemetry=False)
117
+
118
+ if FORCE_REBUILD_DB and os.path.exists(DB_PATH):
119
+ logging.warning("Đang xóa DB cũ theo yêu cầu FORCE_REBUILD...")
120
+ rmtree(DB_PATH, ignore_errors=True)
121
+
122
+ if os.path.exists(DB_PATH) and os.listdir(DB_PATH):
123
+ try:
124
+ vectorstore = Chroma(
125
+ persist_directory=DB_PATH,
126
+ embedding_function=embedding_model,
127
+ client_settings=chroma_settings
128
+ )
129
+
130
+ existing_data = vectorstore.get()
131
+ if existing_data['documents']:
132
+ for text, meta in zip(existing_data['documents'], existing_data['metadatas']):
133
+ splits.append(Document(page_content=text, metadata=meta))
134
+ logging.info(f"Đã khôi phục {len(splits)} chunks từ DB.")
135
+ else:
136
+ logging.warning("DB rỗng, sẽ tạo mới.")
137
+ vectorstore = None
138
+ except Exception as e:
139
+ logging.error(f"DB lỗi: {e}. Đang reset...")
140
+ rmtree(DB_PATH, ignore_errors=True)
141
+ vectorstore = None
142
+
143
+ if not vectorstore:
144
+ logging.info("--- Tạo Index dữ liệu mới ---")
145
+ raw_docs = load_documents_from_folder(DATA_PATH)
146
+ if not raw_docs:
147
+ logging.warning("Không có dữ liệu trong thư mục medical_data.")
148
  return None
149
+
150
+ text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
151
+ splits = text_splitter.split_documents(raw_docs)
152
 
 
 
 
 
 
 
 
 
 
 
 
 
153
  vectorstore = Chroma.from_documents(
154
+ documents=splits,
155
+ embedding=embedding_model,
156
  persist_directory=DB_PATH,
157
+ client_settings=chroma_settings
158
  )
159
+ logging.info("Đã lưu VectorStore thành công.")
 
 
 
 
 
 
160
 
161
+ vector_retriever = vectorstore.as_retriever(search_kwargs={"k": 10})
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
162
 
163
+ if splits:
164
+ bm25_retriever = BM25Retriever.from_documents(splits)
165
+ bm25_retriever.k = 10
166
+ ensemble_retriever = EnsembleRetriever(
167
+ retrievers=[bm25_retriever, vector_retriever],
168
+ weights=[0.4, 0.6]
169
+ )
170
+ else:
171
+ ensemble_retriever = vector_retriever
172
+
173
+ logging.info("--- Tải Reranker Model (BGE-M3) ---")
174
+ reranker_model = HuggingFaceCrossEncoder(model_name="BAAI/bge-reranker-v2-m3")
175
+ compressor = CrossEncoderReranker(model=reranker_model, top_n=5)
176
 
177
+ final_retriever = ContextualCompressionRetriever(
178
+ base_compressor=compressor,
179
+ base_retriever=ensemble_retriever
180
+ )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
181
 
182
+ return final_retriever
183
+
184
+ class DeepMedBot:
185
+ def __init__(self):
186
+ self.rag_chain = None
187
+ self.ready = False
188
 
189
+ if not GOOGLE_API_KEY:
190
+ logging.error("⚠️ Thiếu GOOGLE_API_KEY! Vui lòng thiết lập biến môi trường.")
191
+ return
192
+
193
  try:
194
+ self.retriever = get_retriever_chain()
195
+ if not self.retriever:
196
+ logging.warning("⚠️ Chưa dữ liệu để Retreive. Bot sẽ chỉ trả lời bằng kiến thức nền.")
 
 
197
 
198
+ self.llm = ChatGoogleGenerativeAI(
199
+ model="gemini-2.5-flash",
200
+ temperature=0.11,
201
+ google_api_key=GOOGLE_API_KEY
202
+ )
203
+ self._build_chains()
204
+ self.ready = True
205
+ logging.info("✅ Bot DeepMed đã sẵn sàng phục vụ!")
 
 
 
 
 
 
 
 
 
 
 
 
 
206
  except Exception as e:
207
+ logging.error(f"🔥 Lỗi khởi tạo bot: {e}")
208
+ logging.debug(traceback.format_exc())
209
 
210
+ def _build_chains(self):
211
+ context_system_prompt = (
212
+ "Dựa trên lịch sử chat và câu hỏi mới nhất của người dùng, "
213
+ "hãy viết lại câu hỏi đó thành một câu đầy đủ ngữ cảnh để hệ thống có thể hiểu được. "
214
+ "KHÔNG trả lời câu hỏi, chỉ viết lại nó."
215
+ )
216
+ context_prompt = ChatPromptTemplate.from_messages([
217
+ ("system", context_system_prompt),
218
+ MessagesPlaceholder("chat_history"),
219
+ ("human", "{input}"),
220
+ ])
221
+
222
+ if self.retriever:
223
+ history_aware_retriever = create_history_aware_retriever(
224
+ self.llm, self.retriever, context_prompt
225
+ )
226
+
227
+ qa_system_prompt = (
228
+ "Bạn là 'DeepMed-AI' - Trợ lý Dược lâm sàng tại Trung Tâm Y Tế. "
229
+ "Sử dụng các thông tin được cung cấp trong phần Context dưới đây để trả lời câu hỏi về thuốc, bệnh học và y lệnh.\n"
230
+ "Nếu Context có dữ liệu từ Excel, hãy trình bày dạng bảng hoặc gạch đầu dòng rõ ràng.\n"
231
+ "Nếu không tìm thấy thông tin trong Context, hãy nói 'Tôi không tìm thấy thông tin trong dữ liệu nội bộ' và gợi ý dựa trên kiến thức y khoa chung của bạn.\n\n"
232
+ "Context:\n{context}"
233
+ )
234
+
235
+ qa_prompt = ChatPromptTemplate.from_messages([
236
+ ("system", qa_system_prompt),
237
+ MessagesPlaceholder("chat_history"),
238
+ ("human", "{input}"),
239
+ ])
240
+
241
+ question_answer_chain = create_stuff_documents_chain(self.llm, qa_prompt)
242
+
243
+ if self.retriever:
244
+ self.rag_chain = create_retrieval_chain(history_aware_retriever, question_answer_chain)
245
+ else:
246
+ self.rag_chain = qa_prompt | self.llm
247
 
248
+ def chat_stream(self, message: str, history: list):
249
+ if not self.ready:
250
+ yield "Hệ thống đang khởi động hoặc gặp lỗi cấu hình."
251
+ return
 
 
 
252
 
253
+ chat_history = []
254
+ for u, b in history[-MAX_HISTORY_TURNS:]:
255
+ chat_history.append(HumanMessage(content=str(u)))
256
+ chat_history.append(AIMessage(content=str(b)))
 
 
 
 
 
257
 
258
+ full_response = ""
259
+ retrieved_docs = []
260
+
261
+ try:
262
+ stream_input = {"input": message, "chat_history": chat_history} if self.retriever else {"input": message, "chat_history": chat_history}
263
+
264
+ if self.rag_chain:
265
+ for chunk in self.rag_chain.stream(stream_input):
266
+
267
+ if isinstance(chunk, dict):
268
+ if "answer" in chunk:
269
+ full_response += chunk["answer"]
270
+ yield full_response
271
+
272
+ if "context" in chunk:
273
+ retrieved_docs = chunk["context"]
274
 
275
+ elif hasattr(chunk, 'content'):
276
+ full_response += chunk.content
277
+ yield full_response
278
+
279
+ elif isinstance(chunk, str):
280
+ full_response += chunk
281
+ yield full_response
282
 
283
+ if retrieved_docs:
284
+ refs = self._build_references_text(retrieved_docs)
285
+ if refs:
286
+ full_response += f"\n\n---\n📚 **Nguồn tham khảo:**\n{refs}"
287
+ yield full_response
288
+
289
+ except Exception as e:
290
+ logging.error(f"Lỗi khi chat: {e}")
291
+ logging.debug(traceback.format_exc())
292
+ yield f"Đã xảy ra lỗi: {str(e)}"
293
 
294
+ @staticmethod
295
+ def _build_references_text(docs) -> str:
296
+ lines = []
297
+ seen = set()
298
+ for doc in docs:
299
+ src = doc.metadata.get("source", "Tài liệu")
300
+ row_info = ""
301
+ if "row" in doc.metadata:
302
+ row_info = f"(Dòng {doc.metadata['row']})"
303
+
304
+ ref_str = f"- {src} {row_info}"
305
+
306
+ if ref_str not in seen:
307
+ lines.append(ref_str)
308
+ seen.add(ref_str)
309
+ return "\n".join(lines)
310
 
311
+ bot = DeepMedBot()
 
 
 
 
 
 
312
 
313
+ def gradio_chat_stream(message, history):
314
+ yield from bot.chat_stream(message, history)
 
315
 
316
+ css = """
317
+ .gradio-container {min_height: 600px !important;}
318
+ h1 {text-align: center; color: #2E86C1;}
 
 
 
 
 
 
 
 
 
 
 
319
  """
320
 
321
+ with gr.Blocks(css=css, title="DeepMed AI") as demo:
322
+ gr.Markdown("# 🏥 DeepMed AI - Trợ lý Lâm Sàng")
323
+ gr.Markdown("Hệ thống hỗ trợ lâm sàng tại Trung Tâm Y Tế Khu Vực Thanh Ba.")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
324
 
325
+ chat_interface = gr.ChatInterface(
326
+ fn=gradio_chat_stream,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
327
  )
328
 
 
329
  if __name__ == "__main__":
330
+ demo.launch()