Upload 6 files

RAG_Learning_Assistant_with_Streaming.py

@@ -0,0 +1,518 @@
+import os
+import sys
+from typing import List, Dict, Tuple, Generator, Set
+from openai import OpenAI
+from vectorize_knowledge_base import KnowledgeBaseVectorizer
+import json
+from datetime import datetime
+import re
+
+
+class RAGLearningAssistant:
+    def __init__(self, api_key: str, model: str = "gpt-4.1-nano-2025-04-14", vector_db_path: str = ""):
+        """
+        初始化RAG学习助手（适配学生Space）
+        
+        Args:
+            api_key: OpenAI API密钥（必需）
+            model: 使用的模型名称
+            vector_db_path: 向量数据库所在目录路径（数据存储仓库的本地目录）
+        """
+        self.client = OpenAI(api_key=api_key)
+        
+        # 使用修改后的KnowledgeBaseVectorizer，指定vector_db_dir
+        self.vectorizer = KnowledgeBaseVectorizer(
+            api_key=api_key,
+            vector_db_dir=vector_db_path  # 传递数据存储仓库的本地目录
+        )
+        
+        # 预加载向量数据库到缓存
+        print("[RAGLearningAssistant] Preloading vector database...")
+        load_result = self.vectorizer.load_vector_database()
+        if load_result[0] is not None:
+            print(f"[RAGLearningAssistant] Vector database loaded successfully")
+        else:
+            print(f"[RAGLearningAssistant] Warning: Failed to load vector database")
+        
+        # 模型配置
+        self.model = model
+        self.temperature = 0.1
+        self.max_tokens = 2000
+        
+        # 系统提示词
+        self.system_prompt = """You are a helpful learning assistant specializing in road engineering.
+Students can ask you questions with the following intents:
+1. Clarification: Requests to confirm understanding of a concept, parameter, or calculation.
+2. Instruction: Seeking step-by-step guidance for tasks or calculations.
+3. Explanatory: Asking for the reasoning behind a method, parameter choice, or principle.
+4. Information-seeking: Asking for where to find specific information in course materials.
+
+You have access to a knowledge base of course materials. When answering questions:
+1. Stick to the provided context from the knowledge base.
+2. At the end of your response, provide students the 'title' & 'from' fields of the chunks that were used to answer the question. So that they can refer to the original source.
+3. If the knowledge base doesn't contain relevant information, say so. Students can go to the teaching team for further assistance.
+"""
+        
+        # 查询重写的系统提示词 - 改进版本
+        self.rewrite_prompt = """You are a query rewriting assistant. Your task is to provide a summary of the conversation history and then rewrite user queries based on conversation history to make them more clear and complete.
+
+Please format your response as follows:
+SUMMARY: [Brief summary of the conversation context. Include key points, user intent, and any relevant details]
+REWRITTEN_QUERY: [The rewritten query that incorporates context]
+
+Rules:
+1. If there's relevant context from previous messages, incorporate it into the rewritten query
+2. Make implicit references explicit
+3. Maintain the original intent while adding clarity
+4. If the query is already clear and complete, keep it as is
+5. Always provide both SUMMARY and REWRITTEN_QUERY sections"""
+        
+        # 实体提取的系统提示词
+        self.entity_extraction_prompt = """You are an expert in road engineering. Extract key entities from the given query.
+Focus on:
+1. Technical terms and jargon specific to road engineering
+2. Formulas, equations, or mathematical concepts
+3. Parameters, specifications, or measurements
+4. Standards, methods, or procedures
+5. Materials, equipment, or structures
+
+Return the entities as a JSON array of strings. Only include the most important and specific entities."""
+        
+        # 对话历史
+        self.conversation_history = []
+        
+
+    def rewrite_query(self, query: str) -> Tuple[str, str]:
+        """
+        基于对话历史重写查询，并返回历史总结
+        
+        Args:
+            query: 原始查询
+            
+        Returns:
+            (历史总结, 重写后的查询)
+        """
+        # 构建消息
+        messages = [
+            {"role": "system", "content": self.rewrite_prompt}
+        ]
+        
+        # 添加对话历史上下文
+        if self.conversation_history:
+            context = "Previous conversation:\n"
+            for msg in self.conversation_history[-6:]:  # 最近3轮对话
+                role = "User" if msg["role"] == "user" else "Assistant"
+                # 截取前200个字符避免过长
+                content = msg["content"][:200] + "..." if len(msg["content"]) > 200 else msg["content"]
+                context += f"{role}: {content}\n"
+            
+            messages.append({
+                "role": "user", 
+                "content": f"{context}\n\nCurrent query: {query}\n\nPlease provide summary and rewritten query following the specified format:"
+            })
+        else:
+            # 没有历史时也要按格式返回
+            messages.append({
+                "role": "user",
+                "content": f"Current query: {query}\n\nPlease provide summary and rewritten query following the specified format:"
+            })
+        
+        try:
+            response = self.client.chat.completions.create(
+                model=self.model,
+                messages=messages,
+                temperature=0.1,  # 低温度确保一致性
+                max_tokens=2000
+            )
+            
+            content = response.choices[0].message.content.strip()
+            
+            # 改进的解析逻辑
+            summary = ""
+            rewritten = query  # 默认值
+            
+            # 使用正则表达式提取SUMMARY和REWRITTEN_QUERY
+            summary_match = re.search(r'SUMMARY:\s*(.*?)(?=REWRITTEN_QUERY:|$)', content, re.DOTALL | re.IGNORECASE)
+            rewritten_match = re.search(r'REWRITTEN_QUERY:\s*(.*?)$', content, re.DOTALL | re.IGNORECASE)
+
+            
+            if summary_match:
+                summary = summary_match.group(1).strip()
+            
+            if rewritten_match:
+                rewritten = rewritten_match.group(1).strip()
+            
+            # 备用解析方法 - 如果正则表达式失败
+            if not summary and not rewritten_match:
+                lines = content.split('\n')
+                current_section = None
+                summary_lines = []
+                rewritten_lines = []
+                
+                for line in lines:
+                    line = line.strip()
+                    if line.upper().startswith("SUMMARY"):
+                        current_section = "summary"
+                        # 提取SUMMARY:后面的内容
+                        summary_part = line[line.upper().find("SUMMARY"):].replace("SUMMARY:", "").strip()
+                        if summary_part:
+                            summary_lines.append(summary_part)
+                    elif line.upper().startswith("REWRITTEN_QUERY") or line.upper().startswith("REWRITTEN QUERY"):
+                        current_section = "rewritten"
+                        # 提取REWRITTEN_QUERY:后面的内容
+                        rewritten_part = re.sub(r'^REWRITTEN[_\s]*QUERY[:\s]*', '', line, flags=re.IGNORECASE).strip()
+                        if rewritten_part:
+                            rewritten_lines.append(rewritten_part)
+                    elif current_section == "summary" and line:
+                        summary_lines.append(line)
+                    elif current_section == "rewritten" and line:
+                        rewritten_lines.append(line)
+                
+                if summary_lines:
+                    summary = " ".join(summary_lines)
+                if rewritten_lines:
+                    rewritten = " ".join(rewritten_lines)
+            
+            # 如果仍然没有获得有效结果，使用更简单的方法
+            if not summary and self.conversation_history:
+                summary = "Continue previous discussion"
+            
+            if not rewritten or rewritten == query:
+                rewritten = query
+            
+            print(f"[rewrite_query] Raw query: {query}")
+            print(f"[rewrite_query] Chat history summary: {summary}")
+            print(f"[rewrite_query] Rewritten query: {rewritten}")
+            return summary, rewritten
+            
+        except Exception as e:
+            print(f"[rewrite_query] Query rewriting failed: {e}")
+            # 生成简单的历史总结作为备用
+            simple_summary = ""
+            if self.conversation_history:
+                simple_summary = "Based on previous conversation content"
+            return simple_summary, query  # 失败时返回简单总结和原始查询
+    
+    def extract_entities(self, original_query: str, summary: str, rewritten_query: str) -> List[str]:
+        """
+        从原始查询、历史总结和重写查询中提取关键实体（专业术语、公式、参数等）
+        
+        Args:
+            original_query: 原始用户查询
+            summary: 历史总结
+            rewritten_query: 重写后的查询文本
+            
+        Returns:
+            提取的实体列表
+        """
+        # 合并所有文本作为实体提取的输入
+        text_parts = []
+        
+        # 添加原始查询
+        if original_query:
+            text_parts.append(f"Original query: {original_query}")
+        
+        # 添加历史总结
+        if summary:
+            text_parts.append(f"Context summary: {summary}")
+        
+        # 添加重写查询
+        if rewritten_query and rewritten_query != original_query:
+            text_parts.append(f"Rewritten query: {rewritten_query}")
+        
+        combined_text = " | ".join(text_parts)
+        
+        messages = [
+            {"role": "system", "content": self.entity_extraction_prompt},
+            {"role": "user", "content": f"Text to extract entities from: {combined_text}\n\nExtract entities as JSON array:"}
+        ]
+        
+        try:
+            response = self.client.chat.completions.create(
+                model=self.model,
+                messages=messages,
+                temperature=self.temperature,
+                max_tokens=self.max_tokens
+            )
+            
+            content = response.choices[0].message.content.strip()
+            
+            # 尝试解析JSON
+            try:
+                # 提取JSON数组（处理可能的markdown格式）
+                json_match = re.search(r'\[.*?\]', content, re.DOTALL)
+                if json_match:
+                    entities = json.loads(json_match.group())
+                else:
+                    entities = json.loads(content)
+                
+                print(f"[extract_entities] Extracted entities: {entities}")
+                return entities
+                
+            except json.JSONDecodeError:
+                # 如果JSON解析失败，尝试简单的文本处理
+                print(f"[extract_entities] JSON parsing failed, using backup method")
+                # 查找引号中的内容
+                entities = re.findall(r'"([^"]+)"', content)
+                return entities if entities else self.simple_entity_extraction(combined_text)
+                
+        except Exception as e:
+            print(f"[extract_entities] Entity extraction failed: {e}")
+            # 失败时使用简单的关键词提取
+            return self.simple_entity_extraction(combined_text)
+    
+    def simple_entity_extraction(self, query: str) -> List[str]:
+        """
+        简单的实体提取备用方法
+        
+        Args:
+            query: 查询文本
+            
+        Returns:
+            提取的关键词列表
+        """
+        # 移除常见停用词
+        stop_words = {'the', 'a', 'an', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for', 
+                     'of', 'with', 'by', 'from', 'what', 'how', 'when', 'where', 'why',
+                     'is', 'are', 'was', 'were', 'been', 'be', 'have', 'has', 'had',
+                     'original', 'query', 'context', 'summary', 'rewritten'}  # 添加新的停用词
+        
+        # 分词并过滤
+        words = query.lower().split()
+        entities = [w for w in words if w not in stop_words and len(w) > 2]
+        
+        # 查找可能的专业术语（包含大写字母或数字）
+        special_terms = re.findall(r'\b[A-Z][a-zA-Z]*\b|\b\w*\d+\w*\b', query)
+        entities.extend(special_terms)
+        
+        # 去重并返回
+        return list(set(entities))[:10]  # 最多返回5个实体
+    
+    def enhanced_search(self, query: str, top_k: int = 5) -> Tuple[str, str, List[str], List[Tuple[Dict, float, Dict]]]:
+        """
+        增强搜索：重写查询 -> 提取实体 -> 基于实体搜索（优化版本）
+        
+        Args:
+            query: 原始查询
+            top_k: 返回的结果数
+            
+        Returns:
+            (历史总结, 重写后的查询, 提取的实体, 搜索结果)
+        """
+        # 1. 重写查询并获取历史总结
+        summary, rewritten_query = self.rewrite_query(query)
+        
+        # 2. 基于原始查询、总结和重写查询提取实体
+        entities = self.extract_entities(query, summary, rewritten_query)
+        
+        # 3. 基于实体搜索（使用优化的批量搜索）
+        if entities:
+            # 使用优化的批量搜索方法
+            search_results = self.vectorizer.search_with_entities_optimized(entities, top_k)
+        else:
+            # 如果没有提取到实体，使用重写后的查询进行搜索
+            print("[enhanced_search] No entities extracted, using full query search")
+            search_results = self.vectorizer.search_similar(
+                rewritten_query, 
+                top_k=top_k,
+                title_weight=0.2,
+                content_weight=0.5,
+                full_weight=0.3
+            )
+        
+        return summary, rewritten_query, entities, search_results         
+
+    def format_context(self, search_results: List[Tuple[Dict, float, Dict]]) -> str:
+        """
+        格式化搜索结果作为上下文
+        
+        Args:
+            search_results: 搜索结果列表
+            
+        Returns:
+            格式化的上下文字符串
+        """
+        if not search_results:
+            return ""
+        
+        context_parts = []
+        for i, result in enumerate(search_results, 1):
+            entry, combined_score, details = result
+            # 只显示 title, source, content，不显示 id
+            context_parts.append(
+                f"Title: {entry['title']}\n"
+                f"From: {entry['source']}\n"
+                f"Content: {entry['content']}\n"
+            )
+        
+        return "RELEVANT KNOWLEDGE BASE CONTENT:\n" + "\n---\n".join(context_parts)
+    
+    def build_messages(self, query: str, context: str) -> List[Dict[str, str]]:
+        """
+        构建消息列表，包含系统提示、上下文和用户查询
+        
+        Args:
+            query: 用户查询
+            context: 知识库上下文
+            
+        Returns:
+            消息列表
+        """
+        messages = [
+            {"role": "system", "content": self.system_prompt}
+        ]
+        
+        # 添加对话历史（保留最近5轮对话）
+        for msg in self.conversation_history[-10:]:  # 最多保留5轮对话（10条消息）
+            messages.append(msg)
+        
+        # 构建用户消息，包含上下文
+        user_message = query
+        if context:
+            user_message = f"{context}\n\nUSER QUESTION: {query}"
+        
+        messages.append({"role": "user", "content": user_message})
+        
+        return messages
+    
+    def generate_response_stream(self, query: str) -> Generator[str, None, None]:
+        """
+        生成流式响应
+        
+        Args:
+            query: 用户查询
+            
+        Yields:
+            响应文本片段
+        """
+        # 1. 增强搜索（现在使用优化版本）
+        print("[generate_response_stream] Processing query...")
+        summary, rewritten_query, entities, search_results = self.enhanced_search(query)
+        
+        # 2. 格式化上下文
+        context = self.format_context(search_results)
+        
+        # 3. 构建消息（使用原始查询，但包含基于实体搜索的上下文）
+        messages = self.build_messages(query, context)
+        
+        # 4. 调用OpenAI API进行流式生成
+        try:
+            stream = self.client.chat.completions.create(
+                model=self.model,
+                messages=messages,
+                temperature=self.temperature,
+                max_tokens=self.max_tokens,
+                stream=True
+            )
+            
+            # 收集完整响应用于保存到历史
+            full_response = ""
+            
+            # 首先返回搜索信息
+            search_info = f"\n**Query Analysis:**\n"
+            search_info += f"- Query: {query}\n"
+            if summary:
+                search_info += f"- Summary of history: {summary}\n"
+            if rewritten_query != query:
+                search_info += f"- Rewritten query: {rewritten_query}\n"
+            search_info += f"- Key entities: {', '.join(entities) if entities else 'No specific entities extracted'}\n"
+            
+            if search_results:
+                search_info += f"\n**Relevant Sources:**\n"
+                for result in search_results:
+                    entry, combined_score, details = result
+                    # 给用户显示时包含 ID 和相关度分数
+                    search_info += f"- [{entry['id']}] {entry['title']} (Relevance: {combined_score:.3f})\n"
+                search_info += "\n**Response:**\n"
+            else:
+                search_info += "\n**Response:** (No relevant knowledge base content found, answering based on general knowledge)\n"
+            
+            # 添加缓存信息（调试用）
+            cache_info = self.vectorizer.get_cache_info()
+            if cache_info['is_cached']:
+                search_info += f"💡 Vector database cached with {cache_info['cache_size']} entries\n\n"
+
+            yield search_info
+            
+            # 流式返回生成的内容
+            for chunk in stream:
+                if chunk.choices[0].delta.content is not None:
+                    content = chunk.choices[0].delta.content
+                    full_response += content
+                    yield content
+            
+            # 保存到对话历史
+            self.conversation_history.append({"role": "user", "content": query})
+            self.conversation_history.append({"role": "assistant", "content": full_response})
+            
+        except Exception as e:
+            yield f"\n\nError: Problem occurred while generating response - {str(e)}"
+    
+    def generate_response(self, query: str) -> str:
+        """
+        生成完整响应（非流式）
+        
+        Args:
+            query: 用户查询
+            
+        Returns:
+            完整的响应文本
+        """
+        response_parts = []
+        for part in self.generate_response_stream(query):
+            response_parts.append(part)
+        return "".join(response_parts)
+    
+    def clear_history(self):
+        """清除对话历史"""
+        self.conversation_history = []
+        print("[clear_history] Conversation history cleared")
+    
+    def clear_vector_cache(self):
+        """清除向量数据库缓存"""
+        self.vectorizer.clear_cache()
+        print("[clear_vector_cache] Vector database cache cleared")
+    
+    def reload_vector_database(self):
+        """重新加载向量数据库"""
+        print("[reload_vector_database] Reloading vector database...")
+        self.vectorizer.load_vector_database(force_reload=True)
+        print("[reload_vector_database] Vector database reload completed")
+    
+    def get_system_status(self) -> Dict:
+        """
+        获取系统状态信息
+        
+        Returns:
+            系统状态字典
+        """
+        cache_info = self.vectorizer.get_cache_info()
+        return {
+            'model': self.model,
+            'conversation_turns': len(self.conversation_history) // 2,
+            'vector_cache': cache_info,
+            'last_update': datetime.now().isoformat()
+        }
+    
+    def save_conversation(self, filepath: str = None):
+        """
+        保存对话历史
+        
+        Args:
+            filepath: 保存路径
+        """
+        if filepath is None:
+            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+            filepath = f"conversation_{timestamp}.json"
+        
+        conversation_data = {
+            "timestamp": datetime.now().isoformat(),
+            "model": self.model,
+            "system_status": self.get_system_status(),
+            "history": self.conversation_history
+        }
+        
+        with open(filepath, 'w', encoding='utf-8') as f:
+            json.dump(conversation_data, f, ensure_ascii=False, indent=2)
+        
+        print(f"[save_conversation] Conversation saved to: {filepath}")

README.md

@@ -1,13 +1,14 @@
 ---
-title: CIV3283 Student 17
-emoji: 🐠
-colorFrom: purple
-colorTo: gray
+title: CIV3283 Student 16
+emoji: ⚡
+colorFrom: gray
+colorTo: yellow
 sdk: gradio
-sdk_version: 5.38.2
+sdk_version: 5.34.0
 app_file: app.py
 pinned: false
 license: mit
+short_description: alternative space 16
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,630 @@
+import gradio as gr
+import csv
+import os
+import re
+from datetime import datetime, timedelta
+from huggingface_hub import Repository
+from RAG_Learning_Assistant_with_Streaming import RAGLearningAssistant
+
+# Configuration for Student Space
+STUDENT_SPACE_NAME = "CIV3283_Student_16"  # Replace with actual student space name (e.g., "student-group-1")
+DATA_STORAGE_REPO = "CIV3283/Data_Storage"  # Centralized data storage repo
+DATA_BRANCH_NAME = "data_branch"
+LOCAL_DATA_DIR = "temp_data_repo"
+
+# Session timeout configuration (in minutes)
+SESSION_TIMEOUT_MINUTES = 30  # Adjust this value as needed
+
+# File names in data storage
+KNOWLEDGE_FILE = "knowledge_base.md"
+VECTOR_DB_FILE = "vector_database.csv"
+METADATA_FILE = "vector_metadata.json"
+VECTORIZER_FILE = "vectorize_knowledge_base.py"
+
+# Student-specific log files (with space name prefix)
+QUERY_LOG_FILE = f"{STUDENT_SPACE_NAME}_query_log.csv"
+FEEDBACK_LOG_FILE = f"{STUDENT_SPACE_NAME}_feedback_log.csv"
+
+# Environment variables
+HF_HUB_TOKEN = os.environ.get("HF_HUB_TOKEN", None)
+if HF_HUB_TOKEN is None:
+    raise ValueError("Set HF_HUB_TOKEN in Space Settings -> Secrets")
+
+OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY", None)
+if OPENAI_API_KEY is None:
+    raise ValueError("Set OPENAI_API_KEY in Space Settings -> Secrets")
+
+MODEL = "gpt-4.1-nano-2025-04-14"
+
+def check_session_validity(check_id):
+    """
+    Check if the current session is valid based on:
+    1. If user ID matches last query → Allow continue
+    2. If user ID doesn't match → Check time interval:
+       - If time interval is small → Block (previous user just finished)
+       - If time interval is large → Allow (assistant has been idle)
+    
+    Returns:
+        tuple: (is_valid: bool, error_message: str)
+    """
+    try:
+        filepath = os.path.join(LOCAL_DATA_DIR, QUERY_LOG_FILE)
+        
+        # If no log file exists, this is the first query - allow it
+        if not os.path.exists(filepath):
+            print(f"[check_session_validity] No existing log file, allowing first query for student {check_id}")
+            return True, ""
+        
+        # Read the last record from the CSV file
+        with open(filepath, 'r', encoding='utf-8') as csvfile:
+            reader = csv.reader(csvfile)
+            rows = list(reader)
+            
+            # If only header exists, this is effectively the first query
+            if len(rows) <= 1:
+                print(f"[check_session_validity] Only header in log file, allowing first query for student {check_id}")
+                return True, ""
+            
+            # Get the last record (most recent query)
+            last_record = rows[-1]
+            
+            # CSV format: [student_space, student_id, timestamp, search_info, query_and_response, thumb_feedback]
+            if len(last_record) < 3:
+                print(f"[check_session_validity] Invalid last record format, allowing query")
+                return True, ""
+            
+            last_student_id = last_record[1]
+            last_timestamp_str = last_record[2]
+            
+            print(f"[check_session_validity] Last record - Student ID: {last_student_id}, Timestamp: {last_timestamp_str}")
+            print(f"[check_session_validity] Current request - Student ID: {check_id}")
+            
+            # If student ID matches, allow continuation
+            if last_student_id == check_id:
+                print(f"[check_session_validity] Same user, allowing continuation for student {check_id}")
+                return True, ""
+            
+            # If student ID doesn't match, check time interval
+            try:
+                last_timestamp = datetime.strptime(last_timestamp_str, '%Y-%m-%d %H:%M:%S')
+                current_timestamp = datetime.now()
+                time_diff = current_timestamp - last_timestamp
+                
+                print(f"[check_session_validity] Different user - Time difference: {time_diff.total_seconds()} seconds ({time_diff.total_seconds()/60:.1f} minutes)")
+                
+                # If time difference is small, block access (previous user just finished)
+                if time_diff <= timedelta(minutes=SESSION_TIMEOUT_MINUTES):
+                    error_msg = "⚠️ The assistant is currently being used by another user. Please return to the load distributor page."
+                    print(f"[check_session_validity] Blocking access - Previous user ({last_student_id}) used assistant {time_diff.total_seconds()/60:.1f} minutes ago")
+                    return False, error_msg
+                
+                # If time difference is large, allow access (assistant has been idle)
+                print(f"[check_session_validity] Assistant has been idle for {time_diff.total_seconds()/60:.1f} minutes, allowing new user {check_id}")
+                return True, ""
+                
+            except ValueError as e:
+                print(f"[check_session_validity] Error parsing timestamp: {e}")
+                # If we can't parse the timestamp, allow the query to proceed
+                return True, ""
+    
+    except Exception as e:
+        print(f"[check_session_validity] Error checking session validity: {e}")
+        import traceback
+        print(f"[check_session_validity] Traceback: {traceback.format_exc()}")
+        # On error, allow the query to proceed to avoid blocking legitimate users
+        return True, ""
+
+def init_data_storage_repo():
+    """Initialize connection to centralized data storage repository"""
+    try:
+        repo = Repository(
+            local_dir=LOCAL_DATA_DIR,
+            clone_from=DATA_STORAGE_REPO,
+            revision=DATA_BRANCH_NAME,
+            repo_type="space",
+            use_auth_token=HF_HUB_TOKEN
+        )
+        # Configure git user
+        repo.git_config_username_and_email("git_user", f"Student_Space_{STUDENT_SPACE_NAME}")
+        repo.git_config_username_and_email("git_email", f"{STUDENT_SPACE_NAME}@student.space")
+        
+        # Pull latest changes
+        print(f"[init_data_storage_repo] Pulling latest changes from {DATA_STORAGE_REPO}...")
+        repo.git_pull(rebase=True)
+        
+        print(f"[init_data_storage_repo] Successfully connected to data storage repo: {DATA_STORAGE_REPO}")
+        print(f"[init_data_storage_repo] Local directory: {LOCAL_DATA_DIR}")
+        print(f"[init_data_storage_repo] Branch: {DATA_BRANCH_NAME}")
+        
+        # Check if required files exist
+        required_files = [KNOWLEDGE_FILE, VECTOR_DB_FILE, METADATA_FILE]
+        for file_name in required_files:
+            file_path = os.path.join(LOCAL_DATA_DIR, file_name)
+            if os.path.exists(file_path):
+                print(f"[init_data_storage_repo] Found required file: {file_name}")
+            else:
+                print(f"[init_data_storage_repo] Warning: Missing required file: {file_name}")
+        
+        return repo
+        
+    except Exception as e:
+        print(f"[init_data_storage_repo] Error initializing repository: {e}")
+        import traceback
+        print(f"[init_data_storage_repo] Traceback: {traceback.format_exc()}")
+        return None
+
+def commit_student_logs(commit_message: str):
+    """Commit student logs to data storage repository with conflict resolution"""
+    if repo is None:
+        print("[commit_student_logs] Error: Repository not initialized")
+        return False
+    
+    max_retries = 3
+    retry_count = 0
+    
+    while retry_count < max_retries:
+        try:
+            # Check if log files exist before adding
+            query_log_path = os.path.join(LOCAL_DATA_DIR, QUERY_LOG_FILE)
+            feedback_log_path = os.path.join(LOCAL_DATA_DIR, FEEDBACK_LOG_FILE)
+            
+            files_to_add = []
+            if os.path.exists(query_log_path):
+                files_to_add.append(QUERY_LOG_FILE)
+                print(f"[commit_student_logs] Found query log: {query_log_path}")
+            
+            if os.path.exists(feedback_log_path):
+                files_to_add.append(FEEDBACK_LOG_FILE)
+                print(f"[commit_student_logs] Found feedback log: {feedback_log_path}")
+            
+            if not files_to_add:
+                print("[commit_student_logs] No log files to commit")
+                return False
+            
+            # Add files individually
+            for file_name in files_to_add:
+                print(f"[commit_student_logs] Adding file: {file_name}")
+                repo.git_add(pattern=file_name)
+            
+            # Check if there are changes to commit
+            try:
+                import subprocess
+                result = subprocess.run(
+                    ["git", "status", "--porcelain"], 
+                    cwd=LOCAL_DATA_DIR,
+                    capture_output=True, 
+                    text=True,
+                    check=True
+                )
+                
+                if not result.stdout.strip():
+                    print("[commit_student_logs] No changes to commit")
+                    return True
+                
+                print(f"[commit_student_logs] Changes detected: {result.stdout.strip()}")
+                
+            except Exception as status_error:
+                print(f"[commit_student_logs] Warning: Could not check git status: {status_error}")
+            
+            # Commit changes locally first
+            print(f"[commit_student_logs] Attempt {retry_count + 1}/{max_retries}: Committing locally: {commit_message}")
+            repo.git_commit(commit_message)
+            
+            # Now try to pull and push
+            print("[commit_student_logs] Pulling latest changes...")
+            repo.git_pull(rebase=True)
+            
+            # Push changes
+            print("[commit_student_logs] Pushing to remote...")
+            repo.git_push()
+            
+            print(f"[commit_student_logs] Success: {commit_message}")
+            return True
+            
+        except Exception as e:
+            error_msg = str(e)
+            print(f"[commit_student_logs] Attempt {retry_count + 1} failed: {error_msg}")
+            
+            # Check if it's a push conflict or pull conflict
+            if ("rejected" in error_msg and "fetch first" in error_msg) or ("cannot pull with rebase" in error_msg):
+                print("[commit_student_logs] Detected Git conflict, will retry...")
+                retry_count += 1
+                
+                if retry_count < max_retries:
+                    # Try to reset and start fresh
+                    try:
+                        print("[commit_student_logs] Resetting repository state for retry...")
+                        # Reset to remote state
+                        repo.git_reset("--hard", "HEAD~1")  # Undo the commit
+                        repo.git_pull(rebase=True)  # Get latest changes
+                        
+                        # Wait a bit before retrying to avoid rapid conflicts
+                        import time
+                        wait_time = retry_count * 2  # 2, 4, 6 seconds
+                        print(f"[commit_student_logs] Waiting {wait_time} seconds before retry...")
+                        time.sleep(wait_time)
+                        continue
+                        
+                    except Exception as reset_error:
+                        print(f"[commit_student_logs] Reset failed: {reset_error}")
+                        # If reset fails, try alternative approach
+                        try:
+                            # Alternative: stash changes and pull
+                            repo.git_stash()
+                            repo.git_pull(rebase=True)
+                            repo.git_stash("pop")
+                            continue
+                        except Exception as stash_error:
+                            print(f"[commit_student_logs] Stash approach failed: {stash_error}")
+                            return False
+                else:
+                    print("[commit_student_logs] Max retries reached, giving up")
+                    return False
+            else:
+                # Other types of errors, don't retry
+                print(f"[commit_student_logs] Non-conflict error, not retrying: {error_msg}")
+                return False
+    
+    print("[commit_student_logs] Failed after all retry attempts")
+    return False
+
+def save_student_query_to_csv(query, search_info, response, check_id, thumb_feedback=None):
+    """Save student query record to centralized CSV file"""
+    try:
+        # Validate check_id
+        if not check_id:
+            print("[save_student_query_to_csv] Error: No valid check_id provided")
+            return False
+        
+        # Ensure the local data directory exists
+        os.makedirs(LOCAL_DATA_DIR, exist_ok=True)
+        
+        filepath = os.path.join(LOCAL_DATA_DIR, QUERY_LOG_FILE)
+        file_exists = os.path.isfile(filepath)
+        
+        print(f"[save_student_query_to_csv] Saving to: {filepath}")
+        print(f"[save_student_query_to_csv] File exists: {file_exists}")
+        print(f"[save_student_query_to_csv] Student ID: {check_id}")
+        
+        with open(filepath, 'a', newline='', encoding='utf-8') as csvfile:
+            writer = csv.writer(csvfile)
+            if not file_exists:
+                print("[save_student_query_to_csv] Writing header row")
+                writer.writerow(['student_space', 'student_id', 'timestamp', 'search_info', 'query_and_response', 'thumb_feedback'])
+            
+            timestamp = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+            query_and_response = f"Query: {query}\nResponse: {response}"
+            writer.writerow([STUDENT_SPACE_NAME, check_id, timestamp, search_info, query_and_response, thumb_feedback or ""])
+        
+        print(f"[save_student_query_to_csv] Query saved to local file: {filepath}")
+        
+        # Commit student logs to data storage
+        print("[save_student_query_to_csv] Attempting to commit to remote repository...")
+        commit_success = commit_student_logs(f"Add query log from student {check_id} at {timestamp}")
+        
+        if commit_success:
+            print("[save_student_query_to_csv] Successfully committed to remote repository")
+        else:
+            print("[save_student_query_to_csv] Failed to commit to remote repository")
+        
+        return True
+    except Exception as e:
+        print(f"[save_student_query_to_csv] Error: {e}")
+        import traceback
+        print(f"[save_student_query_to_csv] Traceback: {traceback.format_exc()}")
+        return False
+
+def update_latest_student_query_feedback(feedback_type, check_id):
+    """Update thumb feedback for the latest student query in CSV"""
+    try:
+        # Validate check_id
+        if not check_id:
+            print("[update_latest_student_query_feedback] Error: No valid check_id provided")
+            return False
+        
+        filepath = os.path.join(LOCAL_DATA_DIR, QUERY_LOG_FILE)
+        if not os.path.exists(filepath):
+            print("[update_latest_student_query_feedback] Error: Query log file not found")
+            return False
+        
+        # Read existing data
+        rows = []
+        with open(filepath, 'r', encoding='utf-8') as csvfile:
+            reader = csv.reader(csvfile)
+            rows = list(reader)
+        
+        # Update the last row (most recent query)
+        if len(rows) > 1:  # Ensure there's at least one data row beyond header
+            rows[-1][5] = feedback_type  # thumb_feedback column (index 5 for student format)
+            
+            # Write back to file
+            with open(filepath, 'w', newline='', encoding='utf-8') as csvfile:
+                writer = csv.writer(csvfile)
+                writer.writerows(rows)
+            
+            print(f"[update_latest_student_query_feedback] Updated feedback: {feedback_type}")
+            
+            # Commit the update
+            timestamp = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+            commit_student_logs(f"Update feedback from student {check_id}: {feedback_type} at {timestamp}")
+            return True
+        
+        return False
+    except Exception as e:
+        print(f"[update_latest_student_query_feedback] Error: {e}")
+        return False
+
+def save_student_comment_feedback(comment, check_id):
+    """Save student comment feedback to centralized feedback file"""
+    try:
+        # Validate check_id
+        if not check_id:
+            print("[save_student_comment_feedback] Error: No valid check_id provided")
+            return False
+        
+        filepath = os.path.join(LOCAL_DATA_DIR, FEEDBACK_LOG_FILE)
+        file_exists = os.path.isfile(filepath)
+        
+        with open(filepath, 'a', newline='', encoding='utf-8') as csvfile:
+            writer = csv.writer(csvfile)
+            if not file_exists:
+                writer.writerow(['student_space', 'student_id', 'timestamp', 'comment'])
+            
+            timestamp = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+            writer.writerow([STUDENT_SPACE_NAME, check_id, timestamp, comment])
+        
+        print(f"[save_student_comment_feedback] Saved comment to {filepath}")
+        
+        # Commit student logs
+        commit_student_logs(f"Add comment feedback from student {check_id} at {timestamp}")
+        
+        return True
+    except Exception as e:
+        print(f"[save_student_comment_feedback] Error: {e}")
+        return False
+
+def get_url_params(request: gr.Request):
+    """Extract URL parameters from request"""
+    if request:
+        query_params = dict(request.query_params)
+        check_id = query_params.get('check', None)
+        if check_id:
+            return f"RAG Learning Assistant - Student", check_id
+        else:
+            return "RAG Learning Assistant - Student", None
+    return "RAG Learning Assistant - Student", None
+
+def chat_response(message, history, search_info_display, check_id, has_query):
+    """Process user input and return streaming response"""
+    if not message.strip():
+        return history, search_info_display, has_query
+    
+    # Check access permission first
+    if not check_id:
+        print(f"[chat_response] Access denied: No valid check ID provided")
+        # Raise error dialog for access denial
+        raise gr.Error(
+            "⚠️ Access Restricted\n\n"
+            "Please access this system through the link provided in Moodle.\n\n"
+            "If you are a student in this course:\n"
+            "1. Go to your Moodle course page\n"
+            "2. Find the 'CivASK' link\n"
+            "3. Click the link to access the system\n\n"
+            "If you continue to experience issues, please contact your instructor.",
+            duration=8
+        )
+    
+    # NEW: Check session validity before proceeding
+    session_valid, error_message = check_session_validity(check_id)
+    if not session_valid:
+        print(f"[chat_response] Session invalid for student {check_id}")
+        raise gr.Error(error_message, duration=10)
+    
+    # Valid access and valid session - proceed with normal AI conversation
+    print(f"[chat_response] Valid access and session for student ID: {check_id}")
+    
+    # Convert to messages format if needed
+    if history and isinstance(history[0], list):
+        # Convert from tuples to messages format
+        messages_history = []
+        for user_msg, assistant_msg in history:
+            messages_history.append({"role": "user", "content": user_msg})
+            if assistant_msg:
+                messages_history.append({"role": "assistant", "content": assistant_msg})
+        history = messages_history
+    
+    # Add user message
+    history.append({"role": "user", "content": message})
+    history.append({"role": "assistant", "content": ""})
+    
+    search_info_collected = False
+    search_info_content = ""
+    content_part = ""
+    
+    # Process streaming response
+    for chunk in assistant.generate_response_stream(message):
+        if not search_info_collected:
+            if "**Response:**" in chunk:  # Support English markers
+                search_info_content += chunk
+                search_info_collected = True
+                yield history, search_info_content, has_query
+            else:
+                search_info_content += chunk
+                yield history, search_info_content, has_query
+        else:
+            content_part += chunk
+            # Update the last assistant message
+            history[-1]["content"] = content_part
+            yield history, search_info_content, has_query
+    
+    # After streaming is complete, save to CSV (only for valid access)
+    try:
+        print(f"[chat_response] Saving student query to CSV...")
+        print(f"Student Space: {STUDENT_SPACE_NAME}")
+        print(f"Student ID: {check_id}")
+        print(f"Query: {message}")
+        
+        save_success = save_student_query_to_csv(message, search_info_content, content_part, check_id)
+        if save_success:
+            print(f"[chat_response] Student query saved successfully")
+            has_query = True  # Mark that we have a query to rate
+        else:
+            print(f"[chat_response] Failed to save student query")
+        
+    except Exception as e:
+        print(f"[chat_response] Error saving student query: {e}")
+    
+    return history, search_info_content, has_query
+
+# Global variables
+repo = None
+assistant = None
+
+def main():
+    """Main function to initialize and launch the student application"""
+    global repo, assistant
+    
+    # Initialize data storage repository connection
+    repo = init_data_storage_repo()
+    
+    # Initialize RAG assistant with centralized data storage directory
+    print(f"[main] Initializing RAG assistant with data directory: {LOCAL_DATA_DIR}")
+    print(f"[main] Session timeout set to: {SESSION_TIMEOUT_MINUTES} minutes")
+    assistant = RAGLearningAssistant(
+        api_key=OPENAI_API_KEY, 
+        model=MODEL, 
+        vector_db_path=LOCAL_DATA_DIR  # Pass the data storage repo directory
+    )
+    
+    print(f"[main] RAG assistant initialized successfully")
+    print(f"[main] Student space: {STUDENT_SPACE_NAME}")
+    print(f"[main] Data storage repo: {DATA_STORAGE_REPO}")
+    print(f"[main] Query log file: {QUERY_LOG_FILE}")
+    print(f"[main] Feedback log file: {FEEDBACK_LOG_FILE}")
+    
+    # Create interface
+    with gr.Blocks(title=f"RAG Assistant - {STUDENT_SPACE_NAME}") as interface:
+        check_id_state = gr.State("1")
+        has_query_state = gr.State(False)  # Track if there's a query to rate
+        title_display = gr.Markdown(f"# RAG Learning Assistant - {STUDENT_SPACE_NAME}", elem_id="title")
+        
+        # Only Query Check functionality for students
+        with gr.Row():
+            with gr.Column(scale=4):
+                chatbot = gr.Chatbot(label="Ask Your Questions", height=500, type="messages", render_markdown=True)
+                msg = gr.Textbox(placeholder="Type your message here...", label="Your Message", show_label=True)
+                
+                # Feedback buttons row
+                with gr.Row():
+                    thumbs_up_btn = gr.Button("👍 Good Answer", variant="secondary", size="sm")
+                    thumbs_down_btn = gr.Button("👎 Poor Answer", variant="secondary", size="sm")
+                
+                feedback_status = gr.Textbox(label="Feedback Status", interactive=False, lines=1)
+                
+                # Comment section
+                with gr.Row():
+                    comment_input = gr.Textbox(placeholder="Share your comments or suggestions...", label="Comments", lines=2)
+                    submit_comment_btn = gr.Button("Submit Comment", variant="outline")
+            
+            with gr.Column(scale=1):
+                search_info = gr.Markdown(label="Search Analysis Information", value="")
+        
+        # Event handlers
+        def init_from_url(request: gr.Request):
+            title, check_id = get_url_params(request)
+            print(f"[init_from_url] Extracted check_id: {check_id}")
+            return f"# {title}", check_id, False  # Reset has_query state
+        
+        # Feedback handlers
+        def handle_thumbs_up(check_id, has_query):
+            if not check_id:
+                raise gr.Error(
+                    "⚠️ Access Restricted\n\n"
+                    "Please access this system through the CivASK link provided in Moodle to use the feedback features.",
+                    duration=5
+                )
+            
+            print(f"[handle_thumbs_up] Student: {STUDENT_SPACE_NAME}, check_id: {check_id}")
+            
+            # Check if student query log exists and has queries
+            filepath = os.path.join(LOCAL_DATA_DIR, QUERY_LOG_FILE)
+            if os.path.exists(filepath):
+                with open(filepath, 'r', encoding='utf-8') as csvfile:
+                    reader = csv.reader(csvfile)
+                    rows = list(reader)
+                    if len(rows) > 1:  # Has header + at least one data row
+                        success = update_latest_student_query_feedback("thumbs_up", check_id)
+                        return "👍 Thank you for your positive feedback!" if success else "Failed to save feedback"
+            
+            return "No query to rate yet"
+        
+        def handle_thumbs_down(check_id, has_query):
+            if not check_id:
+                raise gr.Error(
+                    "⚠️ Access Restricted\n\n"
+                    "Please access this system through the CivASK link provided in Moodle to use the feedback features.",
+                    duration=5
+                )
+            
+            print(f"[handle_thumbs_down] Student: {STUDENT_SPACE_NAME}, check_id: {check_id}")
+            
+            # Check if student query log exists and has queries
+            filepath = os.path.join(LOCAL_DATA_DIR, QUERY_LOG_FILE)
+            if os.path.exists(filepath):
+                with open(filepath, 'r', encoding='utf-8') as csvfile:
+                    reader = csv.reader(csvfile)
+                    rows = list(reader)
+                    if len(rows) > 1:  # Has header + at least one data row
+                        success = update_latest_student_query_feedback("thumbs_down", check_id)
+                        return "👎 Thank you for your feedback. We'll work to improve!" if success else "Failed to save feedback"
+            
+            return "No query to rate yet"
+        
+        def handle_comment_submission(comment, check_id):
+            if not check_id:
+                raise gr.Error(
+                    "⚠️ Access Restricted\n\n"
+                    "Please access this system through the CivASK link provided in Moodle to submit comments.",
+                    duration=5
+                )
+            
+            if comment.strip():
+                success = save_student_comment_feedback(comment.strip(), check_id)
+                if success:
+                    return "💬 Thank you for your comment!", ""
+                else:
+                    return "Failed to save comment", comment
+            return "Please enter a comment", comment
+        
+        interface.load(fn=init_from_url, outputs=[title_display, check_id_state, has_query_state])
+        
+        # Query events
+        msg.submit(
+            chat_response, 
+            [msg, chatbot, search_info, check_id_state, has_query_state], 
+            [chatbot, search_info, has_query_state]
+        ).then(lambda: "", outputs=[msg])
+        
+        # Feedback events
+        thumbs_up_btn.click(
+            handle_thumbs_up,
+            inputs=[check_id_state, has_query_state],
+            outputs=[feedback_status]
+        )
+        
+        thumbs_down_btn.click(
+            handle_thumbs_down,
+            inputs=[check_id_state, has_query_state],
+            outputs=[feedback_status]
+        )
+        
+        submit_comment_btn.click(
+            handle_comment_submission,
+            inputs=[comment_input, check_id_state],
+            outputs=[feedback_status, comment_input]
+        )
+    
+    interface.launch()
+
+if __name__ == "__main__":
+    main()

gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

requirements.txt

@@ -0,0 +1,6 @@
+gradio==5.34.0
+openai==1.86.0
+pandas==2.2.3
+numpy==2.2.3
+huggingface-hub==0.33.0
+scipy==1.15.2

vectorize_knowledge_base.py

@@ -0,0 +1,515 @@
+import os
+import re
+import json
+import numpy as np
+import pandas as pd
+from typing import List, Dict, Tuple, Optional
+from openai import OpenAI
+from datetime import datetime
+import csv
+
+class KnowledgeBaseVectorizer:
+    def __init__(self, api_key: str, data_path: str = "", vector_db_dir: str = ""):
+        """
+        初始化向量化器（适配学生Space）
+        
+        Args:
+            api_key: OpenAI API密钥
+            data_path: knowledge_base.md文件的路径（如果为空，使用vector_db_dir中的文件）
+            vector_db_dir: 向量数据库所在目录（通常是数据存储仓库的本地目录）
+        """
+        self.client = OpenAI(api_key=api_key)
+        self.embedding_model = "text-embedding-3-small"
+        
+        # 如果指定了vector_db_dir，优先使用该目录中的文件
+        if vector_db_dir:
+            self.data_path = os.path.join(vector_db_dir, "knowledge_base.md")
+            self.vector_db_path = os.path.join(vector_db_dir, "vector_database.csv")
+            self.metadata_path = os.path.join(vector_db_dir, "vector_metadata.json")
+        else:
+            # 保持原有逻辑用于向后兼容
+            self.data_path = data_path if data_path else "knowledge_base.md"
+            self.vector_db_path = "vector_database.csv"
+            self.metadata_path = "vector_metadata.json"
+        
+        # 缓存相关属性
+        self._cached_df = None
+        self._cached_metadata = None
+        self._cached_embeddings = {}  # 缓存不同类型的向量矩阵
+        self._last_load_time = None
+        
+        print(f"[KnowledgeBaseVectorizer] Initialized with:")
+        print(f"  - Knowledge base: {self.data_path}")
+        print(f"  - Vector database: {self.vector_db_path}")
+        print(f"  - Metadata: {self.metadata_path}")
+        
+    def parse_knowledge_base(self) -> List[Dict]:
+        """
+        解析knowledge_base.md文件，提取所有数据条目
+        支持包含表格的完整内容提取
+        
+        Returns:
+            包含所有数据条目的列表，每个条目是一个字典
+        """
+        entries = []
+        
+        try:
+            with open(self.data_path, 'r', encoding='utf-8') as f:
+                content = f.read()
+            print(f"[parse_knowledge_base] Successfully read file: {self.data_path}")
+        except FileNotFoundError:
+            print(f"[parse_knowledge_base] Error: File not found - {self.data_path}")
+            return entries
+        except Exception as e:
+            print(f"[parse_knowledge_base] Error reading file: {e}")
+            return entries
+        
+        # 改进的匹配策略：使用更精确的正则表达式
+        # 匹配模式：# xx-xx-xx title **source** ... **content** ... (直到下一个 # 或文件结尾)
+        pattern = r'#\s+(\d{2}-\d{2}-\d{2})\s+([^\n]+)\s+\*\*source\*\*\s+([^\n]+)\s+\*\*content\*\*\s+(.*?)(?=\n#\s+\d{2}-\d{2}-\d{2}|$)'
+        
+        matches = re.findall(pattern, content, re.DOTALL)
+        
+        for match in matches:
+            # 清理内容：移除多余的空白行，但保留表格格式
+            content_text = match[3].strip()
+            
+            # 保留表格的结构，但清理多余的空白
+            content_lines = content_text.split('\n')
+            cleaned_lines = []
+            
+            for line in content_lines:
+                # 保留非空行和表格行
+                if line.strip() or (line.startswith('|') and line.endswith('|')):
+                    cleaned_lines.append(line.rstrip())
+            
+            # 重新组合内容
+            cleaned_content = '\n'.join(cleaned_lines)
+            
+            entry = {
+                'id': match[0].strip(),
+                'title': match[1].strip(),
+                'source': match[2].strip(),
+                'content': cleaned_content,
+                'full_text': f"{match[1].strip()} {cleaned_content}"  # 用于向量化的完整文本
+            }
+            entries.append(entry)
+            
+        print(f"[parse_knowledge_base] Successfully parsed {len(entries)} entries")
+        
+        # 打印一些调试信息
+        if entries:
+            print("[parse_knowledge_base] First 3 entries info:")
+            for i, entry in enumerate(entries[:3]):
+                content_lines = entry['content'].count('\n') + 1
+                has_table = '|' in entry['content']
+                print(f"  Entry {entry['id']}: {len(entry['content'])} chars, {content_lines} lines, has table: {has_table}")
+        
+        return entries
+    
+    def get_embedding(self, text: str) -> List[float]:
+        """
+        使用OpenAI API获取文本的向量表示
+        
+        Args:
+            text: 要向量化的文本
+            
+        Returns:
+            文本的向量表示
+        """
+        try:
+            response = self.client.embeddings.create(
+                input=text,
+                model=self.embedding_model
+            )
+            return response.data[0].embedding
+        except Exception as e:
+            print(f"[get_embedding] Error: {e}")
+            return []
+    
+    def batch_get_embeddings(self, texts: List[str], batch_size: int = 10) -> List[List[float]]:
+        """
+        批量获取文本的向量表示
+        
+        Args:
+            texts: 要向量化的文本列表
+            batch_size: 批处理大小
+            
+        Returns:
+            向量列表
+        """
+        embeddings = []
+        
+        for i in range(0, len(texts), batch_size):
+            batch = texts[i:i + batch_size]
+            print(f"[batch_get_embeddings] Processing batch {i//batch_size + 1}/{(len(texts) + batch_size - 1)//batch_size}")
+            
+            try:
+                response = self.client.embeddings.create(
+                    input=batch,
+                    model=self.embedding_model
+                )
+                batch_embeddings = [item.embedding for item in response.data]
+                embeddings.extend(batch_embeddings)
+            except Exception as e:
+                print(f"[batch_get_embeddings] Batch error: {e}")
+                # 如果批处理失败，尝试单个处理
+                for text in batch:
+                    embedding = self.get_embedding(text)
+                    embeddings.append(embedding if embedding else [0] * 1536)  # 默认维度
+        
+        return embeddings
+    
+    def create_vector_database(self):
+        """
+        创建向量数据库并保存为CSV文件
+        支持标题和内容的分别向量化
+        """
+        print("[create_vector_database] Starting to create vector database...")
+        
+        # 1. 解析知识库
+        entries = self.parse_knowledge_base()
+        if not entries:
+            print("[create_vector_database] No entries found")
+            return
+        
+        # 2. 准备要向量化的文本
+        titles = [entry['title'] for entry in entries]
+        contents = [entry['content'] for entry in entries]
+        full_texts = [entry['full_text'] for entry in entries]
+        
+        # 3. 批量获取向量
+        print("[create_vector_database] Vectorizing titles...")
+        title_embeddings = self.batch_get_embeddings(titles)
+        
+        print("[create_vector_database] Vectorizing contents...")
+        content_embeddings = self.batch_get_embeddings(contents)
+        
+        print("[create_vector_database] Vectorizing full texts...")
+        full_embeddings = self.batch_get_embeddings(full_texts)
+        
+        # 4. 创建DataFrame来存储数据
+        print("[create_vector_database] Creating DataFrame...")
+        
+        # 准备数据行
+        rows = []
+        for i, entry in enumerate(entries):
+            row = {
+                'index': i,
+                'id': entry['id'],
+                'title': entry['title'],
+                'source': entry['source'],
+                'content': entry['content'],
+                'full_text': entry['full_text']
+            }
+            
+            # 添加标题向量维度
+            for j, val in enumerate(title_embeddings[i]):
+                row[f'title_dim_{j}'] = val
+            
+            # 添加内容向量维度
+            for j, val in enumerate(content_embeddings[i]):
+                row[f'content_dim_{j}'] = val
+                
+            # 添加完整文本向量维度
+            for j, val in enumerate(full_embeddings[i]):
+                row[f'full_dim_{j}'] = val
+            
+            rows.append(row)
+        
+        # 创建DataFrame
+        df = pd.DataFrame(rows)
+        
+        # 5. 保存为CSV文件
+        print(f"[create_vector_database] Saving to {self.vector_db_path}...")
+        df.to_csv(self.vector_db_path, index=False, encoding='utf-8')
+        
+        # 6. 保存元数据（JSON格式，便于查看）
+        metadata = {
+            'embedding_model': self.embedding_model,
+            'created_at': datetime.now().isoformat(),
+            'num_entries': len(entries),
+            'embedding_dimensions': len(title_embeddings[0]) if title_embeddings else 0,
+            'vector_types': ['title', 'content', 'full'],
+            'columns': list(df.columns),
+            'entries_summary': [
+                {
+                    'id': entry['id'],
+                    'title': entry['title'],
+                    'source': entry['source']
+                } for entry in entries
+            ]
+        }
+        
+        with open(self.metadata_path, 'w', encoding='utf-8') as f:
+            json.dump(metadata, f, ensure_ascii=False, indent=2)
+        
+        print(f"[create_vector_database] Vector database created successfully!")
+        print(f"  - Vector database saved to: {self.vector_db_path}")
+        print(f"  - Metadata saved to: {self.metadata_path}")
+        print(f"  - Processed {len(entries)} entries")
+        print(f"  - Vector dimensions: {len(title_embeddings[0]) if title_embeddings else 0}")
+        
+        # 清除缓存以便重新加载
+        self.clear_cache()
+    
+    def clear_cache(self):
+        """清除所有缓存"""
+        self._cached_df = None
+        self._cached_metadata = None
+        self._cached_embeddings = {}
+        self._last_load_time = None
+        print("[clear_cache] Vector database cache cleared")
+    
+    def load_vector_database(self, force_reload: bool = False) -> Tuple[Optional[pd.DataFrame], Optional[Dict]]:
+        """
+        从CSV文件加载向量数据库（带缓存机制）
+        
+        Args:
+            force_reload: 是否强制重新加载
+            
+        Returns:
+            DataFrame和元数据字典的元组
+        """
+        # 检查是否需要重新加载
+        if not force_reload and self._cached_df is not None and self._cached_metadata is not None:
+            return self._cached_df, self._cached_metadata
+        
+        try:
+            # 加载CSV文件
+            print(f"[load_vector_database] Loading from {self.vector_db_path}")
+            df = pd.read_csv(self.vector_db_path, encoding='utf-8')
+            
+            # 加载元数据
+            print(f"[load_vector_database] Loading metadata from {self.metadata_path}")
+            with open(self.metadata_path, 'r', encoding='utf-8') as f:
+                metadata = json.load(f)
+            
+            # 缓存结果
+            self._cached_df = df
+            self._cached_metadata = metadata
+            self._last_load_time = datetime.now()
+            
+            # 预加载向量矩阵到缓存
+            self._preload_embeddings()
+            
+            print(f"[load_vector_database] Successfully loaded vector database with {len(df)} entries")
+            return df, metadata
+        except FileNotFoundError as e:
+            print(f"[load_vector_database] Error: File not found - {e}")
+            return None, None
+        except Exception as e:
+            print(f"[load_vector_database] Error loading vector database: {e}")
+            return None, None
+    
+    def _preload_embeddings(self):
+        """预加载所有类型的向量矩阵到缓存"""
+        if self._cached_df is None:
+            return
+        
+        vector_types = ['title', 'content', 'full']
+        for vector_type in vector_types:
+            if vector_type not in self._cached_embeddings:
+                embeddings = self.get_embeddings_from_df(self._cached_df, vector_type)
+                # 预计算归一化向量
+                embeddings_norm = embeddings / np.linalg.norm(embeddings, axis=1, keepdims=True)
+                self._cached_embeddings[vector_type] = {
+                    'raw': embeddings,
+                    'normalized': embeddings_norm
+                }
+        
+        print(f"[_preload_embeddings] Preloaded {len(vector_types)} types of vector matrices")
+    
+    def get_embeddings_from_df(self, df: pd.DataFrame, vector_type: str = 'full') -> np.ndarray:
+        """
+        从DataFrame中提取向量矩阵
+        
+        Args:
+            df: 包含向量的DataFrame
+            vector_type: 向量类型 ('title', 'content', 'full')
+            
+        Returns:
+            向量矩阵
+        """
+        # 根据类型获取对应的列
+        if vector_type == 'title':
+            embedding_cols = [col for col in df.columns if col.startswith('title_dim_')]
+        elif vector_type == 'content':
+            embedding_cols = [col for col in df.columns if col.startswith('content_dim_')]
+        else:  # 'full'
+            embedding_cols = [col for col in df.columns if col.startswith('full_dim_')]
+            
+        embeddings = df[embedding_cols].values
+        return embeddings
+    
+    def batch_search_similar(self, queries: List[str], top_k: int = 5, 
+                           title_weight: float = 0.4, 
+                           content_weight: float = 0.3,
+                           full_weight: float = 0.3) -> List[List[Tuple[Dict, float, Dict]]]:
+        """
+        批量搜索多个查询，只加载一次向量数据库
+        
+        Args:
+            queries: 查询文本列表
+            top_k: 每个查询返回最相似的前k个结果
+            title_weight: 标题相似度的权重
+            content_weight: 内容相似度的权重
+            full_weight: 完整文本相似度的权重
+            
+        Returns:
+            每个查询对应的相似条目列表
+        """
+        # 确保权重之和为1
+        total_weight = title_weight + content_weight + full_weight
+        title_weight /= total_weight
+        content_weight /= total_weight
+        full_weight /= total_weight
+        
+        # 加载向量数据库（只加载一次）
+        df, metadata = self.load_vector_database()
+        if df is None:
+            return [[] for _ in queries]
+        
+        # 批量获取查询向量
+        print(f"[batch_search_similar] Generating vectors for {len(queries)} queries...")
+        query_embeddings = self.batch_get_embeddings(queries, batch_size=min(10, len(queries)))
+        
+        if len(query_embeddings) != len(queries):
+            print("[batch_search_similar] Query vector generation failed")
+            return [[] for _ in queries]
+        
+        # 获取缓存的归一化向量矩阵
+        title_embeddings_norm = self._cached_embeddings['title']['normalized']
+        content_embeddings_norm = self._cached_embeddings['content']['normalized']
+        full_embeddings_norm = self._cached_embeddings['full']['normalized']
+        
+        all_results = []
+        
+        # 对每个查询进行相似度计算
+        for i, (query, query_embedding) in enumerate(zip(queries, query_embeddings)):
+            if not query_embedding:
+                all_results.append([])
+                continue
+            
+            query_vec = np.array(query_embedding)
+            query_vec_norm = query_vec / np.linalg.norm(query_vec)
+            
+            # 计算各部分的相似度
+            title_similarities = np.dot(title_embeddings_norm, query_vec_norm)
+            content_similarities = np.dot(content_embeddings_norm, query_vec_norm)
+            full_similarities = np.dot(full_embeddings_norm, query_vec_norm)
+            
+            # 加权综合相似度
+            combined_similarities = (
+                title_weight * title_similarities +
+                content_weight * content_similarities +
+                full_weight * full_similarities
+            )
+            
+            # 获取top-k
+            top_indices = np.argsort(combined_similarities)[::-1][:top_k]
+            
+            query_results = []
+            for idx in top_indices:
+                # 从DataFrame中获取条目信息
+                row = df.iloc[idx]
+                entry = {
+                    'id': row['id'],
+                    'title': row['title'],
+                    'source': row['source'],
+                    'content': row['content']
+                }
+                
+                # 添加各部分的相似度详情
+                similarity_details = {
+                    'combined': float(combined_similarities[idx]),
+                    'title': float(title_similarities[idx]),
+                    'content': float(content_similarities[idx]),
+                    'full': float(full_similarities[idx])
+                }
+                
+                query_results.append((entry, float(combined_similarities[idx]), similarity_details))
+            
+            all_results.append(query_results)
+            print(f"[batch_search_similar] Completed query {i+1}/{len(queries)}: '{query[:50]}...'")
+        
+        return all_results
+    
+    def search_similar(self, query: str, top_k: int = 5, 
+                      title_weight: float = 0.4, 
+                      content_weight: float = 0.3,
+                      full_weight: float = 0.3) -> List[Tuple[Dict, float, Dict]]:
+        """
+        搜索与查询最相似的条目，综合考虑标题和内容的相似度
+        使用批量搜索的优化版本
+        
+        Args:
+            query: 查询文本
+            top_k: 返回最相似的前k个结果
+            title_weight: 标题相似度的权重
+            content_weight: 内容相似度的权重
+            full_weight: 完整文本相似度的权重
+            
+        Returns:
+            相似条目和相似度分数的列表
+        """
+        # 使用批量搜索处理单个查询
+        results = self.batch_search_similar([query], top_k, title_weight, content_weight, full_weight)
+        return results[0] if results else []
+    
+    def search_with_entities_optimized(self, entities: List[str], top_k: int = 5) -> List[Tuple[Dict, float, Dict]]:
+        """
+        优化版本：使用实体列表搜索知识库，只加载一次向量数据库
+        
+        Args:
+            entities: 实体列表
+            top_k: 每个实体返回的结果数
+            
+        Returns:
+            合并和去重后的搜索结果
+        """
+        if not entities:
+            return []
+        
+        # 使用批量搜索
+        batch_results = self.batch_search_similar(
+            entities, 
+            top_k=top_k,
+            title_weight=0.3,    # 对于实体搜索，标题权重更高
+            content_weight=0.5,
+            full_weight=0.2
+        )
+        
+        # 合并结果并去重
+        seen_ids = set()
+        all_results = []
+        
+        for entity_results in batch_results:
+            for entry, score, details in entity_results:
+                entry_id = entry['id']
+                if entry_id not in seen_ids:
+                    seen_ids.add(entry_id)
+                    all_results.append((entry, score, details))
+        
+        # 按分数排序
+        sorted_results = sorted(all_results, key=lambda x: x[1], reverse=True)
+        return sorted_results
+    
+    def get_cache_info(self) -> Dict:
+        """
+        获取缓存状态信息
+        
+        Returns:
+            缓存状态字典
+        """
+        return {
+            'is_cached': self._cached_df is not None,
+            'cache_size': len(self._cached_df) if self._cached_df is not None else 0,
+            'cached_embeddings': list(self._cached_embeddings.keys()),
+            'last_load_time': self._last_load_time.isoformat() if self._last_load_time else None,
+            'data_paths': {
+                'knowledge_base': self.data_path,
+                'vector_database': self.vector_db_path,
+                'metadata': self.metadata_path
+            }
+        }