Upload 7 files

RAG_Learning_Assistant_with_Streaming.py

@@ -0,0 +1,515 @@
+import os
+import sys
+from typing import List, Dict, Tuple, Generator, Set
+from openai import OpenAI
+from vectorize_knowledge_base import KnowledgeBaseVectorizer
+import json
+from datetime import datetime
+import re
+
+
+class RAGLearningAssistant:
+    def __init__(self, api_key: str, model: str = "gpt-4.1-nano-2025-04-14", vector_db_path: str = ""):
+        """
+        初始化RAG学习助手
+        
+        Args:
+            api_key: OpenAI API密钥（必需）
+            model: 使用的模型名称
+            vector_db_path: 向量数据库路径
+        """
+        self.client = OpenAI(api_key=api_key)
+        self.vectorizer = KnowledgeBaseVectorizer(
+            api_key=api_key,
+            #data_path=os.path.join(vector_db_path, "knowledge_base.md")
+            data_path="knowledge_base.md"
+        )
+        
+        # 预加载向量数据库到缓存
+        print("预加载向量数据库...")
+        self.vectorizer.load_vector_database()
+        
+        # 模型配置
+        self.model = model
+        self.temperature = 0.2
+        self.max_tokens = 2000
+        
+        # 系统提示词
+        self.system_prompt = """You are a helpful learning assistant specializing in road engineering.
+Students can ask you questions with the following intents:
+1. Clarification: Requests to confirm understanding of a concept, parameter, or calculation.
+2. Instruction: Seeking step-by-step guidance for tasks or calculations.
+3. Explanatory: Asking for the reasoning behind a method, parameter choice, or principle.
+4. Information-seeking: Asking for where to find specific information in course materials.
+
+You have access to a knowledge base of course materials. When answering questions:
+1. Stick to the provided context from the knowledge base.
+2. At the end of your response, provide students the 'title' & 'from' fields of the chunks that were used to answer the question. So that they can refer to the original source.
+3. If the knowledge base doesn't contain relevant information, say so. Students can go to the teaching team for further assistance.
+"""
+        
+        # 查询重写的系统提示词 - 改进版本
+        self.rewrite_prompt = """You are a query rewriting assistant. Your task is to provide a summary of the conversation history and then rewrite user queries based on conversation history to make them more clear and complete.
+
+Please format your response as follows:
+SUMMARY: [Brief summary of the conversation context. Include key points, user intent, and any relevant details]
+REWRITTEN_QUERY: [The rewritten query that incorporates context]
+
+Rules:
+1. If there's relevant context from previous messages, incorporate it into the rewritten query
+2. Make implicit references explicit
+3. Maintain the original intent while adding clarity
+4. If the query is already clear and complete, keep it as is
+5. Always provide both SUMMARY and REWRITTEN_QUERY sections"""
+        
+        # 实体提取的系统提示词
+        self.entity_extraction_prompt = """You are an expert in road engineering. Extract key entities from the given query.
+Focus on:
+1. Technical terms and jargon specific to road engineering
+2. Formulas, equations, or mathematical concepts
+3. Parameters, specifications, or measurements
+4. Standards, methods, or procedures
+5. Materials, equipment, or structures
+
+Return the entities as a JSON array of strings. Only include the most important and specific entities."""
+        
+        # 对话历史
+        self.conversation_history = []
+        
+
+    def rewrite_query(self, query: str) -> Tuple[str, str]:
+        """
+        基于对话历史重写查询，并返回历史总结
+        
+        Args:
+            query: 原始查询
+            
+        Returns:
+            (历史总结, 重写后的查询)
+        """
+        # 构建消息
+        messages = [
+            {"role": "system", "content": self.rewrite_prompt}
+        ]
+        
+        # 添加对话历史上下文
+        if self.conversation_history:
+            context = "Previous conversation:\n"
+            for msg in self.conversation_history[-6:]:  # 最近3轮对话
+                role = "User" if msg["role"] == "user" else "Assistant"
+                # 截取前200个字符避免过长
+                content = msg["content"][:200] + "..." if len(msg["content"]) > 200 else msg["content"]
+                context += f"{role}: {content}\n"
+            
+            messages.append({
+                "role": "user", 
+                "content": f"{context}\n\nCurrent query: {query}\n\nPlease provide summary and rewritten query following the specified format:"
+            })
+        else:
+            # 没有历史时也要按格式返回
+            messages.append({
+                "role": "user",
+                "content": f"Current query: {query}\n\nPlease provide summary and rewritten query following the specified format:"
+            })
+        
+        try:
+            response = self.client.chat.completions.create(
+                model=self.model,
+                messages=messages,
+                temperature=0.3,  # 低温度确保一致性
+                max_tokens=2000
+            )
+            
+            content = response.choices[0].message.content.strip()
+            
+            # 改进的解析逻辑
+            summary = ""
+            rewritten = query  # 默认值
+            
+            # 使用正则表达式提取SUMMARY和REWRITTEN_QUERY
+            summary_match = re.search(r'SUMMARY:\s*(.*?)(?=REWRITTEN_QUERY:|$)', content, re.DOTALL | re.IGNORECASE)
+            rewritten_match = re.search(r'REWRITTEN_QUERY:\s*(.*?)$', content, re.DOTALL | re.IGNORECASE)
+
+            
+            if summary_match:
+                summary = summary_match.group(1).strip()
+            
+            if rewritten_match:
+                rewritten = rewritten_match.group(1).strip()
+            
+            # 备用解析方法 - 如果正则表达式失败
+            if not summary and not rewritten_match:
+                lines = content.split('\n')
+                current_section = None
+                summary_lines = []
+                rewritten_lines = []
+                
+                for line in lines:
+                    line = line.strip()
+                    if line.upper().startswith("SUMMARY"):
+                        current_section = "summary"
+                        # 提取SUMMARY:后面的内容
+                        summary_part = line[line.upper().find("SUMMARY"):].replace("SUMMARY:", "").strip()
+                        if summary_part:
+                            summary_lines.append(summary_part)
+                    elif line.upper().startswith("REWRITTEN_QUERY") or line.upper().startswith("REWRITTEN QUERY"):
+                        current_section = "rewritten"
+                        # 提取REWRITTEN_QUERY:后面的内容
+                        rewritten_part = re.sub(r'^REWRITTEN[_\s]*QUERY[:\s]*', '', line, flags=re.IGNORECASE).strip()
+                        if rewritten_part:
+                            rewritten_lines.append(rewritten_part)
+                    elif current_section == "summary" and line:
+                        summary_lines.append(line)
+                    elif current_section == "rewritten" and line:
+                        rewritten_lines.append(line)
+                
+                if summary_lines:
+                    summary = " ".join(summary_lines)
+                if rewritten_lines:
+                    rewritten = " ".join(rewritten_lines)
+            
+            # 如果仍然没有获得有效结果，使用更简单的方法
+            if not summary and self.conversation_history:
+                summary = "继续之前的讨论"
+            
+            if not rewritten or rewritten == query:
+                rewritten = query
+            
+            print(f"Raw query: {query}")
+            print(f"Chat history summary: {summary}")
+            print(f"Rewrite query: {rewritten}")
+            return summary, rewritten
+            
+        except Exception as e:
+            print(f"查询重写失败: {e}")
+            # 生成简单的历史总结作为备用
+            simple_summary = ""
+            if self.conversation_history:
+                simple_summary = "基于之前的对话内容"
+            return simple_summary, query  # 失败时返回简单总结和原始查询
+    
+    def extract_entities(self, original_query: str, summary: str, rewritten_query: str) -> List[str]:
+        """
+        从原始查询、历史总结和重写查询中提取关键实体（专业术语、公式、参数等）
+        
+        Args:
+            original_query: 原始用户查询
+            summary: 历史总结
+            rewritten_query: 重写后的查询文本
+            
+        Returns:
+            提取的实体列表
+        """
+        # 合并所有文本作为实体提取的输入
+        text_parts = []
+        
+        # 添加原始查询
+        if original_query:
+            text_parts.append(f"Original query: {original_query}")
+        
+        # 添加历史总结
+        if summary:
+            text_parts.append(f"Context summary: {summary}")
+        
+        # 添加重写查询
+        if rewritten_query and rewritten_query != original_query:
+            text_parts.append(f"Rewritten query: {rewritten_query}")
+        
+        combined_text = " | ".join(text_parts)
+        
+        messages = [
+            {"role": "system", "content": self.entity_extraction_prompt},
+            {"role": "user", "content": f"Text to extract entities from: {combined_text}\n\nExtract entities as JSON array:"}
+        ]
+        
+        try:
+            response = self.client.chat.completions.create(
+                model=self.model,
+                messages=messages,
+                temperature=0.3,
+                max_tokens=200
+            )
+            
+            content = response.choices[0].message.content.strip()
+            
+            # 尝试解析JSON
+            try:
+                # 提取JSON数组（处理可能的markdown格式）
+                json_match = re.search(r'\[.*?\]', content, re.DOTALL)
+                if json_match:
+                    entities = json.loads(json_match.group())
+                else:
+                    entities = json.loads(content)
+                
+                print(f"Extracted entities: {entities}")
+                return entities
+                
+            except json.JSONDecodeError:
+                # 如果JSON解析失败，尝试简单的文本处理
+                print(f"JSON解析失败，使用备用方法")
+                # 查找引号中的内容
+                entities = re.findall(r'"([^"]+)"', content)
+                return entities if entities else self.simple_entity_extraction(combined_text)
+                
+        except Exception as e:
+            print(f"实体提取失败: {e}")
+            # 失败时使用简单的关键词提取
+            return self.simple_entity_extraction(combined_text)
+    
+    def simple_entity_extraction(self, query: str) -> List[str]:
+        """
+        简单的实体提取备用方法
+        
+        Args:
+            query: 查询文本
+            
+        Returns:
+            提取的关键词列表
+        """
+        # 移除常见停用词
+        stop_words = {'the', 'a', 'an', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for', 
+                     'of', 'with', 'by', 'from', 'what', 'how', 'when', 'where', 'why',
+                     'is', 'are', 'was', 'were', 'been', 'be', 'have', 'has', 'had',
+                     'original', 'query', 'context', 'summary', 'rewritten'}  # 添加新的停用词
+        
+        # 分词并过滤
+        words = query.lower().split()
+        entities = [w for w in words if w not in stop_words and len(w) > 2]
+        
+        # 查找可能的专业术语（包含大写字母或数字）
+        special_terms = re.findall(r'\b[A-Z][a-zA-Z]*\b|\b\w*\d+\w*\b', query)
+        entities.extend(special_terms)
+        
+        # 去重并返回
+        return list(set(entities))[:5]  # 最多返回5个实体
+    
+    def enhanced_search(self, query: str, top_k: int = 3) -> Tuple[str, str, List[str], List[Tuple[Dict, float, Dict]]]:
+        """
+        增强搜索：重写查询 -> 提取实体 -> 基于实体搜索（优化版本）
+        
+        Args:
+            query: 原始查询
+            top_k: 返回的结果数
+            
+        Returns:
+            (历史总结, 重写后的查询, 提取的实体, 搜索结果)
+        """
+        # 1. 重写查询并获取历史总结
+        summary, rewritten_query = self.rewrite_query(query)
+        
+        # 2. 基于原始查询、总结和重写查询提取实体
+        entities = self.extract_entities(query, summary, rewritten_query)
+        
+        # 3. 基于实体搜索（使用优化的批量搜索）
+        if entities:
+            # 使用优化的批量搜索方法
+            search_results = self.vectorizer.search_with_entities_optimized(entities, top_k)
+        else:
+            # 如果没有提取到实体，使用重写后的查询进行搜索
+            print("未提取到实体，使用完整查询搜索")
+            search_results = self.vectorizer.search_similar(
+                rewritten_query, 
+                top_k=top_k,
+                title_weight=0.4,
+                content_weight=0.3,
+                full_weight=0.3
+            )
+        
+        return summary, rewritten_query, entities, search_results         
+
+    def format_context(self, search_results: List[Tuple[Dict, float, Dict]]) -> str:
+        """
+        格式化搜索结果作为上下文
+        
+        Args:
+            search_results: 搜索结果列表
+            
+        Returns:
+            格式化的上下文字符串
+        """
+        if not search_results:
+            return ""
+        
+        context_parts = []
+        for i, result in enumerate(search_results, 1):
+            entry, combined_score, details = result
+            # 只显示 title, source, content，不显示 id
+            context_parts.append(
+                #f"[Source {i}]\n"
+                f"Title: {entry['title']}\n"
+                f"From: {entry['source']}\n"
+                f"Content: {entry['content']}\n"
+            )
+        
+        return "RELEVANT KNOWLEDGE BASE CONTENT:\n" + "\n---\n".join(context_parts)
+    
+    def build_messages(self, query: str, context: str) -> List[Dict[str, str]]:
+        """
+        构建消息列表，包含系统提示、上下文和用户查询
+        
+        Args:
+            query: 用户查询
+            context: 知识库上下文
+            
+        Returns:
+            消息列表
+        """
+        messages = [
+            {"role": "system", "content": self.system_prompt}
+        ]
+        
+        # 添加对话历史（保留最近5轮对话）
+        for msg in self.conversation_history[-10:]:  # 最多保留5轮对话（10条消息）
+            messages.append(msg)
+        
+        # 构建用户消息，包含上下文
+        user_message = query
+        if context:
+            user_message = f"{context}\n\nUSER QUESTION: {query}"
+        
+        messages.append({"role": "user", "content": user_message})
+        
+        return messages
+    
+    def generate_response_stream(self, query: str) -> Generator[str, None, None]:
+        """
+        生成流式响应
+        
+        Args:
+            query: 用户查询
+            
+        Yields:
+            响应文本片段
+        """
+        # 1. 增强搜索（现在使用优化版本）
+        print("正在处理查询...")
+        summary, rewritten_query, entities, search_results = self.enhanced_search(query)
+        
+        # 2. 格式化上下文
+        context = self.format_context(search_results)
+        
+        # 3. 构建消息（使用原始查询，但包含基于实体搜索的上下文）
+        messages = self.build_messages(query, context)
+        
+        # 4. 调用OpenAI API进行流式生成
+        try:
+            stream = self.client.chat.completions.create(
+                model=self.model,
+                messages=messages,
+                temperature=self.temperature,
+                max_tokens=self.max_tokens,
+                stream=True
+            )
+            
+            # 收集完整响应用于保存到历史
+            full_response = ""
+            
+            # 首先返回搜索信息
+            search_info = f"\n**Query Analysis:**\n"
+            search_info += f"- Query: {query}\n"
+            if summary:
+                search_info += f"- Summary of history: {summary}\n"
+            if rewritten_query != query:
+                search_info += f"- Rewrite query: {rewritten_query}\n"
+            search_info += f"- Key entities: {', '.join(entities) if entities else 'No specific entities extracted'}\n"
+            
+            if search_results:
+                search_info += f"\n**Relevant Sources:**\n"
+                for result in search_results:
+                    entry, combined_score, details = result
+                    # 给用户显示时包含 ID 和相关度分数
+                    search_info += f"- [{entry['id']}] {entry['title']} (Relevance: {combined_score:.3f})\n"
+                search_info += "\n**Response:**\n"
+            else:
+                search_info += "\n**Response:** (No relevant knowledge base content found, answering based on general knowledge)\n"
+            
+            # 添加缓存信息（调试用）
+            cache_info = self.vectorizer.get_cache_info()
+            if cache_info['is_cached']:
+                search_info += f"The vector db has been cached, containing {cache_info['cache_size']} entries\n\n"
+
+            yield search_info
+            
+            # 流式返回生成的内容
+            for chunk in stream:
+                if chunk.choices[0].delta.content is not None:
+                    content = chunk.choices[0].delta.content
+                    full_response += content
+                    yield content
+            
+            # 保存到对话历史
+            self.conversation_history.append({"role": "user", "content": query})
+            self.conversation_history.append({"role": "assistant", "content": full_response})
+            
+        except Exception as e:
+            yield f"\n\n错误：生成响应时出现问题 - {str(e)}"
+    
+    def generate_response(self, query: str) -> str:
+        """
+        生成完整响应（非流式）
+        
+        Args:
+            query: 用户查询
+            
+        Returns:
+            完整的响应文本
+        """
+        response_parts = []
+        for part in self.generate_response_stream(query):
+            response_parts.append(part)
+        return "".join(response_parts)
+    
+    def clear_history(self):
+        """清除对话历史"""
+        self.conversation_history = []
+        print("对话历史已清除")
+    
+    def clear_vector_cache(self):
+        """清除向量数据库缓存"""
+        self.vectorizer.clear_cache()
+        print("向量数据库缓存已清除")
+    
+    def reload_vector_database(self):
+        """重新加载向量数据库"""
+        print("重新加载向量数据库...")
+        self.vectorizer.load_vector_database(force_reload=True)
+        print("向量数据库重新加载完成")
+    
+    def get_system_status(self) -> Dict:
+        """
+        获取系统状态信息
+        
+        Returns:
+            系统状态字典
+        """
+        cache_info = self.vectorizer.get_cache_info()
+        return {
+            'model': self.model,
+            'conversation_turns': len(self.conversation_history) // 2,
+            'vector_cache': cache_info,
+            'last_update': datetime.now().isoformat()
+        }
+    
+    def save_conversation(self, filepath: str = None):
+        """
+        保存对话历史
+        
+        Args:
+            filepath: 保存路径
+        """
+        if filepath is None:
+            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+            filepath = f"conversation_{timestamp}.json"
+        
+        conversation_data = {
+            "timestamp": datetime.now().isoformat(),
+            "model": self.model,
+            "system_status": self.get_system_status(),
+            "history": self.conversation_history
+        }
+        
+        with open(filepath, 'w', encoding='utf-8') as f:
+            json.dump(conversation_data, f, ensure_ascii=False, indent=2)
+        
+        print(f"对话已保存到: {filepath}")
+

knowledge_base.md

@@ -0,0 +1,340 @@
+# 00-01-00 What this unit is about
+**source** unit infomation on Moodle
+**content** This unit introduces the fundamentals of road engineering theory and practice. It examines a number of issues related to the planning, geometric and structural design of roads, with due consideration to relevant economic, social and environmental aspects.
+
+# 00-02-00 Expected learning outcomes
+**source** unit infomation on Moodle
+**content** On successful completion of this unit, you should be able to:
+- 01 Design road geometry with due consideration to horizontal and vertical alignment, cross-section profile, earthworks and cost implications.
+- 02 Assess different components of a road project, within the context of a safe systems approach to design, based on economic, social and environmental considerations. 
+- 03 Design flexible pavements giving consideration to economic, social and environmental factors.
+
+# 00-03-00 Checklist of important items you should do
+**source** learning on Moodle - getting started
+**content** All listed items are on Moodle.
+- The recommended readings, located in the "Additional information and resources" page (under "Learning" tag).
+- The forums, located in the "Forums" page.
+- Important assessments, located in the "Assessments" page.
+- Link for online workshops, located in the "Getting Started" page.
+- The content for each week, located in each week's pages.
+- Sign up with a group for your project assignment. See the "Week 13 - Project Assignment" page.
+
+# 00-04-00 Timetable and location for class
+**source** learning on Moodle - getting started
+**content** - Workshop: Monday 7pm-9pm, Learning and Teaching Building, G81 (main) and G56 (overflow)
+- Practice Session 1, G29/33 B72, Wednesday 10am-12pm
+- Practice Session 2, G29/33 B72, Wednesday 1pm-3pm
+Please attend the practice session allocated in your timetable. Attendance will be marked.
+
+# 00-05-00 Online link for class
+**source** learning on Moodle - getting started
+**content** - Workshop: Please click this URL to start or join: https://monash.zoom.us/j/82685794107?pwd=hHFu5jrl5YBAKQS60FU5bsMdBONox5.1. Or, go to https://monash.zoom.us/join and enter meeting ID: 826 8579 4107 and passcode: 209261
+- Practice Session: In-person only.
+
+# 00-06-00 Overview for assessment
+**source** learning on Moodle - getting started
+**content** This unit has two general assessment tasks: continuous assessment and final assessment. 
+For continuous assessment, there are three components, 60%:
+- Assignment: Road design project, 45%
+  - Geometric design (stage 1, stage 2), 10%+25%
+  - Pavement design (stage 3), 10%
+- Mid-term quiz, 10%
+- Participation, 5%
+  - Attend practice classes, 2% (TA will mark your attendance)
+  - Participate in activities, 2% (do pre-class quiz and contribute to your group works)
+  - Active engagement, 1% (e.g. effectively help resolve questions/problems in discussion forums)
+
+For final assessment, there is a final exam, 40%
+
+# 00-07-00 Unit schedule and class info
+**source** learning on Moodle - getting started
+**content** The following table shows the schedule of this unit. Note that the schedule of guest lectures are subject to change.
+| Week | Topic | Notes | Milestones |
+|---|---|---|---|
+| Week 1 | Road System | A guest lecture will be given on Safe System Approach by Mr. Ross Thompson from TraffixGroup | Team be formed by Week 1 |
+| Week 2 | Road planning |  |  |
+| Week 3 | Road Design Considerations | A guest lecture will be given on "North East Link Project" by colleagues from SMEC. <br>A prac activity will be led by colleagues from SMEC on "Multi-criteria analysis" | Registration close for “Future Road” competition by Traffic Accident Commission, Victoria |
+| Week 4 | Road Geometry Design 1 |  | Road Design Project<br>Stage 1 submission DUE |
+| Week 5 | Road Geometry Design 2 | A guest lecture will be given on “geometric design” by Mr. Malcom Mak from Australia Road Research Board (ARRB) |  |
+| Week 6 | Road Geometry Design 3 |  |  |
+| Week 7 | Road Geometry Design 4 |  |  |
+| Week 8 | Road Geometry Design 5 | A guest lecture and a prac activity will be led by colleagues from SMEC on “detailed geometric design” |  |
+| Week 9 | Road Pavement Design 1 | A guest lecture will be given on “pavement materials” by Dr.Tareq and Dr. Shan from Department Transport Planning, Victoria | Road Design Project<br>Stage 2 submission DUE |
+|  | Semester Break |  |  |
+| Week 10 | Road Pavement Design 2 | A guest lecture will be given on weigh-in-motion technology and its application in pavement design   | Mid-term Quiz during workshop session |
+| Week 11 | Road Pavement Design 3 | A guest lecture will be given on the Austroads design methods by Prof. Jayantha Kodikara from SPARC Hub Monash University |  |
+| Week 12 | Unit Summary |  | Road Design Project<br>Stage 3 submission DUE |
+
+# 00-08-00 Unit contacts
+**source** learning on Moodle - getting started
+**content** - For technical matters such as knowledge points and practice questions, please use the appropriate forum.
+- For administrative matters such as grouping and Moodle operations, please reach out to Akbar (akbar.rizqiansyah@monash.edu)
+- For “Future Road” competition and mentoring, please contact Nan (nan.zheng@monash.edu)
+- For other inquiries, please contact either Akbar and/or Nan.
+
+# 01-01-00 Learning objectives of week 1
+**source** learning on Moodle - week 1 road system
+**content** - Outline some historical developments in road design
+- Understand the role of road design and classification of roads (road and street hierarchies)
+- Discuss road design considerations
+- Understand how the Safe System approach influence road design
+
+# 01-02-00 Modern road design
+**source** W 1.1 Introduction to Road Engineering, page 6
+**content** The process of designing and managing the safe and efficient movement of people and goods on the road network. The key points are to design and manage the road network. It is a tradeoff between the needs to develop the network that is safe but also efficient. These can often be competing demands, particularly when we discuss travel speed. We also need to consider the movement of not only poeple but also ranges of goods on the road network. In particular, we need to consider the range of vehicles, from bicycles, motorcycles, private cars, through the large vehicles.
+
+# 01-03-00 Road design process
+**source** W 1.1 Introduction to Road Engineering, page 7
+**content** Road design is only one component of the transportation engineering system. From diagram [image/01-03-00 Road design process.jpg], the focuses of this unit, mostly, are Geometric and Pavements Design. We will touch a little of Drainage and Structures. We will also briefly consider some of the other elements in this framework.
+
+# 01-04-00 Benefits of a good road network
+**source** W 1.1 Introduction to Road Engineering, page 8
+**content** Why do we need to plan our road network？There are many benefits associated with the developing road network that operates efficiently and allow people and goods to move freely. The 
+- The road network can provide easy access to commercial, retail and recreational activities.
+- This can stimulate economic growth.
+- When designed well, the road network can improve safety, access and efficiency.
+- Thus to improve road freight productivity.
+- And improve public transport operating conditions.
+
+# 01-05-00 Road design considerations
+**source** W 1.1 Introduction to Road Engineering, page 9
+**content** In order to develop a good road network, we need to make sure we take into account key road design considerations.
+- three key things of road design: Safety, efficiency and accessibility issues for road users. We need to make tradeoffs between these three sometime competing needs. For example, it may be more efficient to let vehicles travel at higher speed. This can have safety issues because crashes that occur at higher speed are likely to cause more severe injuries. At the same time, it may not be practical to provide adjacent land use if travel speed is set too high.
+- Performance and types of vehicles (traffic). Design not just for cars, but also for large truck and small or more vulnerable road users such as bicycles.
+- Behaviour of road users. To understand human factors and how they need to be consider. 
+- Geometry of the road and limitations (vertical and horizontal profile of the road). 
+- Impact of road on environment
+- Economics – road cost/efficiency, economic benefits associated with construction and management
+
+# 01-06-00 Road context in Road traffic design system
+**source** W 1.2 The Road Traffic Design System, page 5
+**content** Road is not isolated from the rest of the world. And often, the road design decisions are influenced by range of other factors including political, institutional, and planning considerations. These factors need to be considered in almost every road design projects.
+ When we look at road design, some of the key considerations include:
+ - How much is spent
+ - What gets done, and where,
+ - Who designs and builds the road project.
+
+ These considerations are determined by:
+ - Political process (safety, environmental and social impacts & election)
+ - Institutional factors (set up by politicians to build and design roads)
+ - Planning process (not designed to give one result that must be followed)
+
+ These factors may not be very technical. But in practice, senior engineers might spend much of their time working on this.
+
+# 01-07-00 Classification of roads
+**source** W 1.2 The Road Traffic Design System, page 6
+**content** Road networks fulfill four basic functions: 
+- Caters for moving passenger and freight traffic,
+- Caters for parked vehicles,
+- Caters for pedestrians and non-vehicular traffic and
+- Allows development and access to adjacent land use.
+
+This functions are inconsistent and competing. For example, a road suit for carrying large volumne of traffic may not be suited for pedestrians or providing convenient access. Thus, it is necessary to have various types of road, each serving predominent functions.
+
+# 01-08-00 Road functions
+**source** W 1.2 The Road Traffic Design System, page 7
+**content** Roads have two main functions: movement and access.
+Movement involves the movement of people or goods.
+Access sometimes refers to activity that looks for other functions that the road might facilitate such as providing access to shops, houses, parking, or other facilites in the road reserve.
+
+# 01-09-00 Road hierarchies and functional classification
+**source** W 1.2 The Road Traffic Design System, page 8-9
+**content** Based on the two basic funcions of road, Movement & Access, the diagram [image/01-09-00 Road hierarchies and functional classification.jpg] shows the classification based on movement v.s. access function. Here is the more detailed description of each range of different road types.
+- Freeway:Those roads with full access control and grade separated instructions, whose primary function is to service large traffic movements.
+- Primary Arterials: Those arterial roads whose function is to form the principle avenue for communication for metropolitan traffic movementsnot catered by for freeways.
+- Secondary Arterials: Those roads which supplement the primary arterial roads in providing for through traffic movement, to an individually determined limit that is sensitive to both roadway characteristics and abutting land uses.
+- Collector Roads: Those non-arterial roads which distribute traffic between the arterial roads and the local street system which provide local connection between arterial roads and which provide accessto abutting property.
+- Local Access Streets: Those streets not being arterials or collectors whose main function is to provide accessto abutting property.
+
+# 01-10-00 Other road classifications
+**source** W 1.2 The Road Traffic Design System, page 11
+**content** Two other consideration for classifying roads:
+- Administration (who is responsible for the road)
+  - State Goverment (major roads), e.g., major roads/freeways in Victoria state
+  - Local Goverment (local roads) fund and maintained by local councils
+- Funding
+  - Federal funds portion of roads of “major economic benefit” (major projects, e.g., a new freeway)
+  - State covers the rest of funding of major roads
+  - Local government / developers fund local roads
+
+# 01-11-00 brief introduction of design components
+**source** W 1.2 The Road Traffic Design System, page 13
+**content** There are three components in road design: humans, vehicles, and road itself. The characteristics impact the design. There are two critical issues: Range of characteristics & The operating (traffic) environment.
+Looking at range of characteristics, road design involves the standard input values, such as: driver eye height, vehicle performance, speed, and reaction time. These are given in design manuals. However, in reality, none of the characteristics is a single fixed value. And all of them distributed across ranges of values. Clearly, we should not design for the average condition, which leaves half of population fail. However, on the other hand, we typically do not design for 100% of the population, which will cause overdesign. For instance, for driver eye height, it is very uncommon that someone drive in a low sport car travelling at a very high speed and have incrediblly short reaction time and brake distance. So, it is not necessary to use all extreme values. Therefore, use 85th percentile values in general design. 
+The other system input is to realise the road vehicle driver system must be capable of operating in a range of environmental conditions. The enviroment in which traffic movemnet occurs has an important affect of both design and operation of the traffic system. It includes a range of physical, traffic, and social environment. The following table shows the factors: 
+| Operating environment | Factors |
+|---|---|
+| Physical environment | day and night (light conditions), the terrain, weather conditions, abutting development (construction cost and standard to use) |
+| Traffic environment | traffic flow (daily and hourly), directional split, traffic composition, seasonality of traffic demands (e.g., recreational locations) |
+| Social environment | driver's understanding of regulations (e.g., the system should run under consistent rules), community attitudes to driver responsibility, degree of enforcement by police |
+
+# 01-12-00 brief overview of design process
+**source** W 1.2 The Road Traffic Design System, page 17
+**content** The objectives of design process are: Safety, Efficiency, and Amenity. We can call it SEA.
+Safety: involves providing a safe facility for both users and society as a whole. It should always be checked
+Efficiency: related to the cost of providing, maintaining, and using the road. e.g., If we were to design a bypass of a rural town, the efficiency objetive must ensure the cost building and maintaining the road, cost saving, travel time, running cost, and social & environmental cost of the project as such the project is efficient in use resources that can be used in other places.
+Amenity: for both road users and non-road users.
+
+# 01-13-00 brief overview of design stages
+**source** W 1.2 The Road Traffic Design System, page 18
+**content** Here is a simple structure of a design project.
+- Conceptual design (Phase 0)
+  - Low details
+  - Large number of alternatives
+  - e.g., if we consider a highway bypass of a rural town. Alteratives with horizontal alignment and general design parameters such as design speed will be set. And each option will be evaluated at a general level
+- Preliminary design (Phase 1)
+  - Smaller number of options (3-6)
+  - More detailed level
+  - e.g., for the rural bypass, it may include horizontal and vertical profiles, road cross section, and others... Each option will be evaluated to assess economical, social, and environmental viability and recommendation may be for small or one alternatives to go to the next stage, the detailed design.
+- Detailed/Final design (Phase 2)
+  - Suitable level of design details for construction purposes
+  - e.g., for the bypass example, the geometry, cross section, and all, will be finalised. All elements will be ready for construction.
+
+# 01-14-00 brief introduction of design elements
+**source** W 1.2 The Road Traffic Design System, page 20
+**content** design elements are the components of the physical road that require design. Here is the list: horizontal & vertical geometry, road reserve, traffic lanes, shoulders, roadside, road furniture, intersections, pavement, drainage.
+
+# 01-15-00 National road safety strategy
+**source** W 1.3 The Safe System Approach to Road Safety, page 8
+**content** National Road Safety Strategy 2011/2020
+- No person should be killed or seriously injured on Australia’s roads
+- Reduce the annual number of road crash fatalities by at least 30 per cent
+- Reduce the annual number of serious road crash injuries by at least 30 per cent.
+- The strategy embraces the Safe System approach to road safety improvement
+
+# 01-16-00 brief introduction of Safe system approach
+**source** W 1.3 The Safe System Approach to Road Safety, page 9
+**content** - Derived from Sweden’s ‘Vision Zero’ and the Dutch ‘Sustainable Safety’ approaches
+- Crashes will continue to occur in spite of best efforts to prevent them (human error)
+- Aims to eliminate fatal crashes and reduce serious injury crashes
+- Safety interventions based on the best available scientific evidence of what is effective
+- A shared responsibility:
+  - Road users responsible for complying with system rules
+  - System designers and operators responsible for the road-transport system
+
+# 01-17-00 Safe system principles
+**source** W 1.3 The Safe System Approach to Road Safety, page 10
+**content** When crashes do occur, crash energies need to reduce to levels that minimize the probability of death and serious injury
+- Provision of a safe, crashworthy system that is forgiving of human error and accommodates vulnerability to serious injury
+- Recognition that all aspects of the transport system work together
+- Manage the road infrastructure, speeds, vehicles and road users and the interactions between these components to achieve safe mobility
+
+# 01-18-00 key ideals of the Safe System approach
+**source** W 1.3 The Safe System Approach to Road Safety, page 11
+**content** The Safe System approach aims to create a forgiving road system based on four key ideals:
+- People make mistakes – and some crashes are inevitable.
+- People are vulnerable –Our bodies have a limited ability to withstand crash forces without being seriously injured or killed.
+- We need to share responsibility –System designers and people who use the roads must all share responsibility for creating a road system where crash forces do not result in death or serious injury.
+- We need to strengthen all parts of the system – We need to improve the safety of all parts of the system – roads and roadsides, speeds, vehicles, and road use so that if one part fails, other parts will still protect the people involved.
+
+# 01-19-00 Components of the Safe System
+**source** W 1.3 The Safe System Approach to Road Safety, page 16
+**content** As the diagram [image/01-19-00 Components of the Safe System.jpg]，there are four components of the "SAFE SYSTEM":
+- Safe roads – that are predictable and forgiving of mistakes. They are self-explaining in that their design encourages safe travel speeds (road factor).
+- Safe speeds – travel speeds suit the function and level of safety of the road. People understand and comply with the speed limits and drive to the conditions (speed factor).
+- Safe vehicles – that prevent crashes and protect road users, including pedestrians and cyclists, in the event of a crash (vehicle factor).
+- Safe road use – road users that are skilled and competent, alert and unimpaired. They comply with road rules, take steps to improve safety, and demand and expect safety improvements (driver or rider factor).
+
+# 01-20-00 Design safe system must consider vulnerable road users
+**source** W 1.3 The Safe System Approach to Road Safety, page 17
+**content** A transport system which aspires to achieve Safe System objectives cannot ignore: The vulnerability of some road user groups (a major determinant of injury outcome) 
+Any safe system needs to be designed to protect our most vulnerable: Pedestrians, Cyclists, Motorcyclists, Children, The elderly.
+
+# 01-21-00 Speed factor
+**source** W 1.3 The Safe System Approach to Road Safety, page 19 & 20
+**content** In a 60km/h zone, travelling at:
+- 65 km/h, you are twiceas likely to have a serious crashthan if you drive at 60 km/h
+- 70 km/h, fourtimes
+- 75 km/h, tentimes
+- 80 km/h, 32 times
+Vehicle speed is the crucial safety issue for vulnerable road users: Speed is an even more powerful determinant of injury severity in the event of a collision.
+
+# 01-22-00 Speed management
+**source** W 1.3 The Safe System Approach to Road Safety, page 21
+**content** Historically, the emphasis has been on mobility
+- that is, we strive for the highest mobility until safety is compromised “too much” (may impact the level of road)
+- current speed limits and road design tend to favour high mobility often at the expense of safety
+- is this the balance we want?
+In general, Australian urban speed limits are 10 km/h higher than in other Western countries
+- However, speeds should be at levels that do not exceed human tolerance to violent forces
+
+# 01-23-00 Speed choice factors
+**source** W 1.3 The Safe System Approach to Road Safety, page 22
+**content** Factors that impact the choice of driver speed include: Driver factors, road & traffic factors, speed zone, crash & injury risk, police enforcement, education & publicity & promotion, and vehicle factors.
+
+# 01-24-00 Speed limits
+**source** W 1.3 The Safe System Approach to Road Safety, page 23
+**content** - Speed limits, arguably, have the single largest influence on driver/rider choice of speed.
+- Studies consistently show that crash incidence or crash severity decline whenever speed limits have been reduced
+  - After 50 km/h introduced in local streets in Victoria, pedestrian casualty crashes fell by 20% and pedestrian serious casualty crashes by 45%
+  - Freeway speed limit reduced from 110km/h to 100km/h (Victoria, 1989) and resulted in a 19% drop in casualty crash rate on freeways
+- Negligible time savings by driving faster
+
+# 01-25-00 Driver or Rider factors
+**source** W 1.3 The Safe System Approach to Road Safety, page 24
+**content** Drivers and riders:
+- Are human and,therefore, make errors
+- Consistently over-estimate what is a “safe” speed(poor risk perception)
+- Are influenced by perceptions about risks of being “caught”
+- Are concerned more about crash than injury risk
+- Drive faster due to others around them -thus, a contagious effect
+- Young drivers (less skilled) with young passengers travel faster than when alone
+- Knowledge and attitudesabout speed only partly determine behaviour–other motives are often at play
+
+# 01-26-00 The licensing system: entry to exit
+**source** W 1.3 The Safe System Approach to Road Safety, page 25
+**content** Licensing systems are a key foundation in Australia road transport system in order to manage the driver/rider factors
+Licensing systems can:
+- Establish/assess entry level ability/skill
+- Monitor maintenance of those skills
+- Decide on continued fitness to drive
+- Promote overall safe mobility of drivers
+- Regulate who can drive
+- Manage illegal driving (combined with enforcement, demerit and sanctions)
+
+# 01-27-00 Vehicle factors
+**source** W 1.3 The Safe System Approach to Road Safety, page 26
+**content** Even the best vehicles cannot protect their occupants in many common crash types.
+“Safe” impact speeds generally fall below 30 to 50 km/h for:
+- side-impacts at intersections
+- collisions with trees and poles
+- pedestrians and other vulnerable road users
+- crashes between incompatible vehicle types
+Low short-term potential for raising the level of “safe” impact speeds through improvements in crashworthiness.
+It is probable that drivers over-estimate the protective capabilities of their vehicles.
+
+# 01-28-00 Threshold impact speeds (human tolerance to violence)
+**source** W 1.3 The Safe System Approach to Road Safety, page 27
+**content** Vehicles cannot protect us in high-speed crashes. The following table provides the threshold impact speeds (human tolerance to violence) under different conditions:
+| Scenarios | Speed range |
+|---|---|
+| car/pedestrian | 20-30 km/h |
+| car/motorcyclist | 20-30 km/h |
+| car/tree or pole | 30-40 km/h |
+| car/car (side-impact) | 50 km/h |
+| car/car (head-on) | 70 km/h |
+
+Estimated that if all drivers were in safer vehicles, we could achieve a 40% reduction in fatal crashes.
+
+# 01-29-00 Road factors
+**source** W 1.3 The Safe System Approach to Road Safety, page 30 & 31
+**content** Road features play a vital role in a driver/rider’s speed choice:
+- Roadsides and the presence of trees, poles, etc.
+- Intersections, and their design and operational features
+- Alignment of roads and their design speeds
+- Road surface condition
+- "Road appearance" conditions: road width, number of lanes, roadside activity, divided/undivided, alignment (vertical and horizontal)
+
+We can also involve the Traffic conditions (e.g., presence of other road users, general congestion or lack of congestion).
+
+# 01-30-00 Roads and Roadsides
+**source** W 1.3 The Safe System Approach to Road Safety, page 32
+**content** Ways addressing high priority crashes types (high priority crash types typically involve incidents with a high risk of serious injury or death. Examples include fatal pedestrian crashes, single-vehicle frontal crashes with objects, and front-to-front or front-to-side collisions):
+- Intersection design:
+  - Adequate sight distance and speed reduction
+  - Well designed roundabouts: low entry angle, good sight zone
+- Separation and safer roadsides:
+  - Separate motorcycle/bicycle lanes, footpaths
+  - Use of good barrier systems
+  - Major grade separation of railway crossings on arterials
+

query_log.csv

@@ -0,0 +1,122 @@
+id,timestamp,search_info,query_and_response,thumb_feedback
+1,2025-06-16 06:03:20,"
+**Query Analysis:**
+- Query: what is safe system
+- Summary of history: The user is asking for the definition or explanation of the term ""safe system,"" but has not provided additional context or details about the specific field or application they are referring to.
+- Rewrite query: What is a ""safe system,"" and can you explain its meaning or significance in a general or specific context?
+- Key entities: safe system
+
+**Relevant Sources:**
+- [01-17-00] Safe system principles (Relevance: 0.652)
+- [01-18-00] key ideals of the Safe System approach (Relevance: 0.595)
+- [01-19-00] Components of the Safe System (Relevance: 0.575)
+
+**Response:**
+The vector db has been cached, containing 39 entries
+
+","Query: what is safe system
+Response: The Safe System is an approach to road safety that aims to create a forgiving and resilient road environment, recognizing that crashes can happen but should result in minimal harm. It involves managing all aspects of the transport system—roads, speeds, vehicles, and road users—so that if one component fails, others will still protect people from death or serious injury. The approach emphasizes that crash energies need to be reduced to safe levels, and it promotes a system where human error is accommodated and vulnerability to injury is minimized.
+
+Key principles include:
+- Designing roads that are predictable and forgiving of mistakes.
+- Ensuring travel speeds are appropriate for safety.
+- Using vehicles that prevent crashes and protect occupants.
+- Promoting skilled, alert, and responsible road users.
+
+The overall goal is to share responsibility among system designers and users to achieve safer mobility for everyone.
+
+**Sources:**  
+- ""W 1.3 The Safe System Approach to Road Safety,"" pages 10, 11, and 16.
+
+**Title:** Safe system principles  
+**From:** W 1.3 The Safe System Approach to Road Safety, page 10",
+1,2025-06-16 06:12:07,"
+**Query Analysis:**
+- Query: list the guest lectures
+- Summary of history: The user has asked to list the guest lectures, but the request is brief and lacks details such as the context (e.g., event, course, conference) or specific information about which guest lectures they are referring to.
+- Rewrite query: Please provide a list of the guest lectures for the upcoming university seminar series.
+- Key entities: guest lectures, university seminar series
+
+**Relevant Sources:**
+- [00-07-00] Unit schedule and class info (Relevance: 0.363)
+- [00-04-00] Timetable and location for class (Relevance: 0.356)
+- [00-05-00] Online link for class (Relevance: 0.328)
+
+**Response:**
+The vector db has been cached, containing 39 entries
+
+","Query: list the guest lectures
+Response: The guest lectures for this unit are as follows:
+
+1. Week 1: Safe System Approach by Mr. Ross Thompson from TraffixGroup.
+2. Week 3: ""North East Link Project"" by colleagues from SMEC.
+3. Week 5: ""Geometric Design"" by Mr. Malcom Mak from Australia Road Research Board (ARRB).
+4. Week 8: ""Detailed Geometric Design"" by colleagues from SMEC.
+5. Week 10: Weigh-in-Motion Technology and its application in pavement design (guest lecturer not specified).
+6. Week 11: Austroads Design Methods by Prof. Jayantha Kodikara from SPARC Hub Monash University.
+
+Please note that the schedule of guest lectures is subject to change.
+
+**Sources:**
+- Title: Unit schedule and class info
+- From: learning on Moodle - getting started",
+1,2025-06-16 06:12:53,"
+**Query Analysis:**
+- Query: where should i take class
+- Summary of history: The user previously asked for a list of guest lectures, which included topics and speakers for specific weeks. Now, they are asking about the location or venue where they should attend a class, likely related to the lectures or the course in general.
+- Rewrite query: Where should I attend the class related to these guest lectures?
+- Key entities: class, guest lectures, topics, speakers, weeks, location, venue, course
+
+**Relevant Sources:**
+- [01-31-00] test (Relevance: 0.389)
+- [00-08-00] Unit contacts (Relevance: 0.349)
+- [00-05-00] Online link for class (Relevance: 0.328)
+- [00-07-00] Unit schedule and class info (Relevance: 0.327)
+- [00-03-00] Checklist of important items you should do (Relevance: 0.309)
+- [00-04-00] Timetable and location for class (Relevance: 0.296)
+- [01-08-00] Road functions (Relevance: 0.291)
+- [01-29-00] Road factors (Relevance: 0.250)
+- [01-14-00] brief introduction of design elements (Relevance: 0.243)
+
+**Response:**
+The vector db has been cached, containing 39 entries
+
+","Query: where should i take class
+Response: You should attend the workshop class in the Learning and Teaching Building, G81 (main) and G56 (overflow). The class is scheduled for Monday from 7 pm to 9 pm. Additionally, there are practice sessions scheduled in G29/33 B72 on Wednesdays at 10 am-12 pm and 1 pm-3 pm, which you should attend according to your timetable.
+
+**Sources:**
+- Title: Timetable and location for class
+- From: learning on Moodle - getting started",
+1,2025-06-16 06:27:47,"
+**Query Analysis:**
+- Query: list the guest lecturers
+- Summary of history: The user has asked for a list of guest lecturers, but the context of which event, course, or institution is not specified. They likely want a comprehensive or specific list related to a particular program or organization.
+- Rewrite query: Please provide a list of guest lecturers for the [specific event, course, or institution], or clarify which guest lecturers you are referring to.
+- Key entities: guest lecturers, event, course, institution
+
+**Relevant Sources:**
+- [01-31-00] test (Relevance: 0.447)
+- [00-07-00] Unit schedule and class info (Relevance: 0.357)
+- [00-05-00] Online link for class (Relevance: 0.354)
+- [00-04-00] Timetable and location for class (Relevance: 0.335)
+- [00-08-00] Unit contacts (Relevance: 0.327)
+- [00-01-00] What this unit is about (Relevance: 0.200)
+
+**Response:**
+The vector db has been cached, containing 39 entries
+
+","Query: list the guest lecturers
+Response: The guest lecturers for this unit are:
+
+1. Mr. Ross Thompson from TraffixGroup, who will give a lecture on the Safe System Approach in Week 1.
+2. Colleagues from SMEC, who will give a lecture on ""North East Link Project"" in Week 3 and lead a practical activity on ""Multi-criteria analysis.""
+3. Mr. Malcolm Mak from Australia Road Research Board (ARRB), who will lecture on “geometric design” in Week 5.
+4. Colleagues from SMEC, who will lead a guest lecture and practical activity on “detailed geometric design” in Week 8.
+5. Dr. Tareq and Dr. Shan from the Department of Transport Planning, Victoria, who will lecture on “pavement materials” in Week 9.
+6. Prof. Jayantha Kodikara from SPARC Hub Monash University, who will give a lecture on “Austroads design methods” in Week 11.
+
+Title: Unit schedule and class info
+From: learning on Moodle - getting started
+Content: The schedule of guest lectures is mentioned in the weekly topics.
+
+",thumbs_up

requirements.txt

@@ -0,0 +1,6 @@
+gradio==5.34.0
+openai==1.86.0
+pandas==2.2.3
+numpy==2.2.3
+huggingface-hub==0.33.0
+scipy==1.15.2

vectorize_knowledge_base.py

@@ -0,0 +1,584 @@
+import os
+import re
+import json
+import numpy as np
+import pandas as pd
+from typing import List, Dict, Tuple, Optional
+from openai import OpenAI
+from datetime import datetime
+import csv
+
+class KnowledgeBaseVectorizer:
+    def __init__(self, api_key: str, data_path: str = ""):
+        """
+        初始化向量化器
+        
+        Args:
+            api_key: OpenAI API密钥
+            data_path: knowledge_base.md文件的路径
+        """
+        self.client = OpenAI(api_key=api_key)
+        self.data_path = data_path
+        self.embedding_model = "text-embedding-3-small"
+        #self.vector_db_path = os.path.join(os.path.dirname(data_path), "vector_database.csv")
+        #self.metadata_path = os.path.join(os.path.dirname(data_path), "vector_metadata.json")
+        self.vector_db_path = "vector_database.csv"
+        self.metadata_path = "vector_metadata.json"
+        
+        # 缓存相关属性
+        self._cached_df = None
+        self._cached_metadata = None
+        self._cached_embeddings = {}  # 缓存不同类型的向量矩阵
+        self._last_load_time = None
+        
+    def parse_knowledge_base(self) -> List[Dict]:
+        """
+        解析knowledge_base.md文件，提取所有数据条目
+        支持包含表格的完整内容提取
+        
+        Returns:
+            包含所有数据条目的列表，每个条目是一个字典
+        """
+        entries = []
+        
+        try:
+            with open(self.data_path, 'r', encoding='utf-8') as f:
+                content = f.read()
+        except FileNotFoundError:
+            print(f"错误：找不到文件 {self.data_path}")
+            return entries
+        
+        # 改进的匹配策略：使用更精确的正则表达式
+        # 匹配模式：# xx-xx-xx title **source** ... **content** ... (直到下一个 # 或文件结尾)
+        pattern = r'#\s+(\d{2}-\d{2}-\d{2})\s+([^\n]+)\s+\*\*source\*\*\s+([^\n]+)\s+\*\*content\*\*\s+(.*?)(?=\n#\s+\d{2}-\d{2}-\d{2}|$)'
+        
+        matches = re.findall(pattern, content, re.DOTALL)
+        
+        for match in matches:
+            # 清理内容：移除多余的空白行，但保留表格格式
+            content_text = match[3].strip()
+            
+            # 保留表格的结构，但清理多余的空白
+            content_lines = content_text.split('\n')
+            cleaned_lines = []
+            
+            for line in content_lines:
+                # 保留非空行和表格行
+                if line.strip() or (line.startswith('|') and line.endswith('|')):
+                    cleaned_lines.append(line.rstrip())
+            
+            # 重新组合内容
+            cleaned_content = '\n'.join(cleaned_lines)
+            
+            entry = {
+                'id': match[0].strip(),
+                'title': match[1].strip(),
+                'source': match[2].strip(),
+                'content': cleaned_content,
+                'full_text': f"{match[1].strip()} {cleaned_content}"  # 用于向量化的完整文本
+            }
+            entries.append(entry)
+            
+        print(f"成功解析 {len(entries)} 个数据条目")
+        
+        # 打印一些调试信息
+        if entries:
+            print("前3个条目的内容长度:")
+            for i, entry in enumerate(entries[:3]):
+                content_lines = entry['content'].count('\n') + 1
+                has_table = '|' in entry['content']
+                print(f"  条目 {entry['id']}: {len(entry['content'])} 字符, {content_lines} 行, 包含表格: {has_table}")
+        
+        return entries
+    
+    def get_embedding(self, text: str) -> List[float]:
+        """
+        使用OpenAI API获取文本的向量表示
+        
+        Args:
+            text: 要向量化的文本
+            
+        Returns:
+            文本的向量表示
+        """
+        try:
+            response = self.client.embeddings.create(
+                input=text,
+                model=self.embedding_model
+            )
+            return response.data[0].embedding
+        except Exception as e:
+            print(f"获取向量时出错: {e}")
+            return []
+    
+    def batch_get_embeddings(self, texts: List[str], batch_size: int = 10) -> List[List[float]]:
+        """
+        批量获取文本的向量表示
+        
+        Args:
+            texts: 要向量化的文本列表
+            batch_size: 批处理大小
+            
+        Returns:
+            向量列表
+        """
+        embeddings = []
+        
+        for i in range(0, len(texts), batch_size):
+            batch = texts[i:i + batch_size]
+            print(f"处理批次 {i//batch_size + 1}/{(len(texts) + batch_size - 1)//batch_size}")
+            
+            try:
+                response = self.client.embeddings.create(
+                    input=batch,
+                    model=self.embedding_model
+                )
+                batch_embeddings = [item.embedding for item in response.data]
+                embeddings.extend(batch_embeddings)
+            except Exception as e:
+                print(f"批次处理出错: {e}")
+                # 如果批处理失败，尝试单个处理
+                for text in batch:
+                    embedding = self.get_embedding(text)
+                    embeddings.append(embedding if embedding else [0] * 1536)  # 默认维度
+        
+        return embeddings
+    
+    def create_vector_database(self):
+        """
+        创建向量数据库并保存为CSV文件
+        支持标题和内容的分别向量化
+        """
+        print("开始创建向量数据库...")
+        
+        # 1. 解析知识库
+        entries = self.parse_knowledge_base()
+        if not entries:
+            print("没有找到任何数据条目")
+            return
+        
+        # 2. 准备要向量化的文本
+        titles = [entry['title'] for entry in entries]
+        contents = [entry['content'] for entry in entries]
+        full_texts = [entry['full_text'] for entry in entries]
+        
+        # 3. 批量获取向量
+        print("开始向量化标题...")
+        title_embeddings = self.batch_get_embeddings(titles)
+        
+        print("开始向量化内容...")
+        content_embeddings = self.batch_get_embeddings(contents)
+        
+        print("开始向量化完整文本...")
+        full_embeddings = self.batch_get_embeddings(full_texts)
+        
+        # 4. 创建DataFrame来存储数据
+        print("创建向量数据库DataFrame...")
+        
+        # 准备数据行
+        rows = []
+        for i, entry in enumerate(entries):
+            row = {
+                'index': i,
+                'id': entry['id'],
+                'title': entry['title'],
+                'source': entry['source'],
+                'content': entry['content'],
+                'full_text': entry['full_text']
+            }
+            
+            # 添加标题向量维度
+            for j, val in enumerate(title_embeddings[i]):
+                row[f'title_dim_{j}'] = val
+            
+            # 添加内容向量维度
+            for j, val in enumerate(content_embeddings[i]):
+                row[f'content_dim_{j}'] = val
+                
+            # 添加完整文本向量维度
+            for j, val in enumerate(full_embeddings[i]):
+                row[f'full_dim_{j}'] = val
+            
+            rows.append(row)
+        
+        # 创建DataFrame
+        df = pd.DataFrame(rows)
+        
+        # 5. 保存为CSV文件
+        print("保存向量数据库到CSV...")
+        df.to_csv(self.vector_db_path, index=False, encoding='utf-8')
+        
+        # 6. 保存元数据（JSON格式，便于查看）
+        metadata = {
+            'embedding_model': self.embedding_model,
+            'created_at': datetime.now().isoformat(),
+            'num_entries': len(entries),
+            'embedding_dimensions': len(title_embeddings[0]) if title_embeddings else 0,
+            'vector_types': ['title', 'content', 'full'],
+            'columns': list(df.columns),
+            'entries_summary': [
+                {
+                    'id': entry['id'],
+                    'title': entry['title'],
+                    'source': entry['source']
+                } for entry in entries
+            ]
+        }
+        
+        with open(self.metadata_path, 'w', encoding='utf-8') as f:
+            json.dump(metadata, f, ensure_ascii=False, indent=2)
+        
+        print(f"向量数据库创建完成！")
+        print(f"向量数据库保存在: {self.vector_db_path}")
+        print(f"元数据保存在: {self.metadata_path}")
+        print(f"总共处理了 {len(entries)} 个条目")
+        print(f"每个向量的维度: {len(title_embeddings[0]) if title_embeddings else 0}")
+        
+        # 清除缓存以便重新加载
+        self.clear_cache()
+    
+    def clear_cache(self):
+        """清除所有缓存"""
+        self._cached_df = None
+        self._cached_metadata = None
+        self._cached_embeddings = {}
+        self._last_load_time = None
+        print("向量数据库缓存已清除")
+    
+    def load_vector_database(self, force_reload: bool = False) -> Tuple[Optional[pd.DataFrame], Optional[Dict]]:
+        """
+        从CSV文件加载向量数据库（带缓存机制）
+        
+        Args:
+            force_reload: 是否强制重新加载
+            
+        Returns:
+            DataFrame和元数据字典的元组
+        """
+        # 检查是否需要重新加载
+        if not force_reload and self._cached_df is not None and self._cached_metadata is not None:
+            return self._cached_df, self._cached_metadata
+        
+        try:
+            # 加载CSV文件
+            df = pd.read_csv(self.vector_db_path, encoding='utf-8')
+            
+            # 加载元数据
+            with open(self.metadata_path, 'r', encoding='utf-8') as f:
+                metadata = json.load(f)
+            
+            # 缓存结果
+            self._cached_df = df
+            self._cached_metadata = metadata
+            self._last_load_time = datetime.now()
+            
+            # 预加载向量矩阵到缓存
+            self._preload_embeddings()
+            
+            print(f"成功加载向量数据库，包含 {len(df)} 个条目")
+            return df, metadata
+        except FileNotFoundError as e:
+            print(f"错误：找不到文件 - {e}")
+            return None, None
+        except Exception as e:
+            print(f"加载向量数据库时出错: {e}")
+            return None, None
+    
+    def _preload_embeddings(self):
+        """预加载所有类型的向量矩阵到缓存"""
+        if self._cached_df is None:
+            return
+        
+        vector_types = ['title', 'content', 'full']
+        for vector_type in vector_types:
+            if vector_type not in self._cached_embeddings:
+                embeddings = self.get_embeddings_from_df(self._cached_df, vector_type)
+                # 预计算归一化向量
+                embeddings_norm = embeddings / np.linalg.norm(embeddings, axis=1, keepdims=True)
+                self._cached_embeddings[vector_type] = {
+                    'raw': embeddings,
+                    'normalized': embeddings_norm
+                }
+        
+        print(f"预加载了 {len(vector_types)} 种类型的向量矩阵")
+    
+    def get_embeddings_from_df(self, df: pd.DataFrame, vector_type: str = 'full') -> np.ndarray:
+        """
+        从DataFrame中提取向量矩阵
+        
+        Args:
+            df: 包含向量的DataFrame
+            vector_type: 向量类型 ('title', 'content', 'full')
+            
+        Returns:
+            向量矩阵
+        """
+        # 根据类型获取对应的列
+        if vector_type == 'title':
+            embedding_cols = [col for col in df.columns if col.startswith('title_dim_')]
+        elif vector_type == 'content':
+            embedding_cols = [col for col in df.columns if col.startswith('content_dim_')]
+        else:  # 'full'
+            embedding_cols = [col for col in df.columns if col.startswith('full_dim_')]
+            
+        embeddings = df[embedding_cols].values
+        return embeddings
+    
+    def batch_search_similar(self, queries: List[str], top_k: int = 5, 
+                           title_weight: float = 0.4, 
+                           content_weight: float = 0.3,
+                           full_weight: float = 0.3) -> List[List[Tuple[Dict, float, Dict]]]:
+        """
+        批量搜索多个查询，只加载一次向量数据库
+        
+        Args:
+            queries: 查询文本列表
+            top_k: 每个查询返回最相似的前k个结果
+            title_weight: 标题相似度的权重
+            content_weight: 内容相似度的权重
+            full_weight: 完整文本相似度的权重
+            
+        Returns:
+            每个查询对应的相似条目列表
+        """
+        # 确保权重之和为1
+        total_weight = title_weight + content_weight + full_weight
+        title_weight /= total_weight
+        content_weight /= total_weight
+        full_weight /= total_weight
+        
+        # 加载向量数据库（只加载一次）
+        df, metadata = self.load_vector_database()
+        if df is None:
+            return [[] for _ in queries]
+        
+        # 批量获取查询向量
+        print(f"批量生成 {len(queries)} 个查询的向量...")
+        query_embeddings = self.batch_get_embeddings(queries, batch_size=min(10, len(queries)))
+        
+        if len(query_embeddings) != len(queries):
+            print("查询向量生成失败")
+            return [[] for _ in queries]
+        
+        # 获取缓存的归一化向量矩阵
+        title_embeddings_norm = self._cached_embeddings['title']['normalized']
+        content_embeddings_norm = self._cached_embeddings['content']['normalized']
+        full_embeddings_norm = self._cached_embeddings['full']['normalized']
+        
+        all_results = []
+        
+        # 对每个查询进行相似度计算
+        for i, (query, query_embedding) in enumerate(zip(queries, query_embeddings)):
+            if not query_embedding:
+                all_results.append([])
+                continue
+            
+            query_vec = np.array(query_embedding)
+            query_vec_norm = query_vec / np.linalg.norm(query_vec)
+            
+            # 计算各部分的相似度
+            title_similarities = np.dot(title_embeddings_norm, query_vec_norm)
+            content_similarities = np.dot(content_embeddings_norm, query_vec_norm)
+            full_similarities = np.dot(full_embeddings_norm, query_vec_norm)
+            
+            # 加权综合相似度
+            combined_similarities = (
+                title_weight * title_similarities +
+                content_weight * content_similarities +
+                full_weight * full_similarities
+            )
+            
+            # 获取top-k
+            top_indices = np.argsort(combined_similarities)[::-1][:top_k]
+            
+            query_results = []
+            for idx in top_indices:
+                # 从DataFrame中获取条目信息
+                row = df.iloc[idx]
+                entry = {
+                    'id': row['id'],
+                    'title': row['title'],
+                    'source': row['source'],
+                    'content': row['content']
+                }
+                
+                # 添加各部分的相似度详情
+                similarity_details = {
+                    'combined': float(combined_similarities[idx]),
+                    'title': float(title_similarities[idx]),
+                    'content': float(content_similarities[idx]),
+                    'full': float(full_similarities[idx])
+                }
+                
+                query_results.append((entry, float(combined_similarities[idx]), similarity_details))
+            
+            all_results.append(query_results)
+            print(f"完成查询 {i+1}/{len(queries)}: '{query[:50]}...'")
+        
+        return all_results
+    
+    def search_similar(self, query: str, top_k: int = 5, 
+                      title_weight: float = 0.4, 
+                      content_weight: float = 0.3,
+                      full_weight: float = 0.3) -> List[Tuple[Dict, float, Dict]]:
+        """
+        搜索与查询最相似的条目，综合考虑标题和内容的相似度
+        使用批量搜索的优化版本
+        
+        Args:
+            query: 查询文本
+            top_k: 返回最相似的前k个结果
+            title_weight: 标题相似度的权重
+            content_weight: 内容相似度的权重
+            full_weight: 完整文本相似度的权重
+            
+        Returns:
+            相似条目和相似度分数的列表
+        """
+        # 使用批量搜索处理单个查询
+        results = self.batch_search_similar([query], top_k, title_weight, content_weight, full_weight)
+        return results[0] if results else []
+    
+    def search_with_entities_optimized(self, entities: List[str], top_k: int = 3) -> List[Tuple[Dict, float, Dict]]:
+        """
+        优化版本：使用实体列表搜索知识库，只加载一次向量数据库
+        
+        Args:
+            entities: 实体列表
+            top_k: 每个实体返回的结果数
+            
+        Returns:
+            合并和去重后的搜索结果
+        """
+        if not entities:
+            return []
+        
+        # 使用批量搜索
+        batch_results = self.batch_search_similar(
+            entities, 
+            top_k=top_k,
+            title_weight=0.5,    # 对于实体搜索，标题权重更高
+            content_weight=0.3,
+            full_weight=0.2
+        )
+        
+        # 合并结果并去重
+        seen_ids = set()
+        all_results = []
+        
+        for entity_results in batch_results:
+            for entry, score, details in entity_results:
+                entry_id = entry['id']
+                if entry_id not in seen_ids:
+                    seen_ids.add(entry_id)
+                    all_results.append((entry, score, details))
+        
+        # 按分数排序
+        sorted_results = sorted(all_results, key=lambda x: x[1], reverse=True)
+        return sorted_results
+    
+    def add_new_entry(self, id: str, title: str, source: str, content: str):
+        """
+        添加新条目到向量数据库
+        
+        Args:
+            id: 条目ID
+            title: 标题
+            source: 来源
+            content: 内容
+        """
+        # 加载现有数据库
+        df, metadata = self.load_vector_database()
+        
+        if df is None:
+            print("向量数据库不存在，将创建新的数据库")
+            df = pd.DataFrame()
+        
+        # 创建新条目
+        full_text = f"{title} {content}"
+        
+        # 获取三种类型的向量
+        print(f"正在为新条目 {id} 生成向量...")
+        title_embedding = self.get_embedding(title)
+        content_embedding = self.get_embedding(content)
+        full_embedding = self.get_embedding(full_text)
+        
+        if not all([title_embedding, content_embedding, full_embedding]):
+            print("无法生成向量")
+            return
+        
+        # 创建新条目
+        new_entry = {
+            'index': len(df),
+            'id': id,
+            'title': title,
+            'source': source,
+            'content': content,
+            'full_text': full_text
+        }
+        
+        # 添加向量维度
+        for j, val in enumerate(title_embedding):
+            new_entry[f'title_dim_{j}'] = val
+        for j, val in enumerate(content_embedding):
+            new_entry[f'content_dim_{j}'] = val
+        for j, val in enumerate(full_embedding):
+            new_entry[f'full_dim_{j}'] = val
+        
+        # 添加到DataFrame
+        new_df = pd.DataFrame([new_entry])
+        df = pd.concat([df, new_df], ignore_index=True)
+        
+        # 保存更新后的数据库
+        df.to_csv(self.vector_db_path, index=False, encoding='utf-8')
+        
+        # 更新元数据
+        if metadata:
+            metadata['num_entries'] = len(df)
+            metadata['updated_at'] = datetime.now().isoformat()
+            with open(self.metadata_path, 'w', encoding='utf-8') as f:
+                json.dump(metadata, f, ensure_ascii=False, indent=2)
+        
+        # 清除缓存以便重新加载
+        self.clear_cache()
+        
+        print(f"成功添加新条目 {id}")
+    
+    def export_to_readable_format(self, output_path: str = None):
+        """
+        导出向量数据库到更易读的格式（不包含向量维度）
+        
+        Args:
+            output_path: 输出文件路径
+        """
+        df, _ = self.load_vector_database()
+        if df is None:
+            return
+        
+        if output_path is None:
+            output_path = os.path.join(
+                os.path.dirname(self.data_path), 
+                "vector_database_readable.csv"
+            )
+        
+        # 只保留非向量列
+        non_vector_cols = [col for col in df.columns if not any(col.startswith(prefix) for prefix in ['title_dim_', 'content_dim_', 'full_dim_'])]
+        readable_df = df[non_vector_cols]
+        
+        # 保存
+        readable_df.to_csv(output_path, index=False, encoding='utf-8')
+        print(f"可读格式的数据库已保存到: {output_path}")
+    
+    def get_cache_info(self) -> Dict:
+        """
+        获取缓存状态信息
+        
+        Returns:
+            缓存状态字典
+        """
+        return {
+            'is_cached': self._cached_df is not None,
+            'cache_size': len(self._cached_df) if self._cached_df is not None else 0,
+            'cached_embeddings': list(self._cached_embeddings.keys()),
+            'last_load_time': self._last_load_time.isoformat() if self._last_load_time else None
+        }