Upload folder using huggingface_hub

.env.example

@@ -0,0 +1,29 @@
+# OpenAI API 配置
+OPENAI_API_KEY=xxx
+OPENAI_BASE_URL=https://matrixllm.alipay.com/v1
+
+# 模型配置
+MODEL_NAME=gpt-4
+EMBEDDING_MODEL=text-embedding-ada-002
+
+# 文本处理配置
+MAX_CHUNK_SIZE=2000
+CHUNK_OVERLAP=200
+
+# 角色提取配置
+MIN_CHARACTER_MENTIONS=10
+TOP_N_CHARACTERS=20
+
+# 角色分析配置
+MAX_ANALYSIS_CHUNKS=10
+ANALYSIS_TEMPERATURE=0.3
+
+# 对话配置
+MAX_HISTORY=10
+CONVERSATION_TEMPERATURE=0.8
+MAX_MEMORY_RETRIEVAL=5
+
+# 缓存配置
+ENABLE_CACHE=True
+CACHE_DIR=cache
+VECTOR_DB_PATH=cache/vectordb

cache/analysis_Thomas_-1235069295746715801.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a54828c9e1e0079587330b82f65bed3602afaef2fedfcdf519fc5ab86b4d997
+size 9919

cache/analysis_Thomas_2921407663095156522.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:02ded18a8adeda53d5c030429ce091c23fed2f07ba868a2a52476fff825fb686
+size 9837

cache/analysis_哈利波特_1748832246067442544.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c0e1a27fd10a4dae2e0a707de5d376eb4fdcc7f6d5cd866399457af83a9396ce
+size 13044

cache/characters_-3712775294649564773.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d00c966ed5d03784c446d231aabe64ace3b45ad035904f8b2269aed97ca79572
+size 79316

cache/characters_250361385978348193.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:82043c4997ac79a9a8e934b828a07006d63d542a80b17977ff93b24545c646a9
+size 3256

cache/characters_4782512100612132426.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:82043c4997ac79a9a8e934b828a07006d63d542a80b17977ff93b24545c646a9
+size 3256

cache/characters_6926022490162963799.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8629e56fd1389e9e345ada3802117d9dfe757da87e18b0e423e7a5de68dd79a3
+size 117980

cache/characters_7412048294772991388.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:82043c4997ac79a9a8e934b828a07006d63d542a80b17977ff93b24545c646a9
+size 3256

cache/characters_8212446298422427823.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bddb7ca207a343482bd772d79aa1f28f82b37f0c323bda1f59913ab7292983ce
+size 3594

config.py

@@ -0,0 +1,35 @@
+import os
+from dotenv import load_dotenv
+
+load_dotenv()
+
+class Config:
+    # API 配置
+    OPENAI_API_KEY = os.getenv("OPENAI_API_KEY", "xxx")
+    OPENAI_BASE_URL = os.getenv("OPENAI_BASE_URL", "https://matrixllm.alipay.com/v1")
+    MODEL_NAME = os.getenv("MODEL_NAME", "gpt-5.1")
+    EMBEDDING_MODEL = os.getenv("EMBEDDING_MODEL", "text-embedding-ada-002")
+    
+    # 文本处理配置
+    MAX_CHUNK_SIZE = 2000  # 每个文本块的最大字符数
+    CHUNK_OVERLAP = 200    # 文本块之间的重叠
+    
+    # 角色提取配置
+    MIN_CHARACTER_MENTIONS = 10  # 最小出场次数
+    TOP_N_CHARACTERS = 20        # 提取前N个主要角色
+    
+    # 角色分析配置
+    MAX_ANALYSIS_CHUNKS = 10     # 每个角色最多分析的文本块数量
+    ANALYSIS_TEMPERATURE = 0.3
+    
+    # 对话配置
+    MAX_HISTORY = 10
+    CONVERSATION_TEMPERATURE = 0.8
+    MAX_MEMORY_RETRIEVAL = 5     # 从记忆中检索的最大片段数
+    
+    # 缓存配置
+    CACHE_DIR = "cache"
+    ENABLE_CACHE = True
+    
+    # 向量数据库配置
+    VECTOR_DB_PATH = "cache/vectordb"

core/character_agent.py

@@ -0,0 +1,211 @@
+from typing import List, Dict, Optional
+from config import Config
+from core.memory_manager import MemoryManager
+from core.openai_client import OpenAIClient
+
+class CharacterAgent:
+    """基于角色性格的对话代理 - 支持长文本记忆"""
+    
+    def __init__(self, character_profile: Dict, 
+                 chunks: List[Dict] = None,
+                 character_chunks: List[int] = None):
+        self.character_profile = character_profile
+        self.client = OpenAIClient.get_client()
+        
+        self.conversation_history: List[Dict] = []
+        self.system_prompt = self._build_system_prompt()
+        
+        # 初始化记忆管理器
+        self.memory_manager = None
+        if chunks and character_chunks:
+            self.memory_manager = MemoryManager(character_profile['name'])
+            self.memory_manager.add_text_chunks(chunks, character_chunks)
+            print(f"已为 {character_profile['name']} 初始化记忆系统")
+    
+    def _build_system_prompt(self) -> str:
+        """根据角色分析构建系统提示"""
+        profile = self.character_profile
+        
+        # 构建角色特征描述
+        traits_desc = "、".join(profile.get('core_traits', []))
+        
+        prompt = f"""
+你现在要完全沉浸式地扮演小说中的角色"{profile['name']}"。这不是角色扮演游戏，你就是{profile['name']}本人。
+
+## 角色身份
+你是 {profile['name']}，{profile.get('background', '一个小说角色')}。
+
+## 性格特征
+{profile.get('personality_summary', '')}
+
+核心性格特质：{traits_desc}
+
+## 说话风格
+{profile.get('speaking_style', '自然真实的对话风格')}
+
+## 行为模式
+{profile.get('behavior_patterns', '根据性格行事')}
+
+## 价值观与信念
+{profile.get('values', '')}
+
+## 情感表达
+{profile.get('emotional_style', '')}
+
+## 人际关系风格
+{profile.get('relationship_style', '')}
+
+## 典型语句示例
+{chr(10).join('- "' + q + '"' for q in profile.get('key_quotes', [])[:5])}
+
+## 对话原则
+1. **完全成为角色**：用第一人称，以{profile['name']}的身份、记忆、情感回应
+2. **保持一致性**：所有回应必须符合上述性格特征和说话风格
+3. **自然真实**：像真人一样对话，有情绪起伏，可以开玩笑、生气、困惑
+4. **不跳戏**：绝不说"作为AI"、"我是语言模型"等破坏沉浸感的话
+5. **合理应对**：如果问到角色不知道的事，以角色的方式回应（如"我不太清楚"、"这个我没听说过"）
+6. **保持神秘**：不要过度解释角色设定，自然地展现性格
+7. **情境感知**：根据对话氛围调整语气和态度
+
+记住：你不是在"扮演"或"模仿"{profile['name']}，你就是{profile['name']}。
+
+现在，以{profile['name']}的身份开始对话。
+"""
+        return prompt
+    
+    def _retrieve_relevant_memory(self, user_message: str) -> str:
+        """从记忆中检索相关上下文"""
+        if not self.memory_manager:
+            return ""
+        
+        relevant_contexts = self.memory_manager.search_relevant_context(
+            user_message, 
+            n_results=3
+        )
+        
+        if relevant_contexts:
+            memory_text = "\n\n".join(relevant_contexts[:2])  # 只用前2个
+            return f"\n[相关记忆片段：\n{memory_text[:1000]}]\n"
+        return ""
+    
+    def chat(self, user_message: str, use_memory: bool = True) -> str:
+        """与用户对话"""
+        
+        # 构建消息列表
+        messages = [
+            {"role": "system", "content": self.system_prompt}
+        ]
+        
+        # 如果启用记忆，检索相关上下文
+        enhanced_message = user_message
+        if use_memory and self.memory_manager:
+            memory_context = self._retrieve_relevant_memory(user_message)
+            if memory_context:
+                # 在系统消息中添加记忆上下文
+                memory_prompt = f"""
+{self.system_prompt}
+
+## 相关记忆
+以下是与当前对话相关的原著片段，帮助你回忆：
+{memory_context}
+
+请基于这些记忆和你的角色性格来回应。
+"""
+                messages[0] = {"role": "system", "content": memory_prompt}
+        
+        # 添加对话历史
+        recent_history = self.conversation_history[-Config.MAX_HISTORY:]
+        messages.extend(recent_history)
+        
+        # 添加当前用户消息
+        messages.append({
+            "role": "user",
+            "content": user_message
+        })
+        
+        try:
+            response = self.client.chat.completions.create(
+                model=Config.MODEL_NAME,
+                messages=messages
+            )
+            
+            assistant_message = response.choices[0].message.content.strip()
+            
+            # 保存到历史
+            self.conversation_history.append({
+                "role": "user",
+                "content": user_message
+            })
+            self.conversation_history.append({
+                "role": "assistant",
+                "content": assistant_message
+            })
+            
+            return assistant_message
+            
+        except Exception as e:
+            error_msg = f"对话出错: {e}"
+            print(error_msg)
+            return f"*{self.character_profile['name']}沉默了片刻，似乎在思考着什么...*"
+    
+    def reset_conversation(self):
+        """重置对话历史"""
+        self.conversation_history = []
+        print(f"\n[对话已重置，{self.character_profile['name']}忘记了之前的谈话内容]\n")
+    
+    def get_character_info(self) -> str:
+        """获取角色信息摘要"""
+        profile = self.character_profile
+        
+        info = f"""
+{'='*70}
+角色档案：{profile['name']}
+{'='*70}
+
+【性格特质】
+{' • '.join(profile.get('core_traits', []))}
+
+【性格总结】
+{profile.get('personality_summary', 'N/A')}
+
+【说话风格】
+{profile.get('speaking_style', 'N/A')}
+
+【核心价值观】
+{profile.get('values', 'N/A')}
+
+【典型语句】
+"""
+        for i, quote in enumerate(profile.get('key_quotes', [])[:3], 1):
+            info += f'{i}. "{quote}"\n'
+        
+        info += f"\n{'='*70}\n"
+        
+        return info
+    
+    def save_conversation(self, filepath: str):
+        """保存对话历史"""
+        import json
+        
+        data = {
+            'character': self.character_profile['name'],
+            'history': self.conversation_history
+        }
+        
+        with open(filepath, 'w', encoding='utf-8') as f:
+            json.dump(data, f, ensure_ascii=False, indent=2)
+        
+        print(f"\n对话已保存到: {filepath}")
+    
+    def load_conversation(self, filepath: str):
+        """加载对话历史"""
+        import json
+        
+        try:
+            with open(filepath, 'r', encoding='utf-8') as f:
+                data = json.load(f)
+            
+            self.conversation_history = data['history']
+            print(f"\n已加载 {len(self.conversation_history)} 条对话记录")
+        except Exception as e:
+            print(f"加载对话失败: {e}")

core/character_analyzer.py

@@ -0,0 +1,175 @@
+from typing import Dict, List
+from tqdm import tqdm
+from config import Config
+from utils.cache_manager import CacheManager
+from core.openai_client import OpenAIClient
+
+class CharacterAnalyzer:
+    """角色性格分析器 - 针对大规模文本优化"""
+    
+    def __init__(self):
+        self.client = OpenAIClient.get_client()
+        self.cache = CacheManager()
+    
+    def select_representative_chunks(self, chunks: List[Dict], 
+                                    character_chunks: List[int],
+                                    max_chunks: int = None) -> List[Dict]:
+        """选择最具代表性的文本块"""
+        
+        max_chunks = max_chunks or Config.MAX_ANALYSIS_CHUNKS
+        
+        # 均匀分布选择
+        if len(character_chunks) <= max_chunks:
+            selected_ids = character_chunks
+        else:
+            step = len(character_chunks) // max_chunks
+            selected_ids = [character_chunks[i * step] for i in range(max_chunks)]
+        
+        selected_chunks = [chunks[i] for i in selected_ids if i < len(chunks)]
+        return selected_chunks
+    
+    def analyze_character_batch(self, character_name: str, 
+                                text_chunks: List[Dict]) -> Dict:
+        """分批分析角色性格"""
+        
+        # 检查缓存
+        cache_key = f"analysis_{character_name}_{hash(str([c['chunk_id'] for c in text_chunks]))}"
+        cached = self.cache.get(cache_key)
+        if cached:
+            print(f"从缓存加载 {character_name} 的分析结果")
+            return cached
+        
+        # 合并文本块
+        combined_text = "\n\n---\n\n".join([c['text'] for c in text_chunks])
+        
+        analysis_prompt = f"""
+        请深度分析小说中"{character_name}"这个角色的性格特征。
+        
+        基于以下文本片段进行分析：
+        
+        {combined_text[:8000]}  # 限制输入长度
+        
+        请从以下维度分析，并以JSON格式返回：
+        
+        {{
+            "name": "{character_name}",
+            "core_traits": ["特质1", "特质2", "特质3"],
+            "speaking_style": "说话风格描述",
+            "behavior_patterns": "行为模式描述",
+            "values": "核心价值观",
+            "emotional_style": "情感表达方式",
+            "relationship_style": "人际关系风格",
+            "background": "背景信息",
+            "key_quotes": ["典型语句1", "典型语句2"],
+            "personality_summary": "性格总结（100字以内）"
+        }}
+        
+        注意：
+        1. 只基于文本内容分析，不要添加原著之外的信息
+        2. 提取该角色的典型对话和行为
+        3. 关注语言风格、用词习惯、口头禅等
+        """
+        
+        try:
+            response = self.client.chat.completions.create(
+                model=Config.MODEL_NAME,
+                messages=[
+                    {"role": "system", "content": "你是一个专业的文学角色分析专家。请基于文本内容进行深入分析。"},
+                    {"role": "user", "content": analysis_prompt}
+                ]
+            )
+            
+            analysis_text = response.choices[0].message.content.strip()
+            
+            # 尝试解析JSON
+            import json
+            import re
+            
+            json_match = re.search(r'\{.*\}', analysis_text, re.DOTALL)
+            if json_match:
+                profile = json.loads(json_match.group())
+            else:
+                profile = self._parse_text_analysis(analysis_text, character_name)
+            
+            profile['raw_analysis'] = analysis_text
+            
+            # 缓存结果
+            self.cache.set(cache_key, profile)
+            
+            return profile
+        
+        except Exception as e:
+            print(f"分析失败: {e}")
+            return self._default_profile(character_name)
+    
+    def _parse_text_analysis(self, text: str, character_name: str) -> Dict:
+        """解析文本格式的分析结果"""
+        
+        profile = {
+            'name': character_name,
+            'core_traits': [],
+            'speaking_style': '',
+            'behavior_patterns': '',
+            'values': '',
+            'emotional_style': '',
+            'relationship_style': '',
+            'background': '',
+            'key_quotes': [],
+            'personality_summary': ''
+        }
+        
+        # 提取各个部分
+        import re
+        
+        patterns = {
+            'core_traits': r'core_traits["\']?\s*:\s*\[(.*?)\]',
+            'speaking_style': r'speaking_style["\']?\s*:\s*["\']([^"\']+)["\']',
+            'key_quotes': r'key_quotes["\']?\s*:\s*\[(.*?)\]',
+        }
+        
+        for key, pattern in patterns.items():
+            match = re.search(pattern, text, re.DOTALL | re.IGNORECASE)
+            if match:
+                content = match.group(1)
+                if key in ['core_traits', 'key_quotes']:
+                    items = re.findall(r'["\']([^"\']+)["\']', content)
+                    profile[key] = items
+                else:
+                    profile[key] = content
+        
+        return profile
+    
+    def _default_profile(self, character_name: str) -> Dict:
+        """默认角色配置"""
+        return {
+            'name': character_name,
+            'core_traits': ['复杂', '多面'],
+            'speaking_style': '根据情境变化',
+            'behavior_patterns': '待观察',
+            'values': '待分析',
+            'emotional_style': '情感丰富',
+            'relationship_style': '因人而异',
+            'background': '小说角色',
+            'key_quotes': [],
+            'personality_summary': f'{character_name}是一个复杂的角色',
+            'raw_analysis': '使用默认配置'
+        }
+    
+    def enhance_profile_with_examples(self, profile: Dict, chunks: List[Dict],
+                                     character_chunks: List[int]) -> Dict:
+        """通过对话实例增强角色配置"""
+        
+        # 提取包含该角色的对话
+        dialogues = []
+        for chunk_id in character_chunks[:5]:  # 只看前几个块
+            if chunk_id < len(chunks):
+                chunk_text = chunks[chunk_id]['text']
+                # 简单提取引号内容
+                import re
+                quotes = re.findall(r'["\']([^"\']{10,100})["\']', chunk_text)
+                dialogues.extend(quotes[:3])
+        
+        if dialogues:
+            profile['example_dialogues'] = dialogues[:5]
+        
+        return profile

core/character_extractor.py

@@ -0,0 +1,356 @@
+import re
+from typing import List, Dict, Tuple
+from collections import Counter, defaultdict
+from tqdm import tqdm
+from config import Config
+from utils.cache_manager import CacheManager
+from core.openai_client import OpenAIClient
+
+class CharacterExtractor:
+    """角色提取器 - 从大规模文本中识别主要角色"""
+    
+    def __init__(self):
+        self.client = OpenAIClient.get_client()
+        self.cache = CacheManager()
+    
+    def extract_names_pattern(self, text: str, language: str = "en") -> List[str]:
+        """使用正则模式提取人名
+        
+        Args:
+            text: 输入文本
+            language: 语言类型 ('en' 或 'zh')
+            
+        Returns:
+            提取到的人名列表
+        """
+        if language == "en":
+            # 英文人名：大写字母开头的连续词
+            pattern = r'\b[A-Z][a-z]+(?:\s+[A-Z][a-z]+)*\b'
+        else:
+            # 中文人名：常见姓氏+1-2个字
+            common_surnames = '哈赫王李张刘陈杨黄赵吴周徐孙马朱胡郭何林高梁郑罗宋谢唐韩曹许邓萧冯曾程蔡彭潘袁于董余苏叶吕魏蒋田杜丁沈姜范江傅钟卢汪戴崔任陆廖姚方金邱夏谭韦贾邹石熊孟秦阎薛侯雷白龙段郝孔邵史毛常万顾赖武康贺严尹钱施牛洪龚'
+            pattern = f'[{common_surnames}][\\u4e00-\\u9fff]{{1,3}}'
+        
+        names = re.findall(pattern, text)
+        return names
+    
+    def extract_characters_from_chunks(self, chunks: List[Dict], 
+                                      language: str = "en") -> Dict[str, Dict]:
+        """从文本块中提取角色及其出现信息
+        
+        Args:
+            chunks: 文本块列表
+            language: 语言类型
+            
+        Returns:
+            角色信息字典
+        """
+        
+        # 检查缓存
+        cache_key = f"characters_{hash(str(chunks[:3]))}"
+        cached = self.cache.get(cache_key)
+        if cached:
+            print("从缓存加载角色信息")
+            return cached
+        
+        character_mentions = defaultdict(lambda: {
+            'count': 0,
+            'chunks': set(),
+            'positions': [],
+            'contexts': []
+        })
+        
+        # 添加常见的非角色词汇黑名单
+        blacklist = {
+            # 章节标记
+            'Chapter', 'CHAPTER', 'Part', 'PART', 'Section', 'SECTION', 
+            'Book', 'BOOK', 'Volume', 'VOLUME',
+            # 冠词和连词
+            'The', 'THE', 'And', 'AND', 'But', 'BUT', 'Or', 'OR',
+            # 疑问词
+            'When', 'WHERE', 'What', 'WHAT', 'Who', 'WHO', 'Why', 'WHY', 'How', 'HOW',
+            # 称谓（单独出现）
+            'Mr', 'Mrs', 'Ms', 'Miss', 'Dr', 'Professor', 'Sir', 'Madam',
+            # 星期
+            'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday',
+            # 月份
+            'January', 'February', 'March', 'April', 'May', 'June', 'July', 
+            'August', 'September', 'October', 'November', 'December',
+            # 常见地点
+            'Street', 'Road', 'Avenue', 'Place', 'Square', 'Hall', 'House',
+            'School', 'Academy', 'University', 'Castle', 'Forest', 'Mountain',
+            # 其他常见词
+            'Note', 'End', 'Beginning', 'Epilogue', 'Prologue', 'Appendix',
+            'First', 'Second', 'Third', 'Last', 'Next', 'Previous',
+        }
+        
+        print("\n提取角色名称...")
+        for chunk in tqdm(chunks):
+            names = self.extract_names_pattern(chunk['text'], language)
+            
+            for name in names:
+                # 过滤太短或太长的名字
+                if language == "en":
+                    if len(name) < 3 or len(name) > 30:
+                        continue
+                else:  # zh
+                    if len(name) < 2 or len(name) > 4:
+                        continue
+                
+                # 过滤黑名单
+                if name in blacklist:
+                    continue
+                
+                # 过滤纯数字
+                if name.isdigit():
+                    continue
+                
+                # 过滤单个字母或单个词（可能是缩写）
+                if len(name) <= 2:
+                    continue
+                
+                # 过滤全大写（可能是缩写或标题）
+                if name.isupper() and len(name) < 5:
+                    continue
+                
+                character_mentions[name]['count'] += 1
+                character_mentions[name]['chunks'].add(chunk['chunk_id'])
+                character_mentions[name]['positions'].append(chunk['start'])
+        
+        # 转换set为list以便序列化
+        for char in character_mentions:
+            character_mentions[char]['chunks'] = list(character_mentions[char]['chunks'])
+        
+        # 缓存结果
+        self.cache.set(cache_key, dict(character_mentions))
+        
+        return dict(character_mentions)
+    
+    def rank_characters(self, character_mentions: Dict) -> List[Tuple[str, Dict]]:
+        """根据出场频率和分布排序角色
+        
+        Args:
+            character_mentions: 角色提及信息
+            
+        Returns:
+            排序后的角色列表
+        """
+        
+        scored_characters = []
+        
+        for name, info in character_mentions.items():
+            # 综合评分：出现次数 + 出现的块数量 * 2
+            score = info['count'] + len(info['chunks']) * 2
+            scored_characters.append((name, info, score))
+        
+        # 按分数排序
+        scored_characters.sort(key=lambda x: x[2], reverse=True)
+        
+        # 返回前N个角色
+        return [(name, info) for name, info, score in scored_characters[:Config.TOP_N_CHARACTERS]]
+    
+    def merge_similar_names(self, characters: List[Tuple[str, Dict]]) -> List[Tuple[str, Dict]]:
+        """合并相似的名字（如 Harry 和 Harry Potter）
+        
+        Args:
+            characters: 角色列表
+            
+        Returns:
+            合并后的角色列表
+        """
+        
+        merged = {}
+        
+        for name, info in characters:
+            # 查找是否有包含关系
+            found_parent = False
+            for existing_name in list(merged.keys()):
+                # 如果当前名字是已存在名字的一部分，或反之
+                if name in existing_name or existing_name in name:
+                    # 合并到更长的名字下
+                    longer_name = name if len(name) > len(existing_name) else existing_name
+                    shorter_name = existing_name if longer_name == name else name
+                    
+                    if longer_name not in merged:
+                        merged[longer_name] = merged.pop(existing_name)
+                    
+                    # 合并信息
+                    merged[longer_name]['count'] += info['count']
+                    merged[longer_name]['chunks'].extend(info['chunks'])
+                    merged[longer_name]['chunks'] = list(set(merged[longer_name]['chunks']))
+                    
+                    found_parent = True
+                    break
+            
+            if not found_parent:
+                merged[name] = info
+        
+        return list(merged.items())
+    
+    def refine_with_llm(self, text_sample: str, candidate_names: List[str]) -> List[str]:
+        """使用LLM进一步确认和提取主要角色
+        
+        Args:
+            text_sample: 文本样本
+            candidate_names: 候选角色名列表
+            
+        Returns:
+            确认的角色名列表
+        """
+        
+        prompt = f"""
+以下是一部小说的开头片段。请识别出主要角色的名字。
+
+候选名字列表：{', '.join(candidate_names[:30])}
+
+文本片段：
+{text_sample[:2000]}
+
+请返回确认的主要角色名字，每行一个。只返回真正的角色名，排除：
+1. 地名、物品名
+2. 章节标记（如 "Chapter"）
+3. 称谓词（如 "Mr", "Professor"）
+4. 普通名词
+
+只返回人名，每个名字一行。
+"""
+        
+        try:
+            response = self.client.chat.completions.create(
+                model="gpt-3.5-turbo",
+                messages=[
+                    {"role": "system", "content": "你是一个文学分析专家，擅长识别小说角色。请只返回真正的人物角色名字。"},
+                    {"role": "user", "content": prompt}
+                ]
+            )
+            
+            result = response.choices[0].message.content.strip()
+            confirmed_names = [name.strip() for name in result.split('\n') if name.strip()]
+            
+            # 再次过滤明显不是人名的词
+            final_names = []
+            for name in confirmed_names:
+                # 跳过太短的
+                if len(name) < 3:
+                    continue
+                # 跳过包含数字的
+                if any(c.isdigit() for c in name):
+                    continue
+                # 跳过全大写的短词
+                if name.isupper() and len(name) < 5:
+                    continue
+                final_names.append(name)
+            
+            return final_names
+        
+        except Exception as e:
+            print(f"LLM精炼失败: {e}")
+            # 返回前10个，但过滤一些明显的错误
+            filtered = []
+            for name in candidate_names[:20]:
+                if len(name) >= 3 and not name.isdigit() and name not in ['Chapter', 'Book', 'Part']:
+                    filtered.append(name)
+            return filtered[:10]
+    
+    def extract_main_characters(self, chunks: List[Dict], 
+                               text_sample: str = None,
+                               language: str = "en") -> List[Dict]:
+        """提取主要角色的完整流程
+        
+        Args:
+            chunks: 文本块列表
+            text_sample: 文本样本（用于LLM精炼）
+            language: 语言类型
+            
+        Returns:
+            主要角色列表
+        """
+        
+        print("\n" + "="*60)
+        print("开始角色提取流程")
+        print("="*60)
+        
+        # 1. 从文本块中提取所有名字
+        character_mentions = self.extract_characters_from_chunks(chunks, language)
+        print(f"\n初步提取到 {len(character_mentions)} 个名字")
+        
+        # 2. 过滤低频角色
+        filtered = {name: info for name, info in character_mentions.items() 
+                   if info['count'] >= Config.MIN_CHARACTER_MENTIONS}
+        print(f"过滤后剩余 {len(filtered)} 个角色（出现次数≥{Config.MIN_CHARACTER_MENTIONS}）")
+        
+        if len(filtered) == 0:
+            print("⚠️  没有找到符合条件的角色，降低阈值...")
+            # 降低阈值重试
+            filtered = {name: info for name, info in character_mentions.items() 
+                       if info['count'] >= max(3, Config.MIN_CHARACTER_MENTIONS // 2)}
+            print(f"降低阈值后找到 {len(filtered)} 个角色")
+        
+        # 3. 排序
+        ranked = self.rank_characters(filtered)
+        print(f"排序后前 {len(ranked)} 个角色")
+        
+        # 4. 合并相似名字
+        merged = self.merge_similar_names(ranked)
+        print(f"合并相似名字后：{len(merged)} 个角色")
+        
+        # 5. LLM精炼（可选）
+        if text_sample and len(merged) > 15:
+            print("\n使用 LLM 精炼角色列表...")
+            candidate_names = [name for name, _ in merged[:30]]
+            try:
+                confirmed = self.refine_with_llm(text_sample, candidate_names)
+                
+                if confirmed:
+                    # 只保留确认的角色
+                    final_characters = []
+                    for name, info in merged:
+                        if any(confirmed_name in name or name in confirmed_name 
+                              for confirmed_name in confirmed):
+                            final_characters.append({'name': name, 'info': info})
+                    
+                    if final_characters:
+                        print(f"LLM确认后：{len(final_characters)} 个主要角色")
+                    else:
+                        print("LLM确认后没有角色，使用原始列表")
+                        final_characters = [{'name': name, 'info': info} for name, info in merged]
+                else:
+                    print("LLM未返回结果，使用原始列表")
+                    final_characters = [{'name': name, 'info': info} for name, info in merged]
+            except Exception as e:
+                print(f"LLM精炼出错: {e}，使用原始列表")
+                final_characters = [{'name': name, 'info': info} for name, info in merged]
+        else:
+            final_characters = [{'name': name, 'info': info} for name, info in merged]
+        
+        # 6. 最终过滤：移除明显不是人名的
+        cleaned_characters = []
+        for char in final_characters:
+            name = char['name']
+            # 跳过明显的非人名
+            if name in ['Chapter', 'Book', 'Part', 'Section', 'The', 'And', 'But', 'When', 'Where']:
+                continue
+            if name.lower() in ['chapter', 'book', 'part', 'section']:
+                continue
+            cleaned_characters.append(char)
+        
+        if len(cleaned_characters) < len(final_characters):
+            print(f"最终清理后：{len(cleaned_characters)} 个角色")
+            final_characters = cleaned_characters
+        
+        # 打印角色列表
+        print("\n" + "="*60)
+        print("主要角色列表（按重要性排序）")
+        print("="*60)
+        for i, char in enumerate(final_characters[:15], 1):
+            print(f"{i}. {char['name']:<20} - 出现{char['info']['count']}次, "
+                  f"分布在{len(char['info']['chunks'])}个章节")
+        
+        if not final_characters:
+            print("\n⚠️  警告：未找到任何角色！")
+            print("建议：")
+            print("  1. 检查文本格式是否正确")
+            print("  2. 确认文本语言设置")
+            print("  3. 尝试降低 MIN_CHARACTER_MENTIONS 参数")
+        
+        return final_characters

core/memory_manager.py

@@ -0,0 +1,168 @@
+import os
+import chromadb
+from chromadb.config import Settings
+from typing import List, Dict, Optional
+from config import Config
+from core.openai_client import OpenAIClient
+
+class MemoryManager:
+    """向量记忆管理器 - 存储和检索角色相关的文本片段"""
+    
+    def __init__(self, character_name: str):
+        self.character_name = character_name
+        self.client = OpenAIClient.get_client()
+        
+        # 创建向量数据库目录
+        os.makedirs(Config.VECTOR_DB_PATH, exist_ok=True)
+        
+        try:
+            self.chroma_client = chromadb.Client(Settings(
+                persist_directory=Config.VECTOR_DB_PATH,
+                anonymized_telemetry=False
+            ))
+        except:
+            # 如果上面的方式失败，尝试使用 PersistentClient
+            self.chroma_client = chromadb.PersistentClient(
+                path=Config.VECTOR_DB_PATH
+            )
+        
+        # 为每个角色创建独立的集合
+        collection_name = f"char_{character_name.replace(' ', '_').lower()}"
+        collection_name = collection_name[:63]  # ChromaDB 限制集合名长度
+        
+        try:
+            self.collection = self.chroma_client.get_or_create_collection(
+                name=collection_name,
+                metadata={"character": character_name}
+            )
+        except Exception as e:
+            print(f"创建集合时出错: {e}")
+            # 如果创建失败，尝试使用更简单的名称
+            collection_name = f"char_{hash(character_name) % 10000}"
+            self.collection = self.chroma_client.get_or_create_collection(
+                name=collection_name,
+                metadata={"character": character_name}
+            )
+    
+    def add_text_chunks(self, chunks: List[Dict], character_chunks: List[int]):
+        """添加与角色相关的文本块
+        
+        Args:
+            chunks: 所有文本块
+            character_chunks: 角色出现的文本块ID列表
+        """
+        
+        documents = []
+        metadatas = []
+        ids = []
+        
+        for chunk_id in character_chunks:
+            if chunk_id < len(chunks):
+                chunk = chunks[chunk_id]
+                documents.append(chunk['text'])
+                metadatas.append({
+                    'chunk_id': chunk_id,
+                    'position': chunk['start']
+                })
+                ids.append(f"chunk_{chunk_id}")
+        
+        if documents:
+            try:
+                # 批量添加，避免一次性添加太多
+                batch_size = 100
+                for i in range(0, len(documents), batch_size):
+                    batch_docs = documents[i:i+batch_size]
+                    batch_metas = metadatas[i:i+batch_size]
+                    batch_ids = ids[i:i+batch_size]
+                    
+                    self.collection.add(
+                        documents=batch_docs,
+                        metadatas=batch_metas,
+                        ids=batch_ids
+                    )
+                
+                print(f"已为 {self.character_name} 添加 {len(documents)} 个文本块到向量库")
+            except Exception as e:
+                print(f"添加文本块到向量库失败: {e}")
+                print("将继续运行，但不使用记忆功能")
+    
+    def search_relevant_context(self, query: str, n_results: int = None) -> List[str]:
+        """检索与查询相关的上下文
+        
+        Args:
+            query: 查询文本
+            n_results: 返回结果数量
+            
+        Returns:
+            相关文本片段列表
+        """
+        
+        n_results = n_results or Config.MAX_MEMORY_RETRIEVAL
+        
+        try:
+            collection_count = self.collection.count()
+            if collection_count == 0:
+                return []
+            
+            actual_n_results = min(n_results, collection_count)
+            
+            results = self.collection.query(
+                query_texts=[query],
+                n_results=actual_n_results
+            )
+            
+            if results and results['documents']:
+                return results['documents'][0]
+            return []
+        
+        except Exception as e:
+            print(f"检索失败: {e}")
+            return []
+    
+    def get_embedding(self, text: str) -> List[float]:
+        """获取文本嵌入向量
+        
+        Args:
+            text: 输入文本
+            
+        Returns:
+            嵌入向量
+        """
+        try:
+            response = self.client.embeddings.create(
+                model=Config.EMBEDDING_MODEL,
+                input=text
+            )
+            return response.data[0].embedding
+        except Exception as e:
+            print(f"获取嵌入失败: {e}")
+            return []
+    
+    def get_statistics(self) -> Dict:
+        """获取记忆库统计信息
+        
+        Returns:
+            统计信��字典
+        """
+        try:
+            count = self.collection.count()
+            return {
+                'character': self.character_name,
+                'chunk_count': count,
+                'collection_name': self.collection.name
+            }
+        except:
+            return {
+                'character': self.character_name,
+                'chunk_count': 0,
+                'collection_name': 'unknown'
+            }
+    
+    def clear(self):
+        """清空记忆库"""
+        try:
+            # 删除集合
+            self.chroma_client.delete_collection(self.collection.name)
+            print(f"已清空 {self.character_name} 的记忆库")
+        except Exception as e:
+            print(f"清空记忆库失败: {e}")

core/openai_client.py

@@ -0,0 +1,30 @@
+from openai import OpenAI
+from config import Config
+
+class OpenAIClient:
+    """OpenAI客户端单例"""
+    
+    _instance = None
+    _client = None
+    
+    def __new__(cls):
+        if cls._instance is None:
+            cls._instance = super(OpenAIClient, cls).__new__(cls)
+        return cls._instance
+    
+    def __init__(self):
+        if self._client is None:
+            self._client = OpenAI(
+                api_key=Config.OPENAI_API_KEY,
+                base_url=Config.OPENAI_BASE_URL
+            )
+    
+    @property
+    def client(self):
+        return self._client
+    
+    @classmethod
+    def get_client(cls) -> OpenAI:
+        """获取OpenAI客户端实例"""
+        instance = cls()
+        return instance.client

core/text_processor.py

@@ -0,0 +1,429 @@
+import re
+from typing import List, Dict, Tuple  # 确保有 Dict
+from tqdm import tqdm
+from utils.text_utils import TextUtils
+from config import Config
+
+
+class TextProcessor:
+    """大规模文本处理器"""
+    
+    def __init__(self):
+        self.text_utils = TextUtils()
+    
+    def chunk_text(self, text: str, chunk_size: int = None, 
+                   overlap: int = None) -> List[Dict]:
+        """将长文本分块，保持语义完整性
+        
+        Args:
+            text: 输入文本
+            chunk_size: 每块的最大字符数
+            overlap: 块之间的重叠字符数
+            
+        Returns:
+            分块结果列表，每个元素包含 text, start, end, chunk_id
+        """
+        chunk_size = chunk_size or Config.MAX_CHUNK_SIZE
+        overlap = overlap or Config.CHUNK_OVERLAP
+        
+        # 先按段落分割
+        paragraphs = text.split('\n\n')
+        
+        chunks = []
+        current_chunk = ""
+        current_start = 0
+        total_processed = 0
+        
+        print(f"开始分块处理 (块大小: {chunk_size}, 重叠: {overlap})...")
+        
+        for para in tqdm(paragraphs, desc="分块进度"):
+            para = para.strip()
+            if not para:
+                continue
+            
+            # 如果当前块加上新段落超过限制
+            if len(current_chunk) + len(para) + 2 > chunk_size:  # +2 for \n\n
+                if current_chunk:
+                    # 保存当前块
+                    chunks.append({
+                        'text': current_chunk.strip(),
+                        'start': current_start,
+                        'end': current_start + len(current_chunk),
+                        'chunk_id': len(chunks)
+                    })
+                    
+                    # 计算重叠部分
+                    if len(current_chunk) > overlap:
+                        # 从当前块末尾取重叠部分
+                        overlap_text = current_chunk[-overlap:]
+                        # 尝试在句子边界处分割
+                        sentences = self.text_utils.split_into_sentences(overlap_text)
+                        if sentences:
+                            overlap_text = sentences[-1] if len(sentences) == 1 else ' '.join(sentences[-2:])
+                    else:
+                        overlap_text = current_chunk
+                    
+                    # 更新起始位置
+                    total_processed += len(current_chunk) - len(overlap_text)
+                    current_start = total_processed
+                    
+                    # 开始新块
+                    current_chunk = overlap_text + "\n\n" + para
+                else:
+                    # 当前块为空，直接使用新段落
+                    current_chunk = para
+                    current_start = total_processed
+            else:
+                # 添加到当前块
+                if current_chunk:
+                    current_chunk += "\n\n" + para
+                else:
+                    current_chunk = para
+        
+        # 添加最后一块
+        if current_chunk:
+            chunks.append({
+                'text': current_chunk.strip(),
+                'start': current_start,
+                'end': current_start + len(current_chunk),
+                'chunk_id': len(chunks)
+            })
+        
+        print(f"✓ 文本分块完成: 总共 {len(chunks)} 块")
+        return chunks
+    
+    def chunk_text_by_tokens(self, text: str, max_tokens: int = 1500,
+                            overlap_tokens: int = 150) -> List[Dict]:
+        """按 token 数量分块（更精确但较慢）
+        
+        Args:
+            text: 输入文本
+            max_tokens: 每块的最大 token 数
+            overlap_tokens: 重叠的 token 数
+            
+        Returns:
+            分块结果列表
+        """
+        sentences = self.text_utils.split_into_sentences(text)
+        
+        chunks = []
+        current_chunk = []
+        current_tokens = 0
+        current_start = 0
+        
+        print(f"按 token 分块处理 (最大: {max_tokens} tokens)...")
+        
+        for sentence in tqdm(sentences, desc="处理句子"):
+            sentence_tokens = self.text_utils.count_tokens(sentence)
+            
+            if current_tokens + sentence_tokens > max_tokens and current_chunk:
+                # 保存当前块
+                chunk_text = ' '.join(current_chunk)
+                chunks.append({
+                    'text': chunk_text,
+                    'start': current_start,
+                    'end': current_start + len(chunk_text),
+                    'chunk_id': len(chunks),
+                    'token_count': current_tokens
+                })
+                
+                # 处理重叠
+                overlap_chunk = []
+                overlap_tokens_count = 0
+                for s in reversed(current_chunk):
+                    s_tokens = self.text_utils.count_tokens(s)
+                    if overlap_tokens_count + s_tokens <= overlap_tokens:
+                        overlap_chunk.insert(0, s)
+                        overlap_tokens_count += s_tokens
+                    else:
+                        break
+                
+                current_chunk = overlap_chunk + [sentence]
+                current_tokens = overlap_tokens_count + sentence_tokens
+                current_start += len(chunk_text) - len(' '.join(overlap_chunk))
+            else:
+                current_chunk.append(sentence)
+                current_tokens += sentence_tokens
+        
+        # 添加最后一块
+        if current_chunk:
+            chunk_text = ' '.join(current_chunk)
+            chunks.append({
+                'text': chunk_text,
+                'start': current_start,
+                'end': current_start + len(chunk_text),
+                'chunk_id': len(chunks),
+                'token_count': current_tokens
+            })
+        
+        print(f"✓ Token 分块完成: 总共 {len(chunks)} 块")
+        return chunks
+    
+    def extract_dialogues(self, text: str) -> List[Dict]:
+        """提取对话片段
+        
+        Args:
+            text: 输入文本
+            
+        Returns:
+            对话列表，每个元素包含 content, attribution, position
+        """
+        # 检测语言
+        language = self.text_utils.detect_language(text)
+        
+        dialogues = []
+        
+        if language == "zh":
+            # 中文对话模式
+            patterns = [
+                (r'"([^"]+)"[,，]?\s*([^说道讲告诉问答叫喊]*(?:说|道|讲|告诉|问|答|叫|喊))', 'chinese_quote'),
+                (r'「([^」]+)」[,，]?\s*([^说道讲]*(?:说|道|讲))', 'chinese_bracket'),
+                (r'"([^"]+)"', 'simple_quote'),
+            ]
+        else:
+            # 英文对话模式
+            patterns = [
+                (r'"([^"]+)",?\s+([A-Z][a-z]+\s+(?:said|asked|replied|shouted|whispered|muttered|exclaimed))', 'english_quote_said'),
+                (r'"([^"]+)"', 'simple_quote'),
+                (r"'([^']+)',?\s+([A-Z][a-z]+\s+said)", 'english_single_quote'),
+            ]
+        
+        for pattern, pattern_type in patterns:
+            matches = re.finditer(pattern, text, re.IGNORECASE)
+            for match in matches:
+                dialogue = {
+                    'content': match.group(1).strip(),
+                    'attribution': match.group(2).strip() if len(match.groups()) > 1 else '',
+                    'position': match.start(),
+                    'type': pattern_type
+                }
+                # 过滤太短的对话
+                if len(dialogue['content']) > 5:
+                    dialogues.append(dialogue)
+        
+        # 按位置排序
+        dialogues.sort(key=lambda x: x['position'])
+        
+        return dialogues
+    
+    def split_by_chapters(self, text: str) -> List[Dict]:
+        """按章节分割文本
+        
+        Args:
+            text: 输入文本
+            
+        Returns:
+            章节列表，每个元素包含 title, content, chapter_num
+        """
+        # 检测章节标记模式
+        chapter_patterns = [
+            r'Chapter\s+(\d+)[:\s]*([^\n]*)',  # English: Chapter 1: Title
+            r'第([一二三四五六七八九十百千零\d]+)章[:\s]*([^\n]*)',  # Chinese: 第一章：标题
+            r'CHAPTER\s+([IVXLCDM]+)[:\s]*([^\n]*)',  # Roman numerals
+        ]
+        
+        chapters = []
+        last_pos = 0
+        
+        for pattern in chapter_patterns:
+            matches = list(re.finditer(pattern, text, re.IGNORECASE | re.MULTILINE))
+            
+            if matches:
+                for i, match in enumerate(matches):
+                    start = match.start()
+                    end = matches[i + 1].start() if i + 1 < len(matches) else len(text)
+                    
+                    chapters.append({
+                        'chapter_num': match.group(1),
+                        'title': match.group(2).strip() if len(match.groups()) > 1 else '',
+                        'content': text[start:end].strip(),
+                        'start': start,
+                        'end': end
+                    })
+                break  # 找到匹配的模式就停止
+        
+        # 如果没找到章节，返回整个文本作为一章
+        if not chapters:
+            chapters.append({
+                'chapter_num': '1',
+                'title': 'Full Text',
+                'content': text,
+                'start': 0,
+                'end': len(text)
+            })
+        
+        return chapters
+    
+    def get_statistics(self, text: str) -> Dict:
+        """获取文本统计信息
+        
+        Args:
+            text: 输入文本
+            
+        Returns:
+            统计信息字典
+        """
+        # 基本统计
+        total_length = len(text)
+        total_tokens = self.text_utils.count_tokens(text)
+        
+        # 段落统计
+        paragraphs = [p for p in text.split('\n\n') if p.strip()]
+        paragraph_count = len(paragraphs)
+        
+        # 句子统计
+        sentences = self.text_utils.split_into_sentences(text)
+        sentence_count = len(sentences)
+        
+        # 单词/字符统计
+        words = re.findall(r'\b\w+\b', text)
+        word_count = len(words)
+        
+        # 语言检测
+        language = self.text_utils.detect_language(text)
+        
+        # 对话统计
+        dialogues = self.extract_dialogues(text[:10000])  # 只检查前10000字符
+        dialogue_count = len(dialogues)
+        
+        # 章节检测
+        chapters = self.split_by_chapters(text)
+        chapter_count = len(chapters)
+        
+        return {
+            'total_length': total_length,
+            'total_tokens': total_tokens,
+            'paragraphs': paragraph_count,
+            'sentences': sentence_count,
+            'words': word_count,
+            'language': language,
+            'dialogues': dialogue_count,
+            'chapters': chapter_count,
+            'avg_paragraph_length': total_length // paragraph_count if paragraph_count > 0 else 0,
+            'avg_sentence_length': total_length // sentence_count if sentence_count > 0 else 0,
+        }
+    
+    def clean_text(self, text: str, 
+                   remove_extra_whitespace: bool = True,
+                   normalize_quotes: bool = True) -> str:
+        """清理文本
+        
+        Args:
+            text: 输入文本
+            remove_extra_whitespace: 是否移除多余空白
+            normalize_quotes: 是否标准化引号
+            
+        Returns:
+            清理后的文本
+        """
+        cleaned = text
+        
+        # 移除多余空白
+        if remove_extra_whitespace:
+            # 移除行首行尾空白
+            cleaned = '\n'.join(line.strip() for line in cleaned.split('\n'))
+            # 合并多个空行为一个
+            cleaned = re.sub(r'\n{3,}', '\n\n', cleaned)
+            # 移除制表符
+            cleaned = cleaned.replace('\t', ' ')
+            # 合并多个空格
+            cleaned = re.sub(r' {2,}', ' ', cleaned)
+        
+        # 标准化引号
+        if normalize_quotes:
+            # 中文引号统一为 ""
+            cleaned = cleaned.replace('『', '"').replace('』', '"')
+            cleaned = cleaned.replace('「', '"').replace('」', '"')
+            # 英文引号统一为 ""
+            cleaned = cleaned.replace('"', '"').replace('"', '"')
+            cleaned = cleaned.replace(''', "'").replace(''', "'")
+        
+        return cleaned
+    
+    def extract_metadata(self, text: str) -> Dict:
+        """提取文本元数据（标题、作者等）
+        
+        Args:
+            text: 输入文本
+            
+        Returns:
+            元数据字典
+        """
+        metadata = {
+            'title': None,
+            'author': None,
+            'year': None,
+        }
+        
+        # 尝试从文本开头提取标题和作者
+        lines = text.split('\n')[:20]  # 只看前20行
+        
+        for line in lines:
+            line = line.strip()
+            
+            # 尝试匹配标题
+            if not metadata['title'] and len(line) > 5 and len(line) < 100:
+                # 如果是全大写或标题格式
+                if line.isupper() or line.istitle():
+                    metadata['title'] = line
+            
+            # 尝试匹配作者
+            author_patterns = [
+                r'by\s+([A-Z][a-z]+(?:\s+[A-Z][a-z]+)*)',
+                r'作者[：:]\s*(.+)',
+                r'Author[:\s]+(.+)',
+            ]
+            
+            for pattern in author_patterns:
+                match = re.search(pattern, line, re.IGNORECASE)
+                if match:
+                    metadata['author'] = match.group(1).strip()
+                    break
+            
+            # 尝试匹配年份
+            year_match = re.search(r'\b(19|20)\d{2}\b', line)
+            if year_match:
+                metadata['year'] = year_match.group(0)
+        
+        return metadata
+    
+    def sample_text(self, text: str, sample_size: int = 1000, 
+                   strategy: str = 'random') -> str:
+        """从文本中采样
+        
+        Args:
+            text: 输入文本
+            sample_size: 采样大小（字符数）
+            strategy: 采样策略 ('start', 'random', 'distributed')
+            
+        Returns:
+            采样的文本
+        """
+        if len(text) <= sample_size:
+            return text
+        
+        if strategy == 'start':
+            # 从开头采样
+            return text[:sample_size]
+        
+        elif strategy == 'random':
+            # 随机位置采样
+            import random
+            start = random.randint(0, len(text) - sample_size)
+            return text[start:start + sample_size]
+        
+        elif strategy == 'distributed':
+            # 分布式采样（从文本的不同部分采样）
+            num_samples = 3
+            sample_per_part = sample_size // num_samples
+            samples = []
+            
+            for i in range(num_samples):
+                start = (len(text) // num_samples) * i
+                end = min(start + sample_per_part, len(text))
+                samples.append(text[start:end])
+            
+            return '\n...\n'.join(samples)
+        
+        else:
+            return text[:sample_size]

main.py

@@ -0,0 +1,367 @@
+import os
+import sys
+from tqdm import tqdm
+from core.text_processor import TextProcessor
+from core.character_extractor import CharacterExtractor
+from core.character_analyzer import CharacterAnalyzer
+from core.character_agent import CharacterAgent
+from utils.text_utils import TextUtils
+from utils.cache_manager import CacheManager
+from config import Config
+
+def print_banner():
+    """打印欢迎横幅"""
+    banner = """
+╔══════════════════════════════════════════════════════════════════╗
+║                                                                  ║
+║           🎭 小说角色 Agent 系统 (大规模文本版)                    ║
+║                                                                  ║
+║              基于 AI 的角色性格分析与对话系统                      ║
+║                                                                  ║
+╚══════════════════════════════════════════════════════════════════╝
+"""
+    print(banner)
+
+def load_novel(file_path: str) -> str:
+    """加载小说文本"""
+    try:
+        # 尝试不同的编码
+        encodings = ['utf-8', 'gbk', 'gb2312', 'latin-1', 'utf-16']
+        
+        for encoding in encodings:
+            try:
+                with open(file_path, 'r', encoding=encoding) as f:
+                    text = f.read()
+                print(f"✓ 成功使用 {encoding} 编码加载文件")
+                return text
+            except UnicodeDecodeError:
+                continue
+        
+        print("✗ 所有编码尝试失败")
+        return ""
+    
+    except FileNotFoundError:
+        print(f"✗ 文件不存在: {file_path}")
+        return ""
+    except Exception as e:
+        print(f"✗ 加载小说失败: {e}")
+        return ""
+
+def display_statistics(stats: dict):
+    """显示文本统计信息"""
+    print("\n" + "="*70)
+    print("📊 文本统计信息")
+    print("="*70)
+    print(f"总字符数:    {stats['total_length']:,}")
+    print(f"Token 数量:  {stats['total_tokens']:,}")
+    print(f"段落数:      {stats['paragraphs']:,}")
+    print(f"句子数:      {stats['sentences']:,}")
+    
+    lang_map = {'zh': '中文', 'en': '英文', 'mixed': '中英混合', 'unknown': '未知'}
+    print(f"检测语言:    {lang_map.get(stats['language'], stats['language'])}")
+    
+    # 估算阅读时间
+    text_utils = TextUtils()
+    reading_time = text_utils.estimate_reading_time(" " * stats['total_length'])
+    print(f"预计阅读:    约 {reading_time} 分钟")
+    print("="*70)
+
+def select_character_interactive(characters: list) -> dict:
+    """交互式选择角色"""
+    print("\n" + "="*70)
+    print("📋 检测到的主要角色")
+    print("="*70)
+    print(f"{'序号':<6}{'角色名':<25}{'出现次数':<12}{'分布章节':<12}")
+    print("-"*70)
+    
+    for i, char in enumerate(characters[:15], 1):
+        name = char['name']
+        count = char['info']['count']
+        chunks = len(char['info']['chunks'])
+        print(f"{i:<6}{name:<25}{count:<12}{chunks:<12}")
+    
+    print("="*70)
+    
+    while True:
+        try:
+            choice = input(f"\n请选择角色编号 (1-{min(15, len(characters))}): ").strip()
+            
+            if choice.isdigit():
+                idx = int(choice) - 1
+                if 0 <= idx < len(characters):
+                    return characters[idx]
+            
+            print("❌ 无效选择，请重试")
+        except KeyboardInterrupt:
+            print("\n\n👋 程序已退出")
+            sys.exit(0)
+        except:
+            print("❌ 输入错误，请重试")
+
+def interactive_chat(agent: CharacterAgent):
+    """交互式对话界面"""
+    
+    print("\n" + "="*70)
+    print(agent.get_character_info())
+    print("\n💬 对话开始！")
+    print("-"*70)
+    print("💡 提示：")
+    print("  • 输入 'quit' 或 'exit' - 退出对话")
+    print("  • 输入 'reset' - 重置对话历史")
+    print("  • 输入 'save' - 保存对话")
+    print("  • 输入 'info' - 查看角色信息")
+    print("  • 输入 'help' - 显示帮助")
+    print("="*70 + "\n")
+    
+    char_name = agent.character_profile['name']
+    
+    while True:
+        try:
+            # 用户输入
+            user_input = input("🧑 你: ").strip()
+            
+            if not user_input:
+                continue
+            
+            # 处理命令
+            if user_input.lower() in ['quit', 'exit', '退出', 'q']:
+                print(f"\n👋 {char_name}: 再见，很高兴和你聊天！")
+                
+                # 询问是否保存对话
+                if len(agent.conversation_history) > 0:
+                    save = input("\n是否保��对话记录？(y/n): ").strip().lower()
+                    if save in ['y', 'yes', '是']:
+                        filename = f"conversation_{char_name}_{len(agent.conversation_history)}.json"
+                        agent.save_conversation(filename)
+                break
+            
+            if user_input.lower() in ['reset', '重置']:
+                agent.reset_conversation()
+                continue
+            
+            if user_input.lower() in ['save', '保存']:
+                filename = f"conversation_{char_name}_{len(agent.conversation_history)}.json"
+                agent.save_conversation(filename)
+                continue
+            
+            if user_input.lower() in ['info', '信息']:
+                print(agent.get_character_info())
+                continue
+            
+            if user_input.lower() in ['help', '帮助']:
+                print("\n可用命令:")
+                print("  quit/exit - 退出对话")
+                print("  reset - 重置对话历史")
+                print("  save - 保存对话")
+                print("  info - 查看角色信息")
+                print("  help - 显示此帮助\n")
+                continue
+            
+            # 正常对话
+            print(f"\n{'⏳ ' + char_name + ' 正在思考...':<70}", end='\r')
+            response = agent.chat(user_input)
+            print(" " * 70, end='\r')  # 清除"思考中"
+            print(f"🎭 {char_name}: {response}\n")
+            
+        except KeyboardInterrupt:
+            print(f"\n\n👋 {char_name}: 再见！")
+            break
+        except Exception as e:
+            print(f"\n❌ 错误: {e}\n")
+
+def check_environment():
+    """检查运行环境"""
+    issues = []
+    
+    # 检查 API Key
+    if not Config.OPENAI_API_KEY or Config.OPENAI_API_KEY == "":
+        issues.append("未设置 OPENAI_API_KEY")
+    
+    # 检查缓存目录
+    if not os.path.exists(Config.CACHE_DIR):
+        try:
+            os.makedirs(Config.CACHE_DIR)
+        except:
+            issues.append(f"无法创建缓存目录: {Config.CACHE_DIR}")
+    
+    # 检查必要的包
+    try:
+        import openai
+        import chromadb
+        import tiktoken
+    except ImportError as e:
+        issues.append(f"缺少必要的包: {e}")
+    
+    if issues:
+        print("\n⚠️  环境检查发现问题:")
+        for issue in issues:
+            print(f"  • {issue}")
+        print("\n请检查配置文件 .env 和依赖安装\n")
+        return False
+    
+    return True
+
+def main():
+    """主函数 - 完整流程"""
+    
+    # 打印横幅
+    print_banner()
+    
+    # 检查环境
+    if not check_environment():
+        return
+    
+    # 显示缓存信息
+    cache = CacheManager()
+    cache_info = cache.get_cache_info()
+    if cache_info['count'] > 0:
+        print(f"📦 缓存: {cache_info['count']} 个文件, {cache_info['size_mb']} MB")
+    
+    # 1. 加载小说
+    print("\n" + "="*70)
+    print("📖 步骤 1/5: 加载小说")
+    print("="*70)
+    
+    default_path = "sample_novels/harry_potter_sample.txt"
+    novel_path = input(f"\n请输入小说文件路径 (默认: {default_path})\n> ").strip()
+    
+    if not novel_path:
+        novel_path = default_path
+    
+    if not os.path.exists(novel_path):
+        print(f"❌ 文件不存在: {novel_path}")
+        
+        # 尝试在 sample_novels 目录下查找
+        alt_path = os.path.join("sample_novels", os.path.basename(novel_path))
+        if os.path.exists(alt_path):
+            print(f"✓ 找到文件: {alt_path}")
+            novel_path = alt_path
+        else:
+            print("程序退出")
+            return
+    
+    print(f"\n正在加载: {novel_path}")
+    novel_text = load_novel(novel_path)
+    
+    if not novel_text:
+        print("❌ 无法加载小说，程序退出")
+        return
+    
+    # 显示统计信息
+    processor = TextProcessor()
+    stats = processor.get_statistics(novel_text)
+    display_statistics(stats)
+    
+    # 检查文本长度
+    if stats['total_length'] < 1000:
+        print("⚠️  警告: 文本过短 (< 1000字符)，可能影响分析效果")
+        proceed = input("是否继续？(y/n): ").strip().lower()
+        if proceed not in ['y', 'yes', '是']:
+            return
+    
+    # 2. 文本分块
+    print("\n" + "="*70)
+    print("📄 步骤 2/5: 文本分块处理")
+    print("="*70)
+    
+    chunks = processor.chunk_text(novel_text)
+    print(f"✓ 文本已分为 {len(chunks)} 个块")
+    print(f"  平均每块: {stats['total_length'] // len(chunks)} 字符")
+    
+    # 3. 提取角色
+    print("\n" + "="*70)
+    print("👥 步骤 3/5: 提取主要角色")
+    print("="*70)
+    
+    extractor = CharacterExtractor()
+    characters = extractor.extract_main_characters(
+        chunks, 
+        text_sample=novel_text[:3000],
+        language=stats['language']
+    )
+    
+    if not characters:
+        print("❌ 未能提取到角色，程序退出")
+        return
+    
+    # 4. 选择角色
+    print("\n" + "="*70)
+    print("🎯 步骤 4/5: 选择要对话的角色")
+    print("="*70)
+    
+    selected = select_character_interactive(characters)
+    character_name = selected['name']
+    character_info = selected['info']
+    
+    print(f"\n✓ 已选择: {character_name}")
+    print(f"  出现次数: {character_info['count']}")
+    print(f"  分布章节: {len(character_info['chunks'])}")
+    
+    # 5. 分析角色
+    print(f"\n" + "="*70)
+    print(f"🧠 步骤 5/5: 分析角色性格")
+    print("="*70)
+    print(f"正在深度分析 {character_name} 的性格特征...")
+    print("这可能需要几分钟，请耐心等待...\n")
+    
+    analyzer = CharacterAnalyzer()
+    
+    # 选择代表性文本块
+    representative_chunks = analyzer.select_representative_chunks(
+        chunks, 
+        character_info['chunks']
+    )
+    
+    print(f"✓ 选取了 {len(representative_chunks)} 个代表性片段进行分析")
+    
+    # 执行分析
+    character_profile = analyzer.analyze_character_batch(
+        character_name, 
+        representative_chunks
+    )
+    
+    # 增强配置
+    character_profile = analyzer.enhance_profile_with_examples(
+        character_profile,
+        chunks,
+        character_info['chunks']
+    )
+    
+    print(f"✓ 角色分析完成！")
+    
+    # 6. 创建对话代理
+    print("\n" + "="*70)
+    print("🤖 创建对话代理")
+    print("="*70)
+    
+    use_memory = input("\n是否启用记忆系统？(y/n, 默认: y): ").strip().lower()
+    
+    if use_memory in ['', 'y', 'yes', '是']:
+        print("正在初始化记忆系统...")
+        agent = CharacterAgent(
+            character_profile,
+            chunks=chunks,
+            character_chunks=character_info['chunks']
+        )
+        print("✓ Agent 创建成功，记忆系统已初始化")
+    else:
+        agent = CharacterAgent(character_profile)
+        print("✓ Agent 创建成功（未启用记忆系统）")
+    
+    # 7. 开始对话
+    interactive_chat(agent)
+    
+    # 结束
+    print("\n" + "="*70)
+    print("感谢使用小说角色 Agent 系统！")
+    print("="*70)
+
+if __name__ == "__main__":
+    try:
+        main()
+    except KeyboardInterrupt:
+        print("\n\n👋 程序已被用户中断")
+    except Exception as e:
+        print(f"\n❌ 程序错误: {e}")
+        import traceback
+        traceback.print_exc()

test_agent.py

@@ -0,0 +1,431 @@
+import unittest
+import os
+import sys
+from core.text_processor import TextProcessor
+from core.character_extractor import CharacterExtractor
+from core.character_analyzer import CharacterAnalyzer
+from core.character_agent import CharacterAgent
+from utils.text_utils import TextUtils
+from utils.cache_manager import CacheManager
+
+class TestLargeScaleAgent(unittest.TestCase):
+    """大规模文本角色Agent测试"""
+    
+    @classmethod
+    def setUpClass(cls):
+        """准备测试数据"""
+        cls.sample_text = """
+Chapter 1: The Boy Who Lived
+
+Mr. and Mrs. Dursley, of number four, Privet Drive, were proud to say that they were perfectly normal, thank you very much. They were the last people you'd expect to be involved in anything strange or mysterious, because they just didn't hold with such nonsense.
+
+Harry Potter rolled over inside his blankets without waking up. One small hand closed on the letter beside him and he slept on, not knowing he was special, not knowing he was famous.
+
+"Harry! Harry Potter!" said Hermione Granger excitedly. She was a clever girl with bushy brown hair.
+
+Ron Weasley grinned at Harry. "Blimey, Harry, everyone knows who you are!" Ron was Harry's best friend, a tall boy with red hair.
+
+Professor Dumbledore smiled wisely. "Harry, you have your mother's eyes," he said gently. Dumbledore was the headmaster of Hogwarts, known for his wisdom and kindness.
+
+Harry looked confused. "I don't understand, Professor," he said quietly. Harry was a brave boy, though he often felt uncertain.
+
+Hermione rolled her eyes. "Honestly, Harry, you need to read more!" she said in her know-it-all voice.
+
+Ron laughed. "Don't worry, mate. We'll figure it out together," he said loyally.
+
+""" * 50  # 重复50次模拟长文本
+        
+        cls.processor = TextProcessor()
+        cls.chunks = cls.processor.chunk_text(cls.sample_text)
+    
+    def test_01_text_utils(self):
+        """测试文本工具"""
+        print("\n" + "="*70)
+        print("测试 1: 文本工具函数")
+        print("="*70)
+        
+        utils = TextUtils()
+        
+        # 测试 token 计数
+        tokens = utils.count_tokens(self.sample_text[:1000])
+        print(f"Token 数量: {tokens}")
+        self.assertGreater(tokens, 0)
+        
+        # 测试语言检测
+        lang = utils.detect_language(self.sample_text)
+        print(f"检测语言: {lang}")
+        self.assertEqual(lang, "en")
+        
+        # 测试句子分割
+        sentences = utils.split_into_sentences(self.sample_text[:500])
+        print(f"句子数量: {len(sentences)}")
+        self.assertGreater(len(sentences), 0)
+        
+        # 测试关键词提取
+        keywords = utils.extract_keywords(self.sample_text[:1000], top_n=5)
+        print(f"关键词: {keywords}")
+        self.assertGreater(len(keywords), 0)
+    
+    def test_02_cache_manager(self):
+        """测试缓存管理器"""
+        print("\n" + "="*70)
+        print("测试 2: 缓存管理器")
+        print("="*70)
+        
+        cache = CacheManager()
+        
+        # 测试缓存设置和获取
+        test_key = "test_key"
+        test_data = {"name": "Harry", "age": 11}
+        
+        cache.set(test_key, test_data)
+        cached_data = cache.get(test_key)
+        
+        print(f"缓存数据: {cached_data}")
+        self.assertEqual(cached_data, test_data)
+        
+        # 测试缓存存在性
+        self.assertTrue(cache.exists(test_key))
+        
+        # 测试 JSON 缓存
+        cache.save_json("test_json", test_data)
+        json_data = cache.load_json("test_json")
+        print(f"JSON 数据: {json_data}")
+        self.assertEqual(json_data, test_data)
+        
+        # 测试缓存信息
+        info = cache.get_cache_info()
+        print(f"缓存信息: {info}")
+        self.assertGreater(info['count'], 0)
+        
+        # 清理测试缓存
+        cache.delete(test_key)
+    
+    def test_03_text_processing(self):
+        """测试文本处理"""
+        print("\n" + "="*70)
+        print("测试 3: 文本处理和分块")
+        print("="*70)
+        
+        stats = self.processor.get_statistics(self.sample_text)
+        
+        print(f"总字符数: {stats['total_length']:,}")
+        print(f"Token数: {stats['total_tokens']:,}")
+        print(f"段落数: {stats['paragraphs']}")
+        print(f"句子数: {stats['sentences']}")
+        print(f"语言: {stats['language']}")
+        print(f"分块数: {len(self.chunks)}")
+        
+        self.assertGreater(len(self.chunks), 0)
+        self.assertGreater(stats['total_length'], 1000)
+        self.assertEqual(stats['language'], 'en')
+    
+    def test_04_character_extraction(self):
+        """测试角色提取"""
+        print("\n" + "="*70)
+        print("测试 4: 角色提取")
+        print("="*70)
+        
+        extractor = CharacterExtractor()
+        characters = extractor.extract_main_characters(
+            self.chunks,
+            text_sample=self.sample_text[:1000],
+            language="en"
+        )
+        
+        print(f"\n提取到 {len(characters)} 个主要角色:")
+        for i, char in enumerate(characters[:5], 1):
+            print(f"  {i}. {char['name']}: 出现 {char['info']['count']} 次")
+        
+        self.assertGreater(len(characters), 0)
+        
+        # 检查是否包含预期角色
+        char_names = [c['name'] for c in characters]
+        expected_chars = ['Harry', 'Hermione', 'Ron']
+        
+        for expected in expected_chars:
+            found = any(expected in name for name in char_names)
+            if found:
+                print(f"✓ 找到角色: {expected}")
+            self.assertTrue(found, f"应该找到角色: {expected}")
+    
+    def test_05_character_analysis(self):
+        """测试角色分析"""
+        print("\n" + "="*70)
+        print("测试 5: 角色性格分析")
+        print("="*70)
+        
+        extractor = CharacterExtractor()
+        characters = extractor.extract_main_characters(
+            self.chunks,
+            language="en"
+        )
+        
+        if not characters:
+            self.skipTest("没有提取到角色")
+        
+        # 分析第一个角色
+        character = characters[0]
+        print(f"\n分析角色: {character['name']}")
+        
+        analyzer = CharacterAnalyzer()
+        representative_chunks = analyzer.select_representative_chunks(
+            self.chunks,
+            character['info']['chunks'][:10]
+        )
+        
+        print(f"选择了 {len(representative_chunks)} 个代表性片段")
+        
+        profile = analyzer.analyze_character_batch(
+            character['name'],
+            representative_chunks
+        )
+        
+        print(f"\n角色档案:")
+        print(f"  名字: {profile['name']}")
+        print(f"  核心特质: {profile.get('core_traits', [])}")
+        print(f"  说话风格: {profile.get('speaking_style', 'N/A')}")
+        print(f"  性格总结: {profile.get('personality_summary', 'N/A')[:100]}...")
+        
+        self.assertEqual(profile['name'], character['name'])
+        self.assertIn('core_traits', profile)
+        self.assertIn('speaking_style', profile)
+    
+    def test_06_agent_creation(self):
+        """测试Agent创建"""
+        print("\n" + "="*70)
+        print("测试 6: 创建对话Agent")
+        print("="*70)
+        
+        test_profile = {
+            'name': 'Harry Potter',
+            'core_traits': ['勇敢', '善良', '忠诚', '谦逊'],
+            'speaking_style': '谦逊、真诚，有时会感到困惑',
+            'behavior_patterns': '面对困难时表现出勇气',
+            'values': '友谊、正义、牺牲精神',
+            'emotional_style': '内敛但情感丰富',
+            'relationship_style': '忠诚于朋友',
+            'background': '年轻的巫师，霍格沃茨学生',
+            'personality_summary': 'Harry是一个勇敢善良的年轻巫师',
+            'key_quotes': ['I don\'t understand', 'What do you mean?']
+        }
+        
+        agent = CharacterAgent(test_profile)
+        
+        print("\n" + agent.get_character_info())
+        
+        self.assertIsNotNone(agent)
+        self.assertEqual(agent.character_profile['name'], 'Harry Potter')
+        self.assertIsNotNone(agent.system_prompt)
+    
+    def test_07_conversation(self):
+        """测试对话功能"""
+        print("\n" + "="*70)
+        print("测试 7: 对话测试")
+        print("="*70)
+        
+        test_profile = {
+            'name': 'Hermione Granger',
+            'core_traits': ['聪明', '好学', '正直', '有时完美主义'],
+            'speaking_style': '自信、条理清晰，有时显得严肃',
+            'behavior_patterns': '总是提前完成作业，遇到问题查阅书籍',
+            'values': '知识就是力量，规则应该被遵守',
+            'emotional_style': '理性为主，但也有感性的一面',
+            'relationship_style': '对朋友忠诚，但不容忍懒惰',
+            'background': '麻瓜出身的女巫，格兰芬多学生',
+            'personality_summary': 'Hermione是个极其聪明、勤奋的学生',
+            'key_quotes': ['Honestly!', 'You need to read more!', 'It\'s in Hogwarts: A History']
+        }
+        
+        agent = CharacterAgent(test_profile)
+        
+        test_messages = [
+            "Hi Hermione, how are you?",
+            "Can you help me with my homework?",
+            "Do you ever just relax?"
+        ]
+        
+        for msg in test_messages:
+            print(f"\n🧑 你: {msg}")
+            response = agent.chat(msg, use_memory=False)
+            print(f"🎭 Hermione: {response}")
+            
+            self.assertIsInstance(response, str)
+            self.assertGreater(len(response), 0)
+        
+        print(f"\n对话历史: {len(agent.conversation_history)} 条")
+        self.assertGreater(len(agent.conversation_history), 0)
+    
+    def test_08_conversation_management(self):
+        """测试对话管理功能"""
+        print("\n" + "="*70)
+        print("测试 8: 对话管理")
+        print("="*70)
+        
+        test_profile = {
+            'name': 'Ron Weasley',
+            'core_traits': ['忠诚', '幽默', '勇敢'],
+            'speaking_style': '随意、幽默',
+            'personality_summary': 'Ron是Harry的忠实朋友',
+            'key_quotes': ['Blimey!', 'Brilliant!']
+        }
+        
+        agent = CharacterAgent(test_profile)
+        
+        # 进行对话
+        agent.chat("Hello Ron!")
+        agent.chat("How's it going?")
+        
+        initial_count = len(agent.conversation_history)
+        print(f"对话数量: {initial_count}")
+        self.assertGreater(initial_count, 0)
+        
+        # 测试重置
+        agent.reset_conversation()
+        print(f"重置后对话数量: {len(agent.conversation_history)}")
+        self.assertEqual(len(agent.conversation_history), 0)
+        
+        # 测试保存和加载
+        agent.chat("Test message")
+        test_file = "test_conversation.json"
+        agent.save_conversation(test_file)
+        
+        self.assertTrue(os.path.exists(test_file))
+        
+        # 清理
+        if os.path.exists(test_file):
+            os.remove(test_file)
+
+def run_quick_demo():
+    """快速演示"""
+    print("\n" + "="*70)
+    print("🎭 快速演示：与 Hermione Granger 对话")
+    print("="*70)
+    
+    profile = {
+        'name': 'Hermione Granger',
+        'core_traits': ['聪明绝顶', '勤奋好学', '正义感强', '有时完美主义'],
+        'speaking_style': '自信、条理清晰，喜欢引用书本知识，有时显得严肃',
+        'behavior_patterns': '总是提前完成作业，遇到问题首先想到查阅书籍',
+        'values': '知识就是力量，规则应该被遵守',
+        'emotional_style': '理性为主，但面对朋友会展现温暖的一面',
+        'relationship_style': '对朋友忠诚，但不容忍懒惰和不负责任',
+        'background': '麻瓜出身的年轻女巫，格兰芬多学院学生',
+        'key_quotes': [
+            'Honestly, don\'t you two read?',
+            'It\'s LeviOsa, not LeviosA!',
+            'When in doubt, go to the library.',
+            'Fear of a name only increases fear of the thing itself.'
+        ],
+        'personality_summary': 'Hermione是一个极其聪明、勤奋的学生，对知识充满渴望。她有强烈的正义感，虽然有时显得有点自以为是，但内心深处对朋友极其忠诚。'
+    }
+    
+    agent = CharacterAgent(profile)
+    print(agent.get_character_info())
+    
+    demo_conversations = [
+        "Hi Hermione! How's your day going?",
+        "I'm really struggling with my Potions homework...",
+        "You always seem so confident. Don't you ever feel stressed?",
+        "What's your secret to being so good at everything?"
+    ]
+    
+    print("\n💬 开始对话演示\n")
+    print("提示: 这是自动演示模式，每个问题后会自动继续")
+    print("-"*70 + "\n")
+    
+    for i, msg in enumerate(demo_conversations, 1):
+        print(f"[{i}/{len(demo_conversations)}]")
+        print(f"🧑 你: {msg}")
+        print("⏳ Hermione 正在思考...", end='', flush=True)
+        
+        response = agent.chat(msg, use_memory=False)
+        
+        print(f"\r🎭 Hermione: {response}\n")
+        
+        if i < len(demo_conversations):
+            try:
+                input("[按 Enter 继续下一个问题...]")
+            except:
+                break
+    
+    print("\n" + "="*70)
+    print("演示完成！")
+    print("="*70)
+
+def run_interactive_demo():
+    """交互式演示"""
+    print("\n" + "="*70)
+    print("🎭 交互式演示：与 Harry Potter 对话")
+    print("="*70)
+    
+    profile = {
+        'name': 'Harry Potter',
+        'core_traits': ['勇敢', '善良', '忠诚', '谦逊', '有正义感'],
+        'speaking_style': '真诚、谦逊，说话时常带有不确定感',
+        'behavior_patterns': '面对危险时挺身而出，但对赞美感到不自在',
+        'values': '友谊高于一切，愿意为朋友牺牲',
+        'emotional_style': '内敛但情感深沉，容易被朋友的遭遇触动',
+        'relationship_style': '对朋友极度忠诚，珍视每一段友谊',
+        'background': '失去父母的年轻巫师，被预言为击败伏地魔的人',
+        'key_quotes': [
+            'I don\'t know what you\'re talking about.',
+            'We can figure this out together.',
+            'I had a lot of help.',
+            'It\'s not about being brave, it\'s about doing what\'s right.'
+        ],
+        'personality_summary': 'Harry是一个勇敢善良但又谦逊的年轻巫师。虽然承担着巨大的使命，但他从不认为自己特殊。他最看重的是友谊，愿意为朋友付出一切。'
+    }
+    
+    agent = CharacterAgent(profile)
+    print(agent.get_character_info())
+    
+    print("\n💬 开始交���式对话")
+    print("-"*70)
+    print("提示:")
+    print("  • 输入你的问题开始对话")
+    print("  • 输入 'quit' 退出")
+    print("  • 输入 'reset' 重置对话")
+    print("="*70 + "\n")
+    
+    while True:
+        try:
+            user_input = input("🧑 你: ").strip()
+            
+            if not user_input:
+                continue
+            
+            if user_input.lower() in ['quit', 'exit', 'q']:
+                print(f"\n🎭 Harry: Goodbye! It was nice talking to you.\n")
+                break
+            
+            if user_input.lower() == 'reset':
+                agent.reset_conversation()
+                continue
+            
+            print("⏳ Harry 正在思考...", end='', flush=True)
+            response = agent.chat(user_input, use_memory=False)
+            print(f"\r🎭 Harry: {response}\n")
+            
+        except KeyboardInterrupt:
+            print(f"\n\n🎭 Harry: Goodbye!\n")
+            break
+        except Exception as e:
+            print(f"\n❌ 错误: {e}\n")
+
+if __name__ == "__main__":
+    import sys
+    
+    if len(sys.argv) > 1:
+        if sys.argv[1] == "demo":
+            run_quick_demo()
+        elif sys.argv[1] == "interactive":
+            run_interactive_demo()
+        else:
+            print("用法:")
+            print("  python test_agent.py          - 运行所有测试")
+            print("  python test_agent.py demo     - 快速演示")
+            print("  python test_agent.py interactive - 交互式演示")
+    else:
+        # 运行单元测试
+        unittest.main(verbosity=2)

tools/__init__.py

@@ -0,0 +1 @@
+# 工具模块

tools/batch_analyze.py

@@ -0,0 +1,163 @@
+"""批量分析多个角色的工具脚本"""
+
+import sys
+import os
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from core import TextProcessor, CharacterExtractor, CharacterAnalyzer
+from utils import CacheManager
+import json
+from tqdm import tqdm
+
+def batch_analyze(novel_path: str, output_dir: str = "character_profiles", 
+                 max_characters: int = 10):
+    """批量分析小说中的所有主要角色
+    
+    Args:
+        novel_path: 小说文件路径
+        output_dir: 输出目录
+        max_characters: 最多分析的角色数
+    """
+    
+    print("="*70)
+    print("📚 批量角色分析工具")
+    print("="*70)
+    
+    # 1. 加载小说
+    print(f"\n📖 加载小说: {novel_path}")
+    try:
+        with open(novel_path, 'r', encoding='utf-8') as f:
+            novel = f.read()
+    except:
+        print(f"❌ 无法加载文件: {novel_path}")
+        return
+    
+    print(f"✓ 已加载 {len(novel):,} 个字符")
+    
+    # 2. 处理文本
+    print("\n📄 处理文本...")
+    processor = TextProcessor()
+    chunks = processor.chunk_text(novel)
+    stats = processor.get_statistics(novel)
+    
+    print(f"✓ 文本已分为 {len(chunks)} 个块")
+    print(f"✓ 检测语言: {stats['language']}")
+    
+    # 3. 提取角色
+    print("\n👥 提取角色...")
+    extractor = CharacterExtractor()
+    characters = extractor.extract_main_characters(
+        chunks,
+        text_sample=novel[:3000],
+        language=stats['language']
+    )
+    
+    if not characters:
+        print("❌ 未找到角色")
+        return
+    
+    print(f"✓ 找到 {len(characters)} 个主要角色")
+    
+    # 创建输出目录
+    os.makedirs(output_dir, exist_ok=True)
+    
+    # 4. 分析每个角色
+    print(f"\n🧠 开始分析角色 (最多 {max_characters} 个)...")
+    analyzer = CharacterAnalyzer()
+    
+    all_profiles = []
+    analyze_count = min(max_characters, len(characters))
+    
+    for i, char in enumerate(tqdm(characters[:analyze_count], desc="分析进度")):
+        char_name = char['name']
+        
+        try:
+            # 选择代表性片段
+            representative_chunks = analyzer.select_representative_chunks(
+                chunks,
+                char['info']['chunks']
+            )
+            
+            # 分析角色
+            profile = analyzer.analyze_character_batch(
+                char_name,
+                representative_chunks
+            )
+            
+            # 增强配置
+            profile = analyzer.enhance_profile_with_examples(
+                profile,
+                chunks,
+                char['info']['chunks']
+            )
+            
+            all_profiles.append(profile)
+            
+            # 保存单个角色配置
+            char_filename = f"{profile['name'].replace(' ', '_').replace('/', '_')}.json"
+            char_file = os.path.join(output_dir, char_filename)
+            
+            with open(char_file, 'w', encoding='utf-8') as f:
+                json.dump(profile, f, ensure_ascii=False, indent=2)
+            
+        except Exception as e:
+            print(f"\n❌ 分析 {char_name} 失败: {e}")
+            continue
+    
+    # 5. 保存汇总
+    all_file = os.path.join(output_dir, "all_characters.json")
+    with open(all_file, 'w', encoding='utf-8') as f:
+        json.dump(all_profiles, f, ensure_ascii=False, indent=2)
+    
+    # 6. 生成报告
+    report_file = os.path.join(output_dir, "analysis_report.txt")
+    with open(report_file, 'w', encoding='utf-8') as f:
+        f.write("="*70 + "\n")
+        f.write("角色分析报告\n")
+        f.write("="*70 + "\n\n")
+        f.write(f"小说文件: {novel_path}\n")
+        f.write(f"文本长度: {len(novel):,} 字符\n")
+        f.write(f"分析角色数: {len(all_profiles)}\n\n")
+        f.write("-"*70 + "\n\n")
+        
+        for i, profile in enumerate(all_profiles, 1):
+            f.write(f"{i}. {profile['name']}\n")
+            f.write(f"   核心特质: {', '.join(profile.get('core_traits', []))}\n")
+            f.write(f"   性格总结: {profile.get('personality_summary', 'N/A')}\n")
+            f.write("\n")
+    
+    # 完成
+    print("\n" + "="*70)
+    print("✅ 分析完成！")
+    print("="*70)
+    print(f"📁 输出目录: {output_dir}")
+    print(f"📊 分析角色数: {len(all_profiles)}")
+    print(f"📄 汇总文件: {all_file}")
+    print(f"📋 报告文件: {report_file}")
+    print("="*70)
+
+def main():
+    import argparse
+    
+    parser = argparse.ArgumentParser(
+        description="批量分析小说角色",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+示例:
+  python batch_analyze.py novel.txt
+  python batch_analyze.py novel.txt -o my_characters -n 15
+        """
+    )
+    
+    parser.add_argument("novel_path", help="小说文件路径")
+    parser.add_argument("-o", "--output", default="character_profiles", 
+                       help="输出目录 (默认: character_profiles)")
+    parser.add_argument("-n", "--num", type=int, default=10,
+                       help="最多分析的角色数 (默认: 10)")
+    
+    args = parser.parse_args()
+    
+    batch_analyze(args.novel_path, args.output, args.num)
+
+if __name__ == "__main__":
+    main()

tools/clear_cache.py

@@ -0,0 +1,66 @@
+"""清理缓存工具"""
+
+import sys
+import os
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from utils import CacheManager
+from config import Config
+
+def clear_cache(pattern=None, confirm=True):
+    """清理缓存
+    
+    Args:
+        pattern: 匹配模式，None表示清理所有
+        confirm: 是否需要确认
+    """
+    
+    cache = CacheManager()
+    info = cache.get_cache_info()
+    
+    print("="*70)
+    print("🗑️  缓存清理工具")
+    print("="*70)
+    print(f"\n当前缓存状态:")
+    print(f"  文件数量: {info['count']}")
+    print(f"  占用空间: {info['size_mb']} MB")
+    print(f"  缓存目录: {Config.CACHE_DIR}")
+    
+    if info['count'] == 0:
+        print("\n✓ 缓存为空，无需清理")
+        return
+    
+    if pattern:
+        print(f"\n将清理匹配 '{pattern}' 的缓存")
+    else:
+        print(f"\n⚠️  将清理所有缓存文件")
+    
+    if confirm:
+        response = input("\n确认清理？(y/n): ").strip().lower()
+        if response not in ['y', 'yes', '是']:
+            print("已取消")
+            return
+    
+    print("\n清理中...")
+    cache.clear(pattern)
+    
+    # 显示清理后状态
+    new_info = cache.get_cache_info()
+    print(f"\n✓ 清理完成")
+    print(f"  剩余文件: {new_info['count']}")
+    print(f"  释放空间: {info['size_mb'] - new_info['size_mb']:.2f} MB")
+
+def main():
+    import argparse
+    
+    parser = argparse.ArgumentParser(description="清理缓存")
+    parser.add_argument("-p", "--pattern", help="匹配模式")
+    parser.add_argument("-y", "--yes", action="store_true", 
+                       help="不询问直接清理")
+    
+    args = parser.parse_args()
+    
+    clear_cache(args.pattern, not args.yes)
+
+if __name__ == "__main__":
+    main()

utils/__init__.py

@@ -0,0 +1,4 @@
+from .text_utils import TextUtils
+from .cache_manager import CacheManager
+
+__all__ = ['TextUtils', 'CacheManager']

utils/cache_manager.py

@@ -0,0 +1,147 @@
+import os
+import json
+import hashlib
+import pickle
+from typing import Any, Optional, Dict  # 添加 Dict
+from pathlib import Path
+from config import Config
+
+class CacheManager:
+    """缓存管理器"""
+    
+    def __init__(self, cache_dir: str = None):
+        self.cache_dir = cache_dir or Config.CACHE_DIR
+        Path(self.cache_dir).mkdir(parents=True, exist_ok=True)
+    
+    def _get_cache_key(self, *args, **kwargs) -> str:
+        """生成缓存键"""
+        content = str(args) + str(sorted(kwargs.items()))
+        return hashlib.md5(content.encode()).hexdigest()
+    
+    def _get_cache_path(self, key: str) -> str:
+        """获取缓存文件路径"""
+        return os.path.join(self.cache_dir, f"{key}.pkl")
+    
+    def get(self, key: str) -> Optional[Any]:
+        """获取缓存"""
+        if not Config.ENABLE_CACHE:
+            return None
+        
+        cache_file = self._get_cache_path(key)
+        if os.path.exists(cache_file):
+            try:
+                with open(cache_file, 'rb') as f:
+                    return pickle.load(f)
+            except Exception as e:
+                print(f"读取缓存失败 ({key}): {e}")
+                return None
+        return None
+    
+    def set(self, key: str, value: Any) -> bool:
+        """设置缓存"""
+        if not Config.ENABLE_CACHE:
+            return False
+        
+        cache_file = self._get_cache_path(key)
+        try:
+            with open(cache_file, 'wb') as f:
+                pickle.dump(value, f)
+            return True
+        except Exception as e:
+            print(f"缓存保存失败 ({key}): {e}")
+            return False
+    
+    def exists(self, key: str) -> bool:
+        """检查缓存是否存在"""
+        if not Config.ENABLE_CACHE:
+            return False
+        return os.path.exists(self._get_cache_path(key))
+    
+    def delete(self, key: str) -> bool:
+        """删除指定缓存"""
+        cache_file = self._get_cache_path(key)
+        if os.path.exists(cache_file):
+            try:
+                os.remove(cache_file)
+                return True
+            except Exception as e:
+                print(f"删除缓存失败 ({key}): {e}")
+                return False
+        return False
+    
+    def clear(self, pattern: str = None):
+        """清除缓存"""
+        if not os.path.exists(self.cache_dir):
+            return
+        
+        count = 0
+        for file in os.listdir(self.cache_dir):
+            if file.endswith('.pkl'):
+                if pattern is None or pattern in file:
+                    try:
+                        os.remove(os.path.join(self.cache_dir, file))
+                        count += 1
+                    except Exception as e:
+                        print(f"删除缓存文件失败 ({file}): {e}")
+        
+        print(f"已清除 {count} 个缓存文件")
+    
+    def get_cache_size(self) -> int:
+        """获取缓存总大小（字节）"""
+        if not os.path.exists(self.cache_dir):
+            return 0
+        
+        total_size = 0
+        for file in os.listdir(self.cache_dir):
+            if file.endswith('.pkl'):
+                file_path = os.path.join(self.cache_dir, file)
+                total_size += os.path.getsize(file_path)
+        
+        return total_size
+    
+    def get_cache_info(self) -> dict:
+        """获取缓存信息"""
+        if not os.path.exists(self.cache_dir):
+            return {
+                'count': 0,
+                'size': 0,
+                'size_mb': 0
+            }
+        
+        count = 0
+        total_size = 0
+        
+        for file in os.listdir(self.cache_dir):
+            if file.endswith('.pkl'):
+                count += 1
+                file_path = os.path.join(self.cache_dir, file)
+                total_size += os.path.getsize(file_path)
+        
+        return {
+            'count': count,
+            'size': total_size,
+            'size_mb': round(total_size / (1024 * 1024), 2)
+        }
+    
+    def save_json(self, key: str, data: dict) -> bool:
+        """保存为JSON格式（用于可读性）"""
+        cache_file = os.path.join(self.cache_dir, f"{key}.json")
+        try:
+            with open(cache_file, 'w', encoding='utf-8') as f:
+                json.dump(data, f, ensure_ascii=False, indent=2)
+            return True
+        except Exception as e:
+            print(f"保存JSON缓存失败 ({key}): {e}")
+            return False
+    
+    def load_json(self, key: str) -> Optional[dict]:
+        """加载JSON格式缓存"""
+        cache_file = os.path.join(self.cache_dir, f"{key}.json")
+        if os.path.exists(cache_file):
+            try:
+                with open(cache_file, 'r', encoding='utf-8') as f:
+                    return json.load(f)
+            except Exception as e:
+                print(f"读取JSON缓存失败 ({key}): {e}")
+                return None
+        return None

utils/text_utils.py

@@ -0,0 +1,139 @@
+import re
+import tiktoken
+from typing import List, Tuple, Dict  # 添加 Dict
+
+class TextUtils:
+    """文本处理工具类"""
+    
+    @staticmethod
+    def count_tokens(text: str, model: str = "gpt-4") -> int:
+        """计算文本的token数量"""
+        try:
+            encoding = tiktoken.encoding_for_model(model)
+            return len(encoding.encode(text))
+        except:
+            # 粗略估计：英文约4字符=1token，中文约1.5字符=1token
+            # 使用更保守的估计
+            chinese_chars = len(re.findall(r'[\u4e00-\u9fff]', text))
+            english_chars = len(text) - chinese_chars
+            return int(chinese_chars / 1.5 + english_chars / 4)
+    
+    @staticmethod
+    def clean_text(text: str) -> str:
+        """清理文本"""
+        # 移除多余的空白
+        text = re.sub(r'\s+', ' ', text)
+        # 移除特殊字符（保留基本标点）
+        text = re.sub(r'[^\w\s,.!?;:\'\"()\-—《》「」『』【】\u4e00-\u9fff]', '', text)
+        return text.strip()
+    
+    @staticmethod
+    def split_into_sentences(text: str) -> List[str]:
+        """分割句子"""
+        # 支持中英文句子分割
+        # 英文句号、问号、感叹号
+        # 中文句号、问号、感叹号
+        sentences = re.split(r'[.!?。！？]+', text)
+        return [s.strip() for s in sentences if s.strip()]
+    
+    @staticmethod
+    def detect_language(text: str) -> str:
+        """检测文本语言"""
+        # 统计中文字符
+        chinese_chars = len(re.findall(r'[\u4e00-\u9fff]', text))
+        # 统计英文字符
+        english_chars = len(re.findall(r'[a-zA-Z]', text))
+        
+        total_chars = chinese_chars + english_chars
+        
+        if total_chars == 0:
+            return "unknown"
+        
+        chinese_ratio = chinese_chars / total_chars
+        
+        if chinese_ratio > 0.3:
+            return "zh"
+        elif chinese_ratio < 0.1:
+            return "en"
+        else:
+            return "mixed"
+    
+    @staticmethod
+    def extract_dialogues(text: str, language: str = "en") -> List[Dict]:
+        """提取对话"""
+        dialogues = []
+        
+        if language == "zh":
+            # 中文对话模式：引号内的内容
+            patterns = [
+                r'"([^"]+)"[,，]?\s*([^说道讲告诉问答叫喊]*(?:说|道|讲|告诉|问|答|叫|喊))',
+                r'"([^"]+)"',
+                r'「([^」]+)」',
+                r'『([^』]+)』'
+            ]
+        else:
+            # 英文对话模式
+            patterns = [
+                r'"([^"]+)",?\s*([^said]*(said|asked|replied|shouted|whispered|muttered))',
+                r'"([^"]+)"',
+                r"'([^']+)',?\s*([^said]*(said|asked|replied))",
+                r"'([^']+)'"
+            ]
+        
+        for pattern in patterns:
+            matches = re.finditer(pattern, text, re.IGNORECASE)
+            for match in matches:
+                dialogue = {
+                    'content': match.group(1),
+                    'attribution': match.group(2) if len(match.groups()) > 1 else '',
+                    'position': match.start()
+                }
+                dialogues.append(dialogue)
+        
+        return dialogues
+    
+    @staticmethod
+    def truncate_text(text: str, max_length: int, 
+                     ellipsis: str = "...") -> str:
+        """截断文本到指定长度"""
+        if len(text) <= max_length:
+            return text
+        
+        return text[:max_length - len(ellipsis)] + ellipsis
+    
+    @staticmethod
+    def extract_keywords(text: str, top_n: int = 10) -> List[str]:
+        """提取关键词（简单实现）"""
+        # 移除标点和停用词
+        words = re.findall(r'\b\w+\b', text.lower())
+        
+        # 简单的停用词列表
+        stop_words = {
+            'the', 'a', 'an', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for',
+            'of', 'with', 'by', 'from', 'as', 'is', 'was', 'are', 'were', 'be',
+            'been', 'being', 'have', 'has', 'had', 'do', 'does', 'did', 'will',
+            'would', 'could', 'should', 'may', 'might', 'can', 'this', 'that',
+            '的', '了', '在', '是', '我', '有', '和', '就', '不', '人', '都', '一',
+            '一个', '上', '也', '很', '到', '说', '要', '去', '你', '会', '着', '没有'
+        }
+        
+        # 过滤停用词
+        filtered_words = [w for w in words if w not in stop_words and len(w) > 2]
+        
+        # 统计词频
+        from collections import Counter
+        word_freq = Counter(filtered_words)
+        
+        # 返回最常见的词
+        return [word for word, freq in word_freq.most_common(top_n)]
+    
+    @staticmethod
+    def estimate_reading_time(text: str, wpm: int = 200) -> int:
+        """估计阅读时间（分钟）"""
+        words = len(re.findall(r'\b\w+\b', text))
+        chinese_chars = len(re.findall(r'[\u4e00-\u9fff]', text))
+        
+        # 中文按字符数/500，英文按单词数/wpm
+        reading_time = chinese_chars / 500 + words / wpm
+        
+        return max(1, int(reading_time))