| |
| """ |
| 处理 SQLite 数据库交互,用于存储和管理对话上下文。 |
| 支持文件存储(持久化)和内存存储(临时)。 |
| 包含加载、保存、删除上下文,以及检查 TTL 和清理内存数据库的功能。 |
| """ |
| import aiosqlite |
| import logging |
| import os |
| import json |
| import uuid |
| import asyncio |
| from datetime import datetime, timedelta, timezone |
| from contextlib import asynccontextmanager |
| from typing import Optional, List, Dict, Any, Tuple, Union, AsyncGenerator |
| from sqlalchemy import select, delete, update as sqlalchemy_update |
| from sqlalchemy.ext.asyncio import AsyncSession |
| from sqlalchemy.dialects.sqlite import insert as sqlite_insert |
| from app.core.context.converter import convert_messages |
|
|
| |
| from app.api.models import Message |
| |
| from app.core.database.models import DialogContext |
|
|
| |
| from app.core.database.settings import get_ttl_days, set_ttl_days |
| |
| from app.config import MAX_CONTEXT_RECORDS_MEMORY, CONTEXT_STORAGE_MODE, CONTEXT_DB_PATH |
|
|
| logger = logging.getLogger('my_logger') |
|
|
| |
| |
|
|
| async def get_proxy_key(key: str) -> Optional[Dict[str, Any]]: |
| """ |
| (可能需要审查/移除) 获取单个代理 Key 的信息 (似乎与 Key 管理功能重复)。 |
| """ |
| if not key: return None |
| try: |
| |
| from app.core.database.utils import get_db_connection |
| async with get_db_connection() as conn: |
| async with conn.cursor() as cursor: |
| |
| await cursor.execute("SELECT key, description, created_at, is_active FROM proxy_keys WHERE key = ?", (key,)) |
| row = await cursor.fetchone() |
| |
| return dict(row) if row else None |
| except aiosqlite.Error as e: |
| logger.error(f"获取代理 Key {key[:8]}... 信息失败: {e}", exc_info=True) |
| return None |
|
|
| async def is_valid_proxy_key(key: str) -> bool: |
| """ |
| (可能需要审查/移除) 检查代理 Key 是否有效且处于活动状态 (似乎与 Key 管理功能重复)。 |
| """ |
| key_info = await get_proxy_key(key) |
| |
| return bool(key_info and key_info.get('is_active')) |
|
|
|
|
| async def save_context(proxy_key: str, contents: List[Dict[str, Any]]): |
| """ |
| 异步保存或更新指定代理 Key (通常是 user_id) 的对话上下文到数据库。 |
| 如果记录已存在,则替换;如果不存在,则插入新记录。 |
| 同时会更新 `last_used` 时间戳。 |
| 在内存数据库模式下,会检查并清理超出数量限制的旧记录。 |
| |
| Args: |
| proxy_key (str): 用于标识上下文的键 (通常是 user_id)。 |
| contents (List[Dict[str, Any]]): 要保存的 Gemini 格式的对话内容列表。 |
| """ |
| |
| if not proxy_key or not contents: |
| logger.warning(f"尝试为 Key {proxy_key[:8]}... 保存空的上下文,已跳过。") |
| return |
|
|
| |
| from app.core.dependencies import get_db_session |
|
|
| try: |
| contents_json = await asyncio.to_thread(json.dumps, contents, ensure_ascii=False) |
| except TypeError as json_err: |
| logger.error(f"序列化上下文为 JSON 时失败 (TypeError) (Key: {proxy_key[:8]}...): {json_err}", exc_info=True) |
| return |
|
|
| async for db in get_db_session(): |
| try: |
| now_dt = datetime.now(timezone.utc) |
| |
| |
| stmt = sqlite_insert(DialogContext).values( |
| proxy_key=proxy_key, |
| contents=contents_json, |
| last_used_at=now_dt, |
| created_at=now_dt |
| ) |
| |
| |
| update_dict = { |
| "contents": stmt.excluded.contents, |
| "last_used_at": stmt.excluded.last_used_at |
| } |
| stmt = stmt.on_conflict_do_update( |
| index_elements=[DialogContext.proxy_key], |
| set_=update_dict |
| ) |
| |
| await db.execute(stmt) |
| await db.commit() |
| logger.info(f"上下文已为 Key {proxy_key[:8]}... 使用 ORM 保存/更新。") |
| break |
| except Exception as e: |
| logger.error(f"为 Key {proxy_key[:8]}... 使用 ORM 保存上下文失败: {e}", exc_info=True) |
| await db.rollback() |
| break |
| finally: |
| |
| pass |
|
|
|
|
| async def _is_context_expired(last_used_dt: datetime, ttl_delta: Optional[timedelta], proxy_key: str, db: AsyncSession) -> bool: |
| """ |
| (内部辅助函数) 检查给定的上次使用时间戳是否已过期。 |
| 如果过期,会尝试删除对应的上下文记录。 |
| |
| Args: |
| last_used_dt (datetime): 上下文的上次使用时间 (datetime 对象)。 |
| ttl_delta (Optional[timedelta]): 上下文的生存时间间隔。如果为 None 或 0,表示永不过期。 |
| proxy_key (str): 相关的代理 Key (用于日志记录和删除)。 |
| db (AsyncSession): SQLAlchemy 异步数据库会话。 |
| |
| Returns: |
| bool: 如果上下文已过期或时间戳解析失败,返回 True;否则返回 False。 |
| """ |
| if not ttl_delta: |
| return False |
|
|
| now_utc = datetime.now(timezone.utc) |
| |
| if last_used_dt.tzinfo is None: |
| last_used_dt = last_used_dt.replace(tzinfo=timezone.utc) |
|
|
| if now_utc - last_used_dt > ttl_delta: |
| logger.info(f"Key {proxy_key[:8]}... 的上下文已超过 TTL ({ttl_delta}),将被视为过期。") |
| await delete_context_for_key(proxy_key, db) |
| return True |
| return False |
|
|
| async def delete_context_for_key(proxy_key: str, db: AsyncSession) -> bool: |
| """ |
| 异步删除指定代理 Key (通常是 user_id) 的所有上下文记录。 |
| |
| Args: |
| proxy_key (str): 要删除上下文的键。 |
| |
| Returns: |
| bool: 如果成功删除(或记录本就不存在)返回 True,否则返回 False。 |
| """ |
| if not proxy_key: return False |
| try: |
| stmt = delete(DialogContext).where(DialogContext.proxy_key == proxy_key) |
| result = await db.execute(stmt) |
| await db.commit() |
| if result.rowcount > 0: |
| logger.info(f"上下文已为 Key {proxy_key[:8]}... 使用 ORM 删除。") |
| return True |
| else: |
| logger.warning(f"尝试使用 ORM 删除 Key {proxy_key[:8]}... 的上下文,但未找到记录。") |
| return True |
| except Exception as e: |
| logger.error(f"使用 ORM 删除 Key {proxy_key[:8]}... 的上下文失败: {e}", exc_info=True) |
| await db.rollback() |
| return False |
|
|
| async def _deserialize_context_contents(contents_json: str, proxy_key: str, db: AsyncSession) -> Optional[List[Dict[str, Any]]]: |
| """ |
| (内部辅助函数) 异步反序列化存储在数据库中的上下文 JSON 字符串。 |
| 处理可能的 JSON 解析错误,并在出错时尝试删除损坏的数据。 |
| |
| Args: |
| contents_json (str): 从数据库读取的上下文内容的 JSON 字符串。 |
| proxy_key (str): 相关的代理 Key (用于日志和删除操作)。 |
| |
| Returns: |
| Optional[List[Dict[str, Any]]]: 如果成功反序列化,返回 Python 列表; |
| 如果反序列化失败,返回 None。 |
| """ |
| try: |
| contents = await asyncio.to_thread(json.loads, contents_json) |
| logger.debug(f"上下文 JSON 已为 Key {proxy_key[:8]}... 反序列化。") |
| if isinstance(contents, list): |
| return contents |
| else: |
| logger.error(f"反序列化的上下文格式不正确 (期望列表,得到 {type(contents)}) (Key: {proxy_key[:8]}...)") |
| await delete_context_for_key(proxy_key, db) |
| return None |
| except json.JSONDecodeError as e: |
| logger.error(f"反序列化存储的上下文时失败 (Key: {proxy_key[:8]}...): {e}", exc_info=True) |
| await delete_context_for_key(proxy_key, db) |
| return None |
| except Exception as e: |
| logger.error(f"反序列化上下文时发生意外错误 (Key: {proxy_key[:8]}...): {e}", exc_info=True) |
| await delete_context_for_key(proxy_key, db) |
| return None |
|
|
| async def load_context(proxy_key: str, db: AsyncSession) -> Optional[List[Dict[str, Any]]]: |
| """ |
| 异步加载指定代理 Key (通常是 user_id) 的对话上下文。 |
| 会检查上下文的 TTL (生存时间),如果过期则删除并返回 None。 |
| 如果数据损坏无法解析,也会删除并返回 None。 |
| |
| Args: |
| proxy_key (str): 要加载上下文的键。 |
| |
| Returns: |
| Optional[List[Dict[str, Any]]]: 如果找到有效且未过期的上下文,返回 Gemini 格式的内容列表; |
| 否则返回 None。 |
| """ |
| if not proxy_key: return None |
|
|
| ttl_days = await get_ttl_days() |
| ttl_delta = timedelta(days=ttl_days) if ttl_days > 0 else None |
|
|
| try: |
| stmt = select(DialogContext).where(DialogContext.proxy_key == proxy_key) |
| result = await db.execute(stmt) |
| record: Optional[DialogContext] = result.scalar_one_or_none() |
|
|
| if not record or not record.contents: |
| logger.debug(f"未找到 Key {proxy_key[:8]}... 的上下文 (ORM)。") |
| return None |
|
|
| |
| if await _is_context_expired(record.last_used_at, ttl_delta, proxy_key, db): |
| return None |
|
|
| return await _deserialize_context_contents(record.contents, proxy_key, db) |
|
|
| except Exception as e: |
| logger.error(f"为 Key {proxy_key[:8]}... 使用 ORM 加载上下文失败: {e}", exc_info=True) |
| |
| return None |
|
|
|
|
| async def get_context_info(proxy_key: str, db: AsyncSession) -> Optional[Dict[str, Any]]: |
| """ |
| 异步获取指定代理 Key 上下文的元信息(内容长度和最后使用时间)。 |
| |
| Args: |
| proxy_key (str): 要查询的代理 Key。 |
| |
| Returns: |
| Optional[Dict[str, Any]]: 包含 'content_length' 和 'last_used' 的字典,如果未找到记录则返回 None。 |
| """ |
| if not proxy_key: return None |
| try: |
| stmt = select(DialogContext.contents, DialogContext.last_used_at).where(DialogContext.proxy_key == proxy_key) |
| result = await db.execute(stmt) |
| row = result.one_or_none() |
| |
| if row: |
| |
| content_length = len(row.contents) if row.contents else 0 |
| last_used_iso = row.last_used_at.isoformat() if row.last_used_at else None |
| return {"content_length": content_length, "last_used": last_used_iso} |
| else: |
| return None |
| except Exception as e: |
| logger.error(f"使用 ORM 获取 Key {proxy_key[:8]}... 的上下文信息失败: {e}", exc_info=True) |
| return None |
|
|
| async def list_all_context_keys_info(db: AsyncSession, user_key: Optional[str] = None, is_admin: bool = False) -> List[Dict[str, Any]]: |
| """ |
| 异步获取存储的上下文的 Key 和元信息列表。 |
| 根据用户权限(是否为管理员)和提供的用户 Key 进行过滤。 |
| |
| Args: |
| user_key (Optional[str]): 当前请求用户的 Key (通常是 user_id)。默认为 None。 |
| is_admin (bool): 当前用户是否具有管理员权限。默认为 False。 |
| |
| Returns: |
| List[Dict[str, Any]]: 包含上下文信息的字典列表。每个字典包含 'proxy_key', 'contents' (JSON 字符串), |
| 'content_length', 'last_used'。 |
| 如果无权访问或出错,返回空列表。 |
| """ |
| contexts_info = [] |
| try: |
| stmt = select( |
| DialogContext.proxy_key, |
| DialogContext.contents, |
| DialogContext.last_used_at |
| ) |
| if is_admin: |
| logger.info(f"管理员请求所有上下文信息 (ORM)...") |
| stmt = stmt.order_by(DialogContext.last_used_at.desc()) |
| elif user_key: |
| logger.info(f"用户 {user_key[:8]}... 请求其上下文信息 (ORM)...") |
| stmt = stmt.where(DialogContext.proxy_key == user_key).order_by(DialogContext.last_used_at.desc()) |
| else: |
| logger.warning(f"非管理员尝试列出上下文但未提供 user_key (ORM)。") |
| return [] |
|
|
| result = await db.execute(stmt) |
| rows = result.all() |
|
|
| for row in rows: |
| contexts_info.append({ |
| "proxy_key": row.proxy_key, |
| "contents": row.contents, |
| "content_length": len(row.contents) if row.contents else 0, |
| "last_used": row.last_used_at.isoformat() if row.last_used_at else None |
| }) |
| |
| except Exception as e: |
| log_prefix = f"管理员" if is_admin else f"用户 {user_key[:8]}..." if user_key else "未知用户" |
| logger.error(f"{log_prefix} 使用 ORM 列出上下文信息失败: {e}", exc_info=True) |
| contexts_info = [] |
| return contexts_info |
|
|
| |
| |
| |
| |
| |
|
|
| def convert_openai_to_gemini_contents(history: List[Dict]) -> List[Dict]: |
| """ |
| (辅助函数) 将 OpenAI 格式的对话历史列表转换为 Gemini 的 contents 格式列表。 |
| 主要处理角色映射和将 content 字符串包装在 parts 列表中。 |
| |
| Args: |
| history (List[Dict]): OpenAI 格式的对话历史列表,每个字典包含 'role' 和 'content'。 |
| |
| Returns: |
| List[Dict]: Gemini 格式的 contents 列表,每个字典包含 'role' 和 'parts'。 |
| """ |
| gemini_contents = [] |
| for message in history: |
| openai_role = message.get('role') |
| openai_content = message.get('content') |
|
|
| |
| if openai_role and openai_content is not None: |
| |
| if openai_role == 'user': |
| gemini_role = 'user' |
| elif openai_role == 'assistant': |
| gemini_role = 'model' |
| elif openai_role == 'system': |
| |
| |
| gemini_role = 'user' |
| logger.debug("将 OpenAI 'system' 角色映射为 Gemini 'user' 角色。") |
| else: |
| logger.warning(f"跳过无效的 OpenAI 历史消息角色: {openai_role}") |
| continue |
|
|
| |
| |
| |
| if isinstance(openai_content, str): |
| gemini_parts = [{'text': openai_content}] |
| else: |
| logger.warning(f"跳过非字符串类型的 OpenAI content: {type(openai_content)}") |
| continue |
|
|
| |
| gemini_contents.append({'role': gemini_role, 'parts': gemini_parts}) |
| else: |
| logger.warning(f"跳过无效的 OpenAI 历史消息(缺少 role 或 content): {message}") |
|
|
| return gemini_contents |
|
|
| async def load_context_as_gemini(proxy_key: str, db: AsyncSession) -> Optional[List[Dict[str, Any]]]: |
| """ |
| 异步加载指定代理 Key 的上下文,并将其直接转换为 Gemini 的 contents 格式返回。 |
| 处理 TTL 检查和数据反序列化。现在使用 ORM 和 AsyncSession。 |
| |
| Args: |
| proxy_key (str): 要加载上下文的代理 Key (通常是 user_id)。 |
| db (AsyncSession): SQLAlchemy 异步数据库会话。 |
| |
| Returns: |
| Optional[List[Dict[str, Any]]]: Gemini 格式的 contents 列表,如果未找到、过期或加载/转换失败则返回 None。 |
| """ |
| loaded_context = await load_context(proxy_key, db) |
|
|
| if loaded_context is None: |
| return None |
|
|
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
|
|
| |
| |
| if isinstance(loaded_context, list): |
| |
| return loaded_context |
| else: |
| logger.error(f"加载的上下文格式不正确 (期望列表,得到 {type(loaded_context)}) (Key: {proxy_key[:8]}...)") |
| |
| await delete_context_for_key(proxy_key, db) |
| return None |
|
|
|
|
| def convert_gemini_to_storage_format(request_content: Dict, response_content: Dict) -> List[Dict]: |
| """ |
| (辅助函数) 将 Gemini API 的请求内容 (用户回合) 和响应内容 (模型回合) |
| 转换为用于数据库存储的格式 (目前使用类似 OpenAI 的格式)。 |
| 主要提取文本内容。 |
| |
| Args: |
| request_content (Dict): Gemini 格式的用户请求内容 (包含 role='user' 和 parts)。 |
| response_content (Dict): Gemini 格式的模型响应内容 (包含 role='model' 和 parts)。 |
| |
| Returns: |
| List[Dict]: 包含两个字典的列表,分别代表用户回合和模型回合,格式为 {'role': 'user'/'assistant', 'content': 'text'}。 |
| 如果输入格式无效,可能返回空列表或只包含部分回合。 |
| """ |
| storage_format = [] |
|
|
| |
| user_role = request_content.get('role') |
| user_parts = request_content.get('parts') |
|
|
| if user_role == 'user' and user_parts is not None: |
| |
| storage_format.append({'role': 'user', 'parts': user_parts}) |
| else: |
| logger.warning(f"转换 Gemini 用户请求内容时发现无效格式或缺少 parts: {request_content}") |
|
|
| |
| model_role = response_content.get('role') |
| model_parts = response_content.get('parts') |
|
|
| if model_role == 'model' and model_parts is not None: |
| |
| storage_format.append({'role': 'model', 'parts': model_parts}) |
| else: |
| logger.warning(f"转换 Gemini 模型响应内容时发现无效格式或缺少 parts: {response_content}") |
|
|
| return storage_format |
|
|
|
|
| |
| |
| |
|
|
| async def update_ttl(context_key: str, ttl_seconds: int) -> Optional[bool]: |
| """ |
| (可能需要审查) 异步更新指定上下文记录的 TTL。 |
| 实际上是通过将 `last_used` 时间戳更新为当前时间来实现“刷新”TTL。 |
| |
| Args: |
| context_key (str): 要更新 TTL 的上下文键 (通常是 user_id)。 |
| ttl_seconds (int): 新的 TTL 秒数 (此参数当前未使用,因为只是更新时间戳)。 |
| |
| Returns: |
| Optional[bool]: 如果成功更新时间戳返回 True,如果记录未找到返回 False,如果发生错误返回 None。 |
| """ |
| if not context_key: return False |
| try: |
| |
| from app.core.database.utils import get_db_connection |
| async with get_db_connection() as conn: |
| async with conn.cursor() as cursor: |
| |
| now_utc_iso = datetime.now(timezone.utc).isoformat() |
| |
| await cursor.execute("UPDATE contexts SET last_used = ? WHERE proxy_key = ?", (now_utc_iso, context_key)) |
| rowcount = cursor.rowcount |
| await conn.commit() |
| if rowcount > 0: |
| logger.info(f"成功更新了 Key {context_key[:8]}... 的 last_used 时间戳。") |
| return True |
| else: |
| logger.warning(f"尝试更新 Key {context_key[:8]}... 的 last_used 时间戳,但未找到记录。") |
| return False |
| except aiosqlite.Error as e: |
| logger.error(f"更新 Key {context_key[:8]}... 的 last_used 时间戳失败: {e}", exc_info=True) |
| return None |
|
|
| async def update_global_ttl(ttl_days: int) -> bool: |
| """ |
| 异步更新全局上下文 TTL 的天数设置。 |
| |
| Args: |
| ttl_days (int): 新的全局 TTL 天数 (非负整数)。 |
| |
| Returns: |
| bool: 如果成功更新设置返回 True,否则返回 False。 |
| """ |
| |
| from app.core.database.settings import set_ttl_days |
| try: |
| |
| await set_ttl_days(ttl_days) |
| logger.info(f"全局上下文 TTL 已更新为 {ttl_days} 天。") |
| return True |
| except ValueError as ve: |
| logger.error(f"更新全局上下文 TTL failed: {ve}") |
| return False |
| except Exception as e: |
| logger.error(f"更新全局上下文 TTL 时发生意外错误: {e}", exc_info=True) |
| return False |
|
|
| async def get_all_contexts_with_ttl() -> Dict[str, Dict[str, Any]]: |
| """ |
| 异步获取所有存储的上下文记录及其元信息,包括计算出的剩余 TTL 和内容摘要。 |
| |
| Returns: |
| Dict[str, Dict[str, Any]]: 一个字典,键是 proxy_key (user_id),值是包含以下信息的字典: |
| - 'ttl' (str): 剩余 TTL 的可读字符串 ("x天 y小时", "已过期", "永不", "N/A")。 |
| - 'last_accessed' (str): 最后访问时间的格式化字符串 ("YYYY-MM-DD HH:MM:SS") 或 "N/A"。 |
| - 'context_summary' (str): 上下文内容的摘要 (通常是第一条消息的前 100 个字符) 或 "N/A"。 |
| """ |
| all_contexts_data = {} |
| try: |
| |
| from app.core.database.utils import get_db_connection |
| from app.core.database.settings import get_ttl_days |
|
|
| |
| global_ttl_days = await get_ttl_days() |
| |
| global_ttl_seconds = global_ttl_days * 24 * 60 * 60 if global_ttl_days > 0 else 0 |
|
|
| async with get_db_connection() as conn: |
| conn.row_factory = aiosqlite.Row |
| async with conn.cursor() as cursor: |
| |
| await cursor.execute("SELECT proxy_key, contents, last_used FROM contexts ORDER BY last_used DESC") |
| rows = await cursor.fetchall() |
|
|
| |
| for row in rows: |
| proxy_key = row['proxy_key'] |
| contents_json = row['contents'] |
| last_used_str = row['last_used'] |
|
|
| |
| context_summary = "N/A" |
| if contents_json: |
| try: |
| |
| contents_list = await asyncio.to_thread(json.loads, contents_json) |
| |
| if contents_list and isinstance(contents_list, list) and len(contents_list) > 0: |
| |
| first_message = contents_list[0] |
| if isinstance(first_message, dict) and 'content' in first_message: |
| content_text = str(first_message['content']) |
| context_summary = content_text[:100] + "..." if len(content_text) > 100 else content_text |
| elif isinstance(first_message, dict) and 'parts' in first_message and isinstance(first_message['parts'], list) and len(first_message['parts']) > 0: |
| first_part = first_message['parts'][0] |
| if isinstance(first_part, dict) and 'text' in first_part: |
| content_text = str(first_part['text']) |
| context_summary = content_text[:100] + "..." if len(content_text) > 100 else content_text |
| except json.JSONDecodeError: |
| logger.warning(f"无法解析 Key {proxy_key[:8]}... 的上下文内容 JSON。") |
| except Exception as e: |
| logger.warning(f"提取 Key {proxy_key[:8]}... 的上下文摘要时出错: {e}") |
|
|
| |
| ttl_remaining_str = "N/A" |
| if last_used_str and global_ttl_seconds > 0: |
| try: |
| |
| last_used_dt = datetime.fromisoformat(last_used_str.replace('Z', '+00:00')).replace(tzinfo=timezone.utc) |
| |
| expiry_time = last_used_dt + timedelta(seconds=global_ttl_seconds) |
| now_utc = datetime.now(timezone.utc) |
| |
| remaining_delta = expiry_time - now_utc |
| if remaining_delta.total_seconds() > 0: |
| |
| days = remaining_delta.days |
| hours, remainder = divmod(remaining_delta.seconds, 3600) |
| minutes, seconds = divmod(remainder, 60) |
| if days > 0: |
| ttl_remaining_str = f"{days}天 {hours}小时" |
| elif hours > 0: |
| ttl_remaining_str = f"{hours}小时 {minutes}分钟" |
| elif minutes > 0: |
| ttl_remaining_str = f"{minutes}分钟 {seconds}秒" |
| else: |
| ttl_remaining_str = f"{seconds}秒" |
| else: |
| ttl_remaining_str = "已过期" |
| except ValueError: |
| logger.warning(f"无法解析 Key {proxy_key[:8]}... 的 last_used 时间戳: {last_used_str}") |
| elif global_ttl_seconds <= 0: |
| ttl_remaining_str = "永不" |
|
|
| |
| all_contexts_data[proxy_key] = { |
| "ttl": ttl_remaining_str, |
| |
| "last_accessed": last_used_str.split('.')[0].replace('T', ' ') if last_used_str else "N/A", |
| "context_summary": context_summary |
| } |
| logger.info(f"成功获取了 {len(all_contexts_data)} 条上下文记录及其 TTL 信息。") |
|
|
| except aiosqlite.Error as e: |
| logger.error(f"获取所有上下文及其 TTL 信息 failed: {e}", exc_info=True) |
| except Exception as e: |
| logger.error(f"处理所有上下文数据时发生意外错误: {e}", exc_info=True) |
|
|
| return all_contexts_data |
|
|
|
|
| |
| from app import config as app_config |
|
|
|
|
| class ContextStore: |
| """ |
| 管理对话上下文的存储和检索。 |
| 支持内存和数据库两种存储模式。 |
| """ |
| def __init__(self, storage_mode: str = app_config.CONTEXT_STORAGE_MODE, db_path: str = app_config.CONTEXT_DB_PATH): |
| """ |
| 初始化 ContextStore。 |
| |
| Args: |
| storage_mode (str): 'memory' 或 'database'。 |
| db_path (str): 数据库文件路径 (仅在 database 模式下使用)。 |
| """ |
| self.storage_mode = storage_mode |
| self.db_path = db_path |
| if self.storage_mode == "memory": |
| self.memory_store: Dict[str, Dict[str, Any]] = {} |
| self.memory_lock = asyncio.Lock() |
| logger.info("上下文存储已初始化为内存模式。") |
| elif self.storage_mode == "database": |
| |
| logger.info(f"上下文存储已初始化为数据库模式 (路径: {self.db_path})。") |
| else: |
| raise ValueError(f"未知的上下文存储模式: {storage_mode}") |
|
|
| async def perform_memory_cleanup(self): |
| """ |
| (仅内存模式) 清理内存中存储的过期上下文条目。 |
| """ |
| if self.storage_mode != "memory": |
| return |
|
|
| async with self.memory_lock: |
| now_iso = datetime.now(timezone.utc).isoformat() |
| keys_to_delete = [] |
| for key, data in self.memory_store.items(): |
| expires_at = data.get("expires_at") |
| if expires_at and expires_at < now_iso: |
| keys_to_delete.append(key) |
| |
| for key in keys_to_delete: |
| del self.memory_store[key] |
| logger.info(f"内存中的上下文 Key '{key}' 已过期并被清理。") |
| |
| if keys_to_delete: |
| logger.info(f"内存上下文清理完成,移除了 {len(keys_to_delete)} 个过期条目。") |
| else: |
| logger.debug("内存上下文清理完成,没有找到需要删除的过期条目。") |
|
|
| |
| if MAX_CONTEXT_RECORDS_MEMORY > 0 and len(self.memory_store) > MAX_CONTEXT_RECORDS_MEMORY: |
| num_to_prune = len(self.memory_store) - MAX_CONTEXT_RECORDS_MEMORY |
| |
| sorted_keys = sorted(self.memory_store.items(), key=lambda item: item[1].get('last_used', '')) |
| keys_to_prune = [item[0] for item in sorted_keys[:num_to_prune]] |
| for key in keys_to_prune: |
| del self.memory_store[key] |
| logger.info(f"ContextStore: 内存上下文存储超出最大记录数限制,已清理 {len(keys_to_prune)} 条最旧的记录。") |
|
|
| async def store_context(self, user_id: str, context_key: str, context_value: Any, ttl_seconds: Optional[int] = None): |
| """ |
| 存储上下文信息。 |
| 对于数据库模式,现在使用 DialogContext 模型。 |
| |
| Args: |
| user_id (str): 用户标识。 (注意:在DialogContext中,proxy_key通常是user_id) |
| context_key (str): 上下文的唯一键 (通常是 user_id 或基于 user_id 的键)。 |
| context_value (Any): 要存储的上下文内容 (通常是 Gemini contents 列表)。 |
| ttl_seconds (Optional[int]): 此上下文的 TTL (秒)。如果为 None,则使用全局 TTL。 |
| """ |
| if not context_key or context_value is None: |
| logger.warning(f"ContextStore: 尝试为 Key {context_key[:8]}... 保存空的上下文或内容,已跳过。") |
| return |
|
|
| now = datetime.now(timezone.utc) |
| |
| |
| |
| final_ttl_seconds_for_memory: Optional[int] = ttl_seconds |
| if final_ttl_seconds_for_memory is None: |
| global_ttl_days = await get_ttl_days() |
| final_ttl_seconds_for_memory = global_ttl_days * 24 * 60 * 60 if global_ttl_days > 0 else None |
|
|
| expires_at_iso_for_memory: Optional[str] = None |
| if final_ttl_seconds_for_memory is not None and final_ttl_seconds_for_memory > 0: |
| expires_at_iso_for_memory = (now + timedelta(seconds=final_ttl_seconds_for_memory)).isoformat() |
|
|
| if self.storage_mode == "memory": |
| async with self.memory_lock: |
| self.memory_store[context_key] = { |
| "user_id": user_id, |
| "content": context_value, |
| "last_used": now.isoformat(), |
| "expires_at": expires_at_iso_for_memory, |
| "created_at": now.isoformat() |
| } |
| logger.info(f"ContextStore: 上下文已为 Key {context_key[:8]}... (用户 {user_id}) 存储到内存。") |
| if MAX_CONTEXT_RECORDS_MEMORY > 0 and len(self.memory_store) > MAX_CONTEXT_RECORDS_MEMORY: |
| num_to_prune = len(self.memory_store) - MAX_CONTEXT_RECORDS_MEMORY |
| sorted_keys = sorted(self.memory_store.items(), key=lambda item: item[1].get('last_used', '')) |
| keys_to_prune = [item[0] for item in sorted_keys[:num_to_prune]] |
| for key_to_prune in keys_to_prune: |
| del self.memory_store[key_to_prune] |
| logger.info(f"ContextStore: 内存上下文存储超出最大记录数限制,已清理 {len(keys_to_prune)} 条最旧的记录。") |
|
|
| elif self.storage_mode == "database": |
| from app.core.dependencies import get_db_session |
| async for db in get_db_session(): |
| try: |
| contents_json = await asyncio.to_thread(json.dumps, context_value, ensure_ascii=False) |
| |
| |
| stmt = sqlite_insert(DialogContext).values( |
| proxy_key=context_key, |
| contents=contents_json, |
| last_used_at=now, |
| created_at=now, |
| ttl_seconds=ttl_seconds |
| ) |
| update_values = { |
| "contents": stmt.excluded.contents, |
| "last_used_at": stmt.excluded.last_used_at, |
| } |
| |
| if ttl_seconds is not None: |
| update_values["ttl_seconds"] = stmt.excluded.ttl_seconds |
| |
| stmt = stmt.on_conflict_do_update( |
| index_elements=[DialogContext.proxy_key], |
| set_=update_values |
| ) |
| await db.execute(stmt) |
| await db.commit() |
| logger.info(f"ContextStore: 上下文已为 Key {context_key[:8]}... (用户 {user_id}) 使用 DialogContext ORM 保存/更新。") |
| break |
| except Exception as e: |
| logger.error(f"ContextStore: 为 Key {context_key[:8]}... (用户 {user_id}) 保存上下文到数据库 (DialogContext) failed: {e}", exc_info=True) |
| await db.rollback() |
| break |
| else: |
| logger.error(f"ContextStore: 未知的存储模式 '{self.storage_mode}',无法存储上下文。") |
|
|
| async def retrieve_context(self, user_id: str, context_key: str) -> Optional[Any]: |
| """ |
| 检索上下文信息。 |
| |
| Args: |
| user_id (str): 用户标识。 |
| context_key (str): 上下文的唯一键。 |
| |
| Returns: |
| Optional[Any]: 存储的上下文内容,如果未找到或已过期则返回 None。 |
| """ |
| now_iso = datetime.now(timezone.utc).isoformat() |
|
|
| if self.storage_mode == "memory": |
| async with self.memory_lock: |
| data = self.memory_store.get(context_key) |
| if data: |
| |
| |
| |
| |
|
|
| if data.get("expires_at") and data["expires_at"] < now_iso: |
| logger.info(f"内存中的上下文 Key '{context_key}' 已过期,将被删除。") |
| del self.memory_store[context_key] |
| return None |
| data["last_used"] = now_iso |
| return data.get("content") |
| return None |
| elif self.storage_mode == "database": |
| from app.core.dependencies import get_db_session |
| async for db in get_db_session(): |
| try: |
| |
| |
| retrieved_value = await load_context(proxy_key=context_key, db=db) |
| |
| |
| if retrieved_value is not None: |
| update_stmt = sqlalchemy_update(DialogContext).\ |
| where(DialogContext.proxy_key == context_key).\ |
| values(last_used_at=datetime.now(timezone.utc)) |
| await db.execute(update_stmt) |
| await db.commit() |
| |
| return retrieved_value |
| except Exception as e: |
| logger.error(f"ContextStore: 从数据库检索 Key '{context_key}' (用户 {user_id}) 的上下文 (DialogContext) failed: {e}", exc_info=True) |
| await db.rollback() |
| return None |
| finally: |
| break |
| else: |
| logger.error(f"ContextStore: 未知的存储模式 '{self.storage_mode}',无法检索上下文。") |
| return None |
|
|
| async def delete_context(self, user_id: str, context_key: str) -> bool: |
| """ |
| 删除指定的上下文。 |
| |
| Args: |
| user_id (str): 用户标识。 |
| context_key (str): 要删除的上下文的键。 |
| |
| Returns: |
| bool: 如果成功删除或记录本就不存在则返回 True,否则返回 False。 |
| """ |
| if self.storage_mode == "memory": |
| async with self.memory_lock: |
| if context_key in self.memory_store: |
| |
| |
| |
| |
| del self.memory_store[context_key] |
| logger.info(f"内存中的上下文 Key '{context_key}' 已被用户 {user_id[:8]}... 删除。") |
| return True |
| return False |
| elif self.storage_mode == "database": |
| from app.core.dependencies import get_db_session |
| async for db in get_db_session(): |
| try: |
| |
| |
| deleted = await delete_context_for_key(proxy_key=context_key, db=db) |
| |
| return deleted |
| except Exception as e: |
| logger.error(f"ContextStore: 从数据库删除 Key '{context_key}' (用户 {user_id}) 的上下文 (DialogContext) failed: {e}", exc_info=True) |
| await db.rollback() |
| return False |
| finally: |
| break |
| else: |
| logger.error(f"ContextStore: 未知的存储模式 '{self.storage_mode}',无法删除上下文。") |
| return False |
|
|
| async def get_context_info_for_management(self, user_id: Optional[str] = None, is_admin: bool = False) -> List[Dict[str, Any]]: |
| """ |
| 获取用于管理界面的上下文信息列表。 |
| """ |
| contexts_info = [] |
| now_utc = datetime.now(timezone.utc) |
| global_ttl_days = await get_ttl_days() |
| global_ttl_delta = timedelta(days=global_ttl_days) if global_ttl_days > 0 else None |
|
|
| if self.storage_mode == "memory": |
| async with self.memory_lock: |
| for key, data in self.memory_store.items(): |
| if not is_admin and data.get("user_id") != user_id: |
| continue |
|
|
| summary = "N/A" |
| if data.get("content"): |
| try: |
| first_message_content = "" |
| if isinstance(data["content"], list) and data["content"]: |
| first_message = data["content"][0] |
| if isinstance(first_message, dict) and 'parts' in first_message and first_message['parts']: |
| first_part = first_message['parts'][0] |
| if isinstance(first_part, dict) and 'text' in first_part: |
| first_message_content = str(first_part['text']) |
| elif isinstance(first_message, dict) and 'content' in first_message: |
| first_message_content = str(first_message['content']) |
| summary = first_message_content[:100] + "..." if len(first_message_content) > 100 else first_message_content |
| except Exception: |
| pass |
| |
| last_used_dt = datetime.fromisoformat(data["last_used"].replace('Z', '+00:00')) if data.get("last_used") else now_utc |
| expires_at_dt = datetime.fromisoformat(data["expires_at"].replace('Z', '+00:00')) if data.get("expires_at") else None |
| |
| ttl_str = "永不" |
| if expires_at_dt: |
| if expires_at_dt < now_utc: |
| ttl_str = "已过期" |
| else: |
| remaining_delta = expires_at_dt - now_utc |
| days = remaining_delta.days |
| hours, remainder = divmod(remaining_delta.seconds, 3600) |
| minutes, _ = divmod(remainder, 60) |
| if days > 0: ttl_str = f"{days}天{hours}小时" |
| elif hours > 0: ttl_str = f"{hours}小时{minutes}分钟" |
| else: ttl_str = f"{minutes}分钟" |
| elif global_ttl_delta : |
| effective_expiry = last_used_dt + global_ttl_delta |
| if effective_expiry < now_utc: |
| ttl_str = "已过期 (全局TTL)" |
| else: |
| remaining_delta = effective_expiry - now_utc |
| days = remaining_delta.days |
| hours, remainder = divmod(remaining_delta.seconds, 3600) |
| minutes, _ = divmod(remainder, 60) |
| if days > 0: ttl_str = f"{days}天{hours}小时 (全局TTL)" |
| elif hours > 0: ttl_str = f"{hours}小时{minutes}分钟 (全局TTL)" |
| else: ttl_str = f"{minutes}分钟 (全局TTL)" |
|
|
|
|
| contexts_info.append({ |
| "context_key": key, |
| "user_id": data.get("user_id", "N/A"), |
| "created_at": data.get("created_at", "N/A"), |
| "last_accessed_at": data.get("last_used", "N/A"), |
| "ttl_seconds": (expires_at_dt - datetime.fromisoformat(data["created_at"])).total_seconds() if expires_at_dt and data.get("created_at") else "N/A", |
| "context_value_summary": summary, |
| "ttl_display": ttl_str, |
| "id": key |
| }) |
| |
| contexts_info.sort(key=lambda x: x.get('created_at', ''), reverse=True) |
|
|
| elif self.storage_mode == "database": |
| from app.core.dependencies import get_db_session |
| async for db in get_db_session(): |
| try: |
| |
| stmt = select(DialogContext) |
| if not is_admin and user_id: |
| stmt = stmt.where(DialogContext.proxy_key == user_id) |
| stmt = stmt.order_by(DialogContext.last_used_at.desc()) |
| |
| result = await db.execute(stmt) |
| records = result.scalars().all() |
|
|
| for record in records: |
| summary = "N/A" |
| if record.contents: |
| try: |
| content_list = await asyncio.to_thread(json.loads, record.contents) |
| if content_list and isinstance(content_list, list) and content_list: |
| first_message = content_list[0] |
| if isinstance(first_message, dict) and 'parts' in first_message and first_message['parts']: |
| first_part = first_message['parts'][0] |
| if isinstance(first_part, dict) and 'text' in first_part: |
| summary = str(first_part['text'])[:100] + "..." if len(str(first_part['text'])) > 100 else str(first_part['text']) |
| elif isinstance(first_message, dict) and 'content' in first_message: |
| summary = str(first_message['content'])[:100] + "..." if len(str(first_message['content'])) > 100 else str(first_message['content']) |
| except Exception: |
| pass |
| |
| ttl_str = "永不" |
| effective_ttl_seconds: Optional[int] = record.ttl_seconds |
| if effective_ttl_seconds is None and global_ttl_delta: |
| effective_ttl_seconds = int(global_ttl_delta.total_seconds()) |
|
|
| if record.last_used_at and effective_ttl_seconds is not None and effective_ttl_seconds > 0: |
| last_used_aware = record.last_used_at if record.last_used_at.tzinfo else record.last_used_at.replace(tzinfo=timezone.utc) |
| expiry_time = last_used_aware + timedelta(seconds=effective_ttl_seconds) |
| if expiry_time < now_utc: |
| ttl_str = "已过期" |
| else: |
| remaining_delta = expiry_time - now_utc |
| days, rem_secs = divmod(remaining_delta.total_seconds(), 86400) |
| hours, rem_secs = divmod(rem_secs, 3600) |
| minutes, _ = divmod(rem_secs, 60) |
| if days > 0: ttl_str = f"{int(days)}天{int(hours)}小时" |
| elif hours > 0: ttl_str = f"{int(hours)}小时{int(minutes)}分钟" |
| else: ttl_str = f"{int(minutes)}分钟" |
| |
| contexts_info.append({ |
| "id": record.id, |
| "user_id": record.proxy_key, |
| "context_key": record.proxy_key, |
| "created_at": record.created_at.isoformat() if record.created_at else "N/A", |
| "last_accessed_at": record.last_used_at.isoformat() if record.last_used_at else "N/A", |
| "ttl_seconds": record.ttl_seconds if record.ttl_seconds is not None else (global_ttl_delta.total_seconds() if global_ttl_delta else "N/A"), |
| "context_value_summary": summary, |
| "ttl_display": ttl_str |
| }) |
| break |
| except Exception as e: |
| logger.error(f"ContextStore: 获取数据库上下文信息 (DialogContext) failed: {e}", exc_info=True) |
| await db.rollback() |
| return False |
| finally: |
| break |
| return contexts_info |
|
|
| async def delete_context_by_id(self, context_id: int, user_id: Optional[str] = None, is_admin: bool = False) -> bool: |
| """ |
| 通过数据库 ID 删除上下文条目。 |
| 如果不是管理员,则会校验 user_id。 |
| """ |
| if self.storage_mode == "memory": |
| |
| async with self.memory_lock: |
| key_to_delete = str(context_id) |
| if key_to_delete in self.memory_store: |
| if not is_admin and self.memory_store[key_to_delete].get("user_id") != user_id: |
| logger.warning(f"用户 {user_id} 尝试删除不属于自己的内存上下文 ID {key_to_delete}") |
| return False |
| del self.memory_store[key_to_delete] |
| logger.info(f"内存上下文 ID '{key_to_delete}' 已被删除。") |
| return True |
| return False |
| elif self.storage_mode == "database": |
| from app.core.dependencies import get_db_session |
| async for db in get_db_session(): |
| try: |
| stmt = select(DialogContext).where(DialogContext.id == context_id) |
| if not is_admin and user_id: |
| |
| stmt = stmt.where(DialogContext.proxy_key == user_id) |
| |
| result = await db.execute(stmt) |
| record_to_delete = result.scalar_one_or_none() |
|
|
| if record_to_delete: |
| await db.delete(record_to_delete) |
| await db.commit() |
| logger.info(f"ContextStore: 数据库上下文 ID '{context_id}' (DialogContext) 已被删除。") |
| return True |
| logger.warning(f"ContextStore: 尝试删除数据库上下文 ID '{context_id}' (DialogContext),但未找到或权限不足。") |
| return False |
| except Exception as e: |
| logger.error(f"ContextStore: 通过 ID 删除数据库上下文 (DialogContext) failed: {e}", exc_info=True) |
| await db.rollback() |
| return False |
| finally: |
| break |
| return False |
|
|
| |
|
|
| async def update_ttl(context_key: str, ttl_seconds: int, db: AsyncSession) -> Optional[bool]: |
| """ |
| 异步更新指定 DialogContext 记录的 TTL (通过更新 last_used_at 和可选的 ttl_seconds 字段)。 |
| """ |
| if not context_key: return False |
| try: |
| now_dt = datetime.now(timezone.utc) |
| values_to_update = {"last_used_at": now_dt} |
| if ttl_seconds is not None and ttl_seconds > 0 : |
| values_to_update["ttl_seconds"] = ttl_seconds |
| |
| stmt = ( |
| sqlalchemy_update(DialogContext) |
| .where(DialogContext.proxy_key == context_key) |
| .values(**values_to_update) |
| .execution_options(synchronize_session=False) |
| ) |
| result = await db.execute(stmt) |
| await db.commit() |
| |
| if result.rowcount > 0: |
| logger.info(f"成功使用 ORM 更新了 Key {context_key[:8]}... 的上下文 TTL 信息。") |
| return True |
| else: |
| logger.warning(f"尝试使用 ORM 更新 Key {context_key[:8]}... 的上下文 TTL,但未找到记录。") |
| return False |
| except Exception as e: |
| logger.error(f"使用 ORM 更新 Key {context_key[:8]}... 的上下文 TTL failed: {e}", exc_info=True) |
| await db.rollback() |
| return None |
|
|
| async def update_global_ttl(ttl_days: int) -> bool: |
| """ |
| 异步更新全局上下文 TTL 的天数设置。 |
| """ |
| from app.core.database.settings import set_ttl_days |
| try: |
| await set_ttl_days(ttl_days) |
| logger.info(f"全局上下文 TTL 已更新为 {ttl_days} 天。") |
| return True |
| except ValueError as ve: |
| logger.error(f"更新全局上下文 TTL failed: {ve}") |
| return False |
| except Exception as e: |
| logger.error(f"更新全局上下文 TTL 时发生意外错误: {e}", exc_info=True) |
| return False |
|
|
| async def get_all_contexts_with_ttl(db: AsyncSession) -> Dict[str, Dict[str, Any]]: |
| """ |
| 异步获取所有存储的 DialogContext 记录及其元信息,包括计算出的剩余 TTL 和内容摘要。 |
| 使用 SQLAlchemy ORM。 |
| """ |
| all_contexts_data = {} |
| try: |
| global_ttl_days = await get_ttl_days() |
| |
| stmt = select(DialogContext).order_by(DialogContext.last_used_at.desc()) |
| result = await db.execute(stmt) |
| records = result.scalars().all() |
|
|
| for record in records: |
| proxy_key = record.proxy_key |
| contents_json = record.contents |
| last_used_dt = record.last_used_at |
| record_ttl_seconds = record.ttl_seconds |
|
|
| context_summary = "N/A" |
| if contents_json: |
| try: |
| contents_list = await asyncio.to_thread(json.loads, contents_json) |
| if contents_list and isinstance(contents_list, list) and len(contents_list) > 0: |
| first_message = contents_list[0] |
| if isinstance(first_message, dict) and 'parts' in first_message and \ |
| isinstance(first_message['parts'], list) and len(first_message['parts']) > 0: |
| first_part = first_message['parts'][0] |
| if isinstance(first_part, dict) and 'text' in first_part: |
| content_text = str(first_part['text']) |
| context_summary = content_text[:100] + "..." if len(content_text) > 100 else content_text |
| elif isinstance(first_message, dict) and 'content' in first_message: |
| content_text = str(first_message['content']) |
| context_summary = content_text[:100] + "..." if len(content_text) > 100 else content_text |
| except Exception as e: |
| logger.warning(f"提取 Key {proxy_key[:8]}... 的上下文摘要时出错: {e}") |
|
|
| ttl_remaining_str = "N/A" |
| |
| effective_ttl_seconds: Optional[int] = None |
| if record_ttl_seconds is not None and record_ttl_seconds > 0: |
| effective_ttl_seconds = record_ttl_seconds |
| elif global_ttl_days > 0: |
| effective_ttl_seconds = global_ttl_days * 24 * 60 * 60 |
| |
| if last_used_dt and effective_ttl_seconds is not None and effective_ttl_seconds > 0: |
| |
| if last_used_dt.tzinfo is None: |
| last_used_dt = last_used_dt.replace(tzinfo=timezone.utc) |
| |
| expiry_time = last_used_dt + timedelta(seconds=effective_ttl_seconds) |
| now_utc = datetime.now(timezone.utc) |
| remaining_delta = expiry_time - now_utc |
| |
| if remaining_delta.total_seconds() > 0: |
| days = remaining_delta.days |
| hours, remainder = divmod(remaining_delta.seconds, 3600) |
| minutes, _ = divmod(remainder, 60) |
| if days > 0: ttl_remaining_str = f"{days}天 {hours}小时" |
| elif hours > 0: ttl_remaining_str = f"{hours}小时 {minutes}分钟" |
| else: ttl_remaining_str = f"{minutes}分钟" |
| else: |
| ttl_remaining_str = "已过期" |
| elif effective_ttl_seconds is None or effective_ttl_seconds <= 0 : |
| ttl_remaining_str = "永不" |
| |
| all_contexts_data[proxy_key] = { |
| "ttl": ttl_remaining_str, |
| "last_accessed": last_used_dt.strftime("%Y-%m-%d %H:%M:%S %Z") if last_used_dt else "N/A", |
| "context_summary": context_summary |
| } |
| logger.info(f"成功使用 ORM 获取了 {len(all_contexts_data)} 条上下文记录及其 TTL 信息。") |
|
|
| except Exception as e: |
| logger.error(f"使用 ORM 获取所有上下文及其 TTL 信息 failed: {e}", exc_info=True) |
| return all_contexts_data |
|
|