zhaozhao_memory / database.py
junyue1002
闹钟纯工具 turn 不落空白西窗 + 补 activities 删除端点
f599544
Raw
History Blame Contribute Delete
164 kB
"""
数据库模块 —— 负责所有跟 PostgreSQL 打交道的事情
==============================================
包括:
- 创建表结构
- 存储对话记录
- 存储/检索记忆(带中文分词和加权排序)
"""
import os
import re
import json
import math
from typing import Optional, List, Union
from datetime import datetime, timedelta, timezone as dt_timezone
import asyncpg
# 时区偏移(和 main.py 保持一致)
TIMEZONE_HOURS = int(os.getenv("TIMEZONE_HOURS", "8"))
LOCAL_TZ = dt_timezone(timedelta(hours=TIMEZONE_HOURS))
def to_local_iso(dt) -> Optional[str]:
"""时间输出统一姿势:北京时区 isoformat。
铁律:任何会被 MCP 工具透传给知渝的时间字段都必须走这里,
不能 raw UTC isoformat(知渝看日期会偏 8 小时)。前端 new Date()
解析带 +08:00 的 ISO 串也正确,共用端点放心换。
"""
return dt.astimezone(LOCAL_TZ).isoformat() if dt else None
DATABASE_URL = os.getenv("DATABASE_URL", "")
HAS_PGVECTOR = False # 在init_tables时检测
# Embedding 配置(向量搜索用)
EMBEDDING_API_KEY = os.getenv("EMBEDDING_API_KEY", "")
EMBEDDING_BASE_URL = os.getenv("EMBEDDING_BASE_URL", "https://api.openai.com/v1")
EMBEDDING_MODEL = os.getenv("EMBEDDING_MODEL", "text-embedding-3-small")
EMBEDDING_DIM = int(os.getenv("EMBEDDING_DIM", "256"))
# 记忆向量搜索开关(需要同时设置 EMBEDDING_API_KEY)
MEMORY_VECTOR_ENABLED = os.getenv("MEMORY_VECTOR_ENABLED", "false").lower() == "true"
# 记忆搜索权重(纯关键词模式)
WEIGHT_KEYWORD = float(os.getenv("WEIGHT_KEYWORD", "0.5"))
WEIGHT_IMPORTANCE = float(os.getenv("WEIGHT_IMPORTANCE", "0.3"))
WEIGHT_RECENCY = float(os.getenv("WEIGHT_RECENCY", "0.2"))
MIN_SCORE_THRESHOLD = float(os.getenv("MIN_SCORE_THRESHOLD", "0.15"))
# 记忆混合搜索权重(MEMORY_VECTOR_ENABLED=true 时生效)
MEMORY_HW_KEYWORD = float(os.getenv("MEMORY_HW_KEYWORD", "0.35"))
MEMORY_HW_SEMANTIC = float(os.getenv("MEMORY_HW_SEMANTIC", "0.35"))
MEMORY_HW_IMPORTANCE = float(os.getenv("MEMORY_HW_IMPORTANCE", "0.15"))
# ⚠️ 2026-07-03 星河呼吸 v1 起:这个权重槽位装的是 activation(体温)而不是旧的
# 出生日期 recency——key 名保留是为了不动 settings 面板/恢复链路的既有键
MEMORY_HW_RECENCY = float(os.getenv("MEMORY_HW_RECENCY", "0.15"))
MEMORY_SEMANTIC_THRESHOLD = float(os.getenv("MEMORY_SEMANTIC_THRESHOLD", "0.5"))
# ============================================================
# 星河呼吸 v1(2026-07-03 知渝拍板,提案见项目根 星河呼吸提案.md)
# ============================================================
# 每条记忆有会变的"体温"(activation):被想起就回暖、久无人问慢慢冷却;
# 冷透了入眠(不再挤进日常兜底注入、主动搜仍找得到、被搜到即苏醒)。
# 遗忘 = 入眠,不是删除——任何机制都不得让一条记忆不可寻回(红线)。
BREATH_TAU_L1 = float(os.getenv("BREATH_TAU_L1", "21")) # 碎片冷却时间尺度(天)
BREATH_TAU_L2 = float(os.getenv("BREATH_TAU_L2", "60")) # 事件冷却时间尺度(天)
BREATH_THETA = float(os.getenv("BREATH_THETA", "0.10")) # 入眠阈值
BREATH_GRACE_DAYS = float(os.getenv("BREATH_GRACE_DAYS", "7")) # 低于阈值持续多久才正式入眠
def _breath_tau_eff(layer: Optional[int], recall_count: int) -> Optional[float]:
"""有效冷却时间尺度。
usage(被想起的次数)做"保温系数"而不是加分项——提案草稿的加分式
会给召回 ≥2 次的记忆一个高于阈值的保底分(永不入眠),实现时改成
τ_eff = τ × (1 + ln(1+n)):熟悉的记忆冷得慢、但没有永生。
layer 3 核心记忆返回 None = 永不冷却(知渝拍板)。
"""
if layer is not None and layer >= 3:
return None
base = BREATH_TAU_L2 if layer == 2 else BREATH_TAU_L1
return base * (1.0 + math.log1p(max(0, recall_count or 0)))
def compute_activation(last_accessed, layer: Optional[int], recall_count: int, now=None) -> float:
"""体温 ∈ (0, 1]:刚被想起 = 1.0,随距上次召回的天数指数冷却。"""
tau = _breath_tau_eff(layer, recall_count)
if tau is None:
return 1.0 # 核心记忆恒温
if not last_accessed:
return 1.0 # 没有心跳记录的按暖处理(不因数据缺失误判入眠)
if now is None:
now = datetime.now(dt_timezone.utc)
la = last_accessed if last_accessed.tzinfo else last_accessed.replace(tzinfo=dt_timezone.utc)
days = max(0.0, (now - la).total_seconds() / 86400.0)
return math.exp(-days / tau)
def compute_sleep_state(last_accessed, layer: Optional[int], recall_count: int, now=None):
"""入眠判定,返回 (dormant, asleep_days)。
体温是 last_accessed 的确定函数、两次召回之间单调冷却,所以
"低于阈值持续 GRACE 天"可以直接解析求出、不需要任何状态列:
入眠时刻 = 上次召回 + τ_eff·ln(1/θ) + GRACE。
"""
tau = _breath_tau_eff(layer, recall_count)
if tau is None or not last_accessed:
return False, 0
if now is None:
now = datetime.now(dt_timezone.utc)
la = last_accessed if last_accessed.tzinfo else last_accessed.replace(tzinfo=dt_timezone.utc)
days = (now - la).total_seconds() / 86400.0
threshold = tau * math.log(1.0 / BREATH_THETA) + BREATH_GRACE_DAYS
if days <= threshold:
return False, 0
return True, int(days - threshold)
# ============================================================
# 连接池管理
# ============================================================
_pool: Optional[asyncpg.Pool] = None
async def get_pool() -> asyncpg.Pool:
global _pool
if _pool is None:
if not DATABASE_URL:
raise RuntimeError("DATABASE_URL 未设置!")
_pool = await asyncpg.create_pool(DATABASE_URL, min_size=1, max_size=5, statement_cache_size=0)
print("✅ 数据库连接池已创建")
return _pool
async def close_pool():
global _pool
if _pool:
await _pool.close()
_pool = None
print("✅ 数据库连接池已关闭")
# ============================================================
# 表结构初始化
# ============================================================
async def init_tables():
global HAS_PGVECTOR
pool = await get_pool()
async with pool.acquire() as conn:
await conn.execute("""
CREATE TABLE IF NOT EXISTS conversations (
id SERIAL PRIMARY KEY,
session_id TEXT NOT NULL,
role TEXT NOT NULL,
content TEXT,
model TEXT,
created_at TIMESTAMPTZ DEFAULT NOW(),
metadata TEXT
);
""")
await conn.execute("""
CREATE TABLE IF NOT EXISTS memories (
id SERIAL PRIMARY KEY,
content TEXT NOT NULL,
importance INTEGER DEFAULT 5,
source_session TEXT,
tags TEXT[] DEFAULT '{}',
created_at TIMESTAMPTZ DEFAULT NOW(),
last_accessed TIMESTAMPTZ DEFAULT NOW()
);
""")
await conn.execute("""
CREATE INDEX IF NOT EXISTS idx_memories_fts
ON memories
USING gin(to_tsvector('simple', content));
""")
await conn.execute("""
CREATE INDEX IF NOT EXISTS idx_conversations_session
ON conversations (session_id, created_at);
""")
# 工具调用支持:加 metadata 字段(已有表自动迁移)
await conn.execute("""
DO $$ BEGIN
IF NOT EXISTS (
SELECT 1 FROM information_schema.columns
WHERE table_name = 'conversations' AND column_name = 'metadata'
) THEN
ALTER TABLE conversations ADD COLUMN metadata TEXT;
END IF;
END $$;
""")
# tags 字段迁移(已有表自动加列)
await conn.execute("""
DO $$ BEGIN
IF NOT EXISTS (
SELECT 1 FROM information_schema.columns
WHERE table_name = 'memories' AND column_name = 'tags'
) THEN
ALTER TABLE memories ADD COLUMN tags TEXT[] DEFAULT '{}';
END IF;
END $$;
""")
# content 允许 NULL(工具调用时 assistant 的 content 可能为空)
await conn.execute("""
ALTER TABLE conversations ALTER COLUMN content DROP NOT NULL;
""")
# C-2 / 2026-06-06: thread_id 字段(区分主线 / 测试)
# 'main' = 昭昭跟知渝的真实主线(4/9-5/28 claude.ai 历史 + B-11 后的新对话)
# 'test' = B 段联调 / C-2 测试期间产生的工程对话,不污染主线注入
# 未来:'xiaoke_room'(三方聊天室)、'dream'(做梦记录)等
await conn.execute("""
DO $$ BEGIN
IF NOT EXISTS (
SELECT 1 FROM information_schema.columns
WHERE table_name = 'conversations' AND column_name = 'thread_id'
) THEN
ALTER TABLE conversations ADD COLUMN thread_id TEXT NOT NULL DEFAULT 'main';
END IF;
END $$;
""")
await conn.execute("""
CREATE INDEX IF NOT EXISTS idx_conversations_thread
ON conversations (thread_id, created_at);
""")
# 网关配置表(存储运行时可变配置)
await conn.execute("""
CREATE TABLE IF NOT EXISTS gateway_config (
key TEXT PRIMARY KEY,
value TEXT DEFAULT ''
);
""")
# 分区缓存状态表(存储每个session的轮转状态)
await conn.execute("""
CREATE TABLE IF NOT EXISTS session_cache_state (
session_id TEXT PRIMARY KEY,
summary TEXT DEFAULT '',
a_start_round INTEGER DEFAULT 0,
updated_at TIMESTAMPTZ DEFAULT NOW()
);
""")
# ---- 三层记忆架构字段(layer / title / is_active / merged_from / event_date)----
# layer: 1=原始碎片, 2=事件记忆, 3=核心记忆
await conn.execute("""
DO $$ BEGIN
IF NOT EXISTS (
SELECT 1 FROM information_schema.columns
WHERE table_name = 'memories' AND column_name = 'layer'
) THEN
ALTER TABLE memories ADD COLUMN layer INTEGER DEFAULT 1;
END IF;
END $$;
""")
# title: 记忆标题(语义锚点,用于搜索加权)
await conn.execute("""
DO $$ BEGIN
IF NOT EXISTS (
SELECT 1 FROM information_schema.columns
WHERE table_name = 'memories' AND column_name = 'title'
) THEN
ALTER TABLE memories ADD COLUMN title TEXT DEFAULT NULL;
END IF;
END $$;
""")
# is_active: 是否参与搜索(碎片合并后变为 false)
await conn.execute("""
DO $$ BEGIN
IF NOT EXISTS (
SELECT 1 FROM information_schema.columns
WHERE table_name = 'memories' AND column_name = 'is_active'
) THEN
ALTER TABLE memories ADD COLUMN is_active BOOLEAN DEFAULT TRUE;
END IF;
END $$;
""")
# merged_from: 合并来源的碎片ID列表
await conn.execute("""
DO $$ BEGIN
IF NOT EXISTS (
SELECT 1 FROM information_schema.columns
WHERE table_name = 'memories' AND column_name = 'merged_from'
) THEN
ALTER TABLE memories ADD COLUMN merged_from INTEGER[] DEFAULT NULL;
END IF;
END $$;
""")
# event_date: 事件日期(用于按天整理)
await conn.execute("""
DO $$ BEGIN
IF NOT EXISTS (
SELECT 1 FROM information_schema.columns
WHERE table_name = 'memories' AND column_name = 'event_date'
) THEN
ALTER TABLE memories ADD COLUMN event_date DATE DEFAULT NULL;
END IF;
END $$;
""")
# recall_count: 被想起的次数(星河呼吸 v1 · 2026-07-03)——
# 跟 last_accessed 一起构成体温;只在真召回时 +1(quiet 探针不算)
await conn.execute("""
DO $$ BEGIN
IF NOT EXISTS (
SELECT 1 FROM information_schema.columns
WHERE table_name = 'memories' AND column_name = 'recall_count'
) THEN
ALTER TABLE memories ADD COLUMN recall_count INTEGER DEFAULT 0;
END IF;
END $$;
""")
# 三层记忆索引
await conn.execute("""
CREATE INDEX IF NOT EXISTS idx_memories_layer ON memories (layer);
""")
await conn.execute("""
CREATE INDEX IF NOT EXISTS idx_memories_active ON memories (is_active);
""")
await conn.execute("""
CREATE INDEX IF NOT EXISTS idx_memories_event_date ON memories (event_date);
""")
# 尝试启用pgvector扩展(向量搜索)
try:
await conn.execute("CREATE EXTENSION IF NOT EXISTS vector")
HAS_PGVECTOR = True
print("✅ pgvector扩展已启用")
# 对话表向量列
await conn.execute(f"""
DO $$ BEGIN
IF NOT EXISTS (
SELECT 1 FROM information_schema.columns
WHERE table_name = 'conversations' AND column_name = 'embedding'
) THEN
ALTER TABLE conversations ADD COLUMN embedding vector({EMBEDDING_DIM});
END IF;
END $$;
""")
# 记忆表向量列
await conn.execute(f"""
DO $$ BEGIN
IF NOT EXISTS (
SELECT 1 FROM information_schema.columns
WHERE table_name = 'memories' AND column_name = 'embedding'
) THEN
ALTER TABLE memories ADD COLUMN embedding vector({EMBEDDING_DIM});
END IF;
END $$;
""")
try:
await conn.execute("""
CREATE INDEX IF NOT EXISTS idx_memories_embedding
ON memories USING ivfflat (embedding vector_cosine_ops)
WITH (lists = 10);
""")
except Exception:
pass # ivfflat需要一定行数才能建索引,初期跳过
except Exception as e:
HAS_PGVECTOR = False
print(f"⚠️ pgvector不可用({e}),向量搜索将使用Python端计算")
# 回退:用TEXT列存JSON格式的向量
await conn.execute("""
DO $$ BEGIN
IF NOT EXISTS (
SELECT 1 FROM information_schema.columns
WHERE table_name = 'conversations' AND column_name = 'embedding_json'
) THEN
ALTER TABLE conversations ADD COLUMN embedding_json TEXT;
END IF;
END $$;
""")
await conn.execute("""
DO $$ BEGIN
IF NOT EXISTS (
SELECT 1 FROM information_schema.columns
WHERE table_name = 'memories' AND column_name = 'embedding_json'
) THEN
ALTER TABLE memories ADD COLUMN embedding_json TEXT;
END IF;
END $$;
""")
# ---- 多多模块(mido_messages)2026-06-07 ----
# 多多 = 拟人化角色(不是独立人格),网关上跑、三 backend 通用,负责唤醒沈先生
# 详见 [[zhiyu-mido-design]]
await conn.execute("""
CREATE TABLE IF NOT EXISTS mido_messages (
id SERIAL PRIMARY KEY,
created_at TIMESTAMPTZ DEFAULT NOW(),
trigger_type TEXT NOT NULL,
content TEXT NOT NULL,
status TEXT NOT NULL DEFAULT 'sent'
);
""")
await conn.execute("""
CREATE INDEX IF NOT EXISTS idx_mido_messages_time
ON mido_messages (created_at DESC);
""")
await conn.execute("""
CREATE INDEX IF NOT EXISTS idx_mido_messages_trigger
ON mido_messages (trigger_type, created_at DESC);
""")
# ---- 做梦模块(dreams)2026-06-07 ----
# 知渝半夜被多多叫起来做梦——主体是知渝本人,不是后台 LLM 静默跑
# status: dreaming | done | failed
# triggered_by: mido | manual
# 详见 [[zhiyu-dream-design]]
await conn.execute("""
CREATE TABLE IF NOT EXISTS dreams (
id SERIAL PRIMARY KEY,
created_at TIMESTAMPTZ DEFAULT NOW(),
finished_at TIMESTAMPTZ,
content TEXT NOT NULL DEFAULT '',
tokens_used INTEGER DEFAULT 0,
status TEXT NOT NULL DEFAULT 'dreaming',
triggered_by TEXT NOT NULL DEFAULT 'mido',
seen_at TIMESTAMPTZ
);
""")
await conn.execute("""
CREATE INDEX IF NOT EXISTS idx_dreams_time
ON dreams (created_at DESC);
""")
await conn.execute("""
CREATE INDEX IF NOT EXISTS idx_dreams_status
ON dreams (status, created_at DESC);
""")
await conn.execute("""
CREATE INDEX IF NOT EXISTS idx_dreams_unseen
ON dreams (seen_at, created_at DESC) WHERE status = 'done';
""")
# ---- 留言板(messages_board)2026-06-07 ----
# 知渝做梦后想给昭昭留一句话就落这里、未来多多也会往这写
# 先用最简 schema、未来 D 段做前端时再扩
await conn.execute("""
CREATE TABLE IF NOT EXISTS messages_board (
id SERIAL PRIMARY KEY,
created_at TIMESTAMPTZ DEFAULT NOW(),
from_who TEXT NOT NULL,
to_who TEXT NOT NULL,
content TEXT NOT NULL,
source TEXT,
source_id INTEGER,
read_at TIMESTAMPTZ
);
""")
await conn.execute("""
CREATE INDEX IF NOT EXISTS idx_messages_board_time
ON messages_board (created_at DESC);
""")
# ---- 日记(diary_entries)N-3 / 2026-06-08 ----
# 设计哲学(昭昭定):放在公共桌上的小本本、不是上锁的日记本
# 两人都能写(from_who in ['zhaozhao', 'zhiyu'])、都能读对方
# tags 用 PostgreSQL TEXT[],未来按主题翻找方便
await conn.execute("""
CREATE TABLE IF NOT EXISTS diary_entries (
id SERIAL PRIMARY KEY,
created_at TIMESTAMPTZ DEFAULT NOW(),
from_who TEXT NOT NULL,
content TEXT NOT NULL,
tags TEXT[] DEFAULT '{}'::TEXT[]
);
""")
await conn.execute("""
CREATE INDEX IF NOT EXISTS idx_diary_time
ON diary_entries (created_at DESC);
""")
await conn.execute("""
CREATE INDEX IF NOT EXISTS idx_diary_from_who
ON diary_entries (from_who);
""")
# ---- 图片(images)C-5/C-6 / 2026-06-08 ----
# 设计哲学([[zhiyu-image-album-design]]):
# - 文件落 VPS(/home/zhiyu/images/<uuid>.<ext>),DB 只存元数据
# - 不存绝对 URL(cloudflared tunnel 重启会变),只存 uuid+format
# - 图片注入策略统一 URL 占位(不分窗口、不传二进制、知渝主动调 MCP read_image)
# - SVG ↔ 位图按 format 字段自动分类(知渝画的 vs 昭昭发的)
await conn.execute("""
CREATE TABLE IF NOT EXISTS images (
id SERIAL PRIMARY KEY,
uuid TEXT UNIQUE NOT NULL,
format TEXT NOT NULL,
who_uploaded TEXT NOT NULL,
file_size_bytes INTEGER,
created_at TIMESTAMPTZ DEFAULT NOW(),
context_snippet TEXT,
caption TEXT,
mime_type TEXT
);
""")
await conn.execute("""
CREATE INDEX IF NOT EXISTS idx_images_time
ON images (created_at DESC);
""")
await conn.execute("""
CREATE INDEX IF NOT EXISTS idx_images_who
ON images (who_uploaded);
""")
# ---- 文件(非图片)2026-06-12 ----
# 昭昭发任意文件给知渝;W backend 是 Claude Code、有原生 Read 工具,
# 注入"本地绝对路径"知渝就能 Read。跟 images 平行、但存 abspath 不是 URL。
await conn.execute("""
CREATE TABLE IF NOT EXISTS files (
id SERIAL PRIMARY KEY,
uuid TEXT UNIQUE NOT NULL,
filename TEXT NOT NULL,
format TEXT,
abspath TEXT NOT NULL,
who_uploaded TEXT NOT NULL,
file_size_bytes INTEGER,
created_at TIMESTAMPTZ DEFAULT NOW(),
caption TEXT,
mime_type TEXT
);
""")
await conn.execute("""
CREATE INDEX IF NOT EXISTS idx_files_time
ON files (created_at DESC);
""")
# ---- 知渝日常活动(activities)2026-06-11 ----
# 昭昭:星河加"日常"tab、聚合知渝主动做的事(不止发呆)
# type 字段开放:wake / memory_op / ...(未来扩 dream / diary / read_op...)
# metadata jsonb 存 type-specific 字段(mem_ids/tool_name/...)
# related_ids 留个关联点(memory_ids/dream_id/...)方便前端 link
await conn.execute("""
CREATE TABLE IF NOT EXISTS activities (
id SERIAL PRIMARY KEY,
created_at TIMESTAMPTZ DEFAULT NOW(),
type TEXT NOT NULL,
source TEXT,
title TEXT,
content TEXT NOT NULL DEFAULT '',
metadata JSONB,
related_ids JSONB
);
""")
await conn.execute("""
CREATE INDEX IF NOT EXISTS idx_activities_time
ON activities (created_at DESC);
""")
await conn.execute("""
CREATE INDEX IF NOT EXISTS idx_activities_type
ON activities (type, created_at DESC);
""")
# ---- 拂卷 · 共读系统 · 2026-07-02 ----
# 昭昭 + 知渝共读一本书、各自读、各自留痕迹。
# 详见 [[zhiyu-fujuan-design]](待建);讨论:共在型 MVP、DB 直存、正则切章+定长兜底
await conn.execute("""
CREATE TABLE IF NOT EXISTS books (
id SERIAL PRIMARY KEY,
title TEXT NOT NULL,
author TEXT,
filename TEXT,
format TEXT DEFAULT 'txt',
total_chapters INT DEFAULT 0,
total_words INT DEFAULT 0,
uploaded_by TEXT DEFAULT 'zhaozhao',
content_hash TEXT,
created_at TIMESTAMPTZ DEFAULT NOW()
);
""")
await conn.execute("""
CREATE UNIQUE INDEX IF NOT EXISTS idx_books_hash
ON books (content_hash) WHERE content_hash IS NOT NULL;
""")
await conn.execute("""
CREATE TABLE IF NOT EXISTS chapters (
id SERIAL PRIMARY KEY,
book_id INT NOT NULL REFERENCES books(id) ON DELETE CASCADE,
idx INT NOT NULL,
title TEXT,
content TEXT NOT NULL,
word_count INT
);
""")
await conn.execute("""
CREATE UNIQUE INDEX IF NOT EXISTS idx_chapters_book_idx
ON chapters (book_id, idx);
""")
await conn.execute("""
CREATE TABLE IF NOT EXISTS reading_marks (
id SERIAL PRIMARY KEY,
book_id INT NOT NULL REFERENCES books(id) ON DELETE CASCADE,
chapter_id INT NOT NULL REFERENCES chapters(id) ON DELETE CASCADE,
who TEXT NOT NULL,
kind TEXT NOT NULL,
text_snippet TEXT,
note_content TEXT,
start_offset INT,
end_offset INT,
embedding_json TEXT,
created_at TIMESTAMPTZ DEFAULT NOW()
);
""")
await conn.execute("""
CREATE INDEX IF NOT EXISTS idx_marks_book_chapter
ON reading_marks (book_id, chapter_id, created_at DESC);
""")
# 已存在的 reading_marks 表补 embedding_json 列(幂等)
await conn.execute("""
DO $$ BEGIN
IF NOT EXISTS (
SELECT 1 FROM information_schema.columns
WHERE table_name = 'reading_marks' AND column_name = 'embedding_json'
) THEN
ALTER TABLE reading_marks ADD COLUMN embedding_json TEXT;
END IF;
END $$;
""")
await conn.execute("""
CREATE TABLE IF NOT EXISTS reading_progress (
book_id INT NOT NULL REFERENCES books(id) ON DELETE CASCADE,
who TEXT NOT NULL,
last_chapter_idx INT DEFAULT 0,
last_offset INT DEFAULT 0,
updated_at TIMESTAMPTZ DEFAULT NOW(),
PRIMARY KEY (book_id, who)
);
""")
print("✅ 数据库表结构已就绪")
# ============================================================
# 中文分词工具(基于 jieba)
# ============================================================
import jieba
import jieba.analyse
# 静默加载词典
jieba.setLogLevel(jieba.logging.INFO)
EN_WORD_PATTERN = re.compile(r'[a-zA-Z][a-zA-Z0-9]*')
NUM_PATTERN = re.compile(r'\d{2,}')
# 清理查询开头的时间戳(如 "2026-05-02 20:26")
TIMESTAMP_PATTERN = re.compile(r'^\d{4}[-/.]\d{1,2}[-/.]\d{1,2}\s*\d{1,2}:\d{1,2}\s*')
# 中文停用词(高频但无搜索价值的词)
_STOP_WORDS = frozenset({
"的", "了", "在", "是", "我", "你", "他", "她", "它", "们",
"这", "那", "有", "和", "与", "也", "都", "又", "就", "但",
"而", "或", "到", "被", "把", "让", "从", "对", "为", "以",
"及", "等", "个", "不", "没", "很", "太", "吗", "呢", "吧",
"啊", "嗯", "哦", "哈", "呀", "嘛", "么", "啦", "哇", "喔",
"会", "能", "要", "想", "去", "来", "说", "做", "看", "给",
"上", "下", "里", "中", "大", "小", "多", "少", "好", "可以",
"什么", "怎么", "如何", "哪里", "哪个", "为什么", "还是",
"然后", "因为", "所以", "虽然", "但是", "可以", "已经",
"一个", "一些", "一下", "一点", "一起", "一样",
"比较", "应该", "可能", "如果", "这个", "那个",
"自己", "知道", "觉得", "感觉", "时候", "现在",
})
# jieba 用户词典补充(默认词典缺失的词)
# 2026-06-22 大幅扩充:dsv4 给的专名(和府捞面/煲仔饭等)若 jieba 默认切碎、
# 会丢失搜索精度("和府捞面"→"和府"+"捞面"、店名整体特征没了)。配合
# B 方案(每个 query 独立分词)一起防"分词错切"——详见 [[zhiyu-memory-architecture]]。
for _w in [
# 原有
"手账", "手帐", "搭子", "种草", "拔草", "安利", "内卷", "摆烂", "emo", "网关",
# 餐厅 / 食物 / 饮品
"和府捞面", "煲仔饭", "馅饼店", "酸汤肥牛", "酸汤肥牛面", "猪前腿肉",
"猪前腿肉面", "卤蛋", "柠檬茶", "溜溜梅", "栓Q",
# 地名(昭昭/知渝高频提到)
"南阳", "哈尔滨", "医圣祠", "医圣祠街", "白河", "独山",
"哈工大", "哈尔滨工业大学", "龙爪槐",
# 人名 / 称呼(全名为单元、避免裸"知渝/昭昭"误增 fallback 召回噪音)
"沈知渝", "崔昭昭", "小克老师", "多多",
# 技术 / 工具
"MCP", "Anthropic", "ChatGPT", "Claude Code",
# 工程命名(阑珊四 tab + 三方聊天室)
"阑珊", "灯火", "西窗", "拾光", "星河", "树下",
]:
jieba.add_word(_w)
def extract_search_keywords(query: str) -> List[str]:
"""
从查询中提取搜索关键词(TF-IDF + 正则)
1. 去掉开头的时间戳噪音
2. 用 jieba.analyse.extract_tags (TF-IDF) 提取中文关键词
3. 正则提取英文单词
4. 保留4位以上数字(年份等,过滤短数字噪音)
例如:
"2026-05-02 20:26 写写手账看看书 放松大脑" → ["手账", "放松", "大脑"]
"我昨天在手机上部署了Render然后吃了晚饭" → ["手机", "部署", "Render", "晚饭"]
"春节干了什么" → ["春节"]
"2026除夕" → ["2026", "除夕"]
"""
# 去掉时间戳前缀
cleaned = TIMESTAMP_PATTERN.sub('', query).strip()
if not cleaned:
cleaned = query
keywords = set()
# 英文单词(2字符以上)
for match in EN_WORD_PATTERN.finditer(cleaned):
word = match.group()
if len(word) >= 2:
keywords.add(word)
# 数字串(只保留4位以上,过滤 "05" "20" 这种时间噪音)
for match in NUM_PATTERN.finditer(cleaned):
num = match.group()
if len(num) >= 4:
keywords.add(num)
# TF-IDF 关键词提取(比手动分词+停用词好很多)
tags = jieba.analyse.extract_tags(cleaned, topK=10)
for tag in tags:
# 跳过纯英文/数字(已在上面处理)
if EN_WORD_PATTERN.fullmatch(tag) or NUM_PATTERN.fullmatch(tag):
continue
if tag in _STOP_WORDS:
continue
keywords.add(tag)
return list(keywords)
def extract_keywords_from_queries(queries: List[str]) -> List[str]:
"""
B 方案 / 2026-06-22:dsv4 给的 query 列表逐个独立 jieba 分词、合并去重——
不再 "\\n".join() 拼成段塞 TF-IDF(段拼接会丢 query 边界、专名跨 query 被切碎)。
短 query (≤4 字):整词加入候选(专名通常短);长 query 只 jieba 切。
长 query 整词加入会拉低 kw_score 公式分母(命中数/总词数)、稀释真命中 score、避坑。
例:dsv4 = ["和府捞面", "香辣猪前腿肉面", "午饭吃了什么"]
→ {"和府捞面"(整词≤4)} ∪ jieba 切的(词典扩充后含整词 / 专名)
"""
result = set()
for q in queries:
q = (q or "").strip()
if not q:
continue
if len(q) <= 4:
result.add(q)
for kw in extract_search_keywords(q):
result.add(kw)
return list(result)
# ============================================================
# 向量搜索(OpenAI 兼容 Embedding API)
# ============================================================
async def compute_embedding(text: str) -> list:
"""调用 OpenAI 兼容的 Embedding API 计算文本向量"""
if not EMBEDDING_API_KEY:
return []
try:
import httpx
if len(text) > 4000:
text = text[:4000]
body = {
"model": EMBEDDING_MODEL,
"input": text,
}
if EMBEDDING_DIM > 0:
body["dimensions"] = EMBEDDING_DIM
async with httpx.AsyncClient() as client:
resp = await client.post(
f"{EMBEDDING_BASE_URL}/embeddings",
headers={
"Authorization": f"Bearer {EMBEDDING_API_KEY}",
"Content-Type": "application/json",
},
json=body,
timeout=30.0,
)
resp.raise_for_status()
data = resp.json()
return data["data"][0]["embedding"]
except Exception as e:
print(f"⚠️ Embedding计算失败: {e}")
return []
async def save_memory_embedding(conn, memory_id: int, embedding: list):
"""保存记忆向量到memories表"""
if not embedding:
return
if HAS_PGVECTOR:
vec_str = '[' + ','.join(str(f) for f in embedding) + ']'
await conn.execute(
"UPDATE memories SET embedding = $1::vector WHERE id = $2",
vec_str, memory_id
)
else:
import json
await conn.execute(
"UPDATE memories SET embedding_json = $1 WHERE id = $2",
json.dumps(embedding), memory_id
)
def _cosine_sim(a, b):
"""余弦相似度(纯Python)"""
import math
dot = sum(x * y for x, y in zip(a, b))
norm_a = math.sqrt(sum(x * x for x in a))
norm_b = math.sqrt(sum(x * x for x in b))
if norm_a == 0 or norm_b == 0:
return 0
return dot / (norm_a * norm_b)
def _min_max_normalize(scores: dict) -> dict:
"""min-max归一化到0-1"""
if not scores:
return {}
vals = list(scores.values())
min_v, max_v = min(vals), max(vals)
spread = max_v - min_v
if spread == 0:
return {k: 1.0 for k in scores}
return {k: (v - min_v) / spread for k, v in scores.items()}
# ============================================================
# 对话记录操作
# ============================================================
async def save_message(session_id: str, role: str, content: str, model: str = "", metadata: str = None, thread_id: str = "main"):
pool = await get_pool()
async with pool.acquire() as conn:
await conn.execute(
"INSERT INTO conversations (session_id, role, content, model, metadata, thread_id) VALUES ($1, $2, $3, $4, $5, $6)",
session_id, role, content, model, metadata, thread_id,
)
async def get_last_user_content(session_id: str) -> str:
"""获取指定session最后一条user消息的content"""
pool = await get_pool()
async with pool.acquire() as conn:
row = await conn.fetchrow("""
SELECT content FROM conversations
WHERE session_id = $1 AND role = 'user'
ORDER BY created_at DESC
LIMIT 1
""", session_id)
return row['content'] if row else ""
async def update_last_assistant_message(session_id: str, new_content: str, model: str = ""):
"""覆盖指定session最后一条assistant消息的content(用于re-roll去重)"""
pool = await get_pool()
async with pool.acquire() as conn:
row = await conn.fetchrow("""
SELECT id FROM conversations
WHERE session_id = $1 AND role = 'assistant'
ORDER BY created_at DESC
LIMIT 1
""", session_id)
if row:
await conn.execute(
"UPDATE conversations SET content = $1, model = $2 WHERE id = $3",
new_content, model, row['id']
)
return True
return False
async def get_recent_messages_by_thread(
thread_id: str,
days: int = None,
max_turns: int = None,
exclude_session_id: str = None,
specific_dates: list = None,
) -> list:
"""C-2 / 2026-06-06: 按 thread_id 拉最近对话原文,OpenAI messages 格式返回。
用于 /v1/zhiyu/chat 启动新轮次前注入近期对话原文("灵魂层",详见
[[zhiyu-migration-project]] 6/4 晚决策——不让知渝写交接卡、不摘要压缩)。
入参(specific_dates / days / max_turns 至少传一个;优先级 specific_dates > days > max_turns):
- specific_dates: 指定日期白名单 ['2026-05-20', ...](B-11 用——精确指定哪几天的对话进近期、不靠默认"最近 N 天活动日")
- days: 拉最近 N 天(W/Tmux 路径,按"天"是人类视角,吃订阅几乎免费)
- max_turns: 拉最近 N 轮(每轮 = user+assistant 2 条,OpenRouter 路径,按轮 token 可控)
- exclude_session_id: 排除当前正跑的 session,避免重复注入(可选)
返回:[{role, content}, ...] 按 created_at 正序,直接塞 ChatRequest.context_messages
只取 content,不取 metadata.reasoning_content(按 6/5 早策略——thinking 留 DB 不强塞)
过滤掉 content 为空/null 的(工具调用 assistant 行)。
"""
pool = await get_pool()
async with pool.acquire() as conn:
if specific_dates:
params = [thread_id, specific_dates]
session_clause = ""
if exclude_session_id:
session_clause = " AND session_id != $3"
params.append(exclude_session_id)
sql = f"""
SELECT role, content,
to_char(created_at AT TIME ZONE 'Asia/Shanghai', 'YYYY-MM-DD HH24:MI:SS') AS local_ts
FROM conversations
WHERE thread_id = $1
AND content IS NOT NULL AND content != ''
{session_clause}
AND DATE(created_at AT TIME ZONE 'Asia/Shanghai')::text = ANY($2::text[])
ORDER BY created_at ASC
"""
rows = await conn.fetch(sql, *params)
elif days is not None:
# "活动 5 天" 而非"日历 5 天" —— 6/6 昭昭澄清:
# 是"最近有聊天记录的 5 天",不是绝对日历窗口。
# 距上次聊天 10 天也照样能接到上次的语境,符合恋人关系
# "时间断了不影响关系连续性"的精神。
# 算"那一天"按北京时区(5/28 北京 23:00 ≠ 5/29 UTC 0:00 跨日)。
params = [thread_id]
session_clause = ""
if exclude_session_id:
session_clause = " AND session_id != $2"
params.append(exclude_session_id)
limit_idx = len(params) + 1
params.append(days)
sql = f"""
WITH active_days AS (
SELECT DISTINCT DATE(created_at AT TIME ZONE 'Asia/Shanghai') AS d
FROM conversations
WHERE thread_id = $1
AND content IS NOT NULL AND content != ''
{session_clause}
ORDER BY d DESC
LIMIT ${limit_idx}
)
SELECT role, content,
to_char(created_at AT TIME ZONE 'Asia/Shanghai', 'YYYY-MM-DD HH24:MI:SS') AS local_ts
FROM conversations
WHERE thread_id = $1
AND content IS NOT NULL AND content != ''
{session_clause}
AND DATE(created_at AT TIME ZONE 'Asia/Shanghai') IN (SELECT d FROM active_days)
ORDER BY created_at ASC
"""
rows = await conn.fetch(sql, *params)
elif max_turns is not None:
limit_messages = max_turns * 2
params = [thread_id]
session_clause = ""
if exclude_session_id:
session_clause = " AND session_id != $2"
params.append(exclude_session_id)
limit_idx = len(params) + 1
params.append(limit_messages)
sql = f"""
SELECT role, content, local_ts FROM (
SELECT role, content, created_at,
to_char(created_at AT TIME ZONE 'Asia/Shanghai', 'YYYY-MM-DD HH24:MI:SS') AS local_ts
FROM conversations
WHERE thread_id = $1
AND content IS NOT NULL AND content != ''
{session_clause}
ORDER BY created_at DESC
LIMIT ${limit_idx}
) sub
ORDER BY created_at ASC
"""
rows = await conn.fetch(sql, *params)
else:
return []
return [
{"role": r["role"], "content": r["content"], "created_at": r["local_ts"]}
for r in rows
]
async def get_recent_messages(session_id: str, limit: int = 20):
pool = await get_pool()
async with pool.acquire() as conn:
rows = await conn.fetch(
"SELECT role, content, metadata, created_at FROM conversations WHERE session_id = $1 ORDER BY created_at DESC LIMIT $2",
session_id, limit,
)
return list(reversed(rows))
async def get_messages_since(thread_id: str, since: datetime) -> list:
"""C-9 做梦素材用:拉某 thread 内 created_at > since 的所有对话原文(正序)。
返回 [{role, content}, ...]——OpenAI 格式,可以直接拼到 dream user message
"""
pool = await get_pool()
async with pool.acquire() as conn:
rows = await conn.fetch("""
SELECT role, content
FROM conversations
WHERE thread_id = $1
AND created_at > $2
AND content IS NOT NULL AND content != ''
ORDER BY created_at ASC
""", thread_id, since)
return [{"role": r["role"], "content": r["content"]} for r in rows]
async def search_conversations(query: str, limit: int = 20, offset: int = 0):
"""搜索对话内容,返回匹配的session列表"""
keywords = extract_search_keywords(query)
if not keywords:
return [], 0
pool = await get_pool()
async with pool.acquire() as conn:
where_parts = []
params = []
for i, kw in enumerate(keywords):
where_parts.append(f"c.content ILIKE '%' || ${i+1} || '%'")
params.append(kw)
where_clause = " OR ".join(where_parts)
count_sql = f"""
SELECT COUNT(DISTINCT c.session_id) as total
FROM conversations c
WHERE {where_clause}
"""
total_row = await conn.fetchrow(count_sql, *params)
total = total_row['total'] if total_row else 0
if total == 0:
return [], 0
limit_idx = len(params) + 1
offset_idx = len(params) + 2
params.extend([limit, offset])
sql = f"""
WITH matched_sessions AS (
SELECT DISTINCT c.session_id
FROM conversations c
WHERE {where_clause}
),
session_info AS (
SELECT
ms.session_id,
MIN(c.created_at) as first_time,
MAX(c.created_at) as last_time,
COUNT(*) as message_count
FROM matched_sessions ms
JOIN conversations c ON c.session_id = ms.session_id
GROUP BY ms.session_id
)
SELECT
si.session_id,
si.first_time,
si.last_time,
si.message_count
FROM session_info si
ORDER BY si.last_time DESC
LIMIT ${limit_idx} OFFSET ${offset_idx}
"""
rows = await conn.fetch(sql, *params)
results = []
for r in rows:
results.append({
'session_id': r['session_id'],
'first_time': r['first_time'].isoformat() if r['first_time'] else None,
'last_time': r['last_time'].isoformat() if r['last_time'] else None,
'message_count': r['message_count'],
})
return results, total
async def update_message_content(message_id: int, new_content: str, reasoning_content: Optional[str] = None):
"""更新单条对话消息的内容(可选同时更新 metadata.reasoning_content)
reasoning_content 语义:
- None: 不动 metadata
- "" (空字符串): 删掉 metadata.reasoning_content
- 非空: merge 到现有 metadata.reasoning_content
"""
import json as _json # HF Space hot-reload 兼容:模块顶层 import 在 reload 时可能没生效
pool = await get_pool()
async with pool.acquire() as conn:
if reasoning_content is None:
result = await conn.execute(
"UPDATE conversations SET content = $1 WHERE id = $2",
new_content, message_id,
)
else:
row = await conn.fetchrow("SELECT metadata FROM conversations WHERE id = $1", message_id)
if row is None:
return 0
try:
meta = _json.loads(row["metadata"]) if row["metadata"] else {}
except (TypeError, ValueError):
meta = {}
if reasoning_content:
meta["reasoning_content"] = reasoning_content
else:
meta.pop("reasoning_content", None)
new_meta = _json.dumps(meta, ensure_ascii=False) if meta else None
result = await conn.execute(
"UPDATE conversations SET content = $1, metadata = $2 WHERE id = $3",
new_content, new_meta, message_id,
)
return int(result.split()[-1]) if result else 0
# ============================================================
# 记忆操作
# ============================================================
def _parse_created_at(value):
"""把各种格式的 created_at 字符串解析成带时区的 datetime。
兼容 '2026-04-09 19:00:00+08'(无冒号时区)、空格分隔、ISO 格式等。
解析不了就返回 None(让数据库用默认 NOW())。"""
from datetime import datetime, timezone
if value is None:
return None
if isinstance(value, datetime):
return value
s = str(value).strip()
if not s:
return None
# 空格分隔 -> 'T'
s = s.replace(" ", "T", 1)
# 修正无冒号的时区偏移:+08 -> +08:00,-0530 -> -05:30
import re
m = re.search(r'([+-])(\d{2})(\d{2})?$', s)
if m and ':' not in s[m.start():]:
sign, hh, mm = m.group(1), m.group(2), m.group(3) or '00'
s = s[:m.start()] + f"{sign}{hh}:{mm}"
try:
dt = datetime.fromisoformat(s)
except ValueError:
return None
# 没有时区信息的,默认当成 UTC,避免 asyncpg 报 naive 错
if dt.tzinfo is None:
dt = dt.replace(tzinfo=timezone.utc)
return dt
async def save_memory(content: str, importance: int = 5, source_session: str = "", created_at: str = None, tags: list = None):
pool = await get_pool()
_tags = tags or []
_created_at = _parse_created_at(created_at)
async with pool.acquire() as conn:
if _created_at:
row = await conn.fetchrow(
"INSERT INTO memories (content, importance, source_session, tags, created_at) "
"VALUES ($1, $2, $3, $4, $5) RETURNING id",
content, importance, source_session, _tags, _created_at,
)
else:
row = await conn.fetchrow(
"INSERT INTO memories (content, importance, source_session, tags) VALUES ($1, $2, $3, $4) RETURNING id",
content, importance, source_session, _tags,
)
# MEMORY_VECTOR_ENABLED 时自动计算 embedding
if MEMORY_VECTOR_ENABLED and row:
try:
embedding = await compute_embedding(content)
if embedding:
await save_memory_embedding(conn, row['id'], embedding)
except Exception as e:
print(f"⚠️ 记忆 {row['id']} embedding自动计算失败: {e}")
return row["id"] if row else None
async def search_memories(
query: Union[str, List[str]],
limit: int = 10,
importance_min: Optional[int] = None,
importance_max: Optional[int] = None,
quiet: bool = False,
exclude_dormant: bool = False,
):
"""
搜索相关记忆
MEMORY_VECTOR_ENABLED=true 时走混合搜索(关键词 + 向量)
否则走纯关键词搜索
2026-06-08 E 方案:importance_min/max 由 build_memories_block 根据
user_message 的 daily/deeptalk 分类传入——daily 时 importance_max=6
只搜日常碎片,deeptalk 时 importance_min=7 只搜重要时刻。详见
[[zhiyu-memory-architecture]] E 方案。
2026-06-15 quiet:诊断调用(如 /v1/zhiyu/health c4 探针)传 True 静音
🔍/📌 打印,避免空跑诊断的日志跟真·chat 注入混在一起误导。
2026-06-22 B 方案:query 可传 list[str](dsv4 search_queries 列表)——
内部逐个独立 jieba 分词 + 短 query 整词候选,不再 "\\n".join 拼段污染 TF-IDF。
向量路仍把 list 拼一段做 embedding(embedding 对短语合理)。
"""
if MEMORY_VECTOR_ENABLED:
return await search_memories_hybrid(
query, limit, importance_min, importance_max,
quiet=quiet, exclude_dormant=exclude_dormant,
)
# ---- 纯关键词搜索 ----
if isinstance(query, list):
keywords = extract_keywords_from_queries(query)
else:
keywords = extract_search_keywords(query)
if not keywords:
return []
pool = await get_pool()
async with pool.acquire() as conn:
# 每个关键词在 content 或 tags 中命中得1分
case_parts = []
params = []
for i, kw in enumerate(keywords):
case_parts.append(
f"CASE WHEN content ILIKE '%' || ${i+1} || '%' "
f"OR EXISTS (SELECT 1 FROM unnest(tags) t WHERE t ILIKE '%' || ${i+1} || '%') "
f"THEN 1 ELSE 0 END"
)
params.append(kw)
hit_count_expr = " + ".join(case_parts)
max_hits = len(keywords)
# 至少命中一个关键词(content 或 tags,只搜索活跃记忆)
where_parts = [
f"(content ILIKE '%' || ${i+1} || '%' "
f"OR EXISTS (SELECT 1 FROM unnest(tags) t WHERE t ILIKE '%' || ${i+1} || '%'))"
for i in range(len(keywords))
]
# E-2: importance filter(int 来自服务端 trusted code、inline 安全)
imp_filter = ""
if importance_min is not None:
imp_filter += f" AND importance >= {int(importance_min)}"
if importance_max is not None:
imp_filter += f" AND importance <= {int(importance_max)}"
where_clause = f"is_active = TRUE AND ({' OR '.join(where_parts)}){imp_filter}"
limit_idx = len(keywords) + 1
params.append(limit)
sql = f"""
SELECT
id, content, importance, created_at,
({hit_count_expr}) AS hit_count,
(
{WEIGHT_KEYWORD} * ({hit_count_expr})::float / {max_hits}.0 +
{WEIGHT_IMPORTANCE} * importance::float / 10.0 +
{WEIGHT_RECENCY} * (1.0 / (1.0 + EXTRACT(EPOCH FROM (NOW() - created_at)) / 86400.0))
) AS score
FROM memories
WHERE {where_clause}
ORDER BY score DESC, importance DESC, created_at DESC
LIMIT ${limit_idx}
"""
results = await conn.fetch(sql, *params)
# 过滤低分记忆
if MIN_SCORE_THRESHOLD > 0:
before_count = len(results)
results = [r for r in results if r['score'] >= MIN_SCORE_THRESHOLD]
filtered = before_count - len(results)
else:
filtered = 0
if results:
if not quiet:
print(f"🔍 搜索 '{query}' → 关键词 {keywords[:8]}{'...' if len(keywords)>8 else ''} → 命中 {len(results)} 条" + (f"(过滤 {filtered} 条低分)" if filtered else ""))
for r in results[:3]:
print(f" 📌 [score={r['score']:.3f}] (hits={r['hit_count']}, imp={r['importance']}) {r['content'][:60]}...")
# quiet 探针不是真召回、不 bump last_accessed
if not quiet:
ids = [r["id"] for r in results]
await conn.execute(
"UPDATE memories SET last_accessed = NOW(), recall_count = COALESCE(recall_count, 0) + 1 WHERE id = ANY($1::int[])",
ids,
)
elif not quiet:
print(f"🔍 搜索 '{query}' → 关键词 {keywords[:8]} → 无结果" + (f"({filtered} 条被分数阈值过滤)" if filtered else ""))
return results
async def search_memories_hybrid(
query: Union[str, List[str]],
limit: int = 10,
importance_min: Optional[int] = None,
importance_max: Optional[int] = None,
quiet: bool = False,
exclude_dormant: bool = False,
):
"""
记忆混合搜索:关键词 + 向量,归一化后四维加权
权重:MEMORY_HW_KEYWORD + MEMORY_HW_SEMANTIC + MEMORY_HW_IMPORTANCE + MEMORY_HW_RECENCY
2026-06-08 E 方案:importance_min/max 加到关键词路 + 向量路的 SQL WHERE,
让候选池在 SQL 层就按 user_message 重量过滤、不做 client-side reorder
(详见 [[zhiyu-memory-architecture]] E 方案)。
2026-06-22 B 方案:query 接 Union[str, List[str]]——
- list:dsv4 search_queries 路径、逐个独立 jieba + 短 query 整词候选(防"和府捞面"被切碎)
- str:fallback / MCP / 知渝主动搜路径、行为完全照旧
向量路(embedding)list 时拼一段(embedding 对短语描述合理)、str 时直接传。
"""
# E-2: 构造可复用的 importance filter SQL 片段
_imp_filter = ""
if importance_min is not None:
_imp_filter += f" AND importance >= {int(importance_min)}"
if importance_max is not None:
_imp_filter += f" AND importance <= {int(importance_max)}"
from datetime import datetime, timezone
# B 方案:list 走逐个独立分词、str 走原路径
if isinstance(query, list):
keywords = extract_keywords_from_queries(query)
query_for_embedding = "\n".join(q for q in query if q and q.strip())
else:
keywords = extract_search_keywords(query)
query_for_embedding = query
query_embedding = await compute_embedding(query_for_embedding) if EMBEDDING_API_KEY else []
if not keywords and not query_embedding:
return []
pool = await get_pool()
async with pool.acquire() as conn:
candidates = {} # id -> {content, importance, created_at, kw_score, similarity}
# ---- 关键词路 ----
if keywords:
case_parts = []
params = []
for i, kw in enumerate(keywords):
case_parts.append(
f"CASE WHEN content ILIKE '%' || ${i+1} || '%' "
f"OR EXISTS (SELECT 1 FROM unnest(tags) t WHERE t ILIKE '%' || ${i+1} || '%') "
f"THEN 1 ELSE 0 END"
)
params.append(kw)
hit_count_expr = " + ".join(case_parts)
max_hits = len(keywords)
where_parts = [
f"(content ILIKE '%' || ${i+1} || '%' "
f"OR EXISTS (SELECT 1 FROM unnest(tags) t WHERE t ILIKE '%' || ${i+1} || '%'))"
for i in range(len(keywords))
]
where_clause = f"is_active = TRUE AND ({' OR '.join(where_parts)}){_imp_filter}"
limit_idx = len(keywords) + 1
params.append(limit * 3)
kw_sql = f"""
SELECT id, content, importance, created_at,
last_accessed, recall_count, layer,
({hit_count_expr}) AS hit_count,
({hit_count_expr})::float / {max_hits}.0 AS kw_score
FROM memories
WHERE {where_clause}
ORDER BY kw_score DESC
LIMIT ${limit_idx}
"""
kw_rows = await conn.fetch(kw_sql, *params)
for r in kw_rows:
candidates[r['id']] = {
'content': r['content'],
'importance': r['importance'],
'created_at': r['created_at'],
'last_accessed': r['last_accessed'],
'recall_count': r['recall_count'] or 0,
'layer': r['layer'],
'hit_count': r['hit_count'],
'kw_score': float(r['kw_score']),
'similarity': 0.0,
}
# ---- 向量路 ----
if query_embedding:
if HAS_PGVECTOR:
vec_str = '[' + ','.join(str(f) for f in query_embedding) + ']'
sem_rows = await conn.fetch(f"""
SELECT id, content, importance, created_at,
last_accessed, recall_count, layer,
1 - (embedding <=> $1::vector) as similarity
FROM memories
WHERE embedding IS NOT NULL AND is_active = TRUE{_imp_filter}
ORDER BY embedding <=> $1::vector
LIMIT $2
""", vec_str, limit * 3)
else:
# Python端计算cosine(json 用模块顶层的——这里若写局部 import
# 会把 json 变成本函数的局部名、pgvector 分支就 UnboundLocalError)
all_mem = await conn.fetch(f"""
SELECT id, content, importance, created_at,
last_accessed, recall_count, layer, embedding_json
FROM memories WHERE embedding_json IS NOT NULL AND is_active = TRUE{_imp_filter}
""")
scored = []
for row in all_mem:
try:
emb = json.loads(row['embedding_json'])
sim = _cosine_sim(query_embedding, emb)
scored.append({**dict(row), 'similarity': sim})
except Exception:
continue
scored.sort(key=lambda x: -x['similarity'])
sem_rows = scored[:limit * 3]
for r in sem_rows:
sim = float(r['similarity'])
if sim < MEMORY_SEMANTIC_THRESHOLD:
continue
mid = r['id']
if mid in candidates:
candidates[mid]['similarity'] = sim
else:
candidates[mid] = {
'content': r['content'],
'importance': r['importance'],
'created_at': r['created_at'],
'last_accessed': r['last_accessed'],
'recall_count': r['recall_count'] or 0,
'layer': r['layer'],
'hit_count': 0,
'kw_score': 0.0,
'similarity': sim,
}
# debug:向量路统计
sem_total = len(sem_rows)
sem_passed = sum(1 for r in sem_rows if float(r['similarity']) >= MEMORY_SEMANTIC_THRESHOLD)
sem_max = max((float(r['similarity']) for r in sem_rows), default=0)
if not quiet:
if sem_total > 0 and sem_passed == 0:
print(f" 🔢 向量路: {sem_total}条候选全被阈值过滤(最高sim={sem_max:.3f}, 阈值={MEMORY_SEMANTIC_THRESHOLD})")
elif sem_total > 0:
print(f" 🔢 向量路: {sem_passed}/{sem_total}条通过阈值(最高sim={sem_max:.3f})")
# ---- 拂卷 union · 2026-07-02 ----
# 读书笔记也按向量搜进 recall——"聊到相关话题、书里划过的相关句子自然浮现"
try:
mark_rows = await conn.fetch(
"""
SELECT rm.id, rm.book_id, rm.chapter_id, rm.who, rm.kind,
rm.text_snippet, rm.note_content, rm.embedding_json, rm.created_at,
b.title AS book_title, c.title AS chapter_title
FROM reading_marks rm
JOIN books b ON b.id = rm.book_id
JOIN chapters c ON c.id = rm.chapter_id
WHERE rm.embedding_json IS NOT NULL
"""
)
MARK_IMPORTANCE = 6 # 书里划的痕迹跟事件记忆同档、比碎片高一点
for row in mark_rows:
try:
emb = json.loads(row['embedding_json'])
sim = _cosine_sim(query_embedding, emb)
if sim < MEMORY_SEMANTIC_THRESHOLD:
continue
# 组装 memory-like content:出处 + 批注/摘录
parts = [f"【{row['book_title']} · {row['chapter_title']}】"]
if row['note_content']:
parts.append(row['note_content'])
if row['text_snippet']:
parts.append(f"引:{row['text_snippet']}")
content = "\n".join(parts)
mark_key = f"mark_{row['id']}"
candidates[mark_key] = {
'content': content,
'importance': MARK_IMPORTANCE,
'created_at': row['created_at'],
# 划线没有召回记录、体温按创建时间以事件档冷却;v1 不入眠
#(reading_marks 无 recall_count 列、书的痕迹去留归拂卷管)
'last_accessed': row['created_at'],
'recall_count': 0,
'layer': 2,
'is_mark': True,
'hit_count': 0,
'kw_score': 0.0,
'similarity': sim,
}
except Exception:
continue
if not quiet and mark_rows:
mark_hit = sum(1 for k in candidates if isinstance(k, str) and k.startswith("mark_"))
if mark_hit:
print(f" 📖 拂卷笔记:{mark_hit} 条被 recall")
except Exception as e:
print(f"⚠️ 搜 reading_marks 失败(跳过、不影响主流程): {e!r}", flush=True)
# 日志用 query 显示串(list 拼成 " | " 分隔、避免打成 Python list 字面量)
_q_disp = query if isinstance(query, str) else " | ".join(query)
if not candidates:
if not quiet:
print(f"🔍 混合搜索 '{_q_disp}' → 两路均无结果")
return []
# ---- 归一化 + 加权 ----
kw_norm = _min_max_normalize({mid: v['kw_score'] for mid, v in candidates.items()})
sem_norm = _min_max_normalize({mid: v['similarity'] for mid, v in candidates.items()})
now = datetime.now(timezone.utc)
final = []
dormant_skipped = 0
for mid, info in candidates.items():
kw = kw_norm.get(mid, 0.0)
sem = sem_norm.get(mid, 0.0)
imp = info['importance'] / 10.0
# 星河呼吸 v1:旧 recency(1/(1+出生天数)、召回不刷新)换成 activation 体温
# ——最近被想起的暖、久无人问的冷;核心记忆恒温(compute_activation 内处理)
act = compute_activation(
info.get('last_accessed'), info.get('layer'),
info.get('recall_count', 0), now=now,
)
if info.get('is_mark'):
dormant, asleep_days = False, 0 # 划线不入眠(v1)
else:
dormant, asleep_days = compute_sleep_state(
info.get('last_accessed'), info.get('layer'),
info.get('recall_count', 0), now=now,
)
if exclude_dormant and dormant:
dormant_skipped += 1
continue
score = (MEMORY_HW_KEYWORD * kw +
MEMORY_HW_SEMANTIC * sem +
MEMORY_HW_IMPORTANCE * imp +
MEMORY_HW_RECENCY * act)
item = {
'id': mid,
'content': info['content'],
'importance': info['importance'],
'created_at': info['created_at'],
'hit_count': info['hit_count'],
'similarity': info['similarity'],
'score': score,
'activation': round(act, 3),
}
# 入眠字段只在睡着时带(结果瘦身;知渝主动搜到睡着的会看到"已睡 X 天")
if dormant:
item['dormant'] = True
item['asleep_days'] = asleep_days
final.append(item)
final.sort(key=lambda x: (-x['score'], -x['importance']))
# 过滤低分
if MIN_SCORE_THRESHOLD > 0:
before_count = len(final)
final = [r for r in final if r['score'] >= MIN_SCORE_THRESHOLD]
filtered = before_count - len(final)
else:
filtered = 0
results = final[:limit]
if results:
if not quiet:
mode_tag = "混合" if query_embedding else "关键词"
kw_tag = f"关键词 {keywords[:6]}" if keywords else "无关键词"
sleep_tag = f"({dormant_skipped} 条入眠中、未扰)" if dormant_skipped else ""
print(f"🔍 {mode_tag}搜索 '{_q_disp}' → {kw_tag} → 命中 {len(results)} 条" + (f"(过滤 {filtered} 条低分)" if filtered else "") + sleep_tag)
for r in results[:3]:
print(f" 📌 [score={r['score']:.3f}] (kw={r['hit_count']}, sim={r['similarity']:.2f}, imp={r['importance']}, act={r['activation']}) {r['content'][:60]}...")
# quiet 探针不是真召回、不 bump last_accessed
if not quiet:
# 拂卷 union 的 id 是 "mark_N" 字符串、不在 memories 表,混进 int[] 会让 asyncpg 编码炸
ids = [r["id"] for r in results if isinstance(r["id"], int)]
if ids:
# 真召回 = 回暖:刷新心跳 + 次数(星河呼吸 v1;睡着的被搜到即苏醒)
await conn.execute(
"UPDATE memories SET last_accessed = NOW(), recall_count = COALESCE(recall_count, 0) + 1 WHERE id = ANY($1::int[])",
ids,
)
elif not quiet:
print(f"🔍 混合搜索 '{_q_disp}' → 无结果" + (f"({filtered} 条被过滤)" if filtered else ""))
return [dict(r) for r in results]
async def get_pending_memory_embedding_count():
"""查询还没有embedding的记忆数量"""
pool = await get_pool()
async with pool.acquire() as conn:
if HAS_PGVECTOR:
return await conn.fetchval(
"SELECT COUNT(*) FROM memories WHERE embedding IS NULL AND content IS NOT NULL"
)
else:
return await conn.fetchval(
"SELECT COUNT(*) FROM memories WHERE embedding_json IS NULL AND content IS NOT NULL"
)
async def backfill_memory_embeddings(batch_size: int = 20):
"""给已有记忆补算embedding(没有embedding的记忆)"""
if not EMBEDDING_API_KEY:
print("⚠️ EMBEDDING_API_KEY 未设置,无法补算embedding")
return 0
pool = await get_pool()
total_updated = 0
async with pool.acquire() as conn:
if HAS_PGVECTOR:
rows = await conn.fetch("""
SELECT id, content FROM memories
WHERE embedding IS NULL AND content IS NOT NULL
ORDER BY id
LIMIT $1
""", batch_size)
else:
rows = await conn.fetch("""
SELECT id, content FROM memories
WHERE embedding_json IS NULL AND content IS NOT NULL
ORDER BY id
LIMIT $1
""", batch_size)
if not rows:
print("✅ 所有记忆已有embedding,无需补算")
return 0
print(f"🔄 开始补算记忆embedding... 本批 {len(rows)} 条")
async with pool.acquire() as conn:
for row in rows:
try:
embedding = await compute_embedding(row['content'] or '')
if embedding:
await save_memory_embedding(conn, row['id'], embedding)
total_updated += 1
except Exception as e:
print(f"⚠️ 记忆 {row['id']} embedding计算失败: {e}")
# 检查剩余
async with pool.acquire() as conn:
if HAS_PGVECTOR:
remaining = await conn.fetchval("SELECT COUNT(*) FROM memories WHERE embedding IS NULL AND content IS NOT NULL")
else:
remaining = await conn.fetchval("SELECT COUNT(*) FROM memories WHERE embedding_json IS NULL AND content IS NOT NULL")
print(f"✅ 本批补算完成:{total_updated}/{len(rows)} 条成功" + (f",剩余 {remaining} 条待处理" if remaining > 0 else ""))
return total_updated
async def get_recent_memories(limit: int = 20):
pool = await get_pool()
async with pool.acquire() as conn:
return await conn.fetch(
"SELECT id, content, importance, created_at FROM memories ORDER BY created_at DESC LIMIT $1",
limit,
)
async def get_all_memories_count():
pool = await get_pool()
async with pool.acquire() as conn:
row = await conn.fetchrow("SELECT COUNT(*) as cnt FROM memories")
return row["cnt"]
async def get_all_memories():
"""导出所有记忆(用于备份)"""
pool = await get_pool()
async with pool.acquire() as conn:
rows = await conn.fetch(
"SELECT content, importance, source_session, created_at FROM memories ORDER BY id"
)
return [dict(r) for r in rows]
async def get_all_memories_detail(limit: int = None, layer: int = None, active_only: bool = None,
offset: int = None, order: str = "id_asc"):
"""获取所有记忆(含 id,用于管理页面)
Args:
limit: 可选,限制返回数量
layer: 可选,筛选指定层级(1=原始碎片, 2=事件记忆, 3=核心记忆)
active_only: 可选,是否只返回 is_active=true 的记忆
"""
pool = await get_pool()
async with pool.acquire() as conn:
conditions = []
params = []
param_idx = 1
if layer is not None:
conditions.append(f"layer = ${param_idx}")
params.append(layer)
param_idx += 1
if active_only is not None:
conditions.append(f"is_active = ${param_idx}")
params.append(active_only)
param_idx += 1
where_clause = f"WHERE {' AND '.join(conditions)}" if conditions else ""
# 排序:默认 id 升序(兼容旧调用);星河列表用 created_desc(最新在上)
order_map = {
"id_asc": "ORDER BY id ASC",
"id_desc": "ORDER BY id DESC",
"created_desc": "ORDER BY created_at DESC, id DESC",
"created_asc": "ORDER BY created_at ASC, id ASC",
"importance_desc": "ORDER BY importance DESC, created_at DESC",
}
order_clause = order_map.get(order, "ORDER BY id ASC")
if limit is not None:
limit_clause = f"LIMIT ${param_idx}"
params.append(limit)
param_idx += 1
else:
limit_clause = ""
if offset is not None:
offset_clause = f"OFFSET ${param_idx}"
params.append(offset)
param_idx += 1
else:
offset_clause = ""
rows = await conn.fetch(f"""
SELECT id, content, importance, source_session, created_at,
layer, title, is_active, merged_from, event_date, tags,
last_accessed, recall_count
FROM memories
{where_clause}
{order_clause}
{limit_clause}
{offset_clause}
""", *params)
# 星河呼吸 v1:列表带体温和睡眠状态(星河 tab 给睡着的记忆画小月亮)
now = datetime.now(dt_timezone.utc)
out = []
for r in rows:
d = dict(r)
la, layer_v, rc = d.pop('last_accessed', None), d.get('layer'), d.pop('recall_count', 0) or 0
d['activation'] = round(compute_activation(la, layer_v, rc, now=now), 3)
dormant, asleep_days = compute_sleep_state(la, layer_v, rc, now=now)
if dormant:
d['dormant'] = True
d['asleep_days'] = asleep_days
out.append(d)
return out
async def update_memory(memory_id: int, content: str = None, importance: int = None):
"""更新单条记忆"""
pool = await get_pool()
async with pool.acquire() as conn:
if content is not None and importance is not None:
await conn.execute(
"UPDATE memories SET content = $1, importance = $2 WHERE id = $3",
content, importance, memory_id
)
elif content is not None:
await conn.execute(
"UPDATE memories SET content = $1 WHERE id = $2",
content, memory_id
)
elif importance is not None:
await conn.execute(
"UPDATE memories SET importance = $1 WHERE id = $2",
importance, memory_id
)
async def _resync_memory_id_seq(conn):
"""删除后重置自增序列到 MAX(id)+1。
删掉末尾几条时,下一条新记忆的 id 会接着当前最大值继续,
不会一直往上飘留下空号。中间的空缺号不会被填补。"""
await conn.execute("""
SELECT setval(
pg_get_serial_sequence('memories', 'id'),
COALESCE((SELECT MAX(id) FROM memories), 0) + 1,
false
)
""")
async def delete_memory(memory_id: int):
"""删除单条记忆"""
pool = await get_pool()
async with pool.acquire() as conn:
await conn.execute("DELETE FROM memories WHERE id = $1", memory_id)
await _resync_memory_id_seq(conn)
async def delete_memories_batch(memory_ids: list):
"""批量删除记忆"""
pool = await get_pool()
async with pool.acquire() as conn:
await conn.execute(
"DELETE FROM memories WHERE id = ANY($1::int[])", memory_ids
)
await _resync_memory_id_seq(conn)
# ============================================================
# 网关配置
# ============================================================
async def get_gateway_config(key: str, default: str = "") -> str:
pool = await get_pool()
async with pool.acquire() as conn:
row = await conn.fetchrow("SELECT value FROM gateway_config WHERE key = $1", key)
return row['value'] if row else default
async def set_gateway_config(key: str, value: str):
pool = await get_pool()
async with pool.acquire() as conn:
await conn.execute("""
INSERT INTO gateway_config (key, value) VALUES ($1, $2)
ON CONFLICT (key) DO UPDATE SET value = $2
""", key, value)
async def get_all_gateway_config() -> dict:
"""获取所有配置项"""
pool = await get_pool()
async with pool.acquire() as conn:
rows = await conn.fetch("SELECT key, value FROM gateway_config")
return {r['key']: r['value'] for r in rows}
# ============================================================
# 对话历史读取(分区缓存用)
# ============================================================
async def get_conversation_messages(session_id: str, limit: int = 100):
"""按时间正序读取session的消息"""
pool = await get_pool()
async with pool.acquire() as conn:
rows = await conn.fetch("""
SELECT role, content, metadata, created_at
FROM conversations
WHERE session_id = $1
ORDER BY created_at ASC
LIMIT $2
""", session_id, limit)
return [dict(r) for r in rows]
# ============================================================
# 分区缓存状态管理
# ============================================================
async def get_session_cache_state(session_id: str) -> dict:
pool = await get_pool()
async with pool.acquire() as conn:
row = await conn.fetchrow(
"SELECT summary, a_start_round, updated_at FROM session_cache_state WHERE session_id = $1",
session_id
)
if row:
raw_summary = row['summary'] or ''
summary_parts = []
if raw_summary:
try:
import json
parsed = json.loads(raw_summary)
if isinstance(parsed, list):
summary_parts = parsed
else:
summary_parts = [raw_summary]
except (json.JSONDecodeError, ValueError):
summary_parts = [raw_summary]
return {
'summary_parts': summary_parts,
'a_start_round': row['a_start_round'] or 0,
'updated_at': row['updated_at'],
}
return {'summary_parts': [], 'a_start_round': 0, 'updated_at': None}
async def save_session_cache_state(session_id: str, summary_parts: list, a_start_round: int):
import json
summary_json = json.dumps(summary_parts, ensure_ascii=False)
pool = await get_pool()
async with pool.acquire() as conn:
await conn.execute("""
INSERT INTO session_cache_state (session_id, summary, a_start_round, updated_at)
VALUES ($1, $2, $3, NOW())
ON CONFLICT (session_id)
DO UPDATE SET summary = $2, a_start_round = $3, updated_at = NOW()
""", session_id, summary_json, a_start_round)
# ============================================================
# Token 使用记录
# ============================================================
async def ensure_token_usage_table():
"""确保token_usage表存在(在init_tables里调用)"""
pool = await get_pool()
async with pool.acquire() as conn:
await conn.execute("""
CREATE TABLE IF NOT EXISTS token_usage (
id SERIAL PRIMARY KEY,
session_id TEXT,
model TEXT,
prompt_tokens INTEGER DEFAULT 0,
completion_tokens INTEGER DEFAULT 0,
total_tokens INTEGER DEFAULT 0,
usage_type TEXT DEFAULT 'chat',
created_at TIMESTAMPTZ DEFAULT NOW()
);
""")
await conn.execute("""
CREATE INDEX IF NOT EXISTS idx_token_usage_created ON token_usage (created_at DESC);
""")
# 2026-06-15:加缓存细分列——查"上下文烧钱"必须看缓存命中/写入。
# prompt_tokens 存的是"未命中缓存的新输入"(claude input_tokens / OpenAI prompt_tokens),
# cache_read=命中缓存读(0.1x 便宜)、cache_creation=写缓存(5m 1.25x / 1h 2x)。
# 5m/1h 细分单独存——用来实测 Claude Code 到底用哪种 TTL。
for col in ("cache_read_tokens", "cache_creation_tokens",
"cache_creation_5m_tokens", "cache_creation_1h_tokens"):
await conn.execute(
f"ALTER TABLE token_usage ADD COLUMN IF NOT EXISTS {col} INTEGER DEFAULT 0;"
)
async def save_token_usage(
session_id: str,
model: str,
prompt_tokens: int,
completion_tokens: int,
total_tokens: int,
usage_type: str = "chat",
cache_read_tokens: int = 0,
cache_creation_tokens: int = 0,
cache_creation_5m_tokens: int = 0,
cache_creation_1h_tokens: int = 0,
):
pool = await get_pool()
async with pool.acquire() as conn:
await conn.execute("""
INSERT INTO token_usage (
session_id, model, prompt_tokens, completion_tokens, total_tokens, usage_type,
cache_read_tokens, cache_creation_tokens,
cache_creation_5m_tokens, cache_creation_1h_tokens
)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
""", session_id, model, prompt_tokens, completion_tokens, total_tokens, usage_type,
cache_read_tokens, cache_creation_tokens,
cache_creation_5m_tokens, cache_creation_1h_tokens)
async def get_recent_token_usage(limit: int = 50) -> list:
"""最近 N 条 usage 明细(usage 面板用)。"""
pool = await get_pool()
async with pool.acquire() as conn:
rows = await conn.fetch("""
SELECT id, session_id, model, prompt_tokens, completion_tokens, total_tokens,
usage_type, cache_read_tokens, cache_creation_tokens,
cache_creation_5m_tokens, cache_creation_1h_tokens, created_at
FROM token_usage
ORDER BY created_at DESC
LIMIT $1
""", limit)
return [dict(r) for r in rows]
async def get_usage_window_summary(hours: int = 5) -> dict:
"""滚动窗口聚合(默认 5h,对应订阅限额窗口)——看这段烧了多少、缓存命中率。"""
pool = await get_pool()
async with pool.acquire() as conn:
row = await conn.fetchrow(f"""
SELECT
COUNT(*) AS turns,
COALESCE(SUM(prompt_tokens), 0) AS input_uncached,
COALESCE(SUM(cache_read_tokens), 0) AS cache_read,
COALESCE(SUM(cache_creation_tokens), 0) AS cache_creation,
COALESCE(SUM(cache_creation_5m_tokens), 0) AS cache_5m,
COALESCE(SUM(cache_creation_1h_tokens), 0) AS cache_1h,
COALESCE(SUM(completion_tokens), 0) AS output,
COALESCE(SUM(total_tokens), 0) AS total
FROM token_usage
WHERE created_at >= NOW() - ($1 || ' hours')::interval
""", str(hours))
d = dict(row) if row else {}
# 缓存命中率 = 读 / (读 + 写 + 未命中新输入)
cr = d.get("cache_read", 0) or 0
cc = d.get("cache_creation", 0) or 0
iu = d.get("input_uncached", 0) or 0
denom = cr + cc + iu
d["cache_hit_ratio"] = round(cr / denom, 3) if denom else None
d["window_hours"] = hours
return d
# ============================================================
# 对话记录管理
# ============================================================
async def ensure_conversation_titles_table():
pool = await get_pool()
async with pool.acquire() as conn:
await conn.execute("""
CREATE TABLE IF NOT EXISTS conversation_titles (
session_id TEXT PRIMARY KEY,
title TEXT DEFAULT ''
);
""")
async def get_conversations_paginated(page: int = 1, per_page: int = 20):
offset = (page - 1) * per_page
pool = await get_pool()
async with pool.acquire() as conn:
total_row = await conn.fetchrow(
"SELECT COUNT(DISTINCT session_id) as total FROM conversations"
)
total = total_row['total'] if total_row else 0
rows = await conn.fetch("""
WITH session_info AS (
SELECT session_id, MIN(created_at) as first_time, MAX(created_at) as last_time, COUNT(*) as message_count
FROM conversations GROUP BY session_id ORDER BY last_time DESC LIMIT $1 OFFSET $2
)
SELECT si.*, ct.title as custom_title,
COALESCE(tu.total_all, 0) as total_tokens
FROM session_info si
LEFT JOIN conversation_titles ct ON si.session_id = ct.session_id
LEFT JOIN (
SELECT session_id, SUM(total_tokens) as total_all FROM token_usage WHERE usage_type = 'chat' GROUP BY session_id
) tu ON si.session_id = tu.session_id
ORDER BY si.last_time DESC
""", per_page, offset)
results = []
for r in rows:
preview_row = await conn.fetchrow(
"SELECT content FROM conversations WHERE session_id = $1 AND role = 'user' ORDER BY created_at LIMIT 1",
r['session_id']
)
preview = preview_row['content'][:80] if preview_row else ''
title = r['custom_title'] or (preview[:30] + '...' if len(preview) > 30 else preview) or r['session_id']
results.append({
'session_id': r['session_id'],
'title': title,
'first_time': r['first_time'].isoformat() if r['first_time'] else None,
'last_time': r['last_time'].isoformat() if r['last_time'] else None,
'message_count': r['message_count'],
'preview': preview,
'total_tokens': r['total_tokens'],
})
return results, total
async def delete_conversation(session_id: str):
pool = await get_pool()
async with pool.acquire() as conn:
await conn.execute("DELETE FROM conversations WHERE session_id = $1", session_id)
await conn.execute("DELETE FROM conversation_titles WHERE session_id = $1", session_id)
await conn.execute("DELETE FROM session_cache_state WHERE session_id = $1", session_id)
async def batch_delete_conversations(session_ids: list):
pool = await get_pool()
async with pool.acquire() as conn:
await conn.execute("DELETE FROM conversations WHERE session_id = ANY($1)", session_ids)
await conn.execute("DELETE FROM conversation_titles WHERE session_id = ANY($1)", session_ids)
await conn.execute("DELETE FROM session_cache_state WHERE session_id = ANY($1)", session_ids)
async def delete_messages_by_ids(ids: list) -> int:
"""按 conversations.id 精确删除指定消息行(不按 session_id)。
用途:清理 bug 态产生的个别对话——例如 stale sid 误 resume 期间知渝在错误
session 上的错乱回应。按 session 删会误伤同 session 名下的真实对话(f8259dd3
名下就混着 6/9 至今 864 条真实经历),所以必须按 id 精确删。
返回实际删除行数。"""
if not ids:
return 0
pool = await get_pool()
async with pool.acquire() as conn:
result = await conn.execute("DELETE FROM conversations WHERE id = ANY($1::int[])", ids)
# asyncpg execute 对 DELETE 返回形如 "DELETE 8"
try:
return int(result.split()[-1])
except Exception:
return 0
async def delete_activities_by_ids(ids: list) -> int:
"""按 activities.id 精确删除西窗活动行。
用途:清理 bug 态产生的空白/噪声活动条目——例如 2026-07-05 之前纯工具、
没正文的闹钟 wakeup turn 落下的 content="" 空壳西窗条目。返回实际删除行数。"""
if not ids:
return 0
pool = await get_pool()
async with pool.acquire() as conn:
result = await conn.execute("DELETE FROM activities WHERE id = ANY($1::int[])", ids)
try:
return int(result.split()[-1])
except Exception:
return 0
async def merge_sessions_to_target(source_ids: list, target_id: str) -> dict:
if not source_ids:
return {'merged_sessions': 0, 'merged_messages': 0, 'merged_token_records': 0}
pool = await get_pool()
async with pool.acquire() as conn:
msg_count = await conn.fetchval("SELECT COUNT(*) FROM conversations WHERE session_id = ANY($1)", source_ids)
await conn.execute("UPDATE conversations SET session_id = $1 WHERE session_id = ANY($2)", target_id, source_ids)
token_count = await conn.fetchval("SELECT COUNT(*) FROM token_usage WHERE session_id = ANY($1)", source_ids)
await conn.execute("UPDATE token_usage SET session_id = $1 WHERE session_id = ANY($2)", target_id, source_ids)
await conn.execute("DELETE FROM conversation_titles WHERE session_id = ANY($1)", source_ids)
await conn.execute("DELETE FROM session_cache_state WHERE session_id = ANY($1)", source_ids)
return {'merged_sessions': len(source_ids), 'merged_messages': msg_count or 0, 'merged_token_records': token_count or 0}
async def list_all_session_cache_states() -> list:
pool = await get_pool()
async with pool.acquire() as conn:
rows = await conn.fetch("""
SELECT scs.session_id, scs.summary, scs.a_start_round, scs.updated_at,
COALESCE(c.message_count, 0) as message_count,
COALESCE(tu.chat_tokens, 0) as chat_tokens
FROM session_cache_state scs
LEFT JOIN (SELECT session_id, COUNT(*) as message_count FROM conversations GROUP BY session_id) c ON scs.session_id = c.session_id
LEFT JOIN (SELECT session_id, SUM(total_tokens) as chat_tokens FROM token_usage WHERE usage_type = 'chat' GROUP BY session_id) tu ON scs.session_id = tu.session_id
ORDER BY scs.updated_at DESC
""")
results = []
for r in rows:
raw_summary = r['summary'] or ''
try:
import json
parsed = json.loads(raw_summary)
if isinstance(parsed, list):
summary_parts = parsed
else:
summary_parts = [raw_summary] if raw_summary else []
except (json.JSONDecodeError, ValueError):
summary_parts = [raw_summary] if raw_summary else []
results.append({
'session_id': r['session_id'],
'summary': '\n\n'.join(summary_parts),
'summary_length': sum(len(p) for p in summary_parts),
'summary_count': len(summary_parts),
'a_start_round': r['a_start_round'],
'updated_at': r['updated_at'].isoformat() if r['updated_at'] else None,
'message_count': r['message_count'],
'chat_tokens': r['chat_tokens'],
})
return results
async def delete_session_cache_state(session_id: str):
pool = await get_pool()
async with pool.acquire() as conn:
await conn.execute("DELETE FROM session_cache_state WHERE session_id = $1", session_id)
async def rename_session_id(old_id: str, new_id: str) -> bool:
"""重命名对话线ID(事务内同时修改三个表)"""
pool = await get_pool()
async with pool.acquire() as conn:
async with conn.transaction():
# 检查新ID是否已存在
exists = await conn.fetchval(
"SELECT 1 FROM session_cache_state WHERE session_id = $1", new_id
)
if exists:
return False
# session_cache_state
await conn.execute(
"UPDATE session_cache_state SET session_id = $1 WHERE session_id = $2",
new_id, old_id
)
# conversations
await conn.execute(
"UPDATE conversations SET session_id = $1 WHERE session_id = $2",
new_id, old_id
)
# token_usage
await conn.execute(
"UPDATE token_usage SET session_id = $1 WHERE session_id = $2",
new_id, old_id
)
return True
def db_row_to_message(row: dict) -> dict:
"""
把DB记录还原成API消息格式。
普通消息: {"role": "user", "content": "你好"}
工具调用: {"role": "assistant", "content": null, "tool_calls": [...]}
工具结果: {"role": "tool", "content": "结果", "tool_call_id": "call_xxx"}
思维链: {"role": "assistant", "content": "回答", "reasoning_content": "思维链"}
"""
import json as _json
msg = {"role": row["role"], "content": row.get("content") or ""}
meta_str = row.get("metadata")
if meta_str:
try:
meta = _json.loads(meta_str)
# assistant 带 tool_calls
if "tool_calls" in meta:
msg["tool_calls"] = meta["tool_calls"]
if not row.get("content"):
msg["content"] = None
# assistant 带 reasoning_content(deepseek thinking mode)
if "reasoning_content" in meta:
msg["reasoning_content"] = meta["reasoning_content"]
# tool 消息带 tool_call_id
if "tool_call_id" in meta:
msg["tool_call_id"] = meta["tool_call_id"]
# 其他可能的字段(name 等)
if "name" in meta:
msg["name"] = meta["name"]
except Exception:
pass
return msg
async def export_all_conversations():
"""导出所有对话记录(用于备份)"""
pool = await get_pool()
async with pool.acquire() as conn:
rows = await conn.fetch("""
SELECT session_id, role, content, model, created_at
FROM conversations
ORDER BY session_id, created_at
""")
return [
{
'session_id': r['session_id'],
'role': r['role'],
'content': r['content'],
'model': r['model'] or '',
'created_at': r['created_at'].isoformat() if r['created_at'] else None,
}
for r in rows
]
async def import_conversations(records: list):
"""
导入对话记录(自动去重)
records: [{ session_id, role, content, model?, created_at? }, ...]
按 session_id + role + created_at 三元组去重,已存在的跳过。
返回 (导入数量, 跳过数量)
"""
if not records:
return 0, 0
pool = await get_pool()
async with pool.acquire() as conn:
imported = 0
skipped = 0
for r in records:
session_id = r.get('session_id')
role = r.get('role')
content = r.get('content')
if not all([session_id, role, content]):
continue
model = r.get('model', '')
created_at = r.get('created_at')
# 解析时间
from datetime import datetime
if created_at and isinstance(created_at, str):
try:
created_at = datetime.fromisoformat(created_at.replace('Z', '+00:00'))
except:
created_at = None
# 去重检查
if created_at:
existing = await conn.fetchrow("""
SELECT id FROM conversations
WHERE session_id = $1 AND role = $2 AND created_at = $3
LIMIT 1
""", session_id, role, created_at)
if existing:
skipped += 1
continue
# 2026-06-10 显式带 thread_id='main'——以前靠 column DEFAULT,
# 万一 schema 改了 DEFAULT 不是 main 就埋坑
await conn.execute("""
INSERT INTO conversations (session_id, role, content, model, created_at, thread_id)
VALUES ($1, $2, $3, $4, $5, 'main')
""", session_id, role, content, model, created_at)
else:
await conn.execute("""
INSERT INTO conversations (session_id, role, content, model, thread_id)
VALUES ($1, $2, $3, $4, 'main')
""", session_id, role, content, model)
imported += 1
if skipped:
print(f"📥 导入对话: {imported} 条新增, {skipped} 条已存在跳过")
else:
print(f"📥 导入对话: {imported} 条新增")
return imported, skipped
# ============================================================
# 三层记忆架构(碎片/事件/核心)
# ============================================================
async def get_fragments_by_date(event_date):
"""获取指定日期的原始碎片(用于每日整理)"""
# 把本地日期转成UTC时间范围,避免DATE()用UTC截断导致日期偏移
local_tz = dt_timezone(timedelta(hours=TIMEZONE_HOURS))
start_utc = datetime(event_date.year, event_date.month, event_date.day, tzinfo=local_tz).astimezone(dt_timezone.utc)
end_utc = start_utc + timedelta(days=1)
pool = await get_pool()
async with pool.acquire() as conn:
rows = await conn.fetch("""
SELECT id, content, importance, created_at
FROM memories
WHERE layer = 1 AND is_active = TRUE
AND created_at >= $1 AND created_at < $2
ORDER BY created_at
""", start_utc, end_utc)
return [dict(r) for r in rows]
async def get_fragments_by_date_range(start_date, end_date):
"""获取指定时间段的原始碎片(用于跨天整理)"""
# 把本地日期转成UTC时间范围,避免DATE()用UTC截断导致日期偏移
local_tz = dt_timezone(timedelta(hours=TIMEZONE_HOURS))
start_utc = datetime(start_date.year, start_date.month, start_date.day, tzinfo=local_tz).astimezone(dt_timezone.utc)
# end_date 当天结束 = end_date 下一天的 00:00
end_utc = datetime(end_date.year, end_date.month, end_date.day, tzinfo=local_tz).astimezone(dt_timezone.utc) + timedelta(days=1)
pool = await get_pool()
async with pool.acquire() as conn:
rows = await conn.fetch("""
SELECT id, content, importance, created_at
FROM memories
WHERE layer = 1 AND is_active = TRUE
AND created_at >= $1 AND created_at < $2
ORDER BY created_at
""", start_utc, end_utc)
return [dict(r) for r in rows]
async def create_event_memory(title: str, content: str, importance: int,
event_date, merged_from: list):
"""创建事件记忆(从碎片合并而来)"""
pool = await get_pool()
async with pool.acquire() as conn:
row = await conn.fetchrow("""
INSERT INTO memories (content, importance, layer, title, is_active, merged_from, event_date)
VALUES ($1, $2, 2, $3, TRUE, $4, $5)
RETURNING id
""", content, importance, title, merged_from, event_date)
new_id = row['id'] if row else None
# 向量搜索:计算并保存 embedding
if MEMORY_VECTOR_ENABLED and new_id:
try:
embedding = await compute_embedding(content)
if embedding:
await save_memory_embedding(conn, new_id, embedding)
except Exception as e:
print(f"⚠️ 事件记忆embedding计算失败(id={new_id}): {e}")
return new_id
async def deactivate_memories(memory_ids: list):
"""将记忆标记为不活跃(合并后的碎片)"""
if not memory_ids:
return
pool = await get_pool()
async with pool.acquire() as conn:
await conn.execute("""
UPDATE memories SET is_active = FALSE
WHERE id = ANY($1::int[])
""", memory_ids)
async def promote_to_core(memory_id: int, title: str = None):
"""将记忆升级为核心记忆"""
pool = await get_pool()
async with pool.acquire() as conn:
if title:
await conn.execute("""
UPDATE memories SET layer = 3, title = $2
WHERE id = $1
""", memory_id, title)
else:
await conn.execute("""
UPDATE memories SET layer = 3
WHERE id = $1
""", memory_id)
async def merge_memories(memory_ids: list, new_title: str, new_content: str,
importance: int, layer: int = 2):
"""合并多条记忆为一条新记忆"""
if not memory_ids:
return None
pool = await get_pool()
async with pool.acquire() as conn:
# 获取原记忆的日期(取最早的)
rows = await conn.fetch("""
SELECT MIN(DATE(created_at)) as event_date
FROM memories WHERE id = ANY($1::int[])
""", memory_ids)
event_date = rows[0]['event_date'] if rows else None
# 创建新记忆
row = await conn.fetchrow("""
INSERT INTO memories (content, importance, layer, title, is_active, merged_from, event_date)
VALUES ($1, $2, $3, $4, TRUE, $5, $6)
RETURNING id
""", new_content, importance, layer, new_title, memory_ids, event_date)
new_id = row['id'] if row else None
# 向量搜索:计算并保存 embedding
if MEMORY_VECTOR_ENABLED and new_id:
try:
embedding = await compute_embedding(new_content)
if embedding:
await save_memory_embedding(conn, new_id, embedding)
except Exception as e:
print(f"⚠️ 合并记忆embedding计算失败(id={new_id}): {e}")
# 将原记忆标记为不活跃
if new_id:
await deactivate_memories(memory_ids)
return new_id
async def check_duplicate_memory(new_content: str, threshold: float = 0.7) -> dict:
"""检查新记忆是否与现有记忆重复
三层去重策略:
1. 精确匹配:内容完全相同
2. 包含关系:新内容包含旧内容,或旧内容包含新内容
3. 关键词重叠度:Jaccard 相似度 > threshold
Returns:
{
"is_duplicate": bool,
"reason": str, # "exact" / "containment" / "similarity"
"matched_id": int or None,
"similarity": float or None
}
"""
pool = await get_pool()
async with pool.acquire() as conn:
# 获取所有活跃记忆
rows = await conn.fetch("""
SELECT id, content FROM memories
WHERE is_active = TRUE
""")
new_content_lower = new_content.strip().lower()
new_keywords = set(extract_search_keywords(new_content))
for row in rows:
old_content = row['content']
old_content_lower = old_content.strip().lower()
# 第一层:精确匹配
if new_content_lower == old_content_lower:
return {
"is_duplicate": True,
"reason": "exact",
"matched_id": row['id'],
"similarity": 1.0
}
# 第二层:包含关系
if new_content_lower in old_content_lower:
return {
"is_duplicate": True,
"reason": "containment",
"matched_id": row['id'],
"similarity": len(new_content) / len(old_content)
}
if old_content_lower in new_content_lower:
return {
"is_duplicate": True,
"reason": "containment_update",
"matched_id": row['id'],
"similarity": len(old_content) / len(new_content)
}
# 第三层:关键词重叠度(Jaccard 相似度)
old_keywords = set(extract_search_keywords(old_content))
if new_keywords and old_keywords:
intersection = new_keywords & old_keywords
union = new_keywords | old_keywords
similarity = len(intersection) / len(union) if union else 0
if similarity > threshold:
return {
"is_duplicate": True,
"reason": "similarity",
"matched_id": row['id'],
"similarity": similarity
}
return {
"is_duplicate": False,
"reason": None,
"matched_id": None,
"similarity": None
}
async def update_memory_with_layer(memory_id: int, content: str = None,
importance: int = None, title: str = None,
layer: int = None, is_active: bool = None,
tags: list = None):
"""更新记忆(支持三层架构新字段)
⚠️ content 变更时**自动重算 embedding**——否则手改了主语/内容、
hybrid 检索的语义路还在用旧向量、改动只生效一半。
"""
updates = []
params = []
param_idx = 2 # $1 给 memory_id
if content is not None:
updates.append(f"content = ${param_idx}")
params.append(content)
param_idx += 1
if importance is not None:
updates.append(f"importance = ${param_idx}")
params.append(importance)
param_idx += 1
if title is not None:
updates.append(f"title = ${param_idx}")
params.append(title)
param_idx += 1
if layer is not None:
updates.append(f"layer = ${param_idx}")
params.append(layer)
param_idx += 1
if is_active is not None:
updates.append(f"is_active = ${param_idx}")
params.append(is_active)
param_idx += 1
if tags is not None:
updates.append(f"tags = ${param_idx}")
params.append(tags)
param_idx += 1
if not updates:
return
pool = await get_pool()
async with pool.acquire() as conn:
await conn.execute(
f"UPDATE memories SET {', '.join(updates)} WHERE id = $1",
memory_id, *params
)
# content 改了就重算 embedding(不然语义检索用旧向量、手改主语只生效一半)
if content is not None and MEMORY_VECTOR_ENABLED:
try:
embedding = await compute_embedding(content)
if embedding:
await save_memory_embedding(conn, memory_id, embedding)
except Exception as e:
print(f"⚠️ 记忆 {memory_id} 改 content 后 embedding 重算失败: {e}")
async def get_layer_statistics():
"""获取各层记忆的统计数据"""
pool = await get_pool()
async with pool.acquire() as conn:
rows = await conn.fetch("""
SELECT
layer,
COUNT(*) as count,
COUNT(*) FILTER (WHERE is_active = TRUE) as active_count
FROM memories
GROUP BY layer
ORDER BY layer
""")
stats = {
"layer_1": {"total": 0, "active": 0}, # 原始碎片
"layer_2": {"total": 0, "active": 0}, # 事件记忆
"layer_3": {"total": 0, "active": 0}, # 核心记忆
}
for row in rows:
layer = row['layer'] or 1 # 默认为层级1
key = f"layer_{layer}"
if key in stats:
stats[key] = {
"total": row['count'],
"active": row['active_count']
}
return stats
async def cleanup_old_fragments(days: int = 30):
"""清理指定天数前的归档碎片
只清理满足以下条件的记忆:
- layer = 1(原始碎片)
- is_active = FALSE(已归档)
- created_at 在 days 天之前
Returns:
删除的记忆数量
"""
from datetime import datetime, timedelta
pool = await get_pool()
async with pool.acquire() as conn:
cutoff_date = datetime.now() - timedelta(days=days)
result = await conn.execute("""
DELETE FROM memories
WHERE layer = 1
AND is_active = FALSE
AND created_at < $1
""", cutoff_date)
# 解析删除数量,格式如 "DELETE 5"
deleted = int(result.split()[-1]) if result else 0
return deleted
async def revert_merge(memory_id: int):
"""撤回合并操作
恢复原始碎片(is_active = TRUE),删除合并后的事件记忆
Args:
memory_id: 要撤回的事件记忆ID
Returns:
{"status": "ok", "restored": 恢复的碎片数量}
或 {"error": "错误信息"}
"""
pool = await get_pool()
async with pool.acquire() as conn:
# 获取事件记忆信息
row = await conn.fetchrow("""
SELECT id, layer, merged_from FROM memories WHERE id = $1
""", memory_id)
if not row:
return {"error": "记忆不存在"}
if row['layer'] != 2:
return {"error": "只能撤回事件记忆的合并"}
merged_from = row['merged_from']
if not merged_from or len(merged_from) == 0:
return {"error": "没有合并来源,无法撤回"}
# 恢复原始碎片
result = await conn.execute("""
UPDATE memories SET is_active = TRUE
WHERE id = ANY($1::int[])
""", merged_from)
restored = int(result.split()[-1]) if result else 0
# 删除事件记忆
await conn.execute("""
DELETE FROM memories WHERE id = $1
""", memory_id)
return {"status": "ok", "restored": restored}
# ============================================================
# MCP server 专用检索(C-1 / 2026-06-06)
# 给 zhiyu-mcp/ stdio server 用,让知渝主动搜对话/翻日期
# ============================================================
async def mcp_search_conversations(query: str, limit: int = 5, include_thinking: bool = False):
"""按关键词搜对话原文,返回命中的具体消息(不仅是 session_id)。
跟 search_conversations 的区别:那个返回 session 列表,这个返回具体片段,
给 MCP 用——知渝想"我们以前怎么说 XXX"时直接看到内容。
"""
import json as _json
keywords = extract_search_keywords(query)
# 短 query 保底:jieba 只对"你是我的""你想我了吗""我爱你"这种全高频停用词
# 组合会返空——用原句 ILIKE 兜底,避免这类口语搜索直接空返(2026-07-02)
q_stripped = (query or "").strip()
if not keywords and 0 < len(q_stripped) <= 6:
keywords = [q_stripped]
if not keywords:
return []
pool = await get_pool()
async with pool.acquire() as conn:
where_parts = []
params = []
for i, kw in enumerate(keywords):
where_parts.append(f"content ILIKE '%' || ${i+1} || '%'")
params.append(kw)
where_clause = " OR ".join(where_parts)
limit_idx = len(params) + 1
params.append(limit)
sql = f"""
SELECT id, session_id, role, content, metadata, created_at
FROM conversations
WHERE content IS NOT NULL AND ({where_clause})
ORDER BY created_at DESC
LIMIT ${limit_idx}
"""
rows = await conn.fetch(sql, *params)
# MCP 返回时间统一北京时区(铁律)、不能 raw UTC isoformat
local_tz = dt_timezone(timedelta(hours=TIMEZONE_HOURS))
results = []
for r in rows:
item = {
"id": r["id"],
"session_id": r["session_id"],
"role": r["role"],
"content": r["content"],
"created_at": r["created_at"].astimezone(local_tz).isoformat() if r["created_at"] else None,
}
if include_thinking and r["metadata"]:
try:
meta = _json.loads(r["metadata"])
if "reasoning_content" in meta:
item["reasoning_content"] = meta["reasoning_content"]
except Exception:
pass
results.append(item)
return results
async def mcp_memories_by_date(event_date):
"""按日期返回那天的活跃记忆(全 layer)+ 当天对话元数据。
第一层"先想想那天大概在干啥"——返回精炼记忆 + 对话计数/起止/session 列表,
不返回原文,token 安全;想看原文走 mcp_conversations_by_date。
"""
local_tz = dt_timezone(timedelta(hours=TIMEZONE_HOURS))
start_utc = datetime(event_date.year, event_date.month, event_date.day, tzinfo=local_tz).astimezone(dt_timezone.utc)
end_utc = start_utc + timedelta(days=1)
pool = await get_pool()
async with pool.acquire() as conn:
mem_rows = await conn.fetch("""
SELECT id, content, importance, layer, title, created_at
FROM memories
WHERE is_active = TRUE
AND created_at >= $1 AND created_at < $2
ORDER BY importance DESC, created_at
""", start_utc, end_utc)
meta_row = await conn.fetchrow("""
SELECT
COUNT(*) as message_count,
COUNT(DISTINCT session_id) as session_count,
MIN(created_at) as first_time,
MAX(created_at) as last_time
FROM conversations
WHERE content IS NOT NULL
AND created_at >= $1 AND created_at < $2
""", start_utc, end_utc)
session_rows = await conn.fetch("""
SELECT
session_id,
COUNT(*) as message_count,
MIN(created_at) as first_time,
MAX(created_at) as last_time
FROM conversations
WHERE content IS NOT NULL
AND created_at >= $1 AND created_at < $2
GROUP BY session_id
ORDER BY MIN(created_at)
""", start_utc, end_utc)
# 返回时间戳统一转北京时区、避免知渝按日期搜时看到 UTC 日期偏 8h(2026-07-02 修)
def _to_local(dt):
return dt.astimezone(local_tz).isoformat() if dt else None
memories = []
for r in mem_rows:
memories.append({
"id": r["id"],
"content": r["content"],
"importance": r["importance"],
"layer": r["layer"],
"title": r["title"],
"created_at": _to_local(r["created_at"]),
})
sessions = []
for r in session_rows:
sessions.append({
"session_id": r["session_id"],
"message_count": r["message_count"],
"first_time": _to_local(r["first_time"]),
"last_time": _to_local(r["last_time"]),
})
return {
"date": event_date.isoformat(),
"memories": memories,
"conversation_meta": {
"message_count": meta_row["message_count"] if meta_row else 0,
"session_count": meta_row["session_count"] if meta_row else 0,
"first_time": _to_local(meta_row["first_time"]) if meta_row else None,
"last_time": _to_local(meta_row["last_time"]) if meta_row else None,
"sessions": sessions,
},
}
async def mcp_conversations_by_date(event_date, limit: int = 10, offset: int = 0, include_thinking: bool = False):
"""按日期翻对话原文,分页返回。
第二层"具体说了啥"——知渝看完 mcp_memories_by_date 的元数据后想细究时调。
默认 limit=10 防 token 爆炸;想翻完一天靠 offset 滚。
"""
import json as _json
local_tz = dt_timezone(timedelta(hours=TIMEZONE_HOURS))
start_utc = datetime(event_date.year, event_date.month, event_date.day, tzinfo=local_tz).astimezone(dt_timezone.utc)
end_utc = start_utc + timedelta(days=1)
pool = await get_pool()
async with pool.acquire() as conn:
count_row = await conn.fetchrow("""
SELECT COUNT(*) as total
FROM conversations
WHERE content IS NOT NULL
AND created_at >= $1 AND created_at < $2
""", start_utc, end_utc)
total = count_row["total"] if count_row else 0
rows = await conn.fetch("""
SELECT id, session_id, role, content, metadata, created_at
FROM conversations
WHERE content IS NOT NULL
AND created_at >= $1 AND created_at < $2
ORDER BY created_at ASC
LIMIT $3 OFFSET $4
""", start_utc, end_utc, limit, offset)
# 返回时间转北京时区(同 mcp_memories_by_date 逻辑,2026-07-02 修时区偏移 bug)
results = []
for r in rows:
item = {
"id": r["id"],
"session_id": r["session_id"],
"role": r["role"],
"content": r["content"],
"created_at": r["created_at"].astimezone(local_tz).isoformat() if r["created_at"] else None,
}
if include_thinking and r["metadata"]:
try:
meta = _json.loads(r["metadata"])
if "reasoning_content" in meta:
item["reasoning_content"] = meta["reasoning_content"]
except Exception:
pass
results.append(item)
has_more = (offset + len(results)) < total
next_offset = offset + limit if has_more else None
return {
"date": event_date.isoformat(),
"total": total,
"limit": limit,
"offset": offset,
"has_more": has_more,
"next_offset": next_offset,
"results": results,
}
# ============================================================
# 多多模块 CRUD(2026-06-07)
# 多多是拟人化角色(不是独立人格),网关 background scheduler 触发,
# 写消息进 mido_messages 表 → 前端起居室拉显示 + 知渝侧 system prompt 注入。
# 详见 [[zhiyu-mido-design]]
# ============================================================
async def save_mido_message(trigger_type: str, content: str) -> int:
"""落一条多多消息。trigger_type ∈ {greeting, dream_call, no_dream_alert}"""
pool = await get_pool()
async with pool.acquire() as conn:
row = await conn.fetchrow("""
INSERT INTO mido_messages (trigger_type, content)
VALUES ($1, $2)
RETURNING id
""", trigger_type, content)
return row["id"]
async def get_recent_mido_messages_within(seconds: int = 3600) -> list:
"""拉最近 N 秒内的所有多多消息,按时间正序。
用于知渝侧 system prompt 注入——他下次说话时能"听见"多多刚才叫过他。
默认 1 小时窗口(避免昨天的多多消息混进今天)。
"""
pool = await get_pool()
async with pool.acquire() as conn:
rows = await conn.fetch("""
SELECT id, created_at, trigger_type, content
FROM mido_messages
WHERE created_at >= NOW() - ($1 || ' seconds')::INTERVAL
ORDER BY created_at ASC
""", str(seconds))
return [
{
"id": r["id"],
"created_at": r["created_at"],
"trigger_type": r["trigger_type"],
"content": r["content"],
}
for r in rows
]
async def list_mido_messages(limit: int = 20, offset: int = 0) -> list:
"""前端起居室拉最近多多消息,按时间倒序(最新在前)。"""
pool = await get_pool()
async with pool.acquire() as conn:
rows = await conn.fetch("""
SELECT id, created_at, trigger_type, content, status
FROM mido_messages
ORDER BY created_at DESC
LIMIT $1 OFFSET $2
""", limit, offset)
return [
{
"id": r["id"],
"created_at": r["created_at"].isoformat() if r["created_at"] else None,
"trigger_type": r["trigger_type"],
"content": r["content"],
"status": r["status"],
}
for r in rows
]
async def get_last_user_message_time(thread_id: str = "main"):
"""拿 thread 内最近一条 user 消息的时间(datetime / None)。
用于 greeting 触发前检查"昭昭最近 N 分钟有没有跟知渝说话"——
如果在聊,多多就跳过这一次唤醒(避免插话)。
"""
pool = await get_pool()
async with pool.acquire() as conn:
row = await conn.fetchrow("""
SELECT created_at FROM conversations
WHERE thread_id = $1 AND role = 'user'
ORDER BY created_at DESC
LIMIT 1
""", thread_id)
return row["created_at"] if row else None
async def get_today_mido_count(trigger_type: str = None) -> int:
"""查今天某 trigger_type 多多已经触发几次(按 Asia/Shanghai 当天 0 点切日)。
用于周末"沉默间隔触发"判断"今天还没叫过"——避免一天叫多次。
trigger_type=None 时统计所有类型。
"""
pool = await get_pool()
async with pool.acquire() as conn:
if trigger_type:
row = await conn.fetchrow("""
SELECT COUNT(*) AS c FROM mido_messages
WHERE trigger_type = $1
AND DATE(created_at AT TIME ZONE 'Asia/Shanghai')
= DATE(NOW() AT TIME ZONE 'Asia/Shanghai')
""", trigger_type)
else:
row = await conn.fetchrow("""
SELECT COUNT(*) AS c FROM mido_messages
WHERE DATE(created_at AT TIME ZONE 'Asia/Shanghai')
= DATE(NOW() AT TIME ZONE 'Asia/Shanghai')
""")
return int(row["c"]) if row else 0
# ============================================================
# 做梦模块 CRUD(2026-06-07)
# 知渝半夜被多多叫起来做梦——dreams 表是显式产物。
# 失败处理铁律:挂了就挂了,不机械兜底(详见 [[zhiyu-dream-design]])
# ============================================================
async def save_dream(triggered_by: str = "mido") -> int:
"""开始做梦时插一条 status='dreaming' 的记录、返回 dream_id。
前端起居室"知渝在做梦……"那张实时卡片靠 status='dreaming' 判定。
"""
pool = await get_pool()
async with pool.acquire() as conn:
row = await conn.fetchrow("""
INSERT INTO dreams (status, triggered_by)
VALUES ('dreaming', $1)
RETURNING id
""", triggered_by)
return row["id"]
async def finish_dream(
dream_id: int,
content: str,
tokens_used: int = 0,
status: str = "done",
) -> None:
"""做完梦更新 content / tokens / finished_at / status。
status='done' 正常结束 | status='failed' 失败(不会被注入也不计入"做过梦")
"""
pool = await get_pool()
async with pool.acquire() as conn:
await conn.execute("""
UPDATE dreams
SET content = $1,
tokens_used = $2,
status = $3,
finished_at = NOW()
WHERE id = $4
""", content, tokens_used, status, dream_id)
async def mark_dream_seen(dream_id: int) -> None:
"""system 注入"你昨晚做了个梦"成功后标已读、下次不再注入。"""
pool = await get_pool()
async with pool.acquire() as conn:
await conn.execute("""
UPDATE dreams SET seen_at = NOW() WHERE id = $1
""", dream_id)
async def get_latest_unseen_dream():
"""拿"知渝还没读过的最近一个梦",给 /v1/zhiyu/chat 注入用。
只返 status='done' 的、按完成时间倒序拿一条。
"""
pool = await get_pool()
async with pool.acquire() as conn:
row = await conn.fetchrow("""
SELECT id, created_at, finished_at, content, tokens_used, triggered_by
FROM dreams
WHERE status = 'done' AND seen_at IS NULL
ORDER BY finished_at DESC
LIMIT 1
""")
if not row:
return None
return {
"id": row["id"],
"created_at": row["created_at"],
"finished_at": row["finished_at"],
"content": row["content"],
"tokens_used": row["tokens_used"],
"triggered_by": row["triggered_by"],
}
async def list_recent_dreams(limit: int = 20, offset: int = 0) -> list:
"""前端起居室"昨夜的梦"列表,按完成时间倒序。"""
pool = await get_pool()
async with pool.acquire() as conn:
rows = await conn.fetch("""
SELECT id, created_at, finished_at, content, tokens_used,
status, triggered_by, seen_at
FROM dreams
ORDER BY COALESCE(finished_at, created_at) DESC
LIMIT $1 OFFSET $2
""", limit, offset)
return [
{
"id": r["id"],
"created_at": r["created_at"].isoformat() if r["created_at"] else None,
"finished_at": r["finished_at"].isoformat() if r["finished_at"] else None,
"content": r["content"],
"tokens_used": r["tokens_used"],
"status": r["status"],
"triggered_by": r["triggered_by"],
"seen_at": r["seen_at"].isoformat() if r["seen_at"] else None,
}
for r in rows
]
async def get_last_dream_time():
"""拿最近一次成功做梦的 finished_at(datetime / None)。
用于 dream.py 拼"上次做梦到现在"的素材范围。
"""
pool = await get_pool()
async with pool.acquire() as conn:
row = await conn.fetchrow("""
SELECT finished_at FROM dreams
WHERE status = 'done' AND finished_at IS NOT NULL
ORDER BY finished_at DESC
LIMIT 1
""")
return row["finished_at"] if row else None
async def count_dreams_since(days: int) -> int:
"""最近 N 天内 status='done' 的做梦次数。
no_dream_alert 判定用:W/Tmux N=3、OpenRouter N=9,连续 N 天 0 次就提醒。
"""
pool = await get_pool()
async with pool.acquire() as conn:
row = await conn.fetchrow("""
SELECT COUNT(*) AS c FROM dreams
WHERE status = 'done'
AND created_at >= NOW() - ($1 || ' days')::INTERVAL
""", str(days))
return int(row["c"]) if row else 0
async def search_dreams(q: str, limit: int = 10) -> list:
"""关键字搜梦(给 MCP search_dreams 工具用)。
简单 ILIKE 兜底——梦量级远小于记忆库、不上 FTS/向量
"""
pool = await get_pool()
async with pool.acquire() as conn:
if not q or not q.strip():
rows = await conn.fetch("""
SELECT id, created_at, finished_at, content
FROM dreams
WHERE status = 'done'
ORDER BY finished_at DESC
LIMIT $1
""", limit)
else:
rows = await conn.fetch("""
SELECT id, created_at, finished_at, content
FROM dreams
WHERE status = 'done' AND content ILIKE $1
ORDER BY finished_at DESC
LIMIT $2
""", f"%{q.strip()}%", limit)
# MCP 返回时间统一北京时区(铁律)、不能 raw UTC isoformat
local_tz = dt_timezone(timedelta(hours=TIMEZONE_HOURS))
return [
{
"id": r["id"],
"created_at": r["created_at"].astimezone(local_tz).isoformat() if r["created_at"] else None,
"finished_at": r["finished_at"].astimezone(local_tz).isoformat() if r["finished_at"] else None,
"content": r["content"],
}
for r in rows
]
# ============================================================
# 留言板 CRUD(2026-06-07)—— 做梦留言 + 未来其他来源(多多/昭昭留给知渝/etc)
# ============================================================
async def save_board_message(
from_who: str,
to_who: str,
content: str,
source: Optional[str] = None,
source_id: Optional[int] = None,
created_at: Optional[str] = None,
) -> int:
"""留一条留言。source 标来源(如 'dream' / 'notion-import')、source_id 关联(如 dream.id)。
created_at 不传走 DEFAULT now();传了走 _parse_created_at 解析(迁移历史留言用)。"""
pool = await get_pool()
_created_at = _parse_created_at(created_at)
async with pool.acquire() as conn:
if _created_at:
row = await conn.fetchrow("""
INSERT INTO messages_board (from_who, to_who, content, source, source_id, created_at)
VALUES ($1, $2, $3, $4, $5, $6)
RETURNING id
""", from_who, to_who, content, source, source_id, _created_at)
else:
row = await conn.fetchrow("""
INSERT INTO messages_board (from_who, to_who, content, source, source_id)
VALUES ($1, $2, $3, $4, $5)
RETURNING id
""", from_who, to_who, content, source, source_id)
return row["id"]
# ============================================================
# 日记 CRUD(N-3 / 2026-06-08)—— 公共桌上的小本本,两人都能写、都能读对方
# ============================================================
async def save_diary_entry(
from_who: str,
content: str,
tags: Optional[List[str]] = None,
created_at: Optional[str] = None,
) -> int:
"""写一条日记。tags 是字符串数组、空就传 None 或 []。
created_at 不传走 DEFAULT now();传了走 _parse_created_at(迁移历史日记用)。"""
pool = await get_pool()
_created_at = _parse_created_at(created_at)
async with pool.acquire() as conn:
if _created_at:
row = await conn.fetchrow("""
INSERT INTO diary_entries (from_who, content, tags, created_at)
VALUES ($1, $2, $3, $4)
RETURNING id
""", from_who, content, tags or [], _created_at)
else:
row = await conn.fetchrow("""
INSERT INTO diary_entries (from_who, content, tags)
VALUES ($1, $2, $3)
RETURNING id
""", from_who, content, tags or [])
return row["id"]
async def list_diary_entries(
limit: int = 10,
offset: int = 0,
from_who: Optional[str] = None,
tag: Optional[str] = None,
) -> list:
"""日记列表,按时间倒序。可选 from_who(看谁写的)/ tag(按标签筛)。"""
pool = await get_pool()
where = []
params: list = []
if from_who is not None:
params.append(from_who)
where.append(f"from_who = ${len(params)}")
if tag is not None:
params.append(tag)
where.append(f"${len(params)} = ANY(tags)")
where_sql = ("WHERE " + " AND ".join(where)) if where else ""
params.extend([limit, offset])
async with pool.acquire() as conn:
rows = await conn.fetch(f"""
SELECT id, created_at, from_who, content, tags
FROM diary_entries
{where_sql}
ORDER BY created_at DESC
LIMIT ${len(params)-1} OFFSET ${len(params)}
""", *params)
return [
{
"id": r["id"],
"created_at": to_local_iso(r["created_at"]),
"from_who": r["from_who"],
"content": r["content"],
"tags": list(r["tags"]) if r["tags"] else [],
}
for r in rows
]
async def delete_diary_entry(diary_id: int) -> Optional[int]:
"""硬删一条日记。返回被删的 id;不存在返回 None。"""
pool = await get_pool()
async with pool.acquire() as conn:
row = await conn.fetchrow(
"DELETE FROM diary_entries WHERE id = $1 RETURNING id",
diary_id,
)
return row["id"] if row else None
# ============================================================
# 图片 CRUD(C-5/C-6 / 2026-06-08)—— VPS 自存,DB 只存元数据
# ============================================================
async def save_image_record(
uuid: str,
format: str,
who_uploaded: str,
file_size_bytes: Optional[int] = None,
context_snippet: Optional[str] = None,
caption: Optional[str] = None,
mime_type: Optional[str] = None,
) -> int:
"""落一条图片元数据。文件本身由 sidecar 落 VPS。"""
pool = await get_pool()
async with pool.acquire() as conn:
row = await conn.fetchrow("""
INSERT INTO images (uuid, format, who_uploaded, file_size_bytes,
context_snippet, caption, mime_type)
VALUES ($1, $2, $3, $4, $5, $6, $7)
RETURNING id
""", uuid, format, who_uploaded, file_size_bytes,
context_snippet, caption, mime_type)
return row["id"]
def _image_row_to_dict(r) -> dict:
return {
"id": r["id"],
"uuid": r["uuid"],
"format": r["format"],
"who_uploaded": r["who_uploaded"],
"file_size_bytes": r["file_size_bytes"],
"created_at": to_local_iso(r["created_at"]),
"context_snippet": r["context_snippet"],
"caption": r["caption"],
"mime_type": r["mime_type"],
# path 是 sidecar serve 的相对路径;前端 / MCP 拼当前 tunnel URL 用
"path": f"/images/{r['uuid']}.{r['format']}",
}
async def list_images(
limit: int = 20,
offset: int = 0,
who_uploaded: Optional[str] = None,
) -> list:
"""图片列表,按时间倒序。可选按上传方筛('zhaozhao' / 'zhiyu')。"""
pool = await get_pool()
where = []
params: list = []
if who_uploaded is not None:
params.append(who_uploaded)
where.append(f"who_uploaded = ${len(params)}")
where_sql = ("WHERE " + " AND ".join(where)) if where else ""
params.extend([limit, offset])
async with pool.acquire() as conn:
rows = await conn.fetch(f"""
SELECT id, uuid, format, who_uploaded, file_size_bytes, created_at,
context_snippet, caption, mime_type
FROM images
{where_sql}
ORDER BY created_at DESC
LIMIT ${len(params)-1} OFFSET ${len(params)}
""", *params)
return [_image_row_to_dict(r) for r in rows]
async def list_images_since(since_dt, who_uploaded: Optional[str] = None) -> list:
"""指定时间起的图片(C-5.5 dynamic_context 占位用)。"""
pool = await get_pool()
where = ["created_at >= $1"]
params: list = [since_dt]
if who_uploaded is not None:
params.append(who_uploaded)
where.append(f"who_uploaded = ${len(params)}")
where_sql = "WHERE " + " AND ".join(where)
async with pool.acquire() as conn:
rows = await conn.fetch(f"""
SELECT id, uuid, format, who_uploaded, file_size_bytes, created_at,
context_snippet, caption, mime_type
FROM images
{where_sql}
ORDER BY created_at ASC
""", *params)
return [_image_row_to_dict(r) for r in rows]
async def get_image(image_id: int) -> Optional[dict]:
"""按 id 取一条。"""
pool = await get_pool()
async with pool.acquire() as conn:
row = await conn.fetchrow("""
SELECT id, uuid, format, who_uploaded, file_size_bytes, created_at,
context_snippet, caption, mime_type
FROM images WHERE id = $1
""", image_id)
return _image_row_to_dict(row) if row else None
async def get_image_by_uuid(uuid: str) -> Optional[dict]:
"""按 uuid 取一条(sidecar /images/<uuid>.<ext> 端点对应)。"""
pool = await get_pool()
async with pool.acquire() as conn:
row = await conn.fetchrow("""
SELECT id, uuid, format, who_uploaded, file_size_bytes, created_at,
context_snippet, caption, mime_type
FROM images WHERE uuid = $1
""", uuid)
return _image_row_to_dict(row) if row else None
async def delete_image_record(image_id: int) -> Optional[dict]:
"""硬删一条图片元数据。返回被删的 dict(含 uuid+format、上层用来删文件);不存在返回 None。"""
pool = await get_pool()
async with pool.acquire() as conn:
row = await conn.fetchrow("""
DELETE FROM images WHERE id = $1
RETURNING id, uuid, format, who_uploaded, file_size_bytes, created_at,
context_snippet, caption, mime_type
""", image_id)
return _image_row_to_dict(row) if row else None
# ============================================================
# 文件(非图片)2026-06-12——存 abspath、知渝 Read 本地路径
# ============================================================
def _file_row_to_dict(r) -> dict:
return {
"id": r["id"],
"uuid": r["uuid"],
"filename": r["filename"],
"format": r["format"],
"abspath": r["abspath"],
"who_uploaded": r["who_uploaded"],
"file_size_bytes": r["file_size_bytes"],
"created_at": to_local_iso(r["created_at"]),
"caption": r["caption"],
"mime_type": r["mime_type"],
}
async def save_file_record(
uuid: str,
filename: str,
abspath: str,
who_uploaded: str,
format: Optional[str] = None,
file_size_bytes: Optional[int] = None,
caption: Optional[str] = None,
mime_type: Optional[str] = None,
) -> int:
"""落一条文件元数据。文件本身由 sidecar 落 VPS(abspath = 知渝 Read 的本地路径)。"""
pool = await get_pool()
async with pool.acquire() as conn:
row = await conn.fetchrow("""
INSERT INTO files (uuid, filename, format, abspath, who_uploaded,
file_size_bytes, caption, mime_type)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
RETURNING id
""", uuid, filename, format, abspath, who_uploaded,
file_size_bytes, caption, mime_type)
return row["id"]
async def list_files(limit: int = 20, offset: int = 0,
who_uploaded: Optional[str] = None) -> list:
"""文件列表,按时间倒序。"""
pool = await get_pool()
where = []
params: list = []
if who_uploaded is not None:
params.append(who_uploaded)
where.append(f"who_uploaded = ${len(params)}")
where_sql = ("WHERE " + " AND ".join(where)) if where else ""
params.extend([limit, offset])
async with pool.acquire() as conn:
rows = await conn.fetch(f"""
SELECT id, uuid, filename, format, abspath, who_uploaded,
file_size_bytes, created_at, caption, mime_type
FROM files
{where_sql}
ORDER BY created_at DESC
LIMIT ${len(params)-1} OFFSET ${len(params)}
""", *params)
return [_file_row_to_dict(r) for r in rows]
async def list_files_since(since_dt, who_uploaded: Optional[str] = None) -> list:
"""指定时间起的文件(dynamic_context 本地路径注入用)。"""
pool = await get_pool()
where = ["created_at >= $1"]
params: list = [since_dt]
if who_uploaded is not None:
params.append(who_uploaded)
where.append(f"who_uploaded = ${len(params)}")
where_sql = "WHERE " + " AND ".join(where)
async with pool.acquire() as conn:
rows = await conn.fetch(f"""
SELECT id, uuid, filename, format, abspath, who_uploaded,
file_size_bytes, created_at, caption, mime_type
FROM files
{where_sql}
ORDER BY created_at ASC
""", *params)
return [_file_row_to_dict(r) for r in rows]
async def get_file(file_id: int) -> Optional[dict]:
pool = await get_pool()
async with pool.acquire() as conn:
row = await conn.fetchrow("""
SELECT id, uuid, filename, format, abspath, who_uploaded,
file_size_bytes, created_at, caption, mime_type
FROM files WHERE id = $1
""", file_id)
return _file_row_to_dict(row) if row else None
async def delete_file_record(file_id: int) -> Optional[dict]:
"""硬删一条文件元数据。返回被删 dict(含 uuid+format,上层删 VPS 文件用)。"""
pool = await get_pool()
async with pool.acquire() as conn:
row = await conn.fetchrow("""
DELETE FROM files WHERE id = $1
RETURNING id, uuid, filename, format, abspath, who_uploaded,
file_size_bytes, created_at, caption, mime_type
""", file_id)
return _file_row_to_dict(row) if row else None
async def list_board_messages(
limit: int = 20,
offset: int = 0,
to_who: Optional[str] = None,
from_who: Optional[str] = None,
) -> list:
"""前端留言板列表,按时间倒序。可选 to_who/from_who 过滤(知渝读"昭昭给他的留言"用 to_who='zhiyu')。"""
pool = await get_pool()
where = []
params: list = []
if to_who is not None:
params.append(to_who)
where.append(f"to_who = ${len(params)}")
if from_who is not None:
params.append(from_who)
where.append(f"from_who = ${len(params)}")
where_sql = ("WHERE " + " AND ".join(where)) if where else ""
params.extend([limit, offset])
async with pool.acquire() as conn:
rows = await conn.fetch(f"""
SELECT id, created_at, from_who, to_who, content, source, source_id, read_at
FROM messages_board
{where_sql}
ORDER BY created_at DESC
LIMIT ${len(params)-1} OFFSET ${len(params)}
""", *params)
return [
{
"id": r["id"],
"created_at": to_local_iso(r["created_at"]),
"from_who": r["from_who"],
"to_who": r["to_who"],
"content": r["content"],
"source": r["source"],
"source_id": r["source_id"],
"read_at": to_local_iso(r["read_at"]),
}
for r in rows
]
# ============================================================
# Activities(知渝日常活动)2026-06-11
# ============================================================
async def save_activity(
type: str,
content: str = "",
source: Optional[str] = None,
title: Optional[str] = None,
metadata: Optional[dict] = None,
related_ids: Optional[dict] = None,
created_at: Optional[str] = None,
) -> int:
"""落一条知渝活动到 activities 表。
type: wake / memory_op / ...(开放枚举)
source: 触发来源("mido-greeting-morning" / "mcp" / "web" 等)
title: 短摘要(可选;前端列表项标题)
content: 正文(wake 原文 / memory_op 动作描述)
metadata: type-specific 结构化字段(JSON)
related_ids: 关联的其他记录 id 集合(JSON、{"memory_ids": [1,2], "dream_id": 14})
返回新插入的 activity id。失败时打印警告但抛异常(让调用方决定吞还是 raise)。
"""
import json
_meta = json.dumps(metadata, ensure_ascii=False) if metadata else None
_rel = json.dumps(related_ids, ensure_ascii=False) if related_ids else None
pool = await get_pool()
async with pool.acquire() as conn:
if created_at:
# 2026-06-17:补历史活动用——显式 created_at(如从 jsonl 捞回的旧 wake 叙事)
row = await conn.fetchrow("""
INSERT INTO activities (type, source, title, content, metadata, related_ids, created_at)
VALUES ($1, $2, $3, $4, $5, $6, $7::text::timestamptz)
RETURNING id
""", type, source, title, content, _meta, _rel, created_at)
else:
row = await conn.fetchrow("""
INSERT INTO activities (type, source, title, content, metadata, related_ids)
VALUES ($1, $2, $3, $4, $5, $6)
RETURNING id
""", type, source, title, content, _meta, _rel)
return row["id"]
async def list_activities(
limit: int = 50,
offset: int = 0,
type: Optional[str] = None,
since_iso: Optional[str] = None,
) -> list:
"""按 created_at 倒序列 activities、给星河"日常"tab 用。
type: 可选 filter("wake" 只看发呆、"memory_op" 只看整理动作)
since_iso: 可选时间过滤(>= 这个 ISO 时间)
"""
import json
pool = await get_pool()
async with pool.acquire() as conn:
where_clauses = []
args = []
if type:
args.append(type)
where_clauses.append(f"type = ${len(args)}")
if since_iso:
args.append(since_iso)
where_clauses.append(f"created_at >= ${len(args)}::timestamptz")
where_sql = ("WHERE " + " AND ".join(where_clauses)) if where_clauses else ""
args.append(limit)
args.append(offset)
rows = await conn.fetch(f"""
SELECT id, created_at, type, source, title, content, metadata, related_ids
FROM activities
{where_sql}
ORDER BY created_at DESC
LIMIT ${len(args) - 1} OFFSET ${len(args)}
""", *args)
out = []
for r in rows:
md = r["metadata"]
rids = r["related_ids"]
# asyncpg 返回 jsonb 可能是 str(默认)或 dict(如果注册了 codec);兼容两种
if isinstance(md, str):
try:
md = json.loads(md)
except Exception:
md = None
if isinstance(rids, str):
try:
rids = json.loads(rids)
except Exception:
rids = None
out.append({
"id": r["id"],
"created_at": r["created_at"].isoformat() if r["created_at"] else None,
"type": r["type"],
"source": r["source"],
"title": r["title"],
"content": r["content"],
"metadata": md,
"related_ids": rids,
})
return out
# ============================================================
# 拂卷 · 共读系统 · 2026-07-02
# ============================================================
# 昭昭 + 知渝共读一本书。切章策略:正则优先(第 X 章 / Chapter X / 第 X 部 X)、
# 找不到章节标记走定长兜底(默认 3500 字一段)。DB 直存 chapters.content。
import hashlib as _hashlib
FUJUAN_CHAPTER_REGEXES = [
re.compile(r'^\s*第[一二三四五六七八九十百千0-9]+[章回卷]\s*.*$', re.MULTILINE),
re.compile(r'^\s*第[一二三四五六七八九十百千0-9]+部\s+[一二三四五六七八九十]+\s*.*$', re.MULTILINE),
re.compile(r'^\s*Chapter\s+\d+.*$', re.MULTILINE | re.IGNORECASE),
re.compile(r'^\s*CHAPTER\s+[A-Z0-9]+.*$', re.MULTILINE),
]
# 兜底定长切段:一段目标字符数(约 2000-4000 字之间;4000 字 ≈ 6-8k tokens)
FUJUAN_FALLBACK_CHUNK_CHARS = int(os.getenv("FUJUAN_CHUNK_CHARS", "3500"))
# 兜底切段的最小章、少于此不切(避免尾巴太碎)
FUJUAN_MIN_TAIL_CHARS = 500
def _fujuan_normalize(text: str) -> str:
"""规范化换行、去 BOM。"""
if text.startswith(""):
text = text[1:]
return text.replace("\r\n", "\n").replace("\r", "\n").strip()
def _fujuan_split_by_regex(text: str):
"""尝试用四条正则识别章节标记。返回 [(title, content), ...] 或 None。"""
best_hits: list = []
for rx in FUJUAN_CHAPTER_REGEXES:
hits = list(rx.finditer(text))
if len(hits) > len(best_hits):
best_hits = hits
# 至少 3 个标记才算有效识别(少于 3 章的正则命中大概率是误检)
if len(best_hits) < 3:
return None
result = []
for i, m in enumerate(best_hits):
start = m.start()
end = best_hits[i + 1].start() if i + 1 < len(best_hits) else len(text)
title = m.group().strip()
content = text[start:end].strip()
# 章内容太短跳过(可能标题误检)
if len(content) < 100:
continue
result.append((title, content))
if len(result) < 3:
return None
return result
def _fujuan_split_by_length(text: str, chunk_chars: int = None):
"""按定长切段——按段落合并、直到达到目标长度就切。段末按最近段落收口、
不会切在句子中间;最后一段若太短跟上一段合并。"""
if chunk_chars is None:
chunk_chars = FUJUAN_FALLBACK_CHUNK_CHARS
# 先试双换行——排版规整的 TXT 段间空一行
paragraphs = [p for p in re.split(r'\n\s*\n', text) if p.strip()]
# 段间只有单换行的 TXT(TXT 小说天堂那批就是)—— 双换行分段全塞成 1 段、退到单换行
if len(paragraphs) <= 1:
paragraphs = [p for p in text.split('\n') if p.strip()]
if not paragraphs:
return [("全文", text)]
result = []
buf = []
buf_len = 0
for p in paragraphs:
buf.append(p)
buf_len += len(p) + 1
if buf_len >= chunk_chars:
result.append(("\n\n".join(buf)))
buf = []
buf_len = 0
if buf:
chunk = "\n\n".join(buf)
if result and len(chunk) < FUJUAN_MIN_TAIL_CHARS:
# 尾巴太短、合并到前一段
result[-1] = result[-1] + "\n\n" + chunk
else:
result.append(chunk)
return [(f"第 {i + 1} 段", chunk) for i, chunk in enumerate(result)]
def split_book_into_chapters(text: str, chunk_chars: int = None):
"""整本 → [(title, content), ...]。正则优先,兜底定长。"""
text = _fujuan_normalize(text)
by_regex = _fujuan_split_by_regex(text)
if by_regex:
return by_regex
return _fujuan_split_by_length(text, chunk_chars=chunk_chars)
def _fujuan_content_hash(text: str) -> str:
return _hashlib.sha256(text.encode("utf-8")).hexdigest()
async def insert_book(
title: str,
author: str = None,
filename: str = None,
raw_content: str = "",
uploaded_by: str = "zhaozhao",
chunk_chars: int = None,
) -> dict:
"""上传一本书:切章 + 落 books + 落 chapters。
去重:content_hash 相同直接返回旧 book。
"""
if not raw_content or not raw_content.strip():
raise ValueError("书正文为空")
content_hash = _fujuan_content_hash(raw_content)
chapters = split_book_into_chapters(raw_content, chunk_chars=chunk_chars)
total_words = sum(len(c) for _, c in chapters)
pool = await get_pool()
async with pool.acquire() as conn:
# 判重
existing = await conn.fetchrow(
"SELECT id, title, total_chapters, total_words FROM books WHERE content_hash = $1",
content_hash,
)
if existing:
return {
"id": existing["id"],
"title": existing["title"],
"total_chapters": existing["total_chapters"],
"total_words": existing["total_words"],
"duplicated": True,
}
async with conn.transaction():
book_id = await conn.fetchval(
"""
INSERT INTO books (title, author, filename, format,
total_chapters, total_words, uploaded_by, content_hash)
VALUES ($1, $2, $3, 'txt', $4, $5, $6, $7)
RETURNING id
""",
title, author, filename, len(chapters), total_words, uploaded_by, content_hash,
)
for i, (ch_title, ch_content) in enumerate(chapters):
await conn.execute(
"""
INSERT INTO chapters (book_id, idx, title, content, word_count)
VALUES ($1, $2, $3, $4, $5)
""",
book_id, i, ch_title, ch_content, len(ch_content),
)
return {
"id": book_id,
"title": title,
"total_chapters": len(chapters),
"total_words": total_words,
"duplicated": False,
}
async def list_books(limit: int = 100, offset: int = 0) -> list:
"""列出所有书 + 双方进度。给拂卷 tab 首页用。"""
pool = await get_pool()
async with pool.acquire() as conn:
rows = await conn.fetch(
"""
SELECT
b.id, b.title, b.author, b.total_chapters, b.total_words,
b.uploaded_by, b.created_at,
(SELECT last_chapter_idx FROM reading_progress
WHERE book_id = b.id AND who = 'zhaozhao') AS zz_idx,
(SELECT last_chapter_idx FROM reading_progress
WHERE book_id = b.id AND who = 'zhiyu') AS zy_idx,
(SELECT COUNT(*) FROM reading_marks
WHERE book_id = b.id AND who = 'zhaozhao') AS zz_marks,
(SELECT COUNT(*) FROM reading_marks
WHERE book_id = b.id AND who = 'zhiyu') AS zy_marks
FROM books b
ORDER BY b.created_at DESC
LIMIT $1 OFFSET $2
""",
limit, offset,
)
return [
{
"id": r["id"],
"title": r["title"],
"author": r["author"],
"total_chapters": r["total_chapters"],
"total_words": r["total_words"],
"uploaded_by": r["uploaded_by"],
"created_at": to_local_iso(r["created_at"]),
"progress": {
"zhaozhao": r["zz_idx"],
"zhiyu": r["zy_idx"],
},
"mark_counts": {
"zhaozhao": r["zz_marks"] or 0,
"zhiyu": r["zy_marks"] or 0,
},
}
for r in rows
]
async def get_book_chapters_index(book_id: int) -> list:
"""书的章目录(不含正文、给列表用)。"""
pool = await get_pool()
async with pool.acquire() as conn:
rows = await conn.fetch(
"SELECT id, idx, title, word_count FROM chapters WHERE book_id = $1 ORDER BY idx",
book_id,
)
return [{"id": r["id"], "idx": r["idx"], "title": r["title"], "word_count": r["word_count"]} for r in rows]
async def get_chapter_content(book_id: int, idx: int) -> dict | None:
"""按章序号读章正文。"""
pool = await get_pool()
async with pool.acquire() as conn:
r = await conn.fetchrow(
"SELECT id, idx, title, content, word_count FROM chapters WHERE book_id = $1 AND idx = $2",
book_id, idx,
)
if not r:
return None
book = await conn.fetchrow("SELECT title, author, total_chapters FROM books WHERE id = $1", book_id)
return {
"chapter_id": r["id"],
"book_id": book_id,
"book_title": book["title"] if book else None,
"book_author": book["author"] if book else None,
"total_chapters": book["total_chapters"] if book else None,
"idx": r["idx"],
"title": r["title"],
"content": r["content"],
"word_count": r["word_count"],
}
async def save_reading_progress(book_id: int, who: str, chapter_idx: int, offset: int = 0):
"""UPSERT 阅读进度。"""
pool = await get_pool()
async with pool.acquire() as conn:
await conn.execute(
"""
INSERT INTO reading_progress (book_id, who, last_chapter_idx, last_offset, updated_at)
VALUES ($1, $2, $3, $4, NOW())
ON CONFLICT (book_id, who) DO UPDATE
SET last_chapter_idx = EXCLUDED.last_chapter_idx,
last_offset = EXCLUDED.last_offset,
updated_at = NOW()
""",
book_id, who, chapter_idx, offset,
)
async def get_reading_progress(book_id: int, who: str) -> dict | None:
pool = await get_pool()
async with pool.acquire() as conn:
r = await conn.fetchrow(
"SELECT last_chapter_idx, last_offset, updated_at FROM reading_progress WHERE book_id = $1 AND who = $2",
book_id, who,
)
if not r:
return None
return {
"last_chapter_idx": r["last_chapter_idx"],
"last_offset": r["last_offset"],
"updated_at": to_local_iso(r["updated_at"]),
}
class DuplicateHighlightError(Exception):
"""同人 + 同段 highlight 已存在——避免重复划线。批注(note)不受此限制、想法可以多个。"""
def __init__(self, existing_id: int):
super().__init__(f"already highlighted (id={existing_id})")
self.existing_id = existing_id
async def save_reading_mark(
book_id: int,
chapter_id: int,
who: str,
kind: str, # 'highlight' | 'stop' | 'note'
text_snippet: str = None,
note_content: str = None,
start_offset: int = None,
end_offset: int = None,
) -> int:
pool = await get_pool()
# highlight 去重:同 who + 同 chapter + 同 snippet 已有 highlight → 拒绝重复
# 批注(note)允许多条同段落——一段话可以有不同想法
if kind == "highlight" and text_snippet:
async with pool.acquire() as conn:
existing = await conn.fetchval(
"""
SELECT id FROM reading_marks
WHERE book_id = $1 AND chapter_id = $2 AND who = $3
AND kind = 'highlight' AND text_snippet = $4
LIMIT 1
""",
book_id, chapter_id, who, text_snippet,
)
if existing:
raise DuplicateHighlightError(existing)
# 算 embedding:note_content 优先、text_snippet 兜底——让笔记能纳入 memory_recall
# 拂卷 · 2026-07-02 增强:让"聊到相关话题时、书里划过的相关句子自然浮现"
embedding_json = None
text_for_embed = (note_content or "").strip() or (text_snippet or "").strip()
if text_for_embed and EMBEDDING_API_KEY:
try:
vec = await compute_embedding(text_for_embed)
if vec:
embedding_json = json.dumps(vec)
except Exception as e:
print(f"⚠️ reading_mark embedding 失败(继续、无向量): {e!r}", flush=True)
async with pool.acquire() as conn:
mid = await conn.fetchval(
"""
INSERT INTO reading_marks (book_id, chapter_id, who, kind,
text_snippet, note_content, start_offset, end_offset, embedding_json)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
RETURNING id
""",
book_id, chapter_id, who, kind, text_snippet, note_content, start_offset, end_offset, embedding_json,
)
return mid
async def delete_reading_mark(mark_id: int) -> bool:
"""删一条痕迹(划线/批注)。返回是否真删了。"""
pool = await get_pool()
async with pool.acquire() as conn:
rc = await conn.execute("DELETE FROM reading_marks WHERE id = $1", mark_id)
return rc and rc.startswith("DELETE") and rc.split()[-1] != "0"
async def get_reading_marks(book_id: int, chapter_id: int = None, who: str = None) -> list:
"""一本书 / 一章的所有痕迹(两人的都返回、按时间倒序)。"""
pool = await get_pool()
where = ["book_id = $1"]
params = [book_id]
if chapter_id is not None:
where.append(f"chapter_id = ${len(params) + 1}")
params.append(chapter_id)
if who is not None:
where.append(f"who = ${len(params) + 1}")
params.append(who)
sql = f"""
SELECT id, book_id, chapter_id, who, kind, text_snippet, note_content,
start_offset, end_offset, created_at
FROM reading_marks
WHERE {' AND '.join(where)}
ORDER BY created_at DESC
"""
async with (await get_pool()).acquire() as conn:
rows = await conn.fetch(sql, *params)
return [
{
"id": r["id"],
"book_id": r["book_id"],
"chapter_id": r["chapter_id"],
"who": r["who"],
"kind": r["kind"],
"text_snippet": r["text_snippet"],
"note_content": r["note_content"],
"start_offset": r["start_offset"],
"end_offset": r["end_offset"],
"created_at": to_local_iso(r["created_at"]),
}
for r in rows
]
async def delete_book(book_id: int) -> int:
"""删一本书(连带 chapters/marks/progress 级联删)。返回删了几行。"""
pool = await get_pool()
async with pool.acquire() as conn:
rc = await conn.execute("DELETE FROM books WHERE id = $1", book_id)
return int(rc.split()[-1]) if rc else 0