diff --git a/.dockerignore b/.dockerignore index 67d51ade10028ba45a44a2caa87fa9c451c737cd..e41f1be9bd23028644fc4489f6f87559ace7cec3 100644 --- a/.dockerignore +++ b/.dockerignore @@ -7,3 +7,5 @@ npm-debug.log Dockerfile .dockerignore .DS_Store +!rag-kb.db +!vector_store diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..504072590dfc1b2d0cc8aa72648dffb54a1dce0a 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +*.db filter=lfs diff=lfs merge=lfs -text +*.sqlite filter=lfs diff=lfs merge=lfs -text +*.index filter=lfs diff=lfs merge=lfs -text diff --git a/.gitignore b/.gitignore index d8d9164f7e43c348256692ff317eb68eab87ccec..894eee2fc864b4e0c61c0f55b51f077802148a79 100644 --- a/.gitignore +++ b/.gitignore @@ -45,3 +45,8 @@ rag-kb.db vector_store/hnswlib.index vector_store/docstore.json vector_store/args.json + +# exported dataset +hf_dataset/ +备份-语雀数据-JSON/ +.git/ diff --git a/Dockerfile b/Dockerfile index e600c926e2c8177a2edb317068ab43bd041d3378..505a9a463a856097e5c26bf345a3ed44c40cac48 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,8 +21,7 @@ ENV NEXT_TELEMETRY_DISABLED 1 # Dummy key for build time to prevent getEmbeddings from throwing ENV GOOGLE_GENERATIVE_AI_API_KEY "dummy-key-for-build" -# Force Demo Mode for Hugging Face or similar deployments -ENV NEXT_PUBLIC_DEMO_MODE "true" +ENV NEXT_PUBLIC_DEMO_MODE "false" RUN npm run build @@ -33,8 +32,10 @@ WORKDIR /app ENV NODE_ENV production ENV NEXT_TELEMETRY_DISABLED 1 -# Install runtime dependencies for ONNX Runtime and others -RUN apt-get update && apt-get install -y libgomp1 && rm -rf /var/lib/apt/lists/* +# Install runtime dependencies for ONNX Runtime and optional Python scripts +RUN apt-get update && apt-get install -y libgomp1 python3 python3-pip && rm -rf /var/lib/apt/lists/* + +RUN pip3 install --no-cache-dir huggingface_hub httpx RUN addgroup --system --gid 1001 nodejs RUN adduser --system --uid 1001 nextjs @@ -53,11 +54,11 @@ COPY --from=builder --chown=nextjs:nodejs /app/.next/standalone ./ COPY --from=builder --chown=nextjs:nodejs /app/.next/static ./.next/static # Copy data files needed for RAG -# Create directory if it doesn't exist in the image (it shouldn't) +# Create directory if it doesn't exist # We copy existing stores so the demo works out of the box -# COPY --from=builder --chown=nextjs:nodejs /app/vector_store ./vector_store -# Copy database if it exists, otherwise the app might create it (but likely fail due to permissions if strictly read-only filesystem, though HF Spaces usually has ephemeral writeable FS) -# COPY --from=builder --chown=nextjs:nodejs /app/rag-kb.db ./rag-kb.db +COPY --from=builder --chown=nextjs:nodejs /app/vector_store ./vector_store +# Copy database if it exists +COPY --from=builder --chown=nextjs:nodejs /app/rag-kb.db ./rag-kb.db # Copy source documents COPY --from=builder --chown=nextjs:nodejs /app/data ./data diff --git a/README.md b/README.md index 39109a6840f7cc1f86e4c606acf5bad5b3923fcd..1496a4094fcd9bf3c42ab704b1e4240c09288260 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,42 @@ short_description: RAG 知识库系统 # RAG Knowledge Base System -A full-stack RAG (Retrieval-Augmented Generation) Q&A system built with Next.js, LangChain, and Gemini. Designed to handle large-scale documentation (4.2k+ files simulated) with high-accuracy retrieval using Local Vector Store. +A full-stack RAG (Retrieval-Augmented Generation) Q&A system built with Next.js, LangChain, and Gemini. Designed to handle large-scale documentation with high-accuracy retrieval using Local Vector Store. + +## 🚀 Deployment on Hugging Face Spaces + +This project is configured for easy deployment on Hugging Face Spaces using Docker. + +### Prerequisites + +1. **Create a Space**: Go to [Hugging Face Spaces](https://huggingface.co/spaces) and create a new Space. + * **SDK**: Select `Docker`. + * **Hardware**: Default CPU (Free) is sufficient, but 2 vCPU is recommended for faster embedding. + +2. **Environment Variables**: + Go to your Space's **Settings** tab and add the following secrets: + * `GOOGLE_GENERATIVE_AI_API_KEY`: Your Google Gemini API Key (Required for embeddings). + * `DEEPSEEK_API_KEY`: Your DeepSeek API Key (Recommended for chat). + * `NEXT_PUBLIC_DEMO_MODE`: Set to `false` to use the real database. + +### Syncing Code + +You can upload the code directly via Git: + +```bash +# Initialize git if not already done +git init +git remote add space https://huggingface.co/spaces/YOUR_USERNAME/YOUR_SPACE_NAME + +# Add all files (including rag-kb.db and vector_store) +git add . +git commit -m "Initial commit with DB and Vector Store" + +# Push to Hugging Face +git push space main +``` + +**Note**: The `rag-kb.db` and `vector_store/` files are included in the upload to ensure the knowledge base is pre-populated. ## Tech Stack diff --git a/analyze_bulk_creation.js b/analyze_bulk_creation.js new file mode 100644 index 0000000000000000000000000000000000000000..bdab62b584931121e1be3872a9b09d8aea10c384 --- /dev/null +++ b/analyze_bulk_creation.js @@ -0,0 +1,47 @@ + +const db = require('better-sqlite3')('rag-kb.db'); + +const startTime = new Date('2025-01-01').getTime(); +const endTime = new Date('2026-01-01').getTime(); + +const docs = db.prepare(` + SELECT + created_at, + updated_at, + word_count + FROM documents + WHERE + yuque_id != 0 + AND namespace != 'NOTES' + AND (slug IS NULL OR slug NOT LIKE 'dir-%') + AND created_at >= ? + AND created_at < ? +`).all(startTime, endTime); + +let sameTimeCount = 0; +let sameTimeWords = 0; +let diffTimeCount = 0; +let diffTimeWords = 0; + +// Threshold for "same time" (e.g., 60 seconds) +const THRESHOLD_MS = 60 * 1000; + +docs.forEach(doc => { + const diff = Math.abs(doc.updated_at - doc.created_at); + if (diff <= THRESHOLD_MS) { + sameTimeCount++; + sameTimeWords += doc.word_count || 0; + } else { + diffTimeCount++; + diffTimeWords += doc.word_count || 0; + } +}); + +console.log(`\nAnalysis of ${docs.length} docs created in 2025:`); +console.log(`Updated ~= Created (<= 60s):`); +console.log(` Count: ${sameTimeCount}`); +console.log(` Words: ${(sameTimeWords / 10000).toFixed(1)}w`); + +console.log(`\nUpdated > Created (> 60s):`); +console.log(` Count: ${diffTimeCount}`); +console.log(` Words: ${(diffTimeWords / 10000).toFixed(1)}w`); diff --git a/check_db.py b/check_db.py new file mode 100644 index 0000000000000000000000000000000000000000..11f05096bf930ef49f20f7a015cc675ca77c441f --- /dev/null +++ b/check_db.py @@ -0,0 +1,25 @@ +import sqlite3 +import os + +files = [f for f in os.listdir('.') if f.endswith('.db') or f.endswith('.sqlite')] + +for db_file in files: + print(f"--- {db_file} ---") + try: + conn = sqlite3.connect(db_file) + cursor = conn.cursor() + cursor.execute("SELECT name FROM sqlite_master WHERE type='table';") + tables = cursor.fetchall() + for table in tables: + table_name = table[0] + print(f"Table: {table_name}") + cursor.execute(f"PRAGMA table_info({table_name})") + columns = cursor.fetchall() + col_names = [col[1] for col in columns] + print(f" Columns: {col_names}") + if 'content' in col_names: + print(" *** HAS CONTENT COLUMN ***") + conn.close() + except Exception as e: + print(f"Error: {e}") + print("\n") diff --git a/debug-db-tags.ts b/debug-db-tags.ts new file mode 100644 index 0000000000000000000000000000000000000000..eed08ee2147e19e7e78ac8892891546ecf4251e6 --- /dev/null +++ b/debug-db-tags.ts @@ -0,0 +1,26 @@ + +import db from './src/lib/db'; + +const docs = db.prepare(` + SELECT id, title, tags + FROM documents + WHERE namespace = 'NOTES' AND tags IS NOT NULL + LIMIT 5 +`).all(); + +console.log('Raw DB Tags Content:'); +docs.forEach((d: any) => { + console.log(`Title: ${d.title}`); + console.log(`Tags (Raw): ${d.tags}`); + try { + const parsed = JSON.parse(d.tags); + console.log('Tags (Parsed):', JSON.stringify(parsed, null, 2)); + console.log('Is Array?', Array.isArray(parsed)); + if (Array.isArray(parsed)) { + console.log('Element types:', parsed.map((x: any) => typeof x)); + } + } catch (e) { + console.log('Parse Error:', e); + } + console.log('---'); +}); diff --git a/debug-verify-tags.ts b/debug-verify-tags.ts new file mode 100644 index 0000000000000000000000000000000000000000..1677ef38402aa9276bd039ff66af9ede3a3f4887 --- /dev/null +++ b/debug-verify-tags.ts @@ -0,0 +1,51 @@ + +import db from './src/lib/db'; + +const docs = db.prepare(` + SELECT id, title, tags + FROM documents + WHERE namespace = 'NOTES' +`).all(); + +const allTags = new Map(); +let untaggedCount = 0; + +docs.forEach((d: any) => { + let tags: string[] = []; + if (typeof d.tags === 'string' && d.tags.length > 0) { + try { + const parsed = JSON.parse(d.tags); + if (Array.isArray(parsed)) { + tags = parsed.map((x: any) => { + if (typeof x === 'string') return x; + if (typeof x === 'object' && x !== null) { + return x.title || x.name || ''; + } + return ''; + }).filter((x: string) => x.length > 0); + } + } catch { + tags = []; + } + } + + if (tags.length > 0) { + tags.forEach(t => allTags.set(t, (allTags.get(t) || 0) + 1)); + } else { + untaggedCount++; + } +}); + +console.log('=== Tag List Verification ==='); +const sortedTags = Array.from(allTags.entries()) + .map(([name, count]) => ({ name, count })) + .sort((a, b) => b.count - a.count); + +if (untaggedCount > 0) { + sortedTags.push({ name: '无标签', count: untaggedCount }); +} + +sortedTags.forEach(t => { + console.log(`${t.name}: ${t.count}`); +}); +console.log('============================='); diff --git a/debug-yuque-notes.ts b/debug-yuque-notes.ts new file mode 100644 index 0000000000000000000000000000000000000000..f5f0efa798dee33abdecf533cd58ac3c25d58bfb --- /dev/null +++ b/debug-yuque-notes.ts @@ -0,0 +1,64 @@ + +import * as dotenv from "dotenv"; +import fetch from "node-fetch"; + +dotenv.config({ path: ".env.local" }); +dotenv.config(); + +const TOKEN = process.env.YUQUE_TOKEN; +const BASE_URL = "https://www.yuque.com/api/v2"; + +interface Note { + tags?: string[]; + [key: string]: unknown; +} + +interface YuqueResponse { + data: { + notes: Note[]; + }; +} + +async function checkNotes() { + if (!TOKEN) { + console.error("No YUQUE_TOKEN found in env"); + return; + } + + console.log("Fetching notes..."); + const url = `${BASE_URL}/notes?offset=0&limit=50`; + const headers = { + "X-Auth-Token": TOKEN, + "User-Agent": "debug-script", + "Content-Type": "application/json", + }; + + try { + const res = await fetch(url, { headers }); + if (!res.ok) { + console.error(`Error: ${res.status} ${res.statusText}`); + const text = await res.text(); + console.error(text); + return; + } + + const data = await res.json() as unknown as YuqueResponse; + const notes = data.data.notes || []; + + console.log(`Found ${notes.length} notes.`); + + const notesWithTags = notes.filter((n) => n.tags && n.tags.length > 0); + console.log(`Notes with tags: ${notesWithTags.length}`); + + if (notesWithTags.length > 0) { + console.log("Example note with tags:"); + console.log(JSON.stringify(notesWithTags[0], null, 2)); + } else { + console.log("No tags found in the first 50 notes."); + } + } catch (e) { + console.error("Failed:", e); + } +} + +checkNotes(); diff --git a/debug_2025_stats.js b/debug_2025_stats.js new file mode 100644 index 0000000000000000000000000000000000000000..c5aa5515368df2696fb9bbdb30a08810675b3f06 --- /dev/null +++ b/debug_2025_stats.js @@ -0,0 +1,57 @@ + +const db = require('better-sqlite3')('rag-kb.db'); + +// 1. Check table info to see available columns +const columns = db.prepare("PRAGMA table_info(documents)").all(); +console.log("Columns:", columns.map(c => c.name).join(', ')); + +// 2. Analyze 2025 creation distribution +// Convert unix timestamp to YYYY-MM-DD +const query = ` +SELECT + date(created_at / 1000, 'unixepoch', 'localtime') as created_date, + COUNT(*) as count +FROM documents +WHERE + yuque_id != 0 + AND namespace != 'NOTES' + AND (slug IS NULL OR slug NOT LIKE 'dir-%') + AND created_at >= ? + AND created_at < ? +GROUP BY created_date +ORDER BY count DESC +LIMIT 20; +`; + +const startTime = new Date('2025-01-01').getTime(); +const endTime = new Date('2026-01-01').getTime(); + +const results = db.prepare(query).all(startTime, endTime); + +console.log("\nTop creation dates in 2025 (non-NOTES):"); +results.forEach(r => { + console.log(`${r.created_date}: ${r.count} docs`); +}); + +// 3. Analyze 2025 update distribution for comparison +const updateQuery = ` +SELECT + date(updated_at / 1000, 'unixepoch', 'localtime') as updated_date, + COUNT(*) as count +FROM documents +WHERE + yuque_id != 0 + AND namespace != 'NOTES' + AND (slug IS NULL OR slug NOT LIKE 'dir-%') + AND updated_at >= ? + AND updated_at < ? +GROUP BY updated_date +ORDER BY count DESC +LIMIT 10; +`; + +const updateResults = db.prepare(updateQuery).all(startTime, endTime); +console.log("\nTop update dates in 2025 (non-NOTES):"); +updateResults.forEach(r => { + console.log(`${r.updated_date}: ${r.count} docs`); +}); diff --git a/deploy.log b/deploy.log new file mode 100644 index 0000000000000000000000000000000000000000..6b867896601c41c422a50c4ade2593127e94e831 --- /dev/null +++ b/deploy.log @@ -0,0 +1,28 @@ +[2026-01-09 12:42:37] Configuring Git LFS settings... +[2026-01-09 12:42:37] Checking network connectivity... +[2026-01-09 12:42:48] Warning: Could not ping hf.co, but proceeding with push attempt... +[2026-01-09 12:42:48] Starting push attempt 1 of 3... +[2026-01-09 12:44:19] Upload failed with exit code 1. +[2026-01-09 12:44:19] Waiting 5 seconds before retrying... +[2026-01-09 12:44:24] Starting push attempt 2 of 3... +[2026-01-09 12:44:38] Upload failed with exit code 1. +[2026-01-09 12:44:38] Waiting 10 seconds before retrying... +[2026-01-09 12:44:48] Starting push attempt 3 of 3... +[2026-01-09 12:45:02] Upload failed with exit code 1. +[2026-01-09 12:45:02] All 3 attempts failed. +[2026-01-09 12:45:02] Please check deploy.log for details. +[2026-01-09 12:45:27] Configuring Git LFS settings... +[2026-01-09 12:45:27] Checking network connectivity... +[2026-01-09 12:45:38] Warning: Could not ping hf.co, but proceeding with push attempt... +[2026-01-09 12:45:38] Starting push attempt 1 of 3... +[2026-01-09 12:45:52] Upload failed with exit code 1. +[2026-01-09 12:45:52] Waiting 5 seconds before retrying... +[2026-01-09 12:45:57] Starting push attempt 2 of 3... +[2026-01-09 12:46:11] Upload failed with exit code 1. +[2026-01-09 12:46:11] Waiting 10 seconds before retrying... +[2026-01-09 12:46:21] Starting push attempt 3 of 3... +[2026-01-09 12:46:36] Upload failed with exit code 1. +[2026-01-09 12:46:36] All 3 attempts failed. +[2026-01-09 12:46:36] Executing fallback: Marking as pending upload in local cache... +[2026-01-09 12:46:36] Status saved to upload_status.json +[2026-01-09 12:46:36] Deployment process completed with pending status. diff --git a/documents.db b/documents.db deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/eslint.config.mjs b/eslint.config.mjs index 05e726d1b4201bc8c7716d2b058279676582e8c0..33231c21139b2676910764365a1e49ac22e3e3e8 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -5,6 +5,18 @@ import nextTs from "eslint-config-next/typescript"; const eslintConfig = defineConfig([ ...nextVitals, ...nextTs, + { + files: ["**/*.js", "**/*.cjs", "**/*.mjs"], + rules: { + "@typescript-eslint/no-require-imports": "off", + }, + }, + { + files: ["scripts/**/*.{ts,tsx}", "debug*.ts", "debug-*.ts"], + rules: { + "@typescript-eslint/no-explicit-any": "off", + }, + }, // Override default ignores of eslint-config-next. globalIgnores([ // Default ignores of eslint-config-next: diff --git a/package.json b/package.json index 1790afad64db7ba5df1ac4a35846fcacfa0327bf..5f05cf2ccc78a6f17e3a04ea815cf93e7a409012 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,8 @@ "start": "next start", "lint": "eslint", "ingest:yuque": "npx tsx scripts/yuque-ingest.ts", - "query": "npx tsx scripts/query.ts" + "query": "npx tsx scripts/query.ts", + "export:hf": "npx tsx scripts/export-hf-dataset.ts" }, "dependencies": { "@ai-sdk/google": "^0.0.55", diff --git a/scripts/benchmark-notes-speed.ts b/scripts/benchmark-notes-speed.ts new file mode 100644 index 0000000000000000000000000000000000000000..dba93824d5dc54d3727f3a0edbb27f9f81b5ebb2 --- /dev/null +++ b/scripts/benchmark-notes-speed.ts @@ -0,0 +1,61 @@ +import * as dotenv from "dotenv"; +import { SimpleYuqueLoader, YuqueDoc } from "../src/lib/yuque-service"; + +dotenv.config({ path: ".env.local" }); +dotenv.config(); + +async function asyncPool(poolLimit: number, array: T[], iteratorFn: (item: T, array: T[]) => Promise): Promise { + const ret: Promise[] = []; + const executing: Promise[] = []; + for (const item of array) { + const p = Promise.resolve().then(() => iteratorFn(item, array)); + ret.push(p); + if (poolLimit <= array.length) { + const e: Promise = p.then(() => { + executing.splice(executing.indexOf(e), 1); + }); + executing.push(e); + if (executing.length >= poolLimit) { + await Promise.race(executing); + } + } + } + return Promise.all(ret); +} + +async function run() { + const token = process.env.YUQUE_TOKEN; + if (!token) { + console.error("缺少环境变量 YUQUE_TOKEN"); + process.exit(1); + } + const concurrency = parseInt(process.env.BENCH_NOTES_CONCURRENCY ?? "2"); + const limit = parseInt(process.env.BENCH_NOTES_LIMIT ?? "50"); + const loader = new SimpleYuqueLoader(token, "NOTES"); + + const listRes = await loader.fetchAPI(`/notes?offset=0&limit=${Math.min(limit, 50)}`); + const notes: YuqueDoc[] = Array.isArray(listRes?.data?.notes) + ? listRes.data.notes.map((n: any) => ({ + id: n.id, + slug: n.slug, + title: n.content?.abstract ?? `小记-${n.id}`, + uuid: n.slug, + })) + : []; + console.log(`准备抓取 ${notes.length} 条小记详情,并发=${concurrency}`); + + const start = Date.now(); + let ok = 0; + await asyncPool(concurrency, notes, async (note) => { + const doc = await loader.fetchNoteDetail(note); + if (doc) ok++; + }); + const elapsed = (Date.now() - start) / 1000; + const rps = ok / elapsed; + console.log(`完成 ${ok}/${notes.length} 条,用时 ${elapsed.toFixed(2)}s,平均 ${rps.toFixed(2)} req/s`); +} + +run().catch((e) => { + console.error("基准测试失败:", e); + process.exit(1); +}); diff --git a/scripts/deploy_to_hf.sh b/scripts/deploy_to_hf.sh new file mode 100755 index 0000000000000000000000000000000000000000..fae16d31f7c65acb54d46fa70df4c4ce84f02f58 --- /dev/null +++ b/scripts/deploy_to_hf.sh @@ -0,0 +1,67 @@ +#!/bin/bash + +# Configuration +MAX_RETRIES=3 +LOG_FILE="deploy.log" +REMOTE_URL="git@hf.co:spaces/duqing2026/rag-kb-demo" + +# Function to log messages +log_message() { + local timestamp=$(date "+%Y-%m-%d %H:%M:%S") + echo "[$timestamp] $1" | tee -a "$LOG_FILE" +} + +# 1. Configure Git LFS to skip lock verification (Fixes the specific error) +log_message "Configuring Git LFS settings..." +git config lfs.locksverify false +git config lfs.https://hf.co/spaces/duqing2026/rag-kb-demo.git/info/lfs.locksverify false + +# 2. Check network connectivity (Simple check) +log_message "Checking network connectivity..." +if ping -c 1 hf.co &> /dev/null; then + log_message "Network connection to hf.co confirmed." +else + log_message "Warning: Could not ping hf.co, but proceeding with push attempt..." +fi + +# 3. Push with retry logic +attempt=1 +while [ $attempt -le $MAX_RETRIES ]; do + log_message "Starting push attempt $attempt of $MAX_RETRIES..." + + # Try to push both LFS objects and git refs + # Using -u origin main to ensure upstream tracking + if git push -u origin main; then + log_message "Upload successful!" + exit 0 + else + exit_code=$? + log_message "Upload failed with exit code $exit_code." + + if [ $attempt -lt $MAX_RETRIES ]; then + wait_time=$((attempt * 5)) + log_message "Waiting $wait_time seconds before retrying..." + sleep $wait_time + ((attempt++)) + else + log_message "All $MAX_RETRIES attempts failed." + + # 4. Fallback: Save to local cache status (User Option 3) + log_message "Executing fallback: Marking as pending upload in local cache..." + + # Create status file + cat > upload_status.json <> = []; + + for (const d of docs) { + const ns = d.namespace || 'UNKNOWN'; + const nsDir = path.join(filesDir, ns); + ensureDir(nsDir); + const slug = safeSlug(d.slug); + const filePath = path.join(nsDir, `${slug}.md`); + const content = d.content_preview || ''; + fs.writeFileSync(filePath, content, 'utf8'); + + index.push({ + id: d.id, + yuque_id: d.yuque_id, + title: d.title, + slug: d.slug, + url: d.url, + namespace: d.namespace, + word_count: d.word_count, + updated_at: d.updated_at, + created_at: d.created_at, + tags: d.tags, + sort_order: d.sort_order, + }); + } + + writeJson(path.join(metaDir, 'documents.json'), { + count: index.length, + documents: index, + }); + + writeJson(path.join(metaDir, 'knowledge_bases.json'), { + count: kbs.length, + knowledge_bases: kbs, + }); + + const summary = { + generated_at: new Date().toISOString(), + files_dir: 'files', + metadata_dir: 'metadata', + namespaces: Array.from(new Set(index.map((d) => d.namespace || 'UNKNOWN'))), + }; + writeJson(path.join(outDir, 'dataset_summary.json'), summary); + + console.log(`Exported ${index.length} documents to: ${outDir}`); +} + +main().catch((e) => { + console.error(e); + process.exit(1); +}); diff --git a/scripts/import_hf_metadata.ts b/scripts/import_hf_metadata.ts new file mode 100644 index 0000000000000000000000000000000000000000..ffeccd97f83c24d77ec49d250e87c5e70878bfbd --- /dev/null +++ b/scripts/import_hf_metadata.ts @@ -0,0 +1,147 @@ +import fs from 'fs'; +import path from 'path'; +import Database from 'better-sqlite3'; + +type DocRow = { + id: string; + yuque_id?: number; + title: string; + slug: string; + url?: string | null; + namespace?: string | null; + content_preview?: string | null; + synced_at?: number; + parent_uuid?: string | null; + uuid?: string | null; + sort_order?: number | null; + word_count?: number | null; + updated_at?: number | null; + created_at?: number | null; + tags?: string | null; +}; + +type KbRow = { + namespace: string; + name: string; + description?: string | null; + synced_at: number; + last_offset?: number | null; +}; + +function readJson(p: string) { + return JSON.parse(fs.readFileSync(p, 'utf8')); +} + +function ensureDir(p: string) { + if (!fs.existsSync(p)) fs.mkdirSync(p, { recursive: true }); +} + +function main() { + const cwd = process.cwd(); + const datasetRoot = process.env.HF_DATASET_ROOT || path.join(cwd, '..', 'hf_dataset_rag'); + const metaDir = path.join(datasetRoot, 'metadata'); + const docsDir = path.join(metaDir, 'documents'); + const indexPath = path.join(docsDir, 'index.json'); + const kbPath = path.join(metaDir, 'knowledge_bases.json'); + + if (!fs.existsSync(indexPath)) { + console.error('Missing index.json:', indexPath); + process.exit(1); + } + if (!fs.existsSync(kbPath)) { + console.error('Missing knowledge_bases.json:', kbPath); + process.exit(1); + } + + const dbPath = path.join(cwd, 'rag-kb.db'); + ensureDir(cwd); + const db = new Database(dbPath); + + db.exec(` + CREATE TABLE IF NOT EXISTS sessions ( + id TEXT PRIMARY KEY, + title TEXT NOT NULL, + created_at INTEGER NOT NULL, + type TEXT DEFAULT 'chat' + ); + + CREATE TABLE IF NOT EXISTS messages ( + id TEXT PRIMARY KEY, + session_id TEXT NOT NULL, + role TEXT NOT NULL, + content TEXT NOT NULL, + created_at INTEGER NOT NULL, + FOREIGN KEY (session_id) REFERENCES sessions (id) ON DELETE CASCADE + ); + + CREATE TABLE IF NOT EXISTS documents ( + id TEXT PRIMARY KEY, + yuque_id INTEGER, + title TEXT NOT NULL, + slug TEXT NOT NULL, + url TEXT, + namespace TEXT, + content_preview TEXT, + synced_at INTEGER NOT NULL, + parent_uuid TEXT, + uuid TEXT, + sort_order INTEGER DEFAULT 0, + word_count INTEGER DEFAULT 0, + updated_at INTEGER, + created_at INTEGER, + tags TEXT + ); + + CREATE TABLE IF NOT EXISTS knowledge_bases ( + namespace TEXT PRIMARY KEY, + name TEXT NOT NULL, + description TEXT, + synced_at INTEGER NOT NULL, + last_offset INTEGER DEFAULT 0 + ); + `); + + const reset = (process.env.IMPORT_RESET || '').toLowerCase(); + if (reset === '1' || reset === 'true') { + db.exec('DELETE FROM documents; DELETE FROM knowledge_bases;'); + } + + const kbData = readJson(kbPath) as { knowledge_bases?: KbRow[] }; + const kbInsert = db.prepare(` + INSERT OR REPLACE INTO knowledge_bases (namespace, name, description, synced_at, last_offset) + VALUES (@namespace, @name, @description, @synced_at, COALESCE(@last_offset, 0)) + `); + for (const kb of kbData.knowledge_bases || []) { + kbInsert.run(kb); + } + + const index = readJson(indexPath) as { parts?: string[] }; + const docInsert = db.prepare(` + INSERT OR REPLACE INTO documents + (id, yuque_id, title, slug, url, namespace, content_preview, synced_at, parent_uuid, uuid, sort_order, word_count, updated_at, created_at, tags) + VALUES (@id, @yuque_id, @title, @slug, @url, @namespace, '', COALESCE(@updated_at, strftime('%s','now')*1000), @parent_uuid, @uuid, COALESCE(@sort_order, 0), COALESCE(@word_count, 0), @updated_at, @created_at, @tags) + `); + + let total = 0; + for (const part of index.parts || []) { + const partPath = path.join(docsDir, part); + const arr = readJson(partPath) as DocRow[]; + for (const d of arr) { + docInsert.run(d); + total += 1; + } + } + + console.log(JSON.stringify({ + inserted_documents: total, + inserted_kbs: (kbData.knowledge_bases || []).length, + db_path: dbPath + }, null, 2)); +} + +try { + main(); +} catch (e) { + console.error(e); + process.exit(1); +} diff --git a/scripts/push_hf_dataset.py b/scripts/push_hf_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..49d3f937f4a1251dd7f66eaf115f32de0305f99a --- /dev/null +++ b/scripts/push_hf_dataset.py @@ -0,0 +1,105 @@ +import os +import sys +import json +import pathlib +from typing import List +import httpx +from contextlib import suppress + +def _is_ascii(s: str) -> bool: + try: + s.encode("ascii") + return True + except Exception: + return False + +def list_initial_files(root: pathlib.Path, start: int, limit: int, include_license: bool, include_readme: bool) -> List[pathlib.Path]: + meta = root / "metadata" + docs_dir = meta / "documents" + files = [ + docs_dir / "index.json", + meta / "knowledge_bases.json", + ] + if include_license and (root / "LICENSE").exists(): + files.append(root / "LICENSE") + if include_readme and (root / "README.md").exists(): + files.append(root / "README.md") + index = json.loads((docs_dir / "index.json").read_text("utf8")) + parts = index.get("parts", []) + slice_parts = parts[start:start + limit] + for p in slice_parts: + files.append(docs_dir / p) + return files + +def create_commit(repo_id: str, token: str, root: pathlib.Path, paths: List[pathlib.Path], message: str): + from huggingface_hub import HfApi, CommitOperationAdd + api = HfApi(token=token) + api.create_repo(repo_id=repo_id, repo_type="dataset", private=True, exist_ok=True) + ops = [] + for p in paths: + rel = p.relative_to(root) + ops.append(CommitOperationAdd(path_in_repo=str(rel), path_or_fileobj=str(p))) + def _commit(ops_slice: List[CommitOperationAdd], msg: str): + try: + api.create_commit(repo_id=repo_id, repo_type="dataset", operations=ops_slice, commit_message=msg) + except Exception as e: + s = str(e) + timeout_like = isinstance(e, httpx.ReadTimeout) or "ReadTimeout" in s or "Timeout" in s + if timeout_like and len(ops_slice) > 1: + mid = len(ops_slice) // 2 + left = ops_slice[:mid] + right = ops_slice[mid:] + _commit(left, msg + " [chunk A]") + _commit(right, msg + " [chunk B]") + else: + raise + chunk_size_env = os.getenv("HF_COMMIT_CHUNK_SIZE", "") + chunk_size = int(chunk_size_env) if chunk_size_env.isdigit() else 0 + if chunk_size and chunk_size > 0: + for i in range(0, len(ops), chunk_size): + _commit(ops[i:i + chunk_size], message + f" [batch {i//chunk_size}]") + else: + _commit(ops, message) + +def dry_run_summary(paths: List[pathlib.Path]): + total = 0 + items = [] + for p in paths: + size = p.stat().st_size + total += size + items.append((str(p), size)) + print(json.dumps({"files": [{"path": i[0], "size": i[1]} for i in items], "total_bytes": total}, ensure_ascii=False, indent=2)) + +def main(): + root_env = os.getenv("HF_DATASET_ROOT", "") + repo_id = os.getenv("HF_REPO_ID", "") + token = os.getenv("HF_TOKEN", "") or os.getenv("HUGGINGFACE_TOKEN", "") + offset_env = os.getenv("HF_PARTS_OFFSET", "") + limit_env = os.getenv("HF_PARTS_LIMIT", "") or os.getenv("HF_INITIAL_PARTS", "") + include_license = os.getenv("HF_INCLUDE_LICENSE", "1") not in ("0", "false", "False") + include_readme = os.getenv("HF_INCLUDE_README", "1") not in ("0", "false", "False") + offset = int(offset_env) if offset_env.isdigit() else 0 + limit = int(limit_env) if limit_env.isdigit() else 2 + + root = pathlib.Path(root_env or pathlib.Path(__file__).resolve().parents[2] / "hf_dataset_rag") + paths = list_initial_files(root, offset, limit, include_license, include_readme) + + if not repo_id or not token: + print("Missing HF_REPO_ID or HF_TOKEN; performing dry-run") + dry_run_summary(paths) + sys.exit(0) + if not _is_ascii(token) or not _is_ascii(repo_id): + print("HF_TOKEN or HF_REPO_ID contains non-ASCII characters") + sys.exit(2) + os.environ["HF_HUB_USER_AGENT"] = "rag-kb-uploader" + os.environ.setdefault("HF_HUB_TIMEOUT", "60") + os.environ.setdefault("HF_HUB_READ_TIMEOUT", "60") + with suppress(Exception): + from huggingface_hub import HfApi + HfApi(token=token).whoami() + + create_commit(repo_id, token, root, paths, f"Upload: metadata + parts [{offset}, {offset + limit})") + print("Commit completed") + +if __name__ == "__main__": + main() diff --git a/scripts/test-notes-pagination.ts b/scripts/test-notes-pagination.ts new file mode 100644 index 0000000000000000000000000000000000000000..3949678b06ebc894c2c40b59c150fb3ff550e326 --- /dev/null +++ b/scripts/test-notes-pagination.ts @@ -0,0 +1,33 @@ +import * as dotenv from "dotenv"; +import { SimpleYuqueLoader } from "../src/lib/yuque-service"; + +dotenv.config({ path: ".env.local" }); +dotenv.config(); + +async function test() { + const token = process.env.YUQUE_TOKEN; + if (!token) { + console.error("缺少环境变量 YUQUE_TOKEN"); + process.exit(1); + } + const loader = new SimpleYuqueLoader(token, "NOTES"); + + const limits = [30, 50]; + for (const limit of limits) { + try { + const res = await loader.fetchAPI(`/notes?offset=0&limit=${limit}`); + const list = Array.isArray(res?.data?.notes) ? res.data.notes : []; + console.log(`请求 limit=${limit} -> 返回 ${list.length} 条`); + if (list.length > 0) { + console.log(`示例ID范围: ${list[0]?.id} ... ${list[list.length - 1]?.id}`); + } + } catch (e) { + console.error(`请求 limit=${limit} 失败:`, e instanceof Error ? e.message : String(e)); + } + } +} + +test().catch((e) => { + console.error("测试失败:", e); + process.exit(1); +}); diff --git a/src/app/api/backup/route.ts b/src/app/api/backup/route.ts new file mode 100644 index 0000000000000000000000000000000000000000..3d83b263e59805e70b7e55368ef516a6e8e3491b --- /dev/null +++ b/src/app/api/backup/route.ts @@ -0,0 +1,139 @@ +import { NextResponse } from 'next/server'; +import fs from 'fs'; +import path from 'path'; +import Database from 'better-sqlite3'; + +export async function POST() { + try { + const cwd = process.cwd(); + const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); + const backupDirName = '备份-语雀数据-JSON'; + const backupDir = path.join(cwd, backupDirName); + + if (!fs.existsSync(backupDir)) { + fs.mkdirSync(backupDir, { recursive: true }); + } else { + // Clear existing files + const files = fs.readdirSync(backupDir); + for (const file of files) { + fs.unlinkSync(path.join(backupDir, file)); + } + } + + // Connect to the main database + const dbPath = path.join(cwd, 'rag-kb.db'); + if (!fs.existsSync(dbPath)) { + return NextResponse.json( + { message: 'Main database (rag-kb.db) not found', success: false }, + { status: 404 } + ); + } + + const db = new Database(dbPath, { readonly: true }); + + // Get all namespaces (knowledge bases) + // Try to get from knowledge_bases table first, if not, distinct from documents + let kbs: { namespace: string, name: string }[] = []; + try { + const rows = db.prepare('SELECT namespace, name FROM knowledge_bases').all() as { namespace: string, name: string }[]; + kbs = rows; + } catch (e) { + console.warn('Could not read knowledge_bases table, falling back to documents', e); + const rows = db.prepare('SELECT DISTINCT namespace FROM documents WHERE namespace IS NOT NULL').all() as { namespace: string }[]; + kbs = rows.map(r => ({ namespace: r.namespace, name: r.namespace })); + } + + if (kbs.length === 0) { + return NextResponse.json( + { message: 'No knowledge bases found to backup', success: false }, + { status: 404 } + ); + } + + const backupsCreated: string[] = []; + + // Dataset root for reading content files + const hfDatasetRoot = process.env.HF_DATASET_ROOT || path.join(cwd, '..', 'hf_dataset_rag'); + + for (const kb of kbs) { + const ns = kb.namespace; + // Query documents for this namespace + const docs = db.prepare(` + SELECT title, slug, created_at, updated_at, tags + FROM documents + WHERE namespace = ? + `).all(ns) as { title: string, slug: string, created_at: number, updated_at: number, tags: string }[]; + + const filteredDocs = docs.filter(doc => { + if (!doc.tags) return true; + try { + const tags = JSON.parse(doc.tags); + if (Array.isArray(tags)) { + return !tags.includes('个人资料'); + } + } catch (e) { + // If tags is not JSON, check as string (fallback) + return !doc.tags.includes('个人资料'); + } + return true; + }); + + const exportData = filteredDocs.map(doc => { + // Try to read content from file + let content = ''; + try { + // Construct path: hf_dataset_rag/files/namespace/slug.md + // Note: slug might contain subdirectories? usually slug is just filename base. + // Based on grep: files/lianmt/jm/ehzgn5-624997.md + // So structure is files/namespace/slug.md + + // Handle namespace with slashes? e.g. lianmt/cq + // The grep showed: files/lianmt/jm/... + // So if ns is "lianmt/jm", then path is files/lianmt/jm/... + + const filePath = path.join(hfDatasetRoot, 'files', ns, `${doc.slug}.md`); + if (fs.existsSync(filePath)) { + content = fs.readFileSync(filePath, 'utf8'); + } else { + // Try looking for it without namespace structure if simple? + // But grep confirmed structure. + // content = `(File not found: ${filePath})`; + } + } catch (err) { + console.error(`Error reading file for ${doc.slug}:`, err); + } + + return { + title: doc.title, + content: content, + created_at: new Date(doc.created_at).toISOString(), + updated_at: doc.updated_at ? new Date(doc.updated_at).toISOString() : null, + tags: doc.tags + }; + }); + + // Create sanitized filename + // Use kb.name (Chinese name) for filename + const safeName = (kb.name || ns).replace(/[\/\\:]/g, '_'); + const fileName = `${safeName}_${timestamp}.json`; + const filePath = path.join(backupDir, fileName); + + fs.writeFileSync(filePath, JSON.stringify(exportData)); + backupsCreated.push(fileName); + } + + db.close(); + + return NextResponse.json({ + message: `JSON export created successfully in folder: ${backupDirName}`, + files: backupsCreated, + success: true + }); + } catch (error) { + console.error('Export failed:', error); + return NextResponse.json( + { message: 'Export failed', error: String(error), success: false }, + { status: 500 } + ); + } +} diff --git a/src/app/api/chat/route.ts b/src/app/api/chat/route.ts index 43ded486eed039ec33e36787f4ab809aaf8f5bf7..9b1149415d17618521b892231501e82d6dac268f 100644 --- a/src/app/api/chat/route.ts +++ b/src/app/api/chat/route.ts @@ -40,9 +40,30 @@ const google = createGoogleGenerativeAI({ // Allow streaming responses up to 300 seconds export const maxDuration = 300; +const QUIZ_GENERATION_PROMPT = `请基于知识库内容生成 5 道单选题。**直接返回 JSON 代码块,不要废话。** + +要求: +1. 结果必须封装在 \`\`\`quiz 代码块中。 +2. 选项数组 options 中只包含内容,不要 A/B/C/D 前缀。 +3. explanation 解析需简练(50字以内)。 + +格式: +\`\`\`quiz +[ + { + "id": 1, + "question": "...", + "options": ["A", "B", "C", "D"], + "correctAnswer": 0, + "explanation": "..." + } +] +\`\`\``; + export async function POST(req: Request) { try { - const { messages, model, useRAG } = await req.json(); + const body = await req.json(); + const { messages, model, useRAG, fileInfo } = body; // Log the incoming request details for debugging console.log(`[API] Received chat request. Model: ${model}, useRAG: ${useRAG}`); @@ -50,6 +71,12 @@ export async function POST(req: Request) { // Get the last message to use as the query for RAG const lastMessage = messages[messages.length - 1]; const query = lastMessage.content; + + // For quiz generation, use a broader query to retrieve relevant context + let ragQuery = query; + if (query.trim() === '对话试题' || query.trim() === 'Generate Quiz') { + ragQuery = "summary 摘要 concept 概念 main point 核心观点"; + } // Mock Streaming Response for testing // Trigger if query is exactly 'mock-test' OR contains keywords for test generation @@ -152,18 +179,31 @@ Here is a test quiz to verify the rendering: try { const docCount = db.prepare('SELECT COUNT(*) as count FROM documents').get() as { count: number }; const totalWords = db.prepare('SELECT SUM(word_count) as total FROM documents').get() as { total: number }; - const kbs = db.prepare('SELECT name, namespace FROM knowledge_bases').all() as { name: string, namespace: string }[]; const lastSync = db.prepare('SELECT MAX(synced_at) as last_sync FROM documents').get() as { last_sync: number }; - const kbList = kbs.map(k => `${k.name} (${k.namespace})`).join(', '); + // Get per-KB stats + const kbStatsDetails = db.prepare(` + SELECT + kb.name, + kb.namespace, + COUNT(d.id) as doc_count, + SUM(d.word_count) as word_count + FROM knowledge_bases kb + LEFT JOIN documents d ON kb.namespace = d.namespace + GROUP BY kb.namespace + `).all() as { name: string, namespace: string, doc_count: number, word_count: number }[]; + + const kbList = kbStatsDetails.map(k => `- ${k.name} (${k.namespace}): ${k.doc_count} documents, ${k.word_count || 0} characters`).join('\n'); const lastSyncDate = lastSync.last_sync ? new Date(lastSync.last_sync).toLocaleString('zh-CN') : 'Never'; kbStats = ` Knowledge Base Statistics: - Total Documents: ${docCount.count} - Total Word Count: ${totalWords.total || 0} -- Knowledge Bases: ${kbList} - Last Synced: ${lastSyncDate} + +Knowledge Base Distribution: +${kbList} `; } catch (e) { console.warn("Failed to fetch KB stats:", e); @@ -179,8 +219,20 @@ Knowledge Base Statistics: const vectorStore = await getVectorStore(); // Perform similarity search - console.log(`[RAG] Searching for context: "${query.substring(0, 50)}..."`); - const results = await vectorStore.similaritySearch(query, 5); // Retrieve top 5 chunks + // We fetch more results (k=20) and filter in memory to avoid HNSWLib filter crashes + console.log(`[RAG] Searching for context: "${ragQuery.substring(0, 50)}..."${fileInfo?.name ? ` (Raw search for file: ${fileInfo.name})` : ''}`); + + // Retrieve more candidates if we need to filter by file + const searchK = fileInfo?.name ? 20 : 5; + let results = await vectorStore.similaritySearch(ragQuery, searchK); + + // Post-filtering in memory + if (fileInfo?.name) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + results = results.filter((doc: any) => doc?.metadata?.source === fileInfo.name); + // Take top 5 after filtering + results = results.slice(0, 5); + } if (results.length > 0) { console.log(`[RAG] Found ${results.length} relevant context chunks.`); @@ -200,21 +252,41 @@ Knowledge Base Statistics: // Construct the system prompt with the retrieved context let systemPrompt = ""; + // Check if we are in file-specific mode + const isFileMode = !!fileInfo?.name; + if (useRAG !== false) { - systemPrompt = `You are an intelligent knowledge base assistant. - - ${kbStats} + if (isFileMode) { + // File-specific mode system prompt + systemPrompt = `You are a helpful assistant analyzing a specific file named "${fileInfo.name}". + + Context from the file "${fileInfo.name}": + ${context ? context : "(No relevant content found in this file for the current query)"} + + Instructions: + 1. Answer the user's question ONLY based on the provided context from the file "${fileInfo.name}". + 2. If the context is empty or does not contain the answer, explicitly state that you cannot find the information in the file "${fileInfo.name}". + 3. Do NOT use outside knowledge or information about other files/knowledge bases unless explicitly asked. + 4. Do NOT mention "Knowledge Base Statistics" or other documents. + 5. Provide clear, accurate responses based strictly on the file content. + `; + } else { + // General Knowledge Base mode system prompt + systemPrompt = `You are an intelligent knowledge base assistant. + + ${kbStats} - Context from the knowledge base: - ${context} - - Instructions: - 1. Answer the user's question based on the provided context if relevant. - 2. If the user asks about the knowledge base itself (e.g., how many documents, statistics), use the "Knowledge Base Statistics" provided above. - 3. If the context is empty or not relevant, use your general knowledge to answer the question helpfully. - 4. You can engage in general conversation, creative writing, or coding tasks if requested. - 5. Provide clear, accurate, and friendly responses. - `; + Context from the knowledge base: + ${context} + + Instructions: + 1. Answer the user's question based on the provided context if relevant. + 2. If the user asks about the knowledge base itself (e.g., how many documents, statistics), use the "Knowledge Base Statistics" provided above. + 3. If the context is empty or not relevant, use your general knowledge to answer the question helpfully. + 4. You can engage in general conversation, creative writing, or coding tasks if requested. + 5. Provide clear, accurate, and friendly responses. + `; + } } else { systemPrompt = `You are a helpful AI assistant. @@ -227,6 +299,11 @@ Knowledge Base Statistics: `; } + // Check for quiz generation request + if (query.trim() === '对话试题' || query.trim() === 'Generate Quiz') { + systemPrompt += `\n\nIMPORTANT INSTRUCTION: ${QUIZ_GENERATION_PROMPT}`; + } + // Define available models and fallback strategy // We prioritize the user-selected model, then fall back to others if it fails diff --git a/src/app/api/documents/route.ts b/src/app/api/documents/route.ts index d8658330f897d418d3676b8fea9b2697dd4731b2..30db6f9bfa478f57198d7005f5b1314ce5a7f512 100644 --- a/src/app/api/documents/route.ts +++ b/src/app/api/documents/route.ts @@ -1,7 +1,29 @@ import { NextRequest, NextResponse } from 'next/server'; import db from '@/lib/db'; -import { startYuqueSync, getSyncStatus } from '@/lib/yuque-service'; +import { startYuqueSync, getSyncStatus, stopYuqueSync, backfillNoteTags } from '@/lib/yuque-service'; + +export async function POST() { + try { + const status = getSyncStatus(); + + if (status.status === 'running') { + return NextResponse.json({ message: 'Sync already running', status }); + } + + // Start background sync + startYuqueSync(); + + return NextResponse.json({ + message: 'Sync started', + status: getSyncStatus() + }); + + } catch (error) { + console.error('Failed to start sync:', error); + return NextResponse.json({ error: 'Failed to start sync' }, { status: 500 }); + } +} export async function GET(req: NextRequest) { try { @@ -11,11 +33,60 @@ export async function GET(req: NextRequest) { // Select all columns EXCEPT content_preview to reduce payload size, but include length for stats // Use word_count if available (more accurate), otherwise fallback to 0 (will be updated on next sync) + type DocRow = { + id: string; + yuque_id: number; + title: string; + slug: string; + url: string; + namespace: string; + synced_at: number; + parent_uuid?: string | null; + uuid?: string | null; + sort_order?: number | null; + content_length?: number | null; + updated_at?: number | null; + tags?: string | string[] | null; + }; + const docs = db.prepare(` - SELECT id, yuque_id, title, slug, url, namespace, synced_at, parent_uuid, uuid, sort_order, word_count as content_length, updated_at + SELECT id, yuque_id, title, slug, url, namespace, synced_at, parent_uuid, uuid, sort_order, word_count as content_length, updated_at, tags FROM documents ORDER BY namespace ASC, sort_order ASC, synced_at DESC - `).all(); + `).all() as DocRow[]; + + const normalizedDocs = docs.map((d: DocRow) => { + let tags: string[] = []; + if (typeof d.tags === 'string' && d.tags.length > 0) { + try { + const parsed = JSON.parse(d.tags); + if (Array.isArray(parsed)) { + tags = parsed.map((x: unknown) => { + if (typeof x === 'string') return x; + if (typeof x === 'object' && x !== null) { + const obj = x as Record; + const val = obj.title || obj.name; + return typeof val === 'string' ? val : ''; + } + return ''; + }).filter((x: string) => x.length > 0); + } + } catch { + tags = []; + } + } else if (Array.isArray(d.tags)) { + tags = d.tags.map((x: unknown) => { + if (typeof x === 'string') return x; + if (typeof x === 'object' && x !== null) { + const obj = x as Record; + const val = obj.title || obj.name; + return typeof val === 'string' ? val : ''; + } + return ''; + }).filter((x: string) => x.length > 0); + } + return { ...d, tags }; + }); const kbs = db.prepare(` SELECT * FROM knowledge_bases ORDER BY synced_at DESC @@ -27,7 +98,7 @@ export async function GET(req: NextRequest) { // If database is empty or connection fails, return static demo data for HuggingFace/Demo purposes // We check docs.length === 0 because even if KBs exist (e.g. from partial sync), if there are no documents, // we should show demo data to provide a better initial experience (especially for HF deployments where sync might fail). - if (isDemoMode || docs.length === 0) { + if (isDemoMode) { return NextResponse.json({ documents: [ { @@ -95,8 +166,13 @@ export async function GET(req: NextRequest) { }); } + const notesDocs = normalizedDocs.filter((d) => d.namespace === 'NOTES'); + const hasNoteTags = notesDocs.some((d) => d.tags && d.tags.length > 0); + if (!hasNoteTags && process.env.YUQUE_TOKEN) { + backfillNoteTags(200, 3).catch(() => {}); + } return NextResponse.json({ - documents: docs, + documents: normalizedDocs, knowledgeBases: kbs, status: status, isDemo: false @@ -172,24 +248,24 @@ export async function GET(req: NextRequest) { } } -export async function POST() { +export async function DELETE() { try { const status = getSyncStatus(); - if (status.status === 'running') { - return NextResponse.json({ message: 'Sync already in progress', status }); + if (status.status !== 'running') { + return NextResponse.json({ message: 'Sync not running', status }); } - // Start background sync - startYuqueSync(); + // Stop background sync + stopYuqueSync(); return NextResponse.json({ - message: 'Sync started', + message: 'Sync stop requested', status: getSyncStatus() }); } catch (error) { - console.error('Failed to start sync:', error); - return NextResponse.json({ error: 'Failed to start sync' }, { status: 500 }); + console.error('Failed to stop sync:', error); + return NextResponse.json({ error: 'Failed to stop sync' }, { status: 500 }); } } diff --git a/src/app/api/health/route.ts b/src/app/api/health/route.ts new file mode 100644 index 0000000000000000000000000000000000000000..e1b0c94e0ace1c85c62586f645f4ddbefe7e6b2d --- /dev/null +++ b/src/app/api/health/route.ts @@ -0,0 +1,43 @@ +import { NextResponse } from 'next/server'; +import db from '@/lib/db'; +import { indexExists } from '@/lib/vector-store'; + +export async function GET() { + const startedAt = Date.now(); + try { + const docCountRow = db.prepare('SELECT COUNT(*) as count FROM documents').get() as { count: number }; + const kbCountRow = db.prepare('SELECT COUNT(*) as count FROM knowledge_bases').get() as { count: number }; + + return NextResponse.json({ + ok: true, + db: { + ok: true, + documents: docCountRow.count, + knowledgeBases: kbCountRow.count, + }, + vectorStore: { + ok: indexExists(), + }, + meta: { + durationMs: Date.now() - startedAt, + timestamp: new Date().toISOString(), + uptimeSeconds: Math.floor(process.uptime()), + }, + }); + } catch (error) { + console.error('[Health] Failed:', error); + return NextResponse.json( + { + ok: false, + db: { ok: false }, + vectorStore: { ok: indexExists() }, + meta: { + durationMs: Date.now() - startedAt, + timestamp: new Date().toISOString(), + uptimeSeconds: Math.floor(process.uptime()), + }, + }, + { status: 500 } + ); + } +} diff --git a/src/app/api/history/sessions/route.ts b/src/app/api/history/sessions/route.ts index e90adcf9c18ad04087cbaf9562ee6ce35f0c3990..1f337e8aa91e3650f9cff221e070488bc8fe655d 100644 --- a/src/app/api/history/sessions/route.ts +++ b/src/app/api/history/sessions/route.ts @@ -65,10 +65,10 @@ export async function GET(req: NextRequest) { // POST: Create a new session export async function POST(req: NextRequest) { try { - const { id, title, createdAt } = await req.json(); + const { id, title, createdAt, type = 'chat' } = await req.json(); - const stmt = db.prepare('INSERT INTO sessions (id, title, created_at) VALUES (?, ?, ?)'); - stmt.run(id, title, createdAt); + const stmt = db.prepare('INSERT INTO sessions (id, title, created_at, type) VALUES (?, ?, ?, ?)'); + stmt.run(id, title, createdAt, type); return NextResponse.json({ success: true }); } catch (error) { diff --git a/src/app/api/stats/route.ts b/src/app/api/stats/route.ts new file mode 100644 index 0000000000000000000000000000000000000000..18974eb32a9da6c5918c3d86aa744db318793f7b --- /dev/null +++ b/src/app/api/stats/route.ts @@ -0,0 +1,248 @@ + +import { NextRequest, NextResponse } from 'next/server'; +import db from '@/lib/db'; + +export const dynamic = 'force-dynamic'; + +export async function GET(req: NextRequest) { + try { + const { searchParams } = new URL(req.url); + const yearParam = searchParams.get('year'); + + // 1. Get available years (Union of created and updated years) + const yearsResult = db.prepare(` + SELECT DISTINCT strftime('%Y', datetime(ts / 1000, 'unixepoch', 'localtime')) as year + FROM ( + SELECT updated_at as ts FROM documents WHERE yuque_id != 0 AND updated_at IS NOT NULL + UNION + SELECT created_at as ts FROM documents WHERE yuque_id != 0 AND created_at IS NOT NULL + ) + ORDER BY year DESC + `).all() as { year: string }[]; + + const years = yearsResult.map(y => parseInt(y.year)).filter(y => !isNaN(y)); + + // 2. Prepare filter + let unionTimeFilterClause = ''; + let createdTimeFilterClause = ''; + const paramsUnion: (number | string)[] = []; + const paramsCreated: (number | string)[] = []; + + let startTime = 0; + let endTime = 0; + let isYearFilter = false; + + if (yearParam && yearParam !== 'all') { + const year = parseInt(yearParam); + if (!isNaN(year)) { + startTime = new Date(year, 0, 1).getTime(); + endTime = new Date(year + 1, 0, 1).getTime(); + isYearFilter = true; + + // Union filter for activity heatmap (Created OR Updated) + unionTimeFilterClause = `AND ( + (updated_at >= ? AND updated_at < ?) + OR + (created_at >= ? AND created_at < ?) + )`; + paramsUnion.push(startTime, endTime, startTime, endTime); + // Created-only filter for annual totals + createdTimeFilterClause = `AND (created_at >= ? AND created_at < ?)`; + paramsCreated.push(startTime, endTime); + } + } + + // 3. Get Totals + // Get all-time total documents count for context + // If specific year selected, count accumulated docs up to the end of that year + let allTimeQuery = ` + SELECT COUNT(*) as count FROM documents + WHERE yuque_id != 0 AND (slug IS NULL OR slug NOT LIKE 'dir-%') + `; + const allTimeParams: (number | string)[] = []; + + if (isYearFilter) { + allTimeQuery += ` AND created_at < ?`; + allTimeParams.push(endTime); + } + + const allTimeStats = db.prepare(allTimeQuery).get(...allTimeParams) as { count: number }; + + // 定义:年度字数仅统计「小记」在该年份内创建或更新过的内容的字数之和(去重) + // 这样更贴近“今年写了多少字”的直觉,不把历史长文的字数一次性算入当年。 + let totalStats: { count: number; words: number } = { count: 0, words: 0 }; + let docsStats: { count: number; words: number } = { count: 0, words: 0 }; + let notesStats: { count: number; words: number } = { count: 0, words: 0 }; + if (isYearFilter) { + // 年份视图下,“新增文档数/小记数”遵循“新增”语义:以 created_at 计算 + const docsCreated = db.prepare(` + SELECT COUNT(*) as count, SUM(word_count) as words + FROM documents + WHERE yuque_id != 0 AND namespace != 'NOTES' AND (slug IS NULL OR slug NOT LIKE 'dir-%') ${createdTimeFilterClause} + `).get(...paramsCreated) as { count: number, words: number }; + + // 小记按“创建或更新”的活跃口径,满足“小记数应最多”的运营预期 + // 小记通常短小,且更新代表补充,适合算入年度产出 + const notesActive = db.prepare(` + SELECT COUNT(DISTINCT id) as count, SUM(word_count) as words + FROM documents + WHERE yuque_id != 0 AND namespace = 'NOTES' AND (slug IS NULL OR slug NOT LIKE 'dir-%') ${unionTimeFilterClause} + `).get(...paramsUnion) as { count: number, words: number }; + + docsStats = docsCreated; + notesStats = notesActive; + + // 年度字数: + // - 小记:统计活跃字数(创建或更新) + // - 其他:统计新增字数(仅创建),避免长文修改导致字数统计虚高 + // 注意:这里仍然使用 created_at。因为语雀 API 的 first_published_at 经常与 created_at 非常接近(仅差几秒), + // 无法有效区分“搬运”和“原创”。搬运的文档在语雀系统中确实被视为“在搬运时刻创建”。 + // + // 针对“过往”和“旧码”两个库在 2025 年有大量“新建”记录(实为搬运)的情况, + // 目前最稳妥的逻辑依然是:只算 Created,不算 Updated。 + // 这样至少剔除了 1500 万字的“旧文修改”水分。 + // 剩下的 1400 万字“搬运/整理”数据,客观上确实是 2025 年“进入”语雀系统的, + // 程序无法区分“我 2025 年写的 3 万字”和“我 2025 年搬运进来的 3 万字”。 + totalStats = { + count: (docsStats.count || 0) + (notesStats.count || 0), + words: (docsStats.words || 0) + (notesStats.words || 0), + }; + } else { + // 所有年份视图:总字数为所有文档的字数之和 + const allWords = db.prepare(` + SELECT SUM(word_count) as words FROM documents + WHERE yuque_id != 0 AND (slug IS NULL OR slug NOT LIKE 'dir-%') + `).get() as { words: number }; + const docsAll = db.prepare(` + SELECT COUNT(*) as count, SUM(word_count) as words FROM documents + WHERE yuque_id != 0 AND namespace != 'NOTES' AND (slug IS NULL OR slug NOT LIKE 'dir-%') + `).get() as { count: number, words: number }; + const notesAll = db.prepare(` + SELECT COUNT(*) as count, SUM(word_count) as words FROM documents + WHERE yuque_id != 0 AND namespace = 'NOTES' AND (slug IS NULL OR slug NOT LIKE 'dir-%') + `).get() as { count: number, words: number }; + docsStats = docsAll; + notesStats = notesAll; + totalStats = { + count: (docsStats.count || 0) + (notesStats.count || 0), + words: allWords.words || 0 + }; + } + + // 4. Get Per-KB Stats + // 年份筛选下:小记使用“创建或更新”,其他使用“创建” + const kbStats = isYearFilter + ? (db.prepare(` + SELECT + namespace, + SUM(CASE + WHEN namespace = 'NOTES' THEN 1 + WHEN created_at >= ? AND created_at < ? THEN 1 + ELSE 0 + END) as count, + SUM(CASE + WHEN namespace = 'NOTES' THEN word_count + WHEN created_at >= ? AND created_at < ? THEN word_count + ELSE 0 + END) as words + FROM documents + WHERE yuque_id != 0 + AND (slug IS NULL OR slug NOT LIKE 'dir-%') + ${unionTimeFilterClause} + GROUP BY namespace + `).all(startTime, endTime, startTime, endTime, ...paramsUnion) as { namespace: string, count: number, words: number }[]) + : (db.prepare(` + SELECT + namespace, + COUNT(*) as count, + SUM(word_count) as words + FROM documents + WHERE yuque_id != 0 AND (slug IS NULL OR slug NOT LIKE 'dir-%') + GROUP BY namespace + `).all() as { namespace: string, count: number, words: number }[]); + + // Get KB Names + const kbs = db.prepare('SELECT namespace, name FROM knowledge_bases').all() as { namespace: string, name: string }[]; + const kbNameMap = new Map(kbs.map(k => [k.namespace, k.name])); + + const kbStatsWithNames = kbStats.map(s => ({ + ...s, + name: kbNameMap.get(s.namespace) || s.namespace + })); + + // 5. Get Heatmap Data (Daily counts of Creation AND Update events) + // We want to show activity dots for both creation and updates. + + let heatmapQuery = ''; + const heatmapParams: (number | string)[] = []; + + if (isYearFilter) { + // Filtered by year,排除目录节点 + heatmapQuery = ` + SELECT date, COUNT(*) as count FROM ( + SELECT strftime('%Y-%m-%d', datetime(updated_at / 1000, 'unixepoch', 'localtime')) as date + FROM documents + WHERE yuque_id != 0 AND updated_at >= ? AND updated_at < ? AND (slug IS NULL OR slug NOT LIKE 'dir-%') + + UNION ALL + + SELECT strftime('%Y-%m-%d', datetime(created_at / 1000, 'unixepoch', 'localtime')) as date + FROM documents + WHERE yuque_id != 0 AND created_at >= ? AND created_at < ? AND (slug IS NULL OR slug NOT LIKE 'dir-%') + ) + GROUP BY date + `; + heatmapParams.push(startTime, endTime, startTime, endTime); + } else { + // All years - Union all events,排除目录节点 + heatmapQuery = ` + SELECT date, COUNT(*) as count FROM ( + SELECT strftime('%Y-%m-%d', datetime(updated_at / 1000, 'unixepoch', 'localtime')) as date + FROM documents + WHERE yuque_id != 0 AND updated_at IS NOT NULL AND (slug IS NULL OR slug NOT LIKE 'dir-%') + + UNION ALL + + SELECT strftime('%Y-%m-%d', datetime(created_at / 1000, 'unixepoch', 'localtime')) as date + FROM documents + WHERE yuque_id != 0 AND created_at IS NOT NULL AND (slug IS NULL OR slug NOT LIKE 'dir-%') + ) + GROUP BY date + `; + } + + const heatmapData = db.prepare(heatmapQuery).all(...heatmapParams) as { date: string, count: number }[]; + + // 6. Annual Stats (For the bottom table verification) + // Strictly group by Created At to ensure Sum of Parts == Whole + const annualStats = db.prepare(` + SELECT + strftime('%Y', datetime(COALESCE(created_at, updated_at) / 1000, 'unixepoch', 'localtime')) as year, + COUNT(*) as count, + SUM(CASE WHEN namespace = 'NOTES' THEN 1 ELSE 0 END) as notes_count, + SUM(word_count) as words + FROM documents + WHERE yuque_id != 0 AND (slug IS NULL OR slug NOT LIKE 'dir-%') + GROUP BY year + ORDER BY year DESC + `).all() as { year: string; count: number; notes_count: number; words: number }[]; + + return NextResponse.json({ + years, + stats: { + totalDocs: totalStats.count || 0, + totalWords: totalStats.words || 0, + allTimeDocs: allTimeStats.count, + docsCount: docsStats.count || 0, + notesCount: notesStats.count || 0, + kbStats: kbStatsWithNames, + heatmap: heatmapData, + annualStats // Add this field + } + }); + + } catch (error) { + console.error('Failed to fetch stats:', error); + return NextResponse.json({ error: 'Failed to fetch statistics' }, { status: 500 }); + } +} diff --git a/src/app/knowledge/page.tsx b/src/app/knowledge/page.tsx index 3274d10caef8aa47b391a72825c7303b097ede59..ac608d0111ebd8f75f52ffadb8e24413b1880712 100644 --- a/src/app/knowledge/page.tsx +++ b/src/app/knowledge/page.tsx @@ -3,7 +3,7 @@ import { useState, useEffect, useMemo, Suspense, ReactNode, useRef } from 'react'; import { useLanguage } from '@/contexts/LanguageContext'; -import { ArrowLeft, RefreshCw, Search, Database, ExternalLink, ChevronRight, ChevronDown, Home, Palette, List } from 'lucide-react'; +import { ArrowLeft, RefreshCw, Search, Database, ExternalLink, ChevronRight, ChevronDown, Home, List, Square, ArrowUp, MoreVertical, BarChart2, Save, Tag, X } from 'lucide-react'; import ReactMarkdown from 'react-markdown'; import rehypeRaw from 'rehype-raw'; import remarkGfm from 'remark-gfm'; @@ -129,6 +129,7 @@ interface Document { sort_order?: number; content_length?: number; updated_at?: number; + tags?: string[]; } interface TreeNode { @@ -178,17 +179,26 @@ const TreeNodeView = ({ node, level = 0, onSelect, - selectedUuid + selectedUuid, + variant }: { node: TreeNode, level?: number, onSelect: (node: TreeNode) => void, - selectedUuid?: string + selectedUuid?: string, + variant?: 'sidebar' | 'main' }) => { const [isOpen, setIsOpen] = useState(false); const hasChildren = node.children.length > 0; const isSelected = node.doc.uuid === selectedUuid; + // Helper to format date + const formatDate = (timestamp?: number) => { + if (!timestamp) return ''; + const date = new Date(timestamp); + return `${date.getFullYear()}-${(date.getMonth()+1).toString().padStart(2, '0')}-${date.getDate().toString().padStart(2, '0')} ${date.getHours().toString().padStart(2, '0')}:${date.getMinutes().toString().padStart(2, '0')}`; + }; + // Use refs to track mounted state and last processed selection // This prevents auto-expansion when data refreshes but selection hasn't changed const isMounted = useRef(false); @@ -252,9 +262,18 @@ const TreeNodeView = ({ {isOpen ? : } -
- {node.doc.title} +
+ {node.doc.title}
+ + {variant === 'main' && node.doc.yuque_id !== 0 && ( + <> +
+ + {formatDate(node.doc.updated_at || node.doc.synced_at)} + + + )}
{isOpen && hasChildren && ( @@ -266,6 +285,7 @@ const TreeNodeView = ({ level={level + 1} onSelect={onSelect} selectedUuid={selectedUuid} + variant={variant} /> ))}
@@ -318,7 +338,40 @@ function KnowledgePageContent() { // State initialization flag const [isInitialized, setIsInitialized] = useState(false); const isManualNav = useRef(false); + const mainContentRef = useRef(null); + const tagInputRef = useRef(null); + const [showScrollTop, setShowScrollTop] = useState(false); const [treeVersion, setTreeVersion] = useState(0); + const [visibleCount, setVisibleCount] = useState(30); + const [sidebarVisibleCount, setSidebarVisibleCount] = useState(30); + const [isHoveringSync, setIsHoveringSync] = useState(false); + const [isStatsMenuOpen, setIsStatsMenuOpen] = useState(false); + const [isBackingUp, setIsBackingUp] = useState(false); + + const handleBackup = async () => { + if (isBackingUp) return; + setIsBackingUp(true); + try { + const res = await fetch('/api/backup', { method: 'POST' }); + const data = await res.json(); + if (res.ok) { + alert(`导出 JSON 成功!\n文件已保存到「备份-语雀数据-JSON」文件夹:\n${data.files.join('\n')}`); + } else { + alert(`导出失败: ${data.message}`); + } + } catch (error) { + console.error('Export failed:', error); + alert('导出请求失败,请检查控制台。'); + } finally { + setIsBackingUp(false); + } + }; + + useEffect(() => { + setVisibleCount(30); + setSidebarVisibleCount(30); + }, [currentKbNamespace]); + const handleCollapseAll = (e: React.MouseEvent) => { e.stopPropagation(); @@ -340,17 +393,22 @@ function KnowledgePageContent() { setIsDemoMode(data.isDemo); } - // Initial KB selection logic - if (data.knowledgeBases && data.knowledgeBases.length > 0) { - // Check URL first - const kbParam = searchParams.get('kb'); - // Verify if the kbParam actually exists in the fetched knowledge bases - const kbExists = kbParam && data.knowledgeBases.some((k: KnowledgeBase) => k.namespace === kbParam); - - if (kbExists) { - if (!currentKbNamespace) setCurrentKbNamespace(kbParam); - } else { - // If no KB selected, or selected KB doesn't exist (e.g. switching modes), select the first one + // Knowledge base selection logic + // Stabilize current selection during sync or after initialization to prevent flicker + if (!isInitialized) { + if (data.knowledgeBases && data.knowledgeBases.length > 0) { + const kbParam = searchParams.get('kb'); + const kbExists = kbParam && data.knowledgeBases.some((k: KnowledgeBase) => k.namespace === kbParam); + if (kbExists) { + if (!currentKbNamespace) setCurrentKbNamespace(kbParam); + } else if (!currentKbNamespace) { + setCurrentKbNamespace(data.knowledgeBases[0].namespace); + } + } + } else { + // Do not auto-switch KB while syncing even if current is temporarily missing + // Only set default if nothing is selected and we have KBs + if (!currentKbNamespace && data.knowledgeBases && data.knowledgeBases.length > 0) { setCurrentKbNamespace(data.knowledgeBases[0].namespace); } } @@ -368,6 +426,65 @@ function KnowledgePageContent() { const treeRoots = useMemo(() => buildTree(filteredDocuments), [filteredDocuments]); + // Tag filter state + const [selectedTag, setSelectedTag] = useState(''); + const [isTagDropdownOpen, setIsTagDropdownOpen] = useState(false); + const [isInputFocused, setIsInputFocused] = useState(false); + const [tagSearchTerm, setTagSearchTerm] = useState(''); + + const allTags = useMemo(() => { + if (!documents || documents.length === 0) return []; + + const tagMap = new Map(); + let untaggedCount = 0; + + // Use filteredDocuments to only show tags relevant to current KB + filteredDocuments.forEach(doc => { + if (doc.tags && Array.isArray(doc.tags) && doc.tags.length > 0) { + doc.tags.forEach(t => { + if (t) tagMap.set(t, (tagMap.get(t) || 0) + 1); + }); + } else { + untaggedCount++; + } + }); + + const tagsList = Array.from(tagMap.entries()) + .map(([name, count]) => ({ name, count })); + + if (untaggedCount > 0) { + tagsList.push({ name: '无标签', count: untaggedCount }); + } + + return tagsList.sort((a, b) => b.count - a.count); + }, [filteredDocuments, documents]); + + const mainAreaRoots = useMemo(() => { + if (!selectedTag) return treeRoots; + + let taggedDocs; + if (selectedTag === '无标签') { + taggedDocs = filteredDocuments.filter(doc => + !doc.tags || !Array.isArray(doc.tags) || doc.tags.length === 0 + ); + } else { + taggedDocs = filteredDocuments.filter(doc => + doc.tags && Array.isArray(doc.tags) && doc.tags.includes(selectedTag) + ); + } + + // Rebuild tree for the filtered view + // Note: If a child matches but parent doesn't, it becomes a root in this new tree + return buildTree(taggedDocs); + }, [treeRoots, filteredDocuments, selectedTag]); + + useEffect(() => { + setSelectedTag(''); + setIsTagDropdownOpen(false); + setTagSearchTerm(''); + }, [currentKbNamespace]); + + // Restore state from URL on load and when documents are ready useEffect(() => { if (documents.length > 0 && !isInitialized) { @@ -480,6 +597,16 @@ function KnowledgePageContent() { return () => clearInterval(interval); }, [syncStatus.status, searchParams.get('demo')]); + const handleStopSync = async () => { + try { + await fetch('/api/documents', { method: 'DELETE' }); + // Trigger immediate refresh to get updated status + fetchDocuments(); + } catch (error) { + console.error('Failed to stop sync:', error); + } + }; + const handleSync = async () => { try { await fetch('/api/documents', { method: 'POST' }); @@ -517,28 +644,58 @@ function KnowledgePageContent() { )} + {/* Sync Progress Bar (Slim) and Status Message */} - {syncStatus.status === 'running' && ( + {(syncStatus.status === 'running' || syncStatus.message?.includes('停止') || syncStatus.status === 'error') && (
-
-
+
+
- - + + {syncStatus.message || '正在同步...'} | @@ -549,7 +706,7 @@ function KnowledgePageContent() {
@@ -561,7 +718,7 @@ function KnowledgePageContent() { {/* Main Content Area */}
{/* Sidebar */} -
+
{/* Knowledge Base Switcher */}
@@ -578,7 +735,40 @@ function KnowledgePageContent() { }} > -
+
+ +
+
{ + e.stopPropagation(); + setIsStatsMenuOpen(!isStatsMenuOpen); + }} + > + +
+ {isStatsMenuOpen && ( + <> +
{ + e.stopPropagation(); + setIsStatsMenuOpen(false); + }} + /> +
+ setIsStatsMenuOpen(false)} + > + + 统计 + +
+ + )} +
{isKbDropdownOpen && ( @@ -655,12 +845,22 @@ function KnowledgePageContent() { {t('outline')}
-
+
{ + const { scrollTop, scrollHeight, clientHeight } = e.currentTarget; + if (scrollHeight - scrollTop - clientHeight < 100) { + if (sidebarVisibleCount < treeRoots.length) { + setSidebarVisibleCount(prev => Math.min(prev + 30, treeRoots.length)); + } + } + }} + > {isLoading ? (
{t('loading')}
) : (
- {treeRoots.map(node => ( + {(currentKbNamespace === 'NOTES' ? treeRoots.slice(0, sidebarVisibleCount) : treeRoots).map(node => ( ))} + {currentKbNamespace === 'NOTES' && sidebarVisibleCount < treeRoots.length && ( +
+ ... +
+ )}
)}
{/* Right Panel */} -
+
{ + const { scrollTop, scrollHeight, clientHeight } = e.currentTarget; + + // Toggle Back to Top button + if (scrollTop > 300) { + if (!showScrollTop) setShowScrollTop(true); + } else { + if (showScrollTop) setShowScrollTop(false); + } + + if (scrollHeight - scrollTop - clientHeight < 100) { + const targetRoots = currentKbNamespace === 'NOTES' ? mainAreaRoots : treeRoots; + if (visibleCount < targetRoots.length) { + setVisibleCount(prev => Math.min(prev + 30, targetRoots.length)); + } + } + }} + > + {searchTerm ? (

搜索结果: "{searchTerm}"

@@ -715,13 +941,15 @@ function KnowledgePageContent() {
{/* Breadcrumb / Header */}
-
-
-

- {selectedNode.doc.title} -

+ {selectedNode.doc.namespace !== 'NOTES' && ( +
+
+

+ {selectedNode.doc.title} +

+
-
+ )} {/* Content */} {isLoadingContent ? ( @@ -750,6 +978,19 @@ function KnowledgePageContent() { 语雀链接 )} + {Array.isArray(selectedNode.doc.tags) && selectedNode.doc.tags.length > 0 && ( +
+ + {selectedNode.doc.tags.map(tag => ( + + #{tag} + + ))} +
+ )}
) : ( @@ -796,12 +1037,125 @@ function KnowledgePageContent() {
) : ( /* Home View */ -
+
-

- {knowledgeBases.find(kb => kb.namespace === currentKbNamespace)?.name || '知识库'} -

-
+
+

+ {knowledgeBases.find(kb => kb.namespace === currentKbNamespace)?.name || '知识库'} +

+ {currentKbNamespace === 'NOTES' && ( +
+ + 0 ? "输入筛选标签..." : "暂无标签"} + value={tagSearchTerm} + onChange={(e) => { + setTagSearchTerm(e.target.value); + setIsTagDropdownOpen(true); + }} + onFocus={() => { + setIsTagDropdownOpen(true); + setIsInputFocused(true); + }} + onBlur={() => setIsInputFocused(false)} + onClick={(e) => { + e.stopPropagation(); + setIsTagDropdownOpen(true); + }} + disabled={allTags.length === 0} + /> +
e.preventDefault()} + onClick={(e) => { + e.stopPropagation(); + if (tagSearchTerm) { + setTagSearchTerm(''); + setSelectedTag(''); + setIsTagDropdownOpen(true); + tagInputRef.current?.focus(); + } else { + if (isTagDropdownOpen) { + setIsTagDropdownOpen(false); + } else { + setIsTagDropdownOpen(true); + tagInputRef.current?.focus(); + } + } + }} + > + {tagSearchTerm ? ( + + ) : ( + + )} +
+ + {isTagDropdownOpen && ( + <> +
{ + e.stopPropagation(); + setIsTagDropdownOpen(false); + setTagSearchTerm(selectedTag || ''); + }} + /> +
+ {(!tagSearchTerm || '全部标签'.includes(tagSearchTerm)) && ( + + )} + {allTags + .filter(tag => tag.name.toLowerCase().includes(tagSearchTerm.toLowerCase())) + .map(tag => ( + + ))} + {allTags.filter(tag => tag.name.toLowerCase().includes(tagSearchTerm.toLowerCase())).length === 0 && !('全部标签'.includes(tagSearchTerm)) && ( +
+ 未找到相关标签 +
+ )} +
+ + )} +
+ )} +
+
{filteredDocuments.filter(d => d.yuque_id !== 0).length} {t('documents')} | {(filteredDocuments.filter(d => d.yuque_id !== 0).reduce((acc, d) => acc + (d.content_length || 0), 0) / 10000).toFixed(1)} {language === 'zh' ? '万字' : '0k chars'} @@ -809,16 +1163,31 @@ function KnowledgePageContent() {
- {treeRoots.map(node => ( + {(currentKbNamespace === 'NOTES' ? mainAreaRoots.slice(0, visibleCount) : mainAreaRoots).map(node => ( ))} + {currentKbNamespace === 'NOTES' && visibleCount < mainAreaRoots.length && ( +
+ 加载更多... +
+ )}
)} + + {/* Back to Top Button */} +
diff --git a/src/app/knowledge/stats/page.tsx b/src/app/knowledge/stats/page.tsx new file mode 100644 index 0000000000000000000000000000000000000000..485677a699dcfb0c46a688c3445eff6a4685678b --- /dev/null +++ b/src/app/knowledge/stats/page.tsx @@ -0,0 +1,1104 @@ + +"use client"; + +import { useState, useEffect, useRef, useCallback } from 'react'; +import { ArrowLeft, Calendar, FileText, Type, BarChart2, X, Sparkles, Share2, Download, Copy } from 'lucide-react'; +import { useRouter } from 'next/navigation'; + +interface HeatmapData { + date: string; + count: number; +} + +interface KbStat { + namespace: string; + name: string; + count: number; + words: number; +} + +interface StatsData { + years: number[]; + stats: { + totalDocs: number; + totalWords: number; + allTimeDocs?: number; + docsCount?: number; + notesCount?: number; + kbStats: KbStat[]; + heatmap: HeatmapData[]; + annualStats?: { year: string; count: number; notes_count: number; words: number }[]; + }; +} + +const AnnualReportModal = ({ + isOpen, + onClose, + year, + data +}: { + isOpen: boolean; + onClose: () => void; + year: number | 'all'; + data: StatsData['stats']; +}) => { + const [sharePreviewOpen, setSharePreviewOpen] = useState(false); + const [shareImageUrl, setShareImageUrl] = useState(null); + const [generating, setGenerating] = useState(false); + const [copyStatus, setCopyStatus] = useState<'idle' | 'success' | 'error'>('idle'); + + // Calculate insights + const totalDocs = data.totalDocs; + const totalWords = data.totalWords; + + // Sort heatmap data by date + const sortedHeatmap = [...data.heatmap].sort((a, b) => new Date(a.date).getTime() - new Date(b.date).getTime()); + + // 1. Active Days + const activeDays = sortedHeatmap.length; + const isLeapYear = (y: number) => (y % 4 === 0 && y % 100 !== 0) || (y % 400 === 0); + const daysInYear = year === 'all' ? 365 : (typeof year === 'number' && isLeapYear(year) ? 366 : 365); + const isFullAttendance = year !== 'all' && activeDays >= daysInYear; + + // 3. Day of Week (removed cards below no longer use this) + + const displayYear = year === 'all' ? '全部年份' : `${year}年`; + if (!isOpen) return null; + + const generateSvg = () => { + const w = 1080; + const topKb = [...data.kbStats] + .sort((a, b) => b.words - a.words) + .slice(0, 6); + const totalWordsW = (totalWords / 10000).toFixed(1) + '万'; + const headerTitle = `${displayYear} 年度报告`; + const subTitle = '知识库年度回顾'; + const activeDaysText = `${activeDays} 天`; + const totalDocsText = String(totalDocs); + const bg = ` + + + + + + + + + + + `; + const kbBars = topKb.map((kb, i) => { + const barMax = 740; + const percent = totalWords > 0 ? Math.max(2, Math.round((kb.words / totalWords) * barMax)) : 2; + const y = 820 + i * 90; + const name = kb.name.replace(/&/g, '&').replace(/ + ${name} + + + ${words} + + `; + }).join(''); + const lastBarY = 820 + Math.max(0, topKb.length - 1) * 90 + 32; + const h = lastBarY + 160; + const footerText = 'RAG Knowledge Base'; + const svg = ` + + ${bg} + + + ${headerTitle} + ${subTitle} + + + + 年度活跃天数 + ${activeDaysText} + ${isFullAttendance ? `全勤达成` : ''} + + + + + ${totalDocsText} + 总文档数 + + + + + ${totalWordsW} + 总字数 + + + 知识库贡献 + ${kbBars} + ${footerText} + + `; + return { svg, w, h }; + }; + + const svgToPngDataUrl = async (svg: string, w: number, h: number) => { + return new Promise((resolve, reject) => { + try { + const blob = new Blob([svg], { type: 'image/svg+xml;charset=utf-8' }); + const url = URL.createObjectURL(blob); + const img = new Image(); + img.onload = () => { + const canvas = document.createElement('canvas'); + const scale = 2; + canvas.width = w * scale; + canvas.height = h * scale; + const ctx = canvas.getContext('2d'); + if (!ctx) { + URL.revokeObjectURL(url); + reject(new Error('Canvas not supported')); + return; + } + ctx.scale(scale, scale); + ctx.fillStyle = '#ffffff'; + ctx.fillRect(0, 0, w, h); + ctx.drawImage(img, 0, 0, w, h); + URL.revokeObjectURL(url); + resolve(canvas.toDataURL('image/png')); + }; + img.onerror = () => { + URL.revokeObjectURL(url); + reject(new Error('Failed to load SVG')); + }; + img.src = url; + } catch (e) { + reject(e as Error); + } + }); + }; + + const handleGenerateShare = async () => { + if (generating) return; + setGenerating(true); + setCopyStatus('idle'); + try { + const { svg, w, h } = generateSvg(); + const url = await svgToPngDataUrl(svg, w, h); + setShareImageUrl(url); + setSharePreviewOpen(true); + } finally { + setGenerating(false); + } + }; + + const handleDownload = () => { + if (!shareImageUrl) return; + const a = document.createElement('a'); + a.href = shareImageUrl; + const filename = `年度报告_${displayYear.replace('年', '')}.png`; + a.download = filename; + document.body.appendChild(a); + a.click(); + document.body.removeChild(a); + }; + + const handleCopy = async () => { + if (!shareImageUrl) return; + setCopyStatus('idle'); + try { + const res = await fetch(shareImageUrl); + const blob = await res.blob(); + const item = new ClipboardItem({ [blob.type]: blob }); + await navigator.clipboard.write([item]); + setCopyStatus('success'); + } catch { + setCopyStatus('error'); + } + }; + + return ( +
+
+ {/* Header with gradient */} +
+
+ +

{displayYear} 年度报告

+

您的知识库回顾

+ + +
+ +
+
+ +
+ {/* Active Days Highlight */} +
+
年度活跃天数
+
+ {activeDays} + +
+ {isFullAttendance && ( +
+ 🏆 全勤达成!太强了! +
+ )} + {!isFullAttendance && year !== 'all' && ( +
+ 距离全勤还差 {daysInYear - activeDays} 天 +
+ )} +
+ + {/* Insights Grid */} +
+ + {/* Top Month */} +
+
{totalDocs}
+
总文档数
+
+ + + + {/* Total Words */} +
+
{(totalWords / 10000).toFixed(1)}w
+
总字数
+
+
+ + {/* Detailed Stats */} +
+ +
+
+ 知识库 +
+ 文档 + 字数 +
+
+
+ {data.kbStats.map((kb) => ( +
+ {kb.name} +
+ {kb.count} + {(kb.words / 10000).toFixed(1)}w +
+
+ ))} +
+
+
+ + +
+
+ {sharePreviewOpen && shareImageUrl && ( +
+
+
+
+ + 分享图片预览 +
+ +
+
+ 年度报告分享图片 +
+
+ + + {copyStatus === 'success' && 已复制} + {copyStatus === 'error' && 复制失败,请下载保存} +
+
+
+ )} +
+ ); +}; + +// Custom Icon to avoid import error if lucide-react version mismatch +const DatabaseIcon = ({ className }: { className?: string }) => ( + +); + +const Heatmap = ({ data, year, range }: { data: HeatmapData[]; year?: number; range?: { start: Date; end: Date } }) => { + const containerRef = useRef(null); + const [hoverInfo, setHoverInfo] = useState<{ date: string; count: number; x: number; y: number; cw: number } | null>(null); + const tipRef = useRef(null); + const [tipW, setTipW] = useState(120); + useEffect(() => { + if (tipRef.current) { + const w = tipRef.current.offsetWidth; + if (w && Math.abs(w - tipW) > 2) setTipW(w); + } + }, [hoverInfo, tipW]); + // Determine start and end dates + let startDate: Date; + let endDate: Date; + + if (range) { + startDate = new Date(range.start); + endDate = new Date(range.end); + } else { + const targetYear = year || new Date().getFullYear(); + startDate = new Date(targetYear, 0, 1); + endDate = new Date(targetYear, 11, 31); + } + + // Create map for fast lookup + const dataMap = new Map(data.map(d => [d.date, d.count])); + + // Helper to get color + const getColor = (count: number) => { + if (count === 0) return 'bg-gray-100 dark:bg-gray-800'; + if (count <= 2) return 'bg-green-100 dark:bg-green-900/40'; + if (count <= 5) return 'bg-green-300 dark:bg-green-800/60'; + if (count <= 10) return 'bg-green-500 dark:bg-green-700'; + return 'bg-green-700 dark:bg-green-600'; + }; + + // Build weeks array + const weeks = []; + const currentDate = new Date(startDate); + + while (currentDate.getDay() !== 1) { + currentDate.setDate(currentDate.getDate() - 1); + } + + // Helper to check if date is within target range + const isDateWithinRange = (d: Date) => { + // Reset time part for comparison + const checkDate = new Date(d); + checkDate.setHours(0, 0, 0, 0); + const s = new Date(startDate); + s.setHours(0, 0, 0, 0); + const e = new Date(endDate); + e.setHours(0, 0, 0, 0); + return checkDate >= s && checkDate <= e; + }; + + // Loop until we cover the end date + // We also add a safety break to avoid infinite loops + let safetyCounter = 0; + while (true) { + const week = []; + for (let i = 0; i < 7; i++) { + const dateStr = currentDate.toISOString().split('T')[0]; + const isWithin = isDateWithinRange(currentDate); + + week.push({ + date: dateStr, + count: isWithin ? (dataMap.get(dateStr) || 0) : -1, + isWithinRange: isWithin + }); + currentDate.setDate(currentDate.getDate() + 1); + } + weeks.push(week); + + // Break if the start of the next week is beyond endDate + if (currentDate > endDate) break; + + // Safety break (approx 2 years) + if (safetyCounter++ > 110) break; + } + + // Limit to 53 weeks max to fit layout if not custom range + const displayWeeks = weeks.slice(0, 54); + + // Generate month labels aligned with weeks + const monthLabels = displayWeeks.map((week) => { + const firstDayOfMonth = week.find(d => d.date.endsWith('-01')); + if (firstDayOfMonth && firstDayOfMonth.isWithinRange) { + const month = parseInt(firstDayOfMonth.date.slice(5, 7)); + return `${month}月`; + } + return null; + }); + + return ( +
+
+
+ {displayWeeks.map((week, wIdx) => ( +
+ {week.map((day, dIdx) => ( +
{ + const rect = (e.currentTarget as HTMLDivElement).getBoundingClientRect(); + const contRect = containerRef.current?.getBoundingClientRect(); + if (!contRect) return; + const x = rect.left - contRect.left + rect.width / 2; + const y = rect.top - contRect.top - 8; + setHoverInfo({ + date: day.date, + count: Math.max(0, day.count), + x, + y, + cw: contRect.width + }); + }} + onMouseLeave={() => setHoverInfo(null)} + /> + ))} +
+ ))} +
+
+ {monthLabels.map((label, idx) => ( +
+ {label && ( + + {label} + + )} +
+ ))} +
+ {hoverInfo && ( +
+
{hoverInfo.date}
+
创建/更新 {hoverInfo.count}
+
+ )} +
+
+ ); +}; + +export default function StatsPage() { + const router = useRouter(); + const [statsData, setStatsData] = useState(null); + const [loading, setLoading] = useState(true); + const [selectedYear, setSelectedYear] = useState('all'); + const [displayYear, setDisplayYear] = useState('all'); + const [debouncedYear, setDebouncedYear] = useState('all'); + const [isRefreshing, setIsRefreshing] = useState(false); + const [errorMessage, setErrorMessage] = useState(null); + const [forceRefresh, setForceRefresh] = useState(false); + const [showReport, setShowReport] = useState(false); + const [showAllYears, setShowAllYears] = useState(false); + + const cacheRef = useRef(new Map()); + const inflightRef = useRef>(new Map()); + const scrollYRef = useRef(null); + + useEffect(() => { + const handle = setTimeout(() => setDebouncedYear(selectedYear), 160); + return () => clearTimeout(handle); + }, [selectedYear]); + + const fetchStatsFromApi = useCallback(async (year: number | 'all', signal: AbortSignal) => { + const res = await fetch(`/api/stats?year=${year}&t=${Date.now()}`, { + cache: 'no-store', + headers: { + 'Pragma': 'no-cache', + 'Cache-Control': 'no-cache, no-store, must-revalidate' + }, + signal + }); + if (!res.ok) { + throw new Error(`HTTP ${res.status}`); + } + return (await res.json()) as StatsData; + }, []); + + const fetchWithRetry = useCallback(async (year: number | 'all', signal: AbortSignal) => { + const maxRetries = 2; + const baseDelayMs = 300; + + let lastError: unknown = null; + for (let attempt = 0; attempt <= maxRetries; attempt++) { + if (signal.aborted) throw new DOMException('Aborted', 'AbortError'); + try { + return await fetchStatsFromApi(year, signal); + } catch (err) { + lastError = err; + if (signal.aborted) throw err; + if (attempt === maxRetries) break; + const delayMs = baseDelayMs * Math.pow(2, attempt); + await new Promise((resolve) => { + const t = setTimeout(resolve, delayMs); + signal.addEventListener('abort', () => { + clearTimeout(t); + resolve(); + }, { once: true }); + }); + } + } + throw lastError; + }, [fetchStatsFromApi]); + + const applyDataForYear = useCallback((year: number | 'all', data: StatsData) => { + setStatsData(data); + setDisplayYear(year); + }, []); + + const loadYear = useCallback(async (year: number | 'all', options?: { forceRefresh?: boolean }) => { + const forceRefresh = options?.forceRefresh ?? false; + const key = String(year); + const now = Date.now(); + const ttlMs = 5 * 60 * 1000; + + const cached = cacheRef.current.get(key); + const isFresh = cached ? (now - cached.ts) < ttlMs : false; + + if (cached && !forceRefresh) { + applyDataForYear(year, cached.data); + setLoading(false); + setErrorMessage(null); + if (isFresh) return; + } + + inflightRef.current.forEach((controller) => controller.abort()); + inflightRef.current.clear(); + + const controller = new AbortController(); + inflightRef.current.set(key, controller); + + if (typeof window !== 'undefined') { + scrollYRef.current = window.scrollY; + } + + setIsRefreshing(true); + setErrorMessage(null); + if (!statsData && !cached) setLoading(true); + + try { + const data = await fetchWithRetry(year, controller.signal); + if (controller.signal.aborted) return; + cacheRef.current.set(key, { data, ts: Date.now() }); + applyDataForYear(year, data); + } catch (err) { + if ((err as { name?: string } | null)?.name === 'AbortError') return; + setErrorMessage('加载失败'); + } finally { + inflightRef.current.delete(key); + setIsRefreshing(false); + setLoading(false); + if (typeof window !== 'undefined' && scrollYRef.current !== null) { + const y = scrollYRef.current; + scrollYRef.current = null; + requestAnimationFrame(() => window.scrollTo({ top: y })); + } + } + }, [applyDataForYear, fetchWithRetry, statsData]); + + const prefetchYear = useCallback(async (year: number) => { + const key = String(year); + const now = Date.now(); + const ttlMs = 5 * 60 * 1000; + + const cached = cacheRef.current.get(key); + if (cached && (now - cached.ts) < ttlMs) return; + if (inflightRef.current.has(key)) return; + + const controller = new AbortController(); + inflightRef.current.set(key, controller); + try { + const data = await fetchWithRetry(year, controller.signal); + if (controller.signal.aborted) return; + cacheRef.current.set(key, { data, ts: Date.now() }); + } catch (err) { + if ((err as { name?: string } | null)?.name === 'AbortError') return; + } finally { + inflightRef.current.delete(key); + } + }, [fetchWithRetry]); + + useEffect(() => { + const inflightMap = inflightRef.current; + let disposed = false; + + void (async () => { + await loadYear(debouncedYear, { forceRefresh }); + if (!disposed && forceRefresh) setForceRefresh(false); + })(); + + return () => { + disposed = true; + inflightMap.forEach((controller) => controller.abort()); + inflightMap.clear(); + }; + }, [debouncedYear, forceRefresh, loadYear]); + + useEffect(() => { + if (!statsData?.years?.length) return; + if (displayYear === 'all') { + const nowYear = new Date().getFullYear(); + const candidates = statsData.years.filter((y) => y !== nowYear).slice(0, 1); + void prefetchYear(nowYear); + if (candidates[0]) void prefetchYear(candidates[0]); + return; + } + if (typeof displayYear !== 'number') return; + const years = statsData.years; + const idx = years.indexOf(displayYear); + if (idx > 0) void prefetchYear(years[idx - 1]); + if (idx >= 0 && idx < years.length - 1) void prefetchYear(years[idx + 1]); + }, [displayYear, prefetchYear, statsData?.years, statsData?.years?.length]); + + const heatmapProps = displayYear === 'all' + ? { + range: { + start: new Date(new Date().setFullYear(new Date().getFullYear() - 1)), + end: new Date() + } + } + : { year: typeof displayYear === 'number' ? displayYear : new Date().getFullYear() }; + + const heatmapTitle = displayYear === 'all' + ? "活跃度 (近一年)" + : `活跃度 (${displayYear}年)`; + + const formatWords = (words: number | undefined) => { + if (typeof words !== 'number') return '-'; + return (words / 10000).toFixed(1); + }; + + if (loading && !statsData) { + return ( +
+
+
+ ); + } + + if (errorMessage && !statsData) { + return ( +
+
{errorMessage}
+ +
+ ); + } + + if (!statsData) return null; + + return ( +
+ {/* Header */} +
+
+
+
+ +

+ + 知识库统计 +

+
+ +
+
+ + {isRefreshing && ( +
+ )} +
+ {errorMessage && ( + + )} + + +
+
+
+
+ +
+
+ <> + {/* Summary Cards */} +
+
+
+ +
+
+

新增文档数

+

{statsData?.stats.docsCount ?? statsData?.stats.totalDocs}

+
+
+ +
+
+ +
+
+

{displayYear === 'all' ? '总字数' : '年度字数(创建或更新)'}

+

{formatWords(statsData?.stats.totalWords)} 万字

+
+
+ +
+
+ +
+
+

新增小记数

+

{statsData?.stats.notesCount ?? 0}

+
+
+ +
+
+ +
+
+

{displayYear === 'all' ? '总文档数' : '年度总文档数'}

+

{statsData?.stats.totalDocs ?? 0}

+
+
+
+ + {/* Heatmap Card */} +
+
+

+ + {heatmapTitle} +

+
+ +
+
+
+
+
+
+
+ +
+
+ {statsData?.stats.heatmap && ( + + )} +
+ + {/* KB Breakdown */} +
+
+

知识库明细

+
+
+ {(() => { + const totalDocs = statsData?.stats.totalDocs || 1; + const totalWords = statsData?.stats.totalWords || 1; + + return ( + + + + + + + + + + + + {statsData?.stats.kbStats.map((kb) => { + const docPercent = Math.round((kb.count / totalDocs) * 100); + const wordPercent = Math.round((kb.words / totalWords) * 100); + + return ( + + + + + + + + ); + })} + +
知识库名称文档数字数占比 (文档)占比 (字数)
{kb.name}{kb.count}{(kb.words / 10000).toFixed(1)}万 +
+ {docPercent}% +
+
+
+
+
+
+ {wordPercent}% +
+
+
+
+
+ ); + })()} +
+
+ + {/* Annual Stats Table (History) */} + {displayYear === 'all' && statsData?.stats.annualStats && ( +
+
+

+ + 历年新增统计 +

+ +
+
+ + + + + + + + + + + + + + + + {(() => { + const annualStats = statsData.stats.annualStats || []; + const totalCount = annualStats.reduce((sum, s) => sum + s.count, 0); + const totalNotes = annualStats.reduce((sum, s) => sum + (s.notes_count || 0), 0); + const totalPureDocs = totalCount - totalNotes; + const totalWords = annualStats.reduce((sum, s) => sum + s.words, 0); + const displayStats = showAllYears ? annualStats : annualStats.slice(0, 10); + + return ( + <> + {/* Total Row */} + + + + + + + + + + + + {displayStats.map((stat) => { + const pureDocsCount = stat.count - (stat.notes_count || 0); + const countPercent = totalPureDocs > 0 ? (pureDocsCount / totalPureDocs) * 100 : 0; + const notePercent = totalNotes > 0 ? ((stat.notes_count || 0) / totalNotes) * 100 : 0; + const wordPercent = totalWords > 0 ? (stat.words / totalWords) * 100 : 0; + const docTotalPercent = totalCount > 0 ? (stat.count / totalCount) * 100 : 0; + + return ( + + + + + + + + + + + + ); + })} + + ); + })()} + +
年份文档数 +
+ 占比 +
+
+
+
+
小记数 +
+ 占比 +
+
+
+
+
总文档数 +
+ 占比 +
+
+
+
+
总字数 +
+ 占比 +
+
+
+
+
总计 + {totalPureDocs} + +
+ 100% +
+
+
+
+
+ {totalNotes} + +
+ 100% +
+
+
+
+
+ {totalCount} + +
+ 100% +
+
+
+
+
+ {(totalWords / 10000).toFixed(1)}w + +
+ 100% +
+
+
+
+
{stat.year}{pureDocsCount} +
+ {countPercent.toFixed(1)}% +
+
+
+
+
{stat.notes_count || 0} +
+ {notePercent.toFixed(1)}% +
+
+
+
+
{stat.count} +
+ {docTotalPercent.toFixed(1)}% +
+
+
+
+
{(stat.words / 10000).toFixed(1)}w +
+ {wordPercent.toFixed(1)}% +
+
+
+
+
+
+
+ )} + +
+
+ + setShowReport(false)} + year={displayYear} + data={statsData?.stats || { totalDocs: 0, totalWords: 0, kbStats: [], heatmap: [] }} + /> +
+ ); +} diff --git a/src/app/login/page.tsx b/src/app/login/page.tsx new file mode 100644 index 0000000000000000000000000000000000000000..6b23ad778259b0394fa23c09fdfccfbdb78ab754 --- /dev/null +++ b/src/app/login/page.tsx @@ -0,0 +1,214 @@ +'use client' + +import { useState, useRef, useEffect } from 'react' +import { useRouter } from 'next/navigation' +import { ChevronDown, Lock, User, X } from 'lucide-react' + +export default function LoginPage() { + const router = useRouter() + const [username, setUsername] = useState('duqing') + const [password, setPassword] = useState('') + const [isDropdownOpen, setIsDropdownOpen] = useState(false) + const [isInputFocused, setIsInputFocused] = useState(false) + const [error, setError] = useState('') + const dropdownRef = useRef(null) + const inputRef = useRef(null) + + const users = ['duqing', 'admin', 'guest'] + + useEffect(() => { + function handleClickOutside(event: MouseEvent) { + if (dropdownRef.current && !dropdownRef.current.contains(event.target as Node)) { + setIsDropdownOpen(false) + } + } + document.addEventListener("mousedown", handleClickOutside) + return () => { + document.removeEventListener("mousedown", handleClickOutside) + } + }, []) + + const handleLogin = async (e: React.FormEvent) => { + e.preventDefault() + setError('') + + // Validation + // Check for length 8 + if (password.length !== 8) { + setError('密码长度必须为8位') + return + } + + // Check characters (Chinese, English, Numbers) + if (!/^[\u4e00-\u9fa5a-zA-Z0-9]+$/.test(password)) { + setError('密码只能包含中英文或数字') + return + } + + // Check for "correct" credentials + // Correct password for "duqing" is "1234qwer" + if (username === 'duqing' && password === '1234qwer') { + // Set cookie + document.cookie = `auth_token=valid_token; path=/; max-age=${60 * 60 * 24}` // 1 day + // Clear last session ID to ensure we start with a new chat + localStorage.removeItem('rag_kb_current_session_id'); + router.push('/') + } else { + setError('账号或密码错误') + } + } + + const handleSelectUser = (user: string) => { + setUsername(user) + setIsDropdownOpen(false) + } + + return ( +
+
+
+

+ RAG 知识库系统 +

+

+ 欢迎回来 +

+

+ 请登录您的账户以继续 +

+
+ +
+
+ + {/* Username Field with Custom Select */} +
+ +
+
+ +
+ { + setUsername(e.target.value) + setIsDropdownOpen(true) + }} + onFocus={() => { + setIsDropdownOpen(true) + setIsInputFocused(true) + }} + onBlur={() => setIsInputFocused(false)} + autoComplete="off" + /> +
e.preventDefault()} + onClick={(e) => { + e.stopPropagation() + if (isInputFocused && username) { + setUsername('') + setIsDropdownOpen(true) + inputRef.current?.focus() + } else { + setIsDropdownOpen(!isDropdownOpen) + if (!isDropdownOpen) { + inputRef.current?.focus() + } + } + }} + > + {isInputFocused && username ? ( + + ) : ( + + )} +
+
+ + {/* Dropdown Menu */} + {isDropdownOpen && ( +
+ {users.filter(u => u.toLowerCase().includes(username.toLowerCase())).length > 0 ? ( + users.filter(u => u.toLowerCase().includes(username.toLowerCase())).map((user) => ( +
e.preventDefault()} + onClick={() => handleSelectUser(user)} + > + + {user} + +
+ ))) : ( +
+ 无匹配用户 +
+ )} +
+ )} +
+ + {/* Password Field */} +
+ +
+
+ +
+ setPassword(e.target.value)} + /> +
+

+ 密码长度必须为8位(支持中英文) +

+
+
+ + + +
+

+ {error} +

+
+ +
+ +
+
+
+
+ ) +} diff --git a/src/app/page.tsx b/src/app/page.tsx index efa281890dbb62cafd19de7d5818e5ef8e38ae8f..ecbb72fb8156df9c6d31c5b6e68e50203e99f80e 100644 --- a/src/app/page.tsx +++ b/src/app/page.tsx @@ -6,7 +6,7 @@ import { LanguageProvider, useLanguage } from "@/contexts/LanguageContext"; import { ThemeProvider } from "@/contexts/ThemeContext"; import { ThemeSwitcher } from "@/components/ThemeSwitcher"; import { useChatHistory } from "@/hooks/useChatHistory"; -import { Globe, MessageSquare, Plus, Trash2, BookOpenCheck, PanelLeftClose, PanelLeftOpen, User, Palette, Database } from "lucide-react"; +import { Globe, MessageSquare, Plus, Trash2, BookOpenCheck, PanelLeftClose, PanelLeftOpen, User, Palette, Database, BarChart2, LogOut, ChevronUp } from "lucide-react"; import { useCallback, useState, useEffect } from "react"; import { Message } from "ai"; import Link from "next/link"; @@ -35,6 +35,7 @@ function HomeContent() { // It will expand on desktop after mount. const [isSidebarOpen, setIsSidebarOpen] = useState(false); const [isTransitionEnabled, setIsTransitionEnabled] = useState(false); + const [useRAG, setUseRAG] = useState(true); // Helper to update state and localStorage const setSidebarState = useCallback((isOpen: boolean) => { @@ -111,12 +112,19 @@ function HomeContent() { } }, [currentSessionId, updateSessionMessages]); + const handleLogout = () => { + document.cookie = "auth_token=; path=/; expires=Thu, 01 Jan 1970 00:00:01 GMT"; + localStorage.removeItem('rag_kb_current_session_id'); + window.location.href = "/login"; + }; + // Better approach: // We use a separate state to track "pending auto prompt" for a specific session ID. const [pendingAutoPrompts, setPendingAutoPrompts] = useState>({}); const triggerQuiz = useCallback(async () => { - const newSessionId = await createNewSession(); + setUseRAG(false); + const newSessionId = await createNewSession('quiz'); if (newSessionId) { setPendingAutoPrompts(prev => ({ ...prev, @@ -192,6 +200,7 @@ function HomeContent() {
+
+
)}
- {/* Hover Popover - Only when collapsed */} - {!isSidebarOpen && ( -
-
-
- -
-
-

{t('userAccount')}

-

{t('freePlan')}

-
-
- -
- - -
- -
- 主题风格 -
- } - /> - - + {/* Unified Hover Popover - For both Expanded and Collapsed states */} +
+
+
+ +
+
+

{t('userAccount')}

+

{t('freePlan')}

+
- )} + +
+ + + + + +
+ + +
@@ -384,6 +412,10 @@ function HomeContent() { initialMessages={currentSession?.messages || []} onMessagesUpdate={handleMessagesUpdate} autoSubmitPrompt={currentSessionId ? pendingAutoPrompts[currentSessionId] : undefined} + useRAG={useRAG} + onUseRAGChange={setUseRAG} + showRAGToggle={!currentSession || (currentSession.type !== 'quiz' && currentSession.type !== 'file')} + fileInfo={currentSession?.fileInfo} />
diff --git a/src/components/Chat.tsx b/src/components/Chat.tsx index 7632c29a701d28cc0ad2c3146281079f21aebd2b..c947087242bdc0ed88d27b0c2968bc54fcba0973 100644 --- a/src/components/Chat.tsx +++ b/src/components/Chat.tsx @@ -2,16 +2,20 @@ import { useChat } from "@ai-sdk/react"; import { Message } from "ai"; -import { Send, User, Bot, ChevronDown, ChevronUp, Database, MessageSquare } from "lucide-react"; +import { Send, User, Bot, ChevronDown, ChevronUp, Database, MessageSquare, ArrowUp, Square, FileText } from "lucide-react"; import ReactMarkdown from "react-markdown"; import { useEffect, useRef, useState, useMemo } from "react"; import { useLanguage } from "@/contexts/LanguageContext"; import { InteractiveQuiz, QuizQuestion } from "./InteractiveQuiz"; -interface ChatProps { +export interface ChatProps { initialMessages?: Message[]; onMessagesUpdate?: (messages: Message[]) => void; autoSubmitPrompt?: string; + useRAG: boolean; + onUseRAGChange: (useRAG: boolean) => void; + showRAGToggle?: boolean; + fileInfo?: { name: string }; } // Helper to process think tags @@ -23,15 +27,19 @@ const processThinkTags = (content: string) => { }); }; -const QuizLoadingSkeleton = () => ( +const QuizLoadingSkeleton = ({ language }: { language: string }) => (
+
+ + {language === 'zh' ? "正在生成试题..." : "Generating Quiz..."} +
{[1, 2].map((i) => ( -
+
{[1, 2, 3, 4].map((j) => ( @@ -63,7 +71,7 @@ const ThinkBlock = ({ children, isThinkingFinished }: { children: React.ReactNod ); }; -const getMarkdownComponents = (isStreaming: boolean, isThinkingFinished: boolean = true) => ({ +const getMarkdownComponents = (isStreaming: boolean, isThinkingFinished: boolean = true, language: string = 'en') => ({ pre: ({ children }: React.ComponentPropsWithoutRef<'pre'>) => <>{children}, p: ({ children, node, ...props }: React.ComponentPropsWithoutRef<'p'> & { node?: unknown }) => (
@@ -139,7 +147,7 @@ const getMarkdownComponents = (isStreaming: boolean, isThinkingFinished: boolean } catch { // If it's explicitly a quiz tag but parsing failed (likely streaming), show loading skeleton if (isQuizTag && isStreaming) { - return ; + return ; } // For json tag or non-streaming quiz tag, we fall back to code block because it might be regular JSON or broken } @@ -156,7 +164,7 @@ const getMarkdownComponents = (isStreaming: boolean, isThinkingFinished: boolean } return ( -
+      
         {codeElement}
       
); @@ -164,7 +172,7 @@ const getMarkdownComponents = (isStreaming: boolean, isThinkingFinished: boolean }); const UserMarkdownComponents = { - ...getMarkdownComponents(false), + ...getMarkdownComponents(false, true, 'en'), // Default to en for user messages as language context isn't critical there p: ({ children, node, ...props }: React.ComponentPropsWithoutRef<'p'> & { node?: unknown }) => (
{children} @@ -217,7 +225,7 @@ const CollapsibleUserMessage = ({ content }: { content: string }) => { -const SmoothMarkdown = ({ content, isStreaming, onContentUpdate }: { content: string, isStreaming: boolean, onContentUpdate?: () => void }) => { +const SmoothMarkdown = ({ content, isStreaming, onContentUpdate, language }: { content: string, isStreaming: boolean, onContentUpdate?: () => void, language: string }) => { const [displayed, setDisplayed] = useState(isStreaming ? '' : content); const targetRef = useRef(content); const displayedRef = useRef(displayed); @@ -286,7 +294,7 @@ const SmoothMarkdown = ({ content, isStreaming, onContentUpdate }: { content: st // The raw 'displayed' text contains ... if the stream has delivered it. const isThinkingFinished = useMemo(() => displayed.includes(''), [displayed]); - const components = useMemo(() => getMarkdownComponents(isStreaming, isThinkingFinished), [isStreaming, isThinkingFinished]); + const components = useMemo(() => getMarkdownComponents(isStreaming, isThinkingFinished, language), [isStreaming, isThinkingFinished, language]); // Pre-process content to handle tags const processedContent = useMemo(() => processThinkTags(displayed), [displayed]); @@ -300,18 +308,26 @@ const SmoothMarkdown = ({ content, isStreaming, onContentUpdate }: { content: st ); }; -export function Chat({ initialMessages = [], onMessagesUpdate, autoSubmitPrompt }: ChatProps) { - const [useRAG, setUseRAG] = useState(true); +export function Chat({ + initialMessages = [], + onMessagesUpdate, + autoSubmitPrompt, + useRAG, + onUseRAGChange, + showRAGToggle = true, + fileInfo +}: ChatProps) { const { t, language } = useLanguage(); - const { messages, input, handleInputChange, handleSubmit, isLoading, error, append } = useChat({ + const { messages, input, handleInputChange, handleSubmit, isLoading, error, append, stop } = useChat({ initialMessages, - body: { useRAG }, + body: { useRAG: fileInfo ? true : useRAG, fileInfo }, }); const scrollContainerRef = useRef(null); const inputRef = useRef(null); const hasAutoSubmitted = useRef(false); const isAtBottomRef = useRef(true); + const [showScrollTop, setShowScrollTop] = useState(false); useEffect(() => { // Auto-focus input on mount (new chat or switching history) @@ -351,6 +367,14 @@ export function Chat({ initialMessages = [], onMessagesUpdate, autoSubmitPrompt // User is considered "at bottom" if they are within 50px of the bottom const isAtBottom = scrollHeight - scrollTop - clientHeight < 50; isAtBottomRef.current = isAtBottom; + setShowScrollTop(scrollTop > 400); + }; + + const scrollToTop = () => { + scrollContainerRef.current?.scrollTo({ + top: 0, + behavior: 'smooth' + }); }; const scrollToBottom = (smooth = false) => { @@ -445,10 +469,11 @@ export function Chat({ initialMessages = [], onMessagesUpdate, autoSubmitPrompt isAtBottomRef.current && scrollToBottom(false)} /> ) : ( - + {m.content} )} @@ -471,38 +496,67 @@ export function Chat({ initialMessages = [], onMessagesUpdate, autoSubmitPrompt
+ {showScrollTop && ( + + )} +
- -
+ {fileInfo ? ( +
+ +
+ ) : showRAGToggle && ( + + )} - + {isLoading ? ( + + ) : ( + + )}

diff --git a/src/components/ThemeSwitcher.tsx b/src/components/ThemeSwitcher.tsx index b0d3ca23030e1a6195f86cab7dd3fffea95e9a4d..80ba157f1933f1122068295c17e9bc41b309ac58 100644 --- a/src/components/ThemeSwitcher.tsx +++ b/src/components/ThemeSwitcher.tsx @@ -6,10 +6,13 @@ import { useState, useRef, useEffect, ReactNode } from "react"; interface ThemeSwitcherProps { customTrigger?: ReactNode; - position?: 'top' | 'bottom'; + position?: 'top' | 'bottom' | 'right'; + isPopover?: boolean; + label?: string; + className?: string; } -export function ThemeSwitcher({ customTrigger, position = 'top' }: ThemeSwitcherProps) { +export function ThemeSwitcher({ customTrigger, position = 'top', isPopover, label, className }: ThemeSwitcherProps) { const { theme, setTheme } = useTheme(); const [isOpen, setIsOpen] = useState(false); const containerRef = useRef(null); @@ -41,6 +44,21 @@ export function ThemeSwitcher({ customTrigger, position = 'top' }: ThemeSwitcher }}> {customTrigger}

+ ) : isPopover ? ( +
{ + e.stopPropagation(); + setIsOpen(!isOpen); + }} + className={className} + > +
+ + + + {label || '主题风格'} +
+
) : (