Update hf-sync-manager.mjs

hf-sync-manager.mjs

@@ -11,11 +11,7 @@ const DATASET_ID     = (process.env.HF_DATASET_ID || "").trim();
 const REPO_DIR       = "/tmp/hf-memory-sync";
 const UPLOAD_INTERVAL = 3 * 60 * 60 * 1000; // 3小时
 
-const FILES = {
-  conversations: path.join(MEMORY_DIR, "conversations.jsonl"),
-  memory:        path.join(MEMORY_DIR, "memory.jsonl"),
-  logs:          path.join(MEMORY_DIR, "logs.jsonl")
-};
+const FILES = ["conversations", "memory", "logs"];
 
 // ===================== 内存对象 =====================
 const botMemory = {
@@ -24,13 +20,9 @@ const botMemory = {
   logs: []
 };
 
-// ===================== Helper 函数 =====================
+// ===================== Helper =====================
 function log(msg) { console.log(`[hf-sync] ${msg}`); }
-
-function repoUrl() {
-  return `https://user:${HF_TOKEN}@huggingface.co/datasets/${DATASET_ID}`;
-}
-
+function repoUrl() { return `https://user:${HF_TOKEN}@huggingface.co/datasets/${DATASET_ID}`; }
 function git(args, cwd) {
   return execSync("git " + args, {
     cwd: cwd || REPO_DIR,
@@ -43,9 +35,8 @@ function git(args, cwd) {
 function ensureRepo() {
   if (fs.existsSync(path.join(REPO_DIR, ".git"))) return;
   execSync("rm -rf " + REPO_DIR, { stdio: "pipe" });
-  try {
-    git("clone --depth 1 " + repoUrl() + " " + REPO_DIR, "/tmp");
-  } catch {
+  try { git("clone --depth 1 " + repoUrl() + " " + REPO_DIR, "/tmp"); }
+  catch {
     fs.mkdirSync(REPO_DIR, { recursive: true });
     git("init", REPO_DIR);
     git("remote add origin " + repoUrl(), REPO_DIR);
@@ -64,9 +55,7 @@ function pullLatest() {
 function readJsonl(filePath, targetArray) {
   if (!fs.existsSync(filePath)) return;
   const lines = fs.readFileSync(filePath, "utf-8").split("\n").filter(Boolean);
-  lines.forEach(line => {
-    try { targetArray.push(JSON.parse(line)); } catch(e) { log(`Invalid JSON line in ${filePath}`); }
-  });
+  lines.forEach(line => { try { targetArray.push(JSON.parse(line)); } catch{} });
 }
 
 function writeJsonl(filePath, dataArray) {
@@ -74,7 +63,7 @@ function writeJsonl(filePath, dataArray) {
   fs.writeFileSync(filePath, lines, "utf-8");
 }
 
-// ===================== 内存操作接口 =====================
+// ===================== 内存接口 =====================
 export function addConversation(sessionId, userMsg, botMsg) {
   const ts = new Date().toISOString();
   botMemory.conversations.push({ session_id: sessionId, user: userMsg, bot: botMsg, timestamp: ts });
@@ -89,27 +78,41 @@ export function addLog(sessionId, event) {
   botMemory.logs.push({ session_id: sessionId, event, timestamp: ts });
 }
 
+// ===================== 路径管理 =====================
+function getDateDir(date=new Date()) {
+  const y = date.getFullYear(), m = String(date.getMonth()+1).padStart(2,"0"), d = String(date.getDate()).padStart(2,"0");
+  return path.join(MEMORY_DIR, `${y}-${m}-${d}`);
+}
+
+function getFilesForDate(date=new Date()) {
+  const dir = getDateDir(date);
+  fs.mkdirSync(dir, { recursive: true });
+  const fileMap = {};
+  FILES.forEach(f => { fileMap[f] = path.join(dir, f + ".jsonl"); });
+  return fileMap;
+}
+
 // ===================== 载入 & 保存 =====================
 function loadMemoryFromRepo() {
   if (!fs.existsSync(REPO_DIR)) return;
   const repoMemoryDir = path.join(REPO_DIR, "dataset-memory");
   if (!fs.existsSync(repoMemoryDir)) return;
 
-  fs.mkdirSync(MEMORY_DIR, { recursive: true });
-  for (const key of Object.keys(FILES)) {
-    const src = path.join(repoMemoryDir, key + ".jsonl");
-    const dst = FILES[key];
-    if (fs.existsSync(src)) fs.copyFileSync(src, dst);
-    else fs.writeFileSync(dst, "", "utf-8");
-  }
+  // 遍历每个日期文件夹
+  const dateDirs = fs.readdirSync(repoMemoryDir).filter(f => fs.statSync(path.join(repoMemoryDir,f)).isDirectory());
+  dateDirs.forEach(dateDir => {
+    const files = {};
+    FILES.forEach(f => { files[f] = path.join(repoMemoryDir, dateDir, f + ".jsonl"); });
+    FILES.forEach(f => readJsonl(files[f], botMemory[f]));
+  });
+  log(`Loaded: conversations=${botMemory.conversations.length}, memory=${botMemory.memory.length}, logs=${botMemory.logs.length}`);
 }
 
 function saveMemoryToRepo() {
-  const repoMemoryDir = path.join(REPO_DIR, "dataset-memory");
-  fs.mkdirSync(repoMemoryDir, { recursive: true });
-  for (const key of Object.keys(FILES)) {
-    writeJsonl(path.join(repoMemoryDir, key + ".jsonl"), botMemory[key]);
-  }
+  const files = getFilesForDate();
+  const repoDateDir = path.join(REPO_DIR, "dataset-memory", path.basename(getDateDir()));
+  fs.mkdirSync(repoDateDir, { recursive: true });
+  FILES.forEach(f => writeJsonl(path.join(repoDateDir, f + ".jsonl"), botMemory[f]));
 }
 
 function commitAndPush() {
@@ -117,14 +120,11 @@ function commitAndPush() {
     git("add -A", REPO_DIR);
     const changed = git("diff --cached --name-only", REPO_DIR);
     if (!changed) return 0;
-    const ts = new Date().toISOString().replace("T", " ").substring(0, 19);
+    const ts = new Date().toISOString().replace("T"," ").substring(0,19);
     git(`commit -m "memory-sync ${ts}"`, REPO_DIR);
     git("push --quiet origin HEAD:main", REPO_DIR);
     return changed.split("\n").filter(Boolean).length;
-  } catch(e) {
-    log("Push failed: " + e.message);
-    return 0;
-  }
+  } catch(e){ log("Push failed: "+e.message); return 0; }
 }
 
 // ===================== 启动 & 定时 =====================
@@ -134,31 +134,29 @@ function bootRestore() {
   ensureRepo();
   pullLatest();
   loadMemoryFromRepo();
-  for (const key of Object.keys(FILES)) readJsonl(FILES[key], botMemory[key]);
-  log(`Loaded: conversations=${botMemory.conversations.length}, memory=${botMemory.memory.length}, logs=${botMemory.logs.length}`);
 }
 
 async function startUploadLoop() {
   while(true){
-    await new Promise(r => setTimeout(r, UPLOAD_INTERVAL));
-    try {
+    await new Promise(r=>setTimeout(r, UPLOAD_INTERVAL));
+    try{
       log("Saving memory to repo...");
       saveMemoryToRepo();
       const count = commitAndPush();
-      log("Upload done: " + count + " file(s) changed");
-    } catch(e){ log("Upload error: " + e.message); }
+      log("Upload done: "+count+" file(s) changed");
+    } catch(e){ log("Upload error: "+e.message); }
   }
 }
 
 // ===================== Main =====================
-async function main() {
+async function main(){
   fs.mkdirSync(MEMORY_DIR, { recursive: true });
   bootRestore();
   log("Memory ready.");
 
   if(HF_TOKEN && DATASET_ID){
     log(`Starting ${UPLOAD_INTERVAL/3600000}h upload loop`);
-    startUploadLoop().catch(e => log("Upload loop error: "+e.message));
+    startUploadLoop().catch(e=>log("Upload loop error: "+e.message));
   }
 }