Update scripts/backup_to_dataset.py

scripts/backup_to_dataset.py

@@ -2,15 +2,49 @@
 import os
 import sys
 import datetime
+import json
+import sqlite3
+import tarfile
 from huggingface_hub import HfApi, login
 
 # ===== 配置 =====
-DB_PATH = "/app/server/data/freeapi.db"                       # 数据库路径（与 DATABASE_URL 一致）
-DATASET_REPO = "lydgs/freellm-backup"          # 替换为你的数据集名称
-BACKUP_PREFIX = "freeapi_backup"                  # 备份文件前缀
-RETENTION_DAYS = 30                               # 保留天数
+DB_PATH = "/app/server/data/freeapi.db"          # 数据库路径
+DATASET_REPO = "lydgs/freellm-backup"           # 数据集名称
+BACKUP_PREFIX = "freeapi_backup"                # 二进制备份前缀
+CONFIG_PREFIX = "config_export"                 # 配置导出前缀
+RETENTION_DAYS = 30
 # ================
 
+def export_db_to_json(db_path, temp_dir):
+    """将数据库中所有表导出为JSON文件，保存在temp_dir下，返回生成的文件列表"""
+    if not os.path.exists(db_path):
+        print(f"⚠️ 数据库文件不存在: {db_path}")
+        return []
+    
+    conn = sqlite3.connect(db_path)
+    cursor = conn.cursor()
+    
+    # 获取所有表名
+    cursor.execute("SELECT name FROM sqlite_master WHERE type='table';")
+    tables = cursor.fetchall()
+    
+    exported_files = []
+    for (table_name,) in tables:
+        cursor.execute(f"SELECT * FROM {table_name}")
+        rows = cursor.fetchall()
+        # 获取列名
+        col_names = [desc[0] for desc in cursor.description]
+        # 转换为字典列表
+        data = [dict(zip(col_names, row)) for row in rows]
+        json_path = os.path.join(temp_dir, f"{table_name}.json")
+        with open(json_path, 'w', encoding='utf-8') as f:
+            json.dump(data, f, indent=2, ensure_ascii=False)
+        exported_files.append(json_path)
+        print(f"📄 导出表 {table_name} 到 {json_path} ({len(data)} 行)")
+    
+    conn.close()
+    return exported_files
+
 def backup_database():
     if not os.path.exists(DB_PATH):
         print(f"❌ 数据库文件不存在: {DB_PATH}")
@@ -23,11 +57,12 @@ def backup_database():
 
     timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
     backup_name = f"{BACKUP_PREFIX}_{timestamp}.db"
+    config_tar_name = f"{CONFIG_PREFIX}_{timestamp}.tar.gz"
 
     login(token=token)
     api = HfApi()
 
-    # 上传当前备份
+    # 1. 上传二进制备份
     try:
         api.upload_file(
             path_or_fileobj=DB_PATH,
@@ -35,29 +70,69 @@ def backup_database():
             repo_id=DATASET_REPO,
             repo_type="dataset"
         )
-        print(f"✅ 备份成功: {backup_name}")
+        print(f"✅ 二进制备份成功: {backup_name}")
     except Exception as e:
-        print(f"❌ 上传失败: {e}")
-        return False
+        print(f"❌ 二进制上传失败: {e}")
+        # 继续尝试导出配置，不中断
+
+    # 2. 导出 JSON 配置并打包上传
+    try:
+        # 创建临时目录
+        temp_export_dir = f"/tmp/config_export_{timestamp}"
+        os.makedirs(temp_export_dir, exist_ok=True)
+        
+        export_db_to_json(DB_PATH, temp_export_dir)
+        
+        # 打包成 tar.gz
+        tar_path = f"/tmp/{config_tar_name}"
+        with tarfile.open(tar_path, "w:gz") as tar:
+            for file in os.listdir(temp_export_dir):
+                full_path = os.path.join(temp_export_dir, file)
+                tar.add(full_path, arcname=file)
+        
+        # 上传压缩包
+        api.upload_file(
+            path_or_fileobj=tar_path,
+            path_in_repo=config_tar_name,
+            repo_id=DATASET_REPO,
+            repo_type="dataset"
+        )
+        print(f"✅ 配置导出包上传成功: {config_tar_name}")
+        
+        # 清理临时文件
+        import shutil
+        shutil.rmtree(temp_export_dir)
+        os.remove(tar_path)
+    except Exception as e:
+        print(f"❌ 配置导出失败: {e}")
 
-    # 清理旧备份（保留最近 RETENTION_DAYS 天）
+    # 3. 清理旧备份（二进制和配置包）
     try:
         files = api.list_repo_files(repo_id=DATASET_REPO, repo_type="dataset")
         now = datetime.datetime.now()
         deleted = 0
         for f in files:
-            if not f.startswith(BACKUP_PREFIX) or not f.endswith(".db"):
-                continue
-            # 解析时间戳，文件名格式: freeapi_backup_YYYYMMDD_HHMMSS.db
-            try:
-                ts_str = f.replace(BACKUP_PREFIX + "_", "").replace(".db", "")
-                file_time = datetime.datetime.strptime(ts_str, "%Y%m%d_%H%M%S")
-                if (now - file_time).days > RETENTION_DAYS:
-                    api.delete_file(path_in_repo=f, repo_id=DATASET_REPO, repo_type="dataset")
-                    print(f"🗑️ 已删除旧备份: {f}")
-                    deleted += 1
-            except Exception:
-                pass
+            # 删除超过 RETENTION_DAYS 天的二进制备份和配置包
+            if f.startswith(BACKUP_PREFIX) or f.startswith(CONFIG_PREFIX):
+                # 提取时间戳
+                parts = f.replace(".db", "").replace(".tar.gz", "").split("_")
+                if len(parts) >= 3:
+                    ts_str = f"{parts[2]}_{parts[3]}" if len(parts) > 3 else parts[2]
+                    # 文件名格式: freeapi_backup_20260602_061721.db 或 config_export_20260602_061721.tar.gz
+                    # 简化：取最后两个部分（日期_时间）
+                    try:
+                        # 尝试解析时间戳
+                        if f.startswith(BACKUP_PREFIX):
+                            ts_part = f.replace(BACKUP_PREFIX + "_", "").replace(".db", "")
+                        else:
+                            ts_part = f.replace(CONFIG_PREFIX + "_", "").replace(".tar.gz", "")
+                        file_time = datetime.datetime.strptime(ts_part, "%Y%m%d_%H%M%S")
+                        if (now - file_time).days > RETENTION_DAYS:
+                            api.delete_file(path_in_repo=f, repo_id=DATASET_REPO, repo_type="dataset")
+                            print(f"🗑️ 已删除旧备份: {f}")
+                            deleted += 1
+                    except Exception as parse_err:
+                        print(f"⚠️ 跳过无法解析的文件: {f} ({parse_err})")
         if deleted:
             print(f"✅ 清理完成，共删除 {deleted} 个旧备份")
         else:
@@ -68,7 +143,7 @@ def backup_database():
     return True
 
 if __name__ == "__main__":
-    print(f"[{datetime.datetime.now()}] 开始备份...")
+    print(f"[{datetime.datetime.now()}] 开始备份及配置导出...")
     if backup_database():
         print("备份任务完成")
         sys.exit(0)