Create scripts/backup_to_dataset.py

scripts/backup_to_dataset.py

@@ -0,0 +1,83 @@
+#!/usr/bin/env python3
+import os
+import sys
+import datetime
+from huggingface_hub import HfApi, login
+
+# ===== 配置区域 =====
+DB_PATH = "/data/freellm/database.sqlite"          # 数据库文件路径
+DATASET_REPO = "lydgs/freellm-backup"         # 替换成你的私有数据集
+BACKUP_PREFIX = "freellm_backup"                  # 备份文件前缀
+RETENTION_DAYS = 30                               # 保留天数（超过则删除）
+# ===================
+
+def backup_database():
+    # 检查数据库文件
+    if not os.path.exists(DB_PATH):
+        print(f"❌ 数据库文件不存在: {DB_PATH}")
+        return False
+
+    # 生成备份文件名
+    timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
+    backup_name = f"{BACKUP_PREFIX}_{timestamp}.sqlite"
+
+    # 获取 Hugging Face Token
+    token = os.getenv("HF_TOKEN")
+    if not token:
+        print("❌ 环境变量 HF_TOKEN 未设置")
+        return False
+
+    # 登录
+    login(token=token)
+    api = HfApi()
+
+    # 1. 上传当前备份
+    try:
+        api.upload_file(
+            path_or_fileobj=DB_PATH,
+            path_in_repo=backup_name,
+            repo_id=DATASET_REPO,
+            repo_type="dataset"
+        )
+        print(f"✅ 备份成功: https://huggingface.co/datasets/{DATASET_REPO}/blob/main/{backup_name}")
+    except Exception as e:
+        print(f"❌ 上传失败: {e}")
+        return False
+
+    # 2. 清理旧备份（保留最近 RETENTION_DAYS 天的文件）
+    try:
+        files = api.list_repo_files(repo_id=DATASET_REPO, repo_type="dataset")
+        now = datetime.datetime.now()
+        deleted_count = 0
+        for f in files:
+            if not f.startswith(BACKUP_PREFIX) or not f.endswith(".sqlite"):
+                continue
+            # 解析时间戳部分，文件名格式: prefix_YYYYMMDD_HHMMSS.sqlite
+            try:
+                parts = f.replace(".sqlite", "").split("_")
+                if len(parts) >= 3:
+                    ts_str = f"{parts[1]}_{parts[2]}"   # YYYYMMDD_HHMMSS
+                    file_time = datetime.datetime.strptime(ts_str, "%Y%m%d_%H%M%S")
+                    if (now - file_time).days > RETENTION_DAYS:
+                        api.delete_file(path_in_repo=f, repo_id=DATASET_REPO, repo_type="dataset")
+                        print(f"🗑️ 已删除旧备份: {f}")
+                        deleted_count += 1
+            except Exception as parse_err:
+                print(f"⚠️ 跳过无法解析的文件: {f}, 原因: {parse_err}")
+        if deleted_count:
+            print(f"✅ 清理完成，共删除 {deleted_count} 个旧备份")
+        else:
+            print("✅ 没有需要清理的旧备份")
+    except Exception as e:
+        print(f"⚠️ 清理旧备份时出错: {e}")
+
+    return True
+
+if __name__ == "__main__":
+    print(f"[{datetime.datetime.now()}] 开始备份并清理...")
+    if backup_database():
+        print("任务完成")
+        sys.exit(0)
+    else:
+        print("任务失败")
+        sys.exit(1)