Spaces:
Running
Running
sync from GitHub d79b7e2: feat: Implement user account management and file synchronization with HF Hub
Browse files- app/api/user_files.py +221 -78
- app/hf_storage.py +87 -0
- app/main.py +53 -1
- app/services/user_files_store.py +139 -0
- app/services/user_files_sync.py +146 -0
app/api/user_files.py
CHANGED
|
@@ -10,62 +10,49 @@
|
|
| 10 |
- GET /u/api/files/{key} 下载文件(仅 owner)
|
| 11 |
- DELETE /u/api/files/{key} 删除文件(仅 owner)
|
| 12 |
|
| 13 |
-
持久化:
|
| 14 |
-
-
|
| 15 |
-
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 16 |
"""
|
| 17 |
from __future__ import annotations
|
| 18 |
|
|
|
|
| 19 |
import hashlib
|
|
|
|
| 20 |
import re
|
| 21 |
import time
|
|
|
|
| 22 |
import uuid
|
| 23 |
-
from
|
| 24 |
|
| 25 |
import bcrypt
|
| 26 |
import jwt
|
| 27 |
-
from fastapi import APIRouter,
|
| 28 |
-
from fastapi.responses import
|
| 29 |
|
| 30 |
from ..config import get_settings
|
| 31 |
from ..database import get_conn
|
| 32 |
from ..errors import HttpError
|
| 33 |
-
from .
|
|
|
|
|
|
|
| 34 |
|
| 35 |
router = APIRouter(prefix="/u/api", tags=["user-files"])
|
| 36 |
|
| 37 |
ALGORITHM = "HS256"
|
| 38 |
MAX_FILE_SIZE = 10 * 1024 * 1024 # 10MB
|
| 39 |
-
NS_USER_FILES = "user_files"
|
| 40 |
TOKEN_TTL_SEC = 7 * 24 * 3600 # 7 天
|
| 41 |
|
| 42 |
_USERNAME_RE = re.compile(r"^[A-Za-z0-9_\u4e00-\u9fff.\-]{2,32}$")
|
| 43 |
|
| 44 |
-
|
| 45 |
-
# ===== 文件存储路径 =====
|
| 46 |
-
|
| 47 |
-
def _uploads_root() -> Path:
|
| 48 |
-
"""上传根目录:优先 /data/uploads,不可写回退 /tmp/uploads。"""
|
| 49 |
-
for candidate in ("/data/uploads", "/tmp/uploads"):
|
| 50 |
-
try:
|
| 51 |
-
p = Path(candidate)
|
| 52 |
-
p.mkdir(parents=True, exist_ok=True)
|
| 53 |
-
test = p / ".wtest"
|
| 54 |
-
test.touch(exist_ok=True)
|
| 55 |
-
test.unlink(missing_ok=True)
|
| 56 |
-
return p
|
| 57 |
-
except Exception:
|
| 58 |
-
continue
|
| 59 |
-
p = Path("/tmp/uploads")
|
| 60 |
-
p.mkdir(parents=True, exist_ok=True)
|
| 61 |
-
return p
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
def _file_path(key: str) -> Path:
|
| 65 |
-
"""根据 key 返回磁盘路径,做严格白名单校验防穿越。"""
|
| 66 |
-
if not key or not re.fullmatch(r"[A-Za-z0-9_-]{8,64}", key):
|
| 67 |
-
raise HttpError("invalid key", status=400, code="bad_request")
|
| 68 |
-
return _uploads_root() / key
|
| 69 |
|
| 70 |
|
| 71 |
# ===== JWT =====
|
|
@@ -134,10 +121,89 @@ def _verify_password(password: str, hashed: str) -> bool:
|
|
| 134 |
return False
|
| 135 |
|
| 136 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 137 |
# ===== 注册 =====
|
| 138 |
|
| 139 |
@router.post("/register")
|
| 140 |
-
async def register(
|
|
|
|
| 141 |
username = str(body.get("username") or "").strip()
|
| 142 |
password = str(body.get("password") or "")
|
| 143 |
if not username or not password:
|
|
@@ -150,18 +216,39 @@ async def register(body: dict = Body(default={})):
|
|
| 150 |
if len(password) < 6 or len(password) > 128:
|
| 151 |
raise HttpError("password length must be 6-128", status=400, code="bad_request")
|
| 152 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 153 |
password_hash = _hash_password(password)
|
| 154 |
now = int(time.time())
|
| 155 |
try:
|
| 156 |
-
|
| 157 |
-
cur = conn.execute(
|
| 158 |
-
"INSERT INTO user_accounts(username, password_hash, created_at) VALUES(?,?,?)",
|
| 159 |
-
(username, password_hash, now),
|
| 160 |
-
)
|
| 161 |
-
user_id = cur.lastrowid
|
| 162 |
except Exception:
|
| 163 |
raise HttpError("username already exists", status=409, code="conflict")
|
| 164 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 165 |
token = _issue_user_token(user_id, username)
|
| 166 |
return ok_with_cors({
|
| 167 |
"token": token,
|
|
@@ -172,24 +259,27 @@ async def register(body: dict = Body(default={})):
|
|
| 172 |
# ===== 登录 =====
|
| 173 |
|
| 174 |
@router.post("/login")
|
| 175 |
-
async def login(
|
|
|
|
| 176 |
username = str(body.get("username") or "").strip()
|
| 177 |
password = str(body.get("password") or "")
|
| 178 |
if not username or not password:
|
| 179 |
raise HttpError("username and password required", status=400, code="bad_request")
|
| 180 |
|
| 181 |
-
|
| 182 |
-
|
| 183 |
-
|
| 184 |
-
|
| 185 |
-
|
| 186 |
-
|
|
|
|
|
|
|
| 187 |
raise HttpError("invalid username or password", status=401, code="unauthorized")
|
| 188 |
|
| 189 |
-
token = _issue_user_token(
|
| 190 |
return ok_with_cors({
|
| 191 |
"token": token,
|
| 192 |
-
"user": {"id":
|
| 193 |
})
|
| 194 |
|
| 195 |
|
|
@@ -207,14 +297,12 @@ async def me(user: dict = Depends(require_user)):
|
|
| 207 |
@router.get("/files")
|
| 208 |
async def list_my_files(user: dict = Depends(require_user)):
|
| 209 |
user_id = str(user["sub"])
|
| 210 |
-
|
| 211 |
-
|
| 212 |
-
|
| 213 |
-
|
| 214 |
-
|
| 215 |
-
(user_id
|
| 216 |
-
).fetchall()
|
| 217 |
-
items = [dict(r) for r in rows]
|
| 218 |
return ok_with_cors({"items": items, "count": len(items)})
|
| 219 |
|
| 220 |
|
|
@@ -239,7 +327,6 @@ async def upload_file(
|
|
| 239 |
|
| 240 |
key = uuid.uuid4().hex
|
| 241 |
filename = (file.filename or "unnamed").strip()
|
| 242 |
-
# 截断过长文件名,避免存储异常
|
| 243 |
if len(filename) > 255:
|
| 244 |
filename = filename[:255]
|
| 245 |
mime = (file.content_type or "application/octet-stream").strip()
|
|
@@ -247,11 +334,30 @@ async def upload_file(
|
|
| 247 |
sha256 = hashlib.sha256(content).hexdigest()
|
| 248 |
now = int(time.time())
|
| 249 |
|
| 250 |
-
# 写
|
| 251 |
-
|
| 252 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 253 |
|
| 254 |
-
# 写元信息
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 255 |
with get_conn() as conn:
|
| 256 |
conn.execute(
|
| 257 |
"INSERT INTO file_meta(key, namespace, filename, mime, size, sha256, "
|
|
@@ -260,6 +366,13 @@ async def upload_file(
|
|
| 260 |
(key, NS_USER_FILES, filename, mime, size, sha256, now, user_id, username, ""),
|
| 261 |
)
|
| 262 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 263 |
return ok_with_cors({
|
| 264 |
"key": key,
|
| 265 |
"filename": filename,
|
|
@@ -283,20 +396,49 @@ async def download_file(
|
|
| 283 |
"SELECT filename, mime, uploaded_by FROM file_meta WHERE key = ?",
|
| 284 |
(key,),
|
| 285 |
).fetchone()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 286 |
if not row:
|
| 287 |
raise HttpError("file not found", status=404, code="not_found")
|
| 288 |
if row["uploaded_by"] != user_id:
|
| 289 |
raise HttpError("forbidden", status=403, code="forbidden")
|
| 290 |
|
| 291 |
-
|
| 292 |
-
|
| 293 |
-
|
| 294 |
-
|
| 295 |
-
|
| 296 |
-
|
| 297 |
-
|
| 298 |
-
|
| 299 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 300 |
|
| 301 |
|
| 302 |
# ===== 删除 =====
|
|
@@ -318,13 +460,14 @@ async def delete_file(
|
|
| 318 |
raise HttpError("forbidden", status=403, code="forbidden")
|
| 319 |
conn.execute("DELETE FROM file_meta WHERE key = ?", (key,))
|
| 320 |
|
| 321 |
-
# 删
|
| 322 |
try:
|
| 323 |
-
|
| 324 |
-
|
| 325 |
-
|
| 326 |
-
|
| 327 |
-
|
| 328 |
-
|
|
|
|
| 329 |
|
| 330 |
return ok_with_cors({"deleted": True, "key": key})
|
|
|
|
| 10 |
- GET /u/api/files/{key} 下载文件(仅 owner)
|
| 11 |
- DELETE /u/api/files/{key} 删除文件(仅 owner)
|
| 12 |
|
| 13 |
+
持久化(与 config_store 同样的混合策略):
|
| 14 |
+
- 热存储:SQLite(/data/xtc.db)+ 本地磁盘缓存(/data/hf_bucket/...)
|
| 15 |
+
- 冷备份:HF Hub dataset 仓库,按用户隔离的 namespace
|
| 16 |
+
- user_accounts/<username> 用户账号 JSON
|
| 17 |
+
- user_files/<user_id>/<file_key> 文件内容
|
| 18 |
+
- user_files_meta/<user_id>/<file_key> 文件元信息 JSON
|
| 19 |
+
- 写入时先写本地,再异步推 Hub(Hub 失败不影响主流程)
|
| 20 |
+
- 读取时本地命中即返回;未命中从 Hub 拉取并写本地(用于 Space 重建后恢复)
|
| 21 |
+
- 未配置 HF_TOKEN / HF_CONFIG_REPO 时自动降级为纯本地
|
| 22 |
"""
|
| 23 |
from __future__ import annotations
|
| 24 |
|
| 25 |
+
import asyncio
|
| 26 |
import hashlib
|
| 27 |
+
import logging
|
| 28 |
import re
|
| 29 |
import time
|
| 30 |
+
import urllib.parse
|
| 31 |
import uuid
|
| 32 |
+
from typing import Optional
|
| 33 |
|
| 34 |
import bcrypt
|
| 35 |
import jwt
|
| 36 |
+
from fastapi import APIRouter, Depends, File, Request, UploadFile
|
| 37 |
+
from fastapi.responses import Response
|
| 38 |
|
| 39 |
from ..config import get_settings
|
| 40 |
from ..database import get_conn
|
| 41 |
from ..errors import HttpError
|
| 42 |
+
from ..hf_storage import is_hub_enabled
|
| 43 |
+
from ..services import user_files_store
|
| 44 |
+
from ._common import CORS_HEADERS, ok_with_cors, read_json_body
|
| 45 |
|
| 46 |
router = APIRouter(prefix="/u/api", tags=["user-files"])
|
| 47 |
|
| 48 |
ALGORITHM = "HS256"
|
| 49 |
MAX_FILE_SIZE = 10 * 1024 * 1024 # 10MB
|
| 50 |
+
NS_USER_FILES = "user_files" # file_meta.namespace 字段值
|
| 51 |
TOKEN_TTL_SEC = 7 * 24 * 3600 # 7 天
|
| 52 |
|
| 53 |
_USERNAME_RE = re.compile(r"^[A-Za-z0-9_\u4e00-\u9fff.\-]{2,32}$")
|
| 54 |
|
| 55 |
+
logger = logging.getLogger(__name__)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 56 |
|
| 57 |
|
| 58 |
# ===== JWT =====
|
|
|
|
| 121 |
return False
|
| 122 |
|
| 123 |
|
| 124 |
+
# ===== 用户账号:SQLite + Hub 备份/恢复 =====
|
| 125 |
+
|
| 126 |
+
def _insert_user_to_db(username: str, password_hash: str, now: int) -> int:
|
| 127 |
+
with get_conn() as conn:
|
| 128 |
+
cur = conn.execute(
|
| 129 |
+
"INSERT INTO user_accounts(username, password_hash, created_at) VALUES(?,?,?)",
|
| 130 |
+
(username, password_hash, now),
|
| 131 |
+
)
|
| 132 |
+
return int(cur.lastrowid)
|
| 133 |
+
|
| 134 |
+
|
| 135 |
+
def _find_user_in_db(username: str) -> Optional[dict]:
|
| 136 |
+
with get_conn() as conn:
|
| 137 |
+
row = conn.execute(
|
| 138 |
+
"SELECT id, username, password_hash, created_at FROM user_accounts WHERE username = ?",
|
| 139 |
+
(username,),
|
| 140 |
+
).fetchone()
|
| 141 |
+
return dict(row) if row else None
|
| 142 |
+
|
| 143 |
+
|
| 144 |
+
def _restore_user_from_hub(username: str) -> Optional[dict]:
|
| 145 |
+
"""Space 重建后 SQLite 丢失,从 Hub 拉取账号并写回 SQLite。"""
|
| 146 |
+
obj = user_files_store.restore_user_account(username)
|
| 147 |
+
if not obj:
|
| 148 |
+
return None
|
| 149 |
+
try:
|
| 150 |
+
# 写回 SQLite(若已存在则忽略)
|
| 151 |
+
with get_conn() as conn:
|
| 152 |
+
conn.execute(
|
| 153 |
+
"INSERT OR IGNORE INTO user_accounts(id, username, password_hash, created_at) "
|
| 154 |
+
"VALUES(?,?,?,?)",
|
| 155 |
+
(int(obj["id"]), obj["username"], obj["password_hash"], int(obj["created_at"])),
|
| 156 |
+
)
|
| 157 |
+
return obj
|
| 158 |
+
except Exception as e:
|
| 159 |
+
print(f"[user_files] restore_user_from_hub write db failed: {e}")
|
| 160 |
+
return obj
|
| 161 |
+
|
| 162 |
+
|
| 163 |
+
# ===== 文件元信息:SQLite + Hub 备份/恢复 =====
|
| 164 |
+
|
| 165 |
+
def _list_files_from_db(user_id: str) -> list[dict]:
|
| 166 |
+
with get_conn() as conn:
|
| 167 |
+
rows = conn.execute(
|
| 168 |
+
"SELECT key, filename, mime, size, sha256, uploaded_at "
|
| 169 |
+
"FROM file_meta WHERE uploaded_by = ? AND namespace = ? "
|
| 170 |
+
"ORDER BY uploaded_at DESC",
|
| 171 |
+
(user_id, NS_USER_FILES),
|
| 172 |
+
).fetchall()
|
| 173 |
+
return [dict(r) for r in rows]
|
| 174 |
+
|
| 175 |
+
|
| 176 |
+
async def _restore_files_from_hub_async(user_id: str) -> int:
|
| 177 |
+
metas = await user_files_store.restore_file_metas(user_id)
|
| 178 |
+
if not metas:
|
| 179 |
+
return 0
|
| 180 |
+
inserted = 0
|
| 181 |
+
with get_conn() as conn:
|
| 182 |
+
for m in metas:
|
| 183 |
+
try:
|
| 184 |
+
conn.execute(
|
| 185 |
+
"INSERT OR IGNORE INTO file_meta(key, namespace, filename, mime, size, "
|
| 186 |
+
"sha256, uploaded_at, uploaded_by, access_key, refs) "
|
| 187 |
+
"VALUES(?,?,?,?,?,?,?,?,?,?)",
|
| 188 |
+
(
|
| 189 |
+
m["key"], NS_USER_FILES, m.get("filename", "unnamed"),
|
| 190 |
+
m.get("mime", "application/octet-stream"),
|
| 191 |
+
int(m.get("size", 0)), m.get("sha256"),
|
| 192 |
+
int(m.get("uploaded_at", 0)), str(user_id),
|
| 193 |
+
m.get("access_key", ""), "",
|
| 194 |
+
),
|
| 195 |
+
)
|
| 196 |
+
inserted += 1
|
| 197 |
+
except Exception as e:
|
| 198 |
+
print(f"[user_files] restore file_meta {m.get('key')} failed: {e}")
|
| 199 |
+
return inserted
|
| 200 |
+
|
| 201 |
+
|
| 202 |
# ===== 注册 =====
|
| 203 |
|
| 204 |
@router.post("/register")
|
| 205 |
+
async def register(request: Request):
|
| 206 |
+
body = await read_json_body(request)
|
| 207 |
username = str(body.get("username") or "").strip()
|
| 208 |
password = str(body.get("password") or "")
|
| 209 |
if not username or not password:
|
|
|
|
| 216 |
if len(password) < 6 or len(password) > 128:
|
| 217 |
raise HttpError("password length must be 6-128", status=400, code="bad_request")
|
| 218 |
|
| 219 |
+
# 先查 Hub 是否已有该用户(SQLite 重建场景)
|
| 220 |
+
if not _find_user_in_db(username):
|
| 221 |
+
hub_user = user_files_store.restore_user_account(username)
|
| 222 |
+
if hub_user:
|
| 223 |
+
# 写回 SQLite,之后走"已存在"分支
|
| 224 |
+
try:
|
| 225 |
+
with get_conn() as conn:
|
| 226 |
+
conn.execute(
|
| 227 |
+
"INSERT OR IGNORE INTO user_accounts(id, username, password_hash, created_at) "
|
| 228 |
+
"VALUES(?,?,?,?)",
|
| 229 |
+
(int(hub_user["id"]), hub_user["username"],
|
| 230 |
+
hub_user["password_hash"], int(hub_user["created_at"])),
|
| 231 |
+
)
|
| 232 |
+
except Exception:
|
| 233 |
+
pass
|
| 234 |
+
|
| 235 |
password_hash = _hash_password(password)
|
| 236 |
now = int(time.time())
|
| 237 |
try:
|
| 238 |
+
user_id = _insert_user_to_db(username, password_hash, now)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 239 |
except Exception:
|
| 240 |
raise HttpError("username already exists", status=409, code="conflict")
|
| 241 |
|
| 242 |
+
# 备份到 Hub(本地缓存 + 异步推远端)
|
| 243 |
+
try:
|
| 244 |
+
user_files_store.backup_user_account(
|
| 245 |
+
user_id=user_id, username=username,
|
| 246 |
+
password_hash=password_hash, created_at=now,
|
| 247 |
+
)
|
| 248 |
+
asyncio.create_task(user_files_store.push_user_account(username))
|
| 249 |
+
except Exception as e:
|
| 250 |
+
print(f"[user_files] backup_user_account failed: {e}")
|
| 251 |
+
|
| 252 |
token = _issue_user_token(user_id, username)
|
| 253 |
return ok_with_cors({
|
| 254 |
"token": token,
|
|
|
|
| 259 |
# ===== 登录 =====
|
| 260 |
|
| 261 |
@router.post("/login")
|
| 262 |
+
async def login(request: Request):
|
| 263 |
+
body = await read_json_body(request)
|
| 264 |
username = str(body.get("username") or "").strip()
|
| 265 |
password = str(body.get("password") or "")
|
| 266 |
if not username or not password:
|
| 267 |
raise HttpError("username and password required", status=400, code="bad_request")
|
| 268 |
|
| 269 |
+
user = _find_user_in_db(username)
|
| 270 |
+
# SQLite 没有则尝试从 Hub 恢复(Space 重建场景)
|
| 271 |
+
if not user:
|
| 272 |
+
restored = _restore_user_from_hub(username)
|
| 273 |
+
if restored:
|
| 274 |
+
user = _find_user_in_db(username) or restored
|
| 275 |
+
|
| 276 |
+
if not user or not _verify_password(password, user["password_hash"]):
|
| 277 |
raise HttpError("invalid username or password", status=401, code="unauthorized")
|
| 278 |
|
| 279 |
+
token = _issue_user_token(int(user["id"]), user["username"])
|
| 280 |
return ok_with_cors({
|
| 281 |
"token": token,
|
| 282 |
+
"user": {"id": int(user["id"]), "username": user["username"]},
|
| 283 |
})
|
| 284 |
|
| 285 |
|
|
|
|
| 297 |
@router.get("/files")
|
| 298 |
async def list_my_files(user: dict = Depends(require_user)):
|
| 299 |
user_id = str(user["sub"])
|
| 300 |
+
items = _list_files_from_db(user_id)
|
| 301 |
+
# SQLite 为空时尝试从 Hub 恢复(Space 重建场景)
|
| 302 |
+
if not items:
|
| 303 |
+
restored = await _restore_files_from_hub_async(user_id)
|
| 304 |
+
if restored > 0:
|
| 305 |
+
items = _list_files_from_db(user_id)
|
|
|
|
|
|
|
| 306 |
return ok_with_cors({"items": items, "count": len(items)})
|
| 307 |
|
| 308 |
|
|
|
|
| 327 |
|
| 328 |
key = uuid.uuid4().hex
|
| 329 |
filename = (file.filename or "unnamed").strip()
|
|
|
|
| 330 |
if len(filename) > 255:
|
| 331 |
filename = filename[:255]
|
| 332 |
mime = (file.content_type or "application/octet-stream").strip()
|
|
|
|
| 334 |
sha256 = hashlib.sha256(content).hexdigest()
|
| 335 |
now = int(time.time())
|
| 336 |
|
| 337 |
+
# 写文件内容到本地缓存 + 异步推 Hub
|
| 338 |
+
try:
|
| 339 |
+
user_files_store.upload_user_file_content(user_id, key, content)
|
| 340 |
+
logger.info(
|
| 341 |
+
"[upload] user=%s uid=%s key=%s filename=%s size=%d saved locally, hub_enabled=%s",
|
| 342 |
+
username, user_id, key, filename, size, is_hub_enabled(),
|
| 343 |
+
)
|
| 344 |
+
asyncio.create_task(user_files_store.push_user_file_content(user_id, key))
|
| 345 |
+
except Exception as e:
|
| 346 |
+
logger.error("[upload] store failed user=%s filename=%s: %s", username, filename, e)
|
| 347 |
+
raise HttpError("failed to store file", status=500, code="internal_error") from e
|
| 348 |
|
| 349 |
+
# 写元信息到 SQLite
|
| 350 |
+
meta = {
|
| 351 |
+
"key": key,
|
| 352 |
+
"namespace": NS_USER_FILES,
|
| 353 |
+
"filename": filename,
|
| 354 |
+
"mime": mime,
|
| 355 |
+
"size": size,
|
| 356 |
+
"sha256": sha256,
|
| 357 |
+
"uploaded_at": now,
|
| 358 |
+
"uploaded_by": user_id,
|
| 359 |
+
"access_key": username,
|
| 360 |
+
}
|
| 361 |
with get_conn() as conn:
|
| 362 |
conn.execute(
|
| 363 |
"INSERT INTO file_meta(key, namespace, filename, mime, size, sha256, "
|
|
|
|
| 366 |
(key, NS_USER_FILES, filename, mime, size, sha256, now, user_id, username, ""),
|
| 367 |
)
|
| 368 |
|
| 369 |
+
# 备份元信息到 Hub
|
| 370 |
+
try:
|
| 371 |
+
user_files_store.backup_file_meta(user_id, key, meta)
|
| 372 |
+
asyncio.create_task(user_files_store.push_file_meta(user_id, key))
|
| 373 |
+
except Exception as e:
|
| 374 |
+
print(f"[user_files] backup_file_meta failed: {e}")
|
| 375 |
+
|
| 376 |
return ok_with_cors({
|
| 377 |
"key": key,
|
| 378 |
"filename": filename,
|
|
|
|
| 396 |
"SELECT filename, mime, uploaded_by FROM file_meta WHERE key = ?",
|
| 397 |
(key,),
|
| 398 |
).fetchone()
|
| 399 |
+
# SQLite 没有则尝试从 Hub 恢复单条 meta(Space 重建场景)
|
| 400 |
+
if not row:
|
| 401 |
+
metas = await user_files_store.restore_file_metas(user_id)
|
| 402 |
+
target = next((m for m in metas if m.get("key") == key), None)
|
| 403 |
+
if target:
|
| 404 |
+
try:
|
| 405 |
+
with get_conn() as conn:
|
| 406 |
+
conn.execute(
|
| 407 |
+
"INSERT OR IGNORE INTO file_meta(key, namespace, filename, mime, size, "
|
| 408 |
+
"sha256, uploaded_at, uploaded_by, access_key, refs) "
|
| 409 |
+
"VALUES(?,?,?,?,?,?,?,?,?,?)",
|
| 410 |
+
(target["key"], NS_USER_FILES, target.get("filename", "unnamed"),
|
| 411 |
+
target.get("mime", "application/octet-stream"),
|
| 412 |
+
int(target.get("size", 0)), target.get("sha256"),
|
| 413 |
+
int(target.get("uploaded_at", 0)), str(user_id),
|
| 414 |
+
target.get("access_key", ""), ""),
|
| 415 |
+
)
|
| 416 |
+
except Exception:
|
| 417 |
+
pass
|
| 418 |
+
with get_conn() as conn:
|
| 419 |
+
row = conn.execute(
|
| 420 |
+
"SELECT filename, mime, uploaded_by FROM file_meta WHERE key = ?",
|
| 421 |
+
(key,),
|
| 422 |
+
).fetchone()
|
| 423 |
if not row:
|
| 424 |
raise HttpError("file not found", status=404, code="not_found")
|
| 425 |
if row["uploaded_by"] != user_id:
|
| 426 |
raise HttpError("forbidden", status=403, code="forbidden")
|
| 427 |
|
| 428 |
+
# 读文件内容:本地优先,Hub 回退
|
| 429 |
+
content = user_files_store.download_user_file_content(user_id, key)
|
| 430 |
+
if content is None:
|
| 431 |
+
raise HttpError("file content missing", status=410, code="gone")
|
| 432 |
+
|
| 433 |
+
# 用 Response 直接返回 bytes(避免依赖本地磁盘路径,兼容 Hub 回退场景)
|
| 434 |
+
encoded_name = urllib.parse.quote(row["filename"])
|
| 435 |
+
headers = {
|
| 436 |
+
**CORS_HEADERS,
|
| 437 |
+
"Content-Disposition": f"attachment; filename=\"{row['filename']}\"; filename*=UTF-8''{encoded_name}",
|
| 438 |
+
"Content-Type": row["mime"] or "application/octet-stream",
|
| 439 |
+
"Content-Length": str(len(content)),
|
| 440 |
+
}
|
| 441 |
+
return Response(content=content, status_code=200, headers=headers, media_type=row["mime"] or "application/octet-stream")
|
| 442 |
|
| 443 |
|
| 444 |
# ===== 删除 =====
|
|
|
|
| 460 |
raise HttpError("forbidden", status=403, code="forbidden")
|
| 461 |
conn.execute("DELETE FROM file_meta WHERE key = ?", (key,))
|
| 462 |
|
| 463 |
+
# 删文件内容 + 元信息(本地 + Hub)
|
| 464 |
try:
|
| 465 |
+
user_files_store.delete_user_file_content(user_id, key)
|
| 466 |
+
except Exception as e:
|
| 467 |
+
print(f"[user_files] delete_user_file_content failed: {e}")
|
| 468 |
+
try:
|
| 469 |
+
user_files_store.delete_file_meta(user_id, key)
|
| 470 |
+
except Exception as e:
|
| 471 |
+
print(f"[user_files] delete_file_meta failed: {e}")
|
| 472 |
|
| 473 |
return ok_with_cors({"deleted": True, "key": key})
|
app/hf_storage.py
CHANGED
|
@@ -105,6 +105,49 @@ def download_bytes(namespace: str, key: str) -> Optional[bytes]:
|
|
| 105 |
return None
|
| 106 |
|
| 107 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 108 |
# ===== 异步接口(推送 Hub 用)=====
|
| 109 |
|
| 110 |
async def push_to_hub(namespace: str, key: str) -> bool:
|
|
@@ -171,3 +214,47 @@ def _hub_download(namespace: str, key: str) -> Optional[bytes]:
|
|
| 171 |
return None
|
| 172 |
print(f"[hf_storage] hub_download failed: {e}")
|
| 173 |
return None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 105 |
return None
|
| 106 |
|
| 107 |
|
| 108 |
+
def delete_local(namespace: str, key: str) -> bool:
|
| 109 |
+
"""仅删本地缓存。返回是否删除了文件。"""
|
| 110 |
+
p = _local_path(namespace, key)
|
| 111 |
+
try:
|
| 112 |
+
if p.exists() and p.is_file():
|
| 113 |
+
p.unlink(missing_ok=True)
|
| 114 |
+
return True
|
| 115 |
+
except Exception:
|
| 116 |
+
pass
|
| 117 |
+
return False
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
def delete_from_hub(namespace: str, key: str) -> bool:
|
| 121 |
+
"""删除:本地缓存 + Hub 远端。Hub 失败不影响本地。"""
|
| 122 |
+
delete_local(namespace, key)
|
| 123 |
+
if not is_hub_enabled():
|
| 124 |
+
return True
|
| 125 |
+
try:
|
| 126 |
+
_hub_delete(namespace, key)
|
| 127 |
+
return True
|
| 128 |
+
except Exception as e:
|
| 129 |
+
print(f"[hf_storage] delete_from_hub failed ns={namespace} key={key}: {e}")
|
| 130 |
+
return False
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
def list_local_keys(namespace: str) -> list[str]:
|
| 134 |
+
"""列出某命名空间下本地缓存的所有 key(用于启动恢复)。"""
|
| 135 |
+
root = _local_root()
|
| 136 |
+
ns_parts = _safe_parts(namespace)
|
| 137 |
+
ns_dir = root.joinpath(*ns_parts) if ns_parts else root
|
| 138 |
+
if not ns_dir.exists() or not ns_dir.is_dir():
|
| 139 |
+
return []
|
| 140 |
+
out = []
|
| 141 |
+
for p in ns_dir.rglob("*"):
|
| 142 |
+
if p.is_file():
|
| 143 |
+
try:
|
| 144 |
+
rel = p.relative_to(ns_dir).as_posix()
|
| 145 |
+
out.append(rel)
|
| 146 |
+
except Exception:
|
| 147 |
+
continue
|
| 148 |
+
return out
|
| 149 |
+
|
| 150 |
+
|
| 151 |
# ===== 异步接口(推送 Hub 用)=====
|
| 152 |
|
| 153 |
async def push_to_hub(namespace: str, key: str) -> bool:
|
|
|
|
| 214 |
return None
|
| 215 |
print(f"[hf_storage] hub_download failed: {e}")
|
| 216 |
return None
|
| 217 |
+
|
| 218 |
+
|
| 219 |
+
def _hub_delete(namespace: str, key: str) -> None:
|
| 220 |
+
"""从 Hub 仓库删除一个文件(不存在视为成功)。"""
|
| 221 |
+
api, repo_id = _get_hf_api()
|
| 222 |
+
_ensure_repo(api, repo_id)
|
| 223 |
+
path_in_repo = _hub_path(namespace, key)
|
| 224 |
+
try:
|
| 225 |
+
api.delete_file(
|
| 226 |
+
path_in_repo=path_in_repo,
|
| 227 |
+
repo_id=repo_id,
|
| 228 |
+
repo_type="dataset",
|
| 229 |
+
)
|
| 230 |
+
except Exception as e:
|
| 231 |
+
msg = str(e).lower()
|
| 232 |
+
if "404" in msg or "not found" in msg or "no such file" in msg:
|
| 233 |
+
return
|
| 234 |
+
raise
|
| 235 |
+
|
| 236 |
+
|
| 237 |
+
def list_hub_keys(namespace: str) -> list[str]:
|
| 238 |
+
"""列出 Hub 仓库中某命名空间下的所有 key(同步,需在线程中调用)。
|
| 239 |
+
|
| 240 |
+
返回 key 列表(已去掉 namespace 前缀)。Hub 未启用或失败时返回空列表。
|
| 241 |
+
"""
|
| 242 |
+
if not is_hub_enabled():
|
| 243 |
+
return []
|
| 244 |
+
try:
|
| 245 |
+
api, repo_id = _get_hf_api()
|
| 246 |
+
_ensure_repo(api, repo_id)
|
| 247 |
+
files = api.list_repo_files(repo_id=repo_id, repo_type="dataset")
|
| 248 |
+
except Exception as e:
|
| 249 |
+
print(f"[hf_storage] list_hub_keys failed: {e}")
|
| 250 |
+
return []
|
| 251 |
+
ns_parts = _safe_parts(namespace)
|
| 252 |
+
prefix = "/".join(ns_parts) + "/" if ns_parts else ""
|
| 253 |
+
out = []
|
| 254 |
+
for f in files:
|
| 255 |
+
if not f.startswith(prefix):
|
| 256 |
+
continue
|
| 257 |
+
rel = f[len(prefix):]
|
| 258 |
+
if rel:
|
| 259 |
+
out.append(rel)
|
| 260 |
+
return out
|
app/main.py
CHANGED
|
@@ -6,6 +6,7 @@ import logging
|
|
| 6 |
from contextlib import asynccontextmanager
|
| 7 |
|
| 8 |
from fastapi import FastAPI, Request
|
|
|
|
| 9 |
from fastapi.middleware.cors import CORSMiddleware
|
| 10 |
from fastapi.responses import JSONResponse
|
| 11 |
|
|
@@ -13,8 +14,41 @@ from .config import get_settings
|
|
| 13 |
from .database import close_db, init_db
|
| 14 |
from .db_writer import start as start_db_writer, stop as stop_db_writer
|
| 15 |
from .errors import HttpError, http_error_handler, unhandled_exception_handler
|
|
|
|
| 16 |
from .http_client import close_http_client
|
| 17 |
-
from .services import config_store, pseudo_store
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 18 |
|
| 19 |
|
| 20 |
@asynccontextmanager
|
|
@@ -35,6 +69,21 @@ async def lifespan(app: FastAPI):
|
|
| 35 |
logging.getLogger(__name__).warning("config preload failed: %s", e)
|
| 36 |
# 启动伪流式清理任务
|
| 37 |
cleanup_task = asyncio.create_task(pseudo_store.cleanup_loop())
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 38 |
yield
|
| 39 |
# 关闭
|
| 40 |
cleanup_task.cancel()
|
|
@@ -42,6 +91,8 @@ async def lifespan(app: FastAPI):
|
|
| 42 |
await cleanup_task
|
| 43 |
except asyncio.CancelledError:
|
| 44 |
pass
|
|
|
|
|
|
|
| 45 |
await close_http_client()
|
| 46 |
stop_db_writer()
|
| 47 |
close_db()
|
|
@@ -78,6 +129,7 @@ def create_app() -> FastAPI:
|
|
| 78 |
|
| 79 |
# 异常处理
|
| 80 |
app.add_exception_handler(HttpError, http_error_handler)
|
|
|
|
| 81 |
app.add_exception_handler(Exception, unhandled_exception_handler)
|
| 82 |
|
| 83 |
# 路由注册
|
|
|
|
| 6 |
from contextlib import asynccontextmanager
|
| 7 |
|
| 8 |
from fastapi import FastAPI, Request
|
| 9 |
+
from fastapi.exceptions import RequestValidationError
|
| 10 |
from fastapi.middleware.cors import CORSMiddleware
|
| 11 |
from fastapi.responses import JSONResponse
|
| 12 |
|
|
|
|
| 14 |
from .database import close_db, init_db
|
| 15 |
from .db_writer import start as start_db_writer, stop as stop_db_writer
|
| 16 |
from .errors import HttpError, http_error_handler, unhandled_exception_handler
|
| 17 |
+
from .hf_storage import is_hub_enabled
|
| 18 |
from .http_client import close_http_client
|
| 19 |
+
from .services import config_store, pseudo_store, user_files_sync
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
async def validation_error_handler(request: Request, exc: RequestValidationError) -> JSONResponse:
|
| 23 |
+
"""把 FastAPI 422 校验错误转为统一格式,避免前端无法解析。"""
|
| 24 |
+
import json
|
| 25 |
+
details = []
|
| 26 |
+
for err in getattr(exc, "errors", lambda: [])():
|
| 27 |
+
details.append(err)
|
| 28 |
+
msg = "request validation failed"
|
| 29 |
+
if details:
|
| 30 |
+
try:
|
| 31 |
+
msg = json.dumps(details, ensure_ascii=False, default=str)
|
| 32 |
+
except Exception:
|
| 33 |
+
msg = str(details)
|
| 34 |
+
return JSONResponse(
|
| 35 |
+
status_code=422,
|
| 36 |
+
content={
|
| 37 |
+
"ok": False,
|
| 38 |
+
"error": {
|
| 39 |
+
"code": "unprocessable_entity",
|
| 40 |
+
"message": msg,
|
| 41 |
+
"status": 422,
|
| 42 |
+
"retryable": False,
|
| 43 |
+
"hint": "请求参数格式有误,请检查字段",
|
| 44 |
+
},
|
| 45 |
+
},
|
| 46 |
+
headers={
|
| 47 |
+
"Access-Control-Allow-Origin": "*",
|
| 48 |
+
"Access-Control-Allow-Methods": "*",
|
| 49 |
+
"Access-Control-Allow-Headers": "*",
|
| 50 |
+
},
|
| 51 |
+
)
|
| 52 |
|
| 53 |
|
| 54 |
@asynccontextmanager
|
|
|
|
| 69 |
logging.getLogger(__name__).warning("config preload failed: %s", e)
|
| 70 |
# 启动伪流式清理任务
|
| 71 |
cleanup_task = asyncio.create_task(pseudo_store.cleanup_loop())
|
| 72 |
+
# 启动用户文件 Hub 定时同步任务
|
| 73 |
+
s = get_settings()
|
| 74 |
+
if is_hub_enabled():
|
| 75 |
+
logging.getLogger(__name__).info(
|
| 76 |
+
"[startup] Hub sync enabled: repo=%s token=***%s",
|
| 77 |
+
s.hf_config_repo, s.hf_token[-4:] if s.hf_token else "(empty)",
|
| 78 |
+
)
|
| 79 |
+
sync_task = user_files_sync.start_sync_task()
|
| 80 |
+
else:
|
| 81 |
+
logging.getLogger(__name__).warning(
|
| 82 |
+
"[startup] Hub sync DISABLED: HF_TOKEN/HF_CONFIG_REPO not set, "
|
| 83 |
+
"user files will only persist locally (lost on Space rebuild). "
|
| 84 |
+
"Set HF_CONFIG_REPO=<your-dataset-repo> and HF_TOKEN=<your-token> to enable."
|
| 85 |
+
)
|
| 86 |
+
sync_task = None
|
| 87 |
yield
|
| 88 |
# 关闭
|
| 89 |
cleanup_task.cancel()
|
|
|
|
| 91 |
await cleanup_task
|
| 92 |
except asyncio.CancelledError:
|
| 93 |
pass
|
| 94 |
+
if sync_task is not None:
|
| 95 |
+
await user_files_sync.stop_sync_task()
|
| 96 |
await close_http_client()
|
| 97 |
stop_db_writer()
|
| 98 |
close_db()
|
|
|
|
| 129 |
|
| 130 |
# 异常处理
|
| 131 |
app.add_exception_handler(HttpError, http_error_handler)
|
| 132 |
+
app.add_exception_handler(RequestValidationError, validation_error_handler)
|
| 133 |
app.add_exception_handler(Exception, unhandled_exception_handler)
|
| 134 |
|
| 135 |
# 路由注册
|
app/services/user_files_store.py
ADDED
|
@@ -0,0 +1,139 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""用户文件与账号的 HF Hub 持久化备份。
|
| 2 |
+
|
| 3 |
+
与 config_store 同样的混合策略:
|
| 4 |
+
- SQLite / 本地磁盘为热存储
|
| 5 |
+
- HF Hub dataset 仓库做备份,解决 HF Spaces 免费档 ephemeral 问题
|
| 6 |
+
- 写入时先写本地,再异步推 Hub(Hub 失败不影响主流程)
|
| 7 |
+
- 读取时本地命中即返回;未命中从 Hub 拉取并写本地
|
| 8 |
+
- 未配置 HF_TOKEN / HF_CONFIG_REPO 时自动降级为纯本地
|
| 9 |
+
|
| 10 |
+
命名空间设计(按用户隔离):
|
| 11 |
+
- user_accounts 用户账号备份,key=username,value=JSON{id,username,password_hash,created_at}
|
| 12 |
+
- user_files/<user_id> 文件内容,key=file_key,value=bytes
|
| 13 |
+
- user_files_meta/<user_id> 文件元信息,key=file_key,value=JSON
|
| 14 |
+
"""
|
| 15 |
+
from __future__ import annotations
|
| 16 |
+
|
| 17 |
+
import asyncio
|
| 18 |
+
import json
|
| 19 |
+
from typing import Optional
|
| 20 |
+
|
| 21 |
+
from ..hf_storage import (
|
| 22 |
+
delete_from_hub,
|
| 23 |
+
download_bytes,
|
| 24 |
+
list_hub_keys,
|
| 25 |
+
push_to_hub,
|
| 26 |
+
upload_bytes,
|
| 27 |
+
)
|
| 28 |
+
|
| 29 |
+
NS_USER_ACCOUNTS = "user_accounts"
|
| 30 |
+
NS_USER_FILES_PREFIX = "user_files" # 实际: user_files/<user_id>
|
| 31 |
+
NS_USER_FILES_META_PREFIX = "user_files_meta" # 实际: user_files_meta/<user_id>
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def _files_ns(user_id: str) -> str:
|
| 35 |
+
return f"{NS_USER_FILES_PREFIX}/{user_id}"
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def _meta_ns(user_id: str) -> str:
|
| 39 |
+
return f"{NS_USER_FILES_META_PREFIX}/{user_id}"
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
# ===== 用户账号备份 =====
|
| 43 |
+
|
| 44 |
+
def backup_user_account(
|
| 45 |
+
*, user_id: int, username: str, password_hash: str, created_at: int
|
| 46 |
+
) -> None:
|
| 47 |
+
"""同步备份用户账号到 Hub 本地缓存(远端推送用 async 版)。"""
|
| 48 |
+
payload = json.dumps(
|
| 49 |
+
{
|
| 50 |
+
"id": user_id,
|
| 51 |
+
"username": username,
|
| 52 |
+
"password_hash": password_hash,
|
| 53 |
+
"created_at": created_at,
|
| 54 |
+
},
|
| 55 |
+
ensure_ascii=False,
|
| 56 |
+
).encode("utf-8")
|
| 57 |
+
upload_bytes(NS_USER_ACCOUNTS, username, payload)
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
async def push_user_account(username: str) -> bool:
|
| 61 |
+
"""异步推送用户账号到 Hub 远端。"""
|
| 62 |
+
return await push_to_hub(NS_USER_ACCOUNTS, username)
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def restore_user_account(username: str) -> Optional[dict]:
|
| 66 |
+
"""从 Hub(本地缓存优先)拉取用户账号。返回 dict 或 None。"""
|
| 67 |
+
data = download_bytes(NS_USER_ACCOUNTS, username)
|
| 68 |
+
if not data:
|
| 69 |
+
return None
|
| 70 |
+
try:
|
| 71 |
+
obj = json.loads(data.decode("utf-8"))
|
| 72 |
+
if isinstance(obj, dict) and obj.get("username") and obj.get("password_hash"):
|
| 73 |
+
return obj
|
| 74 |
+
except Exception as e:
|
| 75 |
+
print(f"[user_files_store] parse user_account failed: {e}")
|
| 76 |
+
return None
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
# ===== 文件内容 =====
|
| 80 |
+
|
| 81 |
+
def upload_user_file_content(user_id: str, key: str, content: bytes) -> None:
|
| 82 |
+
"""写文件内容到本地缓存(Hub 远端推送用 async 版)。"""
|
| 83 |
+
upload_bytes(_files_ns(user_id), key, content)
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
async def push_user_file_content(user_id: str, key: str) -> bool:
|
| 87 |
+
"""异步推送文件内容到 Hub 远端。"""
|
| 88 |
+
return await push_to_hub(_files_ns(user_id), key)
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
def download_user_file_content(user_id: str, key: str) -> Optional[bytes]:
|
| 92 |
+
"""下载文件内容:本地优先,Hub 回退。"""
|
| 93 |
+
return download_bytes(_files_ns(user_id), key)
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
def delete_user_file_content(user_id: str, key: str) -> bool:
|
| 97 |
+
"""删除文件内容:本地 + Hub。"""
|
| 98 |
+
return delete_from_hub(_files_ns(user_id), key)
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
# ===== 文件元信息 =====
|
| 102 |
+
|
| 103 |
+
def backup_file_meta(user_id: str, key: str, meta: dict) -> None:
|
| 104 |
+
"""同步备份文件元信息到本地缓存。"""
|
| 105 |
+
payload = json.dumps(meta, ensure_ascii=False).encode("utf-8")
|
| 106 |
+
upload_bytes(_meta_ns(user_id), key, payload)
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
async def push_file_meta(user_id: str, key: str) -> bool:
|
| 110 |
+
"""异步推送文件元信息到 Hub 远端。"""
|
| 111 |
+
return await push_to_hub(_meta_ns(user_id), key)
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
def delete_file_meta(user_id: str, key: str) -> bool:
|
| 115 |
+
"""删除文件元信息:本地 + Hub。"""
|
| 116 |
+
return delete_from_hub(_meta_ns(user_id), key)
|
| 117 |
+
|
| 118 |
+
|
| 119 |
+
async def restore_file_metas(user_id: str) -> list[dict]:
|
| 120 |
+
"""从 Hub 拉取该用户所有文件元信息(用于 Space 重建后恢复)。
|
| 121 |
+
|
| 122 |
+
流程:list_hub_keys 列出 meta namespace 下所有 key → 逐个 download。
|
| 123 |
+
"""
|
| 124 |
+
ns = _meta_ns(user_id)
|
| 125 |
+
keys = await asyncio.to_thread(list_hub_keys, ns)
|
| 126 |
+
if not keys:
|
| 127 |
+
return []
|
| 128 |
+
out: list[dict] = []
|
| 129 |
+
for k in keys:
|
| 130 |
+
data = download_bytes(ns, k)
|
| 131 |
+
if not data:
|
| 132 |
+
continue
|
| 133 |
+
try:
|
| 134 |
+
obj = json.loads(data.decode("utf-8"))
|
| 135 |
+
if isinstance(obj, dict) and obj.get("key"):
|
| 136 |
+
out.append(obj)
|
| 137 |
+
except Exception:
|
| 138 |
+
continue
|
| 139 |
+
return out
|
app/services/user_files_sync.py
ADDED
|
@@ -0,0 +1,146 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""用户文件 Hub 同步后台任务。
|
| 2 |
+
|
| 3 |
+
定时扫描本地缓存,把未同步到 Hub 的文件推上去。
|
| 4 |
+
作为 asyncio.create_task(push_to_hub) 的兜底:如果异步推送失败或进程崩溃,
|
| 5 |
+
下次定时任务会重新扫描并补推。
|
| 6 |
+
|
| 7 |
+
仅在 Hub 启用时实际工作(is_hub_enabled()=True)。
|
| 8 |
+
"""
|
| 9 |
+
from __future__ import annotations
|
| 10 |
+
|
| 11 |
+
import asyncio
|
| 12 |
+
import logging
|
| 13 |
+
from pathlib import Path
|
| 14 |
+
|
| 15 |
+
from ..hf_storage import (
|
| 16 |
+
_local_path,
|
| 17 |
+
_local_root,
|
| 18 |
+
is_hub_enabled,
|
| 19 |
+
list_hub_keys,
|
| 20 |
+
push_to_hub,
|
| 21 |
+
)
|
| 22 |
+
|
| 23 |
+
logger = logging.getLogger(__name__)
|
| 24 |
+
|
| 25 |
+
# 同步间隔(秒)
|
| 26 |
+
SYNC_INTERVAL_SEC = 300 # 5 分钟
|
| 27 |
+
|
| 28 |
+
# 需要定时扫描同步的 namespace 前缀
|
| 29 |
+
SYNC_NS_PREFIXES = ["user_accounts", "user_files/", "user_files_meta/"]
|
| 30 |
+
|
| 31 |
+
_sync_task: asyncio.Task | None = None
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def _collect_local_files() -> list[tuple[str, str]]:
|
| 35 |
+
"""扫描本地缓存,返回 (namespace, key) 列表。
|
| 36 |
+
|
| 37 |
+
namespace 从路径前缀推断:
|
| 38 |
+
- user_accounts/<username> → ns="user_accounts", key=<username>
|
| 39 |
+
- user_files/<user_id>/<file_key> → ns="user_files/<user_id>", key=<file_key>
|
| 40 |
+
- user_files_meta/<user_id>/<key> → ns="user_files_meta/<user_id>", key=<key>
|
| 41 |
+
"""
|
| 42 |
+
root = _local_root()
|
| 43 |
+
out: list[tuple[str, str]] = []
|
| 44 |
+
if not root.exists():
|
| 45 |
+
return out
|
| 46 |
+
|
| 47 |
+
for ns_prefix in SYNC_NS_PREFIXES:
|
| 48 |
+
parts = ns_prefix.rstrip("/").split("/")
|
| 49 |
+
ns_dir = root.joinpath(*parts)
|
| 50 |
+
if not ns_dir.exists() or not ns_dir.is_dir():
|
| 51 |
+
continue
|
| 52 |
+
for p in ns_dir.rglob("*"):
|
| 53 |
+
if not p.is_file():
|
| 54 |
+
continue
|
| 55 |
+
try:
|
| 56 |
+
rel = p.relative_to(ns_dir).as_posix()
|
| 57 |
+
if not rel:
|
| 58 |
+
continue
|
| 59 |
+
if ns_prefix == "user_accounts/":
|
| 60 |
+
ns = "user_accounts"
|
| 61 |
+
else:
|
| 62 |
+
ns = ns_prefix.rstrip("/")
|
| 63 |
+
out.append((ns, rel))
|
| 64 |
+
except Exception:
|
| 65 |
+
continue
|
| 66 |
+
return out
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
async def _sync_once() -> int:
|
| 70 |
+
"""执行一轮同步:扫描本地 → 对比 Hub → 补推缺失文件。返回补推数量。"""
|
| 71 |
+
if not is_hub_enabled():
|
| 72 |
+
return 0
|
| 73 |
+
|
| 74 |
+
local_files = await asyncio.to_thread(_collect_local_files)
|
| 75 |
+
if not local_files:
|
| 76 |
+
return 0
|
| 77 |
+
|
| 78 |
+
# 按 namespace 分组,批量 list_hub_keys 减少调用
|
| 79 |
+
ns_set: dict[str, list[str]] = {}
|
| 80 |
+
for ns, key in local_files:
|
| 81 |
+
ns_set.setdefault(ns, []).append(key)
|
| 82 |
+
|
| 83 |
+
pushed = 0
|
| 84 |
+
for ns, keys in ns_set.items():
|
| 85 |
+
try:
|
| 86 |
+
hub_keys = set(await asyncio.to_thread(list_hub_keys, ns))
|
| 87 |
+
except Exception as e:
|
| 88 |
+
logger.warning("[user_files_sync] list_hub_keys failed ns=%s: %s", ns, e)
|
| 89 |
+
continue
|
| 90 |
+
|
| 91 |
+
for key in keys:
|
| 92 |
+
if key in hub_keys:
|
| 93 |
+
continue
|
| 94 |
+
# 本地有但 Hub 没有,补推
|
| 95 |
+
try:
|
| 96 |
+
ok = await push_to_hub(ns, key)
|
| 97 |
+
if ok:
|
| 98 |
+
pushed += 1
|
| 99 |
+
logger.info("[user_files_sync] pushed ns=%s key=%s", ns, key)
|
| 100 |
+
except Exception as e:
|
| 101 |
+
logger.warning("[user_files_sync] push failed ns=%s key=%s: %s", ns, key, e)
|
| 102 |
+
|
| 103 |
+
if pushed > 0:
|
| 104 |
+
logger.info("[user_files_sync] sync round done, pushed %d files", pushed)
|
| 105 |
+
return pushed
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
async def sync_loop():
|
| 109 |
+
"""定时同步循环。"""
|
| 110 |
+
logger.info(
|
| 111 |
+
"[user_files_sync] started, interval=%ds, hub_enabled=%s",
|
| 112 |
+
SYNC_INTERVAL_SEC, is_hub_enabled(),
|
| 113 |
+
)
|
| 114 |
+
# 启动后先等一小段时间,让应用初始化完成
|
| 115 |
+
await asyncio.sleep(10)
|
| 116 |
+
while True:
|
| 117 |
+
try:
|
| 118 |
+
await _sync_once()
|
| 119 |
+
except Exception as e:
|
| 120 |
+
logger.warning("[user_files_sync] round failed: %s", e)
|
| 121 |
+
try:
|
| 122 |
+
await asyncio.sleep(SYNC_INTERVAL_SEC)
|
| 123 |
+
except asyncio.CancelledError:
|
| 124 |
+
break
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
def start_sync_task() -> asyncio.Task:
|
| 128 |
+
"""启动同步后台任务。"""
|
| 129 |
+
global _sync_task
|
| 130 |
+
if _sync_task is not None and not _sync_task.done():
|
| 131 |
+
return _sync_task
|
| 132 |
+
_sync_task = asyncio.create_task(sync_loop())
|
| 133 |
+
return _sync_task
|
| 134 |
+
|
| 135 |
+
|
| 136 |
+
async def stop_sync_task() -> None:
|
| 137 |
+
"""停止同步后台任务。"""
|
| 138 |
+
global _sync_task
|
| 139 |
+
if _sync_task is None:
|
| 140 |
+
return
|
| 141 |
+
_sync_task.cancel()
|
| 142 |
+
try:
|
| 143 |
+
await _sync_task
|
| 144 |
+
except asyncio.CancelledError:
|
| 145 |
+
pass
|
| 146 |
+
_sync_task = None
|