a3216 commited on
Commit
5b716a5
·
verified ·
1 Parent(s): 10300d9

sync from GitHub d79b7e2: feat: Implement user account management and file synchronization with HF Hub

Browse files
app/api/user_files.py CHANGED
@@ -10,62 +10,49 @@
10
  - GET /u/api/files/{key} 下载文件(仅 owner)
11
  - DELETE /u/api/files/{key} 删除文件(仅 owner)
12
 
13
- 持久化:
14
- - 文件元信息SQLite file_meta 表(/data/xtc.db,与现有 DB 同卷
15
- - 文件内容存于 /data/uploads/(同一持久卷,HF Spaces restart 不丢)
 
 
 
 
 
 
16
  """
17
  from __future__ import annotations
18
 
 
19
  import hashlib
 
20
  import re
21
  import time
 
22
  import uuid
23
- from pathlib import Path
24
 
25
  import bcrypt
26
  import jwt
27
- from fastapi import APIRouter, Body, Depends, File, Request, UploadFile
28
- from fastapi.responses import FileResponse
29
 
30
  from ..config import get_settings
31
  from ..database import get_conn
32
  from ..errors import HttpError
33
- from ._common import CORS_HEADERS, ok_with_cors
 
 
34
 
35
  router = APIRouter(prefix="/u/api", tags=["user-files"])
36
 
37
  ALGORITHM = "HS256"
38
  MAX_FILE_SIZE = 10 * 1024 * 1024 # 10MB
39
- NS_USER_FILES = "user_files"
40
  TOKEN_TTL_SEC = 7 * 24 * 3600 # 7 天
41
 
42
  _USERNAME_RE = re.compile(r"^[A-Za-z0-9_\u4e00-\u9fff.\-]{2,32}$")
43
 
44
-
45
- # ===== 文件存储路径 =====
46
-
47
- def _uploads_root() -> Path:
48
- """上传根目录:优先 /data/uploads,不可写回退 /tmp/uploads。"""
49
- for candidate in ("/data/uploads", "/tmp/uploads"):
50
- try:
51
- p = Path(candidate)
52
- p.mkdir(parents=True, exist_ok=True)
53
- test = p / ".wtest"
54
- test.touch(exist_ok=True)
55
- test.unlink(missing_ok=True)
56
- return p
57
- except Exception:
58
- continue
59
- p = Path("/tmp/uploads")
60
- p.mkdir(parents=True, exist_ok=True)
61
- return p
62
-
63
-
64
- def _file_path(key: str) -> Path:
65
- """根据 key 返回磁盘路径,做严格白名单校验防穿越。"""
66
- if not key or not re.fullmatch(r"[A-Za-z0-9_-]{8,64}", key):
67
- raise HttpError("invalid key", status=400, code="bad_request")
68
- return _uploads_root() / key
69
 
70
 
71
  # ===== JWT =====
@@ -134,10 +121,89 @@ def _verify_password(password: str, hashed: str) -> bool:
134
  return False
135
 
136
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
137
  # ===== 注册 =====
138
 
139
  @router.post("/register")
140
- async def register(body: dict = Body(default={})):
 
141
  username = str(body.get("username") or "").strip()
142
  password = str(body.get("password") or "")
143
  if not username or not password:
@@ -150,18 +216,39 @@ async def register(body: dict = Body(default={})):
150
  if len(password) < 6 or len(password) > 128:
151
  raise HttpError("password length must be 6-128", status=400, code="bad_request")
152
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
153
  password_hash = _hash_password(password)
154
  now = int(time.time())
155
  try:
156
- with get_conn() as conn:
157
- cur = conn.execute(
158
- "INSERT INTO user_accounts(username, password_hash, created_at) VALUES(?,?,?)",
159
- (username, password_hash, now),
160
- )
161
- user_id = cur.lastrowid
162
  except Exception:
163
  raise HttpError("username already exists", status=409, code="conflict")
164
 
 
 
 
 
 
 
 
 
 
 
165
  token = _issue_user_token(user_id, username)
166
  return ok_with_cors({
167
  "token": token,
@@ -172,24 +259,27 @@ async def register(body: dict = Body(default={})):
172
  # ===== 登录 =====
173
 
174
  @router.post("/login")
175
- async def login(body: dict = Body(default={})):
 
176
  username = str(body.get("username") or "").strip()
177
  password = str(body.get("password") or "")
178
  if not username or not password:
179
  raise HttpError("username and password required", status=400, code="bad_request")
180
 
181
- with get_conn() as conn:
182
- row = conn.execute(
183
- "SELECT id, username, password_hash FROM user_accounts WHERE username = ?",
184
- (username,),
185
- ).fetchone()
186
- if not row or not _verify_password(password, row["password_hash"]):
 
 
187
  raise HttpError("invalid username or password", status=401, code="unauthorized")
188
 
189
- token = _issue_user_token(row["id"], row["username"])
190
  return ok_with_cors({
191
  "token": token,
192
- "user": {"id": row["id"], "username": row["username"]},
193
  })
194
 
195
 
@@ -207,14 +297,12 @@ async def me(user: dict = Depends(require_user)):
207
  @router.get("/files")
208
  async def list_my_files(user: dict = Depends(require_user)):
209
  user_id = str(user["sub"])
210
- with get_conn() as conn:
211
- rows = conn.execute(
212
- "SELECT key, filename, mime, size, sha256, uploaded_at "
213
- "FROM file_meta WHERE uploaded_by = ? AND namespace = ? "
214
- "ORDER BY uploaded_at DESC",
215
- (user_id, NS_USER_FILES),
216
- ).fetchall()
217
- items = [dict(r) for r in rows]
218
  return ok_with_cors({"items": items, "count": len(items)})
219
 
220
 
@@ -239,7 +327,6 @@ async def upload_file(
239
 
240
  key = uuid.uuid4().hex
241
  filename = (file.filename or "unnamed").strip()
242
- # 截断过长文件名,避免存储异常
243
  if len(filename) > 255:
244
  filename = filename[:255]
245
  mime = (file.content_type or "application/octet-stream").strip()
@@ -247,11 +334,30 @@ async def upload_file(
247
  sha256 = hashlib.sha256(content).hexdigest()
248
  now = int(time.time())
249
 
250
- # 写磁盘
251
- p = _file_path(key)
252
- p.write_bytes(content)
 
 
 
 
 
 
 
 
253
 
254
- # 写元信息
 
 
 
 
 
 
 
 
 
 
 
255
  with get_conn() as conn:
256
  conn.execute(
257
  "INSERT INTO file_meta(key, namespace, filename, mime, size, sha256, "
@@ -260,6 +366,13 @@ async def upload_file(
260
  (key, NS_USER_FILES, filename, mime, size, sha256, now, user_id, username, ""),
261
  )
262
 
 
 
 
 
 
 
 
263
  return ok_with_cors({
264
  "key": key,
265
  "filename": filename,
@@ -283,20 +396,49 @@ async def download_file(
283
  "SELECT filename, mime, uploaded_by FROM file_meta WHERE key = ?",
284
  (key,),
285
  ).fetchone()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
286
  if not row:
287
  raise HttpError("file not found", status=404, code="not_found")
288
  if row["uploaded_by"] != user_id:
289
  raise HttpError("forbidden", status=403, code="forbidden")
290
 
291
- p = _file_path(key)
292
- if not p.exists():
293
- raise HttpError("file content missing on disk", status=410, code="gone")
294
-
295
- return FileResponse(
296
- path=str(p),
297
- media_type=row["mime"] or "application/octet-stream",
298
- filename=row["filename"],
299
- )
 
 
 
 
 
300
 
301
 
302
  # ===== 删除 =====
@@ -318,13 +460,14 @@ async def delete_file(
318
  raise HttpError("forbidden", status=403, code="forbidden")
319
  conn.execute("DELETE FROM file_meta WHERE key = ?", (key,))
320
 
321
- # 删磁盘文件
322
  try:
323
- p = _file_path(key)
324
- p.unlink(missing_ok=True)
325
- except HttpError:
326
- raise
327
- except Exception:
328
- pass
 
329
 
330
  return ok_with_cors({"deleted": True, "key": key})
 
10
  - GET /u/api/files/{key} 下载文件(仅 owner)
11
  - DELETE /u/api/files/{key} 删除文件(仅 owner)
12
 
13
+ 持久化(与 config_store 同样的混合策略)
14
+ - 储:SQLite(/data/xtc.db)+ 本地磁盘缓存(/data/hf_bucket/...
15
+ - 冷备份:HF Hub dataset 仓库,按用户隔离的 namespace
16
+ - user_accounts/<username> 用户账号 JSON
17
+ - user_files/<user_id>/<file_key> 文件内容
18
+ - user_files_meta/<user_id>/<file_key> 文件元信息 JSON
19
+ - 写入时先写本地,再异步推 Hub(Hub 失败不影响主流程)
20
+ - 读取时本地命中即返回;未命中从 Hub 拉取并写本地(用于 Space 重建后恢复)
21
+ - 未配置 HF_TOKEN / HF_CONFIG_REPO 时自动降级为纯本地
22
  """
23
  from __future__ import annotations
24
 
25
+ import asyncio
26
  import hashlib
27
+ import logging
28
  import re
29
  import time
30
+ import urllib.parse
31
  import uuid
32
+ from typing import Optional
33
 
34
  import bcrypt
35
  import jwt
36
+ from fastapi import APIRouter, Depends, File, Request, UploadFile
37
+ from fastapi.responses import Response
38
 
39
  from ..config import get_settings
40
  from ..database import get_conn
41
  from ..errors import HttpError
42
+ from ..hf_storage import is_hub_enabled
43
+ from ..services import user_files_store
44
+ from ._common import CORS_HEADERS, ok_with_cors, read_json_body
45
 
46
  router = APIRouter(prefix="/u/api", tags=["user-files"])
47
 
48
  ALGORITHM = "HS256"
49
  MAX_FILE_SIZE = 10 * 1024 * 1024 # 10MB
50
+ NS_USER_FILES = "user_files" # file_meta.namespace 字段值
51
  TOKEN_TTL_SEC = 7 * 24 * 3600 # 7 天
52
 
53
  _USERNAME_RE = re.compile(r"^[A-Za-z0-9_\u4e00-\u9fff.\-]{2,32}$")
54
 
55
+ logger = logging.getLogger(__name__)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
56
 
57
 
58
  # ===== JWT =====
 
121
  return False
122
 
123
 
124
+ # ===== 用户账号:SQLite + Hub 备份/恢复 =====
125
+
126
+ def _insert_user_to_db(username: str, password_hash: str, now: int) -> int:
127
+ with get_conn() as conn:
128
+ cur = conn.execute(
129
+ "INSERT INTO user_accounts(username, password_hash, created_at) VALUES(?,?,?)",
130
+ (username, password_hash, now),
131
+ )
132
+ return int(cur.lastrowid)
133
+
134
+
135
+ def _find_user_in_db(username: str) -> Optional[dict]:
136
+ with get_conn() as conn:
137
+ row = conn.execute(
138
+ "SELECT id, username, password_hash, created_at FROM user_accounts WHERE username = ?",
139
+ (username,),
140
+ ).fetchone()
141
+ return dict(row) if row else None
142
+
143
+
144
+ def _restore_user_from_hub(username: str) -> Optional[dict]:
145
+ """Space 重建后 SQLite 丢失,从 Hub 拉取账号并写回 SQLite。"""
146
+ obj = user_files_store.restore_user_account(username)
147
+ if not obj:
148
+ return None
149
+ try:
150
+ # 写回 SQLite(若已存在则忽略)
151
+ with get_conn() as conn:
152
+ conn.execute(
153
+ "INSERT OR IGNORE INTO user_accounts(id, username, password_hash, created_at) "
154
+ "VALUES(?,?,?,?)",
155
+ (int(obj["id"]), obj["username"], obj["password_hash"], int(obj["created_at"])),
156
+ )
157
+ return obj
158
+ except Exception as e:
159
+ print(f"[user_files] restore_user_from_hub write db failed: {e}")
160
+ return obj
161
+
162
+
163
+ # ===== 文件元信息:SQLite + Hub 备份/恢复 =====
164
+
165
+ def _list_files_from_db(user_id: str) -> list[dict]:
166
+ with get_conn() as conn:
167
+ rows = conn.execute(
168
+ "SELECT key, filename, mime, size, sha256, uploaded_at "
169
+ "FROM file_meta WHERE uploaded_by = ? AND namespace = ? "
170
+ "ORDER BY uploaded_at DESC",
171
+ (user_id, NS_USER_FILES),
172
+ ).fetchall()
173
+ return [dict(r) for r in rows]
174
+
175
+
176
+ async def _restore_files_from_hub_async(user_id: str) -> int:
177
+ metas = await user_files_store.restore_file_metas(user_id)
178
+ if not metas:
179
+ return 0
180
+ inserted = 0
181
+ with get_conn() as conn:
182
+ for m in metas:
183
+ try:
184
+ conn.execute(
185
+ "INSERT OR IGNORE INTO file_meta(key, namespace, filename, mime, size, "
186
+ "sha256, uploaded_at, uploaded_by, access_key, refs) "
187
+ "VALUES(?,?,?,?,?,?,?,?,?,?)",
188
+ (
189
+ m["key"], NS_USER_FILES, m.get("filename", "unnamed"),
190
+ m.get("mime", "application/octet-stream"),
191
+ int(m.get("size", 0)), m.get("sha256"),
192
+ int(m.get("uploaded_at", 0)), str(user_id),
193
+ m.get("access_key", ""), "",
194
+ ),
195
+ )
196
+ inserted += 1
197
+ except Exception as e:
198
+ print(f"[user_files] restore file_meta {m.get('key')} failed: {e}")
199
+ return inserted
200
+
201
+
202
  # ===== 注册 =====
203
 
204
  @router.post("/register")
205
+ async def register(request: Request):
206
+ body = await read_json_body(request)
207
  username = str(body.get("username") or "").strip()
208
  password = str(body.get("password") or "")
209
  if not username or not password:
 
216
  if len(password) < 6 or len(password) > 128:
217
  raise HttpError("password length must be 6-128", status=400, code="bad_request")
218
 
219
+ # 先查 Hub 是否已有该用户(SQLite 重建场景)
220
+ if not _find_user_in_db(username):
221
+ hub_user = user_files_store.restore_user_account(username)
222
+ if hub_user:
223
+ # 写回 SQLite,之后走"已存在"分支
224
+ try:
225
+ with get_conn() as conn:
226
+ conn.execute(
227
+ "INSERT OR IGNORE INTO user_accounts(id, username, password_hash, created_at) "
228
+ "VALUES(?,?,?,?)",
229
+ (int(hub_user["id"]), hub_user["username"],
230
+ hub_user["password_hash"], int(hub_user["created_at"])),
231
+ )
232
+ except Exception:
233
+ pass
234
+
235
  password_hash = _hash_password(password)
236
  now = int(time.time())
237
  try:
238
+ user_id = _insert_user_to_db(username, password_hash, now)
 
 
 
 
 
239
  except Exception:
240
  raise HttpError("username already exists", status=409, code="conflict")
241
 
242
+ # 备份到 Hub(本地缓存 + 异步推远端)
243
+ try:
244
+ user_files_store.backup_user_account(
245
+ user_id=user_id, username=username,
246
+ password_hash=password_hash, created_at=now,
247
+ )
248
+ asyncio.create_task(user_files_store.push_user_account(username))
249
+ except Exception as e:
250
+ print(f"[user_files] backup_user_account failed: {e}")
251
+
252
  token = _issue_user_token(user_id, username)
253
  return ok_with_cors({
254
  "token": token,
 
259
  # ===== 登录 =====
260
 
261
  @router.post("/login")
262
+ async def login(request: Request):
263
+ body = await read_json_body(request)
264
  username = str(body.get("username") or "").strip()
265
  password = str(body.get("password") or "")
266
  if not username or not password:
267
  raise HttpError("username and password required", status=400, code="bad_request")
268
 
269
+ user = _find_user_in_db(username)
270
+ # SQLite 没有则尝试从 Hub 恢复(Space 重建场景)
271
+ if not user:
272
+ restored = _restore_user_from_hub(username)
273
+ if restored:
274
+ user = _find_user_in_db(username) or restored
275
+
276
+ if not user or not _verify_password(password, user["password_hash"]):
277
  raise HttpError("invalid username or password", status=401, code="unauthorized")
278
 
279
+ token = _issue_user_token(int(user["id"]), user["username"])
280
  return ok_with_cors({
281
  "token": token,
282
+ "user": {"id": int(user["id"]), "username": user["username"]},
283
  })
284
 
285
 
 
297
  @router.get("/files")
298
  async def list_my_files(user: dict = Depends(require_user)):
299
  user_id = str(user["sub"])
300
+ items = _list_files_from_db(user_id)
301
+ # SQLite 为空时尝试从 Hub 恢复(Space 重建场景)
302
+ if not items:
303
+ restored = await _restore_files_from_hub_async(user_id)
304
+ if restored > 0:
305
+ items = _list_files_from_db(user_id)
 
 
306
  return ok_with_cors({"items": items, "count": len(items)})
307
 
308
 
 
327
 
328
  key = uuid.uuid4().hex
329
  filename = (file.filename or "unnamed").strip()
 
330
  if len(filename) > 255:
331
  filename = filename[:255]
332
  mime = (file.content_type or "application/octet-stream").strip()
 
334
  sha256 = hashlib.sha256(content).hexdigest()
335
  now = int(time.time())
336
 
337
+ # 写文件内容到本地缓存 + 异步推 Hub
338
+ try:
339
+ user_files_store.upload_user_file_content(user_id, key, content)
340
+ logger.info(
341
+ "[upload] user=%s uid=%s key=%s filename=%s size=%d saved locally, hub_enabled=%s",
342
+ username, user_id, key, filename, size, is_hub_enabled(),
343
+ )
344
+ asyncio.create_task(user_files_store.push_user_file_content(user_id, key))
345
+ except Exception as e:
346
+ logger.error("[upload] store failed user=%s filename=%s: %s", username, filename, e)
347
+ raise HttpError("failed to store file", status=500, code="internal_error") from e
348
 
349
+ # 写元信息到 SQLite
350
+ meta = {
351
+ "key": key,
352
+ "namespace": NS_USER_FILES,
353
+ "filename": filename,
354
+ "mime": mime,
355
+ "size": size,
356
+ "sha256": sha256,
357
+ "uploaded_at": now,
358
+ "uploaded_by": user_id,
359
+ "access_key": username,
360
+ }
361
  with get_conn() as conn:
362
  conn.execute(
363
  "INSERT INTO file_meta(key, namespace, filename, mime, size, sha256, "
 
366
  (key, NS_USER_FILES, filename, mime, size, sha256, now, user_id, username, ""),
367
  )
368
 
369
+ # 备份元信息到 Hub
370
+ try:
371
+ user_files_store.backup_file_meta(user_id, key, meta)
372
+ asyncio.create_task(user_files_store.push_file_meta(user_id, key))
373
+ except Exception as e:
374
+ print(f"[user_files] backup_file_meta failed: {e}")
375
+
376
  return ok_with_cors({
377
  "key": key,
378
  "filename": filename,
 
396
  "SELECT filename, mime, uploaded_by FROM file_meta WHERE key = ?",
397
  (key,),
398
  ).fetchone()
399
+ # SQLite 没有则尝试从 Hub 恢复单条 meta(Space 重建场景)
400
+ if not row:
401
+ metas = await user_files_store.restore_file_metas(user_id)
402
+ target = next((m for m in metas if m.get("key") == key), None)
403
+ if target:
404
+ try:
405
+ with get_conn() as conn:
406
+ conn.execute(
407
+ "INSERT OR IGNORE INTO file_meta(key, namespace, filename, mime, size, "
408
+ "sha256, uploaded_at, uploaded_by, access_key, refs) "
409
+ "VALUES(?,?,?,?,?,?,?,?,?,?)",
410
+ (target["key"], NS_USER_FILES, target.get("filename", "unnamed"),
411
+ target.get("mime", "application/octet-stream"),
412
+ int(target.get("size", 0)), target.get("sha256"),
413
+ int(target.get("uploaded_at", 0)), str(user_id),
414
+ target.get("access_key", ""), ""),
415
+ )
416
+ except Exception:
417
+ pass
418
+ with get_conn() as conn:
419
+ row = conn.execute(
420
+ "SELECT filename, mime, uploaded_by FROM file_meta WHERE key = ?",
421
+ (key,),
422
+ ).fetchone()
423
  if not row:
424
  raise HttpError("file not found", status=404, code="not_found")
425
  if row["uploaded_by"] != user_id:
426
  raise HttpError("forbidden", status=403, code="forbidden")
427
 
428
+ # 读文件内容:本地优先,Hub 回退
429
+ content = user_files_store.download_user_file_content(user_id, key)
430
+ if content is None:
431
+ raise HttpError("file content missing", status=410, code="gone")
432
+
433
+ # 用 Response 直接返回 bytes(避免依赖本地磁盘路径,兼容 Hub 回退场景)
434
+ encoded_name = urllib.parse.quote(row["filename"])
435
+ headers = {
436
+ **CORS_HEADERS,
437
+ "Content-Disposition": f"attachment; filename=\"{row['filename']}\"; filename*=UTF-8''{encoded_name}",
438
+ "Content-Type": row["mime"] or "application/octet-stream",
439
+ "Content-Length": str(len(content)),
440
+ }
441
+ return Response(content=content, status_code=200, headers=headers, media_type=row["mime"] or "application/octet-stream")
442
 
443
 
444
  # ===== 删除 =====
 
460
  raise HttpError("forbidden", status=403, code="forbidden")
461
  conn.execute("DELETE FROM file_meta WHERE key = ?", (key,))
462
 
463
+ # 删文件内容 + 元信息(本地 + Hub)
464
  try:
465
+ user_files_store.delete_user_file_content(user_id, key)
466
+ except Exception as e:
467
+ print(f"[user_files] delete_user_file_content failed: {e}")
468
+ try:
469
+ user_files_store.delete_file_meta(user_id, key)
470
+ except Exception as e:
471
+ print(f"[user_files] delete_file_meta failed: {e}")
472
 
473
  return ok_with_cors({"deleted": True, "key": key})
app/hf_storage.py CHANGED
@@ -105,6 +105,49 @@ def download_bytes(namespace: str, key: str) -> Optional[bytes]:
105
  return None
106
 
107
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
108
  # ===== 异步接口(推送 Hub 用)=====
109
 
110
  async def push_to_hub(namespace: str, key: str) -> bool:
@@ -171,3 +214,47 @@ def _hub_download(namespace: str, key: str) -> Optional[bytes]:
171
  return None
172
  print(f"[hf_storage] hub_download failed: {e}")
173
  return None
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
105
  return None
106
 
107
 
108
+ def delete_local(namespace: str, key: str) -> bool:
109
+ """仅删本地缓存。返回是否删除了文件。"""
110
+ p = _local_path(namespace, key)
111
+ try:
112
+ if p.exists() and p.is_file():
113
+ p.unlink(missing_ok=True)
114
+ return True
115
+ except Exception:
116
+ pass
117
+ return False
118
+
119
+
120
+ def delete_from_hub(namespace: str, key: str) -> bool:
121
+ """删除:本地缓存 + Hub 远端。Hub 失败不影响本地。"""
122
+ delete_local(namespace, key)
123
+ if not is_hub_enabled():
124
+ return True
125
+ try:
126
+ _hub_delete(namespace, key)
127
+ return True
128
+ except Exception as e:
129
+ print(f"[hf_storage] delete_from_hub failed ns={namespace} key={key}: {e}")
130
+ return False
131
+
132
+
133
+ def list_local_keys(namespace: str) -> list[str]:
134
+ """列出某命名空间下本地缓存的所有 key(用于启动恢复)。"""
135
+ root = _local_root()
136
+ ns_parts = _safe_parts(namespace)
137
+ ns_dir = root.joinpath(*ns_parts) if ns_parts else root
138
+ if not ns_dir.exists() or not ns_dir.is_dir():
139
+ return []
140
+ out = []
141
+ for p in ns_dir.rglob("*"):
142
+ if p.is_file():
143
+ try:
144
+ rel = p.relative_to(ns_dir).as_posix()
145
+ out.append(rel)
146
+ except Exception:
147
+ continue
148
+ return out
149
+
150
+
151
  # ===== 异步接口(推送 Hub 用)=====
152
 
153
  async def push_to_hub(namespace: str, key: str) -> bool:
 
214
  return None
215
  print(f"[hf_storage] hub_download failed: {e}")
216
  return None
217
+
218
+
219
+ def _hub_delete(namespace: str, key: str) -> None:
220
+ """从 Hub 仓库删除一个文件(不存在视为成功)。"""
221
+ api, repo_id = _get_hf_api()
222
+ _ensure_repo(api, repo_id)
223
+ path_in_repo = _hub_path(namespace, key)
224
+ try:
225
+ api.delete_file(
226
+ path_in_repo=path_in_repo,
227
+ repo_id=repo_id,
228
+ repo_type="dataset",
229
+ )
230
+ except Exception as e:
231
+ msg = str(e).lower()
232
+ if "404" in msg or "not found" in msg or "no such file" in msg:
233
+ return
234
+ raise
235
+
236
+
237
+ def list_hub_keys(namespace: str) -> list[str]:
238
+ """列出 Hub 仓库中某命名空间下的所有 key(同步,需在线程中调用)。
239
+
240
+ 返回 key 列表(已去掉 namespace 前缀)。Hub 未启用或失败时返回空列表。
241
+ """
242
+ if not is_hub_enabled():
243
+ return []
244
+ try:
245
+ api, repo_id = _get_hf_api()
246
+ _ensure_repo(api, repo_id)
247
+ files = api.list_repo_files(repo_id=repo_id, repo_type="dataset")
248
+ except Exception as e:
249
+ print(f"[hf_storage] list_hub_keys failed: {e}")
250
+ return []
251
+ ns_parts = _safe_parts(namespace)
252
+ prefix = "/".join(ns_parts) + "/" if ns_parts else ""
253
+ out = []
254
+ for f in files:
255
+ if not f.startswith(prefix):
256
+ continue
257
+ rel = f[len(prefix):]
258
+ if rel:
259
+ out.append(rel)
260
+ return out
app/main.py CHANGED
@@ -6,6 +6,7 @@ import logging
6
  from contextlib import asynccontextmanager
7
 
8
  from fastapi import FastAPI, Request
 
9
  from fastapi.middleware.cors import CORSMiddleware
10
  from fastapi.responses import JSONResponse
11
 
@@ -13,8 +14,41 @@ from .config import get_settings
13
  from .database import close_db, init_db
14
  from .db_writer import start as start_db_writer, stop as stop_db_writer
15
  from .errors import HttpError, http_error_handler, unhandled_exception_handler
 
16
  from .http_client import close_http_client
17
- from .services import config_store, pseudo_store
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
18
 
19
 
20
  @asynccontextmanager
@@ -35,6 +69,21 @@ async def lifespan(app: FastAPI):
35
  logging.getLogger(__name__).warning("config preload failed: %s", e)
36
  # 启动伪流式清理任务
37
  cleanup_task = asyncio.create_task(pseudo_store.cleanup_loop())
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
38
  yield
39
  # 关闭
40
  cleanup_task.cancel()
@@ -42,6 +91,8 @@ async def lifespan(app: FastAPI):
42
  await cleanup_task
43
  except asyncio.CancelledError:
44
  pass
 
 
45
  await close_http_client()
46
  stop_db_writer()
47
  close_db()
@@ -78,6 +129,7 @@ def create_app() -> FastAPI:
78
 
79
  # 异常处理
80
  app.add_exception_handler(HttpError, http_error_handler)
 
81
  app.add_exception_handler(Exception, unhandled_exception_handler)
82
 
83
  # 路由注册
 
6
  from contextlib import asynccontextmanager
7
 
8
  from fastapi import FastAPI, Request
9
+ from fastapi.exceptions import RequestValidationError
10
  from fastapi.middleware.cors import CORSMiddleware
11
  from fastapi.responses import JSONResponse
12
 
 
14
  from .database import close_db, init_db
15
  from .db_writer import start as start_db_writer, stop as stop_db_writer
16
  from .errors import HttpError, http_error_handler, unhandled_exception_handler
17
+ from .hf_storage import is_hub_enabled
18
  from .http_client import close_http_client
19
+ from .services import config_store, pseudo_store, user_files_sync
20
+
21
+
22
+ async def validation_error_handler(request: Request, exc: RequestValidationError) -> JSONResponse:
23
+ """把 FastAPI 422 校验错误转为统一格式,避免前端无法解析。"""
24
+ import json
25
+ details = []
26
+ for err in getattr(exc, "errors", lambda: [])():
27
+ details.append(err)
28
+ msg = "request validation failed"
29
+ if details:
30
+ try:
31
+ msg = json.dumps(details, ensure_ascii=False, default=str)
32
+ except Exception:
33
+ msg = str(details)
34
+ return JSONResponse(
35
+ status_code=422,
36
+ content={
37
+ "ok": False,
38
+ "error": {
39
+ "code": "unprocessable_entity",
40
+ "message": msg,
41
+ "status": 422,
42
+ "retryable": False,
43
+ "hint": "请求参数格式有误,请检查字段",
44
+ },
45
+ },
46
+ headers={
47
+ "Access-Control-Allow-Origin": "*",
48
+ "Access-Control-Allow-Methods": "*",
49
+ "Access-Control-Allow-Headers": "*",
50
+ },
51
+ )
52
 
53
 
54
  @asynccontextmanager
 
69
  logging.getLogger(__name__).warning("config preload failed: %s", e)
70
  # 启动伪流式清理任务
71
  cleanup_task = asyncio.create_task(pseudo_store.cleanup_loop())
72
+ # 启动用户文件 Hub 定时同步任务
73
+ s = get_settings()
74
+ if is_hub_enabled():
75
+ logging.getLogger(__name__).info(
76
+ "[startup] Hub sync enabled: repo=%s token=***%s",
77
+ s.hf_config_repo, s.hf_token[-4:] if s.hf_token else "(empty)",
78
+ )
79
+ sync_task = user_files_sync.start_sync_task()
80
+ else:
81
+ logging.getLogger(__name__).warning(
82
+ "[startup] Hub sync DISABLED: HF_TOKEN/HF_CONFIG_REPO not set, "
83
+ "user files will only persist locally (lost on Space rebuild). "
84
+ "Set HF_CONFIG_REPO=<your-dataset-repo> and HF_TOKEN=<your-token> to enable."
85
+ )
86
+ sync_task = None
87
  yield
88
  # 关闭
89
  cleanup_task.cancel()
 
91
  await cleanup_task
92
  except asyncio.CancelledError:
93
  pass
94
+ if sync_task is not None:
95
+ await user_files_sync.stop_sync_task()
96
  await close_http_client()
97
  stop_db_writer()
98
  close_db()
 
129
 
130
  # 异常处理
131
  app.add_exception_handler(HttpError, http_error_handler)
132
+ app.add_exception_handler(RequestValidationError, validation_error_handler)
133
  app.add_exception_handler(Exception, unhandled_exception_handler)
134
 
135
  # 路由注册
app/services/user_files_store.py ADDED
@@ -0,0 +1,139 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """用户文件与账号的 HF Hub 持久化备份。
2
+
3
+ 与 config_store 同样的混合策略:
4
+ - SQLite / 本地磁盘为热存储
5
+ - HF Hub dataset 仓库做备份,解决 HF Spaces 免费档 ephemeral 问题
6
+ - 写入时先写本地,再异步推 Hub(Hub 失败不影响主流程)
7
+ - 读取时本地命中即返回;未命中从 Hub 拉取并写本地
8
+ - 未配置 HF_TOKEN / HF_CONFIG_REPO 时自动降级为纯本地
9
+
10
+ 命名空间设计(按用户隔离):
11
+ - user_accounts 用户账号备份,key=username,value=JSON{id,username,password_hash,created_at}
12
+ - user_files/<user_id> 文件内容,key=file_key,value=bytes
13
+ - user_files_meta/<user_id> 文件元信息,key=file_key,value=JSON
14
+ """
15
+ from __future__ import annotations
16
+
17
+ import asyncio
18
+ import json
19
+ from typing import Optional
20
+
21
+ from ..hf_storage import (
22
+ delete_from_hub,
23
+ download_bytes,
24
+ list_hub_keys,
25
+ push_to_hub,
26
+ upload_bytes,
27
+ )
28
+
29
+ NS_USER_ACCOUNTS = "user_accounts"
30
+ NS_USER_FILES_PREFIX = "user_files" # 实际: user_files/<user_id>
31
+ NS_USER_FILES_META_PREFIX = "user_files_meta" # 实际: user_files_meta/<user_id>
32
+
33
+
34
+ def _files_ns(user_id: str) -> str:
35
+ return f"{NS_USER_FILES_PREFIX}/{user_id}"
36
+
37
+
38
+ def _meta_ns(user_id: str) -> str:
39
+ return f"{NS_USER_FILES_META_PREFIX}/{user_id}"
40
+
41
+
42
+ # ===== 用户账号备份 =====
43
+
44
+ def backup_user_account(
45
+ *, user_id: int, username: str, password_hash: str, created_at: int
46
+ ) -> None:
47
+ """同步备份用户账号到 Hub 本地缓存(远端推送用 async 版)。"""
48
+ payload = json.dumps(
49
+ {
50
+ "id": user_id,
51
+ "username": username,
52
+ "password_hash": password_hash,
53
+ "created_at": created_at,
54
+ },
55
+ ensure_ascii=False,
56
+ ).encode("utf-8")
57
+ upload_bytes(NS_USER_ACCOUNTS, username, payload)
58
+
59
+
60
+ async def push_user_account(username: str) -> bool:
61
+ """异步推送用户账号到 Hub 远端。"""
62
+ return await push_to_hub(NS_USER_ACCOUNTS, username)
63
+
64
+
65
+ def restore_user_account(username: str) -> Optional[dict]:
66
+ """从 Hub(本地缓存优先)拉取用户账号。返回 dict 或 None。"""
67
+ data = download_bytes(NS_USER_ACCOUNTS, username)
68
+ if not data:
69
+ return None
70
+ try:
71
+ obj = json.loads(data.decode("utf-8"))
72
+ if isinstance(obj, dict) and obj.get("username") and obj.get("password_hash"):
73
+ return obj
74
+ except Exception as e:
75
+ print(f"[user_files_store] parse user_account failed: {e}")
76
+ return None
77
+
78
+
79
+ # ===== 文件内容 =====
80
+
81
+ def upload_user_file_content(user_id: str, key: str, content: bytes) -> None:
82
+ """写文件内容到本地缓存(Hub 远端推送用 async 版)。"""
83
+ upload_bytes(_files_ns(user_id), key, content)
84
+
85
+
86
+ async def push_user_file_content(user_id: str, key: str) -> bool:
87
+ """异步推送文件内容到 Hub 远端。"""
88
+ return await push_to_hub(_files_ns(user_id), key)
89
+
90
+
91
+ def download_user_file_content(user_id: str, key: str) -> Optional[bytes]:
92
+ """下载文件内容:本地优先,Hub 回退。"""
93
+ return download_bytes(_files_ns(user_id), key)
94
+
95
+
96
+ def delete_user_file_content(user_id: str, key: str) -> bool:
97
+ """删除文件内容:本地 + Hub。"""
98
+ return delete_from_hub(_files_ns(user_id), key)
99
+
100
+
101
+ # ===== 文件元信息 =====
102
+
103
+ def backup_file_meta(user_id: str, key: str, meta: dict) -> None:
104
+ """同步备份文件元信息到本地缓存。"""
105
+ payload = json.dumps(meta, ensure_ascii=False).encode("utf-8")
106
+ upload_bytes(_meta_ns(user_id), key, payload)
107
+
108
+
109
+ async def push_file_meta(user_id: str, key: str) -> bool:
110
+ """异步推送文件元信息到 Hub 远端。"""
111
+ return await push_to_hub(_meta_ns(user_id), key)
112
+
113
+
114
+ def delete_file_meta(user_id: str, key: str) -> bool:
115
+ """删除文件元信息:本地 + Hub。"""
116
+ return delete_from_hub(_meta_ns(user_id), key)
117
+
118
+
119
+ async def restore_file_metas(user_id: str) -> list[dict]:
120
+ """从 Hub 拉取该用户所有文件元信息(用于 Space 重建后恢复)。
121
+
122
+ 流程:list_hub_keys 列出 meta namespace 下所有 key → 逐个 download。
123
+ """
124
+ ns = _meta_ns(user_id)
125
+ keys = await asyncio.to_thread(list_hub_keys, ns)
126
+ if not keys:
127
+ return []
128
+ out: list[dict] = []
129
+ for k in keys:
130
+ data = download_bytes(ns, k)
131
+ if not data:
132
+ continue
133
+ try:
134
+ obj = json.loads(data.decode("utf-8"))
135
+ if isinstance(obj, dict) and obj.get("key"):
136
+ out.append(obj)
137
+ except Exception:
138
+ continue
139
+ return out
app/services/user_files_sync.py ADDED
@@ -0,0 +1,146 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """用户文件 Hub 同步后台任务。
2
+
3
+ 定时扫描本地缓存,把未同步到 Hub 的文件推上去。
4
+ 作为 asyncio.create_task(push_to_hub) 的兜底:如果异步推送失败或进程崩溃,
5
+ 下次定时任务会重新扫描并补推。
6
+
7
+ 仅在 Hub 启用时实际工作(is_hub_enabled()=True)。
8
+ """
9
+ from __future__ import annotations
10
+
11
+ import asyncio
12
+ import logging
13
+ from pathlib import Path
14
+
15
+ from ..hf_storage import (
16
+ _local_path,
17
+ _local_root,
18
+ is_hub_enabled,
19
+ list_hub_keys,
20
+ push_to_hub,
21
+ )
22
+
23
+ logger = logging.getLogger(__name__)
24
+
25
+ # 同步间隔(秒)
26
+ SYNC_INTERVAL_SEC = 300 # 5 分钟
27
+
28
+ # 需要定时扫描同步的 namespace 前缀
29
+ SYNC_NS_PREFIXES = ["user_accounts", "user_files/", "user_files_meta/"]
30
+
31
+ _sync_task: asyncio.Task | None = None
32
+
33
+
34
+ def _collect_local_files() -> list[tuple[str, str]]:
35
+ """扫描本地缓存,返回 (namespace, key) 列表。
36
+
37
+ namespace 从路径前缀推断:
38
+ - user_accounts/<username> → ns="user_accounts", key=<username>
39
+ - user_files/<user_id>/<file_key> → ns="user_files/<user_id>", key=<file_key>
40
+ - user_files_meta/<user_id>/<key> → ns="user_files_meta/<user_id>", key=<key>
41
+ """
42
+ root = _local_root()
43
+ out: list[tuple[str, str]] = []
44
+ if not root.exists():
45
+ return out
46
+
47
+ for ns_prefix in SYNC_NS_PREFIXES:
48
+ parts = ns_prefix.rstrip("/").split("/")
49
+ ns_dir = root.joinpath(*parts)
50
+ if not ns_dir.exists() or not ns_dir.is_dir():
51
+ continue
52
+ for p in ns_dir.rglob("*"):
53
+ if not p.is_file():
54
+ continue
55
+ try:
56
+ rel = p.relative_to(ns_dir).as_posix()
57
+ if not rel:
58
+ continue
59
+ if ns_prefix == "user_accounts/":
60
+ ns = "user_accounts"
61
+ else:
62
+ ns = ns_prefix.rstrip("/")
63
+ out.append((ns, rel))
64
+ except Exception:
65
+ continue
66
+ return out
67
+
68
+
69
+ async def _sync_once() -> int:
70
+ """执行一轮同步:扫描本地 → 对比 Hub → 补推缺失文件。返回补推数量。"""
71
+ if not is_hub_enabled():
72
+ return 0
73
+
74
+ local_files = await asyncio.to_thread(_collect_local_files)
75
+ if not local_files:
76
+ return 0
77
+
78
+ # 按 namespace 分组,批量 list_hub_keys 减少调用
79
+ ns_set: dict[str, list[str]] = {}
80
+ for ns, key in local_files:
81
+ ns_set.setdefault(ns, []).append(key)
82
+
83
+ pushed = 0
84
+ for ns, keys in ns_set.items():
85
+ try:
86
+ hub_keys = set(await asyncio.to_thread(list_hub_keys, ns))
87
+ except Exception as e:
88
+ logger.warning("[user_files_sync] list_hub_keys failed ns=%s: %s", ns, e)
89
+ continue
90
+
91
+ for key in keys:
92
+ if key in hub_keys:
93
+ continue
94
+ # 本地有但 Hub 没有,补推
95
+ try:
96
+ ok = await push_to_hub(ns, key)
97
+ if ok:
98
+ pushed += 1
99
+ logger.info("[user_files_sync] pushed ns=%s key=%s", ns, key)
100
+ except Exception as e:
101
+ logger.warning("[user_files_sync] push failed ns=%s key=%s: %s", ns, key, e)
102
+
103
+ if pushed > 0:
104
+ logger.info("[user_files_sync] sync round done, pushed %d files", pushed)
105
+ return pushed
106
+
107
+
108
+ async def sync_loop():
109
+ """定时同步循环。"""
110
+ logger.info(
111
+ "[user_files_sync] started, interval=%ds, hub_enabled=%s",
112
+ SYNC_INTERVAL_SEC, is_hub_enabled(),
113
+ )
114
+ # 启动后先等一小段时间,让应用初始化完成
115
+ await asyncio.sleep(10)
116
+ while True:
117
+ try:
118
+ await _sync_once()
119
+ except Exception as e:
120
+ logger.warning("[user_files_sync] round failed: %s", e)
121
+ try:
122
+ await asyncio.sleep(SYNC_INTERVAL_SEC)
123
+ except asyncio.CancelledError:
124
+ break
125
+
126
+
127
+ def start_sync_task() -> asyncio.Task:
128
+ """启动同步后台任务。"""
129
+ global _sync_task
130
+ if _sync_task is not None and not _sync_task.done():
131
+ return _sync_task
132
+ _sync_task = asyncio.create_task(sync_loop())
133
+ return _sync_task
134
+
135
+
136
+ async def stop_sync_task() -> None:
137
+ """停止同步后台任务。"""
138
+ global _sync_task
139
+ if _sync_task is None:
140
+ return
141
+ _sync_task.cancel()
142
+ try:
143
+ await _sync_task
144
+ except asyncio.CancelledError:
145
+ pass
146
+ _sync_task = None