Upload db.py with huggingface_hub
Browse files
db.py
CHANGED
|
@@ -63,25 +63,66 @@ def schedule_auto_push():
|
|
| 63 |
|
| 64 |
|
| 65 |
def push_upload_to_repo(local_path, filename):
|
| 66 |
-
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 67 |
token = os.environ.get('HF_TOKEN', '')
|
| 68 |
username = os.environ.get('HF_USERNAME', '')
|
| 69 |
if not token or not username:
|
| 70 |
return
|
| 71 |
-
|
| 72 |
-
|
| 73 |
-
|
| 74 |
-
|
| 75 |
-
|
| 76 |
-
|
| 77 |
-
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
)
|
| 81 |
-
|
| 82 |
-
|
| 83 |
-
|
| 84 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 85 |
|
| 86 |
|
| 87 |
def _guarded_push():
|
|
@@ -696,3 +737,8 @@ def update_user(user_id, display_name=None, username=None, password=None, role=N
|
|
| 696 |
|
| 697 |
|
| 698 |
init_db()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 63 |
|
| 64 |
|
| 65 |
def push_upload_to_repo(local_path, filename):
|
| 66 |
+
"""同步把單一上傳 XLS 推到 repo 的 uploads/<filename>,
|
| 67 |
+
確保在 request 結束前完成(避免 daemon thread 被容器閒置/重啟中斷)。
|
| 68 |
+
失敗只記錄,不拋出(本地已存檔,可靠 startup 時再推)。"""
|
| 69 |
+
token = os.environ.get('HF_TOKEN', '')
|
| 70 |
+
username = os.environ.get('HF_USERNAME', '')
|
| 71 |
+
if not token or not username:
|
| 72 |
+
return False
|
| 73 |
+
try:
|
| 74 |
+
from huggingface_hub import HfApi
|
| 75 |
+
api = HfApi(token=token)
|
| 76 |
+
api.upload_file(
|
| 77 |
+
path_or_fileobj=str(local_path),
|
| 78 |
+
path_in_repo=f'uploads/{filename}',
|
| 79 |
+
repo_id=f'{username}/pradsa',
|
| 80 |
+
repo_type='space',
|
| 81 |
+
)
|
| 82 |
+
logger.info(f'Upload pushed to repo: uploads/{filename}')
|
| 83 |
+
return True
|
| 84 |
+
except Exception as e:
|
| 85 |
+
logger.warning(f'Upload push failed ({filename}): {e}')
|
| 86 |
+
return False
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
def pull_uploads_from_repo():
|
| 90 |
+
"""容器啟動時從 repo 拉 uploads/* 到本地 UPLOAD_DIR。
|
| 91 |
+
只下載本地沒有的檔案(以檔名判斷),避免重複下載。"""
|
| 92 |
token = os.environ.get('HF_TOKEN', '')
|
| 93 |
username = os.environ.get('HF_USERNAME', '')
|
| 94 |
if not token or not username:
|
| 95 |
return
|
| 96 |
+
try:
|
| 97 |
+
from huggingface_hub import HfApi
|
| 98 |
+
api = HfApi(token=token)
|
| 99 |
+
repo_id = f'{username}/pradsa'
|
| 100 |
+
UPLOAD_DIR.mkdir(exist_ok=True)
|
| 101 |
+
existing = {p.name for p in UPLOAD_DIR.iterdir() if p.is_file()}
|
| 102 |
+
files = api.list_repo_files(repo_id, repo_type='space')
|
| 103 |
+
pulled = 0
|
| 104 |
+
for rf in files:
|
| 105 |
+
if not rf.startswith('uploads/'):
|
| 106 |
+
continue
|
| 107 |
+
name = rf[len('uploads/'):]
|
| 108 |
+
if not name or name in existing:
|
| 109 |
+
continue
|
| 110 |
+
try:
|
| 111 |
+
local = api.hf_hub_download(
|
| 112 |
+
repo_id=repo_id, filename=rf, repo_type='space',
|
| 113 |
+
local_dir=str(UPLOAD_DIR.parent),
|
| 114 |
+
)
|
| 115 |
+
# hf_hub_download 會照 path_in_repo 結構存放;確保檔案在 UPLOAD_DIR
|
| 116 |
+
dest = UPLOAD_DIR / name
|
| 117 |
+
if not dest.exists():
|
| 118 |
+
shutil.copy2(local, dest)
|
| 119 |
+
pulled += 1
|
| 120 |
+
except Exception as e:
|
| 121 |
+
logger.warning(f'pull upload {name} failed: {e}')
|
| 122 |
+
if pulled:
|
| 123 |
+
logger.info(f'Pulled {pulled} upload(s) from repo')
|
| 124 |
+
except Exception as e:
|
| 125 |
+
logger.warning(f'pull_uploads_from_repo failed: {e}')
|
| 126 |
|
| 127 |
|
| 128 |
def _guarded_push():
|
|
|
|
| 737 |
|
| 738 |
|
| 739 |
init_db()
|
| 740 |
+
# 容器啟動時把 repo 上的 uploads/ 拉下來,確保 list_uploads() 能看到其他 session 上傳的檔案
|
| 741 |
+
try:
|
| 742 |
+
pull_uploads_from_repo()
|
| 743 |
+
except Exception as _e:
|
| 744 |
+
logger.warning(f'startup pull_uploads skipped: {_e}')
|