plokmii commited on
Commit
86fe0e2
·
verified ·
1 Parent(s): a5dd6de

Upload db.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. db.py +61 -15
db.py CHANGED
@@ -63,25 +63,66 @@ def schedule_auto_push():
63
 
64
 
65
  def push_upload_to_repo(local_path, filename):
66
- """背景把單一上傳 XLS 推到 repo 的 uploads/<filename>,讓檔案跨 factory reboot 保留。"""
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
67
  token = os.environ.get('HF_TOKEN', '')
68
  username = os.environ.get('HF_USERNAME', '')
69
  if not token or not username:
70
  return
71
- def _go():
72
- try:
73
- from huggingface_hub import HfApi
74
- api = HfApi(token=token)
75
- api.upload_file(
76
- path_or_fileobj=str(local_path),
77
- path_in_repo=f'uploads/{filename}',
78
- repo_id=f'{username}/pradsa',
79
- repo_type='space',
80
- )
81
- logger.info(f'Upload pushed to repo: uploads/{filename}')
82
- except Exception as e:
83
- logger.warning(f'Upload push failed: {e}')
84
- threading.Thread(target=_go, daemon=True).start()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
85
 
86
 
87
  def _guarded_push():
@@ -696,3 +737,8 @@ def update_user(user_id, display_name=None, username=None, password=None, role=N
696
 
697
 
698
  init_db()
 
 
 
 
 
 
63
 
64
 
65
  def push_upload_to_repo(local_path, filename):
66
+ """同步把單一上傳 XLS 推到 repo 的 uploads/<filename>,
67
+ 確保在 request 結束前完成(避免 daemon thread 被容器閒置/重啟中斷)。
68
+ 失敗只記錄,不拋出(本地已存檔,可靠 startup 時再推)。"""
69
+ token = os.environ.get('HF_TOKEN', '')
70
+ username = os.environ.get('HF_USERNAME', '')
71
+ if not token or not username:
72
+ return False
73
+ try:
74
+ from huggingface_hub import HfApi
75
+ api = HfApi(token=token)
76
+ api.upload_file(
77
+ path_or_fileobj=str(local_path),
78
+ path_in_repo=f'uploads/{filename}',
79
+ repo_id=f'{username}/pradsa',
80
+ repo_type='space',
81
+ )
82
+ logger.info(f'Upload pushed to repo: uploads/{filename}')
83
+ return True
84
+ except Exception as e:
85
+ logger.warning(f'Upload push failed ({filename}): {e}')
86
+ return False
87
+
88
+
89
+ def pull_uploads_from_repo():
90
+ """容器啟動時從 repo 拉 uploads/* 到本地 UPLOAD_DIR。
91
+ 只下載本地沒有的檔案(以檔名判斷),避免重複下載。"""
92
  token = os.environ.get('HF_TOKEN', '')
93
  username = os.environ.get('HF_USERNAME', '')
94
  if not token or not username:
95
  return
96
+ try:
97
+ from huggingface_hub import HfApi
98
+ api = HfApi(token=token)
99
+ repo_id = f'{username}/pradsa'
100
+ UPLOAD_DIR.mkdir(exist_ok=True)
101
+ existing = {p.name for p in UPLOAD_DIR.iterdir() if p.is_file()}
102
+ files = api.list_repo_files(repo_id, repo_type='space')
103
+ pulled = 0
104
+ for rf in files:
105
+ if not rf.startswith('uploads/'):
106
+ continue
107
+ name = rf[len('uploads/'):]
108
+ if not name or name in existing:
109
+ continue
110
+ try:
111
+ local = api.hf_hub_download(
112
+ repo_id=repo_id, filename=rf, repo_type='space',
113
+ local_dir=str(UPLOAD_DIR.parent),
114
+ )
115
+ # hf_hub_download 會照 path_in_repo 結構存放;確保檔案在 UPLOAD_DIR
116
+ dest = UPLOAD_DIR / name
117
+ if not dest.exists():
118
+ shutil.copy2(local, dest)
119
+ pulled += 1
120
+ except Exception as e:
121
+ logger.warning(f'pull upload {name} failed: {e}')
122
+ if pulled:
123
+ logger.info(f'Pulled {pulled} upload(s) from repo')
124
+ except Exception as e:
125
+ logger.warning(f'pull_uploads_from_repo failed: {e}')
126
 
127
 
128
  def _guarded_push():
 
737
 
738
 
739
  init_db()
740
+ # 容器啟動時把 repo 上的 uploads/ 拉下來,確保 list_uploads() 能看到其他 session 上傳的檔案
741
+ try:
742
+ pull_uploads_from_repo()
743
+ except Exception as _e:
744
+ logger.warning(f'startup pull_uploads skipped: {_e}')