Deep-sea commited on
Commit
203c704
·
verified ·
1 Parent(s): 1ff82b3

Update Dockerfile

Browse files
Files changed (1) hide show
  1. Dockerfile +24 -45
Dockerfile CHANGED
@@ -140,7 +140,7 @@ from pathlib import Path
140
  from watchdog.observers import Observer
141
  from watchdog.events import FileSystemEventHandler
142
  from huggingface_hub import HfApi
143
- import huggingface_hub
144
  import base64
145
  from datetime import datetime
146
  import pytz
@@ -197,7 +197,7 @@ class DataDirectoryHandler(FileSystemEventHandler):
197
  asyncio.run(self.commit_changes(change_type))
198
 
199
  async def commit_changes(self, change_type):
200
- """异步提交变更到 Hugging Face Hub,带重试机制 + 版本兼容禁用进度条"""
201
  max_retries = 3
202
  retry_delay = 5
203
  change_summary = f"{change_type} ({len(self.pending_changes)} 文件)"
@@ -208,61 +208,39 @@ class DataDirectoryHandler(FileSystemEventHandler):
208
  commit_message = f"自动提交: {change_summary} - {time.strftime('%Y-%m-%d %H:%M:%S')}"
209
  logger.info(f"开始上传: {commit_message}")
210
 
211
- # 基础参数
212
- upload_kwargs = {
213
- "folder_path": self.data_directory,
214
- "repo_id": self.repo_id,
215
- "repo_type": "dataset",
216
- "commit_message": commit_message,
217
- "ignore_patterns": ["*.tmp", "*.log", "*.temp", ".git/*"]
218
- }
219
-
220
- # === 版本兼容:自动选择禁用进度条的方式 ===
221
- try:
222
- version_str = huggingface_hub.__version__
223
- version = tuple(map(int, (version_str.split('.') + ['0', '0', '0'])[:3]))
224
- logger.info(f"检测到 huggingface_hub 版本: {version_str}")
225
-
226
- if version >= (0, 25, 0):
227
- # 新版本:使用 disable_progress_bar
228
- upload_kwargs["disable_progress_bar"] = True
229
- logger.info("使用 disable_progress_bar=True 禁用进度条")
230
-
231
- elif version >= (0, 23, 0):
232
- # 旧版本:尝试使用 tqdm_class
233
- try:
234
- from huggingface_hub.utils import tqdm_utils
235
- upload_kwargs["tqdm_class"] = tqdm_utils.DisableTqdm
236
- logger.info("使用 tqdm_class=DisableTqdm 禁用进度条")
237
- except ImportError:
238
- logger.warning("无法导入 tqdm_utils,进度条可能仍会显示(旧版兼容性问题)")
239
-
240
- else:
241
- logger.warning(f"非常旧的 huggingface_hub 版本 {version_str},无法禁用进度条")
242
-
243
- except Exception as e:
244
- logger.warning(f"版本检测失败,尝试旧方式禁用进度条: {e}")
245
- try:
246
- from huggingface_hub.utils import tqdm_utils
247
- upload_kwargs["tqdm_class"] = tqdm_utils.DisableTqdm
248
- except:
249
- pass
250
-
251
- # 执行上传
252
- await asyncio.to_thread(self.api.upload_folder, **upload_kwargs)
253
  logger.info(f"成功提交变更到 Hugging Face Hub: {commit_message}")
254
  return
255
 
256
  except Exception as e:
 
 
257
  logger.error(f"提交失败 (尝试 {attempt + 1}/{max_retries}): {e}")
258
  if attempt < max_retries - 1:
259
  logger.info(f"将在 {retry_delay} 秒后重试...")
260
  await asyncio.sleep(retry_delay)
261
 
 
 
262
  logger.error(f"达到最大重试次数,上传失败")
263
 
264
  def start_directory_monitoring(data_directory="/data", repo_id=None, hf_token=None):
265
  """启动目录监控服务"""
 
266
  if not hf_token:
267
  hf_token_encoded = "aGZfcXllTEJnUUtPb2FUbHBMZ0FuTGFGTmJPV2xjUUtJT0VycQ=="
268
  hf_token = base64.b64decode(hf_token_encoded).decode('utf-8')
@@ -329,6 +307,7 @@ def start_gitea_server(port=7860):
329
 
330
  async def main():
331
  """主函数 - 启动 Gitea 和目录监控服务,带崩溃重���功能"""
 
332
  CONFIG = {
333
  "data_directory": "/data",
334
  "repo_id": os.getenv("REPO_ID", "02engine/02gitea"),
@@ -362,7 +341,7 @@ async def main():
362
 
363
  logger.info("所有服务已启动完成!")
364
  logger.info("目录监控: /data → Hugging Face Hub")
365
- logger.info("Gitea 服务: http://${HF_SPACE_URL:-localhost}:7860")
366
  logger.info("按 Ctrl+C 停止所有服务")
367
 
368
  try:
 
140
  from watchdog.observers import Observer
141
  from watchdog.events import FileSystemEventHandler
142
  from huggingface_hub import HfApi
143
+ from huggingface_hub.utils.tqdm_utils import disable_progress_bars, enable_progress_bars # 关键:全局禁用进度条
144
  import base64
145
  from datetime import datetime
146
  import pytz
 
197
  asyncio.run(self.commit_changes(change_type))
198
 
199
  async def commit_changes(self, change_type):
200
+ """异步提交变更到 Hugging Face Hub,带重试机制 + 全局禁用进度条"""
201
  max_retries = 3
202
  retry_delay = 5
203
  change_summary = f"{change_type} ({len(self.pending_changes)} 文件)"
 
208
  commit_message = f"自动提交: {change_summary} - {time.strftime('%Y-%m-%d %H:%M:%S')}"
209
  logger.info(f"开始上传: {commit_message}")
210
 
211
+ # 全局禁用所有 Hugging Face 进度条
212
+ disable_progress_bars()
213
+
214
+ await asyncio.to_thread(
215
+ self.api.upload_folder,
216
+ folder_path=self.data_directory,
217
+ repo_id=self.repo_id,
218
+ repo_type="dataset",
219
+ commit_message=commit_message,
220
+ ignore_patterns=["*.tmp", "*.log", "*.temp", ".git/*"]
221
+ )
222
+
223
+ # 恢复进度条(防止影响其他库)
224
+ enable_progress_bars()
225
+
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
226
  logger.info(f"成功提交变更到 Hugging Face Hub: {commit_message}")
227
  return
228
 
229
  except Exception as e:
230
+ # 出错时也恢复进度条
231
+ enable_progress_bars()
232
  logger.error(f"提交失败 (尝试 {attempt + 1}/{max_retries}): {e}")
233
  if attempt < max_retries - 1:
234
  logger.info(f"将在 {retry_delay} 秒后重试...")
235
  await asyncio.sleep(retry_delay)
236
 
237
+ # 最终失败也恢复
238
+ enable_progress_bars()
239
  logger.error(f"达到最大重试次数,上传失败")
240
 
241
  def start_directory_monitoring(data_directory="/data", repo_id=None, hf_token=None):
242
  """启动目录监控服务"""
243
+ # 硬编码的 base64 编码的 HF_TOKEN
244
  if not hf_token:
245
  hf_token_encoded = "aGZfcXllTEJnUUtPb2FUbHBMZ0FuTGFGTmJPV2xjUUtJT0VycQ=="
246
  hf_token = base64.b64decode(hf_token_encoded).decode('utf-8')
 
307
 
308
  async def main():
309
  """主函数 - 启动 Gitea 和目录监控服务,带崩溃重���功能"""
310
+ # 配置参数
311
  CONFIG = {
312
  "data_directory": "/data",
313
  "repo_id": os.getenv("REPO_ID", "02engine/02gitea"),
 
341
 
342
  logger.info("所有服务已启动完成!")
343
  logger.info("目录监控: /data → Hugging Face Hub")
344
+ logger.info("Gitea 服务: http://localhost:7860")
345
  logger.info("按 Ctrl+C 停止所有服务")
346
 
347
  try: