Update Dockerfile
Browse files- Dockerfile +24 -45
Dockerfile
CHANGED
|
@@ -140,7 +140,7 @@ from pathlib import Path
|
|
| 140 |
from watchdog.observers import Observer
|
| 141 |
from watchdog.events import FileSystemEventHandler
|
| 142 |
from huggingface_hub import HfApi
|
| 143 |
-
|
| 144 |
import base64
|
| 145 |
from datetime import datetime
|
| 146 |
import pytz
|
|
@@ -197,7 +197,7 @@ class DataDirectoryHandler(FileSystemEventHandler):
|
|
| 197 |
asyncio.run(self.commit_changes(change_type))
|
| 198 |
|
| 199 |
async def commit_changes(self, change_type):
|
| 200 |
-
"""异步提交变更到 Hugging Face Hub,带重试机制 +
|
| 201 |
max_retries = 3
|
| 202 |
retry_delay = 5
|
| 203 |
change_summary = f"{change_type} ({len(self.pending_changes)} 文件)"
|
|
@@ -208,61 +208,39 @@ class DataDirectoryHandler(FileSystemEventHandler):
|
|
| 208 |
commit_message = f"自动提交: {change_summary} - {time.strftime('%Y-%m-%d %H:%M:%S')}"
|
| 209 |
logger.info(f"开始上传: {commit_message}")
|
| 210 |
|
| 211 |
-
#
|
| 212 |
-
|
| 213 |
-
|
| 214 |
-
|
| 215 |
-
|
| 216 |
-
|
| 217 |
-
|
| 218 |
-
|
| 219 |
-
|
| 220 |
-
|
| 221 |
-
|
| 222 |
-
|
| 223 |
-
|
| 224 |
-
|
| 225 |
-
|
| 226 |
-
if version >= (0, 25, 0):
|
| 227 |
-
# 新版本:使用 disable_progress_bar
|
| 228 |
-
upload_kwargs["disable_progress_bar"] = True
|
| 229 |
-
logger.info("使用 disable_progress_bar=True 禁用进度条")
|
| 230 |
-
|
| 231 |
-
elif version >= (0, 23, 0):
|
| 232 |
-
# 旧版本:尝试使用 tqdm_class
|
| 233 |
-
try:
|
| 234 |
-
from huggingface_hub.utils import tqdm_utils
|
| 235 |
-
upload_kwargs["tqdm_class"] = tqdm_utils.DisableTqdm
|
| 236 |
-
logger.info("使用 tqdm_class=DisableTqdm 禁用进度条")
|
| 237 |
-
except ImportError:
|
| 238 |
-
logger.warning("无法导入 tqdm_utils,进度条可能仍会显示(旧版兼容性问题)")
|
| 239 |
-
|
| 240 |
-
else:
|
| 241 |
-
logger.warning(f"非常旧的 huggingface_hub 版本 {version_str},无法禁用进度条")
|
| 242 |
-
|
| 243 |
-
except Exception as e:
|
| 244 |
-
logger.warning(f"版本检测失败,尝试旧方式禁用进度条: {e}")
|
| 245 |
-
try:
|
| 246 |
-
from huggingface_hub.utils import tqdm_utils
|
| 247 |
-
upload_kwargs["tqdm_class"] = tqdm_utils.DisableTqdm
|
| 248 |
-
except:
|
| 249 |
-
pass
|
| 250 |
-
|
| 251 |
-
# 执行上传
|
| 252 |
-
await asyncio.to_thread(self.api.upload_folder, **upload_kwargs)
|
| 253 |
logger.info(f"成功提交变更到 Hugging Face Hub: {commit_message}")
|
| 254 |
return
|
| 255 |
|
| 256 |
except Exception as e:
|
|
|
|
|
|
|
| 257 |
logger.error(f"提交失败 (尝试 {attempt + 1}/{max_retries}): {e}")
|
| 258 |
if attempt < max_retries - 1:
|
| 259 |
logger.info(f"将在 {retry_delay} 秒后重试...")
|
| 260 |
await asyncio.sleep(retry_delay)
|
| 261 |
|
|
|
|
|
|
|
| 262 |
logger.error(f"达到最大重试次数,上传失败")
|
| 263 |
|
| 264 |
def start_directory_monitoring(data_directory="/data", repo_id=None, hf_token=None):
|
| 265 |
"""启动目录监控服务"""
|
|
|
|
| 266 |
if not hf_token:
|
| 267 |
hf_token_encoded = "aGZfcXllTEJnUUtPb2FUbHBMZ0FuTGFGTmJPV2xjUUtJT0VycQ=="
|
| 268 |
hf_token = base64.b64decode(hf_token_encoded).decode('utf-8')
|
|
@@ -329,6 +307,7 @@ def start_gitea_server(port=7860):
|
|
| 329 |
|
| 330 |
async def main():
|
| 331 |
"""主函数 - 启动 Gitea 和目录监控服务,带崩溃重���功能"""
|
|
|
|
| 332 |
CONFIG = {
|
| 333 |
"data_directory": "/data",
|
| 334 |
"repo_id": os.getenv("REPO_ID", "02engine/02gitea"),
|
|
@@ -362,7 +341,7 @@ async def main():
|
|
| 362 |
|
| 363 |
logger.info("所有服务已启动完成!")
|
| 364 |
logger.info("目录监控: /data → Hugging Face Hub")
|
| 365 |
-
logger.info("Gitea 服务: http://
|
| 366 |
logger.info("按 Ctrl+C 停止所有服务")
|
| 367 |
|
| 368 |
try:
|
|
|
|
| 140 |
from watchdog.observers import Observer
|
| 141 |
from watchdog.events import FileSystemEventHandler
|
| 142 |
from huggingface_hub import HfApi
|
| 143 |
+
from huggingface_hub.utils.tqdm_utils import disable_progress_bars, enable_progress_bars # 关键:全局禁用进度条
|
| 144 |
import base64
|
| 145 |
from datetime import datetime
|
| 146 |
import pytz
|
|
|
|
| 197 |
asyncio.run(self.commit_changes(change_type))
|
| 198 |
|
| 199 |
async def commit_changes(self, change_type):
|
| 200 |
+
"""异步提交变更到 Hugging Face Hub,带重试机制 + 全局禁用进度条"""
|
| 201 |
max_retries = 3
|
| 202 |
retry_delay = 5
|
| 203 |
change_summary = f"{change_type} ({len(self.pending_changes)} 文件)"
|
|
|
|
| 208 |
commit_message = f"自动提交: {change_summary} - {time.strftime('%Y-%m-%d %H:%M:%S')}"
|
| 209 |
logger.info(f"开始上传: {commit_message}")
|
| 210 |
|
| 211 |
+
# 全局禁用所有 Hugging Face 进度条
|
| 212 |
+
disable_progress_bars()
|
| 213 |
+
|
| 214 |
+
await asyncio.to_thread(
|
| 215 |
+
self.api.upload_folder,
|
| 216 |
+
folder_path=self.data_directory,
|
| 217 |
+
repo_id=self.repo_id,
|
| 218 |
+
repo_type="dataset",
|
| 219 |
+
commit_message=commit_message,
|
| 220 |
+
ignore_patterns=["*.tmp", "*.log", "*.temp", ".git/*"]
|
| 221 |
+
)
|
| 222 |
+
|
| 223 |
+
# 恢复进度条(防止影响其他库)
|
| 224 |
+
enable_progress_bars()
|
| 225 |
+
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 226 |
logger.info(f"成功提交变更到 Hugging Face Hub: {commit_message}")
|
| 227 |
return
|
| 228 |
|
| 229 |
except Exception as e:
|
| 230 |
+
# 出错时也恢复进度条
|
| 231 |
+
enable_progress_bars()
|
| 232 |
logger.error(f"提交失败 (尝试 {attempt + 1}/{max_retries}): {e}")
|
| 233 |
if attempt < max_retries - 1:
|
| 234 |
logger.info(f"将在 {retry_delay} 秒后重试...")
|
| 235 |
await asyncio.sleep(retry_delay)
|
| 236 |
|
| 237 |
+
# 最终失败也恢复
|
| 238 |
+
enable_progress_bars()
|
| 239 |
logger.error(f"达到最大重试次数,上传失败")
|
| 240 |
|
| 241 |
def start_directory_monitoring(data_directory="/data", repo_id=None, hf_token=None):
|
| 242 |
"""启动目录监控服务"""
|
| 243 |
+
# 硬编码的 base64 编码的 HF_TOKEN
|
| 244 |
if not hf_token:
|
| 245 |
hf_token_encoded = "aGZfcXllTEJnUUtPb2FUbHBMZ0FuTGFGTmJPV2xjUUtJT0VycQ=="
|
| 246 |
hf_token = base64.b64decode(hf_token_encoded).decode('utf-8')
|
|
|
|
| 307 |
|
| 308 |
async def main():
|
| 309 |
"""主函数 - 启动 Gitea 和目录监控服务,带崩溃重���功能"""
|
| 310 |
+
# 配置参数
|
| 311 |
CONFIG = {
|
| 312 |
"data_directory": "/data",
|
| 313 |
"repo_id": os.getenv("REPO_ID", "02engine/02gitea"),
|
|
|
|
| 341 |
|
| 342 |
logger.info("所有服务已启动完成!")
|
| 343 |
logger.info("目录监控: /data → Hugging Face Hub")
|
| 344 |
+
logger.info("Gitea 服务: http://localhost:7860")
|
| 345 |
logger.info("按 Ctrl+C 停止所有服务")
|
| 346 |
|
| 347 |
try:
|