Deep-sea commited on
Commit
16b49fc
·
verified ·
1 Parent(s): f9f89b0

Update Dockerfile

Browse files
Files changed (1) hide show
  1. Dockerfile +5 -18
Dockerfile CHANGED
@@ -128,7 +128,6 @@ if __name__ == "__main__":
128
  exit(1)
129
  EOF
130
 
131
- # 复制上传脚本(已禁用进度条)
132
  COPY --chown=git:git <<'EOF' /uploadhf.py
133
  import os
134
  import time
@@ -141,7 +140,7 @@ from pathlib import Path
141
  from watchdog.observers import Observer
142
  from watchdog.events import FileSystemEventHandler
143
  from huggingface_hub import HfApi
144
- from huggingface_hub.utils import tqdm_utils # 用于禁用进度条
145
  import base64
146
  from datetime import datetime
147
  import pytz
@@ -179,30 +178,27 @@ class DataDirectoryHandler(FileSystemEventHandler):
179
  self.data_directory = data_directory
180
  self.api = HfApi(token=hf_token)
181
  self.last_commit_time = 0
182
- self.commit_delay = 1 # 防抖延迟 1 秒
183
- self.pending_changes = [] # 缓冲待上传变更
184
  logger.info(f"初始化监控器,监控目录: {data_directory},目标仓库: {repo_id}")
185
 
186
  def on_any_event(self, event):
187
- """捕获所有文件系统事件"""
188
  if event.is_directory:
189
  return
190
  self.pending_changes.append((event.event_type, event.src_path))
191
  self.schedule_commit(f"文件{event.event_type}")
192
 
193
  def schedule_commit(self, change_type):
194
- """安排提交任务,带有防抖机制"""
195
  current_time = time.time()
196
  if current_time - self.last_commit_time > self.commit_delay:
197
  self.last_commit_time = current_time
198
  asyncio.run(self.commit_changes(change_type))
199
 
200
  async def commit_changes(self, change_type):
201
- """异步提交变更到 Hugging Face Hub,带重试机制"""
202
  max_retries = 3
203
  retry_delay = 5
204
  change_summary = f"{change_type} ({len(self.pending_changes)} 文件)"
205
- self.pending_changes = [] # 清空缓冲区
206
  for attempt in range(max_retries):
207
  try:
208
  commit_message = f"自动提交: {change_summary} - {time.strftime('%Y-%m-%d %H:%M:%S')}"
@@ -214,7 +210,7 @@ class DataDirectoryHandler(FileSystemEventHandler):
214
  repo_type="dataset",
215
  commit_message=commit_message,
216
  ignore_patterns=["*.tmp", "*.log", "*.temp", ".git/*"],
217
- tqdm_class=tqdm_utils.DisableTqdm # 禁用进度条
218
  )
219
  logger.info(f"成功提交变更到 Hugging Face Hub: {commit_message}")
220
  return
@@ -226,8 +222,6 @@ class DataDirectoryHandler(FileSystemEventHandler):
226
  logger.error(f"达到最大重试次数,上传失败")
227
 
228
  def start_directory_monitoring(data_directory="/data", repo_id=None, hf_token=None):
229
- """启动目录监控服务"""
230
- # 硬编码的 base64 编码的 HF_TOKEN
231
  if not hf_token:
232
  hf_token_encoded = "aGZfcXllTEJnUUtPb2FUbHBMZ0FuTGFGTmJPV2xjUUtJT0VycQ=="
233
  hf_token = base64.b64decode(hf_token_encoded).decode('utf-8')
@@ -254,7 +248,6 @@ def start_directory_monitoring(data_directory="/data", repo_id=None, hf_token=No
254
  return observer
255
 
256
  def start_gitea_server(port=7860):
257
- """启动 Gitea 服务器"""
258
  def run_gitea():
259
  try:
260
  gitea_process = subprocess.Popen(
@@ -293,8 +286,6 @@ def start_gitea_server(port=7860):
293
  return gitea_thread
294
 
295
  async def main():
296
- """主函数 - 启动 Gitea 和目录监控服务,带崩溃重启功能"""
297
- # 配置参数
298
  CONFIG = {
299
  "data_directory": "/data",
300
  "repo_id": os.getenv("REPO_ID", "02engine/02gitea"),
@@ -309,17 +300,13 @@ async def main():
309
  try:
310
  logger.info(f"启动集成服务 (尝试 {retry_count + 1}/{max_retries})...")
311
 
312
- # 先运行 pullhf.py 拉取最新数据集
313
  logger.info("运行 pullhf.py 拉取最新数据集")
314
  pull_result = subprocess.run(["python3", "/pullhf.py"], check=True)
315
  if pull_result.returncode != 0:
316
  logger.error("pullhf.py 执行失败,退出")
317
  exit(1)
318
 
319
- # 启动 Gitea 服务器
320
  gitea_thread = start_gitea_server(CONFIG["gitea_port"])
321
-
322
- # 启动目录监控服务
323
  observer = start_directory_monitoring(
324
  data_directory=CONFIG["data_directory"],
325
  repo_id=CONFIG["repo_id"],
 
128
  exit(1)
129
  EOF
130
 
 
131
  COPY --chown=git:git <<'EOF' /uploadhf.py
132
  import os
133
  import time
 
140
  from watchdog.observers import Observer
141
  from watchdog.events import FileSystemEventHandler
142
  from huggingface_hub import HfApi
143
+ # 删除:from huggingface_hub.utils import tqdm_utils
144
  import base64
145
  from datetime import datetime
146
  import pytz
 
178
  self.data_directory = data_directory
179
  self.api = HfApi(token=hf_token)
180
  self.last_commit_time = 0
181
+ self.commit_delay = 1
182
+ self.pending_changes = []
183
  logger.info(f"初始化监控器,监控目录: {data_directory},目标仓库: {repo_id}")
184
 
185
  def on_any_event(self, event):
 
186
  if event.is_directory:
187
  return
188
  self.pending_changes.append((event.event_type, event.src_path))
189
  self.schedule_commit(f"文件{event.event_type}")
190
 
191
  def schedule_commit(self, change_type):
 
192
  current_time = time.time()
193
  if current_time - self.last_commit_time > self.commit_delay:
194
  self.last_commit_time = current_time
195
  asyncio.run(self.commit_changes(change_type))
196
 
197
  async def commit_changes(self, change_type):
 
198
  max_retries = 3
199
  retry_delay = 5
200
  change_summary = f"{change_type} ({len(self.pending_changes)} 文件)"
201
+ self.pending_changes = []
202
  for attempt in range(max_retries):
203
  try:
204
  commit_message = f"自动提交: {change_summary} - {time.strftime('%Y-%m-%d %H:%M:%S')}"
 
210
  repo_type="dataset",
211
  commit_message=commit_message,
212
  ignore_patterns=["*.tmp", "*.log", "*.temp", ".git/*"],
213
+ disable_progress_bar=True # 正确禁用进度条
214
  )
215
  logger.info(f"成功提交变更到 Hugging Face Hub: {commit_message}")
216
  return
 
222
  logger.error(f"达到最大重试次数,上传失败")
223
 
224
  def start_directory_monitoring(data_directory="/data", repo_id=None, hf_token=None):
 
 
225
  if not hf_token:
226
  hf_token_encoded = "aGZfcXllTEJnUUtPb2FUbHBMZ0FuTGFGTmJPV2xjUUtJT0VycQ=="
227
  hf_token = base64.b64decode(hf_token_encoded).decode('utf-8')
 
248
  return observer
249
 
250
  def start_gitea_server(port=7860):
 
251
  def run_gitea():
252
  try:
253
  gitea_process = subprocess.Popen(
 
286
  return gitea_thread
287
 
288
  async def main():
 
 
289
  CONFIG = {
290
  "data_directory": "/data",
291
  "repo_id": os.getenv("REPO_ID", "02engine/02gitea"),
 
300
  try:
301
  logger.info(f"启动集成服务 (尝试 {retry_count + 1}/{max_retries})...")
302
 
 
303
  logger.info("运行 pullhf.py 拉取最新数据集")
304
  pull_result = subprocess.run(["python3", "/pullhf.py"], check=True)
305
  if pull_result.returncode != 0:
306
  logger.error("pullhf.py 执行失败,退出")
307
  exit(1)
308
 
 
309
  gitea_thread = start_gitea_server(CONFIG["gitea_port"])
 
 
310
  observer = start_directory_monitoring(
311
  data_directory=CONFIG["data_directory"],
312
  repo_id=CONFIG["repo_id"],