Deep-sea commited on
Commit
e360c1d
·
verified ·
1 Parent(s): ca40643

Update Dockerfile

Browse files
Files changed (1) hide show
  1. Dockerfile +28 -70
Dockerfile CHANGED
@@ -109,7 +109,7 @@ if __name__ == "__main__":
109
  exit(1)
110
  EOF
111
 
112
- # 复制上传脚本,增强 Gitea 错误日志并在重试上限时抛出错误
113
  COPY --chown=git:git <<'EOF' /uploadhf.py
114
  import os
115
  import time
@@ -151,6 +151,7 @@ class DataDirectoryHandler(FileSystemEventHandler):
151
  self.last_commit_time = 0
152
  self.commit_delay = 1 # 防抖延迟 1 秒
153
  self.pending_changes = [] # 缓冲待上传变更
 
154
  logger.info(f"初始化监控器,监控目录: {data_directory},目标仓库: {repo_id}")
155
 
156
  def on_any_event(self, event):
@@ -165,8 +166,8 @@ class DataDirectoryHandler(FileSystemEventHandler):
165
  current_time = time.time()
166
  if current_time - self.last_commit_time > self.commit_delay:
167
  self.last_commit_time = current_time
168
- # 使用 asyncio.create_task 异步调度提交
169
- asyncio.create_task(self.commit_changes(change_type))
170
 
171
  async def commit_changes(self, change_type):
172
  """异步提交变更到 Hugging Face Hub,带重试机制"""
@@ -183,7 +184,7 @@ class DataDirectoryHandler(FileSystemEventHandler):
183
  repo_id=self.repo_id,
184
  repo_type="dataset",
185
  commit_message=commit_message,
186
- ignore_patterns=["*.tmp", "*._workers", "*.log", "*.temp", ".git/*"]
187
  )
188
  logger.info(f"✅ 成功提交变更到 Hugging Face Hub: {commit_message}")
189
  return
@@ -203,7 +204,8 @@ def start_directory_monitoring(data_directory="/data", repo_id=None, hf_token=No
203
  logger.info(f"HF_TOKEN value: {hf_token}")
204
 
205
  if not repo_id:
206
- raise ValueError("必须提供 repo_id 参数")
 
207
 
208
  if not os.path.exists(data_directory):
209
  logger.warning(f"监控目录 {data_directory} 不存在,正在创建...")
@@ -223,92 +225,49 @@ def start_directory_monitoring(data_directory="/data", repo_id=None, hf_token=No
223
  return observer
224
 
225
  def start_gitea_server(port=7860):
226
- """启动 Gitea 服务器,包含自动重试逻辑和超时机制,达到重试上限时抛出错误"""
227
  def run_gitea():
228
  max_restart_attempts = 5
229
  restart_delay = 10 # 每次重启前的等待时间(秒)
230
- startup_timeout = 30 # 启动超时时间(秒)
231
  attempt = 0
232
 
233
- # 检查 /data 目录权限和数据库状态
234
- logger.info("检查 /data 目录权限和 Gitea 数据库状态")
235
- os.system("ls -la /data 2>/dev/null")
236
- os.system("ls -la /data/gitea 2>/dev/null || echo '/data/gitea 不存在'")
237
- if os.path.exists("/data/gitea/gitea.db"):
238
- logger.info("Gitea 数据库文件存在: /data/gitea/gitea.db")
239
- else:
240
- logger.warning("Gitea 数据库文件不存在: /data/gitea/gitea.db")
241
-
242
  while attempt < max_restart_attempts:
243
  try:
244
- # 启动 Gitea 进程
245
  gitea_process = subprocess.Popen(
246
  ['gitea', 'web', '--port', str(port)],
247
  stdout=subprocess.PIPE,
248
  stderr=subprocess.PIPE,
249
  universal_newlines=True
250
  )
251
- logger.info(f"🚀 Gitea 服务器启动尝试 {attempt + 1}/{max_restart_attempts},端口: {port}")
252
  logger.info(f"📊 访问地址: http://localhost:{port}")
253
-
254
- # 等待 Gitea 启动(超时机制)
255
- start_time = time.time()
256
- while time.time() - start_time < startup_timeout:
257
- if gitea_process.poll() is not None:
258
  break
259
- line = gitea_process.stderr.readline()
260
- if line:
261
- logger.info(f"Gitea stderr: {line.strip()}")
262
- time.sleep(0.1)
263
-
264
- # 检查进程状态
265
  return_code = gitea_process.poll()
266
- if return_code is not None:
267
- # 进程已退出
268
  error_output = gitea_process.stderr.read()
269
  logger.error(f"❌ Gitea 服务器异常退出,返回码: {return_code}")
270
- if error_output:
271
- logger.error(f"错误信息: {error_output.strip()}")
272
  attempt += 1
273
  if attempt < max_restart_attempts:
274
  logger.info(f"将在 {restart_delay} 秒后尝试重启 Gitea (尝试 {attempt + 1}/{max_restart_attempts})")
275
  time.sleep(restart_delay)
276
  else:
277
- error_msg = f"❌ 达到最大重次数 ({max_restart_attempts}),Gitea 启动失败"
278
- logger.error(error_msg)
279
- raise RuntimeError(error_msg)
280
  else:
281
- # 进程仍在运行,开始读取输
282
- logger.info("✅ Gitea 服务器启动成功,开始监控输出")
283
- while True:
284
- output = gitea_process.stdout.readline()
285
- if output == '' and gitea_process.poll() is not None:
286
- break
287
- if output:
288
- logger.info(f"Gitea: {output.strip()}")
289
-
290
- return_code = gitea_process.poll()
291
- if return_code != 0:
292
- error_output = gitea_process.stderr.read()
293
- logger.error(f"❌ Gitea 服务器后期异常退出,返回码: {return_code}")
294
- if error_output:
295
- logger.error(f"错误信息: {error_output.strip()}")
296
- attempt += 1
297
- if attempt < max_restart_attempts:
298
- logger.info(f"将在 {restart_delay} 秒后尝试重启 Gitea (尝试 {attempt + 1}/{max_restart_attempts})")
299
- time.sleep(restart_delay)
300
- else:
301
- error_msg = f"❌ 达到最大重试次数 ({max_restart_attempts}),Gitea 启动失败"
302
- logger.error(error_msg)
303
- raise RuntimeError(error_msg)
304
- else:
305
- logger.info("✅ Gitea 服务器正常退出")
306
- break
307
 
308
  except FileNotFoundError:
309
- error_msg = "❌ 未找到 gitea 命令,请确保 Gitea 已正确安装"
310
- logger.error(error_msg)
311
- raise RuntimeError(error_msg)
312
  except Exception as e:
313
  logger.error(f"❌ 启动 Gitea 服务器时发生错误: {e}")
314
  attempt += 1
@@ -316,9 +275,8 @@ def start_gitea_server(port=7860):
316
  logger.info(f"将在 {restart_delay} 秒后尝试重启 Gitea (尝试 {attempt + 1}/{max_restart_attempts})")
317
  time.sleep(restart_delay)
318
  else:
319
- error_msg = f"❌ 达到最大重试次数 ({max_restart_attempts}),Gitea 启动失败: {e}"
320
- logger.error(error_msg)
321
- raise RuntimeError(error_msg)
322
 
323
  gitea_thread = threading.Thread(target=run_gitea)
324
  gitea_thread.daemon = True
@@ -358,7 +316,7 @@ async def main():
358
 
359
  logger.info("✅ 所有服务已启动完成!")
360
  logger.info("📁 目录监控: /data → Hugging Face Hub")
361
- logger.info(f"🌐 Gitea 服务: http://localhost:{CONFIG['gitea_port']}")
362
  logger.info("🛑 按 Ctrl+C 停止所有服务")
363
 
364
  try:
@@ -369,7 +327,7 @@ async def main():
369
 
370
  except Exception as e:
371
  logger.error(f"❌ 启动服务时发生错误: {e}")
372
- raise # 重新抛出异常以确保容器退出
373
  finally:
374
  if 'observer' in locals():
375
  observer.stop()
 
109
  exit(1)
110
  EOF
111
 
112
+ # 复制上传脚本,恢复第一版的错误抛出逻辑,保留北京时间和 Gitea 重试
113
  COPY --chown=git:git <<'EOF' /uploadhf.py
114
  import os
115
  import time
 
151
  self.last_commit_time = 0
152
  self.commit_delay = 1 # 防抖延迟 1 秒
153
  self.pending_changes = [] # 缓冲待上传变更
154
+ self.loop = asyncio.get_event_loop() # 获取主事件循环
155
  logger.info(f"初始化监控器,监控目录: {data_directory},目标仓库: {repo_id}")
156
 
157
  def on_any_event(self, event):
 
166
  current_time = time.time()
167
  if current_time - self.last_commit_time > self.commit_delay:
168
  self.last_commit_time = current_time
169
+ # 在主事件循环中调度异步任务
170
+ self.loop.call_soon_threadsafe(lambda: asyncio.create_task(self.commit_changes(change_type)))
171
 
172
  async def commit_changes(self, change_type):
173
  """异步提交变更到 Hugging Face Hub,带重试机制"""
 
184
  repo_id=self.repo_id,
185
  repo_type="dataset",
186
  commit_message=commit_message,
187
+ ignore_patterns=["*.tmp", "*.log", "*.temp", ".git/*"]
188
  )
189
  logger.info(f"✅ 成功提交变更到 Hugging Face Hub: {commit_message}")
190
  return
 
204
  logger.info(f"HF_TOKEN value: {hf_token}")
205
 
206
  if not repo_id:
207
+ logger.error("必须提供 repo_id 参数")
208
+ exit(1)
209
 
210
  if not os.path.exists(data_directory):
211
  logger.warning(f"监控目录 {data_directory} 不存在,正在创建...")
 
225
  return observer
226
 
227
  def start_gitea_server(port=7860):
228
+ """启动 Gitea 服务器"""
229
  def run_gitea():
230
  max_restart_attempts = 5
231
  restart_delay = 10 # 每次重启前的等待时间(秒)
 
232
  attempt = 0
233
 
 
 
 
 
 
 
 
 
 
234
  while attempt < max_restart_attempts:
235
  try:
 
236
  gitea_process = subprocess.Popen(
237
  ['gitea', 'web', '--port', str(port)],
238
  stdout=subprocess.PIPE,
239
  stderr=subprocess.PIPE,
240
  universal_newlines=True
241
  )
242
+ logger.info(f"🚀 Gitea 服务器启动,端口: {port}")
243
  logger.info(f"📊 访问地址: http://localhost:{port}")
244
+
245
+ while True:
246
+ output = gitea_process.stdout.readline()
247
+ if output == '' and gitea_process.poll() is not None:
 
248
  break
249
+ if output:
250
+ logger.info(f"Gitea: {output.strip()}")
251
+
 
 
 
252
  return_code = gitea_process.poll()
253
+ if return_code != 0:
 
254
  error_output = gitea_process.stderr.read()
255
  logger.error(f"❌ Gitea 服务器异常退出,返回码: {return_code}")
256
+ logger.error(f"错误信息: {error_output}")
 
257
  attempt += 1
258
  if attempt < max_restart_attempts:
259
  logger.info(f"将在 {restart_delay} 秒后尝试重启 Gitea (尝试 {attempt + 1}/{max_restart_attempts})")
260
  time.sleep(restart_delay)
261
  else:
262
+ logger.error(f"❌ 达到最大重次数,Gitea 启动失败")
263
+ exit(1)
 
264
  else:
265
+ logger.info("✅ Gitea 服务器正常退")
266
+ break
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
267
 
268
  except FileNotFoundError:
269
+ logger.error("❌ 未找到 gitea 命令,请确保 Gitea 已正确安装")
270
+ exit(1)
 
271
  except Exception as e:
272
  logger.error(f"❌ 启动 Gitea 服务器时发生错误: {e}")
273
  attempt += 1
 
275
  logger.info(f"将在 {restart_delay} 秒后尝试重启 Gitea (尝试 {attempt + 1}/{max_restart_attempts})")
276
  time.sleep(restart_delay)
277
  else:
278
+ logger.error(f"❌ 达到最大重试次数,Gitea 启动失败")
279
+ exit(1)
 
280
 
281
  gitea_thread = threading.Thread(target=run_gitea)
282
  gitea_thread.daemon = True
 
316
 
317
  logger.info("✅ 所有服务已启动完成!")
318
  logger.info("📁 目录监控: /data → Hugging Face Hub")
319
+ logger.info("🌐 Gitea 服务: http://localhost:7860")
320
  logger.info("🛑 按 Ctrl+C 停止所有服务")
321
 
322
  try:
 
327
 
328
  except Exception as e:
329
  logger.error(f"❌ 启动服务时发生错误: {e}")
330
+ exit(1)
331
  finally:
332
  if 'observer' in locals():
333
  observer.stop()