Deep-sea commited on
Commit
f9f89b0
·
verified ·
1 Parent(s): a587736

Update Dockerfile

Browse files
Files changed (1) hide show
  1. Dockerfile +28 -25
Dockerfile CHANGED
@@ -1,4 +1,4 @@
1
- FROM gitea/gitea:1.24.6 111
2
 
3
  # 以 root 用户设置权限(Spaces 允许构建时 root)
4
  USER root
@@ -114,21 +114,21 @@ def pull_from_hf_hub(repo_id, data_directory="/data"):
114
  os.system("ls -la /data/gitea/conf 2>/dev/null || echo '/data/gitea/conf 不存在'")
115
  os.system("ls -la /data/gitea/log 2>/dev/null || echo '/data/gitea/log 不存在'")
116
 
117
- logger.info(f"成功从 Hugging Face Hub 拉取数据到 {data_directory}")
118
  return True
119
 
120
  except Exception as e:
121
- logger.error(f"拉取 Hugging Face 数据集失败: {e}")
122
  return False
123
 
124
  if __name__ == "__main__":
125
  repo_id = os.getenv("REPO_ID", "02engine/02gitea")
126
  if not pull_from_hf_hub(repo_id):
127
- logger.error("拉取数据集失败,退出")
128
  exit(1)
129
  EOF
130
 
131
- # 复制上传脚本,移除检测到事件日志,修改日志为北京时间,添加崩溃重启功能
132
  COPY --chown=git:git <<'EOF' /uploadhf.py
133
  import os
134
  import time
@@ -141,6 +141,7 @@ from pathlib import Path
141
  from watchdog.observers import Observer
142
  from watchdog.events import FileSystemEventHandler
143
  from huggingface_hub import HfApi
 
144
  import base64
145
  from datetime import datetime
146
  import pytz
@@ -206,21 +207,23 @@ class DataDirectoryHandler(FileSystemEventHandler):
206
  try:
207
  commit_message = f"自动提交: {change_summary} - {time.strftime('%Y-%m-%d %H:%M:%S')}"
208
  logger.info(f"开始上传: {commit_message}")
209
- await asyncio.to_thread(self.api.upload_folder,
 
210
  folder_path=self.data_directory,
211
  repo_id=self.repo_id,
212
  repo_type="dataset",
213
  commit_message=commit_message,
214
- ignore_patterns=["*.tmp", "*.log", "*.temp", ".git/*"]
 
215
  )
216
- logger.info(f"成功提交变更到 Hugging Face Hub: {commit_message}")
217
  return
218
  except Exception as e:
219
- logger.error(f"提交失败 (尝试 {attempt + 1}/{max_retries}): {e}")
220
  if attempt < max_retries - 1:
221
  logger.info(f"将在 {retry_delay} 秒后重试...")
222
  await asyncio.sleep(retry_delay)
223
- logger.error(f"达到最大重试次数,上传失败")
224
 
225
  def start_directory_monitoring(data_directory="/data", repo_id=None, hf_token=None):
226
  """启动目录监控服务"""
@@ -246,7 +249,7 @@ def start_directory_monitoring(data_directory="/data", repo_id=None, hf_token=No
246
  observer = Observer()
247
  observer.schedule(event_handler, data_directory, recursive=True)
248
  observer.start()
249
- logger.info(f"🎯 目录监控服务已启动: {data_directory}")
250
 
251
  return observer
252
 
@@ -260,8 +263,8 @@ def start_gitea_server(port=7860):
260
  stderr=subprocess.PIPE,
261
  universal_newlines=True
262
  )
263
- logger.info(f"🚀 Gitea 服务器已启动,端口: {port}")
264
- logger.info(f"📊 访问地址: http://localhost:{port}")
265
 
266
  while True:
267
  output = gitea_process.stdout.readline()
@@ -273,15 +276,15 @@ def start_gitea_server(port=7860):
273
  return_code = gitea_process.poll()
274
  if return_code != 0:
275
  error_output = gitea_process.stderr.read()
276
- logger.error(f"Gitea 服务器异常退出,返回码: {return_code}")
277
  logger.error(f"错误信息: {error_output}")
278
  else:
279
- logger.info("Gitea 服务器正常退出")
280
 
281
  except FileNotFoundError:
282
- logger.error("未找到 gitea 命令,请确保 Gitea 已正确安装")
283
  except Exception as e:
284
- logger.error(f"启动 Gitea 服务器时发生错误: {e}")
285
 
286
  gitea_thread = threading.Thread(target=run_gitea)
287
  gitea_thread.daemon = True
@@ -304,13 +307,13 @@ async def main():
304
 
305
  while retry_count < max_retries:
306
  try:
307
- logger.info(f"🚀 启动集成服务 (尝试 {retry_count + 1}/{max_retries})...")
308
 
309
  # 先运行 pullhf.py 拉取最新数据集
310
  logger.info("运行 pullhf.py 拉取最新数据集")
311
  pull_result = subprocess.run(["python3", "/pullhf.py"], check=True)
312
  if pull_result.returncode != 0:
313
- logger.error("pullhf.py 执行失败,退出")
314
  exit(1)
315
 
316
  # 启动 Gitea 服务器
@@ -323,10 +326,10 @@ async def main():
323
  hf_token=CONFIG["hf_token"]
324
  )
325
 
326
- logger.info("所有服务已启动完成!")
327
- logger.info("📁 目录监控: /data Hugging Face Hub")
328
- logger.info("🌐 Gitea 服务: http://localhost:7860")
329
- logger.info("🛑 按 Ctrl+C 停止所有服务")
330
 
331
  try:
332
  while True:
@@ -337,12 +340,12 @@ async def main():
337
 
338
  except Exception as e:
339
  retry_count += 1
340
- logger.error(f"服务崩溃 (尝试 {retry_count}/{max_retries}): {e}")
341
  if retry_count < max_retries:
342
  logger.info(f"将在 5 秒后尝试重启...")
343
  await asyncio.sleep(5)
344
  else:
345
- logger.error(f"达到最大重试次数 ({max_retries}),退出")
346
  exit(1)
347
  finally:
348
  if 'observer' in locals():
 
1
+ FROM gitea/gitea:1.24.6
2
 
3
  # 以 root 用户设置权限(Spaces 允许构建时 root)
4
  USER root
 
114
  os.system("ls -la /data/gitea/conf 2>/dev/null || echo '/data/gitea/conf 不存在'")
115
  os.system("ls -la /data/gitea/log 2>/dev/null || echo '/data/gitea/log 不存在'")
116
 
117
+ logger.info(f"成功从 Hugging Face Hub 拉取数据到 {data_directory}")
118
  return True
119
 
120
  except Exception as e:
121
+ logger.error(f"拉取 Hugging Face 数据集失败: {e}")
122
  return False
123
 
124
  if __name__ == "__main__":
125
  repo_id = os.getenv("REPO_ID", "02engine/02gitea")
126
  if not pull_from_hf_hub(repo_id):
127
+ logger.error("拉取数据集失败,退出")
128
  exit(1)
129
  EOF
130
 
131
+ # 复制上传脚本(已禁用进度条)
132
  COPY --chown=git:git <<'EOF' /uploadhf.py
133
  import os
134
  import time
 
141
  from watchdog.observers import Observer
142
  from watchdog.events import FileSystemEventHandler
143
  from huggingface_hub import HfApi
144
+ from huggingface_hub.utils import tqdm_utils # 用于禁用进度条
145
  import base64
146
  from datetime import datetime
147
  import pytz
 
207
  try:
208
  commit_message = f"自动提交: {change_summary} - {time.strftime('%Y-%m-%d %H:%M:%S')}"
209
  logger.info(f"开始上传: {commit_message}")
210
+ await asyncio.to_thread(
211
+ self.api.upload_folder,
212
  folder_path=self.data_directory,
213
  repo_id=self.repo_id,
214
  repo_type="dataset",
215
  commit_message=commit_message,
216
+ ignore_patterns=["*.tmp", "*.log", "*.temp", ".git/*"],
217
+ tqdm_class=tqdm_utils.DisableTqdm # 禁用进度条
218
  )
219
+ logger.info(f"成功提交变更到 Hugging Face Hub: {commit_message}")
220
  return
221
  except Exception as e:
222
+ logger.error(f"提交失败 (尝试 {attempt + 1}/{max_retries}): {e}")
223
  if attempt < max_retries - 1:
224
  logger.info(f"将在 {retry_delay} 秒后重试...")
225
  await asyncio.sleep(retry_delay)
226
+ logger.error(f"达到最大重试次数,上传失败")
227
 
228
  def start_directory_monitoring(data_directory="/data", repo_id=None, hf_token=None):
229
  """启动目录监控服务"""
 
249
  observer = Observer()
250
  observer.schedule(event_handler, data_directory, recursive=True)
251
  observer.start()
252
+ logger.info(f"目录监控服务已启动: {data_directory}")
253
 
254
  return observer
255
 
 
263
  stderr=subprocess.PIPE,
264
  universal_newlines=True
265
  )
266
+ logger.info(f"Gitea 服务器已启动,端口: {port}")
267
+ logger.info(f"访问地址: http://localhost:{port}")
268
 
269
  while True:
270
  output = gitea_process.stdout.readline()
 
276
  return_code = gitea_process.poll()
277
  if return_code != 0:
278
  error_output = gitea_process.stderr.read()
279
+ logger.error(f"Gitea 服务器异常退出,返回码: {return_code}")
280
  logger.error(f"错误信息: {error_output}")
281
  else:
282
+ logger.info("Gitea 服务器正常退出")
283
 
284
  except FileNotFoundError:
285
+ logger.error("未找到 gitea 命令,请确保 Gitea 已正确安装")
286
  except Exception as e:
287
+ logger.error(f"启动 Gitea 服务器时发生错误: {e}")
288
 
289
  gitea_thread = threading.Thread(target=run_gitea)
290
  gitea_thread.daemon = True
 
307
 
308
  while retry_count < max_retries:
309
  try:
310
+ logger.info(f"启动集成服务 (尝试 {retry_count + 1}/{max_retries})...")
311
 
312
  # 先运行 pullhf.py 拉取最新数据集
313
  logger.info("运行 pullhf.py 拉取最新数据集")
314
  pull_result = subprocess.run(["python3", "/pullhf.py"], check=True)
315
  if pull_result.returncode != 0:
316
+ logger.error("pullhf.py 执行失败,退出")
317
  exit(1)
318
 
319
  # 启动 Gitea 服务器
 
326
  hf_token=CONFIG["hf_token"]
327
  )
328
 
329
+ logger.info("所有服务已启动完成!")
330
+ logger.info("目录监控: /data to Hugging Face Hub")
331
+ logger.info("Gitea 服务: http://localhost:7860")
332
+ logger.info("按 Ctrl+C 停止所有服务")
333
 
334
  try:
335
  while True:
 
340
 
341
  except Exception as e:
342
  retry_count += 1
343
+ logger.error(f"服务崩溃 (尝试 {retry_count}/{max_retries}): {e}")
344
  if retry_count < max_retries:
345
  logger.info(f"将在 5 秒后尝试重启...")
346
  await asyncio.sleep(5)
347
  else:
348
+ logger.error(f"达到最大重试次数 ({max_retries}),退出")
349
  exit(1)
350
  finally:
351
  if 'observer' in locals():