Deep-sea commited on
Commit
97e7588
·
verified ·
1 Parent(s): 068328b

Update Dockerfile

Browse files
Files changed (1) hide show
  1. Dockerfile +75 -111
Dockerfile CHANGED
@@ -1,14 +1,14 @@
1
  FROM gitea/gitea:1.24.6
2
 
3
- # 以 root 用户设置权限(Spaces 允许构建时使用 root)
4
  USER root
5
 
6
- # 安装 Python、pip、git、curl 和异步库
7
  RUN apk update && \
8
- apk add python3 py3-pip git curl && \
9
  pip3 install --break-system-packages --upgrade pip
10
 
11
- # 安装 Python 包,包括 pytz
12
  RUN pip3 install --break-system-packages watchdog huggingface_hub aiohttp pytz
13
 
14
  # 预创建 Gitea 所需目录并设置权限
@@ -23,14 +23,33 @@ import shutil
23
  from huggingface_hub import snapshot_download
24
  import logging
25
  import base64
 
 
26
 
27
- # 配置日志
 
28
  logging.basicConfig(
29
  level=logging.INFO,
30
- format='%(asctime)s - %(levelname)s - %(message)s'
 
31
  )
32
  logger = logging.getLogger(__name__)
33
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
34
  def pull_from_hf_hub(repo_id, data_directory="/data"):
35
  """从 Hugging Face Hub 数据集仓库拉取数据替换 /data 目录"""
36
  # 硬编码的 base64 编码的 HF_TOKEN
@@ -109,7 +128,7 @@ if __name__ == "__main__":
109
  exit(1)
110
  EOF
111
 
112
- # 复制上传脚本,恢复第一版错误抛出逻辑,修复卡在 starting 问题
113
  COPY --chown=git:git <<'EOF' /uploadhf.py
114
  import os
115
  import time
@@ -126,20 +145,30 @@ import base64
126
  from datetime import datetime
127
  import pytz
128
 
129
- # 配置日志,使用北京时间(CST,UTC+8)
130
  beijing_tz = pytz.timezone('Asia/Shanghai')
131
  logging.basicConfig(
132
  level=logging.INFO,
133
  format='%(asctime)s - %(levelname)s - %(message)s',
134
- datefmt='%Y-%m-%d %H:%M:%S %Z',
135
- handlers=[
136
- logging.StreamHandler()
137
- ]
138
  )
139
- # 设置日志的时区
140
- logging.Formatter.converter = lambda *args: datetime.now(beijing_tz).timetuple()
141
  logger = logging.getLogger(__name__)
142
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
143
  class DataDirectoryHandler(FileSystemEventHandler):
144
  """处理 /data 目录文件变化的监控器"""
145
 
@@ -151,7 +180,6 @@ class DataDirectoryHandler(FileSystemEventHandler):
151
  self.last_commit_time = 0
152
  self.commit_delay = 1 # 防抖延迟 1 秒
153
  self.pending_changes = [] # 缓冲待上传变更
154
- self.loop = asyncio.get_event_loop() # 获取主事件循环
155
  logger.info(f"初始化监控器,监控目录: {data_directory},目标仓库: {repo_id}")
156
 
157
  def on_any_event(self, event):
@@ -166,8 +194,7 @@ class DataDirectoryHandler(FileSystemEventHandler):
166
  current_time = time.time()
167
  if current_time - self.last_commit_time > self.commit_delay:
168
  self.last_commit_time = current_time
169
- # 在主事件循环中调度异步任务
170
- self.loop.call_soon_threadsafe(lambda: asyncio.create_task(self.commit_changes(change_type)))
171
 
172
  async def commit_changes(self, change_type):
173
  """异步提交变更到 Hugging Face Hub,带重试机制"""
@@ -177,7 +204,7 @@ class DataDirectoryHandler(FileSystemEventHandler):
177
  self.pending_changes = [] # 清空缓冲区
178
  for attempt in range(max_retries):
179
  try:
180
- commit_message = f"自动提交: {change_summary} - {datetime.now(beijing_tz).strftime('%Y-%m-%d %H:%M:%S')}"
181
  logger.info(f"开始上传: {commit_message}")
182
  await asyncio.to_thread(self.api.upload_folder,
183
  folder_path=self.data_directory,
@@ -204,8 +231,7 @@ def start_directory_monitoring(data_directory="/data", repo_id=None, hf_token=No
204
  logger.info(f"HF_TOKEN value: {hf_token}")
205
 
206
  if not repo_id:
207
- logger.error("必须提供 repo_id 参数")
208
- exit(1)
209
 
210
  if not os.path.exists(data_directory):
211
  logger.warning(f"监控目录 {data_directory} 不存在,正在创建...")
@@ -225,98 +251,37 @@ def start_directory_monitoring(data_directory="/data", repo_id=None, hf_token=No
225
  return observer
226
 
227
  def start_gitea_server(port=7860):
228
- """启动 Gitea 服务器,包含健康检查"""
229
  def run_gitea():
230
- max_restart_attempts = 5
231
- restart_delay = 10 # 每次重启前的等待时间(秒)
232
- health_check_timeout = 30 # 健康检查超时时间(秒)
233
- attempt = 0
234
-
235
- # 检查 /data 目录权限
236
- logger.info("检查 /data 目录权限")
237
- os.system("ls -la /data 2>/dev/null")
238
- os.system("ls -la /data/gitea 2>/dev/null || echo '/data/gitea 不存在'")
239
-
240
- while attempt < max_restart_attempts:
241
- try:
242
- # 启动 Gitea 进程
243
- gitea_process = subprocess.Popen(
244
- ['gitea', 'web', '--port', str(port)],
245
- stdout=subprocess.PIPE,
246
- stderr=subprocess.PIPE,
247
- universal_newlines=True
248
- )
249
- logger.info(f"🚀 Gitea 服务器启动尝试 {attempt + 1}/{max_restart_attempts},端口: {port}")
250
- logger.info(f"📊 访问地址: http://localhost:{port}")
251
-
252
- # 健康检查:等待 Gitea 响应
253
- start_time = time.time()
254
- health_check_passed = False
255
- while time.time() - start_time < health_check_timeout:
256
- try:
257
- result = subprocess.run(
258
- ['curl', '-s', '-f', f'http://localhost:{port}/'],
259
- capture_output=True, text=True, timeout=5
260
- )
261
- if result.returncode == 0:
262
- logger.info("✅ Gitea 健康检查通过")
263
- health_check_passed = True
264
- break
265
- except subprocess.SubprocessError:
266
- pass
267
- time.sleep(1)
268
-
269
- if not health_check_passed:
270
- logger.error("❌ Gitea 健康检查失败,服务未就绪")
271
- gitea_process.terminate()
272
- error_output = gitea_process.stderr.read()
273
- if error_output:
274
- logger.error(f"错误信息: {error_output.strip()}")
275
- attempt += 1
276
- if attempt < max_restart_attempts:
277
- logger.info(f"将在 {restart_delay} 秒后尝试重启 Gitea (尝试 {attempt + 1}/{max_restart_attempts})")
278
- time.sleep(restart_delay)
279
- else:
280
- logger.error(f"❌ 达到最大重试次数,Gitea 启动失败")
281
- exit(1)
282
- continue
283
-
284
- # 监控 Gitea 输出
285
- while True:
286
- output = gitea_process.stdout.readline()
287
- if output == '' and gitea_process.poll() is not None:
288
- break
289
- if output:
290
- logger.info(f"Gitea: {output.strip()}")
291
-
292
- return_code = gitea_process.poll()
293
- if return_code != 0:
294
- error_output = gitea_process.stderr.read()
295
- logger.error(f"❌ Gitea 服务器异常退出,返回码: {return_code}")
296
- logger.error(f"错误信息: {error_output}")
297
- attempt += 1
298
- if attempt < max_restart_attempts:
299
- logger.info(f"将在 {restart_delay} 秒后尝试重启 Gitea (尝试 {attempt + 1}/{max_restart_attempts})")
300
- time.sleep(restart_delay)
301
- else:
302
- logger.error(f"❌ 达到最大重试次数,Gitea 启动失败")
303
- exit(1)
304
- else:
305
- logger.info("✅ Gitea 服务器正常退出")
306
  break
 
 
 
 
 
 
 
 
 
 
307
 
308
- except FileNotFoundError:
309
- logger.error("❌ 未找到 gitea 命令,请确保 Gitea 已正确安装")
310
- exit(1)
311
- except Exception as e:
312
- logger.error(f"❌ 启动 Gitea 服务器时发生错误: {e}")
313
- attempt += 1
314
- if attempt < max_restart_attempts:
315
- logger.info(f"将在 {restart_delay} 秒后尝试重启 Gitea (尝试 {attempt + 1}/{max_restart_attempts})")
316
- time.sleep(restart_delay)
317
- else:
318
- logger.error(f"❌ 达到最大重试次数,Gitea 启动失败")
319
- exit(1)
320
 
321
  gitea_thread = threading.Thread(target=run_gitea)
322
  gitea_thread.daemon = True
@@ -356,7 +321,7 @@ async def main():
356
 
357
  logger.info("✅ 所有服务已启动完成!")
358
  logger.info("📁 目录监控: /data → Hugging Face Hub")
359
- logger.info(f"🌐 Gitea 服务: http://localhost:{CONFIG['gitea_port']}")
360
  logger.info("🛑 按 Ctrl+C 停止所有服务")
361
 
362
  try:
@@ -367,7 +332,6 @@ async def main():
367
 
368
  except Exception as e:
369
  logger.error(f"❌ 启动服务时发生错误: {e}")
370
- exit(1)
371
  finally:
372
  if 'observer' in locals():
373
  observer.stop()
 
1
  FROM gitea/gitea:1.24.6
2
 
3
+ # 以 root 用户设置权限(Spaces 允许构建时 root)
4
  USER root
5
 
6
+ # 安装 Python、pip、git 和异步库
7
  RUN apk update && \
8
+ apk add python3 py3-pip git && \
9
  pip3 install --break-system-packages --upgrade pip
10
 
11
+ # 安装 Python 包,包括 pytz 用于处理时区
12
  RUN pip3 install --break-system-packages watchdog huggingface_hub aiohttp pytz
13
 
14
  # 预创建 Gitea 所需目录并设置权限
 
23
  from huggingface_hub import snapshot_download
24
  import logging
25
  import base64
26
+ from datetime import datetime
27
+ import pytz
28
 
29
+ # 配置日志,使用北京时间
30
+ beijing_tz = pytz.timezone('Asia/Shanghai')
31
  logging.basicConfig(
32
  level=logging.INFO,
33
+ format='%(asctime)s - %(levelname)s - %(message)s',
34
+ datefmt='%Y-%m-%d %H:%M:%S %Z'
35
  )
36
  logger = logging.getLogger(__name__)
37
 
38
+ # 自定义日志格式化器以使用北京时间
39
+ class BeijingTimeFormatter(logging.Formatter):
40
+ def formatTime(self, record, datefmt=None):
41
+ dt = datetime.fromtimestamp(record.created, tz=beijing_tz)
42
+ if datefmt:
43
+ return dt.strftime(datefmt)
44
+ return dt.strftime("%Y-%m-%d %H:%M:%S %Z")
45
+
46
+ # 应用自定义格式化器
47
+ for handler in logging.getLogger().handlers:
48
+ handler.setFormatter(BeijingTimeFormatter(
49
+ fmt='%(asctime)s - %(levelname)s - %(message)s',
50
+ datefmt='%Y-%m-%d %H:%M:%S %Z'
51
+ ))
52
+
53
  def pull_from_hf_hub(repo_id, data_directory="/data"):
54
  """从 Hugging Face Hub 数据集仓库拉取数据替换 /data 目录"""
55
  # 硬编码的 base64 编码的 HF_TOKEN
 
128
  exit(1)
129
  EOF
130
 
131
+ # 复制上传脚本,移除检测到事件日志,修改日志为北京时间
132
  COPY --chown=git:git <<'EOF' /uploadhf.py
133
  import os
134
  import time
 
145
  from datetime import datetime
146
  import pytz
147
 
148
+ # 配置日志,使用北京时间
149
  beijing_tz = pytz.timezone('Asia/Shanghai')
150
  logging.basicConfig(
151
  level=logging.INFO,
152
  format='%(asctime)s - %(levelname)s - %(message)s',
153
+ datefmt='%Y-%m-%d %H:%M:%S %Z'
 
 
 
154
  )
 
 
155
  logger = logging.getLogger(__name__)
156
 
157
+ # 自定义日志格式化器以使用北京时间
158
+ class BeijingTimeFormatter(logging.Formatter):
159
+ def formatTime(self, record, datefmt=None):
160
+ dt = datetime.fromtimestamp(record.created, tz=beijing_tz)
161
+ if datefmt:
162
+ return dt.strftime(datefmt)
163
+ return dt.strftime("%Y-%m-%d %H:%M:%S %Z")
164
+
165
+ # 应用自定义格式化器
166
+ for handler in logging.getLogger().handlers:
167
+ handler.setFormatter(BeijingTimeFormatter(
168
+ fmt='%(asctime)s - %(levelname)s - %(message)s',
169
+ datefmt='%Y-%m-%d %H:%M:%S %Z'
170
+ ))
171
+
172
  class DataDirectoryHandler(FileSystemEventHandler):
173
  """处理 /data 目录文件变化的监控器"""
174
 
 
180
  self.last_commit_time = 0
181
  self.commit_delay = 1 # 防抖延迟 1 秒
182
  self.pending_changes = [] # 缓冲待上传变更
 
183
  logger.info(f"初始化监控器,监控目录: {data_directory},目标仓库: {repo_id}")
184
 
185
  def on_any_event(self, event):
 
194
  current_time = time.time()
195
  if current_time - self.last_commit_time > self.commit_delay:
196
  self.last_commit_time = current_time
197
+ asyncio.run(self.commit_changes(change_type))
 
198
 
199
  async def commit_changes(self, change_type):
200
  """异步提交变更到 Hugging Face Hub,带重试机制"""
 
204
  self.pending_changes = [] # 清空缓冲区
205
  for attempt in range(max_retries):
206
  try:
207
+ commit_message = f"自动提交: {change_summary} - {time.strftime('%Y-%m-%d %H:%M:%S')}"
208
  logger.info(f"开始上传: {commit_message}")
209
  await asyncio.to_thread(self.api.upload_folder,
210
  folder_path=self.data_directory,
 
231
  logger.info(f"HF_TOKEN value: {hf_token}")
232
 
233
  if not repo_id:
234
+ raise ValueError("必须提供 repo_id 参数")
 
235
 
236
  if not os.path.exists(data_directory):
237
  logger.warning(f"监控目录 {data_directory} 不存在,正在创建...")
 
251
  return observer
252
 
253
  def start_gitea_server(port=7860):
254
+ """启动 Gitea 服务器"""
255
  def run_gitea():
256
+ try:
257
+ gitea_process = subprocess.Popen(
258
+ ['gitea', 'web', '--port', str(port)],
259
+ stdout=subprocess.PIPE,
260
+ stderr=subprocess.PIPE,
261
+ universal_newlines=True
262
+ )
263
+ logger.info(f"🚀 Gitea 服务器已启动,端口: {port}")
264
+ logger.info(f"📊 访问地址: http://localhost:{port}")
265
+
266
+ while True:
267
+ output = gitea_process.stdout.readline()
268
+ if output == '' and gitea_process.poll() is not None:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
269
  break
270
+ if output:
271
+ logger.info(f"Gitea: {output.strip()}")
272
+
273
+ return_code = gitea_process.poll()
274
+ if return_code != 0:
275
+ error_output = gitea_process.stderr.read()
276
+ logger.error(f"❌ Gitea 服务器异常退出,返回码: {return_code}")
277
+ logger.error(f"错误信息: {error_output}")
278
+ else:
279
+ logger.info("✅ Gitea 服务器正常退出")
280
 
281
+ except FileNotFoundError:
282
+ logger.error("❌ 未找到 gitea 命令,请确保 Gitea 已正确安装")
283
+ except Exception as e:
284
+ logger.error(f"❌ 启动 Gitea 服务器时发生错误: {e}")
 
 
 
 
 
 
 
 
285
 
286
  gitea_thread = threading.Thread(target=run_gitea)
287
  gitea_thread.daemon = True
 
321
 
322
  logger.info("✅ 所有服务已启动完成!")
323
  logger.info("📁 目录监控: /data → Hugging Face Hub")
324
+ logger.info("🌐 Gitea 服务: http://localhost:7860")
325
  logger.info("🛑 按 Ctrl+C 停止所有服务")
326
 
327
  try:
 
332
 
333
  except Exception as e:
334
  logger.error(f"❌ 启动服务时发生错误: {e}")
 
335
  finally:
336
  if 'observer' in locals():
337
  observer.stop()