Update Dockerfile
Browse files- Dockerfile +28 -70
Dockerfile
CHANGED
|
@@ -109,7 +109,7 @@ if __name__ == "__main__":
|
|
| 109 |
exit(1)
|
| 110 |
EOF
|
| 111 |
|
| 112 |
-
# 复制上传脚本,
|
| 113 |
COPY --chown=git:git <<'EOF' /uploadhf.py
|
| 114 |
import os
|
| 115 |
import time
|
|
@@ -151,6 +151,7 @@ class DataDirectoryHandler(FileSystemEventHandler):
|
|
| 151 |
self.last_commit_time = 0
|
| 152 |
self.commit_delay = 1 # 防抖延迟 1 秒
|
| 153 |
self.pending_changes = [] # 缓冲待上传变更
|
|
|
|
| 154 |
logger.info(f"初始化监控器,监控目录: {data_directory},目标仓库: {repo_id}")
|
| 155 |
|
| 156 |
def on_any_event(self, event):
|
|
@@ -165,8 +166,8 @@ class DataDirectoryHandler(FileSystemEventHandler):
|
|
| 165 |
current_time = time.time()
|
| 166 |
if current_time - self.last_commit_time > self.commit_delay:
|
| 167 |
self.last_commit_time = current_time
|
| 168 |
-
#
|
| 169 |
-
asyncio.create_task(self.commit_changes(change_type))
|
| 170 |
|
| 171 |
async def commit_changes(self, change_type):
|
| 172 |
"""异步提交变更到 Hugging Face Hub,带重试机制"""
|
|
@@ -183,7 +184,7 @@ class DataDirectoryHandler(FileSystemEventHandler):
|
|
| 183 |
repo_id=self.repo_id,
|
| 184 |
repo_type="dataset",
|
| 185 |
commit_message=commit_message,
|
| 186 |
-
ignore_patterns=["*.tmp", "*.
|
| 187 |
)
|
| 188 |
logger.info(f"✅ 成功提交变更到 Hugging Face Hub: {commit_message}")
|
| 189 |
return
|
|
@@ -203,7 +204,8 @@ def start_directory_monitoring(data_directory="/data", repo_id=None, hf_token=No
|
|
| 203 |
logger.info(f"HF_TOKEN value: {hf_token}")
|
| 204 |
|
| 205 |
if not repo_id:
|
| 206 |
-
|
|
|
|
| 207 |
|
| 208 |
if not os.path.exists(data_directory):
|
| 209 |
logger.warning(f"监控目录 {data_directory} 不存在,正在创建...")
|
|
@@ -223,92 +225,49 @@ def start_directory_monitoring(data_directory="/data", repo_id=None, hf_token=No
|
|
| 223 |
return observer
|
| 224 |
|
| 225 |
def start_gitea_server(port=7860):
|
| 226 |
-
"""启动 Gitea 服务器
|
| 227 |
def run_gitea():
|
| 228 |
max_restart_attempts = 5
|
| 229 |
restart_delay = 10 # 每次重启前的等待时间(秒)
|
| 230 |
-
startup_timeout = 30 # 启动超时时间(秒)
|
| 231 |
attempt = 0
|
| 232 |
|
| 233 |
-
# 检查 /data 目录权限和数据库状态
|
| 234 |
-
logger.info("检查 /data 目录权限和 Gitea 数据库状态")
|
| 235 |
-
os.system("ls -la /data 2>/dev/null")
|
| 236 |
-
os.system("ls -la /data/gitea 2>/dev/null || echo '/data/gitea 不存在'")
|
| 237 |
-
if os.path.exists("/data/gitea/gitea.db"):
|
| 238 |
-
logger.info("Gitea 数据库文件存在: /data/gitea/gitea.db")
|
| 239 |
-
else:
|
| 240 |
-
logger.warning("Gitea 数据库文件不存在: /data/gitea/gitea.db")
|
| 241 |
-
|
| 242 |
while attempt < max_restart_attempts:
|
| 243 |
try:
|
| 244 |
-
# 启动 Gitea 进程
|
| 245 |
gitea_process = subprocess.Popen(
|
| 246 |
['gitea', 'web', '--port', str(port)],
|
| 247 |
stdout=subprocess.PIPE,
|
| 248 |
stderr=subprocess.PIPE,
|
| 249 |
universal_newlines=True
|
| 250 |
)
|
| 251 |
-
logger.info(f"🚀 Gitea 服务器启动
|
| 252 |
logger.info(f"📊 访问地址: http://localhost:{port}")
|
| 253 |
-
|
| 254 |
-
|
| 255 |
-
|
| 256 |
-
|
| 257 |
-
if gitea_process.poll() is not None:
|
| 258 |
break
|
| 259 |
-
|
| 260 |
-
|
| 261 |
-
|
| 262 |
-
time.sleep(0.1)
|
| 263 |
-
|
| 264 |
-
# 检查进程状态
|
| 265 |
return_code = gitea_process.poll()
|
| 266 |
-
if return_code
|
| 267 |
-
# 进程已退出
|
| 268 |
error_output = gitea_process.stderr.read()
|
| 269 |
logger.error(f"❌ Gitea 服务器异常退出,返回码: {return_code}")
|
| 270 |
-
|
| 271 |
-
logger.error(f"错误信息: {error_output.strip()}")
|
| 272 |
attempt += 1
|
| 273 |
if attempt < max_restart_attempts:
|
| 274 |
logger.info(f"将在 {restart_delay} 秒后尝试重启 Gitea (尝试 {attempt + 1}/{max_restart_attempts})")
|
| 275 |
time.sleep(restart_delay)
|
| 276 |
else:
|
| 277 |
-
|
| 278 |
-
|
| 279 |
-
raise RuntimeError(error_msg)
|
| 280 |
else:
|
| 281 |
-
|
| 282 |
-
|
| 283 |
-
while True:
|
| 284 |
-
output = gitea_process.stdout.readline()
|
| 285 |
-
if output == '' and gitea_process.poll() is not None:
|
| 286 |
-
break
|
| 287 |
-
if output:
|
| 288 |
-
logger.info(f"Gitea: {output.strip()}")
|
| 289 |
-
|
| 290 |
-
return_code = gitea_process.poll()
|
| 291 |
-
if return_code != 0:
|
| 292 |
-
error_output = gitea_process.stderr.read()
|
| 293 |
-
logger.error(f"❌ Gitea 服务器后期异常退出,返回码: {return_code}")
|
| 294 |
-
if error_output:
|
| 295 |
-
logger.error(f"错误信息: {error_output.strip()}")
|
| 296 |
-
attempt += 1
|
| 297 |
-
if attempt < max_restart_attempts:
|
| 298 |
-
logger.info(f"将在 {restart_delay} 秒后尝试重启 Gitea (尝试 {attempt + 1}/{max_restart_attempts})")
|
| 299 |
-
time.sleep(restart_delay)
|
| 300 |
-
else:
|
| 301 |
-
error_msg = f"❌ 达到最大重试次数 ({max_restart_attempts}),Gitea 启动失败"
|
| 302 |
-
logger.error(error_msg)
|
| 303 |
-
raise RuntimeError(error_msg)
|
| 304 |
-
else:
|
| 305 |
-
logger.info("✅ Gitea 服务器正常退出")
|
| 306 |
-
break
|
| 307 |
|
| 308 |
except FileNotFoundError:
|
| 309 |
-
|
| 310 |
-
|
| 311 |
-
raise RuntimeError(error_msg)
|
| 312 |
except Exception as e:
|
| 313 |
logger.error(f"❌ 启动 Gitea 服务器时发生错误: {e}")
|
| 314 |
attempt += 1
|
|
@@ -316,9 +275,8 @@ def start_gitea_server(port=7860):
|
|
| 316 |
logger.info(f"将在 {restart_delay} 秒后尝试重启 Gitea (尝试 {attempt + 1}/{max_restart_attempts})")
|
| 317 |
time.sleep(restart_delay)
|
| 318 |
else:
|
| 319 |
-
|
| 320 |
-
|
| 321 |
-
raise RuntimeError(error_msg)
|
| 322 |
|
| 323 |
gitea_thread = threading.Thread(target=run_gitea)
|
| 324 |
gitea_thread.daemon = True
|
|
@@ -358,7 +316,7 @@ async def main():
|
|
| 358 |
|
| 359 |
logger.info("✅ 所有服务已启动完成!")
|
| 360 |
logger.info("📁 目录监控: /data → Hugging Face Hub")
|
| 361 |
-
logger.info(
|
| 362 |
logger.info("🛑 按 Ctrl+C 停止所有服务")
|
| 363 |
|
| 364 |
try:
|
|
@@ -369,7 +327,7 @@ async def main():
|
|
| 369 |
|
| 370 |
except Exception as e:
|
| 371 |
logger.error(f"❌ 启动服务时发生错误: {e}")
|
| 372 |
-
|
| 373 |
finally:
|
| 374 |
if 'observer' in locals():
|
| 375 |
observer.stop()
|
|
|
|
| 109 |
exit(1)
|
| 110 |
EOF
|
| 111 |
|
| 112 |
+
# 复制上传脚本,恢复第一版的错误抛出逻辑,保留北京时间和 Gitea 重试
|
| 113 |
COPY --chown=git:git <<'EOF' /uploadhf.py
|
| 114 |
import os
|
| 115 |
import time
|
|
|
|
| 151 |
self.last_commit_time = 0
|
| 152 |
self.commit_delay = 1 # 防抖延迟 1 秒
|
| 153 |
self.pending_changes = [] # 缓冲待上传变更
|
| 154 |
+
self.loop = asyncio.get_event_loop() # 获取主事件循环
|
| 155 |
logger.info(f"初始化监控器,监控目录: {data_directory},目标仓库: {repo_id}")
|
| 156 |
|
| 157 |
def on_any_event(self, event):
|
|
|
|
| 166 |
current_time = time.time()
|
| 167 |
if current_time - self.last_commit_time > self.commit_delay:
|
| 168 |
self.last_commit_time = current_time
|
| 169 |
+
# 在主事件循环中调度异步任务
|
| 170 |
+
self.loop.call_soon_threadsafe(lambda: asyncio.create_task(self.commit_changes(change_type)))
|
| 171 |
|
| 172 |
async def commit_changes(self, change_type):
|
| 173 |
"""异步提交变更到 Hugging Face Hub,带重试机制"""
|
|
|
|
| 184 |
repo_id=self.repo_id,
|
| 185 |
repo_type="dataset",
|
| 186 |
commit_message=commit_message,
|
| 187 |
+
ignore_patterns=["*.tmp", "*.log", "*.temp", ".git/*"]
|
| 188 |
)
|
| 189 |
logger.info(f"✅ 成功提交变更到 Hugging Face Hub: {commit_message}")
|
| 190 |
return
|
|
|
|
| 204 |
logger.info(f"HF_TOKEN value: {hf_token}")
|
| 205 |
|
| 206 |
if not repo_id:
|
| 207 |
+
logger.error("❌ 必须提供 repo_id 参数")
|
| 208 |
+
exit(1)
|
| 209 |
|
| 210 |
if not os.path.exists(data_directory):
|
| 211 |
logger.warning(f"监控目录 {data_directory} 不存在,正在创建...")
|
|
|
|
| 225 |
return observer
|
| 226 |
|
| 227 |
def start_gitea_server(port=7860):
|
| 228 |
+
"""启动 Gitea 服务器"""
|
| 229 |
def run_gitea():
|
| 230 |
max_restart_attempts = 5
|
| 231 |
restart_delay = 10 # 每次重启前的等待时间(秒)
|
|
|
|
| 232 |
attempt = 0
|
| 233 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 234 |
while attempt < max_restart_attempts:
|
| 235 |
try:
|
|
|
|
| 236 |
gitea_process = subprocess.Popen(
|
| 237 |
['gitea', 'web', '--port', str(port)],
|
| 238 |
stdout=subprocess.PIPE,
|
| 239 |
stderr=subprocess.PIPE,
|
| 240 |
universal_newlines=True
|
| 241 |
)
|
| 242 |
+
logger.info(f"🚀 Gitea 服务器已启动,端口: {port}")
|
| 243 |
logger.info(f"📊 访问地址: http://localhost:{port}")
|
| 244 |
+
|
| 245 |
+
while True:
|
| 246 |
+
output = gitea_process.stdout.readline()
|
| 247 |
+
if output == '' and gitea_process.poll() is not None:
|
|
|
|
| 248 |
break
|
| 249 |
+
if output:
|
| 250 |
+
logger.info(f"Gitea: {output.strip()}")
|
| 251 |
+
|
|
|
|
|
|
|
|
|
|
| 252 |
return_code = gitea_process.poll()
|
| 253 |
+
if return_code != 0:
|
|
|
|
| 254 |
error_output = gitea_process.stderr.read()
|
| 255 |
logger.error(f"❌ Gitea 服务器异常退出,返回码: {return_code}")
|
| 256 |
+
logger.error(f"错误信息: {error_output}")
|
|
|
|
| 257 |
attempt += 1
|
| 258 |
if attempt < max_restart_attempts:
|
| 259 |
logger.info(f"将在 {restart_delay} 秒后尝试重启 Gitea (尝试 {attempt + 1}/{max_restart_attempts})")
|
| 260 |
time.sleep(restart_delay)
|
| 261 |
else:
|
| 262 |
+
logger.error(f"❌ 达到最大重启次数,Gitea 启动失败")
|
| 263 |
+
exit(1)
|
|
|
|
| 264 |
else:
|
| 265 |
+
logger.info("✅ Gitea 服务器正常退出")
|
| 266 |
+
break
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 267 |
|
| 268 |
except FileNotFoundError:
|
| 269 |
+
logger.error("❌ 未找到 gitea 命令,请确保 Gitea 已正确安装")
|
| 270 |
+
exit(1)
|
|
|
|
| 271 |
except Exception as e:
|
| 272 |
logger.error(f"❌ 启动 Gitea 服务器时发生错误: {e}")
|
| 273 |
attempt += 1
|
|
|
|
| 275 |
logger.info(f"将在 {restart_delay} 秒后尝试重启 Gitea (尝试 {attempt + 1}/{max_restart_attempts})")
|
| 276 |
time.sleep(restart_delay)
|
| 277 |
else:
|
| 278 |
+
logger.error(f"❌ 达到最大重试次数,Gitea 启动失败")
|
| 279 |
+
exit(1)
|
|
|
|
| 280 |
|
| 281 |
gitea_thread = threading.Thread(target=run_gitea)
|
| 282 |
gitea_thread.daemon = True
|
|
|
|
| 316 |
|
| 317 |
logger.info("✅ 所有服务已启动完成!")
|
| 318 |
logger.info("📁 目录监控: /data → Hugging Face Hub")
|
| 319 |
+
logger.info("🌐 Gitea 服务: http://localhost:7860")
|
| 320 |
logger.info("🛑 按 Ctrl+C 停止所有服务")
|
| 321 |
|
| 322 |
try:
|
|
|
|
| 327 |
|
| 328 |
except Exception as e:
|
| 329 |
logger.error(f"❌ 启动服务时发生错误: {e}")
|
| 330 |
+
exit(1)
|
| 331 |
finally:
|
| 332 |
if 'observer' in locals():
|
| 333 |
observer.stop()
|