File size: 1,499 Bytes
a244197 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 | #!/bin/bash
# preflight_check.sh - 训练前预检脚本
# 用法: bash templates/preflight_check.sh [data_file1] [data_file2] ...
#
# 检查项:
# 1. ONESCIENCE_DATASETS_DIR 环境变量
# 2. 数据文件是否存在
# 3. DCU/GPU 状态
# 4. 打印节点信息
ERRORS=0
echo "========================================="
echo "Node: $(hostname)"
echo "Time: $(date)"
echo "========================================="
# 检查数据集目录环境变量
if [ -z "${ONESCIENCE_DATASETS_DIR:-}" ]; then
echo "[ERROR] ONESCIENCE_DATASETS_DIR 未设置，请先 source env.sh"
ERRORS=$((ERRORS + 1))
fi
# 检查数据文件是否存在
for f in "$@"; do
# 跳过空参数
[ -z "$f" ] && continue
# 展开环境变量后检查
expanded=$(eval echo "$f")
if [ ! -e "$expanded" ]; then
echo "[ERROR] 数据文件不存在: $expanded"
ERRORS=$((ERRORS + 1))
else
echo "[OK] 数据文件: $expanded"
fi
done
# 打印 DCU/GPU 状态
echo "-----------------------------------------"
echo ">>> DCU/GPU 状态:"
if command -v hy-smi &>/dev/null; then
hy-smi
elif command -v rocm-smi &>/dev/null; then
rocm-smi
elif command -v nvidia-smi &>/dev/null; then
nvidia-smi
else
echo "[WARN] 未找到 GPU 监控工具 (hy-smi/rocm-smi/nvidia-smi)"
fi
echo "-----------------------------------------"
if [ $ERRORS -gt 0 ]; then
echo "[FATAL] 预检发现 $ERRORS 个错误，终止训练"
exit 1
fi
echo "[OK] 预检通过"
|