scripts/train.sh

#!/usr/bin/env bash
set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"

CONDA_SH="${CONDA_SH:-/media/home/smx/miniconda3/bin/conda}"
CONDA_ENV="${CONDA_ENV:-sharp}"
if [[ -x "${CONDA_SH}" ]]; then
  eval "$("${CONDA_SH}" shell.bash hook)"
  conda activate "${CONDA_ENV}"
fi

export PYTHONPATH="${REPO_ROOT}:${PYTHONPATH:-}"
export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION="${PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION:-python}"

export OUT_ROOT="${OUT_ROOT:-${REPO_ROOT}/outputs}"
export RUN_NAME="${RUN_NAME:-unisharp_gt_override_$(date +%Y%m%d_%H%M%S)}"

export SEED="${SEED:-260602}"

export STEPS="${STEPS:-1000000}"
export WARMUP="${WARMUP:-75000}"
export BATCH_SIZE="${BATCH_SIZE:-2}"
export NUM_WORKERS="${NUM_WORKERS:-1}"
export GPU_IDS="${GPU_IDS:-0,1,2,3,4,5,6,7}"
export MASTER_PORT="${MASTER_PORT:-29531}"
export DEVICE="${DEVICE:-cuda}"
export DDP_TIMEOUT_HOURS="${DDP_TIMEOUT_HOURS:-8}"

export LR0="${LR0:-1.2e-4}"
export LR1="${LR1:-1.6e-5}"
export UNIK3D_DECODER_LR0="${UNIK3D_DECODER_LR0:-2.5e-5}"
export UNIK3D_DECODER_LR1="${UNIK3D_DECODER_LR1:-2.5e-6}"
export UNIK3D_ENCODER_LR0="${UNIK3D_ENCODER_LR0:-1.5e-6}"
export UNIK3D_ENCODER_LR1="${UNIK3D_ENCODER_LR1:-1.5e-7}"
export GRAD_CLIP_NORM="${GRAD_CLIP_NORM:-1.0}"
export MAX_STEP_GRAD_NORM="${MAX_STEP_GRAD_NORM:-100000.0}"

export INITIALIZER_STRIDE="${INITIALIZER_STRIDE:-1}"
export INITIALIZER_SCALE_FACTOR="${INITIALIZER_SCALE_FACTOR:-1.5}"
export DELTA_RHO_LIMIT="${DELTA_RHO_LIMIT:-2.0}"

export MAX_INDEX_GAP="${MAX_INDEX_GAP:-10}"
export MAX_DEPTH_M="${MAX_DEPTH_M:-100.0}"
export PINHOLE_TRAIN_SIZE="${PINHOLE_TRAIN_SIZE:-0}"
export TRAIN_RESIZE_MULTIPLE="${TRAIN_RESIZE_MULTIPLE:-256}"
export SIM_MAX_LONG_EDGE="${SIM_MAX_LONG_EDGE:-512}"
export RE10K_PSEUDO_FAR_DEPTH_INVALID_M="${RE10K_PSEUDO_FAR_DEPTH_INVALID_M:-30.0}"
export SIM_FAR_DEPTH_INVALID_M="${SIM_FAR_DEPTH_INVALID_M:-30.0}"
export SIM_FAR_DEPTH_INVALID_MAX_FRAC="${SIM_FAR_DEPTH_INVALID_MAX_FRAC:-1.0}"
export SCANETPP_FISHEYE_FAR_DEPTH_INVALID_M="${SCANETPP_FISHEYE_FAR_DEPTH_INVALID_M:-30.0}"

export LAMBDA_COLOR="${LAMBDA_COLOR:-1.0}"
export LAMBDA_ALPHA="${LAMBDA_ALPHA:-1.5}"
export LAMBDA_PERCEP="${LAMBDA_PERCEP:-1.0}"
export LAMBDA_DEPTH="${LAMBDA_DEPTH:-0.5}"
export LAMBDA_TV="${LAMBDA_TV:-1.0}"
export LAMBDA_GRAD="${LAMBDA_GRAD:-1.0}"
export LAMBDA_GRAD_IMG="${LAMBDA_GRAD_IMG:-0.2}"
export LAMBDA_EDGE_RGB="${LAMBDA_EDGE_RGB:-0.0}"
export LAMBDA_DELTA="${LAMBDA_DELTA:-1.0}"
export LAMBDA_DELTA_RHO="${LAMBDA_DELTA_RHO:-0.01}"
export LAMBDA_SPLAT="${LAMBDA_SPLAT:-1.0}"
export LAMBDA_EDGE_SPLAT="${LAMBDA_EDGE_SPLAT:-0.0}"
export LAMBDA_GRID="${LAMBDA_GRID:-0.05}"
export LAMBDA_AUX_RAY="${LAMBDA_AUX_RAY:-3.0}"
export LAMBDA_AUX_DEPTH_SCALE="${LAMBDA_AUX_DEPTH_SCALE:-3.0}"
export LAMBDA_AUX_DEPTH2_SCALE="${LAMBDA_AUX_DEPTH2_SCALE:-1.0}"

export SAVE_EVERY="${SAVE_EVERY:-5000}"
export VIS_EVERY="${VIS_EVERY:-500}"
export LOG_EVERY="${LOG_EVERY:-50}"


export DATASET_WEIGHT_RE10K="${DATASET_WEIGHT_RE10K:-1.0}"
export DATASET_WEIGHT_HM3D="${DATASET_WEIGHT_HM3D:-1.0}"
export DATASET_WEIGHT_SIM="${DATASET_WEIGHT_SIM:-1.0}"
export DATASET_WEIGHT_WILDRGBD="${DATASET_WEIGHT_WILDRGBD:-1.0}"
export DATASET_WEIGHT_DL3DV="${DATASET_WEIGHT_DL3DV:-1.0}"
export DATASET_WEIGHT_SCANETPP="${DATASET_WEIGHT_SCANETPP:-0.0}"

export DATA_ROOT_RE10K="${DATA_ROOT_RE10K:-/media/team_data/ML4_team/datasets/nopose/re10k}"
export RE10K_PSEUDO_DEPTH_ROOT="${RE10K_PSEUDO_DEPTH_ROOT:-/media/team_data/ML4_team/datasets/nopose/re10k_unik3d_pseudo_depth}"
export DATA_ROOT_HM3D="${DATA_ROOT_HM3D:-/media/team_data/ML4_team/datasets/panogs}"
export DATA_ROOT_SIM="${DATA_ROOT_SIM:-/media/team_data/ML4_team/datasets/smx_sim}"
export SIM_POSE_ROOT="${SIM_POSE_ROOT:-/media/team_data/ML4_team/datasets/smx_sim/30cm}"
export DATA_ROOT_DL3DV="${DATA_ROOT_DL3DV:-/media/team_data/ML4_team/datasets/sharp/DL3DV-ALL-960P}"
export DATA_ROOT_DL3DV_DEPTH="${DATA_ROOT_DL3DV_DEPTH:-/media/team_data/ML4_team/datasets/sharp/DL3DV-ALL-960P_da3_outputs}"
export DATA_ROOT_SCANETPP="${DATA_ROOT_SCANETPP:-/media/team_data/ML4_team/datasets/scan}"

DEFAULT_DATASET_MANIFEST_DIR="${REPO_ROOT}/dataset_manifests"
if [[ -d "${REPO_ROOT}/../dataset_manifests" ]]; then
  DEFAULT_DATASET_MANIFEST_DIR="${REPO_ROOT}/../dataset_manifests"
fi
export DATASET_MANIFEST_DIR="${DATASET_MANIFEST_DIR:-${DEFAULT_DATASET_MANIFEST_DIR}}"
if [[ ! -f "${DATASET_MANIFEST_DIR}/sim_train_scenes.txt" && -f "${REPO_ROOT}/../dataset_manifests/sim_train_scenes.txt" ]]; then
  export DATASET_MANIFEST_DIR="${REPO_ROOT}/../dataset_manifests"
fi
export WILD_ROOTS_FILE="${WILD_ROOTS_FILE:-${DATASET_MANIFEST_DIR}/wildrgbd_roots.txt}"

export CUDA_VISIBLE_DEVICES="${GPU_IDS}"
export NCCL_NET="${NCCL_NET:-Socket}"
export NCCL_IB_DISABLE="${NCCL_IB_DISABLE:-1}"
export TORCH_NCCL_ASYNC_ERROR_HANDLING="${TORCH_NCCL_ASYNC_ERROR_HANDLING:-1}"

IFS=',' read -r -a GPU_ID_ARR <<< "${GPU_IDS}"
if [[ "${#GPU_ID_ARR[@]}" -gt 1 ]]; then
  LAUNCH_CMD=(torchrun --nproc_per_node="${#GPU_ID_ARR[@]}" --master_port="${MASTER_PORT}")
else
  LAUNCH_CMD=(python)
fi

echo "UniSharp training: run=${RUN_NAME} out=${OUT_ROOT} gpu=${GPU_IDS}"
echo "  branch=gt-override scratch_unik3d_pretrained"
echo "  datasets: re10k=${DATASET_WEIGHT_RE10K} hm3d=${DATASET_WEIGHT_HM3D} sim=${DATASET_WEIGHT_SIM} wildrgbd=${DATASET_WEIGHT_WILDRGBD} dl3dv=${DATASET_WEIGHT_DL3DV} scanetpp=${DATASET_WEIGHT_SCANETPP}"

exec "${LAUNCH_CMD[@]}" -m unisharp.cli train-feature \
  --out-root "${OUT_ROOT}" \
  --run-name "${RUN_NAME}" \
  --steps "${STEPS}" \
  --warmup "${WARMUP}" \
  --lr0 "${LR0}" \
  --lr1 "${LR1}" \
  --unik3d-lr0 "${UNIK3D_DECODER_LR0}" \
  --unik3d-lr1 "${UNIK3D_DECODER_LR1}" \
  --unik3d-encoder-lr0 "${UNIK3D_ENCODER_LR0}" \
  --unik3d-encoder-lr1 "${UNIK3D_ENCODER_LR1}" \
  --grad-clip-norm "${GRAD_CLIP_NORM}" \
  --max-step-grad-norm "${MAX_STEP_GRAD_NORM}" \
  --batch-size "${BATCH_SIZE}" \
  --num-workers "${NUM_WORKERS}" \
  --device "${DEVICE}" \
  --ddp-timeout-hours "${DDP_TIMEOUT_HOURS}" \
  --max-index-gap "${MAX_INDEX_GAP}" \
  --max-depth-m "${MAX_DEPTH_M}" \
  --sim-far-depth-invalid-m "${SIM_FAR_DEPTH_INVALID_M}" \
  --sim-far-depth-invalid-max-frac "${SIM_FAR_DEPTH_INVALID_MAX_FRAC}" \
  --sim-max-long-edge "${SIM_MAX_LONG_EDGE}" \
  --pinhole-train-size "${PINHOLE_TRAIN_SIZE}" \
  --train-resize-multiple "${TRAIN_RESIZE_MULTIPLE}" \
  --scanetpp-fisheye-far-depth-invalid-m "${SCANETPP_FISHEYE_FAR_DEPTH_INVALID_M}" \
  --initializer-stride "${INITIALIZER_STRIDE}" \
  --initializer-scale-factor "${INITIALIZER_SCALE_FACTOR}" \
  --delta-rho-limit "${DELTA_RHO_LIMIT}" \
  --lambda-color "${LAMBDA_COLOR}" \
  --lambda-alpha "${LAMBDA_ALPHA}" \
  --lambda-percep "${LAMBDA_PERCEP}" \
  --lambda-depth "${LAMBDA_DEPTH}" \
  --lambda-tv "${LAMBDA_TV}" \
  --lambda-grad "${LAMBDA_GRAD}" \
  --lambda-grad-img "${LAMBDA_GRAD_IMG}" \
  --lambda-edge-rgb "${LAMBDA_EDGE_RGB}" \
  --lambda-delta "${LAMBDA_DELTA}" \
  --lambda-delta-rho "${LAMBDA_DELTA_RHO}" \
  --lambda-splat "${LAMBDA_SPLAT}" \
  --lambda-edge-splat "${LAMBDA_EDGE_SPLAT}" \
  --lambda-grid "${LAMBDA_GRID}" \
  --lambda-aux-ray "${LAMBDA_AUX_RAY}" \
  --lambda-aux-depth-scale "${LAMBDA_AUX_DEPTH_SCALE}" \
  --lambda-aux-depth2-scale "${LAMBDA_AUX_DEPTH2_SCALE}" \
  --dataset-weight-re10k "${DATASET_WEIGHT_RE10K}" \
  --dataset-weight-hm3d "${DATASET_WEIGHT_HM3D}" \
  --dataset-weight-sim "${DATASET_WEIGHT_SIM}" \
  --dataset-weight-wildrgbd "${DATASET_WEIGHT_WILDRGBD}" \
  --dataset-weight-dl3dv "${DATASET_WEIGHT_DL3DV}" \
  --dataset-weight-scanetpp "${DATASET_WEIGHT_SCANETPP}" \
  --data-root-re10k "${DATA_ROOT_RE10K}" \
  --re10k-pseudo-depth-root "${RE10K_PSEUDO_DEPTH_ROOT}" \
  --re10k-pseudo-far-depth-invalid-m "${RE10K_PSEUDO_FAR_DEPTH_INVALID_M}" \
  --data-root-hm3d "${DATA_ROOT_HM3D}" \
  --data-root-sim "${DATA_ROOT_SIM}" \
  --sim-pose-root "${SIM_POSE_ROOT}" \
  --wild-roots-file "${WILD_ROOTS_FILE}" \
  --data-root-dl3dv "${DATA_ROOT_DL3DV}" \
  --data-root-dl3dv-depth "${DATA_ROOT_DL3DV_DEPTH}" \
  --data-root-scanetpp "${DATA_ROOT_SCANETPP}" \
  --dataset-manifest-dir "${DATASET_MANIFEST_DIR}" \
  --save-every "${SAVE_EVERY}" \
  --vis-every "${VIS_EVERY}" \
  --log-every "${LOG_EVERY}" \
  --seed "${SEED}"