File size: 1,268 Bytes
53e66de | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 | #!/usr/bin/env bash
set -euo pipefail
# Convert a user-provided finetuning CSV into the JSONL format required by finetuning.
# Run this before scripts/slurm/run_finetune.sh.
# This script does not request SLURM resources.
PROJECT_DIR="${PROJECT_DIR:-/public/home/scnb9biwet/jiangqq/CodonTransformer-main}"
CONDA_ENV="${CONDA_ENV:-struct-evo}"
INPUT_CSV="${INPUT_CSV:-${PROJECT_DIR}/scripts/data/raw/your_data.csv}"
OUTPUT_JSON="${OUTPUT_JSON:-${PROJECT_DIR}/scripts/data/processed/finetune_data.json}"
cd "${PROJECT_DIR}"
mkdir -p "$(dirname "${OUTPUT_JSON}")"
export PYTHONPATH="${PROJECT_DIR}/model:${PYTHONPATH:-}"
export INPUT_CSV
export OUTPUT_JSON
export PYTHONFAULTHANDLER=1
if [[ -n "${CONDA_ENV}" ]] && command -v conda >/dev/null 2>&1; then
# shellcheck disable=SC1091
source "$(conda info --base)/etc/profile.d/conda.sh"
conda activate "${CONDA_ENV}"
fi
if [[ ! -f "${INPUT_CSV}" ]]; then
echo "Missing INPUT_CSV: ${INPUT_CSV}" >&2
exit 1
fi
python - <<'PY'
import os
from CodonTransformer.CodonData import prepare_training_data
input_csv = os.environ["INPUT_CSV"]
output_json = os.environ["OUTPUT_JSON"]
print(f"Input CSV: {input_csv}")
print(f"Output JSONL: {output_json}")
prepare_training_data(input_csv, output_json)
PY
|