diff --git a/gainlora_OT_vMF/SETUP.md b/gainlora_OT_vMF/SETUP.md deleted file mode 100644 index 42289af8033386d0b8288110670e397ca8990764..0000000000000000000000000000000000000000 --- a/gainlora_OT_vMF/SETUP.md +++ /dev/null @@ -1,143 +0,0 @@ -# GainLoRA + OT-SIGN Setup Guide - -## Quick Start - -### Option 1: Using conda + pip (Recommended for GPU) - -```bash -# 1. Create conda environment with CUDA support -conda create -n gainlora \ - python=3.10 \ - pytorch::pytorch \ - pytorch::pytorch-cuda=12.1 \ - pytorch::torchvision \ - pytorch::torchaudio \ - -c pytorch -c nvidia - -# 2. Activate environment -conda activate gainlora - -# 3. Install pip dependencies -cd gainlora -pip install -r requirements.txt -``` - -### Option 2: Using only pip (CPU or if CUDA already setup) - -```bash -# Install all pip dependencies -pip install -r requirements.txt -``` - -### Option 3: From conda environment (existing setup) - -```bash -# If you already have an environment, just update pip packages -conda activate -pip install -r requirements.txt -``` - ---- - -## Key Dependencies - -| Package | Purpose | Version | -|---------|---------|---------| -| transformers | HuggingFace models (T5, LLaMA, etc.) | 4.30.2 | -| torch/vision/audio | PyTorch + CUDA support | 2.1.0 | -| datasets | Dataset loading & processing | 2.14.6 | -| accelerate | Distributed training | 0.24.1 | -| deepspeed | ZeRO optimization | 0.11.2 | -| loralib | LoRA layers | 0.1.2 | -| cupy-cuda12x | GPU-accelerated arrays (for SVD) | 12.1.0 | -| safetensors | Fast model serialization | 0.4.4 | - ---- - -## Troubleshooting - -### CUDA Version Mismatch -If you have a different CUDA version: -```bash -# Change 12.1 to your CUDA version (11.8, 11.6, etc.) -conda create -n gainlora ... pytorch::pytorch-cuda=11.8 ... -``` - -### CuPy Installation Fails -CuPy requires CUDA dev toolkit. If it fails: -```bash -# Option 1: Install it separately -conda install -c conda-forge cupy cuda-version=12.1 - -# Option 2: Comment it out in requirements.txt if not using SVD GPM -``` - -### Out of Memory on 2×T4 -Scripts are already optimized (batch_size=8, fp16). If still OOM: -1. Reduce `per_device_train_batch_size` in bash scripts -2. Increase `gradient_accumulation_steps` - -### Kaggle/Colab Environment (Pre-built Wheels) - -When running on Kaggle or Google Colab, use pre-built wheels to avoid compilation: - -```bash -# Clone and install (Colab/Kaggle cell) -! git clone https://github.com/liangyanshuo/gainlora.git -! cd gainlora && pip install -r requirements.txt --no-build-isolation - -# Or with uv proxy (faster) - already recommended in Colab/Kaggle -! pip install uv -! uv pip install -r requirements.txt -``` - -**Note:** `tokenizers>=0.14.1` has pre-built wheels for Python 3.12. If you still get build errors: -- Add GPU dependencies: `! pip install --upgrade setuptools wheel` -- Or downgrade Python: Colab uses 3.10-3.12; ensure compatibility - ---- - -## Running OT-SIGN + GainLoRA - -```bash -# Order 1 (SuperNI) - ~10 hours on 2×T4 -bash run_ot_sign_order1_t5large.sh 0,1 google/flan-t5-large - -# Order 2 (SuperNI) - ~10 hours -bash run_ot_sign_order2_t5large.sh 0,1 google/flan-t5-large - -# Order 3 (Long) - ~4 hours [run this first to test] -bash run_ot_sign_order3_t5large.sh 0,1 google/flan-t5-large - -# Order 4 (Long) - ~4 hours -bash run_ot_sign_order4_t5large.sh 0,1 google/flan-t5-large -``` - -Results and logs: -- Training outputs: `logs_and_outputs/ot_sign_order{1,2,3,4}_t5large/` -- AP/FT scores: Printed in terminal + `ap_ft_result.json` -- Compare with baseline: See `results/comparison_results.md` - ---- - -## Environment Verification - -```bash -# Check PyTorch + CUDA -python -c "import torch; print(f'PyTorch {torch.__version__}, CUDA {torch.cuda.is_available()}')" - -# Check transformers -python -c "import transformers; print(f'Transformers {transformers.__version__}')" - -# Check other deps -python -c "import cupy; import accelerate; import deepspeed; print('All deps OK')" -``` - ---- - -## Notes - -- `requirements.yaml` = Full conda environment spec (includes system libs) -- `requirements.txt` = Python pip packages only (easier to share and modify) -- Both methods should give identical Python environment -- CUDA/cuDNN installation separate (via conda in setup) diff --git a/gainlora_OT_vMF/requirements.txt b/gainlora_OT_vMF/requirements.txt deleted file mode 100644 index 7a0cb12cc8ec4d020d90bbc7ca2f7e8d428217f9..0000000000000000000000000000000000000000 --- a/gainlora_OT_vMF/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:22ba1705f034261bc2483bcabe3bbf5935a041e6972f2d17e487c54fb56b92c5 -size 1136 diff --git a/gainlora_OT_vMF/run_ot_sign_order1_t5large.sh b/gainlora_OT_vMF/run_ot_sign_order1_t5large.sh deleted file mode 100755 index 4b5d6c53ae84335bf75996cab0c2401d8e6f0a7d..0000000000000000000000000000000000000000 --- a/gainlora_OT_vMF/run_ot_sign_order1_t5large.sh +++ /dev/null @@ -1,175 +0,0 @@ -#!/bin/bash -############################################################################### -# OT-SIGN + GainLoRA (T5-Large) — Order 1 (SuperNI) -# =================================================== -# Task order: samsum → sst2 → xsum → ... → emotion (15 tasks) -# Benchmark: SuperNI (RougeL metric) -# Hardware: 2× T4 16GB | Estimated time: ~9-10h -# Run: bash run_ot_sign_order1_t5large.sh [GPU_IDS] [MODEL_PATH] -############################################################################### - -set -e # exit on first error - -export CUDA_DEVICE_ORDER="PCI_BUS_ID" -GPU_IDS="${1:-0,1}" -MODEL_PATH="${2:-google/flan-t5-large}" - -# ── Config ────────────────────────────────────────────────────────────────── -RUN_NAME="ot_sign_order1_t5large" -OUTPUT_BASE="logs_and_outputs/${RUN_NAME}/outputs" -CONFIG_PREFIX="configs/gen_script_superni_order1_t5_configs" -GEN_DATA_DIR="generated_data/lora_gen_superni_t5" -METRIC_TYPE="rougeL" # RougeL for SuperNI -KL_RATIO=0.5 -EPOCHS=50 - -# T4 settings -BATCH_SIZE=8 -EVAL_BATCH=4 -GRAD_ACCUM=4 -LR=0.0003 - -# OT-SIGN -OT_EPSILON=0.05 -OT_N_ITER=10 -DEFAULT_KAPPA=10.0 -LAMBDA_DRIFT=0.01 -LAMBDA_INV=0.001 -THRESHOLD=2.3 - -# ── Task sequence ──────────────────────────────────────────────────────────── -TASKS=( - "task1572_samsum_summary" - "task363_sst2_polarity_classification" - "task1290_xsum_summarization" - "task181_outcome_extraction" - "task002_quoref_answer_generation" - "task1510_evalution_relation_extraction" - "task639_multi_woz_user_utterance_generation" - "task1729_personachat_generate_next" - "task073_commonsenseqa_answer_generation" - "task1590_diplomacy_text_generation" - "task748_glucose_reverse_cause_event_detection" - "task511_reddit_tifu_long_text_summarization" - "task591_sciq_answer_generation" - "task1687_sentiment140_classification" - "task875_emotion_classification" -) -TASK_ORDER=$(IFS=,; echo "${TASKS[*]}") -T=${#TASKS[@]} - -mkdir -p "${OUTPUT_BASE}" -echo "════════════════════════════════════════════════" -echo " OT-SIGN+GainLoRA | Order 1 | T5-Large" -echo " GPUs: ${GPU_IDS} | Tasks: ${T} | Epochs: ${EPOCHS}" -echo "════════════════════════════════════════════════" - -# ── Training loop ──────────────────────────────────────────────────────────── -PREV_LORA_PATHS="" - -for i in "${!TASKS[@]}"; do - TASK="${TASKS[$i]}" - TASKNUM=$((i + 1)) - OUTDIR="${OUTPUT_BASE}/${TASKNUM}-${TASK}" - - # Metric key (task 1 has no "for_task" suffix) - if [[ $i -eq 0 ]]; then - METRIC_KEY="eval_${METRIC_TYPE}" - else - METRIC_KEY="eval_${METRIC_TYPE}_for_${TASK}" - fi - - echo "" - echo "──────────────────────────────────────────────" - echo " Task ${TASKNUM}/${T}: ${TASK}" - echo "──────────────────────────────────────────────" - - # Build OT-SIGN flags - OT_FLAGS="--use_ot_routing True \ - --ot_epsilon ${OT_EPSILON} \ - --ot_n_iter ${OT_N_ITER} \ - --default_kappa ${DEFAULT_KAPPA} \ - --lambda_drift ${LAMBDA_DRIFT} \ - --lambda_inv ${LAMBDA_INV} \ - --invasion_threshold ${THRESHOLD}" - - # Build previous-task flags (empty for task 1) - PREV_FLAGS="" - if [[ $i -gt 0 ]]; then - PREV_OUTDIR="${OUTPUT_BASE}/${i}-${TASKS[$((i-1))]}/saved_weights" - PREV_FLAGS="--load_checkpoint_from ${PREV_OUTDIR}/trans_input.pt \ - --previous_lora_path ${PREV_LORA_PATHS} \ - --previous_prompt_key_path ${PREV_OUTDIR}/prompts_keys_till_now.pt \ - --previous_vmf_signatures_path ${PREV_OUTDIR}/vmf_signatures.pt \ - --add_instruction_replay \ - --gen_data_dir ${GEN_DATA_DIR} \ - --kl_ratio ${KL_RATIO} \ - --data_replay_freq -1 \ - --replay_after_n_epoch 0" - fi - - CUDA_VISIBLE_DEVICES=${GPU_IDS} python src/run_t5.py \ - --do_train \ - --do_predict \ - --predict_with_generate \ - --model_name_or_path "${MODEL_PATH}" \ - --data_dir CL_Benchmark \ - --task_order "${TASK_ORDER}" \ - --task_config_dir "${CONFIG_PREFIX}/${TASK}" \ - --output_dir "${OUTDIR}" \ - --per_device_train_batch_size ${BATCH_SIZE} \ - --per_device_eval_batch_size ${EVAL_BATCH} \ - --gradient_accumulation_steps ${GRAD_ACCUM} \ - --learning_rate ${LR} \ - --num_train_epochs ${EPOCHS} \ - --run_name "${RUN_NAME}" \ - --max_source_length 512 \ - --max_target_length 50 \ - --generation_max_length 50 \ - --add_task_name False \ - --add_dataset_name False \ - --lr_scheduler_type constant \ - --warmup_steps 0 \ - --logging_strategy steps \ - --logging_steps 10 \ - --eval_strategy steps \ - --save_strategy steps \ - --save_total_limit 1 \ - --load_best_model_at_end \ - --metric_for_best_model "${METRIC_KEY}" \ - --lora_r 4 \ - --lora_alpha 32 \ - --lora_dropout 0.0 \ - --attn_temperature 1 \ - --mlp_hidden_dim 100 \ - --model_name gainlora_inflora \ - --threshold 0.995 \ - --transthreshold 0.995 \ - --fp16 \ - ${OT_FLAGS} \ - ${PREV_FLAGS} - - # Accumulate previous lora paths - if [[ -z "${PREV_LORA_PATHS}" ]]; then - PREV_LORA_PATHS="${OUTDIR}/saved_weights" - else - PREV_LORA_PATHS="${PREV_LORA_PATHS},${OUTDIR}/saved_weights" - fi - - echo " [DONE] Task ${TASKNUM}/${T}: ${TASK}" -done - -# ── Final AP/FT ────────────────────────────────────────────────────────────── -echo "" -echo "════════════════════════════════════════════════" -echo " Computing AP / FT ..." -echo "════════════════════════════════════════════════" -python src/compute_ap_ft.py \ - --output_base "${OUTPUT_BASE}" \ - --task_order "${TASK_ORDER}" \ - --method_name "${RUN_NAME}" \ - --save - -echo "" -echo "✓ All done. Results in: logs_and_outputs/${RUN_NAME}/" -echo " Compare with: results/comparison_results.md" diff --git a/gainlora_OT_vMF/run_ot_sign_order2_t5large.sh b/gainlora_OT_vMF/run_ot_sign_order2_t5large.sh deleted file mode 100755 index fdb05c7802c8db592a2ead6aec290029651b1f69..0000000000000000000000000000000000000000 --- a/gainlora_OT_vMF/run_ot_sign_order2_t5large.sh +++ /dev/null @@ -1,168 +0,0 @@ -#!/bin/bash -############################################################################### -# OT-SIGN + GainLoRA (T5-Large) — Order 2 (SuperNI) -# =================================================== -# Task order: glucose → commonsenseqa → diplomacy → ... → emotion (15 tasks) -# Benchmark: SuperNI (RougeL metric) -# Hardware: 2× T4 16GB | Estimated time: ~9-10h -# Run: bash run_ot_sign_order2_t5large.sh [GPU_IDS] [MODEL_PATH] -############################################################################### - -set -e - -export CUDA_DEVICE_ORDER="PCI_BUS_ID" -GPU_IDS="${1:-0,1}" -MODEL_PATH="${2:-google/flan-t5-large}" - -# ── Config ────────────────────────────────────────────────────────────────── -RUN_NAME="ot_sign_order2_t5large" -OUTPUT_BASE="logs_and_outputs/${RUN_NAME}/outputs" -CONFIG_PREFIX="configs/gen_script_superni_order2_t5_configs" -GEN_DATA_DIR="generated_data/lora_gen_superni_t5" -METRIC_TYPE="rougeL" -KL_RATIO=0.5 -EPOCHS=50 - -BATCH_SIZE=8 -EVAL_BATCH=4 -GRAD_ACCUM=4 -LR=0.0003 - -OT_EPSILON=0.05 -OT_N_ITER=10 -DEFAULT_KAPPA=10.0 -LAMBDA_DRIFT=0.01 -LAMBDA_INV=0.001 -THRESHOLD=2.3 - -# ── Task sequence ──────────────────────────────────────────────────────────── -TASKS=( - "task748_glucose_reverse_cause_event_detection" - "task073_commonsenseqa_answer_generation" - "task1590_diplomacy_text_generation" - "task639_multi_woz_user_utterance_generation" - "task1572_samsum_summary" - "task1687_sentiment140_classification" - "task591_sciq_answer_generation" - "task363_sst2_polarity_classification" - "task1510_evalution_relation_extraction" - "task1729_personachat_generate_next" - "task181_outcome_extraction" - "task511_reddit_tifu_long_text_summarization" - "task002_quoref_answer_generation" - "task1290_xsum_summarization" - "task875_emotion_classification" -) -TASK_ORDER=$(IFS=,; echo "${TASKS[*]}") -T=${#TASKS[@]} - -mkdir -p "${OUTPUT_BASE}" -echo "════════════════════════════════════════════════" -echo " OT-SIGN+GainLoRA | Order 2 | T5-Large" -echo " GPUs: ${GPU_IDS} | Tasks: ${T} | Epochs: ${EPOCHS}" -echo "════════════════════════════════════════════════" - -# ── Training loop ──────────────────────────────────────────────────────────── -PREV_LORA_PATHS="" - -for i in "${!TASKS[@]}"; do - TASK="${TASKS[$i]}" - TASKNUM=$((i + 1)) - OUTDIR="${OUTPUT_BASE}/${TASKNUM}-${TASK}" - - if [[ $i -eq 0 ]]; then - METRIC_KEY="eval_${METRIC_TYPE}" - else - METRIC_KEY="eval_${METRIC_TYPE}_for_${TASK}" - fi - - echo "" - echo "──────────────────────────────────────────────" - echo " Task ${TASKNUM}/${T}: ${TASK}" - echo "──────────────────────────────────────────────" - - OT_FLAGS="--use_ot_routing True \ - --ot_epsilon ${OT_EPSILON} \ - --ot_n_iter ${OT_N_ITER} \ - --default_kappa ${DEFAULT_KAPPA} \ - --lambda_drift ${LAMBDA_DRIFT} \ - --lambda_inv ${LAMBDA_INV} \ - --invasion_threshold ${THRESHOLD}" - - PREV_FLAGS="" - if [[ $i -gt 0 ]]; then - PREV_OUTDIR="${OUTPUT_BASE}/${i}-${TASKS[$((i-1))]}/saved_weights" - PREV_FLAGS="--load_checkpoint_from ${PREV_OUTDIR}/trans_input.pt \ - --previous_lora_path ${PREV_LORA_PATHS} \ - --previous_prompt_key_path ${PREV_OUTDIR}/prompts_keys_till_now.pt \ - --previous_vmf_signatures_path ${PREV_OUTDIR}/vmf_signatures.pt \ - --add_instruction_replay \ - --gen_data_dir ${GEN_DATA_DIR} \ - --kl_ratio ${KL_RATIO} \ - --data_replay_freq -1 \ - --replay_after_n_epoch 0" - fi - - CUDA_VISIBLE_DEVICES=${GPU_IDS} python src/run_t5.py \ - --do_train \ - --do_predict \ - --predict_with_generate \ - --model_name_or_path "${MODEL_PATH}" \ - --data_dir CL_Benchmark \ - --task_order "${TASK_ORDER}" \ - --task_config_dir "${CONFIG_PREFIX}/${TASK}" \ - --output_dir "${OUTDIR}" \ - --per_device_train_batch_size ${BATCH_SIZE} \ - --per_device_eval_batch_size ${EVAL_BATCH} \ - --gradient_accumulation_steps ${GRAD_ACCUM} \ - --learning_rate ${LR} \ - --num_train_epochs ${EPOCHS} \ - --run_name "${RUN_NAME}" \ - --max_source_length 512 \ - --max_target_length 50 \ - --generation_max_length 50 \ - --add_task_name False \ - --add_dataset_name False \ - --lr_scheduler_type constant \ - --warmup_steps 0 \ - --logging_strategy steps \ - --logging_steps 10 \ - --eval_strategy steps \ - --save_strategy steps \ - --save_total_limit 1 \ - --load_best_model_at_end \ - --metric_for_best_model "${METRIC_KEY}" \ - --lora_r 4 \ - --lora_alpha 32 \ - --lora_dropout 0.0 \ - --attn_temperature 1 \ - --mlp_hidden_dim 100 \ - --model_name gainlora_inflora \ - --threshold 0.995 \ - --transthreshold 0.995 \ - --fp16 \ - ${OT_FLAGS} \ - ${PREV_FLAGS} - - if [[ -z "${PREV_LORA_PATHS}" ]]; then - PREV_LORA_PATHS="${OUTDIR}/saved_weights" - else - PREV_LORA_PATHS="${PREV_LORA_PATHS},${OUTDIR}/saved_weights" - fi - - echo " [DONE] Task ${TASKNUM}/${T}: ${TASK}" -done - -# ── Final AP/FT ────────────────────────────────────────────────────────────── -echo "" -echo "════════════════════════════════════════════════" -echo " Computing AP / FT ..." -echo "════════════════════════════════════════════════" -python src/compute_ap_ft.py \ - --output_base "${OUTPUT_BASE}" \ - --task_order "${TASK_ORDER}" \ - --method_name "${RUN_NAME}" \ - --save - -echo "" -echo "✓ All done. Results in: logs_and_outputs/${RUN_NAME}/" diff --git a/gainlora_OT_vMF/run_ot_sign_order3_t5large.sh b/gainlora_OT_vMF/run_ot_sign_order3_t5large.sh deleted file mode 100755 index d521d7d064113c3ef3886ba8a40b54fea57de159..0000000000000000000000000000000000000000 --- a/gainlora_OT_vMF/run_ot_sign_order3_t5large.sh +++ /dev/null @@ -1,169 +0,0 @@ -#!/bin/bash -############################################################################### -# OT-SIGN + GainLoRA (T5-Large) — Order 3 (Long Benchmark) -# ========================================================== -# Task order: yelp → amazon → mnli → ... → wic (15 tasks) -# Benchmark: Long (Exact Match metric) -# Hardware: 2× T4 16GB | Estimated time: ~3-4h -# Run: bash run_ot_sign_order3_t5large.sh [GPU_IDS] [MODEL_PATH] -############################################################################### - -set -e - -export CUDA_DEVICE_ORDER="PCI_BUS_ID" -GPU_IDS="${1:-0,1}" -MODEL_PATH="${2:-google/flan-t5-large}" - -# ── Config ────────────────────────────────────────────────────────────────── -RUN_NAME="ot_sign_order3_t5large" -OUTPUT_BASE="logs_and_outputs/${RUN_NAME}/outputs" -CONFIG_PREFIX="configs/gen_script_long_order3_t5_configs" -GEN_DATA_DIR="generated_data/lora_gen_long_t5" -METRIC_TYPE="exact_match" # Exact Match for Long benchmark -KL_RATIO=0.1 -EPOCHS=10 # Long benchmark uses fewer epochs - -BATCH_SIZE=4 -EVAL_BATCH=16 # Long benchmark uses larger eval batch -GRAD_ACCUM=8 -LR=0.0003 - -OT_EPSILON=0.05 -OT_N_ITER=10 -DEFAULT_KAPPA=10.0 -LAMBDA_DRIFT=0.01 -LAMBDA_INV=0.001 -THRESHOLD=2.3 - -# ── Task sequence ──────────────────────────────────────────────────────────── -TASKS=( - "yelp" - "amazon" - "mnli" - "cb" - "copa" - "qqp" - "rte" - "imdb" - "sst2" - "dbpedia" - "agnews" - "yahoo" - "multirc" - "boolq" - "wic" -) -TASK_ORDER=$(IFS=,; echo "${TASKS[*]}") -T=${#TASKS[@]} - -mkdir -p "${OUTPUT_BASE}" -echo "════════════════════════════════════════════════" -echo " OT-SIGN+GainLoRA | Order 3 | T5-Large" -echo " GPUs: ${GPU_IDS} | Tasks: ${T} | Epochs: ${EPOCHS}" -echo "════════════════════════════════════════════════" - -# ── Training loop ──────────────────────────────────────────────────────────── -PREV_LORA_PATHS="" - -for i in "${!TASKS[@]}"; do - TASK="${TASKS[$i]}" - TASKNUM=$((i + 1)) - OUTDIR="${OUTPUT_BASE}/${TASKNUM}-${TASK}" - - if [[ $i -eq 0 ]]; then - METRIC_KEY="eval_${METRIC_TYPE}" - else - METRIC_KEY="eval_${METRIC_TYPE}_for_${TASK}" - fi - - echo "" - echo "──────────────────────────────────────────────" - echo " Task ${TASKNUM}/${T}: ${TASK}" - echo "──────────────────────────────────────────────" - - OT_FLAGS="--use_ot_routing True \ - --ot_epsilon ${OT_EPSILON} \ - --ot_n_iter ${OT_N_ITER} \ - --default_kappa ${DEFAULT_KAPPA} \ - --lambda_drift ${LAMBDA_DRIFT} \ - --lambda_inv ${LAMBDA_INV} \ - --invasion_threshold ${THRESHOLD}" - - PREV_FLAGS="" - if [[ $i -gt 0 ]]; then - PREV_OUTDIR="${OUTPUT_BASE}/${i}-${TASKS[$((i-1))]}/saved_weights" - PREV_FLAGS="--load_checkpoint_from ${PREV_OUTDIR}/trans_input.pt \ - --previous_lora_path ${PREV_LORA_PATHS} \ - --previous_prompt_key_path ${PREV_OUTDIR}/prompts_keys_till_now.pt \ - --previous_vmf_signatures_path ${PREV_OUTDIR}/vmf_signatures.pt \ - --add_instruction_replay \ - --gen_data_dir ${GEN_DATA_DIR} \ - --kl_ratio ${KL_RATIO} \ - --data_replay_freq -1 \ - --replay_after_n_epoch 0" - fi - - CUDA_VISIBLE_DEVICES=${GPU_IDS} torchrun --nproc_per_node=2 src/run_t5.py \ - --do_train \ - --do_predict \ - --predict_with_generate \ - --model_name_or_path "${MODEL_PATH}" \ - --data_dir CL_Benchmark \ - --task_order "${TASK_ORDER}" \ - --task_config_dir "${CONFIG_PREFIX}/${TASK}" \ - --output_dir "${OUTDIR}" \ - --per_device_train_batch_size ${BATCH_SIZE} \ - --per_device_eval_batch_size ${EVAL_BATCH} \ - --gradient_accumulation_steps ${GRAD_ACCUM} \ - --learning_rate ${LR} \ - --num_train_epochs ${EPOCHS} \ - --run_name "${RUN_NAME}" \ - --max_source_length 512 \ - --max_target_length 50 \ - --generation_max_length 50 \ - --add_task_name False \ - --add_dataset_name False \ - --lr_scheduler_type constant \ - --warmup_steps 0 \ - --logging_strategy steps \ - --logging_steps 10 \ - --eval_strategy steps \ - --save_strategy steps \ - --save_total_limit 1 \ - --load_best_model_at_end \ - --metric_for_best_model "${METRIC_KEY}" \ - --lora_r 4 \ - --lora_alpha 32 \ - --lora_dropout 0.0 \ - --attn_temperature 1 \ - --mlp_hidden_dim 100 \ - --model_name gainlora_inflora \ - --threshold 0.995 \ - --transthreshold 0.995 \ - --fp16 \ - --gradient_checkpointing True \ - ${OT_FLAGS} \ - ${PREV_FLAGS} - - if [[ -z "${PREV_LORA_PATHS}" ]]; then - PREV_LORA_PATHS="${OUTDIR}/saved_weights" - else - PREV_LORA_PATHS="${PREV_LORA_PATHS},${OUTDIR}/saved_weights" - fi - - echo " [DONE] Task ${TASKNUM}/${T}: ${TASK}" -done - -# ── Final AP/FT ────────────────────────────────────────────────────────────── -echo "" -echo "════════════════════════════════════════════════" -echo " Computing AP / FT ..." -echo "════════════════════════════════════════════════" -python src/compute_ap_ft.py \ - --output_base "${OUTPUT_BASE}" \ - --task_order "${TASK_ORDER}" \ - --method_name "${RUN_NAME}" \ - --save - -echo "" -echo "✓ All done. Results in: logs_and_outputs/${RUN_NAME}/" diff --git a/gainlora_OT_vMF/run_ot_sign_order4_t5large.sh b/gainlora_OT_vMF/run_ot_sign_order4_t5large.sh deleted file mode 100755 index 95fbfd60c55b8616335e647847b630545fd25121..0000000000000000000000000000000000000000 --- a/gainlora_OT_vMF/run_ot_sign_order4_t5large.sh +++ /dev/null @@ -1,169 +0,0 @@ -#!/bin/bash -############################################################################### -# OT-SIGN + GainLoRA (T5-Large) — Order 4 (Long Benchmark) -# ========================================================== -# Task order: mnli → cb → wic → ... → yahoo (15 tasks) -# Benchmark: Long (Exact Match metric) -# Hardware: 2× T4 16GB | Estimated time: ~3-4h -# Run: bash run_ot_sign_order4_t5large.sh [GPU_IDS] [MODEL_PATH] -############################################################################### - -set -e - -export CUDA_DEVICE_ORDER="PCI_BUS_ID" -GPU_IDS="${1:-0,1}" -MODEL_PATH="${2:-google/flan-t5-large}" - -# ── Config ────────────────────────────────────────────────────────────────── -RUN_NAME="ot_sign_order4_t5large" -OUTPUT_BASE="logs_and_outputs/${RUN_NAME}/outputs" -CONFIG_PREFIX="configs/gen_script_long_order4_t5_configs" -GEN_DATA_DIR="generated_data/lora_gen_long_t5" -METRIC_TYPE="exact_match" -KL_RATIO=0.1 -EPOCHS=10 - -BATCH_SIZE=4 -EVAL_BATCH=16 -GRAD_ACCUM=8 -LR=0.0003 - -OT_EPSILON=0.05 -OT_N_ITER=10 -DEFAULT_KAPPA=10.0 -LAMBDA_DRIFT=0.01 -LAMBDA_INV=0.001 -THRESHOLD=2.3 - -# ── Task sequence ──────────────────────────────────────────────────────────── -TASKS=( - "mnli" - "cb" - "wic" - "copa" - "qqp" - "boolq" - "rte" - "imdb" - "yelp" - "amazon" - "sst2" - "dbpedia" - "agnews" - "multirc" - "yahoo" -) -TASK_ORDER=$(IFS=,; echo "${TASKS[*]}") -T=${#TASKS[@]} - -mkdir -p "${OUTPUT_BASE}" -echo "════════════════════════════════════════════════" -echo " OT-SIGN+GainLoRA | Order 4 | T5-Large" -echo " GPUs: ${GPU_IDS} | Tasks: ${T} | Epochs: ${EPOCHS}" -echo "════════════════════════════════════════════════" - -# ── Training loop ──────────────────────────────────────────────────────────── -PREV_LORA_PATHS="" - -for i in "${!TASKS[@]}"; do - TASK="${TASKS[$i]}" - TASKNUM=$((i + 1)) - OUTDIR="${OUTPUT_BASE}/${TASKNUM}-${TASK}" - - if [[ $i -eq 0 ]]; then - METRIC_KEY="eval_${METRIC_TYPE}" - else - METRIC_KEY="eval_${METRIC_TYPE}_for_${TASK}" - fi - - echo "" - echo "──────────────────────────────────────────────" - echo " Task ${TASKNUM}/${T}: ${TASK}" - echo "──────────────────────────────────────────────" - - OT_FLAGS="--use_ot_routing True \ - --ot_epsilon ${OT_EPSILON} \ - --ot_n_iter ${OT_N_ITER} \ - --default_kappa ${DEFAULT_KAPPA} \ - --lambda_drift ${LAMBDA_DRIFT} \ - --lambda_inv ${LAMBDA_INV} \ - --invasion_threshold ${THRESHOLD}" - - PREV_FLAGS="" - if [[ $i -gt 0 ]]; then - PREV_OUTDIR="${OUTPUT_BASE}/${i}-${TASKS[$((i-1))]}/saved_weights" - PREV_FLAGS="--load_checkpoint_from ${PREV_OUTDIR}/trans_input.pt \ - --previous_lora_path ${PREV_LORA_PATHS} \ - --previous_prompt_key_path ${PREV_OUTDIR}/prompts_keys_till_now.pt \ - --previous_vmf_signatures_path ${PREV_OUTDIR}/vmf_signatures.pt \ - --add_instruction_replay \ - --gen_data_dir ${GEN_DATA_DIR} \ - --kl_ratio ${KL_RATIO} \ - --data_replay_freq -1 \ - --replay_after_n_epoch 0" - fi - - CUDA_VISIBLE_DEVICES=${GPU_IDS} torchrun --nproc_per_node=2 src/run_t5.py \ - --do_train \ - --do_predict \ - --predict_with_generate \ - --model_name_or_path "${MODEL_PATH}" \ - --data_dir CL_Benchmark \ - --task_order "${TASK_ORDER}" \ - --task_config_dir "${CONFIG_PREFIX}/${TASK}" \ - --output_dir "${OUTDIR}" \ - --per_device_train_batch_size ${BATCH_SIZE} \ - --per_device_eval_batch_size ${EVAL_BATCH} \ - --gradient_accumulation_steps ${GRAD_ACCUM} \ - --learning_rate ${LR} \ - --num_train_epochs ${EPOCHS} \ - --run_name "${RUN_NAME}" \ - --max_source_length 512 \ - --max_target_length 50 \ - --generation_max_length 50 \ - --add_task_name False \ - --add_dataset_name False \ - --lr_scheduler_type constant \ - --warmup_steps 0 \ - --logging_strategy steps \ - --logging_steps 10 \ - --eval_strategy steps \ - --save_strategy steps \ - --save_total_limit 1 \ - --load_best_model_at_end \ - --metric_for_best_model "${METRIC_KEY}" \ - --lora_r 4 \ - --lora_alpha 32 \ - --lora_dropout 0.0 \ - --attn_temperature 1 \ - --mlp_hidden_dim 100 \ - --model_name gainlora_inflora \ - --threshold 0.995 \ - --transthreshold 0.995 \ - --fp16 \ - --gradient_checkpointing True \ - ${OT_FLAGS} \ - ${PREV_FLAGS} - - if [[ -z "${PREV_LORA_PATHS}" ]]; then - PREV_LORA_PATHS="${OUTDIR}/saved_weights" - else - PREV_LORA_PATHS="${PREV_LORA_PATHS},${OUTDIR}/saved_weights" - fi - - echo " [DONE] Task ${TASKNUM}/${T}: ${TASK}" -done - -# ── Final AP/FT ────────────────────────────────────────────────────────────── -echo "" -echo "════════════════════════════════════════════════" -echo " Computing AP / FT ..." -echo "════════════════════════════════════════════════" -python src/compute_ap_ft.py \ - --output_base "${OUTPUT_BASE}" \ - --task_order "${TASK_ORDER}" \ - --method_name "${RUN_NAME}" \ - --save - -echo "" -echo "✓ All done. Results in: logs_and_outputs/${RUN_NAME}/" diff --git a/gainlora_OT_vMF/run_ot_sign_smoke_t5.sh b/gainlora_OT_vMF/run_ot_sign_smoke_t5.sh deleted file mode 100755 index d3b3bf21f13afd3b50a40bb4de4cb36726291367..0000000000000000000000000000000000000000 --- a/gainlora_OT_vMF/run_ot_sign_smoke_t5.sh +++ /dev/null @@ -1,70 +0,0 @@ -#!/bin/bash -# Fast smoke run for early bug detection (2-5 minutes, not 30+ minutes). - -set -euo pipefail - -GPU_IDS="${1:-0}" -MODEL_PATH="${2:-google/flan-t5-small}" - -RUN_NAME="ot_sign_smoke_t5" -OUTDIR="logs_and_outputs/${RUN_NAME}/outputs" -TASK="yelp" -TASK_ORDER="yelp" -CONFIG_DIR="configs/gen_script_long_order3_t5_configs/${TASK}" - -echo "[1/2] Running preflight checks..." -python src/preflight_check.py - -echo "[2/2] Running smoke training (tiny data, tiny steps)..." -CUDA_VISIBLE_DEVICES="${GPU_IDS}" torchrun --nproc_per_node=1 src/run_t5.py \ - --do_train \ - --do_eval \ - --model_name_or_path "${MODEL_PATH}" \ - --data_dir CL_Benchmark \ - --task_order "${TASK_ORDER}" \ - --task_config_dir "${CONFIG_DIR}" \ - --output_dir "${OUTDIR}" \ - --overwrite_output_dir True \ - --per_device_train_batch_size 1 \ - --per_device_eval_batch_size 1 \ - --gradient_accumulation_steps 1 \ - --learning_rate 3e-4 \ - --num_train_epochs 1 \ - --max_steps 3 \ - --max_num_instances_per_task 8 \ - --max_num_instances_per_eval_task 8 \ - --max_train_samples 8 \ - --max_eval_samples 8 \ - --max_predict_samples 8 \ - --run_name "${RUN_NAME}" \ - --max_source_length 128 \ - --max_target_length 32 \ - --generation_max_length 32 \ - --add_task_name False \ - --add_dataset_name False \ - --lr_scheduler_type constant \ - --warmup_steps 0 \ - --logging_strategy steps \ - --logging_steps 1 \ - --eval_strategy steps \ - --eval_steps 1 \ - --save_strategy no \ - --lora_r 4 \ - --lora_alpha 32 \ - --lora_dropout 0.0 \ - --attn_temperature 1 \ - --mlp_hidden_dim 32 \ - --model_name gainlora_inflora \ - --threshold 0.995 \ - --transthreshold 0.995 \ - --fp16 \ - --gradient_checkpointing True \ - --use_ot_routing True \ - --ot_epsilon 0.05 \ - --ot_n_iter 5 \ - --default_kappa 10.0 \ - --lambda_drift 0.01 \ - --lambda_inv 0.001 \ - --invasion_threshold 2.3 - -echo "Smoke run done." diff --git a/gainlora_OT_vMF/run_ot_sign_t4.sh b/gainlora_OT_vMF/run_ot_sign_t4.sh deleted file mode 100755 index c1d1f9686799c958455b8566131290ab2a768e0f..0000000000000000000000000000000000000000 --- a/gainlora_OT_vMF/run_ot_sign_t4.sh +++ /dev/null @@ -1,488 +0,0 @@ -#!/bin/bash -############################################################################### -# OT-SIGN + GainLoRA (T5-Large) — Optimized for 2×T4 (16GB each) -# ================================================================ -# Changes from original: -# - batch_size: 16 → 8 (T4 memory constraint) -# - fp16: enabled (T4 has good FP16 throughput) -# - gradient_accumulation: 4 (compensate for smaller batch) -# - OT-SIGN: use_ot_routing enabled from task 2 onward -# - OT-SIGN: vMF signatures saved/loaded between tasks -# - OT-SIGN: anti-drift + anti-invasion losses activated -# - num_train_epochs: 50 (reduced for faster iteration on T4) -# - ipdb/cupy guards already handled in code -############################################################################### - -export CUDA_DEVICE_ORDER="PCI_BUS_ID" - -# === Configuration === -GPU_IDS="${1:-0,1}" # Default: use GPU 0,1 -MODEL_PATH="${2:-google/flan-t5-large}" # T5-Large -RUN_NAME="ot_sign_t5_gainlora_t4" -TASK_ORDER="task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification" -OUTPUT_BASE="logs_and_outputs/${RUN_NAME}/outputs" - -# T4-optimized hyperparameters -BATCH_SIZE=8 -EVAL_BATCH_SIZE=4 -GRAD_ACCUM=4 -LR=0.0003 -EPOCHS=50 -MAX_SRC=512 -MAX_TGT=50 -LORA_R=4 -LORA_ALPHA=32 - -# OT-SIGN hyperparameters -OT_EPSILON=0.05 -OT_N_ITER=10 -DEFAULT_KAPPA=10.0 -LAMBDA_DRIFT=0.01 -LAMBDA_INV=0.001 -INVASION_THRESHOLD=2.3 - -# Common flags -COMMON_FLAGS="--predict_with_generate \ - --model_name_or_path ${MODEL_PATH} \ - --data_dir CL_Benchmark \ - --task_order ${TASK_ORDER} \ - --per_device_eval_batch_size ${EVAL_BATCH_SIZE} \ - --learning_rate ${LR} \ - --num_train_epochs ${EPOCHS} \ - --run_name ${RUN_NAME} \ - --max_source_length ${MAX_SRC} \ - --max_target_length ${MAX_TGT} \ - --generation_max_length ${MAX_TGT} \ - --add_task_name False \ - --add_dataset_name False \ - --lr_scheduler_type constant \ - --warmup_steps 0 \ - --logging_strategy steps \ - --logging_steps 10 \ - --eval_strategy steps \ - --save_strategy steps \ - --save_total_limit 1 \ - --load_best_model_at_end \ - --lora_r ${LORA_R} \ - --lora_alpha ${LORA_ALPHA} \ - --lora_dropout 0.0 \ - --attn_temperature 1 \ - --mlp_hidden_dim 100 \ - --model_name gainlora_inflora \ - --threshold 0.995 \ - --transthreshold 0.995 \ - --fp16" - -# OT-SIGN flags (used from task 2 onward) -OT_SIGN_FLAGS="--use_ot_routing True \ - --ot_epsilon ${OT_EPSILON} \ - --ot_n_iter ${OT_N_ITER} \ - --default_kappa ${DEFAULT_KAPPA} \ - --lambda_drift ${LAMBDA_DRIFT} \ - --lambda_inv ${LAMBDA_INV} \ - --invasion_threshold ${INVASION_THRESHOLD}" - -echo "=============================================" -echo "OT-SIGN + GainLoRA T5-Large on 2×T4" -echo "Run: ${RUN_NAME}" -echo "Model: ${MODEL_PATH}" -echo "Batch: ${BATCH_SIZE} × ${GRAD_ACCUM} grad_accum" -echo "FP16: enabled" -echo "OT-SIGN: epsilon=${OT_EPSILON}, kappa_def=${DEFAULT_KAPPA}" -echo "=============================================" - -# Create output directories -mkdir -p ${OUTPUT_BASE} -mkdir -p logs_and_outputs/${RUN_NAME}/ot_sign_logs - -############################################################################### -# Task 1: task1572_samsum_summary (NO OT routing — first task, no signatures) -############################################################################### -echo "" -echo "===== Task 1/15: task1572_samsum_summary (baseline, no OT) =====" -CUDA_VISIBLE_DEVICES=${GPU_IDS} python src/run_t5.py \ - --do_train \ - --do_predict \ - ${COMMON_FLAGS} \ - --task_config_dir configs/gen_script_superni_order1_t5_configs/task1572_samsum_summary \ - --output_dir ${OUTPUT_BASE}/1-task1572_samsum_summary \ - --per_device_train_batch_size ${BATCH_SIZE} \ - --gradient_accumulation_steps ${GRAD_ACCUM} \ - --metric_for_best_model eval_rougeL \ - --data_replay_freq -1 \ - --replay_after_n_epoch 0 \ - --kl_ratio 0.5 \ - --use_ot_routing True \ - --ot_epsilon ${OT_EPSILON} \ - --ot_n_iter ${OT_N_ITER} \ - --default_kappa ${DEFAULT_KAPPA} - -############################################################################### -# Task 2: task363_sst2 (OT routing with 1 signature from task 1) -############################################################################### -echo "" -echo "===== Task 2/15: task363_sst2_polarity_classification (OT-SIGN active) =====" -PREV_1="${OUTPUT_BASE}/1-task1572_samsum_summary/saved_weights" -CUDA_VISIBLE_DEVICES=${GPU_IDS} python src/run_t5.py \ - --do_train \ - --do_predict \ - ${COMMON_FLAGS} \ - --load_checkpoint_from ${PREV_1}/trans_input.pt \ - --previous_lora_path ${PREV_1} \ - --previous_prompt_key_path ${PREV_1}/prompts_keys_till_now.pt \ - --previous_vmf_signatures_path ${PREV_1}/vmf_signatures.pt \ - --gen_data_dir generated_data/lora_gen_superni_t5 \ - --task_config_dir configs/gen_script_superni_order1_t5_configs/task363_sst2_polarity_classification \ - --output_dir ${OUTPUT_BASE}/2-task363_sst2_polarity_classification \ - --per_device_train_batch_size ${BATCH_SIZE} \ - --gradient_accumulation_steps ${GRAD_ACCUM} \ - --metric_for_best_model eval_rougeL_for_task363_sst2_polarity_classification \ - --add_instruction_replay \ - --data_replay_freq -1 \ - --replay_after_n_epoch 0 \ - --kl_ratio 0.5 \ - ${OT_SIGN_FLAGS} - -############################################################################### -# Task 3: task1290_xsum_summarization -############################################################################### -echo "" -echo "===== Task 3/15: task1290_xsum_summarization (OT-SIGN active) =====" -PREV_2="${OUTPUT_BASE}/2-task363_sst2_polarity_classification/saved_weights" -CUDA_VISIBLE_DEVICES=${GPU_IDS} python src/run_t5.py \ - --do_train \ - --do_predict \ - ${COMMON_FLAGS} \ - --load_checkpoint_from ${PREV_2}/trans_input.pt \ - --previous_lora_path ${PREV_1},${PREV_2} \ - --previous_prompt_key_path ${PREV_2}/prompts_keys_till_now.pt \ - --previous_vmf_signatures_path ${PREV_2}/vmf_signatures.pt \ - --gen_data_dir generated_data/lora_gen_superni_t5 \ - --task_config_dir configs/gen_script_superni_order1_t5_configs/task1290_xsum_summarization \ - --output_dir ${OUTPUT_BASE}/3-task1290_xsum_summarization \ - --per_device_train_batch_size ${BATCH_SIZE} \ - --gradient_accumulation_steps ${GRAD_ACCUM} \ - --metric_for_best_model eval_rougeL_for_task1290_xsum_summarization \ - --add_instruction_replay \ - --data_replay_freq -1 \ - --replay_after_n_epoch 0 \ - --kl_ratio 0.5 \ - ${OT_SIGN_FLAGS} - -############################################################################### -# Task 4: task181_outcome_extraction -############################################################################### -echo "" -echo "===== Task 4/15: task181_outcome_extraction (OT-SIGN active) =====" -PREV_3="${OUTPUT_BASE}/3-task1290_xsum_summarization/saved_weights" -CUDA_VISIBLE_DEVICES=${GPU_IDS} python src/run_t5.py \ - --do_train \ - --do_predict \ - ${COMMON_FLAGS} \ - --load_checkpoint_from ${PREV_3}/trans_input.pt \ - --previous_lora_path ${PREV_1},${PREV_2},${PREV_3} \ - --previous_prompt_key_path ${PREV_3}/prompts_keys_till_now.pt \ - --previous_vmf_signatures_path ${PREV_3}/vmf_signatures.pt \ - --gen_data_dir generated_data/lora_gen_superni_t5 \ - --task_config_dir configs/gen_script_superni_order1_t5_configs/task181_outcome_extraction \ - --output_dir ${OUTPUT_BASE}/4-task181_outcome_extraction \ - --per_device_train_batch_size ${BATCH_SIZE} \ - --gradient_accumulation_steps ${GRAD_ACCUM} \ - --metric_for_best_model eval_rougeL_for_task181_outcome_extraction \ - --add_instruction_replay \ - --data_replay_freq -1 \ - --replay_after_n_epoch 0 \ - --kl_ratio 0.5 \ - ${OT_SIGN_FLAGS} - -############################################################################### -# Task 5: task002_quoref_answer_generation -############################################################################### -echo "" -echo "===== Task 5/15: task002_quoref_answer_generation (OT-SIGN active) =====" -PREV_4="${OUTPUT_BASE}/4-task181_outcome_extraction/saved_weights" -CUDA_VISIBLE_DEVICES=${GPU_IDS} python src/run_t5.py \ - --do_train \ - --do_predict \ - ${COMMON_FLAGS} \ - --load_checkpoint_from ${PREV_4}/trans_input.pt \ - --previous_lora_path ${PREV_1},${PREV_2},${PREV_3},${PREV_4} \ - --previous_prompt_key_path ${PREV_4}/prompts_keys_till_now.pt \ - --previous_vmf_signatures_path ${PREV_4}/vmf_signatures.pt \ - --gen_data_dir generated_data/lora_gen_superni_t5 \ - --task_config_dir configs/gen_script_superni_order1_t5_configs/task002_quoref_answer_generation \ - --output_dir ${OUTPUT_BASE}/5-task002_quoref_answer_generation \ - --per_device_train_batch_size ${BATCH_SIZE} \ - --gradient_accumulation_steps ${GRAD_ACCUM} \ - --metric_for_best_model eval_rougeL_for_task002_quoref_answer_generation \ - --add_instruction_replay \ - --data_replay_freq -1 \ - --replay_after_n_epoch 0 \ - --kl_ratio 0.5 \ - ${OT_SIGN_FLAGS} - -############################################################################### -# Task 6: task1510_evalution_relation_extraction -############################################################################### -echo "" -echo "===== Task 6/15: task1510_evalution_relation_extraction (OT-SIGN active) =====" -PREV_5="${OUTPUT_BASE}/5-task002_quoref_answer_generation/saved_weights" -CUDA_VISIBLE_DEVICES=${GPU_IDS} python src/run_t5.py \ - --do_train \ - --do_predict \ - ${COMMON_FLAGS} \ - --load_checkpoint_from ${PREV_5}/trans_input.pt \ - --previous_lora_path ${PREV_1},${PREV_2},${PREV_3},${PREV_4},${PREV_5} \ - --previous_prompt_key_path ${PREV_5}/prompts_keys_till_now.pt \ - --previous_vmf_signatures_path ${PREV_5}/vmf_signatures.pt \ - --gen_data_dir generated_data/lora_gen_superni_t5 \ - --task_config_dir configs/gen_script_superni_order1_t5_configs/task1510_evalution_relation_extraction \ - --output_dir ${OUTPUT_BASE}/6-task1510_evalution_relation_extraction \ - --per_device_train_batch_size ${BATCH_SIZE} \ - --gradient_accumulation_steps ${GRAD_ACCUM} \ - --metric_for_best_model eval_rougeL_for_task1510_evalution_relation_extraction \ - --add_instruction_replay \ - --data_replay_freq -1 \ - --replay_after_n_epoch 0 \ - --kl_ratio 0.5 \ - ${OT_SIGN_FLAGS} - -############################################################################### -# Task 7: task639_multi_woz_user_utterance_generation -############################################################################### -echo "" -echo "===== Task 7/15: task639_multi_woz_user_utterance_generation (OT-SIGN active) =====" -PREV_6="${OUTPUT_BASE}/6-task1510_evalution_relation_extraction/saved_weights" -CUDA_VISIBLE_DEVICES=${GPU_IDS} python src/run_t5.py \ - --do_train \ - --do_predict \ - ${COMMON_FLAGS} \ - --load_checkpoint_from ${PREV_6}/trans_input.pt \ - --previous_lora_path ${PREV_1},${PREV_2},${PREV_3},${PREV_4},${PREV_5},${PREV_6} \ - --previous_prompt_key_path ${PREV_6}/prompts_keys_till_now.pt \ - --previous_vmf_signatures_path ${PREV_6}/vmf_signatures.pt \ - --gen_data_dir generated_data/lora_gen_superni_t5 \ - --task_config_dir configs/gen_script_superni_order1_t5_configs/task639_multi_woz_user_utterance_generation \ - --output_dir ${OUTPUT_BASE}/7-task639_multi_woz_user_utterance_generation \ - --per_device_train_batch_size ${BATCH_SIZE} \ - --gradient_accumulation_steps ${GRAD_ACCUM} \ - --metric_for_best_model eval_rougeL_for_task639_multi_woz_user_utterance_generation \ - --add_instruction_replay \ - --data_replay_freq -1 \ - --replay_after_n_epoch 0 \ - --kl_ratio 0.5 \ - ${OT_SIGN_FLAGS} - -############################################################################### -# Task 8: task1729_personachat_generate_next -############################################################################### -echo "" -echo "===== Task 8/15: task1729_personachat_generate_next (OT-SIGN active) =====" -PREV_7="${OUTPUT_BASE}/7-task639_multi_woz_user_utterance_generation/saved_weights" -CUDA_VISIBLE_DEVICES=${GPU_IDS} python src/run_t5.py \ - --do_train \ - --do_predict \ - ${COMMON_FLAGS} \ - --load_checkpoint_from ${PREV_7}/trans_input.pt \ - --previous_lora_path ${PREV_1},${PREV_2},${PREV_3},${PREV_4},${PREV_5},${PREV_6},${PREV_7} \ - --previous_prompt_key_path ${PREV_7}/prompts_keys_till_now.pt \ - --previous_vmf_signatures_path ${PREV_7}/vmf_signatures.pt \ - --gen_data_dir generated_data/lora_gen_superni_t5 \ - --task_config_dir configs/gen_script_superni_order1_t5_configs/task1729_personachat_generate_next \ - --output_dir ${OUTPUT_BASE}/8-task1729_personachat_generate_next \ - --per_device_train_batch_size ${BATCH_SIZE} \ - --gradient_accumulation_steps ${GRAD_ACCUM} \ - --metric_for_best_model eval_rougeL_for_task1729_personachat_generate_next \ - --add_instruction_replay \ - --data_replay_freq -1 \ - --replay_after_n_epoch 0 \ - --kl_ratio 0.5 \ - ${OT_SIGN_FLAGS} - -############################################################################### -# Task 9: task073_commonsenseqa_answer_generation -############################################################################### -echo "" -echo "===== Task 9/15: task073_commonsenseqa_answer_generation (OT-SIGN active) =====" -PREV_8="${OUTPUT_BASE}/8-task1729_personachat_generate_next/saved_weights" -CUDA_VISIBLE_DEVICES=${GPU_IDS} python src/run_t5.py \ - --do_train \ - --do_predict \ - ${COMMON_FLAGS} \ - --load_checkpoint_from ${PREV_8}/trans_input.pt \ - --previous_lora_path ${PREV_1},${PREV_2},${PREV_3},${PREV_4},${PREV_5},${PREV_6},${PREV_7},${PREV_8} \ - --previous_prompt_key_path ${PREV_8}/prompts_keys_till_now.pt \ - --previous_vmf_signatures_path ${PREV_8}/vmf_signatures.pt \ - --gen_data_dir generated_data/lora_gen_superni_t5 \ - --task_config_dir configs/gen_script_superni_order1_t5_configs/task073_commonsenseqa_answer_generation \ - --output_dir ${OUTPUT_BASE}/9-task073_commonsenseqa_answer_generation \ - --per_device_train_batch_size ${BATCH_SIZE} \ - --gradient_accumulation_steps ${GRAD_ACCUM} \ - --metric_for_best_model eval_rougeL_for_task073_commonsenseqa_answer_generation \ - --add_instruction_replay \ - --data_replay_freq -1 \ - --replay_after_n_epoch 0 \ - --kl_ratio 0.5 \ - ${OT_SIGN_FLAGS} - -############################################################################### -# Task 10: task1590_diplomacy_text_generation -############################################################################### -echo "" -echo "===== Task 10/15: task1590_diplomacy_text_generation (OT-SIGN active) =====" -PREV_9="${OUTPUT_BASE}/9-task073_commonsenseqa_answer_generation/saved_weights" -CUDA_VISIBLE_DEVICES=${GPU_IDS} python src/run_t5.py \ - --do_train \ - --do_predict \ - ${COMMON_FLAGS} \ - --load_checkpoint_from ${PREV_9}/trans_input.pt \ - --previous_lora_path ${PREV_1},${PREV_2},${PREV_3},${PREV_4},${PREV_5},${PREV_6},${PREV_7},${PREV_8},${PREV_9} \ - --previous_prompt_key_path ${PREV_9}/prompts_keys_till_now.pt \ - --previous_vmf_signatures_path ${PREV_9}/vmf_signatures.pt \ - --gen_data_dir generated_data/lora_gen_superni_t5 \ - --task_config_dir configs/gen_script_superni_order1_t5_configs/task1590_diplomacy_text_generation \ - --output_dir ${OUTPUT_BASE}/10-task1590_diplomacy_text_generation \ - --per_device_train_batch_size ${BATCH_SIZE} \ - --gradient_accumulation_steps ${GRAD_ACCUM} \ - --metric_for_best_model eval_rougeL_for_task1590_diplomacy_text_generation \ - --add_instruction_replay \ - --data_replay_freq -1 \ - --replay_after_n_epoch 0 \ - --kl_ratio 0.5 \ - ${OT_SIGN_FLAGS} - -############################################################################### -# Task 11: task748_glucose_reverse_cause_event_detection -############################################################################### -echo "" -echo "===== Task 11/15: task748_glucose_reverse_cause_event_detection (OT-SIGN active) =====" -PREV_10="${OUTPUT_BASE}/10-task1590_diplomacy_text_generation/saved_weights" -CUDA_VISIBLE_DEVICES=${GPU_IDS} python src/run_t5.py \ - --do_train \ - --do_predict \ - ${COMMON_FLAGS} \ - --load_checkpoint_from ${PREV_10}/trans_input.pt \ - --previous_lora_path ${PREV_1},${PREV_2},${PREV_3},${PREV_4},${PREV_5},${PREV_6},${PREV_7},${PREV_8},${PREV_9},${PREV_10} \ - --previous_prompt_key_path ${PREV_10}/prompts_keys_till_now.pt \ - --previous_vmf_signatures_path ${PREV_10}/vmf_signatures.pt \ - --gen_data_dir generated_data/lora_gen_superni_t5 \ - --task_config_dir configs/gen_script_superni_order1_t5_configs/task748_glucose_reverse_cause_event_detection \ - --output_dir ${OUTPUT_BASE}/11-task748_glucose_reverse_cause_event_detection \ - --per_device_train_batch_size ${BATCH_SIZE} \ - --gradient_accumulation_steps ${GRAD_ACCUM} \ - --metric_for_best_model eval_rougeL_for_task748_glucose_reverse_cause_event_detection \ - --add_instruction_replay \ - --data_replay_freq -1 \ - --replay_after_n_epoch 0 \ - --kl_ratio 0.5 \ - ${OT_SIGN_FLAGS} - -############################################################################### -# Task 12: task511_reddit_tifu_long_text_summarization -############################################################################### -echo "" -echo "===== Task 12/15: task511_reddit_tifu_long_text_summarization (OT-SIGN active) =====" -PREV_11="${OUTPUT_BASE}/11-task748_glucose_reverse_cause_event_detection/saved_weights" -CUDA_VISIBLE_DEVICES=${GPU_IDS} python src/run_t5.py \ - --do_train \ - --do_predict \ - ${COMMON_FLAGS} \ - --load_checkpoint_from ${PREV_11}/trans_input.pt \ - --previous_lora_path ${PREV_1},${PREV_2},${PREV_3},${PREV_4},${PREV_5},${PREV_6},${PREV_7},${PREV_8},${PREV_9},${PREV_10},${PREV_11} \ - --previous_prompt_key_path ${PREV_11}/prompts_keys_till_now.pt \ - --previous_vmf_signatures_path ${PREV_11}/vmf_signatures.pt \ - --gen_data_dir generated_data/lora_gen_superni_t5 \ - --task_config_dir configs/gen_script_superni_order1_t5_configs/task511_reddit_tifu_long_text_summarization \ - --output_dir ${OUTPUT_BASE}/12-task511_reddit_tifu_long_text_summarization \ - --per_device_train_batch_size ${BATCH_SIZE} \ - --gradient_accumulation_steps ${GRAD_ACCUM} \ - --metric_for_best_model eval_rougeL_for_task511_reddit_tifu_long_text_summarization \ - --add_instruction_replay \ - --data_replay_freq -1 \ - --replay_after_n_epoch 0 \ - --kl_ratio 0.5 \ - ${OT_SIGN_FLAGS} - -############################################################################### -# Task 13: task591_sciq_answer_generation -############################################################################### -echo "" -echo "===== Task 13/15: task591_sciq_answer_generation (OT-SIGN active) =====" -PREV_12="${OUTPUT_BASE}/12-task511_reddit_tifu_long_text_summarization/saved_weights" -CUDA_VISIBLE_DEVICES=${GPU_IDS} python src/run_t5.py \ - --do_train \ - --do_predict \ - ${COMMON_FLAGS} \ - --load_checkpoint_from ${PREV_12}/trans_input.pt \ - --previous_lora_path ${PREV_1},${PREV_2},${PREV_3},${PREV_4},${PREV_5},${PREV_6},${PREV_7},${PREV_8},${PREV_9},${PREV_10},${PREV_11},${PREV_12} \ - --previous_prompt_key_path ${PREV_12}/prompts_keys_till_now.pt \ - --previous_vmf_signatures_path ${PREV_12}/vmf_signatures.pt \ - --gen_data_dir generated_data/lora_gen_superni_t5 \ - --task_config_dir configs/gen_script_superni_order1_t5_configs/task591_sciq_answer_generation \ - --output_dir ${OUTPUT_BASE}/13-task591_sciq_answer_generation \ - --per_device_train_batch_size ${BATCH_SIZE} \ - --gradient_accumulation_steps ${GRAD_ACCUM} \ - --metric_for_best_model eval_rougeL_for_task591_sciq_answer_generation \ - --add_instruction_replay \ - --data_replay_freq -1 \ - --replay_after_n_epoch 0 \ - --kl_ratio 0.5 \ - ${OT_SIGN_FLAGS} - -############################################################################### -# Task 14: task1687_sentiment140_classification -############################################################################### -echo "" -echo "===== Task 14/15: task1687_sentiment140_classification (OT-SIGN active) =====" -PREV_13="${OUTPUT_BASE}/13-task591_sciq_answer_generation/saved_weights" -CUDA_VISIBLE_DEVICES=${GPU_IDS} python src/run_t5.py \ - --do_train \ - --do_predict \ - ${COMMON_FLAGS} \ - --load_checkpoint_from ${PREV_13}/trans_input.pt \ - --previous_lora_path ${PREV_1},${PREV_2},${PREV_3},${PREV_4},${PREV_5},${PREV_6},${PREV_7},${PREV_8},${PREV_9},${PREV_10},${PREV_11},${PREV_12},${PREV_13} \ - --previous_prompt_key_path ${PREV_13}/prompts_keys_till_now.pt \ - --previous_vmf_signatures_path ${PREV_13}/vmf_signatures.pt \ - --gen_data_dir generated_data/lora_gen_superni_t5 \ - --task_config_dir configs/gen_script_superni_order1_t5_configs/task1687_sentiment140_classification \ - --output_dir ${OUTPUT_BASE}/14-task1687_sentiment140_classification \ - --per_device_train_batch_size ${BATCH_SIZE} \ - --gradient_accumulation_steps ${GRAD_ACCUM} \ - --metric_for_best_model eval_rougeL_for_task1687_sentiment140_classification \ - --add_instruction_replay \ - --data_replay_freq -1 \ - --replay_after_n_epoch 0 \ - --kl_ratio 0.5 \ - ${OT_SIGN_FLAGS} - -############################################################################### -# Task 15: task875_emotion_classification -############################################################################### -echo "" -echo "===== Task 15/15: task875_emotion_classification (OT-SIGN active) =====" -PREV_14="${OUTPUT_BASE}/14-task1687_sentiment140_classification/saved_weights" -CUDA_VISIBLE_DEVICES=${GPU_IDS} python src/run_t5.py \ - --do_train \ - --do_predict \ - ${COMMON_FLAGS} \ - --load_checkpoint_from ${PREV_14}/trans_input.pt \ - --previous_lora_path ${PREV_1},${PREV_2},${PREV_3},${PREV_4},${PREV_5},${PREV_6},${PREV_7},${PREV_8},${PREV_9},${PREV_10},${PREV_11},${PREV_12},${PREV_13},${PREV_14} \ - --previous_prompt_key_path ${PREV_14}/prompts_keys_till_now.pt \ - --previous_vmf_signatures_path ${PREV_14}/vmf_signatures.pt \ - --gen_data_dir generated_data/lora_gen_superni_t5 \ - --task_config_dir configs/gen_script_superni_order1_t5_configs/task875_emotion_classification \ - --output_dir ${OUTPUT_BASE}/15-task875_emotion_classification \ - --per_device_train_batch_size ${BATCH_SIZE} \ - --gradient_accumulation_steps ${GRAD_ACCUM} \ - --metric_for_best_model eval_rougeL_for_task875_emotion_classification \ - --add_instruction_replay \ - --data_replay_freq -1 \ - --replay_after_n_epoch 0 \ - --kl_ratio 0.5 \ - ${OT_SIGN_FLAGS} - -echo "" -echo "=============================================" -echo "OT-SIGN Training Complete!" -echo "Logs: logs_and_outputs/${RUN_NAME}/ot_sign_logs/" -echo "=============================================" diff --git a/gainlora_OT_vMF/src/__pycache__/cl_collator.cpython-312.pyc b/gainlora_OT_vMF/src/__pycache__/cl_collator.cpython-312.pyc deleted file mode 100644 index 3d1e99a821f01942886ec830c1c0f5309a0ad47e..0000000000000000000000000000000000000000 --- a/gainlora_OT_vMF/src/__pycache__/cl_collator.cpython-312.pyc +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:7ac24a521ab05c5f986666baca176d46a7b716692db92c57ce86c0b150597b6e -size 10116 diff --git a/gainlora_OT_vMF/src/__pycache__/cl_dataset.cpython-312.pyc b/gainlora_OT_vMF/src/__pycache__/cl_dataset.cpython-312.pyc deleted file mode 100644 index e70e6d759031ec82d902020311250b0df182910b..0000000000000000000000000000000000000000 --- a/gainlora_OT_vMF/src/__pycache__/cl_dataset.cpython-312.pyc +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:f2e7871bd4488d77a1beca52f07be680fe96e2580847792047caf3d76364750d -size 13159 diff --git a/gainlora_OT_vMF/src/__pycache__/cl_trainer_gainlora_inflora.cpython-312.pyc b/gainlora_OT_vMF/src/__pycache__/cl_trainer_gainlora_inflora.cpython-312.pyc deleted file mode 100644 index 801bbfdf78a3cbebec9309ce953d7c98b251965e..0000000000000000000000000000000000000000 --- a/gainlora_OT_vMF/src/__pycache__/cl_trainer_gainlora_inflora.cpython-312.pyc +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:9751f103b0e86c9b3b8f269dae372351263cb0824656c821c90a8254847ee1e4 -size 90854 diff --git a/gainlora_OT_vMF/src/__pycache__/cl_trainer_gainlora_inflora_llama.cpython-312.pyc b/gainlora_OT_vMF/src/__pycache__/cl_trainer_gainlora_inflora_llama.cpython-312.pyc deleted file mode 100644 index 34e2f567c7e30f0c368929f700eb45fca1ec46d6..0000000000000000000000000000000000000000 --- a/gainlora_OT_vMF/src/__pycache__/cl_trainer_gainlora_inflora_llama.cpython-312.pyc +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:d36384d520c11413ee49678a86ebc5bdc3671c64ea4b8ea49631c4ef5037d9d0 -size 85068 diff --git a/gainlora_OT_vMF/src/__pycache__/cl_trainer_gainlora_olora.cpython-312.pyc b/gainlora_OT_vMF/src/__pycache__/cl_trainer_gainlora_olora.cpython-312.pyc deleted file mode 100644 index b2a495f671547c72a1a3fdb114e961f58dab95ff..0000000000000000000000000000000000000000 --- a/gainlora_OT_vMF/src/__pycache__/cl_trainer_gainlora_olora.cpython-312.pyc +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:370b7649906c9cf60e7fb0fd8839938433c42dbfcae79c7ff3e635313b62dcdf -size 76727 diff --git a/gainlora_OT_vMF/src/__pycache__/cl_trainer_gainlora_olora_llama.cpython-312.pyc b/gainlora_OT_vMF/src/__pycache__/cl_trainer_gainlora_olora_llama.cpython-312.pyc deleted file mode 100644 index 0850ac420ef5f1486522352e97bc3a71c9acf96e..0000000000000000000000000000000000000000 --- a/gainlora_OT_vMF/src/__pycache__/cl_trainer_gainlora_olora_llama.cpython-312.pyc +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:f1680b3b17975f4de053e9be0f7eab2ef7902865cbe78059677cd3e02a1b63c2 -size 76991 diff --git a/gainlora_OT_vMF/src/__pycache__/cl_trainer_inflora.cpython-312.pyc b/gainlora_OT_vMF/src/__pycache__/cl_trainer_inflora.cpython-312.pyc deleted file mode 100644 index dc35dec24ceda06ec4e85a8aba990c07c9a4cf88..0000000000000000000000000000000000000000 --- a/gainlora_OT_vMF/src/__pycache__/cl_trainer_inflora.cpython-312.pyc +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:77eefddbbd5e041b41abe092f7004787fd48805b2c8608eaef3278fdab1812bd -size 39965 diff --git a/gainlora_OT_vMF/src/__pycache__/cl_trainer_inflora_llama.cpython-312.pyc b/gainlora_OT_vMF/src/__pycache__/cl_trainer_inflora_llama.cpython-312.pyc deleted file mode 100644 index 027188762be2a881680059c5c64fcc2427a10ba7..0000000000000000000000000000000000000000 --- a/gainlora_OT_vMF/src/__pycache__/cl_trainer_inflora_llama.cpython-312.pyc +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:4fa7568e94200b9196177ed47ea2c4d203ced2796d4f68df9d737b3f8c74fbd4 -size 41975 diff --git a/gainlora_OT_vMF/src/__pycache__/cl_trainer_olora.cpython-312.pyc b/gainlora_OT_vMF/src/__pycache__/cl_trainer_olora.cpython-312.pyc deleted file mode 100644 index a29626a8a9bae69e7cef934c7d8abf7f078ba83e..0000000000000000000000000000000000000000 --- a/gainlora_OT_vMF/src/__pycache__/cl_trainer_olora.cpython-312.pyc +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:5d0394c51dfd84ef4fde8f259a4bd2a5bfba5fc5ded9874c7e48ec016335154e -size 29030 diff --git a/gainlora_OT_vMF/src/__pycache__/cl_trainer_olora_llama.cpython-312.pyc b/gainlora_OT_vMF/src/__pycache__/cl_trainer_olora_llama.cpython-312.pyc deleted file mode 100644 index 270f3b162eb63e1ff625edd276134597cc485b36..0000000000000000000000000000000000000000 --- a/gainlora_OT_vMF/src/__pycache__/cl_trainer_olora_llama.cpython-312.pyc +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:3e7defc47276232e838f31b4b8dce26c968c00b772590be8a2105a77190ffaa6 -size 29042 diff --git a/gainlora_OT_vMF/src/__pycache__/compat_transformers.cpython-312.pyc b/gainlora_OT_vMF/src/__pycache__/compat_transformers.cpython-312.pyc deleted file mode 100644 index cd5c4f91ecebb6cadd1477c0ae24c9627e845f3b..0000000000000000000000000000000000000000 --- a/gainlora_OT_vMF/src/__pycache__/compat_transformers.cpython-312.pyc +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:e1dfbaa2911f50bbc0919979a7abaebff3700e1c54651e42f2c19bf2c4f8e1b5 -size 6959 diff --git a/gainlora_OT_vMF/src/__pycache__/llama_gainlora_inflora.cpython-312.pyc b/gainlora_OT_vMF/src/__pycache__/llama_gainlora_inflora.cpython-312.pyc deleted file mode 100644 index 0209c72b350789e7c923b3dd046852ada022edd4..0000000000000000000000000000000000000000 --- a/gainlora_OT_vMF/src/__pycache__/llama_gainlora_inflora.cpython-312.pyc +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:4bd9b3d05288494ad72b0151d68a39e9381752a5ef49819c9f7576d87a27ac4b -size 83793 diff --git a/gainlora_OT_vMF/src/__pycache__/llama_inflora.cpython-312.pyc b/gainlora_OT_vMF/src/__pycache__/llama_inflora.cpython-312.pyc deleted file mode 100644 index 64bfb981b6cce4864b7c55071b0ddb9173aa2dcc..0000000000000000000000000000000000000000 --- a/gainlora_OT_vMF/src/__pycache__/llama_inflora.cpython-312.pyc +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:df016d6843864aa718d9b408bef534b204cdfd89b6ab8a3a65666f8dc0159245 -size 74109 diff --git a/gainlora_OT_vMF/src/__pycache__/ot_sign_utils.cpython-312.pyc b/gainlora_OT_vMF/src/__pycache__/ot_sign_utils.cpython-312.pyc deleted file mode 100644 index e19d036bf8af8a9984f639f703e59b98f97b2b1f..0000000000000000000000000000000000000000 --- a/gainlora_OT_vMF/src/__pycache__/ot_sign_utils.cpython-312.pyc +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:55761c6d084de04d7ee473dccb92cdffe94d819ce33435580a56fc28297df409 -size 13082 diff --git a/gainlora_OT_vMF/src/__pycache__/ot_sign_utils.cpython-313.pyc b/gainlora_OT_vMF/src/__pycache__/ot_sign_utils.cpython-313.pyc deleted file mode 100644 index 3ec5e3c91a9cf2d5036c9e576f9d38ed99bd9085..0000000000000000000000000000000000000000 --- a/gainlora_OT_vMF/src/__pycache__/ot_sign_utils.cpython-313.pyc +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:64711cfa682645e7d2df1e923d09f0dc35d4ffb8c45807972df75a4c1c1755a1 -size 12377 diff --git a/gainlora_OT_vMF/src/__pycache__/run_llama.cpython-312.pyc b/gainlora_OT_vMF/src/__pycache__/run_llama.cpython-312.pyc deleted file mode 100644 index f20022a60fd6d996d30ce29e09c0b769dc4cc43c..0000000000000000000000000000000000000000 --- a/gainlora_OT_vMF/src/__pycache__/run_llama.cpython-312.pyc +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:ff4021939dae2e3c66abfccb78c774324ec19de43dee71d5c8838fd7e37167c2 -size 44561 diff --git a/gainlora_OT_vMF/src/__pycache__/run_logger.cpython-313.pyc b/gainlora_OT_vMF/src/__pycache__/run_logger.cpython-313.pyc deleted file mode 100644 index b5d2c4220e0ea49d1f23ad28c767977e542d8347..0000000000000000000000000000000000000000 --- a/gainlora_OT_vMF/src/__pycache__/run_logger.cpython-313.pyc +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:96c719d7c1cf1f2bd4dddc4808b5f512a221ec5a4f8ebd1a0c60d4069216339f -size 5510 diff --git a/gainlora_OT_vMF/src/__pycache__/run_t5.cpython-312.pyc b/gainlora_OT_vMF/src/__pycache__/run_t5.cpython-312.pyc deleted file mode 100644 index 4a58ad0a2b74b0aec7090282d713b0d0d32cfb50..0000000000000000000000000000000000000000 --- a/gainlora_OT_vMF/src/__pycache__/run_t5.cpython-312.pyc +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:b490149aa88456c921dbecd66d0b6119a7a6be4cdb8634f4d5efcb2d0dad43e7 -size 48700 diff --git a/gainlora_OT_vMF/src/__pycache__/t5_gainlora_inflora.cpython-312.pyc b/gainlora_OT_vMF/src/__pycache__/t5_gainlora_inflora.cpython-312.pyc deleted file mode 100644 index 120c28d7c82ab43e6ce11337bb2585a12edde460..0000000000000000000000000000000000000000 --- a/gainlora_OT_vMF/src/__pycache__/t5_gainlora_inflora.cpython-312.pyc +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:413a2c9a942c77bbc879103ca0c963de3a66d92b841aa6216a3239b44cf43665 -size 125803 diff --git a/gainlora_OT_vMF/src/__pycache__/t5_inflora.cpython-312.pyc b/gainlora_OT_vMF/src/__pycache__/t5_inflora.cpython-312.pyc deleted file mode 100644 index c29d844d63336dc10a9291240f5c088ff782fed8..0000000000000000000000000000000000000000 --- a/gainlora_OT_vMF/src/__pycache__/t5_inflora.cpython-312.pyc +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:8279a00efc964668a6729e13dcd430f08b4eaf1c1ee5ae31fffe31da9922bb9a -size 111402 diff --git a/gainlora_OT_vMF/src/compat_transformers.py b/gainlora_OT_vMF/src/compat_transformers.py deleted file mode 100644 index 13572da63af8ea57e9c2f5033e1b525aab9b954d..0000000000000000000000000000000000000000 --- a/gainlora_OT_vMF/src/compat_transformers.py +++ /dev/null @@ -1,206 +0,0 @@ -""" -Compatibility shims for transformers 4.x APIs removed in transformers 5.0. - -This module provides fallback definitions for all symbols and attributes -that were removed from the transformers library in version 5.0, allowing -the trainer code (originally written for transformers ~4.28) to run -without modification on transformers 5.0+. - -Usage: - from compat_transformers import ( - ShardedDDPOption, is_sagemaker_mp_enabled, smp_forward_backward, - deepspeed_init, deepspeed_load_checkpoint, HPSearchBackend, - patch_trainer_compat, - ) - - # In your Trainer subclass __init__: - class MyTrainer(Seq2SeqTrainer): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - patch_trainer_compat(self) -""" - -from enum import Enum - -# ============================================================================ -# ShardedDDPOption (removed in transformers 5.0, replaced by FSDP) -# ============================================================================ -try: - from transformers.trainer import ShardedDDPOption -except ImportError: - class ShardedDDPOption(str, Enum): - SIMPLE = "simple" - ZERO_DP_2 = "zero_dp_2" - ZERO_DP_3 = "zero_dp_3" - OFFLOAD = "offload" - -# ============================================================================ -# is_sagemaker_mp_enabled (removed in transformers 5.0) -# ============================================================================ -try: - from transformers.trainer import is_sagemaker_mp_enabled -except ImportError: - def is_sagemaker_mp_enabled(): - return False - -# ============================================================================ -# smp_forward_backward / smp_forward_only / smp_nested_tensor_size -# (SageMaker Model Parallel — removed in transformers 5.0) -# ============================================================================ -try: - from transformers.trainer import smp_forward_backward -except ImportError: - def smp_forward_backward(model, inputs, gradient_accumulation_steps=1): - raise RuntimeError("SageMaker Model Parallel is not available") - -try: - from transformers.trainer import smp_forward_only -except ImportError: - def smp_forward_only(model, inputs): - raise RuntimeError("SageMaker Model Parallel is not available") - -try: - from transformers.trainer import smp_nested_tensor_size -except ImportError: - def smp_nested_tensor_size(tensor): - return tensor.size() - -# ============================================================================ -# deepspeed_init (moved/changed in transformers 5.0) -# ============================================================================ -try: - from transformers.integrations import deepspeed_init -except ImportError: - try: - from transformers.deepspeed import deepspeed_init - except ImportError: - def deepspeed_init(trainer, num_training_steps, resume_from_checkpoint=None, inference=False): - raise RuntimeError("DeepSpeed is not available in this transformers version") - -# ============================================================================ -# deepspeed_load_checkpoint (moved/changed in transformers 5.0) -# ============================================================================ -try: - from transformers.integrations import deepspeed_load_checkpoint -except ImportError: - try: - from transformers.deepspeed import deepspeed_load_checkpoint - except ImportError: - def deepspeed_load_checkpoint(deepspeed_engine, checkpoint_path): - raise RuntimeError("DeepSpeed is not available in this transformers version") - -# ============================================================================ -# HPSearchBackend (removed in transformers 5.0) -# ============================================================================ -try: - from transformers.trainer import HPSearchBackend -except ImportError: - try: - from transformers.trainer_utils import HPSearchBackend - except ImportError: - class HPSearchBackend(str, Enum): - OPTUNA = "optuna" - RAY = "ray" - SIGOPT = "sigopt" - WANDB = "wandb" - -# ============================================================================ -# is_apex_available (removed in transformers 5.0) -# ============================================================================ -try: - from transformers.utils import is_apex_available -except ImportError: - def is_apex_available(): - return False - -# ============================================================================ -# ALL_LAYERNORM_LAYERS (removed from transformers.trainer in 5.0) -# ============================================================================ -try: - from transformers.trainer import ALL_LAYERNORM_LAYERS -except ImportError: - try: - from transformers.pytorch_utils import ALL_LAYERNORM_LAYERS - except ImportError: - import torch.nn as _nn - ALL_LAYERNORM_LAYERS = [_nn.LayerNorm] - -# ============================================================================ -# IS_SAGEMAKER_MP_POST_1_10 (SageMaker detection constant — removed in transformers 5.0) -# ============================================================================ -try: - from transformers.trainer import IS_SAGEMAKER_MP_POST_1_10 -except ImportError: - IS_SAGEMAKER_MP_POST_1_10 = False - -# ============================================================================ -# is_torch_tpu_available (removed in transformers 5.0, replaced by is_torch_xla_available) -# ============================================================================ -try: - from transformers.utils import is_torch_tpu_available -except ImportError: - try: - from transformers.utils import is_torch_xla_available as is_torch_tpu_available - except ImportError: - def is_torch_tpu_available(check_device=True): - return False - - -def patch_trainer_compat(trainer): - """ - Patch a Trainer instance with deprecated attributes that were removed - in transformers 5.0. Call this in __init__ after super().__init__(). - - Sets safe defaults for all deprecated attributes: - - sharded_ddp = None (FSDP replaces this) - - use_apex = False - - do_grad_scaling = False - - scaler = None - - deepspeed = None (the engine reference) - - current_flos = 0 - - hp_name = None - - hp_search_backend = None - - is_in_train = False - """ - if not hasattr(trainer, 'sharded_ddp'): - trainer.sharded_ddp = None - if not hasattr(trainer, 'fsdp'): - trainer.fsdp = None - if not hasattr(trainer, 'use_apex'): - trainer.use_apex = False - if not hasattr(trainer, 'do_grad_scaling'): - trainer.do_grad_scaling = False - if not hasattr(trainer, 'scaler'): - trainer.scaler = None - if not hasattr(trainer, 'deepspeed'): - trainer.deepspeed = None - if not hasattr(trainer, 'current_flos'): - trainer.current_flos = 0 - if not hasattr(trainer, 'hp_name'): - trainer.hp_name = None - if not hasattr(trainer, 'hp_search_backend'): - trainer.hp_search_backend = None - if not hasattr(trainer, 'is_in_train'): - trainer.is_in_train = False - if not hasattr(trainer, 'is_deepspeed_enabled'): - trainer.is_deepspeed_enabled = False - if not hasattr(trainer, '_trial'): - trainer._trial = None - if not hasattr(trainer, 'model_wrapped'): - trainer.model_wrapped = getattr(trainer, 'model', None) - -def patch_args_compat(args): - """ - Patch a TrainingArguments instance with deprecated attributes that were removed - in transformers 5.0. Call this in Trainer.__init__ to ensure args has all needed fields. - - - past_index: -1 (default, no cached predictions) - """ - if not hasattr(args, 'past_index'): - args.past_index = -1 - - # GainLoRA has parameters (prompt_key, previous_prompts_keys, vmf_signatures) - # not used in every forward pass. Transformers 5.0 sets find_unused_parameters=False - # when gradient_checkpointing=True, which causes DDP reduction errors. - if getattr(args, 'ddp_find_unused_parameters', None) is None: - args.ddp_find_unused_parameters = True \ No newline at end of file diff --git a/gainlora_OT_vMF/src/compute_ap_ft.py b/gainlora_OT_vMF/src/compute_ap_ft.py deleted file mode 100644 index 35cf975325cf4a47285caa37b4ca5fe3e475f349..0000000000000000000000000000000000000000 --- a/gainlora_OT_vMF/src/compute_ap_ft.py +++ /dev/null @@ -1,242 +0,0 @@ -#!/usr/bin/env python3 -""" -compute_ap_ft.py — Compute AP and FT from training output directories -====================================================================== -Usage: - python compute_ap_ft.py \\ - --output_base logs_and_outputs/ot_sign_order1/outputs \\ - --task_order "task1572_samsum_summary,task363_sst2_polarity_classification,..." \\ - --method_name "OT-SIGN+GainLoRA(InfLoRA)" - -AP = (1/T) * Σ_j R[T-1][j] — avg score on ALL tasks after training last task -FT = (1/(T-1)) * Σ_j (R[j][j] - R[T-1][j]) — avg performance DROP from peak - -Result matrix R[i][j]: - R[i][j] = score on task j right after model was trained on task i - Diagonal R[i][i] = peak performance on task i - Last row R[T-1][j] = final performance (what we report as AP) - -Score file locations (HuggingFace Trainer): - {output_base}/{i+1}-{task_name}/predict_results.json → has all task scores - {output_base}/{i+1}-{task_name}/eval_results.json → used as fallback peak -""" - -import argparse -import json -import os -import re -import sys - - -# ── Metric key detection ──────────────────────────────────────────────────── - -def _is_long_benchmark(task_list): - long_tasks = {'yelp', 'amazon', 'mnli', 'imdb', 'sst2', 'dbpedia', 'agnews', - 'yahoo', 'multirc', 'boolq', 'wic', 'copa', 'qqp', 'rte', 'cb'} - return any(t.lower() in long_tasks for t in task_list) - - -def _get_score(metrics, task_name, is_long): - """Extract score for `task_name` from a metrics dict.""" - prefix = "exact_match" if is_long else "rougeL" - candidates = [ - f"predict_{prefix}_for_{task_name}", - f"eval_{prefix}_for_{task_name}", - f"predict_{prefix}", - f"eval_{prefix}", - ] - for key in candidates: - if key in metrics: - v = float(metrics[key]) - # rougeL from compute_metrics is 0-1 scale → convert to % - if v <= 1.0 and "rouge" in key.lower(): - v = v * 100.0 - return round(v, 2) - - # broad fallback: any key mentioning the task name - for k, v in metrics.items(): - if task_name in k and isinstance(v, (int, float)): - v = float(v) - if v <= 1.0 and "rouge" in k.lower(): - v = v * 100.0 - return round(v, 2) - return None - - -def _load_json_file(path): - if not os.path.exists(path): - return {} - with open(path) as f: - return json.load(f) - - -# ── Result matrix builder ──────────────────────────────────────────────────── - -def build_result_matrix(output_base, task_order, is_long): - """ - R[i][j] = score on task j after training on task i. - R[i][i] = peak performance on task i (from predict_results when trained on i). - """ - T = len(task_order) - R = {} - - for i, task_i in enumerate(task_order): - # Find output directory for this task (numbered 1-based) - task_dir = _find_task_dir(output_base, i + 1, task_i) - if task_dir is None: - print(f" [WARN] Directory for task {i+1} ({task_i}) not found — skipping", file=sys.stderr) - continue - - predict_file = os.path.join(task_dir, "predict_results.json") - metrics = _load_json_file(predict_file) - - if not metrics: - # Try eval_results.json as fallback - metrics = _load_json_file(os.path.join(task_dir, "eval_results.json")) - - if not metrics: - print(f" [WARN] No metrics found in {task_dir}", file=sys.stderr) - continue - - R[i] = {} - # Extract score for EACH task visible at this training step (tasks 0..i) - for j in range(i + 1): - score = _get_score(metrics, task_order[j], is_long) - if score is not None: - R[i][j] = score - - return R - - -def _find_task_dir(output_base, task_num, task_name): - """Locate task output dir robustly: tries exact match, then prefix match.""" - # Try exact name: "1-task1572_samsum_summary" - candidate = os.path.join(output_base, f"{task_num}-{task_name}") - if os.path.isdir(candidate): - return candidate - - # Try prefix match - if os.path.isdir(output_base): - prefix = f"{task_num}-" - matches = [d for d in os.listdir(output_base) - if d.startswith(prefix) and os.path.isdir(os.path.join(output_base, d))] - if matches: - return os.path.join(output_base, sorted(matches)[0]) - return None - - -# ── AP / FT computation ─────────────────────────────────────────────────────── - -def compute_ap_ft(R, task_order): - """ - Args: - R: dict {i: {j: score}} — result matrix - task_order: list of task names - Returns: - ap, ft as floats (already rounded to 2 dp) - """ - T = len(task_order) - final_row = T - 1 - - # AP: average over final row - final_scores = [R.get(final_row, {}).get(j) for j in range(T)] - valid_final = [s for s in final_scores if s is not None] - ap = round(sum(valid_final) / len(valid_final), 2) if valid_final else 0.0 - - # FT: mean drop from diagonal to final row (exclude last task by convention) - drops = [] - for j in range(T - 1): - diag = R.get(j, {}).get(j) - final = R.get(final_row, {}).get(j) - if diag is not None and final is not None: - drops.append(diag - final) - ft = round(sum(drops) / len(drops), 2) if drops else 0.0 - - return ap, ft - - -# ── Output formatter ────────────────────────────────────────────────────────── - -def print_comparison_table(R, task_order, method_name, ap, ft): - T = len(task_order) - final_row = T - 1 - sep = "─" * 72 - - print(f"\n{'═' * 72}") - print(f" {method_name}") - print(f"{'═' * 72}") - print(f" {'#':<3} {'Task':<46} {'Peak':>6} {'Final':>6} {'Drop':>6}") - print(sep) - - for j, tname in enumerate(task_order): - diag = R.get(j, {}).get(j) - final = R.get(final_row, {}).get(j) - drop = round(diag - final, 2) if diag is not None and final is not None else None - d_str = f"{diag:.2f}" if diag is not None else " ——" - f_str = f"{final:.2f}" if final is not None else " ——" - r_str = f"{drop:.2f}" if drop is not None else " ——" - print(f" {j+1:<3} {tname:<46} {d_str:>6} {f_str:>6} {r_str:>6}") - - print(sep) - print(f" AP = {ap:>6.2f} │ FT = {ft:>5.2f}") - print(f"{'═' * 72}\n") - - -def save_results_json(R, task_order, method_name, ap, ft, output_base): - out = { - "method": method_name, - "ap": ap, - "ft": ft, - "per_task": {} - } - T = len(task_order) - final_row = T - 1 - for j, tname in enumerate(task_order): - out["per_task"][tname] = { - "peak": R.get(j, {}).get(j), - "final": R.get(final_row, {}).get(j), - } - - result_path = os.path.join(output_base, "..", "ap_ft_result.json") - result_path = os.path.normpath(result_path) - with open(result_path, 'w') as f: - json.dump(out, f, indent=2) - print(f"[compute_ap_ft] Saved results to {result_path}") - return result_path - - -# ── Main ───────────────────────────────────────────────────────────────────── - -def main(): - parser = argparse.ArgumentParser(description="Compute AP/FT from GainLoRA output directories") - parser.add_argument("--output_base", required=True, - help="Path to the outputs/ directory containing task subdirectories") - parser.add_argument("--task_order", required=True, - help="Comma-separated list of task names in training order") - parser.add_argument("--method_name", default="OT-SIGN+GainLoRA", - help="Label shown in the result table") - parser.add_argument("--save", action="store_true", - help="Also save results to ap_ft_result.json") - args = parser.parse_args() - - task_order = [t.strip() for t in args.task_order.split(',') if t.strip()] - is_long = _is_long_benchmark(task_order) - - print(f"[compute_ap_ft] Benchmark type: {'long' if is_long else 'superni'}") - print(f"[compute_ap_ft] Tasks: {len(task_order)} | Output base: {args.output_base}") - - R = build_result_matrix(args.output_base, task_order, is_long) - - if not R: - print("[compute_ap_ft] ERROR: No results found. Check --output_base path.", file=sys.stderr) - sys.exit(1) - - ap, ft = compute_ap_ft(R, task_order) - print_comparison_table(R, task_order, args.method_name, ap, ft) - - if args.save: - save_results_json(R, task_order, args.method_name, ap, ft, args.output_base) - - -if __name__ == "__main__": - main() diff --git a/gainlora_OT_vMF/src/ot_sign_utils.py b/gainlora_OT_vMF/src/ot_sign_utils.py deleted file mode 100644 index 12f6c2ea3ed7d0bedf835c4cbbd647d1eaa7b0cd..0000000000000000000000000000000000000000 --- a/gainlora_OT_vMF/src/ot_sign_utils.py +++ /dev/null @@ -1,289 +0,0 @@ -""" -OT-SIGN: Statistical Signatures + Optimal Transport Routing for GainLoRA -========================================================================= -Utility functions for: - 1. vMF (von Mises-Fisher) knowledge signatures - 2. Sinkhorn OT routing with vMF log-likelihood cost - 3. Anti-drift and anti-invasion losses - -Author: OT-SIGN implementation for GainLoRA continual learning -""" - -import torch -import torch.nn as nn -import torch.nn.functional as F -import math - - -# ============================================================================ -# 1. Sinkhorn Optimal Transport (log-domain for numerical stability) -# ============================================================================ - -def sinkhorn_log(C, epsilon=0.05, n_iter=10): - """ - Log-domain Sinkhorn algorithm for entropic OT. - - Args: - C: Cost matrix of shape (B, N) — cost of assigning sample b to expert n - epsilon: Entropic regularization strength (lower = sharper assignment) - n_iter: Number of Sinkhorn iterations - - Returns: - Transport plan of shape (B, N), rows sum to ~1 - """ - B, N = C.shape - - # Log kernel: K = exp(-C / epsilon) - log_K = -C / epsilon # (B, N) - - # Initialize dual variables - u = torch.zeros(B, device=C.device, dtype=C.dtype) - v = torch.zeros(N, device=C.device, dtype=C.dtype) - - for _ in range(n_iter): - # Update u (row normalization in log domain) - u = -torch.logsumexp(log_K + v.unsqueeze(0), dim=1) # (B,) - # Update v (column normalization in log domain) - v = -torch.logsumexp(log_K + u.unsqueeze(1), dim=0) # (N,) - - # Compute transport plan - log_pi = log_K + u.unsqueeze(1) + v.unsqueeze(0) # (B, N) - pi = log_pi.exp() - - # Normalize rows to sum to 1 (for routing weights) - pi = pi / (pi.sum(dim=1, keepdim=True) + 1e-8) - - return pi # (B, N) - - -# ============================================================================ -# 2. vMF Parameter Estimation (MLE) -# ============================================================================ - -def compute_vmf_params(features, d_model, kappa_min=0.1, kappa_max=50.0): - """ - Compute vMF MLE parameters from features on unit sphere. - - Args: - features: Tensor of shape (num_samples, d) — L2-normalized features - d_model: Feature dimension - kappa_min: Minimum allowed kappa (prevents numerical issues) - kappa_max: Maximum allowed kappa (prevents overflow) - - Returns: - mu: Mean direction (d,) — unit vector - kappa: Concentration parameter — scalar tensor - """ - # Mean resultant vector - x_bar = features.mean(dim=0) # (d,) - r_bar = x_bar.norm() # scalar — mean resultant length - - # Mean direction - mu = F.normalize(x_bar, dim=0) # (d,) - - # Concentration parameter (Banerjee et al. 2005 approximation) - d = d_model - r_bar_clamped = r_bar.clamp(min=1e-6, max=1.0 - 1e-6) - kappa = r_bar_clamped * (d - r_bar_clamped ** 2) / (1 - r_bar_clamped ** 2) - - # Clamp kappa for numerical stability - kappa = kappa.clamp(min=kappa_min, max=kappa_max) - - return mu.detach(), kappa.detach() - - -# ============================================================================ -# 3. Cost Matrix Construction -# ============================================================================ - -def vmf_cost_matrix(x, vmf_signatures, prompt_key=None, default_kappa=10.0): - """ - Build OT cost matrix from vMF signatures. - - ORDERING: Current task FIRST, then previous tasks sorted by task_id. - This matches GainLoRA's agg_lora_states ordering: - concat_q = cat([cur_lora, prev_lora_0, prev_lora_1, ...]) - - For current task: C[b, 0] = -default_kappa * (prompt_key · x_b) - For previous tasks: C[b, t+1] = -kappa_t * (mu_t · x_b) - - Args: - x: Normalized input features (B, d) - vmf_signatures: Dict {task_id: (mu, kappa)} for previous tasks - prompt_key: Current task's prompt key (1, d) or None - default_kappa: Kappa value used for the current task's proxy - - Returns: - C: Cost matrix (B, N_total) where N_total = 1 + len(signatures) - Position 0 = current task, positions 1..N-1 = prev tasks in order - """ - device = x.device - dtype = x.dtype - costs = [] - - # Position 0: Cost for current task (using prompt_key as proxy direction) - if prompt_key is not None: - pk = prompt_key.squeeze() # (d,) - pk = F.normalize(pk, dim=-1) - dot_product = (x * pk.unsqueeze(0)).sum(dim=-1) # (B,) - default_kappa_t = torch.tensor(default_kappa, device=device, dtype=dtype) - cost_cur = -default_kappa_t * dot_product # (B,) - costs.append(cost_cur) - - # Positions 1..N: Cost from vMF signatures (previous tasks) — sorted by task_id - for task_id in sorted(vmf_signatures.keys()): - mu_t, kappa_t = vmf_signatures[task_id] - mu_t = mu_t.to(device=device, dtype=dtype) - kappa_t_val = kappa_t.to(device=device, dtype=dtype) if isinstance(kappa_t, torch.Tensor) else torch.tensor(kappa_t, device=device, dtype=dtype) - - # vMF negative log-likelihood (ignoring constant normalization) - dot_product = (x * mu_t.unsqueeze(0)).sum(dim=-1) # (B,) - cost_t = -kappa_t_val * dot_product # (B,) — lower cost = higher likelihood - costs.append(cost_t) - - if len(costs) == 0: - raise ValueError("No signatures or prompt_key provided for cost matrix") - - C = torch.stack(costs, dim=1) # (B, N_total) - return C - - -# ============================================================================ -# 4. Anti-Drift Loss -# ============================================================================ - -def compute_anti_drift_loss(model_encoder, ref_trans_input_state, input_ids, attention_mask): - """ - MSE between current and reference trans_input outputs on replay data. - Protects the shared MLP from drifting away from old task representations. - - Args: - model_encoder: The model's encoder (has embed_tokens, trans_input) - ref_trans_input_state: Frozen state dict of trans_input from previous task - input_ids: Replay input token IDs (B, seq_len) - attention_mask: Replay attention mask (B, seq_len) - - Returns: - drift_loss: Scalar MSE loss - """ - # Get current trans_input output - with torch.no_grad(): - embeddings = model_encoder.embed_tokens(input_ids) - avg_emb = (attention_mask.unsqueeze(-1) * embeddings).mean(dim=1, keepdim=True) # (B, 1, d) - - # Current trans_input output - medium_curr = model_encoder.trans_input[1](model_encoder.trans_input[0](avg_emb)) - x_curr = model_encoder.trans_input[3](model_encoder.trans_input[2](medium_curr)) - x_curr = F.normalize(x_curr, dim=-1) # (B, 1, d) - - # Reference trans_input output (frozen copy) - ref_trans_input = nn.Sequential( - nn.Linear(model_encoder.trans_input[0].in_features, model_encoder.trans_input[0].out_features, bias=False), - nn.SiLU(), - nn.Linear(model_encoder.trans_input[2].in_features, model_encoder.trans_input[2].out_features, bias=False), - nn.SiLU(), - ).to(device=avg_emb.device, dtype=avg_emb.dtype) - ref_trans_input.load_state_dict(ref_trans_input_state) - for p in ref_trans_input.parameters(): - p.requires_grad = False - - with torch.no_grad(): - medium_ref = ref_trans_input[1](ref_trans_input[0](avg_emb)) - x_ref = ref_trans_input[3](ref_trans_input[2](medium_ref)) - x_ref = F.normalize(x_ref, dim=-1) # (B, 1, d) - - drift_loss = F.mse_loss(x_curr, x_ref) - return drift_loss - - -class CachedRefTransInput(nn.Module): - """ - Cached frozen copy of trans_input for efficient anti-drift computation. - Avoids recreating nn.Sequential every step. - """ - def __init__(self, trans_input_module): - super().__init__() - import copy - self.ref = copy.deepcopy(trans_input_module) - for p in self.ref.parameters(): - p.requires_grad = False - - def forward(self, x): - with torch.no_grad(): - medium = self.ref[1](self.ref[0](x)) - out = self.ref[3](self.ref[2](medium)) - return F.normalize(out, dim=-1) - - -def compute_anti_drift_loss_cached(model_encoder, ref_trans_input_cached, input_ids, attention_mask): - """ - Efficient anti-drift loss using a pre-cached frozen trans_input. - - Args: - model_encoder: Model encoder with embed_tokens and trans_input - ref_trans_input_cached: CachedRefTransInput instance - input_ids: Replay input IDs (B, seq_len) - attention_mask: Replay attention mask (B, seq_len) - - Returns: - Scalar MSE loss - """ - with torch.no_grad(): - embeddings = model_encoder.embed_tokens(input_ids) - avg_emb = (attention_mask.unsqueeze(-1) * embeddings).mean(dim=1, keepdim=True) - - # Current trans_input output - medium_curr = model_encoder.trans_input[1](model_encoder.trans_input[0](avg_emb)) - x_curr = model_encoder.trans_input[3](model_encoder.trans_input[2](medium_curr)) - x_curr = F.normalize(x_curr, dim=-1) - - # Reference output (frozen) - x_ref = ref_trans_input_cached(avg_emb) - - return F.mse_loss(x_curr, x_ref) - - -# ============================================================================ -# 5. Anti-Invasion Loss -# ============================================================================ - -def compute_anti_invasion_loss(x_normalized, vmf_signatures, current_task_id, threshold=2.3): - """ - Hinge loss penalizing current task features that invade old task regions. - - Args: - x_normalized: Current task's normalized features (B, d) or (B, 1, d) - vmf_signatures: Dict {task_id: (mu, kappa)} - current_task_id: Current task index - threshold: vMF log-likelihood threshold (default -log(0.1) ≈ 2.3) - - Returns: - invasion_loss: Scalar hinge loss - """ - if x_normalized.dim() == 3: - x_normalized = x_normalized.squeeze(1) # (B, d) - - device = x_normalized.device - dtype = x_normalized.dtype - invasion_loss = torch.tensor(0.0, device=device, dtype=dtype) - n_old_tasks = 0 - - for task_id, (mu_s, kappa_s) in vmf_signatures.items(): - if task_id >= current_task_id: - continue - - mu_s = mu_s.to(device=device, dtype=dtype) - kappa_val = kappa_s.item() if isinstance(kappa_s, torch.Tensor) else kappa_s - - # Log-likelihood of x_new under vMF(mu_s, kappa_s): kappa_s * (mu_s · x_new) - log_lik = kappa_val * (x_normalized * mu_s.unsqueeze(0)).sum(dim=-1) # (B,) - - # Hinge: penalize when log_lik > threshold (feature too close to old task) - violation = F.relu(log_lik - threshold) # (B,) - invasion_loss = invasion_loss + violation.mean() - n_old_tasks += 1 - - if n_old_tasks > 0: - invasion_loss = invasion_loss / n_old_tasks - - return invasion_loss diff --git a/gainlora_OT_vMF/src/preflight_check.py b/gainlora_OT_vMF/src/preflight_check.py deleted file mode 100644 index 967e92dbaa1707c5cdcf8f7e885db1a8ef7aa9c7..0000000000000000000000000000000000000000 --- a/gainlora_OT_vMF/src/preflight_check.py +++ /dev/null @@ -1,199 +0,0 @@ -#!/usr/bin/env python3 -"""Fast preflight checks for GainLoRA training compatibility. - -This script catches common breakages in seconds before launching long runs: -- Python syntax/import issues in critical source files -- transformers/accelerate API mismatches -- Trainer init signature mismatches (processing_class vs tokenizer) -- Known bad patterns that caused recent runtime failures -""" - -import glob -import importlib -import inspect -import os -import py_compile -import sys -from typing import List - - -ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -SRC_DIR = os.path.join(ROOT, "src") - -CRITICAL_FILES = [ - "run_t5.py", - "run_llama.py", - "compat_transformers.py", - "cl_trainer_gainlora_inflora.py", - "cl_trainer_gainlora_inflora_llama.py", - "cl_trainer_gainlora_olora.py", - "cl_trainer_gainlora_olora_llama.py", -] - -GAINLORA_TRAINERS = [ - "cl_trainer_gainlora_inflora.py", - "cl_trainer_gainlora_inflora_llama.py", - "cl_trainer_gainlora_olora.py", - "cl_trainer_gainlora_olora_llama.py", -] - - -def _ok(msg: str) -> None: - print(f"[OK] {msg}") - - -def _warn(msg: str) -> None: - print(f"[WARN] {msg}") - - -def _fail(msg: str, failures: List[str]) -> None: - print(f"[FAIL] {msg}") - failures.append(msg) - - -def check_syntax(failures: List[str]) -> None: - for rel in CRITICAL_FILES: - path = os.path.join(SRC_DIR, rel) - if not os.path.exists(path): - _fail(f"Missing file: {rel}", failures) - continue - try: - py_compile.compile(path, doraise=True) - _ok(f"Syntax compile: {rel}") - except Exception as exc: - _fail(f"Syntax error in {rel}: {exc}", failures) - - -def check_core_api(failures: List[str]) -> None: - try: - import transformers - from transformers import Trainer - - sig = inspect.signature(Trainer.__init__) - params = sig.parameters - version_str = transformers.__version__ - major = int(version_str.split(".")[0]) - - if major >= 5: - if "processing_class" not in params: - _fail("transformers>=5 requires Trainer.__init__.processing_class", failures) - else: - _ok("Trainer.__init__ has processing_class (transformers>=5)") - else: - if "tokenizer" not in params: - _fail("transformers<5 expected Trainer.__init__.tokenizer", failures) - else: - _ok("Trainer.__init__ has tokenizer (transformers<5)") - - _ok(f"transformers version: {version_str}") - except Exception as exc: - _fail(f"transformers API check failed: {exc}", failures) - - try: - import accelerate - from accelerate import Accelerator - - clip_sig = inspect.signature(Accelerator.clip_grad_norm_) - clip_params = set(clip_sig.parameters.keys()) - if "check_grad_overflow" in clip_params: - _warn("Accelerator.clip_grad_norm_ supports check_grad_overflow") - else: - _ok("Accelerator.clip_grad_norm_ does not support check_grad_overflow") - _ok(f"accelerate version: {accelerate.__version__}") - except Exception as exc: - _fail(f"accelerate API check failed: {exc}", failures) - - -def check_text_patterns(failures: List[str]) -> None: - # Patterns that recently caused failures in this codebase. - bad_patterns = { - "check_grad_overflow=": "Unsupported kwarg for accelerate.clip_grad_norm_ in many envs", - } - - for rel in GAINLORA_TRAINERS: - path = os.path.join(SRC_DIR, rel) - try: - text = open(path, "r", encoding="utf-8").read() - except Exception as exc: - _fail(f"Cannot read {rel}: {exc}", failures) - continue - - for pattern, reason in bad_patterns.items(): - if pattern in text: - _fail(f"{rel} contains '{pattern}' ({reason})", failures) - - if "self.accelerator.clip_grad_norm_(" in text: - _fail( - f"{rel} still uses self.accelerator.clip_grad_norm_ (can trigger double unscale)", - failures, - ) - elif "nn.utils.clip_grad_norm_(" in text: - _ok(f"{rel} uses nn.utils.clip_grad_norm_") - - # Check Trainer API migration in run scripts. - for rel in ["run_t5.py", "run_llama.py"]: - path = os.path.join(SRC_DIR, rel) - try: - text = open(path, "r", encoding="utf-8").read() - except Exception as exc: - _fail(f"Cannot read {rel}: {exc}", failures) - continue - - if "tokenizer=tokenizer" in text: - _fail(f"{rel} still contains tokenizer=tokenizer (should be processing_class=tokenizer)", failures) - if "processing_class=tokenizer" in text: - _ok(f"{rel} uses processing_class=tokenizer") - - -def check_custom_trainer_signatures(failures: List[str]) -> None: - sys.path.insert(0, SRC_DIR) - trainer_mods = sorted(glob.glob(os.path.join(SRC_DIR, "cl_trainer_*.py"))) - - for mod_path in trainer_mods: - mod_name = os.path.splitext(os.path.basename(mod_path))[0] - try: - mod = importlib.import_module(mod_name) - except Exception as exc: - _fail(f"Import failed for {mod_name}: {exc}", failures) - continue - - for _, obj in inspect.getmembers(mod, inspect.isclass): - if obj.__module__ != mod_name: - continue - if not hasattr(obj, "__mro__"): - continue - if not any(base.__name__ == "Seq2SeqTrainer" for base in obj.__mro__): - continue - - init_sig = inspect.signature(obj.__init__) - params = init_sig.parameters - if "processing_class" not in params: - _fail(f"{mod_name}.{obj.__name__}.__init__ missing processing_class", failures) - elif "tokenizer" in params: - _fail(f"{mod_name}.{obj.__name__}.__init__ still has tokenizer", failures) - else: - _ok(f"{mod_name}.{obj.__name__}.__init__ signature is compatible") - - -def main() -> int: - failures: List[str] = [] - - print("== GainLoRA Preflight Check ==") - print(f"Root: {ROOT}") - - check_syntax(failures) - check_core_api(failures) - check_text_patterns(failures) - check_custom_trainer_signatures(failures) - - print("\n== Result ==") - if failures: - print(f"FAILED with {len(failures)} issue(s)") - return 1 - - print("PASS: no blocking issues found") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/gainlora_OT_vMF/src/run_logger.py b/gainlora_OT_vMF/src/run_logger.py deleted file mode 100644 index 60591f5dfc537b2e86058bc36b77ebf359649af3..0000000000000000000000000000000000000000 --- a/gainlora_OT_vMF/src/run_logger.py +++ /dev/null @@ -1,229 +0,0 @@ -""" -OT-SIGN Run Logger -================== -Logs each training run with config, vMF signatures, and per-task scores. -Computes AP (Average Performance) and FT (Forgetting) directly comparable -to GainLoRA paper Table 1 / Table 2. - -Output files: - run_log.jsonl — machine-readable full log - scores_matrix.json — R[i][j] = score of task j after training task i - ap_ft_summary.md — final AP/FT table ready for paper comparison -""" - -import json -import os -from datetime import datetime - - -# Keys used for score extraction from HuggingFace Trainer metrics -SCORE_KEY_TEMPLATES = { - "superni": ["predict_rougeL_for_{task}", "eval_rougeL_for_{task}", "predict_rougeL"], - "long": ["predict_exact_match_for_{task}", "eval_exact_match_for_{task}", "predict_exact_match"], -} - - -def _extract_score(metrics_dict, task_name, benchmark_type="superni"): - """Extract the primary score for a given task from a metrics dict.""" - templates = SCORE_KEY_TEMPLATES.get(benchmark_type, SCORE_KEY_TEMPLATES["superni"]) - for tmpl in templates: - key = tmpl.format(task=task_name) - if key in metrics_dict: - v = metrics_dict[key] - # rougeL is typically 0–1 from compute_metrics; multiply to get percentage - if v <= 1.0 and "rouge" in key.lower(): - v = v * 100.0 - return round(v, 2) - # fallback: look for any key containing the task name - for k, v in metrics_dict.items(): - if task_name in k and isinstance(v, (int, float)): - if v <= 1.0 and "rouge" in k.lower(): - v = v * 100.0 - return round(v, 2) - return None - - -class RunLogger: - """ - Logs every task training run and builds the result matrix R[i][j]. - - Result matrix semantics (matching GainLoRA paper Table 1): - R[i][j] = score on task j after model has been trained on tasks 0..i - AP = (1/T) * sum_j R[T-1][j] (final row average) - FT = (1/(T-1)) * sum_j (R[j][j] - R[T-1][j]) (diagonal drop) - """ - - def __init__(self, log_dir="logs_and_outputs/ot_sign_logs", benchmark_type="superni"): - self.log_dir = log_dir - self.benchmark_type = benchmark_type # "superni" or "long" - os.makedirs(log_dir, exist_ok=True) - self.log_file = os.path.join(log_dir, "run_log.jsonl") - self.matrix_file = os.path.join(log_dir, "scores_matrix.json") - self.summary_file = os.path.join(log_dir, "ap_ft_summary.md") - - # In-memory score matrix: {trained_on_task_id: {eval_task_name: score}} - self._scores = self._load_scores_matrix() - - # ------------------------------------------------------------------ - # Public API - # ------------------------------------------------------------------ - - def log_run(self, task_name, task_id, config, vmf_signature=None, metrics=None, extra=None): - """Log a single task's training metadata (config, vMF, train metrics).""" - entry = { - "timestamp": datetime.now().isoformat(), - "task_name": task_name, - "task_id": task_id, - "config": config, - "vmf_signature": vmf_signature, - "metrics": metrics, - "extra": extra, - } - with open(self.log_file, 'a') as f: - f.write(json.dumps(entry, default=str) + "\n") - self._update_md_log(entry) - print(f"[RunLogger] Logged task {task_id}: {task_name}") - - def log_predict_scores(self, trained_on_task_id, predict_metrics_dict, task_order_list): - """ - After training on task `trained_on_task_id`, log prediction scores for all - tasks evaluated so far. - - Args: - trained_on_task_id: index of the just-trained task (0-based) - predict_metrics_dict: HF Trainer predict_results.metrics - task_order_list: full list of task names in sequence order - """ - row = {} - for j, tname in enumerate(task_order_list[:trained_on_task_id + 1]): - score = _extract_score(predict_metrics_dict, tname, self.benchmark_type) - if score is not None: - row[tname] = score - - self._scores[trained_on_task_id] = row - self._save_scores_matrix() - - # Print progress table - self._print_current_scores(trained_on_task_id, task_order_list) - - def print_ap_ft(self, task_order_list, method_name="OT-SIGN+GainLoRA"): - """ - Compute and print AP / FT from the accumulated score matrix. - Prints a table directly comparable to GainLoRA paper. - - Returns: - (ap, ft) as floats - """ - T = len(task_order_list) - if T not in self._scores and (T - 1) not in self._scores: - print("[RunLogger] Not enough data to compute AP/FT yet.") - return None, None - - final_row_id = T - 1 - if final_row_id not in self._scores: - print("[RunLogger] Final task scores not found.") - return None, None - - final_scores = [self._scores[final_row_id].get(t) for t in task_order_list] - final_scores_valid = [s for s in final_scores if s is not None] - ap = round(sum(final_scores_valid) / len(final_scores_valid), 2) if final_scores_valid else 0.0 - - # FT = mean of (diagonal - final row), skip first task - ft_values = [] - for j in range(T - 1): - tname = task_order_list[j] - diag = self._scores.get(j, {}).get(tname) - final = self._scores.get(final_row_id, {}).get(tname) - if diag is not None and final is not None: - ft_values.append(diag - final) - ft = round(sum(ft_values) / len(ft_values), 2) if ft_values else 0.0 - - # ---- Console output ---- - sep = "-" * 60 - print(f"\n{sep}") - print(f" FINAL RESULTS — {method_name}") - print(sep) - print(f" {'Task':<45} {'Peak':>6} {'Final':>6} {'Drop':>6}") - print(sep) - for j, tname in enumerate(task_order_list): - diag = self._scores.get(j, {}).get(tname, "-") - final = self._scores.get(final_row_id, {}).get(tname, "-") - drop = f"{diag - final:.2f}" if isinstance(diag, float) and isinstance(final, float) else "-" - diag_s = f"{diag:.2f}" if isinstance(diag, float) else str(diag) - final_s = f"{final:.2f}" if isinstance(final, float) else str(final) - print(f" {tname:<45} {diag_s:>6} {final_s:>6} {drop:>6}") - print(sep) - print(f" AP = {ap:.2f} | FT = {ft:.2f}") - print(f"{sep}\n") - - # ---- File output ---- - self._write_ap_ft_summary(task_order_list, method_name, ap, ft) - return ap, ft - - # ------------------------------------------------------------------ - # Internal helpers - # ------------------------------------------------------------------ - - def _load_scores_matrix(self): - if os.path.exists(self.matrix_file): - with open(self.matrix_file) as f: - raw = json.load(f) - return {int(k): v for k, v in raw.items()} - return {} - - def _save_scores_matrix(self): - with open(self.matrix_file, 'w') as f: - json.dump({str(k): v for k, v in self._scores.items()}, f, indent=2) - - def _print_current_scores(self, trained_on_task_id, task_order_list): - row = self._scores.get(trained_on_task_id, {}) - tname = task_order_list[trained_on_task_id] if trained_on_task_id < len(task_order_list) else "?" - print(f"\n[RunLogger] After task {trained_on_task_id} ({tname}) — predict scores:") - for t, s in row.items(): - print(f" {t:<50} {s:.2f}") - - def _update_md_log(self, entry): - """Append human-readable training info only (not the score matrix).""" - with open(self.log_file.replace(".jsonl", "_detail.md"), 'a') as f: - f.write(f"\n## Task {entry['task_id']}: {entry['task_name']}\n") - f.write(f"*{entry['timestamp']}*\n\n") - if entry.get('vmf_signature'): - f.write(f"- vMF kappa: `{entry['vmf_signature'].get('kappa', '?')}` " - f"mu_norm: `{entry['vmf_signature'].get('mu_norm', '?')}`\n") - if entry.get('metrics'): - f.write("| Metric | Value |\n|--------|-------|\n") - for k, v in entry['metrics'].items(): - val = f"{v:.4f}" if isinstance(v, float) else str(v) - f.write(f"| `{k}` | {val} |\n") - f.write("\n") - - def _write_ap_ft_summary(self, task_order_list, method_name, ap, ft): - """Write a markdown file ready to paste into the comparison table.""" - T = len(task_order_list) - final_row_id = T - 1 - with open(self.summary_file, 'a') as f: - f.write(f"\n## {method_name}\n") - f.write(f"*Generated: {datetime.now().strftime('%Y-%m-%d %H:%M')}*\n\n") - f.write("| Task | Peak Score (R[i,i]) | Final Score (R[T,i]) | Forgetting |\n") - f.write("|------|---------------------|----------------------|------------|\n") - for j, tname in enumerate(task_order_list): - diag = self._scores.get(j, {}).get(tname, "-") - final = self._scores.get(final_row_id, {}).get(tname, "-") - drop = f"{diag - final:.2f}" if isinstance(diag, float) and isinstance(final, float) else "-" - f.write(f"| {tname} | {diag} | {final} | {drop} |\n") - f.write(f"\n**AP = {ap:.2f}** | **FT = {ft:.2f}**\n\n") - - # ---- Legacy compat ---- - def load_all_runs(self): - runs = [] - if os.path.exists(self.log_file): - with open(self.log_file) as f: - for line in f: - line = line.strip() - if line: - runs.append(json.loads(line)) - return runs - - def get_latest_run(self): - runs = self.load_all_runs() - return runs[-1] if runs else None diff --git a/gainlora_OT_vMF/src/test_ot_sign.py b/gainlora_OT_vMF/src/test_ot_sign.py deleted file mode 100644 index 597aaf146538626be3e95b54aef70e930ddf8c1d..0000000000000000000000000000000000000000 --- a/gainlora_OT_vMF/src/test_ot_sign.py +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env python -"""Quick unit test for OT-SIGN components.""" -import torch -import torch.nn.functional as F -import sys -sys.path.insert(0, '.') - -from ot_sign_utils import sinkhorn_log, vmf_cost_matrix, compute_vmf_params, compute_anti_invasion_loss - -print('=== Test 1: Sinkhorn OT ===') -C = torch.randn(4, 3) -weights = sinkhorn_log(C, epsilon=0.05, n_iter=10) -print(f'Cost shape: {C.shape}, Weights shape: {weights.shape}') -print(f'Row sums: {weights.sum(dim=1)}') -print(f'Weights:\n{weights}') - -print() -print('=== Test 2: vMF Parameter Estimation ===') -d = 1024 -n = 100 -raw = torch.randn(n, d) -features = F.normalize(raw, dim=-1) -mu, kappa = compute_vmf_params(features, d) -print(f'mu shape: {mu.shape}, norm: {mu.norm():.4f}') -print(f'kappa: {kappa.item():.4f}') - -print() -print('=== Test 3: vmf_cost_matrix ===') -x = F.normalize(torch.randn(4, d), dim=-1) -sigs = {0: (mu, kappa)} -prompt_key = torch.randn(1, d) -C2 = vmf_cost_matrix(x, sigs, prompt_key=prompt_key, default_kappa=10.0) -print(f'Cost matrix shape: {C2.shape}') - -weights2 = sinkhorn_log(C2, epsilon=0.05, n_iter=10) -print(f'OT weights shape: {weights2.shape}, row sums: {weights2.sum(dim=1)}') -print(f'Weights:\n{weights2}') - -print() -print('=== Test 4: Anti-invasion loss ===') -x_new = F.normalize(torch.randn(8, d), dim=-1) -sigs2 = {0: (mu, torch.tensor(5.0)), 1: (F.normalize(torch.randn(d), dim=-1), torch.tensor(8.0))} -inv_loss = compute_anti_invasion_loss(x_new, sigs2, current_task_id=2, threshold=2.3) -print(f'Invasion loss: {inv_loss.item():.6f}') - -print() -print('=== Test 5: FP16 compatibility ===') -C_fp16 = torch.randn(8, 5, dtype=torch.float16) -w_fp16 = sinkhorn_log(C_fp16.float(), epsilon=0.05, n_iter=10).half() -print(f'FP16 output: {w_fp16.shape}, dtype: {w_fp16.dtype}') -print(f'Row sums: {w_fp16.sum(dim=1)}') - -print() -print('=== All tests passed! ===') diff --git a/gainlora_OT_vMF/CL_Benchmark/Long_Sequence/agnews/dev.json b/root_gainlora/CL_Benchmark/Long_Sequence/agnews/dev.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/Long_Sequence/agnews/dev.json rename to root_gainlora/CL_Benchmark/Long_Sequence/agnews/dev.json diff --git a/gainlora_OT_vMF/CL_Benchmark/Long_Sequence/agnews/labels.json b/root_gainlora/CL_Benchmark/Long_Sequence/agnews/labels.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/Long_Sequence/agnews/labels.json rename to root_gainlora/CL_Benchmark/Long_Sequence/agnews/labels.json diff --git a/gainlora_OT_vMF/CL_Benchmark/Long_Sequence/agnews/test.json b/root_gainlora/CL_Benchmark/Long_Sequence/agnews/test.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/Long_Sequence/agnews/test.json rename to root_gainlora/CL_Benchmark/Long_Sequence/agnews/test.json diff --git a/gainlora_OT_vMF/CL_Benchmark/Long_Sequence/agnews/train.json b/root_gainlora/CL_Benchmark/Long_Sequence/agnews/train.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/Long_Sequence/agnews/train.json rename to root_gainlora/CL_Benchmark/Long_Sequence/agnews/train.json diff --git a/gainlora_OT_vMF/CL_Benchmark/Long_Sequence/amazon/dev.json b/root_gainlora/CL_Benchmark/Long_Sequence/amazon/dev.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/Long_Sequence/amazon/dev.json rename to root_gainlora/CL_Benchmark/Long_Sequence/amazon/dev.json diff --git a/gainlora_OT_vMF/CL_Benchmark/Long_Sequence/amazon/labels.json b/root_gainlora/CL_Benchmark/Long_Sequence/amazon/labels.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/Long_Sequence/amazon/labels.json rename to root_gainlora/CL_Benchmark/Long_Sequence/amazon/labels.json diff --git a/gainlora_OT_vMF/CL_Benchmark/Long_Sequence/amazon/test.json b/root_gainlora/CL_Benchmark/Long_Sequence/amazon/test.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/Long_Sequence/amazon/test.json rename to root_gainlora/CL_Benchmark/Long_Sequence/amazon/test.json diff --git a/gainlora_OT_vMF/CL_Benchmark/Long_Sequence/amazon/train.json b/root_gainlora/CL_Benchmark/Long_Sequence/amazon/train.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/Long_Sequence/amazon/train.json rename to root_gainlora/CL_Benchmark/Long_Sequence/amazon/train.json diff --git a/gainlora_OT_vMF/CL_Benchmark/Long_Sequence/boolq/dev.json b/root_gainlora/CL_Benchmark/Long_Sequence/boolq/dev.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/Long_Sequence/boolq/dev.json rename to root_gainlora/CL_Benchmark/Long_Sequence/boolq/dev.json diff --git a/gainlora_OT_vMF/CL_Benchmark/Long_Sequence/boolq/labels.json b/root_gainlora/CL_Benchmark/Long_Sequence/boolq/labels.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/Long_Sequence/boolq/labels.json rename to root_gainlora/CL_Benchmark/Long_Sequence/boolq/labels.json diff --git a/gainlora_OT_vMF/CL_Benchmark/Long_Sequence/boolq/test.json b/root_gainlora/CL_Benchmark/Long_Sequence/boolq/test.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/Long_Sequence/boolq/test.json rename to root_gainlora/CL_Benchmark/Long_Sequence/boolq/test.json diff --git a/gainlora_OT_vMF/CL_Benchmark/Long_Sequence/boolq/train.json b/root_gainlora/CL_Benchmark/Long_Sequence/boolq/train.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/Long_Sequence/boolq/train.json rename to root_gainlora/CL_Benchmark/Long_Sequence/boolq/train.json diff --git a/gainlora_OT_vMF/CL_Benchmark/Long_Sequence/cb/dev.json b/root_gainlora/CL_Benchmark/Long_Sequence/cb/dev.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/Long_Sequence/cb/dev.json rename to root_gainlora/CL_Benchmark/Long_Sequence/cb/dev.json diff --git a/gainlora_OT_vMF/CL_Benchmark/Long_Sequence/cb/labels.json b/root_gainlora/CL_Benchmark/Long_Sequence/cb/labels.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/Long_Sequence/cb/labels.json rename to root_gainlora/CL_Benchmark/Long_Sequence/cb/labels.json diff --git a/gainlora_OT_vMF/CL_Benchmark/Long_Sequence/cb/test.json b/root_gainlora/CL_Benchmark/Long_Sequence/cb/test.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/Long_Sequence/cb/test.json rename to root_gainlora/CL_Benchmark/Long_Sequence/cb/test.json diff --git a/gainlora_OT_vMF/CL_Benchmark/Long_Sequence/cb/train.json b/root_gainlora/CL_Benchmark/Long_Sequence/cb/train.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/Long_Sequence/cb/train.json rename to root_gainlora/CL_Benchmark/Long_Sequence/cb/train.json diff --git a/gainlora_OT_vMF/CL_Benchmark/Long_Sequence/copa/dev.json b/root_gainlora/CL_Benchmark/Long_Sequence/copa/dev.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/Long_Sequence/copa/dev.json rename to root_gainlora/CL_Benchmark/Long_Sequence/copa/dev.json diff --git a/gainlora_OT_vMF/CL_Benchmark/Long_Sequence/copa/labels.json b/root_gainlora/CL_Benchmark/Long_Sequence/copa/labels.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/Long_Sequence/copa/labels.json rename to root_gainlora/CL_Benchmark/Long_Sequence/copa/labels.json diff --git a/gainlora_OT_vMF/CL_Benchmark/Long_Sequence/copa/test.json b/root_gainlora/CL_Benchmark/Long_Sequence/copa/test.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/Long_Sequence/copa/test.json rename to root_gainlora/CL_Benchmark/Long_Sequence/copa/test.json diff --git a/gainlora_OT_vMF/CL_Benchmark/Long_Sequence/copa/train.json b/root_gainlora/CL_Benchmark/Long_Sequence/copa/train.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/Long_Sequence/copa/train.json rename to root_gainlora/CL_Benchmark/Long_Sequence/copa/train.json diff --git a/gainlora_OT_vMF/CL_Benchmark/Long_Sequence/dbpedia/dev.json b/root_gainlora/CL_Benchmark/Long_Sequence/dbpedia/dev.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/Long_Sequence/dbpedia/dev.json rename to root_gainlora/CL_Benchmark/Long_Sequence/dbpedia/dev.json diff --git a/gainlora_OT_vMF/CL_Benchmark/Long_Sequence/dbpedia/labels.json b/root_gainlora/CL_Benchmark/Long_Sequence/dbpedia/labels.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/Long_Sequence/dbpedia/labels.json rename to root_gainlora/CL_Benchmark/Long_Sequence/dbpedia/labels.json diff --git a/gainlora_OT_vMF/CL_Benchmark/Long_Sequence/dbpedia/test.json b/root_gainlora/CL_Benchmark/Long_Sequence/dbpedia/test.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/Long_Sequence/dbpedia/test.json rename to root_gainlora/CL_Benchmark/Long_Sequence/dbpedia/test.json diff --git a/gainlora_OT_vMF/CL_Benchmark/Long_Sequence/dbpedia/train.json b/root_gainlora/CL_Benchmark/Long_Sequence/dbpedia/train.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/Long_Sequence/dbpedia/train.json rename to root_gainlora/CL_Benchmark/Long_Sequence/dbpedia/train.json diff --git a/gainlora_OT_vMF/CL_Benchmark/Long_Sequence/imdb/dev.json b/root_gainlora/CL_Benchmark/Long_Sequence/imdb/dev.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/Long_Sequence/imdb/dev.json rename to root_gainlora/CL_Benchmark/Long_Sequence/imdb/dev.json diff --git a/gainlora_OT_vMF/CL_Benchmark/Long_Sequence/imdb/labels.json b/root_gainlora/CL_Benchmark/Long_Sequence/imdb/labels.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/Long_Sequence/imdb/labels.json rename to root_gainlora/CL_Benchmark/Long_Sequence/imdb/labels.json diff --git a/gainlora_OT_vMF/CL_Benchmark/Long_Sequence/imdb/test.json b/root_gainlora/CL_Benchmark/Long_Sequence/imdb/test.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/Long_Sequence/imdb/test.json rename to root_gainlora/CL_Benchmark/Long_Sequence/imdb/test.json diff --git a/gainlora_OT_vMF/CL_Benchmark/Long_Sequence/imdb/train.json b/root_gainlora/CL_Benchmark/Long_Sequence/imdb/train.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/Long_Sequence/imdb/train.json rename to root_gainlora/CL_Benchmark/Long_Sequence/imdb/train.json diff --git a/gainlora_OT_vMF/CL_Benchmark/Long_Sequence/mnli/dev.json b/root_gainlora/CL_Benchmark/Long_Sequence/mnli/dev.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/Long_Sequence/mnli/dev.json rename to root_gainlora/CL_Benchmark/Long_Sequence/mnli/dev.json diff --git a/gainlora_OT_vMF/CL_Benchmark/Long_Sequence/mnli/labels.json b/root_gainlora/CL_Benchmark/Long_Sequence/mnli/labels.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/Long_Sequence/mnli/labels.json rename to root_gainlora/CL_Benchmark/Long_Sequence/mnli/labels.json diff --git a/gainlora_OT_vMF/CL_Benchmark/Long_Sequence/mnli/test.json b/root_gainlora/CL_Benchmark/Long_Sequence/mnli/test.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/Long_Sequence/mnli/test.json rename to root_gainlora/CL_Benchmark/Long_Sequence/mnli/test.json diff --git a/gainlora_OT_vMF/CL_Benchmark/Long_Sequence/mnli/train.json b/root_gainlora/CL_Benchmark/Long_Sequence/mnli/train.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/Long_Sequence/mnli/train.json rename to root_gainlora/CL_Benchmark/Long_Sequence/mnli/train.json diff --git a/gainlora_OT_vMF/CL_Benchmark/Long_Sequence/multirc/dev.json b/root_gainlora/CL_Benchmark/Long_Sequence/multirc/dev.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/Long_Sequence/multirc/dev.json rename to root_gainlora/CL_Benchmark/Long_Sequence/multirc/dev.json diff --git a/gainlora_OT_vMF/CL_Benchmark/Long_Sequence/multirc/labels.json b/root_gainlora/CL_Benchmark/Long_Sequence/multirc/labels.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/Long_Sequence/multirc/labels.json rename to root_gainlora/CL_Benchmark/Long_Sequence/multirc/labels.json diff --git a/gainlora_OT_vMF/CL_Benchmark/Long_Sequence/multirc/test.json b/root_gainlora/CL_Benchmark/Long_Sequence/multirc/test.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/Long_Sequence/multirc/test.json rename to root_gainlora/CL_Benchmark/Long_Sequence/multirc/test.json diff --git a/gainlora_OT_vMF/CL_Benchmark/Long_Sequence/multirc/train.json b/root_gainlora/CL_Benchmark/Long_Sequence/multirc/train.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/Long_Sequence/multirc/train.json rename to root_gainlora/CL_Benchmark/Long_Sequence/multirc/train.json diff --git a/gainlora_OT_vMF/CL_Benchmark/Long_Sequence/qqp/dev.json b/root_gainlora/CL_Benchmark/Long_Sequence/qqp/dev.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/Long_Sequence/qqp/dev.json rename to root_gainlora/CL_Benchmark/Long_Sequence/qqp/dev.json diff --git a/gainlora_OT_vMF/CL_Benchmark/Long_Sequence/qqp/labels.json b/root_gainlora/CL_Benchmark/Long_Sequence/qqp/labels.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/Long_Sequence/qqp/labels.json rename to root_gainlora/CL_Benchmark/Long_Sequence/qqp/labels.json diff --git a/gainlora_OT_vMF/CL_Benchmark/Long_Sequence/qqp/test.json b/root_gainlora/CL_Benchmark/Long_Sequence/qqp/test.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/Long_Sequence/qqp/test.json rename to root_gainlora/CL_Benchmark/Long_Sequence/qqp/test.json diff --git a/gainlora_OT_vMF/CL_Benchmark/Long_Sequence/qqp/train.json b/root_gainlora/CL_Benchmark/Long_Sequence/qqp/train.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/Long_Sequence/qqp/train.json rename to root_gainlora/CL_Benchmark/Long_Sequence/qqp/train.json diff --git a/gainlora_OT_vMF/CL_Benchmark/Long_Sequence/rte/dev.json b/root_gainlora/CL_Benchmark/Long_Sequence/rte/dev.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/Long_Sequence/rte/dev.json rename to root_gainlora/CL_Benchmark/Long_Sequence/rte/dev.json diff --git a/gainlora_OT_vMF/CL_Benchmark/Long_Sequence/rte/labels.json b/root_gainlora/CL_Benchmark/Long_Sequence/rte/labels.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/Long_Sequence/rte/labels.json rename to root_gainlora/CL_Benchmark/Long_Sequence/rte/labels.json diff --git a/gainlora_OT_vMF/CL_Benchmark/Long_Sequence/rte/test.json b/root_gainlora/CL_Benchmark/Long_Sequence/rte/test.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/Long_Sequence/rte/test.json rename to root_gainlora/CL_Benchmark/Long_Sequence/rte/test.json diff --git a/gainlora_OT_vMF/CL_Benchmark/Long_Sequence/rte/train.json b/root_gainlora/CL_Benchmark/Long_Sequence/rte/train.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/Long_Sequence/rte/train.json rename to root_gainlora/CL_Benchmark/Long_Sequence/rte/train.json diff --git a/gainlora_OT_vMF/CL_Benchmark/Long_Sequence/sst2/dev.json b/root_gainlora/CL_Benchmark/Long_Sequence/sst2/dev.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/Long_Sequence/sst2/dev.json rename to root_gainlora/CL_Benchmark/Long_Sequence/sst2/dev.json diff --git a/gainlora_OT_vMF/CL_Benchmark/Long_Sequence/sst2/labels.json b/root_gainlora/CL_Benchmark/Long_Sequence/sst2/labels.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/Long_Sequence/sst2/labels.json rename to root_gainlora/CL_Benchmark/Long_Sequence/sst2/labels.json diff --git a/gainlora_OT_vMF/CL_Benchmark/Long_Sequence/sst2/test.json b/root_gainlora/CL_Benchmark/Long_Sequence/sst2/test.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/Long_Sequence/sst2/test.json rename to root_gainlora/CL_Benchmark/Long_Sequence/sst2/test.json diff --git a/gainlora_OT_vMF/CL_Benchmark/Long_Sequence/sst2/train.json b/root_gainlora/CL_Benchmark/Long_Sequence/sst2/train.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/Long_Sequence/sst2/train.json rename to root_gainlora/CL_Benchmark/Long_Sequence/sst2/train.json diff --git a/gainlora_OT_vMF/CL_Benchmark/Long_Sequence/wic/dev.json b/root_gainlora/CL_Benchmark/Long_Sequence/wic/dev.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/Long_Sequence/wic/dev.json rename to root_gainlora/CL_Benchmark/Long_Sequence/wic/dev.json diff --git a/gainlora_OT_vMF/CL_Benchmark/Long_Sequence/wic/labels.json b/root_gainlora/CL_Benchmark/Long_Sequence/wic/labels.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/Long_Sequence/wic/labels.json rename to root_gainlora/CL_Benchmark/Long_Sequence/wic/labels.json diff --git a/gainlora_OT_vMF/CL_Benchmark/Long_Sequence/wic/test.json b/root_gainlora/CL_Benchmark/Long_Sequence/wic/test.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/Long_Sequence/wic/test.json rename to root_gainlora/CL_Benchmark/Long_Sequence/wic/test.json diff --git a/gainlora_OT_vMF/CL_Benchmark/Long_Sequence/wic/train.json b/root_gainlora/CL_Benchmark/Long_Sequence/wic/train.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/Long_Sequence/wic/train.json rename to root_gainlora/CL_Benchmark/Long_Sequence/wic/train.json diff --git a/gainlora_OT_vMF/CL_Benchmark/Long_Sequence/yahoo/dev.json b/root_gainlora/CL_Benchmark/Long_Sequence/yahoo/dev.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/Long_Sequence/yahoo/dev.json rename to root_gainlora/CL_Benchmark/Long_Sequence/yahoo/dev.json diff --git a/gainlora_OT_vMF/CL_Benchmark/Long_Sequence/yahoo/labels.json b/root_gainlora/CL_Benchmark/Long_Sequence/yahoo/labels.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/Long_Sequence/yahoo/labels.json rename to root_gainlora/CL_Benchmark/Long_Sequence/yahoo/labels.json diff --git a/gainlora_OT_vMF/CL_Benchmark/Long_Sequence/yahoo/test.json b/root_gainlora/CL_Benchmark/Long_Sequence/yahoo/test.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/Long_Sequence/yahoo/test.json rename to root_gainlora/CL_Benchmark/Long_Sequence/yahoo/test.json diff --git a/gainlora_OT_vMF/CL_Benchmark/Long_Sequence/yahoo/train.json b/root_gainlora/CL_Benchmark/Long_Sequence/yahoo/train.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/Long_Sequence/yahoo/train.json rename to root_gainlora/CL_Benchmark/Long_Sequence/yahoo/train.json diff --git a/gainlora_OT_vMF/CL_Benchmark/Long_Sequence/yelp/dev.json b/root_gainlora/CL_Benchmark/Long_Sequence/yelp/dev.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/Long_Sequence/yelp/dev.json rename to root_gainlora/CL_Benchmark/Long_Sequence/yelp/dev.json diff --git a/gainlora_OT_vMF/CL_Benchmark/Long_Sequence/yelp/labels.json b/root_gainlora/CL_Benchmark/Long_Sequence/yelp/labels.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/Long_Sequence/yelp/labels.json rename to root_gainlora/CL_Benchmark/Long_Sequence/yelp/labels.json diff --git a/gainlora_OT_vMF/CL_Benchmark/Long_Sequence/yelp/test.json b/root_gainlora/CL_Benchmark/Long_Sequence/yelp/test.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/Long_Sequence/yelp/test.json rename to root_gainlora/CL_Benchmark/Long_Sequence/yelp/test.json diff --git a/gainlora_OT_vMF/CL_Benchmark/Long_Sequence/yelp/train.json b/root_gainlora/CL_Benchmark/Long_Sequence/yelp/train.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/Long_Sequence/yelp/train.json rename to root_gainlora/CL_Benchmark/Long_Sequence/yelp/train.json diff --git a/gainlora_OT_vMF/CL_Benchmark/SuperNI/task002_quoref_answer_generation/dev.json b/root_gainlora/CL_Benchmark/SuperNI/task002_quoref_answer_generation/dev.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/SuperNI/task002_quoref_answer_generation/dev.json rename to root_gainlora/CL_Benchmark/SuperNI/task002_quoref_answer_generation/dev.json diff --git a/gainlora_OT_vMF/CL_Benchmark/SuperNI/task002_quoref_answer_generation/test.json b/root_gainlora/CL_Benchmark/SuperNI/task002_quoref_answer_generation/test.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/SuperNI/task002_quoref_answer_generation/test.json rename to root_gainlora/CL_Benchmark/SuperNI/task002_quoref_answer_generation/test.json diff --git a/gainlora_OT_vMF/CL_Benchmark/SuperNI/task002_quoref_answer_generation/train.json b/root_gainlora/CL_Benchmark/SuperNI/task002_quoref_answer_generation/train.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/SuperNI/task002_quoref_answer_generation/train.json rename to root_gainlora/CL_Benchmark/SuperNI/task002_quoref_answer_generation/train.json diff --git a/gainlora_OT_vMF/CL_Benchmark/SuperNI/task073_commonsenseqa_answer_generation/dev.json b/root_gainlora/CL_Benchmark/SuperNI/task073_commonsenseqa_answer_generation/dev.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/SuperNI/task073_commonsenseqa_answer_generation/dev.json rename to root_gainlora/CL_Benchmark/SuperNI/task073_commonsenseqa_answer_generation/dev.json diff --git a/gainlora_OT_vMF/CL_Benchmark/SuperNI/task073_commonsenseqa_answer_generation/test.json b/root_gainlora/CL_Benchmark/SuperNI/task073_commonsenseqa_answer_generation/test.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/SuperNI/task073_commonsenseqa_answer_generation/test.json rename to root_gainlora/CL_Benchmark/SuperNI/task073_commonsenseqa_answer_generation/test.json diff --git a/gainlora_OT_vMF/CL_Benchmark/SuperNI/task073_commonsenseqa_answer_generation/train.json b/root_gainlora/CL_Benchmark/SuperNI/task073_commonsenseqa_answer_generation/train.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/SuperNI/task073_commonsenseqa_answer_generation/train.json rename to root_gainlora/CL_Benchmark/SuperNI/task073_commonsenseqa_answer_generation/train.json diff --git a/gainlora_OT_vMF/CL_Benchmark/SuperNI/task1290_xsum_summarization/dev.json b/root_gainlora/CL_Benchmark/SuperNI/task1290_xsum_summarization/dev.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/SuperNI/task1290_xsum_summarization/dev.json rename to root_gainlora/CL_Benchmark/SuperNI/task1290_xsum_summarization/dev.json diff --git a/gainlora_OT_vMF/CL_Benchmark/SuperNI/task1290_xsum_summarization/test.json b/root_gainlora/CL_Benchmark/SuperNI/task1290_xsum_summarization/test.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/SuperNI/task1290_xsum_summarization/test.json rename to root_gainlora/CL_Benchmark/SuperNI/task1290_xsum_summarization/test.json diff --git a/gainlora_OT_vMF/CL_Benchmark/SuperNI/task1290_xsum_summarization/train.json b/root_gainlora/CL_Benchmark/SuperNI/task1290_xsum_summarization/train.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/SuperNI/task1290_xsum_summarization/train.json rename to root_gainlora/CL_Benchmark/SuperNI/task1290_xsum_summarization/train.json diff --git a/gainlora_OT_vMF/CL_Benchmark/SuperNI/task1510_evalution_relation_extraction/dev.json b/root_gainlora/CL_Benchmark/SuperNI/task1510_evalution_relation_extraction/dev.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/SuperNI/task1510_evalution_relation_extraction/dev.json rename to root_gainlora/CL_Benchmark/SuperNI/task1510_evalution_relation_extraction/dev.json diff --git a/gainlora_OT_vMF/CL_Benchmark/SuperNI/task1510_evalution_relation_extraction/test.json b/root_gainlora/CL_Benchmark/SuperNI/task1510_evalution_relation_extraction/test.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/SuperNI/task1510_evalution_relation_extraction/test.json rename to root_gainlora/CL_Benchmark/SuperNI/task1510_evalution_relation_extraction/test.json diff --git a/gainlora_OT_vMF/CL_Benchmark/SuperNI/task1510_evalution_relation_extraction/train.json b/root_gainlora/CL_Benchmark/SuperNI/task1510_evalution_relation_extraction/train.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/SuperNI/task1510_evalution_relation_extraction/train.json rename to root_gainlora/CL_Benchmark/SuperNI/task1510_evalution_relation_extraction/train.json diff --git a/gainlora_OT_vMF/CL_Benchmark/SuperNI/task1572_samsum_summary/dev.json b/root_gainlora/CL_Benchmark/SuperNI/task1572_samsum_summary/dev.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/SuperNI/task1572_samsum_summary/dev.json rename to root_gainlora/CL_Benchmark/SuperNI/task1572_samsum_summary/dev.json diff --git a/gainlora_OT_vMF/CL_Benchmark/SuperNI/task1572_samsum_summary/test.json b/root_gainlora/CL_Benchmark/SuperNI/task1572_samsum_summary/test.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/SuperNI/task1572_samsum_summary/test.json rename to root_gainlora/CL_Benchmark/SuperNI/task1572_samsum_summary/test.json diff --git a/gainlora_OT_vMF/CL_Benchmark/SuperNI/task1572_samsum_summary/train.json b/root_gainlora/CL_Benchmark/SuperNI/task1572_samsum_summary/train.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/SuperNI/task1572_samsum_summary/train.json rename to root_gainlora/CL_Benchmark/SuperNI/task1572_samsum_summary/train.json diff --git a/gainlora_OT_vMF/CL_Benchmark/SuperNI/task1590_diplomacy_text_generation/dev.json b/root_gainlora/CL_Benchmark/SuperNI/task1590_diplomacy_text_generation/dev.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/SuperNI/task1590_diplomacy_text_generation/dev.json rename to root_gainlora/CL_Benchmark/SuperNI/task1590_diplomacy_text_generation/dev.json diff --git a/gainlora_OT_vMF/CL_Benchmark/SuperNI/task1590_diplomacy_text_generation/test.json b/root_gainlora/CL_Benchmark/SuperNI/task1590_diplomacy_text_generation/test.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/SuperNI/task1590_diplomacy_text_generation/test.json rename to root_gainlora/CL_Benchmark/SuperNI/task1590_diplomacy_text_generation/test.json diff --git a/gainlora_OT_vMF/CL_Benchmark/SuperNI/task1590_diplomacy_text_generation/train.json b/root_gainlora/CL_Benchmark/SuperNI/task1590_diplomacy_text_generation/train.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/SuperNI/task1590_diplomacy_text_generation/train.json rename to root_gainlora/CL_Benchmark/SuperNI/task1590_diplomacy_text_generation/train.json diff --git a/gainlora_OT_vMF/CL_Benchmark/SuperNI/task1687_sentiment140_classification/dev.json b/root_gainlora/CL_Benchmark/SuperNI/task1687_sentiment140_classification/dev.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/SuperNI/task1687_sentiment140_classification/dev.json rename to root_gainlora/CL_Benchmark/SuperNI/task1687_sentiment140_classification/dev.json diff --git a/gainlora_OT_vMF/CL_Benchmark/SuperNI/task1687_sentiment140_classification/test.json b/root_gainlora/CL_Benchmark/SuperNI/task1687_sentiment140_classification/test.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/SuperNI/task1687_sentiment140_classification/test.json rename to root_gainlora/CL_Benchmark/SuperNI/task1687_sentiment140_classification/test.json diff --git a/gainlora_OT_vMF/CL_Benchmark/SuperNI/task1687_sentiment140_classification/train.json b/root_gainlora/CL_Benchmark/SuperNI/task1687_sentiment140_classification/train.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/SuperNI/task1687_sentiment140_classification/train.json rename to root_gainlora/CL_Benchmark/SuperNI/task1687_sentiment140_classification/train.json diff --git a/gainlora_OT_vMF/CL_Benchmark/SuperNI/task1729_personachat_generate_next/dev.json b/root_gainlora/CL_Benchmark/SuperNI/task1729_personachat_generate_next/dev.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/SuperNI/task1729_personachat_generate_next/dev.json rename to root_gainlora/CL_Benchmark/SuperNI/task1729_personachat_generate_next/dev.json diff --git a/gainlora_OT_vMF/CL_Benchmark/SuperNI/task1729_personachat_generate_next/test.json b/root_gainlora/CL_Benchmark/SuperNI/task1729_personachat_generate_next/test.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/SuperNI/task1729_personachat_generate_next/test.json rename to root_gainlora/CL_Benchmark/SuperNI/task1729_personachat_generate_next/test.json diff --git a/gainlora_OT_vMF/CL_Benchmark/SuperNI/task1729_personachat_generate_next/train.json b/root_gainlora/CL_Benchmark/SuperNI/task1729_personachat_generate_next/train.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/SuperNI/task1729_personachat_generate_next/train.json rename to root_gainlora/CL_Benchmark/SuperNI/task1729_personachat_generate_next/train.json diff --git a/gainlora_OT_vMF/CL_Benchmark/SuperNI/task181_outcome_extraction/dev.json b/root_gainlora/CL_Benchmark/SuperNI/task181_outcome_extraction/dev.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/SuperNI/task181_outcome_extraction/dev.json rename to root_gainlora/CL_Benchmark/SuperNI/task181_outcome_extraction/dev.json diff --git a/gainlora_OT_vMF/CL_Benchmark/SuperNI/task181_outcome_extraction/test.json b/root_gainlora/CL_Benchmark/SuperNI/task181_outcome_extraction/test.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/SuperNI/task181_outcome_extraction/test.json rename to root_gainlora/CL_Benchmark/SuperNI/task181_outcome_extraction/test.json diff --git a/gainlora_OT_vMF/CL_Benchmark/SuperNI/task181_outcome_extraction/train.json b/root_gainlora/CL_Benchmark/SuperNI/task181_outcome_extraction/train.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/SuperNI/task181_outcome_extraction/train.json rename to root_gainlora/CL_Benchmark/SuperNI/task181_outcome_extraction/train.json diff --git a/gainlora_OT_vMF/CL_Benchmark/SuperNI/task363_sst2_polarity_classification/dev.json b/root_gainlora/CL_Benchmark/SuperNI/task363_sst2_polarity_classification/dev.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/SuperNI/task363_sst2_polarity_classification/dev.json rename to root_gainlora/CL_Benchmark/SuperNI/task363_sst2_polarity_classification/dev.json diff --git a/gainlora_OT_vMF/CL_Benchmark/SuperNI/task363_sst2_polarity_classification/test.json b/root_gainlora/CL_Benchmark/SuperNI/task363_sst2_polarity_classification/test.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/SuperNI/task363_sst2_polarity_classification/test.json rename to root_gainlora/CL_Benchmark/SuperNI/task363_sst2_polarity_classification/test.json diff --git a/gainlora_OT_vMF/CL_Benchmark/SuperNI/task363_sst2_polarity_classification/train.json b/root_gainlora/CL_Benchmark/SuperNI/task363_sst2_polarity_classification/train.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/SuperNI/task363_sst2_polarity_classification/train.json rename to root_gainlora/CL_Benchmark/SuperNI/task363_sst2_polarity_classification/train.json diff --git a/gainlora_OT_vMF/CL_Benchmark/SuperNI/task511_reddit_tifu_long_text_summarization/dev.json b/root_gainlora/CL_Benchmark/SuperNI/task511_reddit_tifu_long_text_summarization/dev.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/SuperNI/task511_reddit_tifu_long_text_summarization/dev.json rename to root_gainlora/CL_Benchmark/SuperNI/task511_reddit_tifu_long_text_summarization/dev.json diff --git a/gainlora_OT_vMF/CL_Benchmark/SuperNI/task511_reddit_tifu_long_text_summarization/test.json b/root_gainlora/CL_Benchmark/SuperNI/task511_reddit_tifu_long_text_summarization/test.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/SuperNI/task511_reddit_tifu_long_text_summarization/test.json rename to root_gainlora/CL_Benchmark/SuperNI/task511_reddit_tifu_long_text_summarization/test.json diff --git a/gainlora_OT_vMF/CL_Benchmark/SuperNI/task511_reddit_tifu_long_text_summarization/train.json b/root_gainlora/CL_Benchmark/SuperNI/task511_reddit_tifu_long_text_summarization/train.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/SuperNI/task511_reddit_tifu_long_text_summarization/train.json rename to root_gainlora/CL_Benchmark/SuperNI/task511_reddit_tifu_long_text_summarization/train.json diff --git a/gainlora_OT_vMF/CL_Benchmark/SuperNI/task591_sciq_answer_generation/dev.json b/root_gainlora/CL_Benchmark/SuperNI/task591_sciq_answer_generation/dev.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/SuperNI/task591_sciq_answer_generation/dev.json rename to root_gainlora/CL_Benchmark/SuperNI/task591_sciq_answer_generation/dev.json diff --git a/gainlora_OT_vMF/CL_Benchmark/SuperNI/task591_sciq_answer_generation/test.json b/root_gainlora/CL_Benchmark/SuperNI/task591_sciq_answer_generation/test.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/SuperNI/task591_sciq_answer_generation/test.json rename to root_gainlora/CL_Benchmark/SuperNI/task591_sciq_answer_generation/test.json diff --git a/gainlora_OT_vMF/CL_Benchmark/SuperNI/task591_sciq_answer_generation/train.json b/root_gainlora/CL_Benchmark/SuperNI/task591_sciq_answer_generation/train.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/SuperNI/task591_sciq_answer_generation/train.json rename to root_gainlora/CL_Benchmark/SuperNI/task591_sciq_answer_generation/train.json diff --git a/gainlora_OT_vMF/CL_Benchmark/SuperNI/task639_multi_woz_user_utterance_generation/dev.json b/root_gainlora/CL_Benchmark/SuperNI/task639_multi_woz_user_utterance_generation/dev.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/SuperNI/task639_multi_woz_user_utterance_generation/dev.json rename to root_gainlora/CL_Benchmark/SuperNI/task639_multi_woz_user_utterance_generation/dev.json diff --git a/gainlora_OT_vMF/CL_Benchmark/SuperNI/task639_multi_woz_user_utterance_generation/test.json b/root_gainlora/CL_Benchmark/SuperNI/task639_multi_woz_user_utterance_generation/test.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/SuperNI/task639_multi_woz_user_utterance_generation/test.json rename to root_gainlora/CL_Benchmark/SuperNI/task639_multi_woz_user_utterance_generation/test.json diff --git a/gainlora_OT_vMF/CL_Benchmark/SuperNI/task639_multi_woz_user_utterance_generation/train.json b/root_gainlora/CL_Benchmark/SuperNI/task639_multi_woz_user_utterance_generation/train.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/SuperNI/task639_multi_woz_user_utterance_generation/train.json rename to root_gainlora/CL_Benchmark/SuperNI/task639_multi_woz_user_utterance_generation/train.json diff --git a/gainlora_OT_vMF/CL_Benchmark/SuperNI/task748_glucose_reverse_cause_event_detection/dev.json b/root_gainlora/CL_Benchmark/SuperNI/task748_glucose_reverse_cause_event_detection/dev.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/SuperNI/task748_glucose_reverse_cause_event_detection/dev.json rename to root_gainlora/CL_Benchmark/SuperNI/task748_glucose_reverse_cause_event_detection/dev.json diff --git a/gainlora_OT_vMF/CL_Benchmark/SuperNI/task748_glucose_reverse_cause_event_detection/test.json b/root_gainlora/CL_Benchmark/SuperNI/task748_glucose_reverse_cause_event_detection/test.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/SuperNI/task748_glucose_reverse_cause_event_detection/test.json rename to root_gainlora/CL_Benchmark/SuperNI/task748_glucose_reverse_cause_event_detection/test.json diff --git a/gainlora_OT_vMF/CL_Benchmark/SuperNI/task748_glucose_reverse_cause_event_detection/train.json b/root_gainlora/CL_Benchmark/SuperNI/task748_glucose_reverse_cause_event_detection/train.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/SuperNI/task748_glucose_reverse_cause_event_detection/train.json rename to root_gainlora/CL_Benchmark/SuperNI/task748_glucose_reverse_cause_event_detection/train.json diff --git a/gainlora_OT_vMF/CL_Benchmark/SuperNI/task875_emotion_classification/dev.json b/root_gainlora/CL_Benchmark/SuperNI/task875_emotion_classification/dev.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/SuperNI/task875_emotion_classification/dev.json rename to root_gainlora/CL_Benchmark/SuperNI/task875_emotion_classification/dev.json diff --git a/gainlora_OT_vMF/CL_Benchmark/SuperNI/task875_emotion_classification/test.json b/root_gainlora/CL_Benchmark/SuperNI/task875_emotion_classification/test.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/SuperNI/task875_emotion_classification/test.json rename to root_gainlora/CL_Benchmark/SuperNI/task875_emotion_classification/test.json diff --git a/gainlora_OT_vMF/CL_Benchmark/SuperNI/task875_emotion_classification/train.json b/root_gainlora/CL_Benchmark/SuperNI/task875_emotion_classification/train.json similarity index 100% rename from gainlora_OT_vMF/CL_Benchmark/SuperNI/task875_emotion_classification/train.json rename to root_gainlora/CL_Benchmark/SuperNI/task875_emotion_classification/train.json diff --git a/gainlora_OT_vMF/README.md b/root_gainlora/README.md similarity index 100% rename from gainlora_OT_vMF/README.md rename to root_gainlora/README.md diff --git a/gainlora_OT_vMF/configs/ds_configs/eval.config b/root_gainlora/configs/ds_configs/eval.config similarity index 100% rename from gainlora_OT_vMF/configs/ds_configs/eval.config rename to root_gainlora/configs/ds_configs/eval.config diff --git a/gainlora_OT_vMF/configs/ds_configs/stage0.config b/root_gainlora/configs/ds_configs/stage0.config similarity index 100% rename from gainlora_OT_vMF/configs/ds_configs/stage0.config rename to root_gainlora/configs/ds_configs/stage0.config diff --git a/gainlora_OT_vMF/configs/ds_configs/stage0_llama.config b/root_gainlora/configs/ds_configs/stage0_llama.config similarity index 100% rename from gainlora_OT_vMF/configs/ds_configs/stage0_llama.config rename to root_gainlora/configs/ds_configs/stage0_llama.config diff --git a/gainlora_OT_vMF/configs/ds_configs/stage1.config b/root_gainlora/configs/ds_configs/stage1.config similarity index 100% rename from gainlora_OT_vMF/configs/ds_configs/stage1.config rename to root_gainlora/configs/ds_configs/stage1.config diff --git a/gainlora_OT_vMF/configs/ds_configs/stage2.config b/root_gainlora/configs/ds_configs/stage2.config similarity index 100% rename from gainlora_OT_vMF/configs/ds_configs/stage2.config rename to root_gainlora/configs/ds_configs/stage2.config diff --git a/gainlora_OT_vMF/configs/ds_configs/stage2_llama.config b/root_gainlora/configs/ds_configs/stage2_llama.config similarity index 100% rename from gainlora_OT_vMF/configs/ds_configs/stage2_llama.config rename to root_gainlora/configs/ds_configs/stage2_llama.config diff --git a/gainlora_OT_vMF/configs/ds_configs/stage2_without_offload.config b/root_gainlora/configs/ds_configs/stage2_without_offload.config similarity index 100% rename from gainlora_OT_vMF/configs/ds_configs/stage2_without_offload.config rename to root_gainlora/configs/ds_configs/stage2_without_offload.config diff --git a/gainlora_OT_vMF/configs/ds_configs/stage3.config b/root_gainlora/configs/ds_configs/stage3.config similarity index 100% rename from gainlora_OT_vMF/configs/ds_configs/stage3.config rename to root_gainlora/configs/ds_configs/stage3.config diff --git a/gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/agnews/dev_tasks.json b/root_gainlora/configs/gen_script_long_order3_t5_configs/agnews/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/agnews/dev_tasks.json rename to root_gainlora/configs/gen_script_long_order3_t5_configs/agnews/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/agnews/test_tasks.json b/root_gainlora/configs/gen_script_long_order3_t5_configs/agnews/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/agnews/test_tasks.json rename to root_gainlora/configs/gen_script_long_order3_t5_configs/agnews/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/agnews/train_tasks.json b/root_gainlora/configs/gen_script_long_order3_t5_configs/agnews/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/agnews/train_tasks.json rename to root_gainlora/configs/gen_script_long_order3_t5_configs/agnews/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/amazon/dev_tasks.json b/root_gainlora/configs/gen_script_long_order3_t5_configs/amazon/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/amazon/dev_tasks.json rename to root_gainlora/configs/gen_script_long_order3_t5_configs/amazon/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/amazon/test_tasks.json b/root_gainlora/configs/gen_script_long_order3_t5_configs/amazon/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/amazon/test_tasks.json rename to root_gainlora/configs/gen_script_long_order3_t5_configs/amazon/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/amazon/train_tasks.json b/root_gainlora/configs/gen_script_long_order3_t5_configs/amazon/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/amazon/train_tasks.json rename to root_gainlora/configs/gen_script_long_order3_t5_configs/amazon/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/boolq/dev_tasks.json b/root_gainlora/configs/gen_script_long_order3_t5_configs/boolq/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/boolq/dev_tasks.json rename to root_gainlora/configs/gen_script_long_order3_t5_configs/boolq/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/boolq/test_tasks.json b/root_gainlora/configs/gen_script_long_order3_t5_configs/boolq/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/boolq/test_tasks.json rename to root_gainlora/configs/gen_script_long_order3_t5_configs/boolq/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/boolq/train_tasks.json b/root_gainlora/configs/gen_script_long_order3_t5_configs/boolq/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/boolq/train_tasks.json rename to root_gainlora/configs/gen_script_long_order3_t5_configs/boolq/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/cb/dev_tasks.json b/root_gainlora/configs/gen_script_long_order3_t5_configs/cb/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/cb/dev_tasks.json rename to root_gainlora/configs/gen_script_long_order3_t5_configs/cb/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/cb/test_tasks.json b/root_gainlora/configs/gen_script_long_order3_t5_configs/cb/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/cb/test_tasks.json rename to root_gainlora/configs/gen_script_long_order3_t5_configs/cb/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/cb/train_tasks.json b/root_gainlora/configs/gen_script_long_order3_t5_configs/cb/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/cb/train_tasks.json rename to root_gainlora/configs/gen_script_long_order3_t5_configs/cb/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/copa/dev_tasks.json b/root_gainlora/configs/gen_script_long_order3_t5_configs/copa/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/copa/dev_tasks.json rename to root_gainlora/configs/gen_script_long_order3_t5_configs/copa/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/copa/test_tasks.json b/root_gainlora/configs/gen_script_long_order3_t5_configs/copa/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/copa/test_tasks.json rename to root_gainlora/configs/gen_script_long_order3_t5_configs/copa/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/copa/train_tasks.json b/root_gainlora/configs/gen_script_long_order3_t5_configs/copa/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/copa/train_tasks.json rename to root_gainlora/configs/gen_script_long_order3_t5_configs/copa/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/dbpedia/dev_tasks.json b/root_gainlora/configs/gen_script_long_order3_t5_configs/dbpedia/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/dbpedia/dev_tasks.json rename to root_gainlora/configs/gen_script_long_order3_t5_configs/dbpedia/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/dbpedia/test_tasks.json b/root_gainlora/configs/gen_script_long_order3_t5_configs/dbpedia/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/dbpedia/test_tasks.json rename to root_gainlora/configs/gen_script_long_order3_t5_configs/dbpedia/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/dbpedia/train_tasks.json b/root_gainlora/configs/gen_script_long_order3_t5_configs/dbpedia/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/dbpedia/train_tasks.json rename to root_gainlora/configs/gen_script_long_order3_t5_configs/dbpedia/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/imdb/dev_tasks.json b/root_gainlora/configs/gen_script_long_order3_t5_configs/imdb/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/imdb/dev_tasks.json rename to root_gainlora/configs/gen_script_long_order3_t5_configs/imdb/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/imdb/test_tasks.json b/root_gainlora/configs/gen_script_long_order3_t5_configs/imdb/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/imdb/test_tasks.json rename to root_gainlora/configs/gen_script_long_order3_t5_configs/imdb/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/imdb/train_tasks.json b/root_gainlora/configs/gen_script_long_order3_t5_configs/imdb/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/imdb/train_tasks.json rename to root_gainlora/configs/gen_script_long_order3_t5_configs/imdb/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/mnli/dev_tasks.json b/root_gainlora/configs/gen_script_long_order3_t5_configs/mnli/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/mnli/dev_tasks.json rename to root_gainlora/configs/gen_script_long_order3_t5_configs/mnli/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/mnli/test_tasks.json b/root_gainlora/configs/gen_script_long_order3_t5_configs/mnli/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/mnli/test_tasks.json rename to root_gainlora/configs/gen_script_long_order3_t5_configs/mnli/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/mnli/train_tasks.json b/root_gainlora/configs/gen_script_long_order3_t5_configs/mnli/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/mnli/train_tasks.json rename to root_gainlora/configs/gen_script_long_order3_t5_configs/mnli/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/multirc/dev_tasks.json b/root_gainlora/configs/gen_script_long_order3_t5_configs/multirc/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/multirc/dev_tasks.json rename to root_gainlora/configs/gen_script_long_order3_t5_configs/multirc/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/multirc/test_tasks.json b/root_gainlora/configs/gen_script_long_order3_t5_configs/multirc/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/multirc/test_tasks.json rename to root_gainlora/configs/gen_script_long_order3_t5_configs/multirc/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/multirc/train_tasks.json b/root_gainlora/configs/gen_script_long_order3_t5_configs/multirc/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/multirc/train_tasks.json rename to root_gainlora/configs/gen_script_long_order3_t5_configs/multirc/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/qqp/dev_tasks.json b/root_gainlora/configs/gen_script_long_order3_t5_configs/qqp/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/qqp/dev_tasks.json rename to root_gainlora/configs/gen_script_long_order3_t5_configs/qqp/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/qqp/test_tasks.json b/root_gainlora/configs/gen_script_long_order3_t5_configs/qqp/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/qqp/test_tasks.json rename to root_gainlora/configs/gen_script_long_order3_t5_configs/qqp/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/qqp/train_tasks.json b/root_gainlora/configs/gen_script_long_order3_t5_configs/qqp/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/qqp/train_tasks.json rename to root_gainlora/configs/gen_script_long_order3_t5_configs/qqp/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/rte/dev_tasks.json b/root_gainlora/configs/gen_script_long_order3_t5_configs/rte/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/rte/dev_tasks.json rename to root_gainlora/configs/gen_script_long_order3_t5_configs/rte/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/rte/test_tasks.json b/root_gainlora/configs/gen_script_long_order3_t5_configs/rte/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/rte/test_tasks.json rename to root_gainlora/configs/gen_script_long_order3_t5_configs/rte/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/rte/train_tasks.json b/root_gainlora/configs/gen_script_long_order3_t5_configs/rte/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/rte/train_tasks.json rename to root_gainlora/configs/gen_script_long_order3_t5_configs/rte/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/sst2/dev_tasks.json b/root_gainlora/configs/gen_script_long_order3_t5_configs/sst2/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/sst2/dev_tasks.json rename to root_gainlora/configs/gen_script_long_order3_t5_configs/sst2/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/sst2/test_tasks.json b/root_gainlora/configs/gen_script_long_order3_t5_configs/sst2/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/sst2/test_tasks.json rename to root_gainlora/configs/gen_script_long_order3_t5_configs/sst2/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/sst2/train_tasks.json b/root_gainlora/configs/gen_script_long_order3_t5_configs/sst2/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/sst2/train_tasks.json rename to root_gainlora/configs/gen_script_long_order3_t5_configs/sst2/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/wic/dev_tasks.json b/root_gainlora/configs/gen_script_long_order3_t5_configs/wic/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/wic/dev_tasks.json rename to root_gainlora/configs/gen_script_long_order3_t5_configs/wic/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/wic/test_tasks.json b/root_gainlora/configs/gen_script_long_order3_t5_configs/wic/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/wic/test_tasks.json rename to root_gainlora/configs/gen_script_long_order3_t5_configs/wic/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/wic/train_tasks.json b/root_gainlora/configs/gen_script_long_order3_t5_configs/wic/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/wic/train_tasks.json rename to root_gainlora/configs/gen_script_long_order3_t5_configs/wic/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/yahoo/dev_tasks.json b/root_gainlora/configs/gen_script_long_order3_t5_configs/yahoo/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/yahoo/dev_tasks.json rename to root_gainlora/configs/gen_script_long_order3_t5_configs/yahoo/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/yahoo/test_tasks.json b/root_gainlora/configs/gen_script_long_order3_t5_configs/yahoo/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/yahoo/test_tasks.json rename to root_gainlora/configs/gen_script_long_order3_t5_configs/yahoo/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/yahoo/train_tasks.json b/root_gainlora/configs/gen_script_long_order3_t5_configs/yahoo/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/yahoo/train_tasks.json rename to root_gainlora/configs/gen_script_long_order3_t5_configs/yahoo/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/yelp/dev_tasks.json b/root_gainlora/configs/gen_script_long_order3_t5_configs/yelp/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/yelp/dev_tasks.json rename to root_gainlora/configs/gen_script_long_order3_t5_configs/yelp/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/yelp/test_tasks.json b/root_gainlora/configs/gen_script_long_order3_t5_configs/yelp/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/yelp/test_tasks.json rename to root_gainlora/configs/gen_script_long_order3_t5_configs/yelp/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/yelp/train_tasks.json b/root_gainlora/configs/gen_script_long_order3_t5_configs/yelp/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order3_t5_configs/yelp/train_tasks.json rename to root_gainlora/configs/gen_script_long_order3_t5_configs/yelp/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/agnews/dev_tasks.json b/root_gainlora/configs/gen_script_long_order4_t5_configs/agnews/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/agnews/dev_tasks.json rename to root_gainlora/configs/gen_script_long_order4_t5_configs/agnews/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/agnews/test_tasks.json b/root_gainlora/configs/gen_script_long_order4_t5_configs/agnews/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/agnews/test_tasks.json rename to root_gainlora/configs/gen_script_long_order4_t5_configs/agnews/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/agnews/train_tasks.json b/root_gainlora/configs/gen_script_long_order4_t5_configs/agnews/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/agnews/train_tasks.json rename to root_gainlora/configs/gen_script_long_order4_t5_configs/agnews/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/amazon/dev_tasks.json b/root_gainlora/configs/gen_script_long_order4_t5_configs/amazon/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/amazon/dev_tasks.json rename to root_gainlora/configs/gen_script_long_order4_t5_configs/amazon/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/amazon/test_tasks.json b/root_gainlora/configs/gen_script_long_order4_t5_configs/amazon/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/amazon/test_tasks.json rename to root_gainlora/configs/gen_script_long_order4_t5_configs/amazon/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/amazon/train_tasks.json b/root_gainlora/configs/gen_script_long_order4_t5_configs/amazon/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/amazon/train_tasks.json rename to root_gainlora/configs/gen_script_long_order4_t5_configs/amazon/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/boolq/dev_tasks.json b/root_gainlora/configs/gen_script_long_order4_t5_configs/boolq/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/boolq/dev_tasks.json rename to root_gainlora/configs/gen_script_long_order4_t5_configs/boolq/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/boolq/test_tasks.json b/root_gainlora/configs/gen_script_long_order4_t5_configs/boolq/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/boolq/test_tasks.json rename to root_gainlora/configs/gen_script_long_order4_t5_configs/boolq/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/boolq/train_tasks.json b/root_gainlora/configs/gen_script_long_order4_t5_configs/boolq/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/boolq/train_tasks.json rename to root_gainlora/configs/gen_script_long_order4_t5_configs/boolq/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/cb/dev_tasks.json b/root_gainlora/configs/gen_script_long_order4_t5_configs/cb/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/cb/dev_tasks.json rename to root_gainlora/configs/gen_script_long_order4_t5_configs/cb/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/cb/test_tasks.json b/root_gainlora/configs/gen_script_long_order4_t5_configs/cb/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/cb/test_tasks.json rename to root_gainlora/configs/gen_script_long_order4_t5_configs/cb/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/cb/train_tasks.json b/root_gainlora/configs/gen_script_long_order4_t5_configs/cb/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/cb/train_tasks.json rename to root_gainlora/configs/gen_script_long_order4_t5_configs/cb/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/copa/dev_tasks.json b/root_gainlora/configs/gen_script_long_order4_t5_configs/copa/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/copa/dev_tasks.json rename to root_gainlora/configs/gen_script_long_order4_t5_configs/copa/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/copa/test_tasks.json b/root_gainlora/configs/gen_script_long_order4_t5_configs/copa/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/copa/test_tasks.json rename to root_gainlora/configs/gen_script_long_order4_t5_configs/copa/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/copa/train_tasks.json b/root_gainlora/configs/gen_script_long_order4_t5_configs/copa/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/copa/train_tasks.json rename to root_gainlora/configs/gen_script_long_order4_t5_configs/copa/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/dbpedia/dev_tasks.json b/root_gainlora/configs/gen_script_long_order4_t5_configs/dbpedia/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/dbpedia/dev_tasks.json rename to root_gainlora/configs/gen_script_long_order4_t5_configs/dbpedia/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/dbpedia/test_tasks.json b/root_gainlora/configs/gen_script_long_order4_t5_configs/dbpedia/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/dbpedia/test_tasks.json rename to root_gainlora/configs/gen_script_long_order4_t5_configs/dbpedia/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/dbpedia/train_tasks.json b/root_gainlora/configs/gen_script_long_order4_t5_configs/dbpedia/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/dbpedia/train_tasks.json rename to root_gainlora/configs/gen_script_long_order4_t5_configs/dbpedia/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/imdb/dev_tasks.json b/root_gainlora/configs/gen_script_long_order4_t5_configs/imdb/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/imdb/dev_tasks.json rename to root_gainlora/configs/gen_script_long_order4_t5_configs/imdb/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/imdb/test_tasks.json b/root_gainlora/configs/gen_script_long_order4_t5_configs/imdb/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/imdb/test_tasks.json rename to root_gainlora/configs/gen_script_long_order4_t5_configs/imdb/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/imdb/train_tasks.json b/root_gainlora/configs/gen_script_long_order4_t5_configs/imdb/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/imdb/train_tasks.json rename to root_gainlora/configs/gen_script_long_order4_t5_configs/imdb/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/mnli/dev_tasks.json b/root_gainlora/configs/gen_script_long_order4_t5_configs/mnli/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/mnli/dev_tasks.json rename to root_gainlora/configs/gen_script_long_order4_t5_configs/mnli/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/mnli/test_tasks.json b/root_gainlora/configs/gen_script_long_order4_t5_configs/mnli/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/mnli/test_tasks.json rename to root_gainlora/configs/gen_script_long_order4_t5_configs/mnli/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/mnli/train_tasks.json b/root_gainlora/configs/gen_script_long_order4_t5_configs/mnli/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/mnli/train_tasks.json rename to root_gainlora/configs/gen_script_long_order4_t5_configs/mnli/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/multirc/dev_tasks.json b/root_gainlora/configs/gen_script_long_order4_t5_configs/multirc/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/multirc/dev_tasks.json rename to root_gainlora/configs/gen_script_long_order4_t5_configs/multirc/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/multirc/test_tasks.json b/root_gainlora/configs/gen_script_long_order4_t5_configs/multirc/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/multirc/test_tasks.json rename to root_gainlora/configs/gen_script_long_order4_t5_configs/multirc/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/multirc/train_tasks.json b/root_gainlora/configs/gen_script_long_order4_t5_configs/multirc/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/multirc/train_tasks.json rename to root_gainlora/configs/gen_script_long_order4_t5_configs/multirc/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/qqp/dev_tasks.json b/root_gainlora/configs/gen_script_long_order4_t5_configs/qqp/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/qqp/dev_tasks.json rename to root_gainlora/configs/gen_script_long_order4_t5_configs/qqp/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/qqp/test_tasks.json b/root_gainlora/configs/gen_script_long_order4_t5_configs/qqp/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/qqp/test_tasks.json rename to root_gainlora/configs/gen_script_long_order4_t5_configs/qqp/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/qqp/train_tasks.json b/root_gainlora/configs/gen_script_long_order4_t5_configs/qqp/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/qqp/train_tasks.json rename to root_gainlora/configs/gen_script_long_order4_t5_configs/qqp/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/rte/dev_tasks.json b/root_gainlora/configs/gen_script_long_order4_t5_configs/rte/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/rte/dev_tasks.json rename to root_gainlora/configs/gen_script_long_order4_t5_configs/rte/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/rte/test_tasks.json b/root_gainlora/configs/gen_script_long_order4_t5_configs/rte/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/rte/test_tasks.json rename to root_gainlora/configs/gen_script_long_order4_t5_configs/rte/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/rte/train_tasks.json b/root_gainlora/configs/gen_script_long_order4_t5_configs/rte/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/rte/train_tasks.json rename to root_gainlora/configs/gen_script_long_order4_t5_configs/rte/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/sst2/dev_tasks.json b/root_gainlora/configs/gen_script_long_order4_t5_configs/sst2/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/sst2/dev_tasks.json rename to root_gainlora/configs/gen_script_long_order4_t5_configs/sst2/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/sst2/test_tasks.json b/root_gainlora/configs/gen_script_long_order4_t5_configs/sst2/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/sst2/test_tasks.json rename to root_gainlora/configs/gen_script_long_order4_t5_configs/sst2/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/sst2/train_tasks.json b/root_gainlora/configs/gen_script_long_order4_t5_configs/sst2/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/sst2/train_tasks.json rename to root_gainlora/configs/gen_script_long_order4_t5_configs/sst2/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/wic/dev_tasks.json b/root_gainlora/configs/gen_script_long_order4_t5_configs/wic/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/wic/dev_tasks.json rename to root_gainlora/configs/gen_script_long_order4_t5_configs/wic/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/wic/test_tasks.json b/root_gainlora/configs/gen_script_long_order4_t5_configs/wic/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/wic/test_tasks.json rename to root_gainlora/configs/gen_script_long_order4_t5_configs/wic/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/wic/train_tasks.json b/root_gainlora/configs/gen_script_long_order4_t5_configs/wic/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/wic/train_tasks.json rename to root_gainlora/configs/gen_script_long_order4_t5_configs/wic/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/yahoo/dev_tasks.json b/root_gainlora/configs/gen_script_long_order4_t5_configs/yahoo/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/yahoo/dev_tasks.json rename to root_gainlora/configs/gen_script_long_order4_t5_configs/yahoo/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/yahoo/test_tasks.json b/root_gainlora/configs/gen_script_long_order4_t5_configs/yahoo/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/yahoo/test_tasks.json rename to root_gainlora/configs/gen_script_long_order4_t5_configs/yahoo/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/yahoo/train_tasks.json b/root_gainlora/configs/gen_script_long_order4_t5_configs/yahoo/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/yahoo/train_tasks.json rename to root_gainlora/configs/gen_script_long_order4_t5_configs/yahoo/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/yelp/dev_tasks.json b/root_gainlora/configs/gen_script_long_order4_t5_configs/yelp/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/yelp/dev_tasks.json rename to root_gainlora/configs/gen_script_long_order4_t5_configs/yelp/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/yelp/test_tasks.json b/root_gainlora/configs/gen_script_long_order4_t5_configs/yelp/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/yelp/test_tasks.json rename to root_gainlora/configs/gen_script_long_order4_t5_configs/yelp/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/yelp/train_tasks.json b/root_gainlora/configs/gen_script_long_order4_t5_configs/yelp/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_long_order4_t5_configs/yelp/train_tasks.json rename to root_gainlora/configs/gen_script_long_order4_t5_configs/yelp/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task002_quoref_answer_generation/dev_tasks.json b/root_gainlora/configs/gen_script_superni_order1_llama_configs/task002_quoref_answer_generation/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task002_quoref_answer_generation/dev_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_llama_configs/task002_quoref_answer_generation/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task002_quoref_answer_generation/test_tasks.json b/root_gainlora/configs/gen_script_superni_order1_llama_configs/task002_quoref_answer_generation/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task002_quoref_answer_generation/test_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_llama_configs/task002_quoref_answer_generation/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task002_quoref_answer_generation/train_tasks.json b/root_gainlora/configs/gen_script_superni_order1_llama_configs/task002_quoref_answer_generation/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task002_quoref_answer_generation/train_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_llama_configs/task002_quoref_answer_generation/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task073_commonsenseqa_answer_generation/dev_tasks.json b/root_gainlora/configs/gen_script_superni_order1_llama_configs/task073_commonsenseqa_answer_generation/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task073_commonsenseqa_answer_generation/dev_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_llama_configs/task073_commonsenseqa_answer_generation/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task073_commonsenseqa_answer_generation/test_tasks.json b/root_gainlora/configs/gen_script_superni_order1_llama_configs/task073_commonsenseqa_answer_generation/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task073_commonsenseqa_answer_generation/test_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_llama_configs/task073_commonsenseqa_answer_generation/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task073_commonsenseqa_answer_generation/train_tasks.json b/root_gainlora/configs/gen_script_superni_order1_llama_configs/task073_commonsenseqa_answer_generation/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task073_commonsenseqa_answer_generation/train_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_llama_configs/task073_commonsenseqa_answer_generation/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task1290_xsum_summarization/dev_tasks.json b/root_gainlora/configs/gen_script_superni_order1_llama_configs/task1290_xsum_summarization/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task1290_xsum_summarization/dev_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_llama_configs/task1290_xsum_summarization/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task1290_xsum_summarization/test_tasks.json b/root_gainlora/configs/gen_script_superni_order1_llama_configs/task1290_xsum_summarization/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task1290_xsum_summarization/test_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_llama_configs/task1290_xsum_summarization/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task1290_xsum_summarization/train_tasks.json b/root_gainlora/configs/gen_script_superni_order1_llama_configs/task1290_xsum_summarization/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task1290_xsum_summarization/train_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_llama_configs/task1290_xsum_summarization/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task1510_evalution_relation_extraction/dev_tasks.json b/root_gainlora/configs/gen_script_superni_order1_llama_configs/task1510_evalution_relation_extraction/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task1510_evalution_relation_extraction/dev_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_llama_configs/task1510_evalution_relation_extraction/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task1510_evalution_relation_extraction/test_tasks.json b/root_gainlora/configs/gen_script_superni_order1_llama_configs/task1510_evalution_relation_extraction/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task1510_evalution_relation_extraction/test_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_llama_configs/task1510_evalution_relation_extraction/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task1510_evalution_relation_extraction/train_tasks.json b/root_gainlora/configs/gen_script_superni_order1_llama_configs/task1510_evalution_relation_extraction/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task1510_evalution_relation_extraction/train_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_llama_configs/task1510_evalution_relation_extraction/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task1572_samsum_summary/dev_tasks.json b/root_gainlora/configs/gen_script_superni_order1_llama_configs/task1572_samsum_summary/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task1572_samsum_summary/dev_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_llama_configs/task1572_samsum_summary/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task1572_samsum_summary/test_tasks.json b/root_gainlora/configs/gen_script_superni_order1_llama_configs/task1572_samsum_summary/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task1572_samsum_summary/test_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_llama_configs/task1572_samsum_summary/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task1572_samsum_summary/train_tasks.json b/root_gainlora/configs/gen_script_superni_order1_llama_configs/task1572_samsum_summary/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task1572_samsum_summary/train_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_llama_configs/task1572_samsum_summary/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task1590_diplomacy_text_generation/dev_tasks.json b/root_gainlora/configs/gen_script_superni_order1_llama_configs/task1590_diplomacy_text_generation/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task1590_diplomacy_text_generation/dev_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_llama_configs/task1590_diplomacy_text_generation/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task1590_diplomacy_text_generation/test_tasks.json b/root_gainlora/configs/gen_script_superni_order1_llama_configs/task1590_diplomacy_text_generation/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task1590_diplomacy_text_generation/test_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_llama_configs/task1590_diplomacy_text_generation/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task1590_diplomacy_text_generation/train_tasks.json b/root_gainlora/configs/gen_script_superni_order1_llama_configs/task1590_diplomacy_text_generation/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task1590_diplomacy_text_generation/train_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_llama_configs/task1590_diplomacy_text_generation/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task1687_sentiment140_classification/dev_tasks.json b/root_gainlora/configs/gen_script_superni_order1_llama_configs/task1687_sentiment140_classification/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task1687_sentiment140_classification/dev_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_llama_configs/task1687_sentiment140_classification/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task1687_sentiment140_classification/test_tasks.json b/root_gainlora/configs/gen_script_superni_order1_llama_configs/task1687_sentiment140_classification/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task1687_sentiment140_classification/test_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_llama_configs/task1687_sentiment140_classification/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task1687_sentiment140_classification/train_tasks.json b/root_gainlora/configs/gen_script_superni_order1_llama_configs/task1687_sentiment140_classification/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task1687_sentiment140_classification/train_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_llama_configs/task1687_sentiment140_classification/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task1729_personachat_generate_next/dev_tasks.json b/root_gainlora/configs/gen_script_superni_order1_llama_configs/task1729_personachat_generate_next/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task1729_personachat_generate_next/dev_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_llama_configs/task1729_personachat_generate_next/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task1729_personachat_generate_next/test_tasks.json b/root_gainlora/configs/gen_script_superni_order1_llama_configs/task1729_personachat_generate_next/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task1729_personachat_generate_next/test_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_llama_configs/task1729_personachat_generate_next/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task1729_personachat_generate_next/train_tasks.json b/root_gainlora/configs/gen_script_superni_order1_llama_configs/task1729_personachat_generate_next/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task1729_personachat_generate_next/train_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_llama_configs/task1729_personachat_generate_next/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task181_outcome_extraction/dev_tasks.json b/root_gainlora/configs/gen_script_superni_order1_llama_configs/task181_outcome_extraction/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task181_outcome_extraction/dev_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_llama_configs/task181_outcome_extraction/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task181_outcome_extraction/test_tasks.json b/root_gainlora/configs/gen_script_superni_order1_llama_configs/task181_outcome_extraction/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task181_outcome_extraction/test_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_llama_configs/task181_outcome_extraction/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task181_outcome_extraction/train_tasks.json b/root_gainlora/configs/gen_script_superni_order1_llama_configs/task181_outcome_extraction/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task181_outcome_extraction/train_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_llama_configs/task181_outcome_extraction/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task363_sst2_polarity_classification/dev_tasks.json b/root_gainlora/configs/gen_script_superni_order1_llama_configs/task363_sst2_polarity_classification/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task363_sst2_polarity_classification/dev_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_llama_configs/task363_sst2_polarity_classification/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task363_sst2_polarity_classification/test_tasks.json b/root_gainlora/configs/gen_script_superni_order1_llama_configs/task363_sst2_polarity_classification/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task363_sst2_polarity_classification/test_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_llama_configs/task363_sst2_polarity_classification/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task363_sst2_polarity_classification/train_tasks.json b/root_gainlora/configs/gen_script_superni_order1_llama_configs/task363_sst2_polarity_classification/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task363_sst2_polarity_classification/train_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_llama_configs/task363_sst2_polarity_classification/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task511_reddit_tifu_long_text_summarization/dev_tasks.json b/root_gainlora/configs/gen_script_superni_order1_llama_configs/task511_reddit_tifu_long_text_summarization/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task511_reddit_tifu_long_text_summarization/dev_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_llama_configs/task511_reddit_tifu_long_text_summarization/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task511_reddit_tifu_long_text_summarization/test_tasks.json b/root_gainlora/configs/gen_script_superni_order1_llama_configs/task511_reddit_tifu_long_text_summarization/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task511_reddit_tifu_long_text_summarization/test_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_llama_configs/task511_reddit_tifu_long_text_summarization/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task511_reddit_tifu_long_text_summarization/train_tasks.json b/root_gainlora/configs/gen_script_superni_order1_llama_configs/task511_reddit_tifu_long_text_summarization/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task511_reddit_tifu_long_text_summarization/train_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_llama_configs/task511_reddit_tifu_long_text_summarization/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task591_sciq_answer_generation/dev_tasks.json b/root_gainlora/configs/gen_script_superni_order1_llama_configs/task591_sciq_answer_generation/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task591_sciq_answer_generation/dev_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_llama_configs/task591_sciq_answer_generation/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task591_sciq_answer_generation/test_tasks.json b/root_gainlora/configs/gen_script_superni_order1_llama_configs/task591_sciq_answer_generation/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task591_sciq_answer_generation/test_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_llama_configs/task591_sciq_answer_generation/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task591_sciq_answer_generation/train_tasks.json b/root_gainlora/configs/gen_script_superni_order1_llama_configs/task591_sciq_answer_generation/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task591_sciq_answer_generation/train_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_llama_configs/task591_sciq_answer_generation/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task639_multi_woz_user_utterance_generation/dev_tasks.json b/root_gainlora/configs/gen_script_superni_order1_llama_configs/task639_multi_woz_user_utterance_generation/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task639_multi_woz_user_utterance_generation/dev_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_llama_configs/task639_multi_woz_user_utterance_generation/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task639_multi_woz_user_utterance_generation/test_tasks.json b/root_gainlora/configs/gen_script_superni_order1_llama_configs/task639_multi_woz_user_utterance_generation/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task639_multi_woz_user_utterance_generation/test_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_llama_configs/task639_multi_woz_user_utterance_generation/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task639_multi_woz_user_utterance_generation/train_tasks.json b/root_gainlora/configs/gen_script_superni_order1_llama_configs/task639_multi_woz_user_utterance_generation/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task639_multi_woz_user_utterance_generation/train_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_llama_configs/task639_multi_woz_user_utterance_generation/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task748_glucose_reverse_cause_event_detection/dev_tasks.json b/root_gainlora/configs/gen_script_superni_order1_llama_configs/task748_glucose_reverse_cause_event_detection/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task748_glucose_reverse_cause_event_detection/dev_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_llama_configs/task748_glucose_reverse_cause_event_detection/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task748_glucose_reverse_cause_event_detection/test_tasks.json b/root_gainlora/configs/gen_script_superni_order1_llama_configs/task748_glucose_reverse_cause_event_detection/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task748_glucose_reverse_cause_event_detection/test_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_llama_configs/task748_glucose_reverse_cause_event_detection/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task748_glucose_reverse_cause_event_detection/train_tasks.json b/root_gainlora/configs/gen_script_superni_order1_llama_configs/task748_glucose_reverse_cause_event_detection/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task748_glucose_reverse_cause_event_detection/train_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_llama_configs/task748_glucose_reverse_cause_event_detection/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task875_emotion_classification/dev_tasks.json b/root_gainlora/configs/gen_script_superni_order1_llama_configs/task875_emotion_classification/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task875_emotion_classification/dev_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_llama_configs/task875_emotion_classification/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task875_emotion_classification/test_tasks.json b/root_gainlora/configs/gen_script_superni_order1_llama_configs/task875_emotion_classification/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task875_emotion_classification/test_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_llama_configs/task875_emotion_classification/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task875_emotion_classification/train_tasks.json b/root_gainlora/configs/gen_script_superni_order1_llama_configs/task875_emotion_classification/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_llama_configs/task875_emotion_classification/train_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_llama_configs/task875_emotion_classification/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task002_quoref_answer_generation/dev_tasks.json b/root_gainlora/configs/gen_script_superni_order1_t5_configs/task002_quoref_answer_generation/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task002_quoref_answer_generation/dev_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_t5_configs/task002_quoref_answer_generation/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task002_quoref_answer_generation/test_tasks.json b/root_gainlora/configs/gen_script_superni_order1_t5_configs/task002_quoref_answer_generation/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task002_quoref_answer_generation/test_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_t5_configs/task002_quoref_answer_generation/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task002_quoref_answer_generation/train_tasks.json b/root_gainlora/configs/gen_script_superni_order1_t5_configs/task002_quoref_answer_generation/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task002_quoref_answer_generation/train_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_t5_configs/task002_quoref_answer_generation/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task073_commonsenseqa_answer_generation/dev_tasks.json b/root_gainlora/configs/gen_script_superni_order1_t5_configs/task073_commonsenseqa_answer_generation/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task073_commonsenseqa_answer_generation/dev_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_t5_configs/task073_commonsenseqa_answer_generation/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task073_commonsenseqa_answer_generation/test_tasks.json b/root_gainlora/configs/gen_script_superni_order1_t5_configs/task073_commonsenseqa_answer_generation/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task073_commonsenseqa_answer_generation/test_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_t5_configs/task073_commonsenseqa_answer_generation/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task073_commonsenseqa_answer_generation/train_tasks.json b/root_gainlora/configs/gen_script_superni_order1_t5_configs/task073_commonsenseqa_answer_generation/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task073_commonsenseqa_answer_generation/train_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_t5_configs/task073_commonsenseqa_answer_generation/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task1290_xsum_summarization/dev_tasks.json b/root_gainlora/configs/gen_script_superni_order1_t5_configs/task1290_xsum_summarization/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task1290_xsum_summarization/dev_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_t5_configs/task1290_xsum_summarization/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task1290_xsum_summarization/test_tasks.json b/root_gainlora/configs/gen_script_superni_order1_t5_configs/task1290_xsum_summarization/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task1290_xsum_summarization/test_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_t5_configs/task1290_xsum_summarization/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task1290_xsum_summarization/train_tasks.json b/root_gainlora/configs/gen_script_superni_order1_t5_configs/task1290_xsum_summarization/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task1290_xsum_summarization/train_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_t5_configs/task1290_xsum_summarization/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task1510_evalution_relation_extraction/dev_tasks.json b/root_gainlora/configs/gen_script_superni_order1_t5_configs/task1510_evalution_relation_extraction/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task1510_evalution_relation_extraction/dev_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_t5_configs/task1510_evalution_relation_extraction/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task1510_evalution_relation_extraction/test_tasks.json b/root_gainlora/configs/gen_script_superni_order1_t5_configs/task1510_evalution_relation_extraction/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task1510_evalution_relation_extraction/test_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_t5_configs/task1510_evalution_relation_extraction/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task1510_evalution_relation_extraction/train_tasks.json b/root_gainlora/configs/gen_script_superni_order1_t5_configs/task1510_evalution_relation_extraction/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task1510_evalution_relation_extraction/train_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_t5_configs/task1510_evalution_relation_extraction/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task1572_samsum_summary/dev_tasks.json b/root_gainlora/configs/gen_script_superni_order1_t5_configs/task1572_samsum_summary/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task1572_samsum_summary/dev_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_t5_configs/task1572_samsum_summary/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task1572_samsum_summary/test_tasks.json b/root_gainlora/configs/gen_script_superni_order1_t5_configs/task1572_samsum_summary/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task1572_samsum_summary/test_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_t5_configs/task1572_samsum_summary/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task1572_samsum_summary/train_tasks.json b/root_gainlora/configs/gen_script_superni_order1_t5_configs/task1572_samsum_summary/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task1572_samsum_summary/train_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_t5_configs/task1572_samsum_summary/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task1590_diplomacy_text_generation/dev_tasks.json b/root_gainlora/configs/gen_script_superni_order1_t5_configs/task1590_diplomacy_text_generation/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task1590_diplomacy_text_generation/dev_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_t5_configs/task1590_diplomacy_text_generation/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task1590_diplomacy_text_generation/test_tasks.json b/root_gainlora/configs/gen_script_superni_order1_t5_configs/task1590_diplomacy_text_generation/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task1590_diplomacy_text_generation/test_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_t5_configs/task1590_diplomacy_text_generation/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task1590_diplomacy_text_generation/train_tasks.json b/root_gainlora/configs/gen_script_superni_order1_t5_configs/task1590_diplomacy_text_generation/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task1590_diplomacy_text_generation/train_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_t5_configs/task1590_diplomacy_text_generation/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task1687_sentiment140_classification/dev_tasks.json b/root_gainlora/configs/gen_script_superni_order1_t5_configs/task1687_sentiment140_classification/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task1687_sentiment140_classification/dev_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_t5_configs/task1687_sentiment140_classification/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task1687_sentiment140_classification/test_tasks.json b/root_gainlora/configs/gen_script_superni_order1_t5_configs/task1687_sentiment140_classification/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task1687_sentiment140_classification/test_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_t5_configs/task1687_sentiment140_classification/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task1687_sentiment140_classification/train_tasks.json b/root_gainlora/configs/gen_script_superni_order1_t5_configs/task1687_sentiment140_classification/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task1687_sentiment140_classification/train_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_t5_configs/task1687_sentiment140_classification/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task1729_personachat_generate_next/dev_tasks.json b/root_gainlora/configs/gen_script_superni_order1_t5_configs/task1729_personachat_generate_next/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task1729_personachat_generate_next/dev_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_t5_configs/task1729_personachat_generate_next/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task1729_personachat_generate_next/test_tasks.json b/root_gainlora/configs/gen_script_superni_order1_t5_configs/task1729_personachat_generate_next/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task1729_personachat_generate_next/test_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_t5_configs/task1729_personachat_generate_next/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task1729_personachat_generate_next/train_tasks.json b/root_gainlora/configs/gen_script_superni_order1_t5_configs/task1729_personachat_generate_next/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task1729_personachat_generate_next/train_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_t5_configs/task1729_personachat_generate_next/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task181_outcome_extraction/dev_tasks.json b/root_gainlora/configs/gen_script_superni_order1_t5_configs/task181_outcome_extraction/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task181_outcome_extraction/dev_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_t5_configs/task181_outcome_extraction/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task181_outcome_extraction/test_tasks.json b/root_gainlora/configs/gen_script_superni_order1_t5_configs/task181_outcome_extraction/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task181_outcome_extraction/test_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_t5_configs/task181_outcome_extraction/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task181_outcome_extraction/train_tasks.json b/root_gainlora/configs/gen_script_superni_order1_t5_configs/task181_outcome_extraction/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task181_outcome_extraction/train_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_t5_configs/task181_outcome_extraction/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task363_sst2_polarity_classification/dev_tasks.json b/root_gainlora/configs/gen_script_superni_order1_t5_configs/task363_sst2_polarity_classification/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task363_sst2_polarity_classification/dev_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_t5_configs/task363_sst2_polarity_classification/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task363_sst2_polarity_classification/test_tasks.json b/root_gainlora/configs/gen_script_superni_order1_t5_configs/task363_sst2_polarity_classification/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task363_sst2_polarity_classification/test_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_t5_configs/task363_sst2_polarity_classification/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task363_sst2_polarity_classification/train_tasks.json b/root_gainlora/configs/gen_script_superni_order1_t5_configs/task363_sst2_polarity_classification/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task363_sst2_polarity_classification/train_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_t5_configs/task363_sst2_polarity_classification/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task511_reddit_tifu_long_text_summarization/dev_tasks.json b/root_gainlora/configs/gen_script_superni_order1_t5_configs/task511_reddit_tifu_long_text_summarization/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task511_reddit_tifu_long_text_summarization/dev_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_t5_configs/task511_reddit_tifu_long_text_summarization/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task511_reddit_tifu_long_text_summarization/test_tasks.json b/root_gainlora/configs/gen_script_superni_order1_t5_configs/task511_reddit_tifu_long_text_summarization/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task511_reddit_tifu_long_text_summarization/test_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_t5_configs/task511_reddit_tifu_long_text_summarization/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task511_reddit_tifu_long_text_summarization/train_tasks.json b/root_gainlora/configs/gen_script_superni_order1_t5_configs/task511_reddit_tifu_long_text_summarization/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task511_reddit_tifu_long_text_summarization/train_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_t5_configs/task511_reddit_tifu_long_text_summarization/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task591_sciq_answer_generation/dev_tasks.json b/root_gainlora/configs/gen_script_superni_order1_t5_configs/task591_sciq_answer_generation/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task591_sciq_answer_generation/dev_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_t5_configs/task591_sciq_answer_generation/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task591_sciq_answer_generation/test_tasks.json b/root_gainlora/configs/gen_script_superni_order1_t5_configs/task591_sciq_answer_generation/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task591_sciq_answer_generation/test_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_t5_configs/task591_sciq_answer_generation/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task591_sciq_answer_generation/train_tasks.json b/root_gainlora/configs/gen_script_superni_order1_t5_configs/task591_sciq_answer_generation/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task591_sciq_answer_generation/train_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_t5_configs/task591_sciq_answer_generation/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task639_multi_woz_user_utterance_generation/dev_tasks.json b/root_gainlora/configs/gen_script_superni_order1_t5_configs/task639_multi_woz_user_utterance_generation/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task639_multi_woz_user_utterance_generation/dev_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_t5_configs/task639_multi_woz_user_utterance_generation/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task639_multi_woz_user_utterance_generation/test_tasks.json b/root_gainlora/configs/gen_script_superni_order1_t5_configs/task639_multi_woz_user_utterance_generation/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task639_multi_woz_user_utterance_generation/test_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_t5_configs/task639_multi_woz_user_utterance_generation/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task639_multi_woz_user_utterance_generation/train_tasks.json b/root_gainlora/configs/gen_script_superni_order1_t5_configs/task639_multi_woz_user_utterance_generation/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task639_multi_woz_user_utterance_generation/train_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_t5_configs/task639_multi_woz_user_utterance_generation/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task748_glucose_reverse_cause_event_detection/dev_tasks.json b/root_gainlora/configs/gen_script_superni_order1_t5_configs/task748_glucose_reverse_cause_event_detection/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task748_glucose_reverse_cause_event_detection/dev_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_t5_configs/task748_glucose_reverse_cause_event_detection/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task748_glucose_reverse_cause_event_detection/test_tasks.json b/root_gainlora/configs/gen_script_superni_order1_t5_configs/task748_glucose_reverse_cause_event_detection/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task748_glucose_reverse_cause_event_detection/test_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_t5_configs/task748_glucose_reverse_cause_event_detection/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task748_glucose_reverse_cause_event_detection/train_tasks.json b/root_gainlora/configs/gen_script_superni_order1_t5_configs/task748_glucose_reverse_cause_event_detection/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task748_glucose_reverse_cause_event_detection/train_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_t5_configs/task748_glucose_reverse_cause_event_detection/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task875_emotion_classification/dev_tasks.json b/root_gainlora/configs/gen_script_superni_order1_t5_configs/task875_emotion_classification/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task875_emotion_classification/dev_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_t5_configs/task875_emotion_classification/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task875_emotion_classification/test_tasks.json b/root_gainlora/configs/gen_script_superni_order1_t5_configs/task875_emotion_classification/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task875_emotion_classification/test_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_t5_configs/task875_emotion_classification/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task875_emotion_classification/train_tasks.json b/root_gainlora/configs/gen_script_superni_order1_t5_configs/task875_emotion_classification/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order1_t5_configs/task875_emotion_classification/train_tasks.json rename to root_gainlora/configs/gen_script_superni_order1_t5_configs/task875_emotion_classification/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task002_quoref_answer_generation/dev_tasks.json b/root_gainlora/configs/gen_script_superni_order2_llama_configs/task002_quoref_answer_generation/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task002_quoref_answer_generation/dev_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_llama_configs/task002_quoref_answer_generation/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task002_quoref_answer_generation/test_tasks.json b/root_gainlora/configs/gen_script_superni_order2_llama_configs/task002_quoref_answer_generation/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task002_quoref_answer_generation/test_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_llama_configs/task002_quoref_answer_generation/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task002_quoref_answer_generation/train_tasks.json b/root_gainlora/configs/gen_script_superni_order2_llama_configs/task002_quoref_answer_generation/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task002_quoref_answer_generation/train_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_llama_configs/task002_quoref_answer_generation/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task073_commonsenseqa_answer_generation/dev_tasks.json b/root_gainlora/configs/gen_script_superni_order2_llama_configs/task073_commonsenseqa_answer_generation/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task073_commonsenseqa_answer_generation/dev_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_llama_configs/task073_commonsenseqa_answer_generation/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task073_commonsenseqa_answer_generation/test_tasks.json b/root_gainlora/configs/gen_script_superni_order2_llama_configs/task073_commonsenseqa_answer_generation/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task073_commonsenseqa_answer_generation/test_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_llama_configs/task073_commonsenseqa_answer_generation/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task073_commonsenseqa_answer_generation/train_tasks.json b/root_gainlora/configs/gen_script_superni_order2_llama_configs/task073_commonsenseqa_answer_generation/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task073_commonsenseqa_answer_generation/train_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_llama_configs/task073_commonsenseqa_answer_generation/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task1290_xsum_summarization/dev_tasks.json b/root_gainlora/configs/gen_script_superni_order2_llama_configs/task1290_xsum_summarization/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task1290_xsum_summarization/dev_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_llama_configs/task1290_xsum_summarization/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task1290_xsum_summarization/test_tasks.json b/root_gainlora/configs/gen_script_superni_order2_llama_configs/task1290_xsum_summarization/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task1290_xsum_summarization/test_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_llama_configs/task1290_xsum_summarization/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task1290_xsum_summarization/train_tasks.json b/root_gainlora/configs/gen_script_superni_order2_llama_configs/task1290_xsum_summarization/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task1290_xsum_summarization/train_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_llama_configs/task1290_xsum_summarization/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task1510_evalution_relation_extraction/dev_tasks.json b/root_gainlora/configs/gen_script_superni_order2_llama_configs/task1510_evalution_relation_extraction/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task1510_evalution_relation_extraction/dev_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_llama_configs/task1510_evalution_relation_extraction/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task1510_evalution_relation_extraction/test_tasks.json b/root_gainlora/configs/gen_script_superni_order2_llama_configs/task1510_evalution_relation_extraction/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task1510_evalution_relation_extraction/test_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_llama_configs/task1510_evalution_relation_extraction/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task1510_evalution_relation_extraction/train_tasks.json b/root_gainlora/configs/gen_script_superni_order2_llama_configs/task1510_evalution_relation_extraction/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task1510_evalution_relation_extraction/train_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_llama_configs/task1510_evalution_relation_extraction/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task1572_samsum_summary/dev_tasks.json b/root_gainlora/configs/gen_script_superni_order2_llama_configs/task1572_samsum_summary/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task1572_samsum_summary/dev_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_llama_configs/task1572_samsum_summary/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task1572_samsum_summary/test_tasks.json b/root_gainlora/configs/gen_script_superni_order2_llama_configs/task1572_samsum_summary/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task1572_samsum_summary/test_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_llama_configs/task1572_samsum_summary/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task1572_samsum_summary/train_tasks.json b/root_gainlora/configs/gen_script_superni_order2_llama_configs/task1572_samsum_summary/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task1572_samsum_summary/train_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_llama_configs/task1572_samsum_summary/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task1590_diplomacy_text_generation/dev_tasks.json b/root_gainlora/configs/gen_script_superni_order2_llama_configs/task1590_diplomacy_text_generation/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task1590_diplomacy_text_generation/dev_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_llama_configs/task1590_diplomacy_text_generation/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task1590_diplomacy_text_generation/test_tasks.json b/root_gainlora/configs/gen_script_superni_order2_llama_configs/task1590_diplomacy_text_generation/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task1590_diplomacy_text_generation/test_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_llama_configs/task1590_diplomacy_text_generation/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task1590_diplomacy_text_generation/train_tasks.json b/root_gainlora/configs/gen_script_superni_order2_llama_configs/task1590_diplomacy_text_generation/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task1590_diplomacy_text_generation/train_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_llama_configs/task1590_diplomacy_text_generation/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task1687_sentiment140_classification/dev_tasks.json b/root_gainlora/configs/gen_script_superni_order2_llama_configs/task1687_sentiment140_classification/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task1687_sentiment140_classification/dev_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_llama_configs/task1687_sentiment140_classification/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task1687_sentiment140_classification/test_tasks.json b/root_gainlora/configs/gen_script_superni_order2_llama_configs/task1687_sentiment140_classification/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task1687_sentiment140_classification/test_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_llama_configs/task1687_sentiment140_classification/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task1687_sentiment140_classification/train_tasks.json b/root_gainlora/configs/gen_script_superni_order2_llama_configs/task1687_sentiment140_classification/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task1687_sentiment140_classification/train_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_llama_configs/task1687_sentiment140_classification/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task1729_personachat_generate_next/dev_tasks.json b/root_gainlora/configs/gen_script_superni_order2_llama_configs/task1729_personachat_generate_next/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task1729_personachat_generate_next/dev_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_llama_configs/task1729_personachat_generate_next/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task1729_personachat_generate_next/test_tasks.json b/root_gainlora/configs/gen_script_superni_order2_llama_configs/task1729_personachat_generate_next/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task1729_personachat_generate_next/test_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_llama_configs/task1729_personachat_generate_next/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task1729_personachat_generate_next/train_tasks.json b/root_gainlora/configs/gen_script_superni_order2_llama_configs/task1729_personachat_generate_next/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task1729_personachat_generate_next/train_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_llama_configs/task1729_personachat_generate_next/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task181_outcome_extraction/dev_tasks.json b/root_gainlora/configs/gen_script_superni_order2_llama_configs/task181_outcome_extraction/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task181_outcome_extraction/dev_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_llama_configs/task181_outcome_extraction/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task181_outcome_extraction/test_tasks.json b/root_gainlora/configs/gen_script_superni_order2_llama_configs/task181_outcome_extraction/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task181_outcome_extraction/test_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_llama_configs/task181_outcome_extraction/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task181_outcome_extraction/train_tasks.json b/root_gainlora/configs/gen_script_superni_order2_llama_configs/task181_outcome_extraction/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task181_outcome_extraction/train_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_llama_configs/task181_outcome_extraction/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task363_sst2_polarity_classification/dev_tasks.json b/root_gainlora/configs/gen_script_superni_order2_llama_configs/task363_sst2_polarity_classification/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task363_sst2_polarity_classification/dev_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_llama_configs/task363_sst2_polarity_classification/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task363_sst2_polarity_classification/test_tasks.json b/root_gainlora/configs/gen_script_superni_order2_llama_configs/task363_sst2_polarity_classification/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task363_sst2_polarity_classification/test_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_llama_configs/task363_sst2_polarity_classification/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task363_sst2_polarity_classification/train_tasks.json b/root_gainlora/configs/gen_script_superni_order2_llama_configs/task363_sst2_polarity_classification/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task363_sst2_polarity_classification/train_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_llama_configs/task363_sst2_polarity_classification/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task511_reddit_tifu_long_text_summarization/dev_tasks.json b/root_gainlora/configs/gen_script_superni_order2_llama_configs/task511_reddit_tifu_long_text_summarization/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task511_reddit_tifu_long_text_summarization/dev_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_llama_configs/task511_reddit_tifu_long_text_summarization/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task511_reddit_tifu_long_text_summarization/test_tasks.json b/root_gainlora/configs/gen_script_superni_order2_llama_configs/task511_reddit_tifu_long_text_summarization/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task511_reddit_tifu_long_text_summarization/test_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_llama_configs/task511_reddit_tifu_long_text_summarization/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task511_reddit_tifu_long_text_summarization/train_tasks.json b/root_gainlora/configs/gen_script_superni_order2_llama_configs/task511_reddit_tifu_long_text_summarization/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task511_reddit_tifu_long_text_summarization/train_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_llama_configs/task511_reddit_tifu_long_text_summarization/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task591_sciq_answer_generation/dev_tasks.json b/root_gainlora/configs/gen_script_superni_order2_llama_configs/task591_sciq_answer_generation/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task591_sciq_answer_generation/dev_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_llama_configs/task591_sciq_answer_generation/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task591_sciq_answer_generation/test_tasks.json b/root_gainlora/configs/gen_script_superni_order2_llama_configs/task591_sciq_answer_generation/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task591_sciq_answer_generation/test_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_llama_configs/task591_sciq_answer_generation/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task591_sciq_answer_generation/train_tasks.json b/root_gainlora/configs/gen_script_superni_order2_llama_configs/task591_sciq_answer_generation/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task591_sciq_answer_generation/train_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_llama_configs/task591_sciq_answer_generation/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task639_multi_woz_user_utterance_generation/dev_tasks.json b/root_gainlora/configs/gen_script_superni_order2_llama_configs/task639_multi_woz_user_utterance_generation/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task639_multi_woz_user_utterance_generation/dev_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_llama_configs/task639_multi_woz_user_utterance_generation/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task639_multi_woz_user_utterance_generation/test_tasks.json b/root_gainlora/configs/gen_script_superni_order2_llama_configs/task639_multi_woz_user_utterance_generation/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task639_multi_woz_user_utterance_generation/test_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_llama_configs/task639_multi_woz_user_utterance_generation/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task639_multi_woz_user_utterance_generation/train_tasks.json b/root_gainlora/configs/gen_script_superni_order2_llama_configs/task639_multi_woz_user_utterance_generation/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task639_multi_woz_user_utterance_generation/train_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_llama_configs/task639_multi_woz_user_utterance_generation/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task748_glucose_reverse_cause_event_detection/dev_tasks.json b/root_gainlora/configs/gen_script_superni_order2_llama_configs/task748_glucose_reverse_cause_event_detection/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task748_glucose_reverse_cause_event_detection/dev_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_llama_configs/task748_glucose_reverse_cause_event_detection/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task748_glucose_reverse_cause_event_detection/test_tasks.json b/root_gainlora/configs/gen_script_superni_order2_llama_configs/task748_glucose_reverse_cause_event_detection/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task748_glucose_reverse_cause_event_detection/test_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_llama_configs/task748_glucose_reverse_cause_event_detection/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task748_glucose_reverse_cause_event_detection/train_tasks.json b/root_gainlora/configs/gen_script_superni_order2_llama_configs/task748_glucose_reverse_cause_event_detection/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task748_glucose_reverse_cause_event_detection/train_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_llama_configs/task748_glucose_reverse_cause_event_detection/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task875_emotion_classification/dev_tasks.json b/root_gainlora/configs/gen_script_superni_order2_llama_configs/task875_emotion_classification/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task875_emotion_classification/dev_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_llama_configs/task875_emotion_classification/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task875_emotion_classification/test_tasks.json b/root_gainlora/configs/gen_script_superni_order2_llama_configs/task875_emotion_classification/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task875_emotion_classification/test_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_llama_configs/task875_emotion_classification/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task875_emotion_classification/train_tasks.json b/root_gainlora/configs/gen_script_superni_order2_llama_configs/task875_emotion_classification/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_llama_configs/task875_emotion_classification/train_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_llama_configs/task875_emotion_classification/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task002_quoref_answer_generation/dev_tasks.json b/root_gainlora/configs/gen_script_superni_order2_t5_configs/task002_quoref_answer_generation/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task002_quoref_answer_generation/dev_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_t5_configs/task002_quoref_answer_generation/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task002_quoref_answer_generation/test_tasks.json b/root_gainlora/configs/gen_script_superni_order2_t5_configs/task002_quoref_answer_generation/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task002_quoref_answer_generation/test_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_t5_configs/task002_quoref_answer_generation/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task002_quoref_answer_generation/train_tasks.json b/root_gainlora/configs/gen_script_superni_order2_t5_configs/task002_quoref_answer_generation/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task002_quoref_answer_generation/train_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_t5_configs/task002_quoref_answer_generation/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task073_commonsenseqa_answer_generation/dev_tasks.json b/root_gainlora/configs/gen_script_superni_order2_t5_configs/task073_commonsenseqa_answer_generation/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task073_commonsenseqa_answer_generation/dev_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_t5_configs/task073_commonsenseqa_answer_generation/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task073_commonsenseqa_answer_generation/test_tasks.json b/root_gainlora/configs/gen_script_superni_order2_t5_configs/task073_commonsenseqa_answer_generation/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task073_commonsenseqa_answer_generation/test_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_t5_configs/task073_commonsenseqa_answer_generation/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task073_commonsenseqa_answer_generation/train_tasks.json b/root_gainlora/configs/gen_script_superni_order2_t5_configs/task073_commonsenseqa_answer_generation/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task073_commonsenseqa_answer_generation/train_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_t5_configs/task073_commonsenseqa_answer_generation/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task1290_xsum_summarization/dev_tasks.json b/root_gainlora/configs/gen_script_superni_order2_t5_configs/task1290_xsum_summarization/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task1290_xsum_summarization/dev_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_t5_configs/task1290_xsum_summarization/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task1290_xsum_summarization/test_tasks.json b/root_gainlora/configs/gen_script_superni_order2_t5_configs/task1290_xsum_summarization/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task1290_xsum_summarization/test_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_t5_configs/task1290_xsum_summarization/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task1290_xsum_summarization/train_tasks.json b/root_gainlora/configs/gen_script_superni_order2_t5_configs/task1290_xsum_summarization/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task1290_xsum_summarization/train_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_t5_configs/task1290_xsum_summarization/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task1510_evalution_relation_extraction/dev_tasks.json b/root_gainlora/configs/gen_script_superni_order2_t5_configs/task1510_evalution_relation_extraction/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task1510_evalution_relation_extraction/dev_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_t5_configs/task1510_evalution_relation_extraction/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task1510_evalution_relation_extraction/test_tasks.json b/root_gainlora/configs/gen_script_superni_order2_t5_configs/task1510_evalution_relation_extraction/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task1510_evalution_relation_extraction/test_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_t5_configs/task1510_evalution_relation_extraction/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task1510_evalution_relation_extraction/train_tasks.json b/root_gainlora/configs/gen_script_superni_order2_t5_configs/task1510_evalution_relation_extraction/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task1510_evalution_relation_extraction/train_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_t5_configs/task1510_evalution_relation_extraction/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task1572_samsum_summary/dev_tasks.json b/root_gainlora/configs/gen_script_superni_order2_t5_configs/task1572_samsum_summary/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task1572_samsum_summary/dev_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_t5_configs/task1572_samsum_summary/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task1572_samsum_summary/test_tasks.json b/root_gainlora/configs/gen_script_superni_order2_t5_configs/task1572_samsum_summary/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task1572_samsum_summary/test_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_t5_configs/task1572_samsum_summary/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task1572_samsum_summary/train_tasks.json b/root_gainlora/configs/gen_script_superni_order2_t5_configs/task1572_samsum_summary/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task1572_samsum_summary/train_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_t5_configs/task1572_samsum_summary/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task1590_diplomacy_text_generation/dev_tasks.json b/root_gainlora/configs/gen_script_superni_order2_t5_configs/task1590_diplomacy_text_generation/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task1590_diplomacy_text_generation/dev_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_t5_configs/task1590_diplomacy_text_generation/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task1590_diplomacy_text_generation/test_tasks.json b/root_gainlora/configs/gen_script_superni_order2_t5_configs/task1590_diplomacy_text_generation/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task1590_diplomacy_text_generation/test_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_t5_configs/task1590_diplomacy_text_generation/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task1590_diplomacy_text_generation/train_tasks.json b/root_gainlora/configs/gen_script_superni_order2_t5_configs/task1590_diplomacy_text_generation/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task1590_diplomacy_text_generation/train_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_t5_configs/task1590_diplomacy_text_generation/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task1687_sentiment140_classification/dev_tasks.json b/root_gainlora/configs/gen_script_superni_order2_t5_configs/task1687_sentiment140_classification/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task1687_sentiment140_classification/dev_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_t5_configs/task1687_sentiment140_classification/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task1687_sentiment140_classification/test_tasks.json b/root_gainlora/configs/gen_script_superni_order2_t5_configs/task1687_sentiment140_classification/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task1687_sentiment140_classification/test_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_t5_configs/task1687_sentiment140_classification/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task1687_sentiment140_classification/train_tasks.json b/root_gainlora/configs/gen_script_superni_order2_t5_configs/task1687_sentiment140_classification/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task1687_sentiment140_classification/train_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_t5_configs/task1687_sentiment140_classification/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task1729_personachat_generate_next/dev_tasks.json b/root_gainlora/configs/gen_script_superni_order2_t5_configs/task1729_personachat_generate_next/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task1729_personachat_generate_next/dev_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_t5_configs/task1729_personachat_generate_next/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task1729_personachat_generate_next/test_tasks.json b/root_gainlora/configs/gen_script_superni_order2_t5_configs/task1729_personachat_generate_next/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task1729_personachat_generate_next/test_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_t5_configs/task1729_personachat_generate_next/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task1729_personachat_generate_next/train_tasks.json b/root_gainlora/configs/gen_script_superni_order2_t5_configs/task1729_personachat_generate_next/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task1729_personachat_generate_next/train_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_t5_configs/task1729_personachat_generate_next/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task181_outcome_extraction/dev_tasks.json b/root_gainlora/configs/gen_script_superni_order2_t5_configs/task181_outcome_extraction/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task181_outcome_extraction/dev_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_t5_configs/task181_outcome_extraction/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task181_outcome_extraction/test_tasks.json b/root_gainlora/configs/gen_script_superni_order2_t5_configs/task181_outcome_extraction/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task181_outcome_extraction/test_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_t5_configs/task181_outcome_extraction/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task181_outcome_extraction/train_tasks.json b/root_gainlora/configs/gen_script_superni_order2_t5_configs/task181_outcome_extraction/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task181_outcome_extraction/train_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_t5_configs/task181_outcome_extraction/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task363_sst2_polarity_classification/dev_tasks.json b/root_gainlora/configs/gen_script_superni_order2_t5_configs/task363_sst2_polarity_classification/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task363_sst2_polarity_classification/dev_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_t5_configs/task363_sst2_polarity_classification/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task363_sst2_polarity_classification/test_tasks.json b/root_gainlora/configs/gen_script_superni_order2_t5_configs/task363_sst2_polarity_classification/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task363_sst2_polarity_classification/test_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_t5_configs/task363_sst2_polarity_classification/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task363_sst2_polarity_classification/train_tasks.json b/root_gainlora/configs/gen_script_superni_order2_t5_configs/task363_sst2_polarity_classification/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task363_sst2_polarity_classification/train_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_t5_configs/task363_sst2_polarity_classification/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task511_reddit_tifu_long_text_summarization/dev_tasks.json b/root_gainlora/configs/gen_script_superni_order2_t5_configs/task511_reddit_tifu_long_text_summarization/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task511_reddit_tifu_long_text_summarization/dev_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_t5_configs/task511_reddit_tifu_long_text_summarization/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task511_reddit_tifu_long_text_summarization/test_tasks.json b/root_gainlora/configs/gen_script_superni_order2_t5_configs/task511_reddit_tifu_long_text_summarization/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task511_reddit_tifu_long_text_summarization/test_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_t5_configs/task511_reddit_tifu_long_text_summarization/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task511_reddit_tifu_long_text_summarization/train_tasks.json b/root_gainlora/configs/gen_script_superni_order2_t5_configs/task511_reddit_tifu_long_text_summarization/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task511_reddit_tifu_long_text_summarization/train_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_t5_configs/task511_reddit_tifu_long_text_summarization/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task591_sciq_answer_generation/dev_tasks.json b/root_gainlora/configs/gen_script_superni_order2_t5_configs/task591_sciq_answer_generation/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task591_sciq_answer_generation/dev_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_t5_configs/task591_sciq_answer_generation/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task591_sciq_answer_generation/test_tasks.json b/root_gainlora/configs/gen_script_superni_order2_t5_configs/task591_sciq_answer_generation/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task591_sciq_answer_generation/test_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_t5_configs/task591_sciq_answer_generation/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task591_sciq_answer_generation/train_tasks.json b/root_gainlora/configs/gen_script_superni_order2_t5_configs/task591_sciq_answer_generation/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task591_sciq_answer_generation/train_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_t5_configs/task591_sciq_answer_generation/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task639_multi_woz_user_utterance_generation/dev_tasks.json b/root_gainlora/configs/gen_script_superni_order2_t5_configs/task639_multi_woz_user_utterance_generation/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task639_multi_woz_user_utterance_generation/dev_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_t5_configs/task639_multi_woz_user_utterance_generation/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task639_multi_woz_user_utterance_generation/test_tasks.json b/root_gainlora/configs/gen_script_superni_order2_t5_configs/task639_multi_woz_user_utterance_generation/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task639_multi_woz_user_utterance_generation/test_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_t5_configs/task639_multi_woz_user_utterance_generation/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task639_multi_woz_user_utterance_generation/train_tasks.json b/root_gainlora/configs/gen_script_superni_order2_t5_configs/task639_multi_woz_user_utterance_generation/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task639_multi_woz_user_utterance_generation/train_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_t5_configs/task639_multi_woz_user_utterance_generation/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task748_glucose_reverse_cause_event_detection/dev_tasks.json b/root_gainlora/configs/gen_script_superni_order2_t5_configs/task748_glucose_reverse_cause_event_detection/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task748_glucose_reverse_cause_event_detection/dev_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_t5_configs/task748_glucose_reverse_cause_event_detection/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task748_glucose_reverse_cause_event_detection/test_tasks.json b/root_gainlora/configs/gen_script_superni_order2_t5_configs/task748_glucose_reverse_cause_event_detection/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task748_glucose_reverse_cause_event_detection/test_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_t5_configs/task748_glucose_reverse_cause_event_detection/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task748_glucose_reverse_cause_event_detection/train_tasks.json b/root_gainlora/configs/gen_script_superni_order2_t5_configs/task748_glucose_reverse_cause_event_detection/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task748_glucose_reverse_cause_event_detection/train_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_t5_configs/task748_glucose_reverse_cause_event_detection/train_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task875_emotion_classification/dev_tasks.json b/root_gainlora/configs/gen_script_superni_order2_t5_configs/task875_emotion_classification/dev_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task875_emotion_classification/dev_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_t5_configs/task875_emotion_classification/dev_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task875_emotion_classification/test_tasks.json b/root_gainlora/configs/gen_script_superni_order2_t5_configs/task875_emotion_classification/test_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task875_emotion_classification/test_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_t5_configs/task875_emotion_classification/test_tasks.json diff --git a/gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task875_emotion_classification/train_tasks.json b/root_gainlora/configs/gen_script_superni_order2_t5_configs/task875_emotion_classification/train_tasks.json similarity index 100% rename from gainlora_OT_vMF/configs/gen_script_superni_order2_t5_configs/task875_emotion_classification/train_tasks.json rename to root_gainlora/configs/gen_script_superni_order2_t5_configs/task875_emotion_classification/train_tasks.json diff --git a/gainlora_OT_vMF/data/generate_labels.py b/root_gainlora/data/generate_labels.py similarity index 100% rename from gainlora_OT_vMF/data/generate_labels.py rename to root_gainlora/data/generate_labels.py diff --git a/gainlora_OT_vMF/data/gpt2tokenizer/config.json b/root_gainlora/data/gpt2tokenizer/config.json similarity index 100% rename from gainlora_OT_vMF/data/gpt2tokenizer/config.json rename to root_gainlora/data/gpt2tokenizer/config.json diff --git a/gainlora_OT_vMF/data/gpt2tokenizer/merges.txt b/root_gainlora/data/gpt2tokenizer/merges.txt similarity index 100% rename from gainlora_OT_vMF/data/gpt2tokenizer/merges.txt rename to root_gainlora/data/gpt2tokenizer/merges.txt diff --git a/gainlora_OT_vMF/data/gpt2tokenizer/special_tokens_map.json b/root_gainlora/data/gpt2tokenizer/special_tokens_map.json similarity index 100% rename from gainlora_OT_vMF/data/gpt2tokenizer/special_tokens_map.json rename to root_gainlora/data/gpt2tokenizer/special_tokens_map.json diff --git a/gainlora_OT_vMF/data/gpt2tokenizer/tokenizer.json b/root_gainlora/data/gpt2tokenizer/tokenizer.json similarity index 100% rename from gainlora_OT_vMF/data/gpt2tokenizer/tokenizer.json rename to root_gainlora/data/gpt2tokenizer/tokenizer.json diff --git a/gainlora_OT_vMF/data/gpt2tokenizer/tokenizer_config.json b/root_gainlora/data/gpt2tokenizer/tokenizer_config.json similarity index 100% rename from gainlora_OT_vMF/data/gpt2tokenizer/tokenizer_config.json rename to root_gainlora/data/gpt2tokenizer/tokenizer_config.json diff --git a/gainlora_OT_vMF/data/gpt2tokenizer/vocab.json b/root_gainlora/data/gpt2tokenizer/vocab.json similarity index 100% rename from gainlora_OT_vMF/data/gpt2tokenizer/vocab.json rename to root_gainlora/data/gpt2tokenizer/vocab.json diff --git a/gainlora_OT_vMF/gainlora.png b/root_gainlora/gainlora.png similarity index 100% rename from gainlora_OT_vMF/gainlora.png rename to root_gainlora/gainlora.png diff --git a/gainlora_OT_vMF/gen_script_long_order3_t5_gainlora_inflora.sh b/root_gainlora/gen_script_long_order3_t5_gainlora_inflora.sh similarity index 100% rename from gainlora_OT_vMF/gen_script_long_order3_t5_gainlora_inflora.sh rename to root_gainlora/gen_script_long_order3_t5_gainlora_inflora.sh diff --git a/gainlora_OT_vMF/gen_script_long_order3_t5_inflora.sh b/root_gainlora/gen_script_long_order3_t5_inflora.sh similarity index 100% rename from gainlora_OT_vMF/gen_script_long_order3_t5_inflora.sh rename to root_gainlora/gen_script_long_order3_t5_inflora.sh diff --git a/gainlora_OT_vMF/gen_script_long_order4_t5_gainlora_inflora.sh b/root_gainlora/gen_script_long_order4_t5_gainlora_inflora.sh similarity index 100% rename from gainlora_OT_vMF/gen_script_long_order4_t5_gainlora_inflora.sh rename to root_gainlora/gen_script_long_order4_t5_gainlora_inflora.sh diff --git a/gainlora_OT_vMF/gen_script_long_order4_t5_inflora.sh b/root_gainlora/gen_script_long_order4_t5_inflora.sh similarity index 100% rename from gainlora_OT_vMF/gen_script_long_order4_t5_inflora.sh rename to root_gainlora/gen_script_long_order4_t5_inflora.sh diff --git a/gainlora_OT_vMF/gen_script_superni_order1_llama_gainlora_inflora.sh b/root_gainlora/gen_script_superni_order1_llama_gainlora_inflora.sh similarity index 100% rename from gainlora_OT_vMF/gen_script_superni_order1_llama_gainlora_inflora.sh rename to root_gainlora/gen_script_superni_order1_llama_gainlora_inflora.sh diff --git a/gainlora_OT_vMF/gen_script_superni_order1_llama_gainlora_olora.sh b/root_gainlora/gen_script_superni_order1_llama_gainlora_olora.sh similarity index 100% rename from gainlora_OT_vMF/gen_script_superni_order1_llama_gainlora_olora.sh rename to root_gainlora/gen_script_superni_order1_llama_gainlora_olora.sh diff --git a/gainlora_OT_vMF/gen_script_superni_order1_llama_inflora.sh b/root_gainlora/gen_script_superni_order1_llama_inflora.sh similarity index 100% rename from gainlora_OT_vMF/gen_script_superni_order1_llama_inflora.sh rename to root_gainlora/gen_script_superni_order1_llama_inflora.sh diff --git a/gainlora_OT_vMF/gen_script_superni_order1_llama_olora.sh b/root_gainlora/gen_script_superni_order1_llama_olora.sh similarity index 100% rename from gainlora_OT_vMF/gen_script_superni_order1_llama_olora.sh rename to root_gainlora/gen_script_superni_order1_llama_olora.sh diff --git a/gainlora_OT_vMF/gen_script_superni_order1_t5_gainlora_inflora.sh b/root_gainlora/gen_script_superni_order1_t5_gainlora_inflora.sh similarity index 100% rename from gainlora_OT_vMF/gen_script_superni_order1_t5_gainlora_inflora.sh rename to root_gainlora/gen_script_superni_order1_t5_gainlora_inflora.sh diff --git a/gainlora_OT_vMF/gen_script_superni_order1_t5_inflora.sh b/root_gainlora/gen_script_superni_order1_t5_inflora.sh similarity index 100% rename from gainlora_OT_vMF/gen_script_superni_order1_t5_inflora.sh rename to root_gainlora/gen_script_superni_order1_t5_inflora.sh diff --git a/gainlora_OT_vMF/gen_script_superni_order2_llama_gainlora_inflora.sh b/root_gainlora/gen_script_superni_order2_llama_gainlora_inflora.sh similarity index 100% rename from gainlora_OT_vMF/gen_script_superni_order2_llama_gainlora_inflora.sh rename to root_gainlora/gen_script_superni_order2_llama_gainlora_inflora.sh diff --git a/gainlora_OT_vMF/gen_script_superni_order2_llama_gainlora_olora.sh b/root_gainlora/gen_script_superni_order2_llama_gainlora_olora.sh similarity index 100% rename from gainlora_OT_vMF/gen_script_superni_order2_llama_gainlora_olora.sh rename to root_gainlora/gen_script_superni_order2_llama_gainlora_olora.sh diff --git a/gainlora_OT_vMF/gen_script_superni_order2_llama_inflora.sh b/root_gainlora/gen_script_superni_order2_llama_inflora.sh similarity index 100% rename from gainlora_OT_vMF/gen_script_superni_order2_llama_inflora.sh rename to root_gainlora/gen_script_superni_order2_llama_inflora.sh diff --git a/gainlora_OT_vMF/gen_script_superni_order2_llama_olora.sh b/root_gainlora/gen_script_superni_order2_llama_olora.sh similarity index 100% rename from gainlora_OT_vMF/gen_script_superni_order2_llama_olora.sh rename to root_gainlora/gen_script_superni_order2_llama_olora.sh diff --git a/gainlora_OT_vMF/gen_script_superni_order2_t5_gainlora_inflora.sh b/root_gainlora/gen_script_superni_order2_t5_gainlora_inflora.sh similarity index 100% rename from gainlora_OT_vMF/gen_script_superni_order2_t5_gainlora_inflora.sh rename to root_gainlora/gen_script_superni_order2_t5_gainlora_inflora.sh diff --git a/gainlora_OT_vMF/gen_script_superni_order2_t5_inflora.sh b/root_gainlora/gen_script_superni_order2_t5_inflora.sh similarity index 100% rename from gainlora_OT_vMF/gen_script_superni_order2_t5_inflora.sh rename to root_gainlora/gen_script_superni_order2_t5_inflora.sh diff --git a/root_gainlora/push_github.sh b/root_gainlora/push_github.sh new file mode 100644 index 0000000000000000000000000000000000000000..92f451951225d1d032d67c932fd8eedaada8d032 --- /dev/null +++ b/root_gainlora/push_github.sh @@ -0,0 +1,4 @@ +cd /Users/nnminh322/Desktop/personal/Continual +git add . +git commit -m "update" +git push origin main diff --git a/gainlora_OT_vMF/requirements.yaml b/root_gainlora/requirements.yaml similarity index 100% rename from gainlora_OT_vMF/requirements.yaml rename to root_gainlora/requirements.yaml diff --git a/root_gainlora/runtest_colab.ipynb b/root_gainlora/runtest_colab.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..cc5c6e40e6ff3a9a270e07e10d9b44b68360c37f --- /dev/null +++ b/root_gainlora/runtest_colab.ipynb @@ -0,0 +1,53 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "bc5f87a4", + "metadata": {}, + "outputs": [], + "source": [ + "# Cell for start run\n", + "!git clone -b main --single-branch --depth 1 https://github.com/nnminh322/Continual.git\n", + "!pip install uv\n", + "%cd Continual/root_gainlora/\n", + "!bash setup_kaggle_colab.sh\n", + "!bash gen_script_superni_order1_t5_specroute.sh 0 google/flan-t5-large\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6257b4db", + "metadata": {}, + "outputs": [], + "source": [ + "# Cell for remove run\n", + "%cd ../..\n", + "!rm -rf ./Continual" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4b2ec1b8", + "metadata": {}, + "outputs": [], + "source": [ + "# Cell for rerun\n", + "!git clone -b main --single-branch --depth 1 https://github.com/nnminh322/Continual.git\n", + "%cd Continual/root_gainlora/\n", + "!bash gen_script_superni_order1_t5_specroute.sh 0 google/flan-t5-large\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/gainlora_OT_vMF/score.py b/root_gainlora/score.py similarity index 100% rename from gainlora_OT_vMF/score.py rename to root_gainlora/score.py diff --git a/root_gainlora/setup_kaggle_colab.sh b/root_gainlora/setup_kaggle_colab.sh new file mode 100755 index 0000000000000000000000000000000000000000..86c1020e5cc61a45965a799f448391a5d1c7ec94 --- /dev/null +++ b/root_gainlora/setup_kaggle_colab.sh @@ -0,0 +1,151 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Conflict-safe setup for Kaggle/Colab notebooks. +# Works with both conda environments and direct pip installs. +# Usage: +# bash setup_kaggle_colab.sh + +echo "==========================================" +echo "SpecRoute Setup for Kaggle/Colab" +echo "==========================================" + +# Detect environment +if [[ -d /kaggle/working ]]; then + ENV_TYPE="kaggle" +elif [[ -d /content ]]; then + ENV_TYPE="colab" +else + ENV_TYPE="local" +fi + +echo "[Env] Detected: ${ENV_TYPE}" + +# Optional conda mode: USE_CONDA=1 bash setup_kaggle_colab.sh +USE_CONDA="${USE_CONDA:-0}" +if command -v conda >/dev/null 2>&1 && [[ "${USE_CONDA}" == "1" ]]; then + echo "[Setup] Using conda environment (USE_CONDA=1)" + CONDA_ENV_NAME="specroute" + conda env remove -n "${CONDA_ENV_NAME}" -y >/dev/null 2>&1 || true + conda create -n "${CONDA_ENV_NAME}" python=3.10 -y + PY_CMD="conda run -n ${CONDA_ENV_NAME} python" + PIP_CMD="conda run -n ${CONDA_ENV_NAME} python -m pip" + ACTIVATE_CMD="conda activate ${CONDA_ENV_NAME}" +else + echo "[Setup] Using current Python environment" + PY_CMD="python" + PIP_CMD="python -m pip" +fi + +# Upgrade pip and core tools +${PIP_CMD} install --upgrade pip setuptools wheel + +echo "" +echo "[Remove] Uninstalling conflicting packages..." +# Remove common preinstalled conflicts (deep clean). +# JAX/jaxlib MUST go because they require numpy>=2.0 which breaks our pin. +# Also remove packages that depend on torch/jax to suppress pip warnings later. +${PIP_CMD} uninstall -y \ + torch torchvision torchaudio pytorch-cuda triton \ + cupy cupy-cuda12x flash-attn xformers bitsandbytes \ + transformers tokenizers datasets accelerate peft \ + sentence-transformers torchtune deepspeed apex \ + cudf-cu12 dask-cudf-cu12 cuml-cu12 cucim-cu12 \ + ydf grain umap-learn hdbscan textblob \ + opentelemetry-proto grpcio-status \ + protobuf \ + jax jaxlib \ + torchmetrics easyocr pytorch-lightning fastai \ + gymnax kaggle-environments dopamine-rl \ + flax optax orbax-checkpoint chex \ + nx-cugraph-cu12 pylibcugraph-cu12 \ + opencv-contrib-python opencv-python opencv-python-headless \ + shap rasterio tobler cesium pytensor \ + tensorflow-decision-forests \ + a2a-sdk bigframes google-adk gcsfs \ + 2>/dev/null || true + +echo "" +echo "[Install] Core CUDA stack (PyTorch cu121)..." +# 1) Pre-pin numpy+protobuf so torch install doesn't drag in wrong versions. +# 2) No --force-reinstall: that flag reinstalls ALL deps (including numpy) → disaster. +# Since torch was uninstalled above, pip will install 2.5.1 fresh. +# Note: pip may show "ERROR: dependency resolver" warnings from other Kaggle packages. +# These are harmless — those packages (kaggle-environments, dopamine-rl, etc.) +# won't be used and don't affect our training. We filter them for readability. +${PIP_CMD} install --no-cache-dir -q \ + "numpy==1.26.4" "protobuf==5.29.3" 2>&1 | grep -v "which is not installed\|which is incompatible\|dependency resolver" || true +${PIP_CMD} install --no-cache-dir -q \ + torch==2.5.1 torchvision==0.20.1 torchaudio==2.5.1 \ + --index-url https://download.pytorch.org/whl/cu121 2>&1 | grep -v "which is not installed\|which is incompatible\|dependency resolver" || true + +# Sanity-check: abort early if wrong torch survived +TORCH_VER=$(${PY_CMD} -c "import torch; print(torch.__version__)" 2>/dev/null || echo "NONE") +echo "[Check] torch version after install: ${TORCH_VER}" +if [[ "${TORCH_VER}" != 2.5.1* ]]; then + echo "[WARN] torch ${TORCH_VER} != 2.5.1 — forcing pip uninstall+reinstall..." + ${PIP_CMD} uninstall -y torch torchvision torchaudio 2>/dev/null || true + ${PIP_CMD} install --no-cache-dir -q --no-deps \ + torch==2.5.1 torchvision==0.20.1 torchaudio==2.5.1 \ + --index-url https://download.pytorch.org/whl/cu121 + TORCH_VER=$(${PY_CMD} -c "import torch; print(torch.__version__)" 2>/dev/null || echo "NONE") + echo "[Check] torch version after force: ${TORCH_VER}" + if [[ "${TORCH_VER}" != 2.5.1* ]]; then + echo "[ERROR] Cannot install torch 2.5.1. Aborting." + exit 1 + fi +fi + +echo "[Install] Project dependencies..." +${PIP_CMD} install --no-cache-dir -q \ + "numpy==1.26.4" "scipy==1.14.1" \ + transformers==4.40.2 tokenizers==0.19.1 \ + "datasets==2.21.0" accelerate==0.34.2 \ + loralib==0.1.2 sentencepiece==0.2.0 ipdb==0.13.13 \ + nltk==3.9.1 scikit-learn==1.6.1 pandas==2.2.2 \ + "pyarrow==17.0.0" "protobuf==5.29.3" tqdm==4.67.1 \ + "fsspec==2024.6.1" pynvml==11.5.3 \ + 2>&1 | grep -v "which is not installed\|which is incompatible\|dependency resolver" || true + +echo "[Install] CuPy (with fallback for Python/CUDA compatibility)..." +if ! ${PIP_CMD} install --no-cache-dir -q cupy-cuda12x==13.6.0; then + ${PIP_CMD} install --no-cache-dir -q 'cupy-cuda12x>=13.0,<15.0' +fi + +echo "[Cache] Clearing stale HuggingFace dataset module cache..." +rm -rf ~/.cache/huggingface/modules/datasets_modules/ 2>/dev/null || true +echo "[Cache] HF dataset module cache cleared" + +echo "" +echo "[Check] Verifying installation..." +${PY_CMD} - <<'PY' +import sys +try: + import torch + import transformers + import datasets + import cupy + print(f'✓ python {sys.version.split()[0]}') + print(f'✓ torch {torch.__version__}') + print(f'✓ cuda available: {torch.cuda.is_available()}') + if torch.cuda.is_available(): + print(f'✓ gpu count: {torch.cuda.device_count()}') + for i in range(torch.cuda.device_count()): + print(f' - GPU {i}: {torch.cuda.get_device_name(i)} ({torch.cuda.get_device_properties(i).total_memory / 1e9:.1f} GB)') + print(f'✓ transformers {transformers.__version__}') + print(f'✓ datasets {datasets.__version__}') + print(f'✓ cupy {cupy.__version__}') + print('\n[Success] All packages installed correctly!') +except Exception as e: + print(f'[Error] {e}', file=sys.stderr) + sys.exit(1) +PY + +echo "" +echo "==========================================" +if [[ -n "${ACTIVATE_CMD:-}" ]]; then + echo "[Ready] Run: ${ACTIVATE_CMD}" +else + echo "[Ready] Environment is ready to use" +fi +echo "==========================================" diff --git a/gainlora_OT_vMF/src/__init__.py b/root_gainlora/src/__init__.py similarity index 100% rename from gainlora_OT_vMF/src/__init__.py rename to root_gainlora/src/__init__.py diff --git a/gainlora_OT_vMF/src/__pycache__/assets.cpython-310.pyc b/root_gainlora/src/__pycache__/assets.cpython-310.pyc similarity index 100% rename from gainlora_OT_vMF/src/__pycache__/assets.cpython-310.pyc rename to root_gainlora/src/__pycache__/assets.cpython-310.pyc diff --git a/gainlora_OT_vMF/src/__pycache__/cl_collator.cpython-310.pyc b/root_gainlora/src/__pycache__/cl_collator.cpython-310.pyc similarity index 100% rename from gainlora_OT_vMF/src/__pycache__/cl_collator.cpython-310.pyc rename to root_gainlora/src/__pycache__/cl_collator.cpython-310.pyc diff --git a/gainlora_OT_vMF/src/__pycache__/cl_dataset.cpython-310.pyc b/root_gainlora/src/__pycache__/cl_dataset.cpython-310.pyc similarity index 100% rename from gainlora_OT_vMF/src/__pycache__/cl_dataset.cpython-310.pyc rename to root_gainlora/src/__pycache__/cl_dataset.cpython-310.pyc diff --git a/gainlora_OT_vMF/src/__pycache__/cl_trainer_gainlora_inflora.cpython-310.pyc b/root_gainlora/src/__pycache__/cl_trainer_gainlora_inflora.cpython-310.pyc similarity index 100% rename from gainlora_OT_vMF/src/__pycache__/cl_trainer_gainlora_inflora.cpython-310.pyc rename to root_gainlora/src/__pycache__/cl_trainer_gainlora_inflora.cpython-310.pyc diff --git a/gainlora_OT_vMF/src/__pycache__/cl_trainer_gainlora_inflora_llama.cpython-310.pyc b/root_gainlora/src/__pycache__/cl_trainer_gainlora_inflora_llama.cpython-310.pyc similarity index 100% rename from gainlora_OT_vMF/src/__pycache__/cl_trainer_gainlora_inflora_llama.cpython-310.pyc rename to root_gainlora/src/__pycache__/cl_trainer_gainlora_inflora_llama.cpython-310.pyc diff --git a/gainlora_OT_vMF/src/__pycache__/cl_trainer_gainlora_olora.cpython-310.pyc b/root_gainlora/src/__pycache__/cl_trainer_gainlora_olora.cpython-310.pyc similarity index 100% rename from gainlora_OT_vMF/src/__pycache__/cl_trainer_gainlora_olora.cpython-310.pyc rename to root_gainlora/src/__pycache__/cl_trainer_gainlora_olora.cpython-310.pyc diff --git a/gainlora_OT_vMF/src/__pycache__/cl_trainer_gainlora_olora_llama.cpython-310.pyc b/root_gainlora/src/__pycache__/cl_trainer_gainlora_olora_llama.cpython-310.pyc similarity index 100% rename from gainlora_OT_vMF/src/__pycache__/cl_trainer_gainlora_olora_llama.cpython-310.pyc rename to root_gainlora/src/__pycache__/cl_trainer_gainlora_olora_llama.cpython-310.pyc diff --git a/gainlora_OT_vMF/src/__pycache__/cl_trainer_inflora.cpython-310.pyc b/root_gainlora/src/__pycache__/cl_trainer_inflora.cpython-310.pyc similarity index 100% rename from gainlora_OT_vMF/src/__pycache__/cl_trainer_inflora.cpython-310.pyc rename to root_gainlora/src/__pycache__/cl_trainer_inflora.cpython-310.pyc diff --git a/gainlora_OT_vMF/src/__pycache__/cl_trainer_inflora_llama.cpython-310.pyc b/root_gainlora/src/__pycache__/cl_trainer_inflora_llama.cpython-310.pyc similarity index 100% rename from gainlora_OT_vMF/src/__pycache__/cl_trainer_inflora_llama.cpython-310.pyc rename to root_gainlora/src/__pycache__/cl_trainer_inflora_llama.cpython-310.pyc diff --git a/gainlora_OT_vMF/src/__pycache__/cl_trainer_olora.cpython-310.pyc b/root_gainlora/src/__pycache__/cl_trainer_olora.cpython-310.pyc similarity index 100% rename from gainlora_OT_vMF/src/__pycache__/cl_trainer_olora.cpython-310.pyc rename to root_gainlora/src/__pycache__/cl_trainer_olora.cpython-310.pyc diff --git a/gainlora_OT_vMF/src/__pycache__/cl_trainer_olora_llama.cpython-310.pyc b/root_gainlora/src/__pycache__/cl_trainer_olora_llama.cpython-310.pyc similarity index 100% rename from gainlora_OT_vMF/src/__pycache__/cl_trainer_olora_llama.cpython-310.pyc rename to root_gainlora/src/__pycache__/cl_trainer_olora_llama.cpython-310.pyc diff --git a/gainlora_OT_vMF/src/__pycache__/compute_metrics.cpython-310.pyc b/root_gainlora/src/__pycache__/compute_metrics.cpython-310.pyc similarity index 100% rename from gainlora_OT_vMF/src/__pycache__/compute_metrics.cpython-310.pyc rename to root_gainlora/src/__pycache__/compute_metrics.cpython-310.pyc diff --git a/gainlora_OT_vMF/src/__pycache__/llama_gainlora_inflora.cpython-310.pyc b/root_gainlora/src/__pycache__/llama_gainlora_inflora.cpython-310.pyc similarity index 100% rename from gainlora_OT_vMF/src/__pycache__/llama_gainlora_inflora.cpython-310.pyc rename to root_gainlora/src/__pycache__/llama_gainlora_inflora.cpython-310.pyc diff --git a/gainlora_OT_vMF/src/__pycache__/llama_inflora.cpython-310.pyc b/root_gainlora/src/__pycache__/llama_inflora.cpython-310.pyc similarity index 100% rename from gainlora_OT_vMF/src/__pycache__/llama_inflora.cpython-310.pyc rename to root_gainlora/src/__pycache__/llama_inflora.cpython-310.pyc diff --git a/gainlora_OT_vMF/src/__pycache__/t5_gainlora_inflora.cpython-310.pyc b/root_gainlora/src/__pycache__/t5_gainlora_inflora.cpython-310.pyc similarity index 100% rename from gainlora_OT_vMF/src/__pycache__/t5_gainlora_inflora.cpython-310.pyc rename to root_gainlora/src/__pycache__/t5_gainlora_inflora.cpython-310.pyc diff --git a/gainlora_OT_vMF/src/__pycache__/t5_inflora.cpython-310.pyc b/root_gainlora/src/__pycache__/t5_inflora.cpython-310.pyc similarity index 100% rename from gainlora_OT_vMF/src/__pycache__/t5_inflora.cpython-310.pyc rename to root_gainlora/src/__pycache__/t5_inflora.cpython-310.pyc diff --git a/gainlora_OT_vMF/src/assets.py b/root_gainlora/src/assets.py similarity index 100% rename from gainlora_OT_vMF/src/assets.py rename to root_gainlora/src/assets.py diff --git a/gainlora_OT_vMF/src/cl_collator.py b/root_gainlora/src/cl_collator.py similarity index 91% rename from gainlora_OT_vMF/src/cl_collator.py rename to root_gainlora/src/cl_collator.py index 151ae6ac4a6e62bcfad97abfb1186f43385db153..ec25828a95c49b1048cc099617b797f9b8ee0407 100644 --- a/gainlora_OT_vMF/src/cl_collator.py +++ b/root_gainlora/src/cl_collator.py @@ -1,6 +1,4 @@ import logging -from dataclasses import dataclass -from typing import Any, Optional, Union import torch from transformers.data.data_collator import * @@ -104,7 +102,7 @@ class DataCollator: instruction = self.get_instruction(instance) source = instruction - tokenized_source = self.tokenizer(source, truncation=True, max_length=self.max_source_length)["input_ids"] + tokenized_source = self.tokenizer(source)["input_ids"] if len(tokenized_source) <= self.max_source_length: sources.append(source) else: @@ -122,14 +120,15 @@ class DataCollator: truncation=True, pad_to_multiple_of=self.pad_to_multiple_of ) - # as_target_tokenizer() removed in transformers 5.0, no longer needed - labels = self.tokenizer( - labels, - max_length=self.max_target_length, - padding=self.padding, - return_tensors=return_tensors, - truncation=True, - ) + with self.tokenizer.as_target_tokenizer(): + labels = self.tokenizer( + labels, + max_length=self.max_target_length, + padding=self.padding, + return_tensors=return_tensors, + truncation=True, + pad_to_multiple_of=self.pad_to_multiple_of + ) label_mask = labels["attention_mask"].bool() model_inputs["labels"] = labels["input_ids"].masked_fill(~label_mask, self.label_pad_token_id) diff --git a/gainlora_OT_vMF/src/cl_dataset.py b/root_gainlora/src/cl_dataset.py similarity index 90% rename from gainlora_OT_vMF/src/cl_dataset.py rename to root_gainlora/src/cl_dataset.py index 7588251d180570baae333a4507c8ce6ddceca4e4..7b090aa0558ebda65422ab30173ece9f3815a3fd 100644 --- a/gainlora_OT_vMF/src/cl_dataset.py +++ b/root_gainlora/src/cl_dataset.py @@ -19,6 +19,8 @@ import json import os import random +from dataclasses import dataclass, field as dc_field +from typing import Optional import datasets from hashlib import md5 @@ -50,39 +52,26 @@ def save_ds(instances, file_name): json.dump(instances, fi, ensure_ascii=False, indent=2) +@dataclass class CLConfig(datasets.BuilderConfig): """ Config dataset load procedure. Args: data_dir: task data dir, which contains the corresponding dataset dirs - prompt_path: prompt json file, which saves task and its prompts map - task_file: task config file, save training and testing split config, and sampling strategies. - Support two sampling strategies: 'random' indicates random sampling, while 'full' means to return all samples. + task_config_dir: directory with train/dev/test task config json files max_num_instances_per_task: max training sample size of each task max_num_instances_per_eval_task: max eval sample size of each task """ - - def __init__( - self, - *args, - data_dir=None, - task_config_dir=None, - num_examples=None, - max_num_instances_per_task=None, - max_num_instances_per_eval_task=None, - over_sampling=None, - **kwargs - ): - super().__init__(*args, **kwargs) - self.data_dir = data_dir - self.num_examples = num_examples - self.over_sampling = over_sampling - self.task_configs = self._parse_task_config(task_config_dir) - self.max_num_instances_per_task = max_num_instances_per_task - self.max_num_instances_per_eval_task = max_num_instances_per_eval_task - - def _parse_task_config(self, task_config_dir): + task_config_dir: Optional[str] = None + num_examples: Optional[int] = None + max_num_instances_per_task: Optional[int] = None + max_num_instances_per_eval_task: Optional[int] = None + over_sampling: Optional[bool] = None + + @staticmethod + def parse_task_config(task_config_dir): + """Parse train/dev/test task config JSON files from the given directory.""" if not task_config_dir: return None @@ -137,13 +126,14 @@ class CLInstructions(datasets.GeneratorBasedBuilder): def _split_generators(self, dl_manager): """Returns SplitGenerators.""" - if self.config.data_dir is None or self.config.task_configs is None: + # Parse task configs lazily here (not in CLConfig) because datasets + # may copy/replace the config object, dropping non-field attributes. + task_configs = CLConfig.parse_task_config(self.config.task_config_dir) + + if self.config.data_dir is None or task_configs is None: logger.error("Please provide right input: data_dir or task_config_dir!") - # split dir save datasets - # task config to specify train,dev,test split_dir = self.config.data_dir - task_configs = self.config.task_configs return [ datasets.SplitGenerator( diff --git a/gainlora_OT_vMF/src/cl_trainer_gainlora_inflora.py b/root_gainlora/src/cl_trainer_gainlora_inflora.py similarity index 88% rename from gainlora_OT_vMF/src/cl_trainer_gainlora_inflora.py rename to root_gainlora/src/cl_trainer_gainlora_inflora.py index e81abed7b099037c9a97c22b3ebbace95c9989cf..c66493dbf28197e919cac361639435e42a209f94 100644 --- a/gainlora_OT_vMF/src/cl_trainer_gainlora_inflora.py +++ b/root_gainlora/src/cl_trainer_gainlora_inflora.py @@ -1,79 +1,16 @@ import torch -import torch.nn as nn -import torch.nn.functional as F_loss -from typing import Dict, Union, Any, Optional, List, Tuple, Callable from transformers import GenerationConfig from transformers.trainer_seq2seq import Seq2SeqTrainer from transformers.trainer import * -from transformers.trainer_utils import IntervalStrategy -from compat_transformers import ShardedDDPOption, is_sagemaker_mp_enabled, smp_forward_backward, deepspeed_init, deepspeed_load_checkpoint, HPSearchBackend, patch_trainer_compat, patch_args_compat, is_torch_tpu_available, ALL_LAYERNORM_LAYERS, IS_SAGEMAKER_MP_POST_1_10 # explicit for transformers 5.x from transformers.trainer_callback import TrainerCallback -from torch.utils.data import DataLoader -from torch.utils.data.distributed import DistributedSampler import numpy as np from cl_collator import SUPPORTED_DECODER_MODELS, check_model from cl_dataset import ANSWER_PREFIX - -# CuPy import with fallback for environments without CUDA CuPy -try: - import cupy as cp - from torch.utils.dlpack import to_dlpack, from_dlpack - from cupy import fromDlpack - HAS_CUPY = True -except ImportError: - HAS_CUPY = False - print("[WARNING] CuPy not available. SVD operations will use PyTorch fallback.") - -try: - import ipdb -except ImportError: - pass - -# OT-SIGN imports -from ot_sign_utils import ( - compute_vmf_params, - compute_anti_drift_loss_cached, - compute_anti_invasion_loss, - CachedRefTransInput, -) - -def robust_svd(matrix, eps=1e-8): - """ - Robust SVD with regularization and QR fallback for ill-conditioned matrices. - - Args: - matrix: input matrix for SVD - eps: regularization parameter (small identity to add for numerical stability) - - Returns: - U, S, V tensors from SVD, or QR decomposition fallback if SVD fails - """ - try: - # Add small regularization to improve conditioning - regularized = matrix + eps * torch.eye(matrix.shape[-1], device=matrix.device, dtype=matrix.dtype) - U, S, V = torch.linalg.svd(regularized, full_matrices=False) - return U, S, V - except RuntimeError as e: - # If SVD fails, use QR decomposition as fallback - if "failed to converge" in str(e) or "ill-conditioned" in str(e): - print(f"[WARNING] SVD convergence failed, using QR decomposition fallback") - try: - Q, R = torch.linalg.qr(matrix) - # Create S and V from QR: approximation U≈Q, S≈diag(|diag(R)|), V≈I - S = torch.abs(torch.diagonal(R)) - V = torch.eye(matrix.shape[-1], device=matrix.device, dtype=matrix.dtype) - return Q, S, V - except Exception as qr_err: - print(f"[WARNING] QR also failed, using identity approximation: {qr_err}") - # Last resort: return identity-like decomposition - n = matrix.shape[-1] - U = torch.eye(n, device=matrix.device, dtype=matrix.dtype) - S = torch.ones(n, device=matrix.device, dtype=matrix.dtype) - V = torch.eye(n, device=matrix.device, dtype=matrix.dtype) - return U, S, V - else: - raise +import cupy as cp +from torch.utils.dlpack import to_dlpack, from_dlpack +from cupy import fromDlpack +import ipdb def skip_instructions(model, predictions_ids, tokenizer, ignore_idx=-100): predictions_ids = np.where(predictions_ids == ignore_idx, tokenizer.pad_token_id, predictions_ids) @@ -115,7 +52,7 @@ class DenserEvalCallback(TrainerCallback): control.should_log = True # Evaluate - if args.eval_strategy == IntervalStrategy.STEPS and state.global_step in log_eval_steps: + if args.evaluation_strategy == IntervalStrategy.STEPS and state.global_step in log_eval_steps: control.should_evaluate = True # Save @@ -126,10 +63,8 @@ class DenserEvalCallback(TrainerCallback): class GainLoRA_InfLoRA_Trainer(Seq2SeqTrainer): - def __init__(self, model, args, train_dataset, cur_task_id, task_order, data_collator_replay=None, replay_dataset_dict=None, replay_label_dict=None, eval_dataset=None, processing_class=None, data_collator=None, compute_metrics=None, callbacks=None): - super().__init__(model=model, args=args, train_dataset=train_dataset, eval_dataset=eval_dataset, processing_class=processing_class, data_collator=data_collator, compute_metrics=compute_metrics, callbacks=callbacks) - patch_trainer_compat(self) - patch_args_compat(self.args) + def __init__(self, model, args, train_dataset, cur_task_id, task_order, data_collator_replay=None, replay_dataset_dict=None, replay_label_dict=None, eval_dataset=None, tokenizer=None, data_collator=None, compute_metrics=None, callbacks=None): + super().__init__(model=model, args=args, train_dataset=train_dataset, eval_dataset=eval_dataset, tokenizer=tokenizer, data_collator=data_collator, compute_metrics=compute_metrics, callbacks=callbacks) self.data_collator_replay = data_collator_replay self.replay_dataset_dict = replay_dataset_dict @@ -137,20 +72,6 @@ class GainLoRA_InfLoRA_Trainer(Seq2SeqTrainer): self.task_order = task_order self.cur_task_id = cur_task_id - # ===== OT-SIGN: Initialize anti-drift reference ===== - self.use_ot_sign = getattr(args, 'use_ot_routing', False) - self.ref_trans_input_cached = None - if self.use_ot_sign and cur_task_id > 0: - # Create frozen snapshot of trans_input from loaded checkpoint - try: - self.ref_trans_input_cached = CachedRefTransInput(model.encoder.trans_input) - self.ref_trans_input_cached = self.ref_trans_input_cached.to(next(model.parameters()).device) - print("[OT-SIGN] Created reference trans_input snapshot for anti-drift") - except Exception as e: - print(f"[OT-SIGN] Warning: Could not create ref_trans_input: {e}") - self.ref_trans_input_cached = None - # ===== End OT-SIGN init ===== - if self.args.data_replay_freq != -1: seed = self.args.data_seed if self.args.data_seed is not None else self.args.seed self.replay_dataloader_dict = {} @@ -251,7 +172,12 @@ class GainLoRA_InfLoRA_Trainer(Seq2SeqTrainer): pre_norm = module.prompt_key.detach().norm() for index in module.matrix_trans_3.keys(): cur_trans_matrix = module.matrix_trans_3[index] - U, S, V = robust_svd(cur_trans_matrix) + try: + U, S, V = torch.linalg.svd(cur_trans_matrix) + except RuntimeError: + # SVD failed due to ill-conditioning; use pseudoinverse fallback + p_inv = torch.pinv(cur_trans_matrix) + U, S, V = torch.linalg.svd(p_inv + 1e-4 * torch.eye(p_inv.shape[0], device=p_inv.device, dtype=p_inv.dtype)) module.prompt_key.data[:,index*module.step:(index+1)*module.step].copy_(U[:,:1].T) # ipdb.set_trace() module.matrix_trans_1[index].zero_() @@ -297,7 +223,7 @@ class GainLoRA_InfLoRA_Trainer(Seq2SeqTrainer): cur_trans_matrix = module.matrix_trans_3[index] cur_trans_matrix = torch.randn_like(cur_trans_matrix) cur_trans_matrix = cur_trans_matrix - torch.mm(self.feature_trans_mat[2][index],cur_trans_matrix) - U, S, V = robust_svd(cur_trans_matrix) + U, S, V = torch.linalg.svd(cur_trans_matrix) module.prompt_key.data[:,index*module.step:(index+1)*module.step].copy_(U[:,:1].T) module.matrix_trans_1[index].zero_() module.matrix_trans_3[index].zero_() @@ -554,7 +480,7 @@ class GainLoRA_InfLoRA_Trainer(Seq2SeqTrainer): torch.save(self.feature_trans_list[i], os.path.join(self.args.output_dir, 'trans_input', 'reg_{}.pt'.format(i))) - def training_step(self, model: nn.Module, inputs: Dict[str, Union[torch.Tensor, Any]], num_items_in_batch=None) -> torch.Tensor: + def training_step(self, model: nn.Module, inputs: Dict[str, Union[torch.Tensor, Any]]) -> torch.Tensor: """ Perform a training step on a batch of inputs. @@ -586,26 +512,6 @@ class GainLoRA_InfLoRA_Trainer(Seq2SeqTrainer): if self.args.n_gpu > 1: loss = loss.mean() # mean() to average on multi-gpu parallel training - # ===== OT-SIGN: Anti-invasion loss (add to CE loss before backward) ===== - if self.use_ot_sign and hasattr(self.model, 'encoder') and len(self.model.encoder.vmf_signatures) > 0: - x_norm = self.model.encoder.last_x_normalized # set during forward - if x_norm is not None: - # Need fresh x_norm with gradient for anti-invasion - avg_emb = (inputs.get('attention_mask', None).unsqueeze(-1) * self.model.encoder.embed_tokens(inputs['input_ids'])).mean(dim=1, keepdim=True) - medium = self.model.encoder.trans_input[1](self.model.encoder.trans_input[0](avg_emb)) - x_fresh = self.model.encoder.trans_input[3](self.model.encoder.trans_input[2](medium)) - x_fresh_norm = F_loss.normalize(x_fresh.squeeze(1), dim=-1) - - inv_loss = compute_anti_invasion_loss( - x_fresh_norm, - self.model.encoder.vmf_signatures, - self.cur_task_id, - threshold=getattr(self.args, 'invasion_threshold', 2.3) - ) - lambda_inv = getattr(self.args, 'lambda_inv', 0.001) - loss = loss + lambda_inv * inv_loss - # ===== End anti-invasion ===== - if self.args.gradient_accumulation_steps > 1 and not self.is_deepspeed_enabled: # deepspeed handles loss scaling by gradient_accumulation_steps in its `backward` loss = loss / self.args.gradient_accumulation_steps @@ -615,10 +521,11 @@ class GainLoRA_InfLoRA_Trainer(Seq2SeqTrainer): elif self.use_apex: with amp.scale_loss(loss, self.optimizer) as scaled_loss: scaled_loss.backward() - else: - # Use accelerator.backward() for all cases (DDP, fp16, deepspeed, etc.) - # This ensures the accelerate GradScaler is properly managed + elif self.is_deepspeed_enabled: + # loss gets scaled under gradient_accumulation_steps in deepspeed self.accelerator.backward(loss) + else: + loss.backward() if self.state.global_step > self.args.replay_after_n_epoch*self.args.step_per_epoch and self.args.data_replay_freq != -1 and self.state.global_step % self.args.data_replay_freq == 0: for item in self.replay_iterator_dict.keys(): @@ -652,75 +559,8 @@ class GainLoRA_InfLoRA_Trainer(Seq2SeqTrainer): else: kl_loss.backward() - # ===== OT-SIGN: Anti-drift loss (on replay data) ===== - if self.use_ot_sign and self.ref_trans_input_cached is not None: - drift_loss = compute_anti_drift_loss_cached( - self.model.encoder, - self.ref_trans_input_cached, - replay_inputs["input_ids"], - replay_inputs.get("attention_mask", torch.ones_like(replay_inputs["input_ids"])) - ) - lambda_drift = getattr(self.args, 'lambda_drift', 0.01) - scaled_drift = lambda_drift * drift_loss - if self.do_grad_scaling: - self.scaler.scale(scaled_drift).backward() - else: - scaled_drift.backward() - # ===== End anti-drift ===== - return loss.detach() - - # ===== OT-SIGN: vMF signature computation ===== - def compute_vmf_signature(self, task_id): - """ - Compute vMF signature for the current task after training completes. - Runs a single pass over training data to collect trans_input features, - then fits vMF parameters (mu, kappa). - - Args: - task_id: Current task index - - Returns: - (mu, kappa) tuple - """ - print(f"[OT-SIGN] Computing vMF signature for task {task_id}...") - self.model.eval() - all_x = [] - - train_dataloader = self.get_train_dataloader() - max_batches = 200 # limit to avoid excessive computation - - with torch.no_grad(): - for i, batch in enumerate(train_dataloader): - if i >= max_batches: - break - batch = self._prepare_inputs(batch) - input_ids = batch['input_ids'] - attention_mask = batch.get('attention_mask', torch.ones_like(input_ids)) - - embeddings = self.model.encoder.embed_tokens(input_ids) - avg_emb = (attention_mask.unsqueeze(-1) * embeddings).mean(dim=1, keepdim=True) - - medium = self.model.encoder.trans_input[1](self.model.encoder.trans_input[0](avg_emb)) - x = self.model.encoder.trans_input[3](self.model.encoder.trans_input[2](medium)) - x = F_loss.normalize(x.squeeze(1), dim=-1) # (B, d) - all_x.append(x.cpu()) - - all_x = torch.cat(all_x, dim=0) - d_model = all_x.shape[1] - - mu, kappa = compute_vmf_params(all_x, d_model) - - # Store in model - self.model.encoder.vmf_signatures[task_id] = (mu, kappa) - - print(f"[OT-SIGN] Task {task_id} signature: kappa={kappa.item():.4f}, " - f"mu_norm={mu.norm().item():.4f}, num_samples={all_x.shape[0]}") - - self.model.train() - return mu, kappa - # ===== End vMF signature computation ===== - + def create_optimizer_and_scheduler(self, num_training_steps: int): """ Setup the optimizer and the learning rate scheduler. @@ -1069,7 +909,6 @@ class GainLoRA_InfLoRA_Trainer(Seq2SeqTrainer): if "attention_mask" in inputs: gen_kwargs["attention_mask"] = inputs.get("attention_mask", None) - synced_gpus = gen_kwargs.pop("synced_gpus", False) generation_config = GenerationConfig(**gen_kwargs) # prepare generation inputs @@ -1081,7 +920,6 @@ class GainLoRA_InfLoRA_Trainer(Seq2SeqTrainer): generated_tokens = self.model.generate( input_ids=generation_inputs, generation_config=generation_config, - synced_gpus=synced_gpus, ) else: generation_inputs = inputs[self.model.main_input_name] @@ -1091,14 +929,12 @@ class GainLoRA_InfLoRA_Trainer(Seq2SeqTrainer): input_ids=generation_inputs, input_ids_wo_label=inputs["input_ids_wo_label"], generation_config=generation_config, - synced_gpus=synced_gpus, ) else: generated_tokens = self.model.generate( input_ids=generation_inputs, generation_config=generation_config, - synced_gpus=synced_gpus, ) bs, source_len = inputs['input_ids'].shape @@ -1220,7 +1056,7 @@ class GainLoRA_InfLoRA_Trainer(Seq2SeqTrainer): # Activate gradient checkpointing if needed if args.gradient_checkpointing: - self.model.gradient_checkpointing_enable(gradient_checkpointing_kwargs={"use_reentrant": False}) + self.model.gradient_checkpointing_enable() model = self._wrap_model(self.model_wrapped) @@ -1346,7 +1182,6 @@ class GainLoRA_InfLoRA_Trainer(Seq2SeqTrainer): self._total_loss_scalar = 0.0 self._globalstep_last_logged = self.state.global_step model.zero_grad() - grad_norm: float | None = None self.control = self.callback_handler.on_train_begin(args, self.state, self.control) @@ -1481,14 +1316,10 @@ class GainLoRA_InfLoRA_Trainer(Seq2SeqTrainer): args.max_grad_norm, ) else: - _grad_norm = nn.utils.clip_grad_norm_( + self.accelerator.clip_grad_norm_( model.parameters(), args.max_grad_norm, ) - if hasattr(_grad_norm, "item"): - grad_norm = _grad_norm.item() - else: - grad_norm = _grad_norm # Optimizer step optimizer_was_run = True @@ -1554,7 +1385,7 @@ class GainLoRA_InfLoRA_Trainer(Seq2SeqTrainer): self.state.epoch = epoch + (step + 1 + steps_skipped) / steps_in_epoch self.control = self.callback_handler.on_step_end(args, self.state, self.control) - self._maybe_log_save_evaluate(tr_loss, grad_norm, model, trial, epoch, ignore_keys_for_eval, start_time) + self._maybe_log_save_evaluate(tr_loss, model, trial, epoch, ignore_keys_for_eval) else: self.control = self.callback_handler.on_substep_end(args, self.state, self.control) @@ -1569,7 +1400,7 @@ class GainLoRA_InfLoRA_Trainer(Seq2SeqTrainer): self.control.should_training_stop = True self.control = self.callback_handler.on_epoch_end(args, self.state, self.control) - self._maybe_log_save_evaluate(tr_loss, grad_norm, model, trial, epoch, ignore_keys_for_eval, start_time) + self._maybe_log_save_evaluate(tr_loss, model, trial, epoch, ignore_keys_for_eval) if DebugOption.TPU_METRICS_DEBUG in self.args.debug: if is_torch_tpu_available(): diff --git a/gainlora_OT_vMF/src/cl_trainer_gainlora_inflora_llama.py b/root_gainlora/src/cl_trainer_gainlora_inflora_llama.py similarity index 97% rename from gainlora_OT_vMF/src/cl_trainer_gainlora_inflora_llama.py rename to root_gainlora/src/cl_trainer_gainlora_inflora_llama.py index 0ef09794434c4de0fa5f9b20f7f70d68cd7151fe..0105cea529c546d6f5405715faeabd89472c9621 100644 --- a/gainlora_OT_vMF/src/cl_trainer_gainlora_inflora_llama.py +++ b/root_gainlora/src/cl_trainer_gainlora_inflora_llama.py @@ -1,30 +1,16 @@ import torch -import torch.nn as nn -from typing import Dict, Union, Any, Optional, List, Tuple, Callable from transformers import GenerationConfig from transformers.trainer_seq2seq import Seq2SeqTrainer from transformers.trainer import * -from transformers.trainer_utils import IntervalStrategy -from compat_transformers import ShardedDDPOption, is_sagemaker_mp_enabled, smp_forward_backward, deepspeed_init, deepspeed_load_checkpoint, HPSearchBackend, patch_trainer_compat, patch_args_compat, is_torch_tpu_available, ALL_LAYERNORM_LAYERS, IS_SAGEMAKER_MP_POST_1_10 # explicit for transformers 5.x from transformers.trainer_callback import TrainerCallback -from torch.utils.data import DataLoader -from torch.utils.data.distributed import DistributedSampler import numpy as np from cl_collator import SUPPORTED_DECODER_MODELS, check_model from cl_dataset import ANSWER_PREFIX -try: - import cupy as cp - from torch.utils.dlpack import to_dlpack, from_dlpack - from cupy import fromDlpack - HAS_CUPY = True -except ImportError: - HAS_CUPY = False - print("[WARNING] CuPy not available. SVD operations will use PyTorch fallback.") -try: - import ipdb -except ImportError: - pass +import cupy as cp +from torch.utils.dlpack import to_dlpack, from_dlpack +from cupy import fromDlpack +import ipdb from copy import deepcopy def skip_instructions(model, predictions_ids, tokenizer, ignore_idx=-100): @@ -67,7 +53,7 @@ class DenserEvalCallback(TrainerCallback): control.should_log = True # Evaluate - if args.eval_strategy == IntervalStrategy.STEPS and state.global_step in log_eval_steps: + if args.evaluation_strategy == IntervalStrategy.STEPS and state.global_step in log_eval_steps: control.should_evaluate = True # Save @@ -78,10 +64,8 @@ class DenserEvalCallback(TrainerCallback): class GainLoRA_InfLoRA_Trainer(Seq2SeqTrainer): - def __init__(self, model, args, train_dataset, cur_task_id, task_order, data_collator_replay=None, replay_dataset_dict=None, replay_label_dict=None, eval_dataset=None, processing_class=None, data_collator=None, compute_metrics=None, callbacks=None): - super().__init__(model=model, args=args, train_dataset=train_dataset, eval_dataset=eval_dataset, processing_class=processing_class, data_collator=data_collator, compute_metrics=compute_metrics, callbacks=callbacks) - patch_trainer_compat(self) - patch_args_compat(self.args) + def __init__(self, model, args, train_dataset, cur_task_id, task_order, data_collator_replay=None, replay_dataset_dict=None, replay_label_dict=None, eval_dataset=None, tokenizer=None, data_collator=None, compute_metrics=None, callbacks=None): + super().__init__(model=model, args=args, train_dataset=train_dataset, eval_dataset=eval_dataset, tokenizer=tokenizer, data_collator=data_collator, compute_metrics=compute_metrics, callbacks=callbacks) self.data_collator_replay = data_collator_replay self.replay_dataset_dict = replay_dataset_dict @@ -527,7 +511,7 @@ class GainLoRA_InfLoRA_Trainer(Seq2SeqTrainer): torch.save(self.feature_trans_list[i], os.path.join(self.args.output_dir, 'trans_input', 'reg_{}.pt'.format(i))) - def training_step(self, model: nn.Module, inputs: Dict[str, Union[torch.Tensor, Any]], num_items_in_batch=None) -> torch.Tensor: + def training_step(self, model: nn.Module, inputs: Dict[str, Union[torch.Tensor, Any]]) -> torch.Tensor: """ Perform a training step on a batch of inputs. @@ -568,9 +552,11 @@ class GainLoRA_InfLoRA_Trainer(Seq2SeqTrainer): elif self.use_apex: with amp.scale_loss(loss, self.optimizer) as scaled_loss: scaled_loss.backward() - else: - # Use accelerator.backward() for all cases (DDP, fp16, deepspeed, etc.) + elif self.is_deepspeed_enabled: + # loss gets scaled under gradient_accumulation_steps in deepspeed self.accelerator.backward(loss) + else: + loss.backward() if self.state.global_step > self.args.replay_after_n_epoch*self.args.step_per_epoch and self.args.data_replay_freq != -1 and self.state.global_step % self.args.data_replay_freq == 0: for item in self.replay_iterator_dict.keys(): @@ -954,7 +940,6 @@ class GainLoRA_InfLoRA_Trainer(Seq2SeqTrainer): if "attention_mask" in inputs: gen_kwargs["attention_mask"] = inputs.get("attention_mask", None) - synced_gpus = gen_kwargs.pop("synced_gpus", False) generation_config = GenerationConfig(**gen_kwargs) # prepare generation inputs @@ -966,7 +951,6 @@ class GainLoRA_InfLoRA_Trainer(Seq2SeqTrainer): generated_tokens = self.model.generate( input_ids=generation_inputs, generation_config=generation_config, - synced_gpus=synced_gpus, ) else: generation_inputs = inputs[self.model.main_input_name] @@ -976,14 +960,12 @@ class GainLoRA_InfLoRA_Trainer(Seq2SeqTrainer): input_ids=generation_inputs, input_ids_wo_label=inputs["input_ids_wo_label"], generation_config=generation_config, - synced_gpus=synced_gpus, ) else: generated_tokens = self.model.generate( input_ids=generation_inputs, generation_config=generation_config, - synced_gpus=synced_gpus, ) bs, source_len = inputs['input_ids'].shape @@ -1103,7 +1085,7 @@ class GainLoRA_InfLoRA_Trainer(Seq2SeqTrainer): # Activate gradient checkpointing if needed if args.gradient_checkpointing: - self.model.gradient_checkpointing_enable(gradient_checkpointing_kwargs={"use_reentrant": False}) + self.model.gradient_checkpointing_enable() model = self._wrap_model(self.model_wrapped) @@ -1229,7 +1211,6 @@ class GainLoRA_InfLoRA_Trainer(Seq2SeqTrainer): self._total_loss_scalar = 0.0 self._globalstep_last_logged = self.state.global_step model.zero_grad() - grad_norm: float | None = None self.control = self.callback_handler.on_train_begin(args, self.state, self.control) @@ -1407,14 +1388,10 @@ class GainLoRA_InfLoRA_Trainer(Seq2SeqTrainer): args.max_grad_norm, ) else: - _grad_norm = nn.utils.clip_grad_norm_( + self.accelerator.clip_grad_norm_( model.parameters(), args.max_grad_norm, ) - if hasattr(_grad_norm, "item"): - grad_norm = _grad_norm.item() - else: - grad_norm = _grad_norm # Optimizer step optimizer_was_run = True @@ -1484,7 +1461,7 @@ class GainLoRA_InfLoRA_Trainer(Seq2SeqTrainer): self.state.epoch = epoch + (step + 1 + steps_skipped) / steps_in_epoch self.control = self.callback_handler.on_step_end(args, self.state, self.control) - self._maybe_log_save_evaluate(tr_loss, grad_norm, model, trial, epoch, ignore_keys_for_eval, start_time) + self._maybe_log_save_evaluate(tr_loss, model, trial, epoch, ignore_keys_for_eval) # print(self.model.model.trans_input[0].weight-old_trans_input_0) else: @@ -1504,7 +1481,7 @@ class GainLoRA_InfLoRA_Trainer(Seq2SeqTrainer): self.control.should_training_stop = True self.control = self.callback_handler.on_epoch_end(args, self.state, self.control) - self._maybe_log_save_evaluate(tr_loss, grad_norm, model, trial, epoch, ignore_keys_for_eval, start_time) + self._maybe_log_save_evaluate(tr_loss, model, trial, epoch, ignore_keys_for_eval) if DebugOption.TPU_METRICS_DEBUG in self.args.debug: if is_torch_tpu_available(): diff --git a/gainlora_OT_vMF/src/cl_trainer_gainlora_olora.py b/root_gainlora/src/cl_trainer_gainlora_olora.py similarity index 94% rename from gainlora_OT_vMF/src/cl_trainer_gainlora_olora.py rename to root_gainlora/src/cl_trainer_gainlora_olora.py index 61e797882d8f85e9f388a94e953ea0ab4b8b73f9..12bc6b41b3e5ec5da80cdd46d3298142a122e221 100644 --- a/gainlora_OT_vMF/src/cl_trainer_gainlora_olora.py +++ b/root_gainlora/src/cl_trainer_gainlora_olora.py @@ -1,71 +1,19 @@ import torch -import torch.nn as nn -from typing import Dict, Union, Any, Optional, List, Tuple, Callable from transformers import GenerationConfig from transformers.trainer_seq2seq import Seq2SeqTrainer from transformers.trainer import * -from transformers.trainer_utils import IntervalStrategy -from compat_transformers import ShardedDDPOption, is_sagemaker_mp_enabled, smp_forward_backward, deepspeed_init, deepspeed_load_checkpoint, HPSearchBackend, patch_trainer_compat, patch_args_compat, is_torch_tpu_available, ALL_LAYERNORM_LAYERS, IS_SAGEMAKER_MP_POST_1_10 # explicit for transformers 5.x from transformers.trainer_callback import TrainerCallback -from torch.utils.data import DataLoader -from torch.utils.data.distributed import DistributedSampler import numpy as np from cl_collator import SUPPORTED_DECODER_MODELS, check_model from cl_dataset import ANSWER_PREFIX -try: - import cupy as cp - from torch.utils.dlpack import to_dlpack, from_dlpack - from cupy import fromDlpack - HAS_CUPY = True -except ImportError: - HAS_CUPY = False - print("[WARNING] CuPy not available. SVD operations will use PyTorch fallback.") +import cupy as cp +from torch.utils.dlpack import to_dlpack, from_dlpack +from cupy import fromDlpack # from t5_olora import T5Attention from t5_gainlora_inflora import T5Attention -try: - import ipdb -except ImportError: - pass - - -def robust_svd(matrix, eps=1e-8): - """ - Robust SVD with regularization and QR fallback for ill-conditioned matrices. - - Args: - matrix: input matrix for SVD - eps: regularization parameter (small identity to add for numerical stability) - - Returns: - U, S, V tensors from SVD, or QR decomposition fallback if SVD fails - """ - try: - # Add small regularization to improve conditioning - regularized = matrix + eps * torch.eye(matrix.shape[-1], device=matrix.device, dtype=matrix.dtype) - U, S, V = torch.linalg.svd(regularized, full_matrices=False) - return U, S, V - except RuntimeError as e: - # If SVD fails, use QR decomposition as fallback - if "failed to converge" in str(e) or "ill-conditioned" in str(e): - print(f"[WARNING] SVD convergence failed, using QR decomposition fallback") - try: - Q, R = torch.linalg.qr(matrix) - # Create S and V from QR: approximation U≈Q, S≈diag(|diag(R)|), V≈I - S = torch.abs(torch.diagonal(R)) - V = torch.eye(matrix.shape[-1], device=matrix.device, dtype=matrix.dtype) - return Q, S, V - except Exception as qr_err: - print(f"[WARNING] QR also failed, using identity approximation: {qr_err}") - # Last resort: return identity-like decomposition - n = matrix.shape[-1] - U = torch.eye(n, device=matrix.device, dtype=matrix.dtype) - S = torch.ones(n, device=matrix.device, dtype=matrix.dtype) - V = torch.eye(n, device=matrix.device, dtype=matrix.dtype) - return U, S, V - else: - raise +import ipdb def skip_instructions(model, predictions_ids, tokenizer, ignore_idx=-100): @@ -108,7 +56,7 @@ class DenserEvalCallback(TrainerCallback): control.should_log = True # Evaluate - if args.eval_strategy == IntervalStrategy.STEPS and state.global_step in log_eval_steps: + if args.evaluation_strategy == IntervalStrategy.STEPS and state.global_step in log_eval_steps: control.should_evaluate = True # Save @@ -119,10 +67,8 @@ class DenserEvalCallback(TrainerCallback): class GainLoRA_OLoRA_Trainer(Seq2SeqTrainer): - def __init__(self, model, args, train_dataset, cur_task_id, task_order, data_collator_replay=None, replay_dataset_dict=None, replay_label_dict=None, eval_dataset=None, processing_class=None, data_collator=None, compute_metrics=None, callbacks=None): - super().__init__(model=model, args=args, train_dataset=train_dataset, eval_dataset=eval_dataset, processing_class=processing_class, data_collator=data_collator, compute_metrics=compute_metrics, callbacks=callbacks) - patch_trainer_compat(self) - patch_args_compat(self.args) + def __init__(self, model, args, train_dataset, cur_task_id, task_order, data_collator_replay=None, replay_dataset_dict=None, replay_label_dict=None, eval_dataset=None, tokenizer=None, data_collator=None, compute_metrics=None, callbacks=None): + super().__init__(model=model, args=args, train_dataset=train_dataset, eval_dataset=eval_dataset, tokenizer=tokenizer, data_collator=data_collator, compute_metrics=compute_metrics, callbacks=callbacks) self.data_collator_replay = data_collator_replay self.replay_dataset_dict = replay_dataset_dict @@ -233,7 +179,7 @@ class GainLoRA_OLoRA_Trainer(Seq2SeqTrainer): pre_norm = module.prompt_key.detach().norm() for index in module.matrix_trans_3.keys(): cur_trans_matrix = module.matrix_trans_3[index] - U, S, V = robust_svd(cur_trans_matrix) + U, S, V = torch.linalg.svd(cur_trans_matrix) module.prompt_key.data[:,index*module.step:(index+1)*module.step].copy_(U[:,:1].T) # ipdb.set_trace() module.matrix_trans_1[index].zero_() @@ -294,8 +240,9 @@ class GainLoRA_OLoRA_Trainer(Seq2SeqTrainer): try: cur_trans_matrix = cur_trans_matrix - torch.mm(self.feature_trans_mat[2][index],cur_trans_matrix) except: - pass # Continue with original matrix if subtraction fails - U, S, V = robust_svd(cur_trans_matrix) + ipdb.set_trace() + raise Exception + U, S, V = torch.linalg.svd(cur_trans_matrix) module.prompt_key.data[:,index*module.step:(index+1)*module.step].copy_(U[:,:1].T) module.matrix_trans_1[index].zero_() module.matrix_trans_3[index].zero_() @@ -546,7 +493,7 @@ class GainLoRA_OLoRA_Trainer(Seq2SeqTrainer): torch.save(self.feature_trans_list[i], os.path.join(self.args.output_dir, 'trans_input', 'reg_{}.pt'.format(i))) - def training_step(self, model: nn.Module, inputs: Dict[str, Union[torch.Tensor, Any]], num_items_in_batch=None) -> torch.Tensor: + def training_step(self, model: nn.Module, inputs: Dict[str, Union[torch.Tensor, Any]]) -> torch.Tensor: """ Perform a training step on a batch of inputs. @@ -612,9 +559,11 @@ class GainLoRA_OLoRA_Trainer(Seq2SeqTrainer): elif self.use_apex: with amp.scale_loss(loss, self.optimizer) as scaled_loss: scaled_loss.backward() - else: - # Use accelerator.backward() for all cases (DDP, fp16, deepspeed, etc.) + elif self.is_deepspeed_enabled: + # loss gets scaled under gradient_accumulation_steps in deepspeed self.accelerator.backward(loss) + else: + loss.backward() if self.state.global_step > self.args.replay_after_n_epoch*self.args.step_per_epoch and self.args.data_replay_freq != -1 and self.state.global_step % self.args.data_replay_freq == 0: for item in self.replay_iterator_dict.keys(): @@ -999,7 +948,6 @@ class GainLoRA_OLoRA_Trainer(Seq2SeqTrainer): if "attention_mask" in inputs: gen_kwargs["attention_mask"] = inputs.get("attention_mask", None) - synced_gpus = gen_kwargs.pop("synced_gpus", False) generation_config = GenerationConfig(**gen_kwargs) # prepare generation inputs @@ -1011,7 +959,6 @@ class GainLoRA_OLoRA_Trainer(Seq2SeqTrainer): generated_tokens = self.model.generate( input_ids=generation_inputs, generation_config=generation_config, - synced_gpus=synced_gpus, ) else: generation_inputs = inputs[self.model.main_input_name] @@ -1021,14 +968,12 @@ class GainLoRA_OLoRA_Trainer(Seq2SeqTrainer): input_ids=generation_inputs, input_ids_wo_label=inputs["input_ids_wo_label"], generation_config=generation_config, - synced_gpus=synced_gpus, ) else: generated_tokens = self.model.generate( input_ids=generation_inputs, generation_config=generation_config, - synced_gpus=synced_gpus, ) bs, source_len = inputs['input_ids'].shape @@ -1148,7 +1093,7 @@ class GainLoRA_OLoRA_Trainer(Seq2SeqTrainer): # Activate gradient checkpointing if needed if args.gradient_checkpointing: - self.model.gradient_checkpointing_enable(gradient_checkpointing_kwargs={"use_reentrant": False}) + self.model.gradient_checkpointing_enable() model = self._wrap_model(self.model_wrapped) @@ -1274,7 +1219,6 @@ class GainLoRA_OLoRA_Trainer(Seq2SeqTrainer): self._total_loss_scalar = 0.0 self._globalstep_last_logged = self.state.global_step model.zero_grad() - grad_norm: float | None = None self.control = self.callback_handler.on_train_begin(args, self.state, self.control) @@ -1409,14 +1353,10 @@ class GainLoRA_OLoRA_Trainer(Seq2SeqTrainer): args.max_grad_norm, ) else: - _grad_norm = nn.utils.clip_grad_norm_( + self.accelerator.clip_grad_norm_( model.parameters(), args.max_grad_norm, ) - if hasattr(_grad_norm, "item"): - grad_norm = _grad_norm.item() - else: - grad_norm = _grad_norm # Optimizer step optimizer_was_run = True @@ -1483,7 +1423,7 @@ class GainLoRA_OLoRA_Trainer(Seq2SeqTrainer): self.state.epoch = epoch + (step + 1 + steps_skipped) / steps_in_epoch self.control = self.callback_handler.on_step_end(args, self.state, self.control) - self._maybe_log_save_evaluate(tr_loss, grad_norm, model, trial, epoch, ignore_keys_for_eval, start_time) + self._maybe_log_save_evaluate(tr_loss, model, trial, epoch, ignore_keys_for_eval) else: self.control = self.callback_handler.on_substep_end(args, self.state, self.control) @@ -1498,7 +1438,7 @@ class GainLoRA_OLoRA_Trainer(Seq2SeqTrainer): self.control.should_training_stop = True self.control = self.callback_handler.on_epoch_end(args, self.state, self.control) - self._maybe_log_save_evaluate(tr_loss, grad_norm, model, trial, epoch, ignore_keys_for_eval, start_time) + self._maybe_log_save_evaluate(tr_loss, model, trial, epoch, ignore_keys_for_eval) if DebugOption.TPU_METRICS_DEBUG in self.args.debug: if is_torch_tpu_available(): diff --git a/gainlora_OT_vMF/src/cl_trainer_gainlora_olora_llama.py b/root_gainlora/src/cl_trainer_gainlora_olora_llama.py similarity index 97% rename from gainlora_OT_vMF/src/cl_trainer_gainlora_olora_llama.py rename to root_gainlora/src/cl_trainer_gainlora_olora_llama.py index 9ef7333d3213eabd4ff08ab82fb9d5a462e48317..7837e7a08c3cdfd3d1a2336616d36318f854898a 100644 --- a/gainlora_OT_vMF/src/cl_trainer_gainlora_olora_llama.py +++ b/root_gainlora/src/cl_trainer_gainlora_olora_llama.py @@ -1,35 +1,21 @@ import torch -import torch.nn as nn -from typing import Dict, Union, Any, Optional, List, Tuple, Callable from transformers import GenerationConfig from transformers.trainer_seq2seq import Seq2SeqTrainer from transformers.trainer import * -from transformers.trainer_utils import IntervalStrategy -from compat_transformers import ShardedDDPOption, is_sagemaker_mp_enabled, smp_forward_backward, deepspeed_init, deepspeed_load_checkpoint, HPSearchBackend, patch_trainer_compat, patch_args_compat, is_torch_tpu_available, ALL_LAYERNORM_LAYERS, IS_SAGEMAKER_MP_POST_1_10 # explicit for transformers 5.x from transformers.trainer_callback import TrainerCallback -from torch.utils.data import DataLoader -from torch.utils.data.distributed import DistributedSampler import numpy as np from cl_collator import SUPPORTED_DECODER_MODELS, check_model from cl_dataset import ANSWER_PREFIX -try: - import cupy as cp - from torch.utils.dlpack import to_dlpack, from_dlpack - from cupy import fromDlpack - HAS_CUPY = True -except ImportError: - HAS_CUPY = False - print("[WARNING] CuPy not available. SVD operations will use PyTorch fallback.") +import cupy as cp +from torch.utils.dlpack import to_dlpack, from_dlpack +from cupy import fromDlpack # from t5_olora import T5Attention from llama_gainlora_inflora import LlamaAttention # from llama13b_inflorap1 import LlamaFlashAttention2 # from llama13b_inflorap1_1 import LlamaFlashAttention2 as LlamaFlashAttention2_1 -try: - import ipdb -except ImportError: - pass +import ipdb from copy import deepcopy @@ -73,7 +59,7 @@ class DenserEvalCallback(TrainerCallback): control.should_log = True # Evaluate - if args.eval_strategy == IntervalStrategy.STEPS and state.global_step in log_eval_steps: + if args.evaluation_strategy == IntervalStrategy.STEPS and state.global_step in log_eval_steps: control.should_evaluate = True # Save @@ -84,10 +70,8 @@ class DenserEvalCallback(TrainerCallback): class GainLoRA_OLoRA_Trainer(Seq2SeqTrainer): - def __init__(self, model, args, train_dataset, cur_task_id, task_order, data_collator_replay=None, replay_dataset_dict=None, replay_label_dict=None, eval_dataset=None, processing_class=None, data_collator=None, compute_metrics=None, callbacks=None): - super().__init__(model=model, args=args, train_dataset=train_dataset, eval_dataset=eval_dataset, processing_class=processing_class, data_collator=data_collator, compute_metrics=compute_metrics, callbacks=callbacks) - patch_trainer_compat(self) - patch_args_compat(self.args) + def __init__(self, model, args, train_dataset, cur_task_id, task_order, data_collator_replay=None, replay_dataset_dict=None, replay_label_dict=None, eval_dataset=None, tokenizer=None, data_collator=None, compute_metrics=None, callbacks=None): + super().__init__(model=model, args=args, train_dataset=train_dataset, eval_dataset=eval_dataset, tokenizer=tokenizer, data_collator=data_collator, compute_metrics=compute_metrics, callbacks=callbacks) self.data_collator_replay = data_collator_replay self.replay_dataset_dict = replay_dataset_dict @@ -492,7 +476,7 @@ class GainLoRA_OLoRA_Trainer(Seq2SeqTrainer): torch.save(self.feature_trans_list[i], os.path.join(self.args.output_dir, 'trans_input', 'reg_{}.pt'.format(i))) - def training_step(self, model: nn.Module, inputs: Dict[str, Union[torch.Tensor, Any]], num_items_in_batch=None) -> torch.Tensor: + def training_step(self, model: nn.Module, inputs: Dict[str, Union[torch.Tensor, Any]]) -> torch.Tensor: """ Perform a training step on a batch of inputs. @@ -558,9 +542,11 @@ class GainLoRA_OLoRA_Trainer(Seq2SeqTrainer): elif self.use_apex: with amp.scale_loss(loss, self.optimizer) as scaled_loss: scaled_loss.backward() - else: - # Use accelerator.backward() for all cases (DDP, fp16, deepspeed, etc.) + elif self.is_deepspeed_enabled: + # loss gets scaled under gradient_accumulation_steps in deepspeed self.accelerator.backward(loss) + else: + loss.backward() if self.state.global_step > self.args.replay_after_n_epoch*self.args.step_per_epoch and self.args.data_replay_freq != -1 and self.state.global_step % self.args.data_replay_freq == 0: for item in self.replay_iterator_dict.keys(): @@ -945,7 +931,6 @@ class GainLoRA_OLoRA_Trainer(Seq2SeqTrainer): if "attention_mask" in inputs: gen_kwargs["attention_mask"] = inputs.get("attention_mask", None) - synced_gpus = gen_kwargs.pop("synced_gpus", False) generation_config = GenerationConfig(**gen_kwargs) # prepare generation inputs @@ -957,7 +942,6 @@ class GainLoRA_OLoRA_Trainer(Seq2SeqTrainer): generated_tokens = self.model.generate( input_ids=generation_inputs, generation_config=generation_config, - synced_gpus=synced_gpus, ) else: generation_inputs = inputs[self.model.main_input_name] @@ -967,14 +951,12 @@ class GainLoRA_OLoRA_Trainer(Seq2SeqTrainer): input_ids=generation_inputs, input_ids_wo_label=inputs["input_ids_wo_label"], generation_config=generation_config, - synced_gpus=synced_gpus, ) else: generated_tokens = self.model.generate( input_ids=generation_inputs, generation_config=generation_config, - synced_gpus=synced_gpus, ) bs, source_len = inputs['input_ids'].shape @@ -1094,7 +1076,7 @@ class GainLoRA_OLoRA_Trainer(Seq2SeqTrainer): # Activate gradient checkpointing if needed if args.gradient_checkpointing: - self.model.gradient_checkpointing_enable(gradient_checkpointing_kwargs={"use_reentrant": False}) + self.model.gradient_checkpointing_enable() model = self._wrap_model(self.model_wrapped) @@ -1220,7 +1202,6 @@ class GainLoRA_OLoRA_Trainer(Seq2SeqTrainer): self._total_loss_scalar = 0.0 self._globalstep_last_logged = self.state.global_step model.zero_grad() - grad_norm: float | None = None self.control = self.callback_handler.on_train_begin(args, self.state, self.control) @@ -1398,14 +1379,10 @@ class GainLoRA_OLoRA_Trainer(Seq2SeqTrainer): args.max_grad_norm, ) else: - _grad_norm = nn.utils.clip_grad_norm_( + self.accelerator.clip_grad_norm_( model.parameters(), args.max_grad_norm, ) - if hasattr(_grad_norm, "item"): - grad_norm = _grad_norm.item() - else: - grad_norm = _grad_norm # Optimizer step optimizer_was_run = True @@ -1475,7 +1452,7 @@ class GainLoRA_OLoRA_Trainer(Seq2SeqTrainer): self.state.epoch = epoch + (step + 1 + steps_skipped) / steps_in_epoch self.control = self.callback_handler.on_step_end(args, self.state, self.control) - self._maybe_log_save_evaluate(tr_loss, grad_norm, model, trial, epoch, ignore_keys_for_eval, start_time) + self._maybe_log_save_evaluate(tr_loss, model, trial, epoch, ignore_keys_for_eval) # print(self.model.model.trans_input[0].weight-old_trans_input_0) else: @@ -1495,7 +1472,7 @@ class GainLoRA_OLoRA_Trainer(Seq2SeqTrainer): self.control.should_training_stop = True self.control = self.callback_handler.on_epoch_end(args, self.state, self.control) - self._maybe_log_save_evaluate(tr_loss, grad_norm, model, trial, epoch, ignore_keys_for_eval, start_time) + self._maybe_log_save_evaluate(tr_loss, model, trial, epoch, ignore_keys_for_eval) if DebugOption.TPU_METRICS_DEBUG in self.args.debug: if is_torch_tpu_available(): diff --git a/gainlora_OT_vMF/src/cl_trainer_inflora.py b/root_gainlora/src/cl_trainer_inflora.py similarity index 95% rename from gainlora_OT_vMF/src/cl_trainer_inflora.py rename to root_gainlora/src/cl_trainer_inflora.py index 2340d8db9fa9f14a9f1f7eaafbb4855616ee419e..53a7cef502a96be43387501b0560dce11b13c9da 100644 --- a/gainlora_OT_vMF/src/cl_trainer_inflora.py +++ b/root_gainlora/src/cl_trainer_inflora.py @@ -1,31 +1,16 @@ import torch -import torch.nn as nn -import torch.nn.functional as F_loss -from typing import Dict, Union, Any, Optional, List, Tuple, Callable from transformers import GenerationConfig from transformers.trainer_seq2seq import Seq2SeqTrainer from transformers.trainer import * -from transformers.trainer_utils import IntervalStrategy -from compat_transformers import ShardedDDPOption, is_sagemaker_mp_enabled, smp_forward_backward, deepspeed_init, deepspeed_load_checkpoint, HPSearchBackend, patch_trainer_compat, patch_args_compat, is_torch_tpu_available, ALL_LAYERNORM_LAYERS, IS_SAGEMAKER_MP_POST_1_10 # explicit for transformers 5.x from transformers.trainer_callback import TrainerCallback -from torch.utils.data import DataLoader -from torch.utils.data.distributed import DistributedSampler import numpy as np from cl_collator import SUPPORTED_DECODER_MODELS, check_model from cl_dataset import ANSWER_PREFIX -try: - import cupy as cp - from torch.utils.dlpack import to_dlpack, from_dlpack - from cupy import fromDlpack - HAS_CUPY = True -except ImportError: - HAS_CUPY = False - print("[WARNING] CuPy not available. SVD operations will use PyTorch fallback.") -try: - import ipdb -except ImportError: - pass +import cupy as cp +from torch.utils.dlpack import to_dlpack, from_dlpack +from cupy import fromDlpack +import ipdb def skip_instructions(model, predictions_ids, tokenizer, ignore_idx=-100): predictions_ids = np.where(predictions_ids == ignore_idx, tokenizer.pad_token_id, predictions_ids) @@ -67,7 +52,7 @@ class DenserEvalCallback(TrainerCallback): control.should_log = True # Evaluate - if args.eval_strategy == IntervalStrategy.STEPS and state.global_step in log_eval_steps: + if args.evaluation_strategy == IntervalStrategy.STEPS and state.global_step in log_eval_steps: control.should_evaluate = True # Save @@ -78,10 +63,8 @@ class DenserEvalCallback(TrainerCallback): class InfLoRATrainer(Seq2SeqTrainer): - def __init__(self, model, args, train_dataset, cur_task_id, task_order, data_collator_replay=None, replay_dataset_dict=None, replay_label_dict=None, eval_dataset=None, processing_class=None, data_collator=None, compute_metrics=None, callbacks=None): - super().__init__(model=model, args=args, train_dataset=train_dataset, eval_dataset=eval_dataset, processing_class=processing_class, data_collator=data_collator, compute_metrics=compute_metrics, callbacks=callbacks) - patch_trainer_compat(self) - patch_args_compat(self.args) + def __init__(self, model, args, train_dataset, cur_task_id, task_order, data_collator_replay=None, replay_dataset_dict=None, replay_label_dict=None, eval_dataset=None, tokenizer=None, data_collator=None, compute_metrics=None, callbacks=None): + super().__init__(model=model, args=args, train_dataset=train_dataset, eval_dataset=eval_dataset, tokenizer=tokenizer, data_collator=data_collator, compute_metrics=compute_metrics, callbacks=callbacks) self.data_collator_replay = data_collator_replay self.replay_dataset_dict = replay_dataset_dict @@ -292,7 +275,7 @@ class InfLoRATrainer(Seq2SeqTrainer): torch.save(self.feature_list[i], os.path.join(self.args.output_dir, 'reg_{}.pt'.format(i))) - def training_step(self, model: nn.Module, inputs: Dict[str, Union[torch.Tensor, Any]], num_items_in_batch=None) -> torch.Tensor: + def training_step(self, model: nn.Module, inputs: Dict[str, Union[torch.Tensor, Any]]) -> torch.Tensor: """ Perform a training step on a batch of inputs. @@ -333,9 +316,11 @@ class InfLoRATrainer(Seq2SeqTrainer): elif self.use_apex: with amp.scale_loss(loss, self.optimizer) as scaled_loss: scaled_loss.backward() - else: - # Use accelerator.backward() for all cases (DDP, fp16, deepspeed, etc.) + elif self.is_deepspeed_enabled: + # loss gets scaled under gradient_accumulation_steps in deepspeed self.accelerator.backward(loss) + else: + loss.backward() if self.state.global_step > self.args.replay_after_n_epoch*self.args.step_per_epoch and self.args.data_replay_freq != -1 and self.state.global_step % self.args.data_replay_freq == 0: for item in self.replay_iterator_dict.keys(): @@ -719,7 +704,6 @@ class InfLoRATrainer(Seq2SeqTrainer): if "attention_mask" in inputs: gen_kwargs["attention_mask"] = inputs.get("attention_mask", None) - synced_gpus = gen_kwargs.pop("synced_gpus", False) generation_config = GenerationConfig(**gen_kwargs) # prepare generation inputs @@ -731,7 +715,6 @@ class InfLoRATrainer(Seq2SeqTrainer): generated_tokens = self.model.generate( input_ids=generation_inputs, generation_config=generation_config, - synced_gpus=synced_gpus, ) else: generation_inputs = inputs[self.model.main_input_name] @@ -741,14 +724,12 @@ class InfLoRATrainer(Seq2SeqTrainer): input_ids=generation_inputs, input_ids_wo_label=inputs["input_ids_wo_label"], generation_config=generation_config, - synced_gpus=synced_gpus, ) else: generated_tokens = self.model.generate( input_ids=generation_inputs, generation_config=generation_config, - synced_gpus=synced_gpus, ) bs, source_len = inputs['input_ids'].shape diff --git a/gainlora_OT_vMF/src/cl_trainer_inflora_llama.py b/root_gainlora/src/cl_trainer_inflora_llama.py similarity index 95% rename from gainlora_OT_vMF/src/cl_trainer_inflora_llama.py rename to root_gainlora/src/cl_trainer_inflora_llama.py index 638415b564818abbed6796ac920d3043b2368f77..abbc004a7f5fce373af9211a7833f344c10e5fbb 100644 --- a/gainlora_OT_vMF/src/cl_trainer_inflora_llama.py +++ b/root_gainlora/src/cl_trainer_inflora_llama.py @@ -1,26 +1,15 @@ import torch -import torch.nn as nn -from typing import Dict, Union, Any, Optional, List, Tuple, Callable from transformers import GenerationConfig from transformers.trainer_seq2seq import Seq2SeqTrainer from transformers.trainer import * -from transformers.trainer_utils import IntervalStrategy -from compat_transformers import ShardedDDPOption, is_sagemaker_mp_enabled, smp_forward_backward, deepspeed_init, deepspeed_load_checkpoint, HPSearchBackend, patch_trainer_compat, patch_args_compat, is_torch_tpu_available, ALL_LAYERNORM_LAYERS, IS_SAGEMAKER_MP_POST_1_10 # explicit for transformers 5.x from transformers.trainer_callback import TrainerCallback -from torch.utils.data import DataLoader -from torch.utils.data.distributed import DistributedSampler import numpy as np from cl_collator import SUPPORTED_DECODER_MODELS, check_model from cl_dataset import ANSWER_PREFIX -try: - import cupy as cp - from torch.utils.dlpack import to_dlpack, from_dlpack - from cupy import fromDlpack - HAS_CUPY = True -except ImportError: - HAS_CUPY = False - print("[WARNING] CuPy not available. SVD operations will use PyTorch fallback.") +import cupy as cp +from torch.utils.dlpack import to_dlpack, from_dlpack +from cupy import fromDlpack def skip_instructions(model, predictions_ids, tokenizer, ignore_idx=-100): predictions_ids = np.where(predictions_ids == ignore_idx, tokenizer.pad_token_id, predictions_ids) @@ -62,7 +51,7 @@ class DenserEvalCallback(TrainerCallback): control.should_log = True # Evaluate - if args.eval_strategy == IntervalStrategy.STEPS and state.global_step in log_eval_steps: + if args.evaluation_strategy == IntervalStrategy.STEPS and state.global_step in log_eval_steps: control.should_evaluate = True # Save @@ -73,10 +62,8 @@ class DenserEvalCallback(TrainerCallback): class InfLoRATrainer(Seq2SeqTrainer): - def __init__(self, model, args, train_dataset, cur_task_id, task_order, data_collator_replay=None, replay_dataset_dict=None, replay_label_dict=None, eval_dataset=None, processing_class=None, data_collator=None, compute_metrics=None, callbacks=None): - super().__init__(model=model, args=args, train_dataset=train_dataset, eval_dataset=eval_dataset, processing_class=processing_class, data_collator=data_collator, compute_metrics=compute_metrics, callbacks=callbacks) - patch_trainer_compat(self) - patch_args_compat(self.args) + def __init__(self, model, args, train_dataset, cur_task_id, task_order, data_collator_replay=None, replay_dataset_dict=None, replay_label_dict=None, eval_dataset=None, tokenizer=None, data_collator=None, compute_metrics=None, callbacks=None): + super().__init__(model=model, args=args, train_dataset=train_dataset, eval_dataset=eval_dataset, tokenizer=tokenizer, data_collator=data_collator, compute_metrics=compute_metrics, callbacks=callbacks) self.data_collator_replay = data_collator_replay self.replay_dataset_dict = replay_dataset_dict @@ -315,7 +302,7 @@ class InfLoRATrainer(Seq2SeqTrainer): torch.save(self.feature_list[i], os.path.join(self.args.output_dir, 'reg_{}.pt'.format(i))) - def training_step(self, model: nn.Module, inputs: Dict[str, Union[torch.Tensor, Any]], num_items_in_batch=None) -> torch.Tensor: + def training_step(self, model: nn.Module, inputs: Dict[str, Union[torch.Tensor, Any]]) -> torch.Tensor: """ Perform a training step on a batch of inputs. @@ -356,9 +343,11 @@ class InfLoRATrainer(Seq2SeqTrainer): elif self.use_apex: with amp.scale_loss(loss, self.optimizer) as scaled_loss: scaled_loss.backward() - else: - # Use accelerator.backward() for all cases (DDP, fp16, deepspeed, etc.) + elif self.is_deepspeed_enabled: + # loss gets scaled under gradient_accumulation_steps in deepspeed self.accelerator.backward(loss) + else: + loss.backward() if self.state.global_step > self.args.replay_after_n_epoch*self.args.step_per_epoch and self.args.data_replay_freq != -1 and self.state.global_step % self.args.data_replay_freq == 0: for item in self.replay_iterator_dict.keys(): @@ -742,7 +731,6 @@ class InfLoRATrainer(Seq2SeqTrainer): if "attention_mask" in inputs: gen_kwargs["attention_mask"] = inputs.get("attention_mask", None) - synced_gpus = gen_kwargs.pop("synced_gpus", False) generation_config = GenerationConfig(**gen_kwargs) # prepare generation inputs @@ -754,7 +742,6 @@ class InfLoRATrainer(Seq2SeqTrainer): generated_tokens = self.model.generate( input_ids=generation_inputs, generation_config=generation_config, - synced_gpus=synced_gpus, ) else: generation_inputs = inputs[self.model.main_input_name] @@ -764,14 +751,12 @@ class InfLoRATrainer(Seq2SeqTrainer): input_ids=generation_inputs, input_ids_wo_label=inputs["input_ids_wo_label"], generation_config=generation_config, - synced_gpus=synced_gpus, ) else: generated_tokens = self.model.generate( input_ids=generation_inputs, generation_config=generation_config, - synced_gpus=synced_gpus, ) bs, source_len = inputs['input_ids'].shape diff --git a/gainlora_OT_vMF/src/cl_trainer_olora.py b/root_gainlora/src/cl_trainer_olora.py similarity index 95% rename from gainlora_OT_vMF/src/cl_trainer_olora.py rename to root_gainlora/src/cl_trainer_olora.py index 6c876335763beebb5b2a32c63057da10c0967ab1..a186bf4b57a1c8afd4861a77e30ded987d77d623 100644 --- a/gainlora_OT_vMF/src/cl_trainer_olora.py +++ b/root_gainlora/src/cl_trainer_olora.py @@ -1,11 +1,7 @@ import torch -import torch.nn as nn -from typing import Dict, Union, Any, Optional, List, Tuple, Callable from transformers import GenerationConfig from transformers.trainer_seq2seq import Seq2SeqTrainer from transformers.trainer import * -from transformers.trainer_utils import IntervalStrategy -from compat_transformers import ShardedDDPOption, is_sagemaker_mp_enabled, smp_forward_backward, deepspeed_init, deepspeed_load_checkpoint, HPSearchBackend, patch_trainer_compat, patch_args_compat, is_torch_tpu_available, ALL_LAYERNORM_LAYERS, IS_SAGEMAKER_MP_POST_1_10 # explicit for transformers 5.x from transformers.trainer_callback import TrainerCallback import numpy as np @@ -13,10 +9,7 @@ from cl_collator import SUPPORTED_DECODER_MODELS, check_model from cl_dataset import ANSWER_PREFIX from t5_inflora import T5Attention -try: - import ipdb -except ImportError: - pass +import ipdb def skip_instructions(model, predictions_ids, tokenizer, ignore_idx=-100): @@ -59,7 +52,7 @@ class DenserEvalCallback(TrainerCallback): control.should_log = True # Evaluate - if args.eval_strategy == IntervalStrategy.STEPS and state.global_step in log_eval_steps: + if args.evaluation_strategy == IntervalStrategy.STEPS and state.global_step in log_eval_steps: control.should_evaluate = True # Save @@ -70,10 +63,8 @@ class DenserEvalCallback(TrainerCallback): class OLoRATrainer(Seq2SeqTrainer): - def __init__(self, model, args, train_dataset, cur_task_id, task_order, data_collator_replay=None, replay_dataset_dict=None, replay_label_dict=None, eval_dataset=None, processing_class=None, data_collator=None, compute_metrics=None, callbacks=None): - super().__init__(model=model, args=args, train_dataset=train_dataset, eval_dataset=eval_dataset, processing_class=processing_class, data_collator=data_collator, compute_metrics=compute_metrics, callbacks=callbacks) - patch_trainer_compat(self) - patch_args_compat(self.args) + def __init__(self, model, args, train_dataset, cur_task_id, task_order, data_collator_replay=None, replay_dataset_dict=None, replay_label_dict=None, eval_dataset=None, tokenizer=None, data_collator=None, compute_metrics=None, callbacks=None): + super().__init__(model=model, args=args, train_dataset=train_dataset, eval_dataset=eval_dataset, tokenizer=tokenizer, data_collator=data_collator, compute_metrics=compute_metrics, callbacks=callbacks) self.data_collator_replay = data_collator_replay self.replay_dataset_dict = replay_dataset_dict @@ -100,7 +91,7 @@ class OLoRATrainer(Seq2SeqTrainer): worker_init_fn=seed_worker) self.replay_iterator_dict = create_memory_replay_generators(task_order[cur_task_id], task_order, self.replay_dataloader_dict) - def training_step(self, model: nn.Module, inputs: Dict[str, Union[torch.Tensor, Any]], num_items_in_batch=None) -> torch.Tensor: + def training_step(self, model: nn.Module, inputs: Dict[str, Union[torch.Tensor, Any]]) -> torch.Tensor: """ Perform a training step on a batch of inputs. @@ -166,9 +157,11 @@ class OLoRATrainer(Seq2SeqTrainer): elif self.use_apex: with amp.scale_loss(loss, self.optimizer) as scaled_loss: scaled_loss.backward() - else: - # Use accelerator.backward() for all cases (DDP, fp16, deepspeed, etc.) + elif self.is_deepspeed_enabled: + # loss gets scaled under gradient_accumulation_steps in deepspeed self.accelerator.backward(loss) + else: + loss.backward() if self.state.global_step > self.args.replay_after_n_epoch*self.args.step_per_epoch and self.args.data_replay_freq != -1 and self.state.global_step % self.args.data_replay_freq == 0: for item in self.replay_iterator_dict.keys(): @@ -553,7 +546,6 @@ class OLoRATrainer(Seq2SeqTrainer): if "attention_mask" in inputs: gen_kwargs["attention_mask"] = inputs.get("attention_mask", None) - synced_gpus = gen_kwargs.pop("synced_gpus", False) generation_config = GenerationConfig(**gen_kwargs) # prepare generation inputs @@ -565,7 +557,6 @@ class OLoRATrainer(Seq2SeqTrainer): generated_tokens = self.model.generate( input_ids=generation_inputs, generation_config=generation_config, - synced_gpus=synced_gpus, ) else: generation_inputs = inputs[self.model.main_input_name] @@ -575,14 +566,12 @@ class OLoRATrainer(Seq2SeqTrainer): input_ids=generation_inputs, input_ids_wo_label=inputs["input_ids_wo_label"], generation_config=generation_config, - synced_gpus=synced_gpus, ) else: generated_tokens = self.model.generate( input_ids=generation_inputs, generation_config=generation_config, - synced_gpus=synced_gpus, ) bs, source_len = inputs['input_ids'].shape diff --git a/gainlora_OT_vMF/src/cl_trainer_olora_llama.py b/root_gainlora/src/cl_trainer_olora_llama.py similarity index 95% rename from gainlora_OT_vMF/src/cl_trainer_olora_llama.py rename to root_gainlora/src/cl_trainer_olora_llama.py index d92b6380ed3d129eb8858a46b346fab38becb827..daf19c238ade76cd05ef70e2efc2d3967d6d1721 100644 --- a/gainlora_OT_vMF/src/cl_trainer_olora_llama.py +++ b/root_gainlora/src/cl_trainer_olora_llama.py @@ -1,11 +1,7 @@ import torch -import torch.nn as nn -from typing import Dict, Union, Any, Optional, List, Tuple, Callable from transformers import GenerationConfig from transformers.trainer_seq2seq import Seq2SeqTrainer from transformers.trainer import * -from transformers.trainer_utils import IntervalStrategy -from compat_transformers import ShardedDDPOption, is_sagemaker_mp_enabled, smp_forward_backward, deepspeed_init, deepspeed_load_checkpoint, HPSearchBackend, patch_trainer_compat, patch_args_compat, is_torch_tpu_available, ALL_LAYERNORM_LAYERS, IS_SAGEMAKER_MP_POST_1_10 # explicit for transformers 5.x from transformers.trainer_callback import TrainerCallback import numpy as np @@ -15,10 +11,7 @@ from cl_dataset import ANSWER_PREFIX from llama_inflora import LlamaAttention # from llama13b_inflora import LlamaFlashAttention2 # from t5_inflora import T5Attention -try: - import ipdb -except ImportError: - pass +import ipdb def skip_instructions(model, predictions_ids, tokenizer, ignore_idx=-100): @@ -61,7 +54,7 @@ class DenserEvalCallback(TrainerCallback): control.should_log = True # Evaluate - if args.eval_strategy == IntervalStrategy.STEPS and state.global_step in log_eval_steps: + if args.evaluation_strategy == IntervalStrategy.STEPS and state.global_step in log_eval_steps: control.should_evaluate = True # Save @@ -72,10 +65,8 @@ class DenserEvalCallback(TrainerCallback): class OLoRATrainer(Seq2SeqTrainer): - def __init__(self, model, args, train_dataset, cur_task_id, task_order, data_collator_replay=None, replay_dataset_dict=None, replay_label_dict=None, eval_dataset=None, processing_class=None, data_collator=None, compute_metrics=None, callbacks=None): - super().__init__(model=model, args=args, train_dataset=train_dataset, eval_dataset=eval_dataset, processing_class=processing_class, data_collator=data_collator, compute_metrics=compute_metrics, callbacks=callbacks) - patch_trainer_compat(self) - patch_args_compat(self.args) + def __init__(self, model, args, train_dataset, cur_task_id, task_order, data_collator_replay=None, replay_dataset_dict=None, replay_label_dict=None, eval_dataset=None, tokenizer=None, data_collator=None, compute_metrics=None, callbacks=None): + super().__init__(model=model, args=args, train_dataset=train_dataset, eval_dataset=eval_dataset, tokenizer=tokenizer, data_collator=data_collator, compute_metrics=compute_metrics, callbacks=callbacks) self.data_collator_replay = data_collator_replay self.replay_dataset_dict = replay_dataset_dict @@ -102,7 +93,7 @@ class OLoRATrainer(Seq2SeqTrainer): worker_init_fn=seed_worker) self.replay_iterator_dict = create_memory_replay_generators(task_order[cur_task_id], task_order, self.replay_dataloader_dict) - def training_step(self, model: nn.Module, inputs: Dict[str, Union[torch.Tensor, Any]], num_items_in_batch=None) -> torch.Tensor: + def training_step(self, model: nn.Module, inputs: Dict[str, Union[torch.Tensor, Any]]) -> torch.Tensor: """ Perform a training step on a batch of inputs. @@ -169,9 +160,11 @@ class OLoRATrainer(Seq2SeqTrainer): elif self.use_apex: with amp.scale_loss(loss, self.optimizer) as scaled_loss: scaled_loss.backward() - else: - # Use accelerator.backward() for all cases (DDP, fp16, deepspeed, etc.) + elif self.is_deepspeed_enabled: + # loss gets scaled under gradient_accumulation_steps in deepspeed self.accelerator.backward(loss) + else: + loss.backward() if self.state.global_step > self.args.replay_after_n_epoch*self.args.step_per_epoch and self.args.data_replay_freq != -1 and self.state.global_step % self.args.data_replay_freq == 0: for item in self.replay_iterator_dict.keys(): @@ -555,7 +548,6 @@ class OLoRATrainer(Seq2SeqTrainer): if "attention_mask" in inputs: gen_kwargs["attention_mask"] = inputs.get("attention_mask", None) - synced_gpus = gen_kwargs.pop("synced_gpus", False) generation_config = GenerationConfig(**gen_kwargs) # prepare generation inputs @@ -567,7 +559,6 @@ class OLoRATrainer(Seq2SeqTrainer): generated_tokens = self.model.generate( input_ids=generation_inputs, generation_config=generation_config, - synced_gpus=synced_gpus, ) else: generation_inputs = inputs[self.model.main_input_name] @@ -577,14 +568,12 @@ class OLoRATrainer(Seq2SeqTrainer): input_ids=generation_inputs, input_ids_wo_label=inputs["input_ids_wo_label"], generation_config=generation_config, - synced_gpus=synced_gpus, ) else: generated_tokens = self.model.generate( input_ids=generation_inputs, generation_config=generation_config, - synced_gpus=synced_gpus, ) bs, source_len = inputs['input_ids'].shape diff --git a/gainlora_OT_vMF/src/compute_metrics.py b/root_gainlora/src/compute_metrics.py similarity index 100% rename from gainlora_OT_vMF/src/compute_metrics.py rename to root_gainlora/src/compute_metrics.py diff --git a/gainlora_OT_vMF/src/llama_gainlora_inflora.py b/root_gainlora/src/llama_gainlora_inflora.py similarity index 99% rename from gainlora_OT_vMF/src/llama_gainlora_inflora.py rename to root_gainlora/src/llama_gainlora_inflora.py index 9446ff8ffebae74821ba6938eecbbc1cd9c3c15e..ccea8ad27927ac70c92f6420c5a455478eec4716 100644 --- a/gainlora_OT_vMF/src/llama_gainlora_inflora.py +++ b/root_gainlora/src/llama_gainlora_inflora.py @@ -639,10 +639,9 @@ class LlamaPreTrainedModel(PreTrainedModel): if module.padding_idx is not None: module.weight.data[module.padding_idx].zero_() - def _set_gradient_checkpointing(self, enable=True, gradient_checkpointing_func=None): - for module in self.modules(): - if isinstance(module, LlamaModel): - module.gradient_checkpointing = enable + def _set_gradient_checkpointing(self, module, value=False): + if isinstance(module, LlamaModel): + module.gradient_checkpointing = value LLAMA_INPUTS_DOCSTRING = r""" @@ -1003,7 +1002,6 @@ class LlamaModel(LlamaPreTrainedModel): attention_mask, position_ids, None, - use_reentrant=False, ) else: layer_outputs = decoder_layer( diff --git a/gainlora_OT_vMF/src/llama_inflora.py b/root_gainlora/src/llama_inflora.py similarity index 99% rename from gainlora_OT_vMF/src/llama_inflora.py rename to root_gainlora/src/llama_inflora.py index 0a15b4360a8b3ecdac7cb3627568c2dba7eceb47..66b00a4e1d40e975cd1e736f5fa7c16ed568b062 100644 --- a/gainlora_OT_vMF/src/llama_inflora.py +++ b/root_gainlora/src/llama_inflora.py @@ -618,10 +618,9 @@ class LlamaPreTrainedModel(PreTrainedModel): if module.padding_idx is not None: module.weight.data[module.padding_idx].zero_() - def _set_gradient_checkpointing(self, enable=True, gradient_checkpointing_func=None): - for module in self.modules(): - if isinstance(module, LlamaModel): - module.gradient_checkpointing = enable + def _set_gradient_checkpointing(self, module, value=False): + if isinstance(module, LlamaModel): + module.gradient_checkpointing = value LLAMA_INPUTS_DOCSTRING = r""" @@ -908,7 +907,6 @@ class LlamaModel(LlamaPreTrainedModel): attention_mask, position_ids, None, - use_reentrant=False, ) else: layer_outputs = decoder_layer( diff --git a/gainlora_OT_vMF/src/rouge/README.md b/root_gainlora/src/rouge/README.md similarity index 100% rename from gainlora_OT_vMF/src/rouge/README.md rename to root_gainlora/src/rouge/README.md diff --git a/gainlora_OT_vMF/src/rouge/__init__.py b/root_gainlora/src/rouge/__init__.py similarity index 100% rename from gainlora_OT_vMF/src/rouge/__init__.py rename to root_gainlora/src/rouge/__init__.py diff --git a/gainlora_OT_vMF/src/rouge/__pycache__/__init__.cpython-310.pyc b/root_gainlora/src/rouge/__pycache__/__init__.cpython-310.pyc similarity index 100% rename from gainlora_OT_vMF/src/rouge/__pycache__/__init__.cpython-310.pyc rename to root_gainlora/src/rouge/__pycache__/__init__.cpython-310.pyc diff --git a/gainlora_OT_vMF/src/rouge/__pycache__/rouge_scorer.cpython-310.pyc b/root_gainlora/src/rouge/__pycache__/rouge_scorer.cpython-310.pyc similarity index 100% rename from gainlora_OT_vMF/src/rouge/__pycache__/rouge_scorer.cpython-310.pyc rename to root_gainlora/src/rouge/__pycache__/rouge_scorer.cpython-310.pyc diff --git a/gainlora_OT_vMF/src/rouge/__pycache__/scoring.cpython-310.pyc b/root_gainlora/src/rouge/__pycache__/scoring.cpython-310.pyc similarity index 100% rename from gainlora_OT_vMF/src/rouge/__pycache__/scoring.cpython-310.pyc rename to root_gainlora/src/rouge/__pycache__/scoring.cpython-310.pyc diff --git a/gainlora_OT_vMF/src/rouge/__pycache__/tokenize.cpython-310.pyc b/root_gainlora/src/rouge/__pycache__/tokenize.cpython-310.pyc similarity index 100% rename from gainlora_OT_vMF/src/rouge/__pycache__/tokenize.cpython-310.pyc rename to root_gainlora/src/rouge/__pycache__/tokenize.cpython-310.pyc diff --git a/gainlora_OT_vMF/src/rouge/__pycache__/tokenizers.cpython-310.pyc b/root_gainlora/src/rouge/__pycache__/tokenizers.cpython-310.pyc similarity index 100% rename from gainlora_OT_vMF/src/rouge/__pycache__/tokenizers.cpython-310.pyc rename to root_gainlora/src/rouge/__pycache__/tokenizers.cpython-310.pyc diff --git a/gainlora_OT_vMF/src/rouge/create_pyrouge_files.py b/root_gainlora/src/rouge/create_pyrouge_files.py similarity index 100% rename from gainlora_OT_vMF/src/rouge/create_pyrouge_files.py rename to root_gainlora/src/rouge/create_pyrouge_files.py diff --git a/gainlora_OT_vMF/src/rouge/io.py b/root_gainlora/src/rouge/io.py similarity index 100% rename from gainlora_OT_vMF/src/rouge/io.py rename to root_gainlora/src/rouge/io.py diff --git a/gainlora_OT_vMF/src/rouge/io_test.py b/root_gainlora/src/rouge/io_test.py similarity index 100% rename from gainlora_OT_vMF/src/rouge/io_test.py rename to root_gainlora/src/rouge/io_test.py diff --git a/gainlora_OT_vMF/src/rouge/requirements.txt b/root_gainlora/src/rouge/requirements.txt similarity index 100% rename from gainlora_OT_vMF/src/rouge/requirements.txt rename to root_gainlora/src/rouge/requirements.txt diff --git a/gainlora_OT_vMF/src/rouge/rouge.py b/root_gainlora/src/rouge/rouge.py similarity index 100% rename from gainlora_OT_vMF/src/rouge/rouge.py rename to root_gainlora/src/rouge/rouge.py diff --git a/gainlora_OT_vMF/src/rouge/rouge_scorer.py b/root_gainlora/src/rouge/rouge_scorer.py similarity index 100% rename from gainlora_OT_vMF/src/rouge/rouge_scorer.py rename to root_gainlora/src/rouge/rouge_scorer.py diff --git a/gainlora_OT_vMF/src/rouge/rouge_scorer_test.py b/root_gainlora/src/rouge/rouge_scorer_test.py similarity index 100% rename from gainlora_OT_vMF/src/rouge/rouge_scorer_test.py rename to root_gainlora/src/rouge/rouge_scorer_test.py diff --git a/gainlora_OT_vMF/src/rouge/run.sh b/root_gainlora/src/rouge/run.sh similarity index 100% rename from gainlora_OT_vMF/src/rouge/run.sh rename to root_gainlora/src/rouge/run.sh diff --git a/gainlora_OT_vMF/src/rouge/scoring.py b/root_gainlora/src/rouge/scoring.py similarity index 100% rename from gainlora_OT_vMF/src/rouge/scoring.py rename to root_gainlora/src/rouge/scoring.py diff --git a/gainlora_OT_vMF/src/rouge/scoring_test.py b/root_gainlora/src/rouge/scoring_test.py similarity index 100% rename from gainlora_OT_vMF/src/rouge/scoring_test.py rename to root_gainlora/src/rouge/scoring_test.py diff --git a/gainlora_OT_vMF/src/rouge/setup.py b/root_gainlora/src/rouge/setup.py similarity index 100% rename from gainlora_OT_vMF/src/rouge/setup.py rename to root_gainlora/src/rouge/setup.py diff --git a/gainlora_OT_vMF/src/rouge/test_util.py b/root_gainlora/src/rouge/test_util.py similarity index 100% rename from gainlora_OT_vMF/src/rouge/test_util.py rename to root_gainlora/src/rouge/test_util.py diff --git a/gainlora_OT_vMF/src/rouge/tokenize.py b/root_gainlora/src/rouge/tokenize.py similarity index 100% rename from gainlora_OT_vMF/src/rouge/tokenize.py rename to root_gainlora/src/rouge/tokenize.py diff --git a/gainlora_OT_vMF/src/rouge/tokenize_test.py b/root_gainlora/src/rouge/tokenize_test.py similarity index 100% rename from gainlora_OT_vMF/src/rouge/tokenize_test.py rename to root_gainlora/src/rouge/tokenize_test.py diff --git a/gainlora_OT_vMF/src/rouge/tokenizers.py b/root_gainlora/src/rouge/tokenizers.py similarity index 100% rename from gainlora_OT_vMF/src/rouge/tokenizers.py rename to root_gainlora/src/rouge/tokenizers.py diff --git a/gainlora_OT_vMF/src/rouge/tokenizers_test.py b/root_gainlora/src/rouge/tokenizers_test.py similarity index 100% rename from gainlora_OT_vMF/src/rouge/tokenizers_test.py rename to root_gainlora/src/rouge/tokenizers_test.py diff --git a/gainlora_OT_vMF/src/run_llama.py b/root_gainlora/src/run_llama.py similarity index 94% rename from gainlora_OT_vMF/src/run_llama.py rename to root_gainlora/src/run_llama.py index 49ccfd4c0f7e27a2cace50595ee5bde30939ca3b..0bce356e288990cfaf8702d46ec0dd2e79931478 100644 --- a/gainlora_OT_vMF/src/run_llama.py +++ b/root_gainlora/src/run_llama.py @@ -32,6 +32,7 @@ import datasets import nltk # Here to have a nice missing dependency error message early on import numpy as np import pickle +from datasets import load_dataset from copy import deepcopy from torch import nn @@ -52,13 +53,7 @@ from assets import task_config, lora_state_dict_A, lora_state_dict_B, lora_state from cl_trainer_gainlora_inflora_llama import DenserEvalCallback, skip_instructions from compute_metrics import compute_metrics, compute_grouped_metrics -# Add src directory to path for local imports -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) - -try: - import ipdb -except ImportError: - pass # ipdb not required +import ipdb # off wandb os.environ['WANDB_DISABLED'] = "True" @@ -66,11 +61,8 @@ os.environ['WANDB_DISABLED'] = "True" logger = logging.getLogger(__name__) CURRENT_DIR = os.path.dirname(__file__) -local_data_path = os.environ.get("NLTK_DATA", os.path.expanduser("~/nltk_data")) -if os.path.exists(local_data_path): - nltk.data.path.append(local_data_path) -else: - nltk.download("punkt") +local_data_path = "/home/work/nltk_data" +nltk.data.path.append(local_data_path) @dataclass class ModelArguments: @@ -372,38 +364,6 @@ class TrainingArguments(Seq2SeqTrainingArguments): metadata={"help": "persent"} ) - -def load_cl_dataset(data_dir, task_config_dir, max_num_instances_per_task, - max_num_instances_per_eval_task, num_examples): - """ - Load CL dataset directly from the CLInstructions builder. - datasets>=4.0 no longer supports loading from a local script via load_dataset(script.py, ...). - """ - from cl_dataset import CLInstructions, CLConfig - - # Create config to parse task configs - config = CLConfig( - data_dir=data_dir, - task_config_dir=task_config_dir, - max_num_instances_per_task=max_num_instances_per_task, - max_num_instances_per_eval_task=max_num_instances_per_eval_task, - num_examples=num_examples, - ) - - # Create builder instance and manually set config (datasets 4.x compatibility) - builder = CLInstructions() - builder.config = config - - # Generate splits - split_generators = builder._split_generators(None) - split_names = ["train", "validation", "test"] - dataset_dict = {} - for split_name, split_gen in zip(split_names, split_generators): - examples_list = [v for _, v in builder._generate_examples(**split_gen.gen_kwargs)] - dataset_dict[split_name] = datasets.Dataset.from_list(examples_list) - return datasets.DatasetDict(dataset_dict) - - def main(): # See all possible arguments in src/transformers/training_args.py # or by passing the --help flag to this script. @@ -463,20 +423,23 @@ def main(): cur_task_id = task_order.index(cur_task) # Get the CL dataset - raw_datasets = load_cl_dataset( + raw_datasets = load_dataset( + os.path.join(CURRENT_DIR, "cl_dataset.py"), data_dir=data_args.data_dir, task_config_dir=data_args.task_config_dir, + # cache_dir=data_cache_dir, # for debug, change dataset size, otherwise open it max_num_instances_per_task=data_args.max_num_instances_per_task, max_num_instances_per_eval_task=data_args.max_num_instances_per_eval_task, - num_examples=data_args.num_examples, + num_examples=data_args.num_examples ) + raw_datasets.cleanup_cache_files() print(raw_datasets) config = AutoConfig.from_pretrained( model_args.model_name_or_path, cache_dir=model_args.cache_dir, revision=model_args.model_revision, - token=True if model_args.use_auth_token else None, + use_auth_token=True if model_args.use_auth_token else None, ) config.bos_token_id = 1 config.eos_token_id = 2 @@ -487,7 +450,7 @@ def main(): cache_dir = model_args.cache_dir, use_fast = model_args.use_fast_tokenizer, revision = model_args.model_revision, - token = True if model_args.use_auth_token else None, + use_auth_token = True if model_args.use_auth_token else None, ) else: tokenizer = transformers.LlamaTokenizer.from_pretrained( @@ -495,7 +458,7 @@ def main(): cache_dir = model_args.cache_dir, use_fast = model_args.use_fast_tokenizer, revision = model_args.model_revision, - token = True if model_args.use_auth_token else None, + use_auth_token = True if model_args.use_auth_token else None, ) tokenizer.bos_token_id = 1 tokenizer.eos_token_id = 2 @@ -554,7 +517,7 @@ def main(): config=config, cache_dir=model_args.cache_dir, revision=model_args.model_revision, - token=True if model_args.use_auth_token else None, + use_auth_token=True if model_args.use_auth_token else None, use_safetensors=True, ) @@ -721,13 +684,14 @@ def main(): if model_args.load_checkpoint_from: replay_dataset_dict = {} for idx in range(cur_task_id): - raw_datasets_gen = load_cl_dataset( + raw_datasets_gen = load_dataset( + os.path.join(CURRENT_DIR, "cl_dataset.py"), data_dir=data_args.gen_data_dir, task_config_dir=task_config[task_order[idx]], + cache_dir=data_cache_dir, # for debug, change dataset size, otherwise open it max_num_instances_per_task=data_args.max_num_instances_per_task, max_num_instances_per_eval_task=data_args.max_num_instances_per_eval_task, - num_examples=data_args.num_examples, - ) + num_examples=data_args.num_examples) replay_dataset_dict[task_order[idx]] = raw_datasets_gen["train"] print(raw_datasets_gen) @@ -768,7 +732,7 @@ def main(): return result print(f"-----Gradient checkpointing: {training_args.gradient_checkpointing} -----") if training_args.gradient_checkpointing: - model.gradient_checkpointing_enable(gradient_checkpointing_kwargs={"use_reentrant": False}) + model.gradient_checkpointing_enable() world_size = int(os.environ.get("WORLD_SIZE", 1)) training_args.step_per_epoch = math.ceil(len(raw_datasets["train"]) / training_args.per_device_train_batch_size / world_size / training_args.gradient_accumulation_steps) @@ -800,7 +764,7 @@ def main(): replay_dataset_dict=replay_dataset_dict, replay_label_dict=replay_label_dict, eval_dataset=eval_dataset if training_args.do_eval else None, - processing_class=tokenizer, + tokenizer=tokenizer, data_collator=data_collator, compute_metrics=compute_rouge_metrics, callbacks=[DenserEvalCallback] if training_args.denser_evaluation else None @@ -822,7 +786,7 @@ def main(): replay_dataset_dict=replay_dataset_dict, replay_label_dict=replay_label_dict, eval_dataset=eval_dataset if training_args.do_eval else None, - processing_class=tokenizer, + tokenizer=tokenizer, data_collator=data_collator, compute_metrics=compute_rouge_metrics, callbacks=[DenserEvalCallback] if training_args.denser_evaluation else None @@ -847,7 +811,7 @@ def main(): replay_dataset_dict=replay_dataset_dict, replay_label_dict=replay_label_dict, eval_dataset=eval_dataset if training_args.do_eval else None, - processing_class=tokenizer, + tokenizer=tokenizer, data_collator=data_collator, compute_metrics=compute_rouge_metrics, callbacks=[DenserEvalCallback] if training_args.denser_evaluation else None @@ -871,7 +835,7 @@ def main(): replay_dataset_dict=replay_dataset_dict, replay_label_dict=replay_label_dict, eval_dataset=eval_dataset if training_args.do_eval else None, - processing_class=tokenizer, + tokenizer=tokenizer, data_collator=data_collator, compute_metrics=compute_rouge_metrics, callbacks=[DenserEvalCallback] if training_args.denser_evaluation else None diff --git a/gainlora_OT_vMF/src/run_t5.py b/root_gainlora/src/run_t5.py similarity index 83% rename from gainlora_OT_vMF/src/run_t5.py rename to root_gainlora/src/run_t5.py index 78456c2ba02e76d89667b7ce6714748e5e12e922..efb0c674dd4a9ba0eacfb0572d2697c9fe069ce6 100644 --- a/gainlora_OT_vMF/src/run_t5.py +++ b/root_gainlora/src/run_t5.py @@ -28,19 +28,16 @@ from typing import Optional import math import torch from torch import nn - -# Add src directory to path for local imports -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) - try: - import ipdb + import ipdb # Optional debug dependency. except ImportError: - pass # ipdb not required + ipdb = None import datasets import nltk # Here to have a nice missing dependency error message early on import numpy as np import pickle +from datasets import load_dataset from copy import deepcopy import transformers @@ -60,9 +57,7 @@ from assets import task_config, lora_state_dict_A, lora_state_dict_B from cl_trainer_gainlora_inflora import DenserEvalCallback, skip_instructions from compute_metrics import compute_metrics, compute_grouped_metrics - -# OT-SIGN run logger -from run_logger import RunLogger +from datasets.download import DownloadConfig # off wandb os.environ['WANDB_DISABLED'] = "True" @@ -70,14 +65,8 @@ os.environ['WANDB_DISABLED'] = "True" logger = logging.getLogger(__name__) CURRENT_DIR = os.path.dirname(__file__) -local_data_path = os.environ.get("NLTK_DATA", os.path.expanduser("~/nltk_data")) -if os.path.exists(local_data_path): - nltk.data.path.append(local_data_path) -else: - try: - nltk.download('punkt', quiet=True) - except Exception: - pass +local_data_path = "/home/work/nltk_data" +nltk.data.path.append(local_data_path) @dataclass(frozen=False) class ModelArguments: @@ -189,28 +178,6 @@ class ModelArguments: metadata={"help": "Path to load previous checkpoints"} ) - # ===== OT-SIGN model arguments ===== - use_ot_routing: bool = field( - default=False, - metadata={"help": "Enable OT-SIGN routing (vMF signatures + Sinkhorn OT)"} - ) - ot_epsilon: float = field( - default=0.05, - metadata={"help": "Sinkhorn entropic regularization strength"} - ) - ot_n_iter: int = field( - default=10, - metadata={"help": "Number of Sinkhorn iterations"} - ) - default_kappa: float = field( - default=10.0, - metadata={"help": "Default kappa for current task's proxy in OT cost"} - ) - previous_vmf_signatures_path: Optional[str] = field( - default=None, - metadata={"help": "Path to load vMF signatures from previous tasks"} - ) - @dataclass(frozen=False) class DataTrainingArguments: @@ -394,53 +361,6 @@ class TrainingArguments(Seq2SeqTrainingArguments): metadata={"help": "persent"} ) - # ===== OT-SIGN training arguments ===== - # NOTE: use_ot_routing is defined in ModelArguments; it is copied to training_args - # after parsing so that the trainer can access it via self.args.use_ot_routing. - lambda_drift: float = field( - default=0.01, - metadata={"help": "Weight for anti-drift loss on replay data"} - ) - lambda_inv: float = field( - default=0.001, - metadata={"help": "Weight for anti-invasion loss on current task data"} - ) - invasion_threshold: float = field( - default=2.3, - metadata={"help": "vMF log-likelihood threshold for invasion detection (~-log(0.1))"} - ) - - -def load_cl_dataset(data_dir, task_config_dir, max_num_instances_per_task, - max_num_instances_per_eval_task, num_examples): - """ - Load CL dataset directly from the CLInstructions builder. - datasets>=4.0 no longer supports loading from a local script via load_dataset(script.py, ...). - """ - from cl_dataset import CLInstructions, CLConfig - - # Create config to parse task configs - config = CLConfig( - data_dir=data_dir, - task_config_dir=task_config_dir, - max_num_instances_per_task=max_num_instances_per_task, - max_num_instances_per_eval_task=max_num_instances_per_eval_task, - num_examples=num_examples, - ) - - # Create builder instance and manually set config (datasets 4.x compatibility) - builder = CLInstructions() - builder.config = config - - # Generate splits - split_generators = builder._split_generators(None) - split_names = ["train", "validation", "test"] - dataset_dict = {} - for split_name, split_gen in zip(split_names, split_generators): - examples_list = [v for _, v in builder._generate_examples(**split_gen.gen_kwargs)] - dataset_dict[split_name] = datasets.Dataset.from_list(examples_list) - return datasets.DatasetDict(dataset_dict) - def main(): # See all possible arguments in src/transformers/training_args.py @@ -455,9 +375,6 @@ def main(): else: model_args, data_args, training_args = parser.parse_args_into_dataclasses() training_args._frozen = False - # Copy OT-SIGN flag from ModelArguments into training_args so the trainer - # (which reads self.args) can see it without duplicating the CLI argument. - training_args.use_ot_routing = model_args.use_ot_routing # Setup logging logging.basicConfig( @@ -503,28 +420,34 @@ def main(): cur_task = data_args.task_config_dir.split('/')[-1] cur_task_id = task_order.index(cur_task) + download_config = DownloadConfig + download_config.local_files_only = True # Get the CL dataset - raw_datasets = load_cl_dataset( + raw_datasets = load_dataset( + os.path.join(CURRENT_DIR, "cl_dataset.py"), data_dir=data_args.data_dir, + download_config=download_config, task_config_dir=data_args.task_config_dir, + # cache_dir=data_cache_dir, # for debug, change dataset size, otherwise open it max_num_instances_per_task=data_args.max_num_instances_per_task, max_num_instances_per_eval_task=data_args.max_num_instances_per_eval_task, - num_examples=data_args.num_examples, + num_examples=data_args.num_examples ) + raw_datasets.cleanup_cache_files() # Load pretrained model and tokenizer config = AutoConfig.from_pretrained( model_args.config_name if model_args.config_name else model_args.model_name_or_path, cache_dir=model_args.cache_dir, revision=model_args.model_revision, - token=True if model_args.use_auth_token else None, + use_auth_token=True if model_args.use_auth_token else None, ) tokenizer = AutoTokenizer.from_pretrained( model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path, cache_dir=model_args.cache_dir, use_fast=model_args.use_fast_tokenizer, revision=model_args.model_revision, - token=True if model_args.use_auth_token else None, + use_auth_token=True if model_args.use_auth_token else None, ) prompt_config = { @@ -537,13 +460,7 @@ def main(): 'run_single': model_args.run_single, 'lora_r': model_args.lora_r, 'lora_alpha': model_args.lora_alpha, - 'lora_dropout': model_args.lora_dropout, - # OT-SIGN parameters - 'use_ot_routing': model_args.use_ot_routing, - 'ot_epsilon': model_args.ot_epsilon, - 'ot_n_iter': model_args.ot_n_iter, - 'default_kappa': model_args.default_kappa, - 'previous_vmf_signatures_path': model_args.previous_vmf_signatures_path, + 'lora_dropout': model_args.lora_dropout } if training_args.model_name in ['inflora', 'olora']: @@ -560,7 +477,7 @@ def main(): config=config, cache_dir=model_args.cache_dir, revision=model_args.model_revision, - token=True if model_args.use_auth_token else None, + use_auth_token=True if model_args.use_auth_token else None, ) model.persent = training_args.persent @@ -736,13 +653,15 @@ def main(): if model_args.load_checkpoint_from: replay_dataset_dict = {} for idx in range(cur_task_id): - raw_datasets_gen = load_cl_dataset( + raw_datasets_gen = load_dataset( + os.path.join(CURRENT_DIR, "cl_dataset.py"), data_dir=data_dir, + download_config=download_config, task_config_dir=task_config[task_order[idx]], + cache_dir=data_cache_dir, # for debug, change dataset size, otherwise open it max_num_instances_per_task=data_args.max_num_instances_per_task, max_num_instances_per_eval_task=data_args.max_num_instances_per_eval_task, - num_examples=data_args.num_examples, - ) + num_examples=data_args.num_examples) replay_dataset_dict[task_order[idx]] = raw_datasets_gen["train"] print(raw_datasets_gen) @@ -782,7 +701,7 @@ def main(): return result print(f"-----Gradient checkpointing: {training_args.gradient_checkpointing} -----") if training_args.gradient_checkpointing: - model.gradient_checkpointing_enable(gradient_checkpointing_kwargs={"use_reentrant": False}) + model.gradient_checkpointing_enable() world_size = int(os.environ.get("WORLD_SIZE", 1)) training_args.step_per_epoch = math.ceil(len(raw_datasets["train"]) / training_args.per_device_train_batch_size / world_size / training_args.gradient_accumulation_steps) @@ -809,7 +728,7 @@ def main(): replay_dataset_dict=replay_dataset_dict, replay_label_dict=replay_label_dict, eval_dataset=eval_dataset if training_args.do_eval else None, - processing_class=tokenizer, + tokenizer=tokenizer, data_collator=data_collator, compute_metrics=compute_rouge_metrics, callbacks=[DenserEvalCallback] if training_args.denser_evaluation else None @@ -828,7 +747,7 @@ def main(): replay_dataset_dict=replay_dataset_dict, replay_label_dict=replay_label_dict, eval_dataset=eval_dataset if training_args.do_eval else None, - processing_class=tokenizer, + tokenizer=tokenizer, data_collator=data_collator, compute_metrics=compute_rouge_metrics, callbacks=[DenserEvalCallback] if training_args.denser_evaluation else None @@ -847,7 +766,7 @@ def main(): replay_dataset_dict=replay_dataset_dict, replay_label_dict=replay_label_dict, eval_dataset=eval_dataset if training_args.do_eval else None, - processing_class=tokenizer, + tokenizer=tokenizer, data_collator=data_collator, compute_metrics=compute_rouge_metrics, callbacks=[DenserEvalCallback] if training_args.denser_evaluation else None @@ -866,7 +785,7 @@ def main(): replay_dataset_dict=replay_dataset_dict, replay_label_dict=replay_label_dict, eval_dataset=eval_dataset if training_args.do_eval else None, - processing_class=tokenizer, + tokenizer=tokenizer, data_collator=data_collator, compute_metrics=compute_rouge_metrics, callbacks=[DenserEvalCallback] if training_args.denser_evaluation else None @@ -929,46 +848,6 @@ def main(): if training_args.model_name in ['inflora', 'gainlora_inflora', 'gainlora_olora']: trainer.get_repsentation() - # ===== OT-SIGN: Compute and save vMF signature for current task ===== - if model_args.use_ot_routing and not prompt_config["run_single"]: - mu, kappa = trainer.compute_vmf_signature(cur_task_id) - - # Save all signatures (previous + current) - vmf_save_path = os.path.join(save_path, 'vmf_signatures.pt') - torch.save(trainer.model.encoder.vmf_signatures, vmf_save_path) - print(f"[OT-SIGN] Saved {len(trainer.model.encoder.vmf_signatures)} vMF signatures to {vmf_save_path}") - - # Log to run logger - try: - run_logger = RunLogger( - log_dir=os.path.join("logs_and_outputs", training_args.run_name, "ot_sign_logs") - ) - run_logger.log_run( - task_name=cur_task, - task_id=cur_task_id, - config={ - "use_ot_routing": model_args.use_ot_routing, - "ot_epsilon": model_args.ot_epsilon, - "ot_n_iter": model_args.ot_n_iter, - "default_kappa": model_args.default_kappa, - "lambda_drift": training_args.lambda_drift, - "lambda_inv": training_args.lambda_inv, - "invasion_threshold": training_args.invasion_threshold, - "lora_r": model_args.lora_r, - "lora_alpha": model_args.lora_alpha, - "batch_size": training_args.per_device_train_batch_size, - "learning_rate": training_args.learning_rate, - "num_train_epochs": training_args.num_train_epochs, - "fp16": training_args.fp16, - "gradient_accumulation_steps": training_args.gradient_accumulation_steps, - }, - vmf_signature={"mu_norm": mu.norm().item(), "kappa": kappa.item()}, - metrics=metrics, - ) - except Exception as e: - print(f"[OT-SIGN] Warning: Logger error: {e}") - # ===== End OT-SIGN save ===== - # Evaluation results = {} # in case the batch is shorter than max length, the output should be padded @@ -1028,23 +907,6 @@ def main(): trainer.save_metrics("predict", metrics) all_metrics.update(metrics) - # ===== Log predict scores for AP/FT computation ===== - try: - task_order_list = [t.strip() for t in data_args.task_order.split(',') if t.strip()] - _is_long = any(t in ['yelp', 'amazon', 'mnli', 'imdb', 'sst2', 'dbpedia', 'agnews'] for t in task_order_list) - _btype = 'long' if _is_long else 'superni' - _score_logger = RunLogger( - log_dir=os.path.join("logs_and_outputs", training_args.run_name, "ot_sign_logs"), - benchmark_type=_btype, - ) - _score_logger.log_predict_scores(cur_task_id, metrics, task_order_list) - # On final task, print AP/FT summary to console + file - if cur_task_id == len(task_order_list) - 1: - _score_logger.print_ap_ft(task_order_list, method_name=training_args.run_name) - except Exception as _e: - print(f"[RunLogger] Score logging error: {_e}") - # ===== End AP/FT logging ===== - with open(os.path.join("logs_and_outputs", training_args.run_name, "outputs", "task_order.txt"), 'w') as f: f.write(data_args.task_order) diff --git a/gainlora_OT_vMF/src/t5_gainlora_inflora.py b/root_gainlora/src/t5_gainlora_inflora.py similarity index 92% rename from gainlora_OT_vMF/src/t5_gainlora_inflora.py rename to root_gainlora/src/t5_gainlora_inflora.py index f0d6c459d126a22e16e0189171836f8e0a3a41d6..bcc97040e66bc64e73984ba2f6a870b30fe061cf 100644 --- a/gainlora_OT_vMF/src/t5_gainlora_inflora.py +++ b/root_gainlora/src/t5_gainlora_inflora.py @@ -23,18 +23,11 @@ import numpy as np import copy import numpy from typing import Optional, Tuple, Union -try: - import ipdb -except ImportError: - pass # ipdb not required for production runs +import ipdb import torch from torch import nn from torch.nn import CrossEntropyLoss -import torch.nn.functional as F - -# OT-SIGN imports -from ot_sign_utils import sinkhorn_log, vmf_cost_matrix from torch.utils.checkpoint import checkpoint from transformers.activations import ACT2FN @@ -45,56 +38,17 @@ from transformers.modeling_outputs import ( Seq2SeqModelOutput, ) from transformers.modeling_utils import PreTrainedModel -from transformers.pytorch_utils import ALL_LAYERNORM_LAYERS - -# Try to import pruning utilities (not available in transformers 5.0+) -try: - from transformers.pytorch_utils import find_pruneable_heads_and_indices, prune_linear_layer - HAS_PRUNING_UTILS = True -except ImportError: - HAS_PRUNING_UTILS = False - -# Try to import utils removed in transformers 5.0+ -try: - from transformers.utils import ( - DUMMY_INPUTS, - DUMMY_MASK, - add_start_docstrings, - add_start_docstrings_to_model_forward, - is_torch_fx_proxy, - logging, - replace_return_docstrings, - ) -except ImportError: - from transformers.utils import add_start_docstrings, add_start_docstrings_to_model_forward, logging - DUMMY_INPUTS = [[7, 6, 0, 0, 1], [1, 2, 3, 0, 0], [0, 0, 0, 4, 5]] - DUMMY_MASK = [[1, 1, 0, 0, 1], [1, 1, 1, 0, 0], [1, 0, 0, 1, 1]] - is_torch_fx_proxy = lambda x: False - replace_return_docstrings = lambda *a, **kw: (lambda f: f) - -# Try to import model parallel utils (removed in transformers 5.0+) -try: - from transformers.utils.model_parallel_utils import assert_device_map, get_device_map -except ImportError: - # Fallback implementations for transformers 5.0+ - def get_device_map(num_blocks, num_devices): - """Distribute blocks across devices.""" - device_map = {} - blocks_per_device = num_blocks // num_devices - remainder = num_blocks % num_devices - for i in range(num_blocks): - device_id = min(i // (blocks_per_device + 1), num_devices - 1) if remainder > 0 else i // max(1, blocks_per_device) - device_map[i] = device_id - return device_map - - def assert_device_map(device_map, num_blocks): - """Validate device map.""" - if device_map is None: - return - for block_idx in range(num_blocks): - if block_idx not in device_map: - raise ValueError(f"Block {block_idx} not in device_map") - +from transformers.pytorch_utils import ALL_LAYERNORM_LAYERS, find_pruneable_heads_and_indices, prune_linear_layer +from transformers.utils import ( + DUMMY_INPUTS, + DUMMY_MASK, + add_start_docstrings, + add_start_docstrings_to_model_forward, + is_torch_fx_proxy, + logging, + replace_return_docstrings, +) +from transformers.utils.model_parallel_utils import assert_device_map, get_device_map from transformers.models.t5.configuration_t5 import T5Config import loralib as lora @@ -516,9 +470,6 @@ class T5Attention(nn.Module): def prune_heads(self, heads): if len(heads) == 0: return - if not HAS_PRUNING_UTILS: - # Pruning not available in transformers 5.0+, skip - return heads, index = find_pruneable_heads_and_indices( heads, self.n_heads, self.key_value_proj_dim, self.pruned_heads ) @@ -1055,25 +1006,9 @@ class T5PreTrainedModel(PreTrainedModel): if module.has_relative_attention_bias: module.relative_attention_bias.weight.data.normal_(mean=0.0, std=factor * ((d_model) ** -0.5)) - def get_head_mask(self, head_mask, num_hidden_layers, is_attention_chunked=False): - """Fallback for transformers 5.0+ where get_head_mask was removed from PreTrainedModel.""" - if head_mask is not None: - head_mask = self._convert_head_mask_to_5d(head_mask, num_hidden_layers) - if is_attention_chunked is True: - head_mask = head_mask.unsqueeze(-1) - else: - head_mask = [None] * num_hidden_layers - return head_mask - - def _convert_head_mask_to_5d(self, head_mask, num_hidden_layers): - if head_mask.dim() == 1: - head_mask = head_mask.unsqueeze(0).unsqueeze(0).unsqueeze(-1).unsqueeze(-1) - head_mask = head_mask.expand(num_hidden_layers, -1, -1, -1, -1) - elif head_mask.dim() == 2: - head_mask = head_mask.unsqueeze(1).unsqueeze(-1).unsqueeze(-1) - assert head_mask.dim() == 5, f"head_mask.dim != 5, instead {head_mask.dim()}" - head_mask = head_mask.to(dtype=self.dtype) - return head_mask + def _set_gradient_checkpointing(self, module, value=False): + if isinstance(module, (T5Attention, T5Stack)): + module.gradient_checkpointing = value def _shift_right(self, input_ids): decoder_start_token_id = self.config.decoder_start_token_id @@ -1156,23 +1091,6 @@ class T5Stack(T5PreTrainedModel): self.all_attn_weights = [] self.key_attention_weights = None - # ===== OT-SIGN: vMF signatures + OT routing ===== - self.use_ot_routing = prompt_config.get('use_ot_routing', False) - self.ot_epsilon = prompt_config.get('ot_epsilon', 0.05) - self.ot_n_iter = prompt_config.get('ot_n_iter', 10) - self.default_kappa = prompt_config.get('default_kappa', 10.0) - self.vmf_signatures = {} # {task_id: (mu, kappa)} - self.last_x_normalized = None # stored for anti-invasion in trainer - - # Load vMF signatures from previous tasks if available - vmf_sig_path = prompt_config.get('previous_vmf_signatures_path', None) - if vmf_sig_path is not None and os.path.exists(vmf_sig_path): - print("----------Loading Previous vMF Signatures----------") - loaded_sigs = torch.load(vmf_sig_path, map_location='cpu') - self.vmf_signatures = loaded_sigs - print(f" Loaded {len(loaded_sigs)} vMF signatures") - # ===== End OT-SIGN init ===== - self.final_layer_norm = T5LayerNorm(config.d_model, eps=config.layer_norm_epsilon) self.dropout = nn.Dropout(config.dropout_rate) @@ -1183,19 +1101,6 @@ class T5Stack(T5PreTrainedModel): self.device_map = None self.gradient_checkpointing = False - @property - def dtype(self): - """ - Override dtype property to handle StopIteration in transformers 5.0. - If model has no floating-point parameters (edge case in DataParallel), - default to torch.float32. - """ - try: - return next(param.dtype for param in self.parameters() if param.is_floating_point()) - except StopIteration: - # Fallback to float32 if no floating-point parameters found - return torch.float32 - def get_chunk(self, chunk): self.chunk_trans = chunk @@ -1206,13 +1111,7 @@ class T5Stack(T5PreTrainedModel): self.matrix_trans_1[index] = torch.zeros(self.step_trans, self.step_trans).cuda() self.matrix_trans_3[index] = torch.zeros(self.step_trans, self.step_trans).cuda() self.n_trans_matrix[index] = 0 - # Handle meta tensors (device_map may create tensors on meta device) - if self.matrix_trans_2 is not None: - if str(self.matrix_trans_2.device) != 'meta': - self.matrix_trans_2 = self.matrix_trans_2.cuda() - else: - # Re-initialize on actual device - self.matrix_trans_2 = torch.zeros(self.config.d_model, self.config.d_model).cuda() + self.matrix_trans_2 = self.matrix_trans_2.cuda() def get_matrix3(self, x, medium, x_final): for index in range(self.index_trans): @@ -1230,13 +1129,7 @@ class T5Stack(T5PreTrainedModel): if self.matrix_trans_2 is None: self.matrix_trans_2 = torch.bmm(medium.detach().permute(0, 2, 1), medium.detach()).sum(dim=0).float()/(medium.shape[0]*medium.shape[1]) else: - # Check shape compatibility before accumulating - new_matrix = torch.bmm(medium.detach().permute(0, 2, 1), medium.detach()).sum(dim=0).float()/(medium.shape[0]*medium.shape[1]) - if self.matrix_trans_2.shape == new_matrix.shape: - self.matrix_trans_2 = (self.matrix_trans_2*self.n_trans_matrix[index] + new_matrix)/(self.n_trans_matrix[index] + medium.shape[0]*medium.shape[1]) - else: - # Shape mismatch, reinitialize with correct shape - self.matrix_trans_2 = new_matrix + self.matrix_trans_2 = (self.matrix_trans_2*self.n_trans_matrix[index] + torch.bmm(medium.detach().permute(0, 2, 1), medium.detach()).sum(dim=0).float())/(self.n_trans_matrix[index] + medium.shape[0]*medium.shape[1]) return @@ -1338,34 +1231,6 @@ class T5Stack(T5PreTrainedModel): x_inv_norm = torch.rsqrt(torch.maximum(square_sum, torch.tensor(epsilon, device=x.device))) return x * x_inv_norm - def cal_attention_ot(self, x_normalized): - """ - OT-SIGN routing using vMF signatures + Sinkhorn OT. - - Args: - x_normalized: Normalized input features (B, 1, d) - - Returns: - weights: Transport plan (B, N_total, 1) — same shape as original cal_attention - """ - x = x_normalized.squeeze(1) # (B, d) - - # Build cost matrix from vMF signatures + current task's prompt_key - C = vmf_cost_matrix( - x, self.vmf_signatures, - prompt_key=self.prompt_key, - default_kappa=self.default_kappa - ) # (B, N_total) - - # Sinkhorn OT (log-domain for FP16 stability) - weights = sinkhorn_log( - C.float(), # compute in FP32 for stability - epsilon=self.ot_epsilon, - n_iter=self.ot_n_iter - ).to(dtype=x.dtype) # (B, N_total) - - return weights.unsqueeze(2) # (B, N_total, 1) - def get_prompt(self): return self.prompt_encoder(self.prompt) + self.prompt @@ -1421,37 +1286,15 @@ class T5Stack(T5PreTrainedModel): # ipdb.set_trace() if not self.is_decoder and not self.prompt_config["run_single"]: self.reduce_sim = None - self.last_x_normalized = None # reset per forward prompt_key = self.prompt_key - - # ===== OT-SIGN routing branch ===== - if self.use_ot_routing and len(self.vmf_signatures) > 0: - # Use SINGLE trans_input for all tasks (no previous_trans_input needed) - avg_inputs_embeds = (attention_mask.unsqueeze(-1)*inputs_embeds).mean(dim=1, keepdim=True) - medium = self.trans_input[1](self.trans_input[0](avg_inputs_embeds)) - x = self.trans_input[3](self.trans_input[2](medium)) - - if self.get_trans_feature: - self.get_matrix3(avg_inputs_embeds, medium, x) - - # Normalize for vMF routing - x_normalized = F.normalize(x, dim=-1) # (B, 1, d) - self.last_x_normalized = x_normalized.detach() # store for anti-invasion - - # OT routing with vMF signatures - key_attention_weights = self.cal_attention_ot(x_normalized) # (B, N_total, 1) - - if self.is_inference: - self.all_attn_weights.append( - key_attention_weights.squeeze().mean(dim=0, keepdim=True).detach().to(torch.float).cpu().numpy() - ) - - # ===== Original routing branch (fallback) ===== - elif self.previous_prompts_keys is not None: + if self.previous_prompts_keys is not None: prompt_key = self.prompt_key.to(prompt_key.device) past_prompt_key = torch.cat([prompt_key.repeat(batch_size, 1, 1), self.previous_prompts_keys.repeat(batch_size, 1, 1)], dim=1) + # avg_inputs_embeds = inputs_embeds.max(dim=1, keepdim=True).values + # avg_inputs_embeds = inputs_embeds.mean(dim=1, keepdim=True) avg_inputs_embeds = (attention_mask.unsqueeze(-1)*inputs_embeds).mean(dim=1, keepdim=True) + # x = self.trans_input(avg_inputs_embeds) medium = self.trans_input[1](self.trans_input[0](avg_inputs_embeds)) x = self.trans_input[3](self.trans_input[2](medium)) @@ -1461,10 +1304,15 @@ class T5Stack(T5PreTrainedModel): past_x = torch.cat([x, self.previous_trans_input(avg_inputs_embeds)], dim=1) key_attention_weights = self.cal_attention(past_prompt_key, past_x) + # if not self.training: print(key_attention_weights[:,0,0].detach()) + + # print(key_attention_weights.squeeze().mean(dim=0).detach().to(torch.float).cpu().numpy()) if self.is_inference: self.all_attn_weights.append(key_attention_weights.squeeze().mean(dim=0, keepdim=True).detach().to(torch.float).cpu().numpy()) else: + # avg_inputs_embeds = inputs_embeds.max(dim=1, keepdim=True).values + # avg_inputs_embeds = inputs_embeds.mean(dim=1, keepdim=True) avg_inputs_embeds = (attention_mask.unsqueeze(-1)*inputs_embeds).mean(dim=1, keepdim=True) medium = self.trans_input[1](self.trans_input[0](avg_inputs_embeds)) x = self.trans_input[3](self.trans_input[2](medium)) @@ -1476,8 +1324,10 @@ class T5Stack(T5PreTrainedModel): if self.is_inference: self.all_attn_weights.append(key_attention_weights.squeeze(2).mean(dim=0, keepdim=True).detach().to(torch.float).cpu().numpy()) + # self.key_attention_weights = torch.ones_like(key_attention_weights) self.key_attention_weights = key_attention_weights else: + # key_attention_weights = torch.ones_like(key_attention_weights) key_attention_weights = key_attention_weights # required mask seq length can be calculated via length of past @@ -1576,7 +1426,6 @@ class T5Stack(T5PreTrainedModel): layer_head_mask, cross_attn_layer_head_mask, None, # past_key_value is always None with gradient checkpointing - use_reentrant=False, ) else: layer_outputs = layer_module( @@ -2015,17 +1864,8 @@ class T5Model(T5PreTrainedModel): encoder_attentions=encoder_outputs.attentions, ) -# Import GenerationMixin for transformers 5.0+ compatibility -try: - from transformers.generation.utils import GenerationMixin as _GenerationMixin -except ImportError: - try: - from transformers import GenerationMixin as _GenerationMixin - except ImportError: - _GenerationMixin = object - @add_start_docstrings("""T5 Model with a `language modeling` head on top.""", T5_START_DOCSTRING) -class T5ForConditionalGeneration(T5PreTrainedModel, _GenerationMixin): +class T5ForConditionalGeneration(T5PreTrainedModel): _keys_to_ignore_on_load_missing = [ r"encoder.embed_tokens.weight", r"decoder.embed_tokens.weight", diff --git a/gainlora_OT_vMF/src/t5_inflora.py b/root_gainlora/src/t5_inflora.py similarity index 96% rename from gainlora_OT_vMF/src/t5_inflora.py rename to root_gainlora/src/t5_inflora.py index dae7773f354dde35f93a78b0202c3d622391ceda..5cbc857fb1f33aa37b55af9151ad4fe920a46d3e 100644 --- a/gainlora_OT_vMF/src/t5_inflora.py +++ b/root_gainlora/src/t5_inflora.py @@ -38,56 +38,17 @@ from transformers.modeling_outputs import ( Seq2SeqModelOutput, ) from transformers.modeling_utils import PreTrainedModel -from transformers.pytorch_utils import ALL_LAYERNORM_LAYERS - -# Try to import pruning utilities (not available in transformers 5.0+) -try: - from transformers.pytorch_utils import find_pruneable_heads_and_indices, prune_linear_layer - HAS_PRUNING_UTILS = True -except ImportError: - HAS_PRUNING_UTILS = False - -# Try to import utils removed in transformers 5.0+ -try: - from transformers.utils import ( - DUMMY_INPUTS, - DUMMY_MASK, - add_start_docstrings, - add_start_docstrings_to_model_forward, - is_torch_fx_proxy, - logging, - replace_return_docstrings, - ) -except ImportError: - from transformers.utils import add_start_docstrings, add_start_docstrings_to_model_forward, logging - DUMMY_INPUTS = [[7, 6, 0, 0, 1], [1, 2, 3, 0, 0], [0, 0, 0, 4, 5]] - DUMMY_MASK = [[1, 1, 0, 0, 1], [1, 1, 1, 0, 0], [1, 0, 0, 1, 1]] - is_torch_fx_proxy = lambda x: False - replace_return_docstrings = lambda *a, **kw: (lambda f: f) - -# Try to import model parallel utils (removed in transformers 5.0+) -try: - from transformers.utils.model_parallel_utils import assert_device_map, get_device_map -except ImportError: - # Fallback implementations for transformers 5.0+ - def get_device_map(num_blocks, num_devices): - """Distribute blocks across devices.""" - device_map = {} - blocks_per_device = num_blocks // num_devices - remainder = num_blocks % num_devices - for i in range(num_blocks): - device_id = min(i // (blocks_per_device + 1), num_devices - 1) if remainder > 0 else i // max(1, blocks_per_device) - device_map[i] = device_id - return device_map - - def assert_device_map(device_map, num_blocks): - """Validate device map.""" - if device_map is None: - return - for block_idx in range(num_blocks): - if block_idx not in device_map: - raise ValueError(f"Block {block_idx} not in device_map") - +from transformers.pytorch_utils import ALL_LAYERNORM_LAYERS, find_pruneable_heads_and_indices, prune_linear_layer +from transformers.utils import ( + DUMMY_INPUTS, + DUMMY_MASK, + add_start_docstrings, + add_start_docstrings_to_model_forward, + is_torch_fx_proxy, + logging, + replace_return_docstrings, +) +from transformers.utils.model_parallel_utils import assert_device_map, get_device_map from transformers.models.t5.configuration_t5 import T5Config import loralib as lora @@ -467,19 +428,6 @@ class T5Attention(nn.Module): self.matrix = torch.zeros(self.d_model, self.d_model) self.n_matrix = 0 - @property - def dtype(self): - """ - Override dtype property to handle StopIteration in transformers 5.0. - If model has no floating-point parameters (edge case in DataParallel), - default to torch.float32. - """ - try: - return next(param.dtype for param in self.parameters() if param.is_floating_point()) - except StopIteration: - # Fallback to float32 if no floating-point parameters found - return torch.float32 - def get_chunk(self, chunk): self.chunk = chunk @@ -501,9 +449,6 @@ class T5Attention(nn.Module): def prune_heads(self, heads): if len(heads) == 0: return - if not HAS_PRUNING_UTILS: - # Pruning not available in transformers 5.0+, skip - return heads, index = find_pruneable_heads_and_indices( heads, self.n_heads, self.key_value_proj_dim, self.pruned_heads ) @@ -1039,25 +984,9 @@ class T5PreTrainedModel(PreTrainedModel): if module.has_relative_attention_bias: module.relative_attention_bias.weight.data.normal_(mean=0.0, std=factor * ((d_model) ** -0.5)) - def get_head_mask(self, head_mask, num_hidden_layers, is_attention_chunked=False): - """Fallback for transformers 5.0+ where get_head_mask was removed from PreTrainedModel.""" - if head_mask is not None: - head_mask = self._convert_head_mask_to_5d(head_mask, num_hidden_layers) - if is_attention_chunked is True: - head_mask = head_mask.unsqueeze(-1) - else: - head_mask = [None] * num_hidden_layers - return head_mask - - def _convert_head_mask_to_5d(self, head_mask, num_hidden_layers): - if head_mask.dim() == 1: - head_mask = head_mask.unsqueeze(0).unsqueeze(0).unsqueeze(-1).unsqueeze(-1) - head_mask = head_mask.expand(num_hidden_layers, -1, -1, -1, -1) - elif head_mask.dim() == 2: - head_mask = head_mask.unsqueeze(1).unsqueeze(-1).unsqueeze(-1) - assert head_mask.dim() == 5, f"head_mask.dim != 5, instead {head_mask.dim()}" - head_mask = head_mask.to(dtype=self.dtype) - return head_mask + def _set_gradient_checkpointing(self, module, value=False): + if isinstance(module, (T5Attention, T5Stack)): + module.gradient_checkpointing = value def _shift_right(self, input_ids): decoder_start_token_id = self.config.decoder_start_token_id @@ -1397,7 +1326,6 @@ class T5Stack(T5PreTrainedModel): layer_head_mask, cross_attn_layer_head_mask, None, # past_key_value is always None with gradient checkpointing - use_reentrant=False, ) else: layer_outputs = layer_module( @@ -1836,17 +1764,8 @@ class T5Model(T5PreTrainedModel): encoder_attentions=encoder_outputs.attentions, ) -# Import GenerationMixin for transformers 5.0+ compatibility -try: - from transformers.generation.utils import GenerationMixin as _GenerationMixin -except ImportError: - try: - from transformers import GenerationMixin as _GenerationMixin - except ImportError: - _GenerationMixin = object - @add_start_docstrings("""T5 Model with a `language modeling` head on top.""", T5_START_DOCSTRING) -class T5ForConditionalGeneration(T5PreTrainedModel, _GenerationMixin): +class T5ForConditionalGeneration(T5PreTrainedModel): _keys_to_ignore_on_load_missing = [ r"encoder.embed_tokens.weight", r"decoder.embed_tokens.weight",