feat: add T5_small benchmark scripts for 4 comparison scenarios

- Create root_gainlora/T5_small/ with 8 scripts (inflora + gainlora_inflora
for long_order3, long_order4, superni_order1, superni_order2)
- Create improve_gainlora/T5_small/ with 12 scripts (same + specroute)
- Model: google/flan-t5-small (passed as $2 arg, no code changes needed)
- Long sequence: train_bsz=32, ga=1, eval_bsz=256 (from bsz=8-16, ga=2-4)
- SuperNI: train_bsz=16, ga=2, eval_bsz=8
- Specroute T4 1GPU: BSZ=32; GA=1; EVAL_BSZ=256 (long); BSZ=16; GA=2 (superni)
- Removed --gradient_checkpointing (flan-t5-small fits in ~2GB VRAM)
- All output_dir/run_name/checkpoint paths updated with t5_small_ prefix
- Added gen_t5_small_scripts.py for reproducibility

gen_t5_small_scripts.py

@@ -0,0 +1,196 @@
+#!/usr/bin/env python3
+"""
+Generate T5_small benchmark scripts from original T5 scripts.
+
+For all scripts:
+  - Rename experiment: t5_METHOD -> t5_small_METHOD  (output_dir, run_name, checkpoint paths)
+  - Remove --gradient_checkpointing standalone flag (not needed for small model)
+
+Batch sizes (flan-t5-small fits easily on T4 with large batches):
+  Long sequence non-specroute : train=32, ga=1, eval=256
+  SuperNI non-specroute        : train=16, ga=2, eval=8
+
+Specroute GPU-mode blocks (long):
+  t4_2gpu : BSZ=16; GA=1; EVAL_BSZ=256
+  t4_1gpu : BSZ=32; GA=1; EVAL_BSZ=256
+  a100    : BSZ=64; GA=1; EVAL_BSZ=512
+
+Specroute GPU-mode blocks (superni):
+  t4_2gpu : BSZ=8 ; GA=2; EVAL_BSZ=16
+  t4_1gpu : BSZ=16; GA=2; EVAL_BSZ=16
+  a100    : BSZ=32; GA=1; EVAL_BSZ=32
+"""
+
+import re
+import os
+from pathlib import Path
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+SPECROUTE_LONG_MODES = {
+    "t4_2gpu": "BSZ=16; GA=1; EVAL_BSZ=256",
+    "t4_1gpu": "BSZ=32; GA=1; EVAL_BSZ=256",
+    "a100":    "BSZ=64; GA=1; EVAL_BSZ=512",
+}
+
+SPECROUTE_SUPERNI_MODES = {
+    "t4_2gpu": "BSZ=8;  GA=2; EVAL_BSZ=16",
+    "t4_1gpu": "BSZ=16; GA=2; EVAL_BSZ=16",
+    "a100":    "BSZ=32; GA=1; EVAL_BSZ=32",
+}
+
+
+def replace_experiment_names(content: str) -> str:
+    """Replace gen_script_X_t5_METHOD  →  gen_script_X_t5_small_METHOD."""
+    # Works in output_dir, run_name, checkpoint paths
+    return re.sub(
+        r'(gen_script_(?:long_order[34]|superni_order[12])_t5_)(?!small_)',
+        r'\1small_',
+        content,
+    )
+
+
+def remove_gradient_checkpointing_flag(content: str) -> str:
+    """Remove   --gradient_checkpointing \\   as a standalone argument line."""
+    # Matches lines like:  "   --gradient_checkpointing \\\n"  or  "   --gradient_checkpointing\n"
+    content = re.sub(r'[ \t]*--gradient_checkpointing \\\n', '', content)
+    content = re.sub(r'[ \t]*--gradient_checkpointing\n', '', content)
+    return content
+
+
+def set_non_specroute_batch_sizes(content: str, script_type: str) -> str:
+    """Replace hardcoded --per_device_* and --gradient_accumulation_steps."""
+    if script_type == 'long':
+        train_bsz, ga, eval_bsz = 32, 1, 256
+    else:  # superni
+        train_bsz, ga, eval_bsz = 16, 2, 8
+
+    content = re.sub(r'--per_device_train_batch_size \d+',
+                     f'--per_device_train_batch_size {train_bsz}', content)
+    content = re.sub(r'--per_device_eval_batch_size \d+',
+                     f'--per_device_eval_batch_size {eval_bsz}', content)
+    content = re.sub(r'--gradient_accumulation_steps \d+',
+                     f'--gradient_accumulation_steps {ga}', content)
+    return content
+
+
+def fix_specroute_gpu_modes(content: str, script_type: str) -> str:
+    """Replace BSZ/GA/EVAL_BSZ inside the GPU-mode if/elif/else block."""
+    modes = SPECROUTE_LONG_MODES if script_type == 'long' else SPECROUTE_SUPERNI_MODES
+
+    # t4_2gpu block: "    BSZ=N; GA=N; EVAL_BSZ=N"
+    content = re.sub(
+        r'(if \[ "\$GPU_MODE" = "t4_2gpu" \]; then\n)[ \t]*BSZ=\d+; GA=\d+; EVAL_BSZ=\d+',
+        r'\g<1>    ' + modes['t4_2gpu'],
+        content,
+    )
+    # t4_1gpu block
+    content = re.sub(
+        r'(elif \[ "\$GPU_MODE" = "t4_1gpu" \]; then\n)[ \t]*BSZ=\d+; GA=\d+; EVAL_BSZ=\d+',
+        r'\g<1>    ' + modes['t4_1gpu'],
+        content,
+    )
+    # a100 block (else)
+    content = re.sub(
+        r'(else\n)[ \t]*BSZ=\d+; GA=\d+; EVAL_BSZ=\d+',
+        r'\g<1>    ' + modes['a100'],
+        content,
+    )
+
+    # Remove gradient_checkpointing from FP16_FLAG  (set it to empty for all modes)
+    content = re.sub(r'FP16_FLAG="--gradient_checkpointing"', 'FP16_FLAG=""', content)
+    return content
+
+
+def transform(content: str, is_specroute: bool, script_type: str) -> str:
+    content = replace_experiment_names(content)
+    content = remove_gradient_checkpointing_flag(content)
+    if is_specroute:
+        content = fix_specroute_gpu_modes(content, script_type)
+    else:
+        content = set_non_specroute_batch_sizes(content, script_type)
+    return content
+
+
+def process_dir(src_dir: Path, dst_dir: Path, scripts_long: list, scripts_superni: list):
+    dst_dir.mkdir(exist_ok=True)
+
+    for script in scripts_long:
+        src = src_dir / script
+        if not src.exists():
+            print(f"  SKIP (not found): {src}")
+            continue
+        content = src.read_text()
+        is_specroute = 'specroute' in script
+        new_content = transform(content, is_specroute, 'long')
+        new_name = script.replace('_t5_', '_t5_small_')
+        dst = dst_dir / new_name
+        dst.write_text(new_content)
+        os.chmod(dst, 0o755)
+        print(f"  Created  {dst.relative_to(src_dir.parent.parent)}")
+
+    for script in scripts_superni:
+        src = src_dir / script
+        if not src.exists():
+            print(f"  SKIP (not found): {src}")
+            continue
+        content = src.read_text()
+        is_specroute = 'specroute' in script
+        new_content = transform(content, is_specroute, 'superni')
+        new_name = script.replace('_t5_', '_t5_small_')
+        dst = dst_dir / new_name
+        dst.write_text(new_content)
+        os.chmod(dst, 0o755)
+        print(f"  Created  {dst.relative_to(src_dir.parent.parent)}")
+
+
+# ---------------------------------------------------------------------------
+# Root gainlora
+# ---------------------------------------------------------------------------
+ROOT = Path('/Users/nnminh322/Desktop/personal/Continual/root_gainlora')
+
+ROOT_LONG = [
+    'gen_script_long_order3_t5_inflora.sh',
+    'gen_script_long_order3_t5_gainlora_inflora.sh',
+    'gen_script_long_order4_t5_inflora.sh',
+    'gen_script_long_order4_t5_gainlora_inflora.sh',
+]
+ROOT_SUPERNI = [
+    'gen_script_superni_order1_t5_inflora.sh',
+    'gen_script_superni_order1_t5_gainlora_inflora.sh',
+    'gen_script_superni_order2_t5_inflora.sh',
+    'gen_script_superni_order2_t5_gainlora_inflora.sh',
+]
+
+print("=== root_gainlora/T5_small/ ===")
+process_dir(ROOT, ROOT / 'T5_small', ROOT_LONG, ROOT_SUPERNI)
+
+# ---------------------------------------------------------------------------
+# Improve gainlora
+# ---------------------------------------------------------------------------
+IMPROVE = Path('/Users/nnminh322/Desktop/personal/Continual/improve_gainlora')
+
+IMPROVE_LONG = [
+    'gen_script_long_order3_t5_inflora.sh',
+    'gen_script_long_order3_t5_gainlora_inflora.sh',
+    'gen_script_long_order3_t5_specroute.sh',
+    'gen_script_long_order4_t5_inflora.sh',
+    'gen_script_long_order4_t5_gainlora_inflora.sh',
+    'gen_script_long_order4_t5_specroute.sh',
+]
+IMPROVE_SUPERNI = [
+    'gen_script_superni_order1_t5_inflora.sh',
+    'gen_script_superni_order1_t5_gainlora_inflora.sh',
+    'gen_script_superni_order1_t5_specroute.sh',
+    'gen_script_superni_order2_t5_inflora.sh',
+    'gen_script_superni_order2_t5_gainlora_inflora.sh',
+    'gen_script_superni_order2_t5_specroute.sh',
+]
+
+print("\n=== improve_gainlora/T5_small/ ===")
+process_dir(IMPROVE, IMPROVE / 'T5_small', IMPROVE_LONG, IMPROVE_SUPERNI)
+
+print("\nDone!")

improve_gainlora/T5_small/gen_script_long_order3_t5_small_gainlora_inflora.sh

@@ -0,0 +1,763 @@
+#!/bin/bash
+#SBATCH -J cl                           
+#SBATCH -o cl-%j.out                       
+#SBATCH -p compute 
+#SBATCH -N 1                           
+#SBATCH -t 20:00:00   
+#SBATCH --mem 128G 
+#SBATCH --gres=gpu:a100-sxm4-80gb:1  
+
+export CUDA_DEVICE_ORDER="PCI_BUS_ID"
+
+port=$(shuf -i25000-30000 -n1)  
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/yelp \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/1-yelp \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/1-yelp/checkpoint*
+
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/1-yelp/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/1-yelp/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/1-yelp/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/amazon \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/2-amazon \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_amazon \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/2-amazon/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/2-amazon/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/1-yelp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/2-amazon/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/2-amazon/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/mnli \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/3-mnli \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_mnli \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/3-mnli/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/3-mnli/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/1-yelp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/2-amazon/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/3-mnli/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/3-mnli/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/cb \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/4-cb \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_cb \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/4-cb/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/4-cb/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/1-yelp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/2-amazon/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/3-mnli/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/4-cb/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/4-cb/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/copa \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/5-copa \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_copa \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/5-copa/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/5-copa/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/1-yelp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/2-amazon/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/3-mnli/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/4-cb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/5-copa/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/5-copa/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/qqp \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/6-qqp \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_qqp \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/6-qqp/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/6-qqp/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/1-yelp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/2-amazon/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/3-mnli/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/4-cb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/5-copa/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/6-qqp/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/6-qqp/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/rte \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/7-rte \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_rte \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/7-rte/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/7-rte/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/1-yelp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/2-amazon/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/3-mnli/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/4-cb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/5-copa/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/6-qqp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/7-rte/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/7-rte/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/imdb \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/8-imdb \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_imdb \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/8-imdb/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/8-imdb/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/1-yelp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/2-amazon/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/3-mnli/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/4-cb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/5-copa/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/6-qqp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/8-imdb/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/8-imdb/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/sst2 \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/9-sst2 \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_sst2 \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/9-sst2/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/9-sst2/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/1-yelp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/2-amazon/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/3-mnli/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/4-cb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/5-copa/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/6-qqp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/8-imdb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/9-sst2/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/9-sst2/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/dbpedia \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/10-dbpedia \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_dbpedia \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/10-dbpedia/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/10-dbpedia/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/1-yelp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/2-amazon/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/3-mnli/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/4-cb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/5-copa/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/6-qqp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/8-imdb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/9-sst2/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/10-dbpedia/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/10-dbpedia/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/agnews \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/11-agnews \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_agnews \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/11-agnews/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/11-agnews/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/1-yelp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/2-amazon/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/3-mnli/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/4-cb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/5-copa/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/6-qqp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/8-imdb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/9-sst2/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/10-dbpedia/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/11-agnews/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/11-agnews/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/yahoo \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/12-yahoo \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_yahoo \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/12-yahoo/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/12-yahoo/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/1-yelp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/2-amazon/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/3-mnli/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/4-cb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/5-copa/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/6-qqp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/8-imdb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/9-sst2/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/10-dbpedia/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/11-agnews/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/12-yahoo/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/12-yahoo/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/multirc \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/13-multirc \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_multirc \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/13-multirc/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/13-multirc/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/1-yelp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/2-amazon/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/3-mnli/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/4-cb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/5-copa/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/6-qqp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/8-imdb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/9-sst2/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/10-dbpedia/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/11-agnews/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/12-yahoo/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/13-multirc/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/13-multirc/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/boolq \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/14-boolq \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_boolq \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/14-boolq/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/14-boolq/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/1-yelp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/2-amazon/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/3-mnli/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/4-cb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/5-copa/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/6-qqp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/8-imdb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/9-sst2/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/10-dbpedia/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/11-agnews/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/12-yahoo/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/13-multirc/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/14-boolq/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/14-boolq/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/wic \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/15-wic \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_wic \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/15-wic/checkpoint*
+   
+sleep 5
+
+CUDA_VISIBLE_DEVICES=$1 python score.py gen_script_long_order3_t5_small_gainlora_inflora gen_script_long_order3_t5_small_gainlora_inflora
+
+
+

improve_gainlora/T5_small/gen_script_long_order3_t5_small_inflora.sh

@@ -0,0 +1,744 @@
+#!/bin/bash
+#SBATCH -J cl                           
+#SBATCH -o cl-%j.out                       
+#SBATCH -p compute 
+#SBATCH -N 1                           
+#SBATCH -t 20:00:00   
+#SBATCH --mem 128G 
+#SBATCH --gres=gpu:a100-sxm4-80gb:1  
+
+export CUDA_DEVICE_ORDER="PCI_BUS_ID"
+
+port=$(shuf -i25000-30000 -n1)  
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/yelp \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/1-yelp \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --model_name inflora \
+   --threshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/1-yelp/checkpoint*
+
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/1-yelp/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/1-yelp/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/1-yelp/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/amazon \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/2-amazon \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_amazon \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/2-amazon/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/2-amazon/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/1-yelp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/2-amazon/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/2-amazon/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/mnli \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/3-mnli \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_mnli \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/3-mnli/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/3-mnli/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/1-yelp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/2-amazon/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/3-mnli/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/3-mnli/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/cb \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/4-cb \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_cb \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/4-cb/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/4-cb/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/1-yelp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/2-amazon/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/3-mnli/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/4-cb/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/4-cb/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/copa \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/5-copa \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_copa \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/5-copa/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/5-copa/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/1-yelp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/2-amazon/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/3-mnli/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/4-cb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/5-copa/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/5-copa/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/qqp \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/6-qqp \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_qqp \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/6-qqp/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/6-qqp/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/1-yelp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/2-amazon/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/3-mnli/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/4-cb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/5-copa/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/6-qqp/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/6-qqp/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/rte \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/7-rte \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_rte \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/7-rte/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/7-rte/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/1-yelp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/2-amazon/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/3-mnli/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/4-cb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/5-copa/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/6-qqp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/7-rte/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/7-rte/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/imdb \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/8-imdb \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_imdb \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/8-imdb/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/8-imdb/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/1-yelp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/2-amazon/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/3-mnli/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/4-cb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/5-copa/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/6-qqp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/8-imdb/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/8-imdb/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/sst2 \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/9-sst2 \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_sst2 \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/9-sst2/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/9-sst2/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/1-yelp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/2-amazon/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/3-mnli/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/4-cb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/5-copa/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/6-qqp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/8-imdb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/9-sst2/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/9-sst2/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/dbpedia \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/10-dbpedia \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_dbpedia \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/10-dbpedia/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/10-dbpedia/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/1-yelp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/2-amazon/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/3-mnli/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/4-cb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/5-copa/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/6-qqp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/8-imdb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/9-sst2/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/10-dbpedia/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/10-dbpedia/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/agnews \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/11-agnews \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_agnews \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/11-agnews/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/11-agnews/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/1-yelp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/2-amazon/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/3-mnli/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/4-cb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/5-copa/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/6-qqp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/8-imdb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/9-sst2/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/10-dbpedia/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/11-agnews/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/11-agnews/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/yahoo \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/12-yahoo \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_yahoo \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/12-yahoo/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/12-yahoo/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/1-yelp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/2-amazon/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/3-mnli/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/4-cb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/5-copa/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/6-qqp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/8-imdb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/9-sst2/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/10-dbpedia/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/11-agnews/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/12-yahoo/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/12-yahoo/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/multirc \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/13-multirc \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_multirc \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/13-multirc/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/13-multirc/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/1-yelp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/2-amazon/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/3-mnli/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/4-cb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/5-copa/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/6-qqp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/8-imdb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/9-sst2/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/10-dbpedia/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/11-agnews/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/12-yahoo/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/13-multirc/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/13-multirc/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/boolq \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/14-boolq \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_boolq \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/14-boolq/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/14-boolq/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/1-yelp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/2-amazon/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/3-mnli/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/4-cb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/5-copa/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/6-qqp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/8-imdb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/9-sst2/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/10-dbpedia/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/11-agnews/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/12-yahoo/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/13-multirc/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/14-boolq/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/14-boolq/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/wic \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/15-wic \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_wic \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/15-wic/checkpoint*
+   
+sleep 5
+
+CUDA_VISIBLE_DEVICES=$1 python score.py gen_script_long_order3_t5_small_inflora gen_script_long_order3_t5_small_inflora

improve_gainlora/T5_small/gen_script_long_order3_t5_small_specroute.sh

@@ -0,0 +1,849 @@
+#!/bin/bash
+#SBATCH -J cl
+#SBATCH -o cl-%j.out
+#SBATCH -p compute
+#SBATCH -N 1
+#SBATCH -t 20:00:00
+#SBATCH --mem 128G
+#SBATCH --gres=gpu:2
+
+export CUDA_DEVICE_ORDER="PCI_BUS_ID"
+
+port=$(shuf -i25000-30000 -n1)
+
+# ============================================================
+# Auto-detect GPU count and type for optimal parallelism
+# ============================================================
+NUM_GPUS=$(nvidia-smi -L 2>/dev/null | wc -l)
+GPU_MEM=$(nvidia-smi --query-gpu=memory.total --format=csv,noheader,nounits 2>/dev/null | head -1)
+
+if [ -z "$GPU_MEM" ]; then
+    echo "ERROR: No GPU detected!"
+    exit 1
+fi
+
+# Determine GPU type
+if [ "$GPU_MEM" -lt 20000 ]; then
+    IS_T4=1
+    echo "[GPU] Detected T4 GPUs (${GPU_MEM}MB VRAM each)"
+else
+    IS_T4=0
+    echo "[GPU] Detected high-memory GPUs (${GPU_MEM}MB VRAM each)"
+fi
+
+# Determine parallelism strategy
+if [ "$IS_T4" -eq 1 ] && [ "$NUM_GPUS" -ge 2 ]; then
+    GPU_MODE="t4_2gpu"
+    GPU_IDS="0,1"
+    FP16_FLAG=""
+    echo "[GPU] Strategy: 2x T4 DataParallel + fp32 + gradient_checkpointing"
+elif [ "$IS_T4" -eq 1 ]; then
+    GPU_MODE="t4_1gpu"
+    GPU_IDS="${1:-0}"
+    FP16_FLAG=""
+    echo "[GPU] Strategy: 1x T4 + fp32 + gradient_checkpointing"
+else
+    GPU_MODE="a100"
+    GPU_IDS="${1:-0}"
+    FP16_FLAG=""
+    echo "[GPU] Strategy: A100 (single GPU, fp32)"
+fi
+
+echo "[GPU] Using CUDA_VISIBLE_DEVICES=$GPU_IDS"
+echo "============================================================"
+echo ""
+
+if [ "$GPU_MODE" = "t4_2gpu" ]; then
+    BSZ=16; GA=1; EVAL_BSZ=256
+elif [ "$GPU_MODE" = "t4_1gpu" ]; then
+    BSZ=32; GA=1; EVAL_BSZ=256
+else
+    BSZ=64; GA=1; EVAL_BSZ=512
+fi
+
+CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
+   --do_train \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/yelp \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/1-yelp \
+   --per_device_train_batch_size $BSZ \
+   --per_device_eval_batch_size $EVAL_BSZ \
+   --gradient_accumulation_steps $GA \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_specroute \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --run_single True \
+   --data_replay_freq -1 \
+   --mlp_hidden_dim 100 \
+   --model_name specroute \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   $FP16_FLAG
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/1-yelp/checkpoint*
+
+sleep 5
+
+if [ "$GPU_MODE" = "t4_2gpu" ]; then
+    BSZ=16; GA=1; EVAL_BSZ=256
+elif [ "$GPU_MODE" = "t4_1gpu" ]; then
+    BSZ=32; GA=1; EVAL_BSZ=256
+else
+    BSZ=64; GA=1; EVAL_BSZ=512
+fi
+
+CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
+   --do_train \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/1-yelp/saved_weights \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/amazon \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/2-amazon \
+   --per_device_train_batch_size $BSZ \
+   --per_device_eval_batch_size $EVAL_BSZ \
+   --gradient_accumulation_steps $GA \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_specroute \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_amazon \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --mlp_hidden_dim 100 \
+   --model_name specroute \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   $FP16_FLAG
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/2-amazon/checkpoint*
+
+sleep 5
+
+if [ "$GPU_MODE" = "t4_2gpu" ]; then
+    BSZ=16; GA=1; EVAL_BSZ=256
+elif [ "$GPU_MODE" = "t4_1gpu" ]; then
+    BSZ=32; GA=1; EVAL_BSZ=256
+else
+    BSZ=64; GA=1; EVAL_BSZ=512
+fi
+
+CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
+   --do_train \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/1-yelp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/2-amazon/saved_weights \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/mnli \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/3-mnli \
+   --per_device_train_batch_size $BSZ \
+   --per_device_eval_batch_size $EVAL_BSZ \
+   --gradient_accumulation_steps $GA \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_specroute \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_mnli \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --mlp_hidden_dim 100 \
+   --model_name specroute \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   $FP16_FLAG
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/3-mnli/checkpoint*
+
+sleep 5
+
+if [ "$GPU_MODE" = "t4_2gpu" ]; then
+    BSZ=16; GA=1; EVAL_BSZ=256
+elif [ "$GPU_MODE" = "t4_1gpu" ]; then
+    BSZ=32; GA=1; EVAL_BSZ=256
+else
+    BSZ=64; GA=1; EVAL_BSZ=512
+fi
+
+CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
+   --do_train \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/1-yelp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/2-amazon/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/3-mnli/saved_weights \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/cb \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/4-cb \
+   --per_device_train_batch_size $BSZ \
+   --per_device_eval_batch_size $EVAL_BSZ \
+   --gradient_accumulation_steps $GA \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_specroute \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_cb \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --mlp_hidden_dim 100 \
+   --model_name specroute \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   $FP16_FLAG
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/4-cb/checkpoint*
+
+sleep 5
+
+if [ "$GPU_MODE" = "t4_2gpu" ]; then
+    BSZ=16; GA=1; EVAL_BSZ=256
+elif [ "$GPU_MODE" = "t4_1gpu" ]; then
+    BSZ=32; GA=1; EVAL_BSZ=256
+else
+    BSZ=64; GA=1; EVAL_BSZ=512
+fi
+
+CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
+   --do_train \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/1-yelp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/2-amazon/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/3-mnli/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/4-cb/saved_weights \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/copa \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/5-copa \
+   --per_device_train_batch_size $BSZ \
+   --per_device_eval_batch_size $EVAL_BSZ \
+   --gradient_accumulation_steps $GA \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_specroute \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_copa \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --mlp_hidden_dim 100 \
+   --model_name specroute \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   $FP16_FLAG
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/5-copa/checkpoint*
+
+sleep 5
+
+if [ "$GPU_MODE" = "t4_2gpu" ]; then
+    BSZ=16; GA=1; EVAL_BSZ=256
+elif [ "$GPU_MODE" = "t4_1gpu" ]; then
+    BSZ=32; GA=1; EVAL_BSZ=256
+else
+    BSZ=64; GA=1; EVAL_BSZ=512
+fi
+
+CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
+   --do_train \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/1-yelp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/2-amazon/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/3-mnli/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/4-cb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/5-copa/saved_weights \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/qqp \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/6-qqp \
+   --per_device_train_batch_size $BSZ \
+   --per_device_eval_batch_size $EVAL_BSZ \
+   --gradient_accumulation_steps $GA \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_specroute \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_qqp \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --mlp_hidden_dim 100 \
+   --model_name specroute \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   $FP16_FLAG
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/6-qqp/checkpoint*
+
+sleep 5
+
+if [ "$GPU_MODE" = "t4_2gpu" ]; then
+    BSZ=16; GA=1; EVAL_BSZ=256
+elif [ "$GPU_MODE" = "t4_1gpu" ]; then
+    BSZ=32; GA=1; EVAL_BSZ=256
+else
+    BSZ=64; GA=1; EVAL_BSZ=512
+fi
+
+CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
+   --do_train \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/1-yelp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/2-amazon/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/3-mnli/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/4-cb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/5-copa/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/6-qqp/saved_weights \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/rte \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/7-rte \
+   --per_device_train_batch_size $BSZ \
+   --per_device_eval_batch_size $EVAL_BSZ \
+   --gradient_accumulation_steps $GA \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_specroute \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_rte \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --mlp_hidden_dim 100 \
+   --model_name specroute \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   $FP16_FLAG
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/7-rte/checkpoint*
+
+sleep 5
+
+if [ "$GPU_MODE" = "t4_2gpu" ]; then
+    BSZ=16; GA=1; EVAL_BSZ=256
+elif [ "$GPU_MODE" = "t4_1gpu" ]; then
+    BSZ=32; GA=1; EVAL_BSZ=256
+else
+    BSZ=64; GA=1; EVAL_BSZ=512
+fi
+
+CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
+   --do_train \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/1-yelp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/2-amazon/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/3-mnli/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/4-cb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/5-copa/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/6-qqp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/7-rte/saved_weights \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/imdb \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/8-imdb \
+   --per_device_train_batch_size $BSZ \
+   --per_device_eval_batch_size $EVAL_BSZ \
+   --gradient_accumulation_steps $GA \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_specroute \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_imdb \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --mlp_hidden_dim 100 \
+   --model_name specroute \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   $FP16_FLAG
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/8-imdb/checkpoint*
+
+sleep 5
+
+if [ "$GPU_MODE" = "t4_2gpu" ]; then
+    BSZ=16; GA=1; EVAL_BSZ=256
+elif [ "$GPU_MODE" = "t4_1gpu" ]; then
+    BSZ=32; GA=1; EVAL_BSZ=256
+else
+    BSZ=64; GA=1; EVAL_BSZ=512
+fi
+
+CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
+   --do_train \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/1-yelp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/2-amazon/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/3-mnli/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/4-cb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/5-copa/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/6-qqp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/8-imdb/saved_weights \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/sst2 \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/9-sst2 \
+   --per_device_train_batch_size $BSZ \
+   --per_device_eval_batch_size $EVAL_BSZ \
+   --gradient_accumulation_steps $GA \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_specroute \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_sst2 \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --mlp_hidden_dim 100 \
+   --model_name specroute \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   $FP16_FLAG
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/9-sst2/checkpoint*
+
+sleep 5
+
+if [ "$GPU_MODE" = "t4_2gpu" ]; then
+    BSZ=16; GA=1; EVAL_BSZ=256
+elif [ "$GPU_MODE" = "t4_1gpu" ]; then
+    BSZ=32; GA=1; EVAL_BSZ=256
+else
+    BSZ=64; GA=1; EVAL_BSZ=512
+fi
+
+CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
+   --do_train \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/1-yelp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/2-amazon/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/3-mnli/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/4-cb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/5-copa/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/6-qqp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/8-imdb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/9-sst2/saved_weights \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/dbpedia \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/10-dbpedia \
+   --per_device_train_batch_size $BSZ \
+   --per_device_eval_batch_size $EVAL_BSZ \
+   --gradient_accumulation_steps $GA \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_specroute \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_dbpedia \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --mlp_hidden_dim 100 \
+   --model_name specroute \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   $FP16_FLAG
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/10-dbpedia/checkpoint*
+
+sleep 5
+
+if [ "$GPU_MODE" = "t4_2gpu" ]; then
+    BSZ=16; GA=1; EVAL_BSZ=256
+elif [ "$GPU_MODE" = "t4_1gpu" ]; then
+    BSZ=32; GA=1; EVAL_BSZ=256
+else
+    BSZ=64; GA=1; EVAL_BSZ=512
+fi
+
+CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
+   --do_train \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/1-yelp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/2-amazon/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/3-mnli/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/4-cb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/5-copa/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/6-qqp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/8-imdb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/9-sst2/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/10-dbpedia/saved_weights \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/agnews \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/11-agnews \
+   --per_device_train_batch_size $BSZ \
+   --per_device_eval_batch_size $EVAL_BSZ \
+   --gradient_accumulation_steps $GA \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_specroute \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_agnews \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --mlp_hidden_dim 100 \
+   --model_name specroute \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   $FP16_FLAG
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/11-agnews/checkpoint*
+
+sleep 5
+
+if [ "$GPU_MODE" = "t4_2gpu" ]; then
+    BSZ=16; GA=1; EVAL_BSZ=256
+elif [ "$GPU_MODE" = "t4_1gpu" ]; then
+    BSZ=32; GA=1; EVAL_BSZ=256
+else
+    BSZ=64; GA=1; EVAL_BSZ=512
+fi
+
+CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
+   --do_train \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/1-yelp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/2-amazon/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/3-mnli/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/4-cb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/5-copa/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/6-qqp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/8-imdb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/9-sst2/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/10-dbpedia/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/11-agnews/saved_weights \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/yahoo \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/12-yahoo \
+   --per_device_train_batch_size $BSZ \
+   --per_device_eval_batch_size $EVAL_BSZ \
+   --gradient_accumulation_steps $GA \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_specroute \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_yahoo \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --mlp_hidden_dim 100 \
+   --model_name specroute \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   $FP16_FLAG
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/12-yahoo/checkpoint*
+
+sleep 5
+
+if [ "$GPU_MODE" = "t4_2gpu" ]; then
+    BSZ=16; GA=1; EVAL_BSZ=256
+elif [ "$GPU_MODE" = "t4_1gpu" ]; then
+    BSZ=32; GA=1; EVAL_BSZ=256
+else
+    BSZ=64; GA=1; EVAL_BSZ=512
+fi
+
+CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
+   --do_train \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/1-yelp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/2-amazon/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/3-mnli/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/4-cb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/5-copa/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/6-qqp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/8-imdb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/9-sst2/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/10-dbpedia/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/11-agnews/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/12-yahoo/saved_weights \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/multirc \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/13-multirc \
+   --per_device_train_batch_size $BSZ \
+   --per_device_eval_batch_size $EVAL_BSZ \
+   --gradient_accumulation_steps $GA \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_specroute \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_multirc \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --mlp_hidden_dim 100 \
+   --model_name specroute \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   $FP16_FLAG
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/13-multirc/checkpoint*
+
+sleep 5
+
+if [ "$GPU_MODE" = "t4_2gpu" ]; then
+    BSZ=16; GA=1; EVAL_BSZ=256
+elif [ "$GPU_MODE" = "t4_1gpu" ]; then
+    BSZ=32; GA=1; EVAL_BSZ=256
+else
+    BSZ=64; GA=1; EVAL_BSZ=512
+fi
+
+CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
+   --do_train \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/1-yelp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/2-amazon/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/3-mnli/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/4-cb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/5-copa/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/6-qqp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/8-imdb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/9-sst2/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/10-dbpedia/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/11-agnews/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/12-yahoo/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/13-multirc/saved_weights \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/boolq \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/14-boolq \
+   --per_device_train_batch_size $BSZ \
+   --per_device_eval_batch_size $EVAL_BSZ \
+   --gradient_accumulation_steps $GA \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_specroute \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_boolq \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --mlp_hidden_dim 100 \
+   --model_name specroute \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   $FP16_FLAG
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/14-boolq/checkpoint*
+
+sleep 5
+
+if [ "$GPU_MODE" = "t4_2gpu" ]; then
+    BSZ=16; GA=1; EVAL_BSZ=256
+elif [ "$GPU_MODE" = "t4_1gpu" ]; then
+    BSZ=32; GA=1; EVAL_BSZ=256
+else
+    BSZ=64; GA=1; EVAL_BSZ=512
+fi
+
+CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
+   --do_train \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/1-yelp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/2-amazon/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/3-mnli/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/4-cb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/5-copa/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/6-qqp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/8-imdb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/9-sst2/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/10-dbpedia/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/11-agnews/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/12-yahoo/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/13-multirc/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/14-boolq/saved_weights \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/wic \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/15-wic \
+   --per_device_train_batch_size $BSZ \
+   --per_device_eval_batch_size $EVAL_BSZ \
+   --gradient_accumulation_steps $GA \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_specroute \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_wic \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --mlp_hidden_dim 100 \
+   --model_name specroute \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   $FP16_FLAG
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_specroute/outputs/15-wic/checkpoint*
+
+sleep 5

improve_gainlora/T5_small/gen_script_long_order4_t5_small_gainlora_inflora.sh

@@ -0,0 +1,774 @@
+#!/bin/bash
+#SBATCH -J cl                           
+#SBATCH -o cl-%j.out                       
+#SBATCH -p compute 
+#SBATCH -N 1                           
+#SBATCH -t 20:00:00   
+#SBATCH --mem 128G 
+#SBATCH --gres=gpu:a100-sxm4-80gb:1  
+
+export CUDA_DEVICE_ORDER="PCI_BUS_ID"
+
+port=$(shuf -i25000-30000 -n1)  
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/mnli \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/1-mnli \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order4_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/1-mnli/checkpoint*
+
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/1-mnli/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/1-mnli/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/1-mnli/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/cb \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/2-cb \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10\
+   --run_name gen_script_long_order4_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_cb \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/2-cb/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/2-cb/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/1-mnli/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/2-cb/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/2-cb/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/wic \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/3-wic \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10\
+   --run_name gen_script_long_order4_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_wic \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/3-wic/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/3-wic/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/1-mnli/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/2-cb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/3-wic/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/3-wic/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/copa \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/4-copa \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10\
+   --run_name gen_script_long_order4_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_copa \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/4-copa/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/4-copa/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/1-mnli/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/2-cb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/3-wic/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/4-copa/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/4-copa/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/qqp \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/5-qqp \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10\
+   --run_name gen_script_long_order4_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_qqp \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/5-qqp/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/5-qqp/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/1-mnli/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/2-cb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/3-wic/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/4-copa/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/5-qqp/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/5-qqp/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/boolq \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/6-boolq \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10\
+   --run_name gen_script_long_order4_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_boolq \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/6-boolq/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/6-boolq/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/1-mnli/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/2-cb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/3-wic/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/4-copa/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/5-qqp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/6-boolq/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/6-boolq/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/rte \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/7-rte \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10\
+   --run_name gen_script_long_order4_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_rte \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/7-rte/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/7-rte/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/1-mnli/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/2-cb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/3-wic/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/4-copa/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/5-qqp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/6-boolq/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/7-rte/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/7-rte/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/imdb \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/8-imdb \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10\
+   --run_name gen_script_long_order4_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_imdb \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/8-imdb/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/8-imdb/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/1-mnli/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/2-cb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/3-wic/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/4-copa/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/5-qqp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/6-boolq/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/8-imdb/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/8-imdb/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/yelp \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/9-yelp \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10\
+   --run_name gen_script_long_order4_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_yelp \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/9-yelp/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/9-yelp/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/1-mnli/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/2-cb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/3-wic/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/4-copa/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/5-qqp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/6-boolq/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/8-imdb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/9-yelp/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/9-yelp/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/amazon \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/10-amazon \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10\
+   --run_name gen_script_long_order4_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_amazon \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/10-amazon/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/10-amazon/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/1-mnli/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/2-cb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/3-wic/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/4-copa/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/5-qqp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/6-boolq/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/8-imdb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/9-yelp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/10-amazon/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/10-amazon/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/sst2 \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/11-sst2 \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10\
+   --run_name gen_script_long_order4_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_sst2 \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/11-sst2/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/11-sst2/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/1-mnli/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/2-cb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/3-wic/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/4-copa/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/5-qqp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/6-boolq/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/8-imdb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/9-yelp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/10-amazon/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/11-sst2/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/11-sst2/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/dbpedia \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/12-dbpedia \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10\
+   --run_name gen_script_long_order4_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_dbpedia \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/12-dbpedia/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/12-dbpedia/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/1-mnli/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/2-cb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/3-wic/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/4-copa/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/5-qqp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/6-boolq/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/8-imdb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/9-yelp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/10-amazon/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/11-sst2/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/12-dbpedia/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/12-dbpedia/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/agnews \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/13-agnews \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10\
+   --run_name gen_script_long_order4_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_agnews \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/13-agnews/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/13-agnews/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/1-mnli/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/2-cb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/3-wic/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/4-copa/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/5-qqp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/6-boolq/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/8-imdb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/9-yelp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/10-amazon/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/11-sst2/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/12-dbpedia/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/13-agnews/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/13-agnews/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/multirc \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/14-multirc \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10\
+   --run_name gen_script_long_order4_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_multirc \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/14-multirc/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/14-multirc/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/1-mnli/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/2-cb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/3-wic/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/4-copa/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/5-qqp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/6-boolq/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/8-imdb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/9-yelp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/10-amazon/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/11-sst2/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/12-dbpedia/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/13-agnews/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/14-multirc/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/14-multirc/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/yahoo \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/15-yahoo \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10\
+   --run_name gen_script_long_order4_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_yahoo \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/15-yahoo/checkpoint*
+   
+sleep 5
+
+CUDA_VISIBLE_DEVICES=$1 python score.py gen_script_long_order4_t5_small_gainlora_inflora gen_script_long_order4_t5_small_gainlora_inflora

improve_gainlora/T5_small/gen_script_long_order4_t5_small_inflora.sh

@@ -0,0 +1,744 @@
+#!/bin/bash
+#SBATCH -J cl                           
+#SBATCH -o cl-%j.out                       
+#SBATCH -p compute 
+#SBATCH -N 1                           
+#SBATCH -t 20:00:00   
+#SBATCH --mem 128G 
+#SBATCH --gres=gpu:a100-sxm4-80gb:1  
+
+export CUDA_DEVICE_ORDER="PCI_BUS_ID"
+
+port=$(shuf -i25000-30000 -n1)  
+
+CUDA_VISIBLE_DEVICES=1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/mnli \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/1-mnli \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order4_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --model_name inflora \
+   --threshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/1-mnli/checkpoint*
+
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/1-mnli/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/1-mnli/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/1-mnli/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/cb \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/2-cb \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10\
+   --run_name gen_script_long_order4_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_cb \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/2-cb/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/2-cb/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/1-mnli/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/2-cb/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/2-cb/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/wic \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/3-wic \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10\
+   --run_name gen_script_long_order4_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_wic \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/3-wic/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/3-wic/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/1-mnli/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/2-cb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/3-wic/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/3-wic/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/copa \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/4-copa \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10\
+   --run_name gen_script_long_order4_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_copa \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/4-copa/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/4-copa/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/1-mnli/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/2-cb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/3-wic/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/4-copa/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/4-copa/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/qqp \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/5-qqp \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10\
+   --run_name gen_script_long_order4_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_qqp \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/5-qqp/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/5-qqp/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/1-mnli/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/2-cb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/3-wic/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/4-copa/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/5-qqp/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/5-qqp/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/boolq \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/6-boolq \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10\
+   --run_name gen_script_long_order4_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_boolq \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/6-boolq/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/6-boolq/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/1-mnli/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/2-cb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/3-wic/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/4-copa/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/5-qqp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/6-boolq/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/6-boolq/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/rte \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/7-rte \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10\
+   --run_name gen_script_long_order4_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_rte \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/7-rte/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/7-rte/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/1-mnli/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/2-cb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/3-wic/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/4-copa/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/5-qqp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/6-boolq/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/7-rte/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/7-rte/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/imdb \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/8-imdb \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10\
+   --run_name gen_script_long_order4_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_imdb \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/8-imdb/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/8-imdb/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/1-mnli/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/2-cb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/3-wic/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/4-copa/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/5-qqp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/6-boolq/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/8-imdb/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/8-imdb/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/yelp \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/9-yelp \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10\
+   --run_name gen_script_long_order4_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_yelp \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/9-yelp/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/9-yelp/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/1-mnli/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/2-cb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/3-wic/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/4-copa/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/5-qqp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/6-boolq/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/8-imdb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/9-yelp/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/9-yelp/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/amazon \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/10-amazon \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10\
+   --run_name gen_script_long_order4_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_amazon \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/10-amazon/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/10-amazon/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/1-mnli/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/2-cb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/3-wic/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/4-copa/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/5-qqp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/6-boolq/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/8-imdb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/9-yelp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/10-amazon/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/10-amazon/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/sst2 \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/11-sst2 \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10\
+   --run_name gen_script_long_order4_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_sst2 \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/11-sst2/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/11-sst2/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/1-mnli/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/2-cb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/3-wic/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/4-copa/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/5-qqp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/6-boolq/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/8-imdb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/9-yelp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/10-amazon/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/11-sst2/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/11-sst2/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/dbpedia \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/12-dbpedia \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10\
+   --run_name gen_script_long_order4_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_dbpedia \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/12-dbpedia/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/12-dbpedia/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/1-mnli/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/2-cb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/3-wic/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/4-copa/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/5-qqp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/6-boolq/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/8-imdb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/9-yelp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/10-amazon/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/11-sst2/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/12-dbpedia/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/12-dbpedia/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/agnews \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/13-agnews \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10\
+   --run_name gen_script_long_order4_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_agnews \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/13-agnews/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/13-agnews/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/1-mnli/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/2-cb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/3-wic/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/4-copa/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/5-qqp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/6-boolq/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/8-imdb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/9-yelp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/10-amazon/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/11-sst2/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/12-dbpedia/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/13-agnews/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/13-agnews/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/multirc \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/14-multirc \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10\
+   --run_name gen_script_long_order4_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_multirc \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/14-multirc/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/14-multirc/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/1-mnli/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/2-cb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/3-wic/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/4-copa/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/5-qqp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/6-boolq/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/8-imdb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/9-yelp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/10-amazon/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/11-sst2/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/12-dbpedia/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/13-agnews/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/14-multirc/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/14-multirc/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/yahoo \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/15-yahoo \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10\
+   --run_name gen_script_long_order4_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_yahoo \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/15-yahoo/checkpoint*
+   
+sleep 5
+
+CUDA_VISIBLE_DEVICES=1 python score.py gen_script_long_order4_t5_small_inflora gen_script_long_order4_t5_small_inflora

improve_gainlora/T5_small/gen_script_long_order4_t5_small_specroute.sh

@@ -0,0 +1,849 @@
+#!/bin/bash
+#SBATCH -J cl
+#SBATCH -o cl-%j.out
+#SBATCH -p compute
+#SBATCH -N 1
+#SBATCH -t 20:00:00
+#SBATCH --mem 128G
+#SBATCH --gres=gpu:2
+
+export CUDA_DEVICE_ORDER="PCI_BUS_ID"
+
+port=$(shuf -i25000-30000 -n1)
+
+# ============================================================
+# Auto-detect GPU count and type for optimal parallelism
+# ============================================================
+NUM_GPUS=$(nvidia-smi -L 2>/dev/null | wc -l)
+GPU_MEM=$(nvidia-smi --query-gpu=memory.total --format=csv,noheader,nounits 2>/dev/null | head -1)
+
+if [ -z "$GPU_MEM" ]; then
+    echo "ERROR: No GPU detected!"
+    exit 1
+fi
+
+# Determine GPU type
+if [ "$GPU_MEM" -lt 20000 ]; then
+    IS_T4=1
+    echo "[GPU] Detected T4 GPUs (${GPU_MEM}MB VRAM each)"
+else
+    IS_T4=0
+    echo "[GPU] Detected high-memory GPUs (${GPU_MEM}MB VRAM each)"
+fi
+
+# Determine parallelism strategy
+if [ "$IS_T4" -eq 1 ] && [ "$NUM_GPUS" -ge 2 ]; then
+    GPU_MODE="t4_2gpu"
+    GPU_IDS="0,1"
+    FP16_FLAG=""
+    echo "[GPU] Strategy: 2x T4 DataParallel + fp32 + gradient_checkpointing"
+elif [ "$IS_T4" -eq 1 ]; then
+    GPU_MODE="t4_1gpu"
+    GPU_IDS="${1:-0}"
+    FP16_FLAG=""
+    echo "[GPU] Strategy: 1x T4 + fp32 + gradient_checkpointing"
+else
+    GPU_MODE="a100"
+    GPU_IDS="${1:-0}"
+    FP16_FLAG=""
+    echo "[GPU] Strategy: A100 (single GPU, fp32)"
+fi
+
+echo "[GPU] Using CUDA_VISIBLE_DEVICES=$GPU_IDS"
+echo "============================================================"
+echo ""
+
+if [ "$GPU_MODE" = "t4_2gpu" ]; then
+    BSZ=16; GA=1; EVAL_BSZ=256
+elif [ "$GPU_MODE" = "t4_1gpu" ]; then
+    BSZ=32; GA=1; EVAL_BSZ=256
+else
+    BSZ=64; GA=1; EVAL_BSZ=512
+fi
+
+CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
+   --do_train \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/mnli \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/1-mnli \
+   --per_device_train_batch_size $BSZ \
+   --per_device_eval_batch_size $EVAL_BSZ \
+   --gradient_accumulation_steps $GA \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order4_t5_small_specroute \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --run_single True \
+   --data_replay_freq -1 \
+   --mlp_hidden_dim 100 \
+   --model_name specroute \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   $FP16_FLAG
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/1-mnli/checkpoint*
+
+sleep 5
+
+if [ "$GPU_MODE" = "t4_2gpu" ]; then
+    BSZ=16; GA=1; EVAL_BSZ=256
+elif [ "$GPU_MODE" = "t4_1gpu" ]; then
+    BSZ=32; GA=1; EVAL_BSZ=256
+else
+    BSZ=64; GA=1; EVAL_BSZ=512
+fi
+
+CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
+   --do_train \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/1-mnli/saved_weights \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/cb \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/2-cb \
+   --per_device_train_batch_size $BSZ \
+   --per_device_eval_batch_size $EVAL_BSZ \
+   --gradient_accumulation_steps $GA \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order4_t5_small_specroute \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_cb \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --mlp_hidden_dim 100 \
+   --model_name specroute \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   $FP16_FLAG
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/2-cb/checkpoint*
+
+sleep 5
+
+if [ "$GPU_MODE" = "t4_2gpu" ]; then
+    BSZ=16; GA=1; EVAL_BSZ=256
+elif [ "$GPU_MODE" = "t4_1gpu" ]; then
+    BSZ=32; GA=1; EVAL_BSZ=256
+else
+    BSZ=64; GA=1; EVAL_BSZ=512
+fi
+
+CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
+   --do_train \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/1-mnli/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/2-cb/saved_weights \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/wic \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/3-wic \
+   --per_device_train_batch_size $BSZ \
+   --per_device_eval_batch_size $EVAL_BSZ \
+   --gradient_accumulation_steps $GA \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order4_t5_small_specroute \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_wic \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --mlp_hidden_dim 100 \
+   --model_name specroute \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   $FP16_FLAG
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/3-wic/checkpoint*
+
+sleep 5
+
+if [ "$GPU_MODE" = "t4_2gpu" ]; then
+    BSZ=16; GA=1; EVAL_BSZ=256
+elif [ "$GPU_MODE" = "t4_1gpu" ]; then
+    BSZ=32; GA=1; EVAL_BSZ=256
+else
+    BSZ=64; GA=1; EVAL_BSZ=512
+fi
+
+CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
+   --do_train \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/1-mnli/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/2-cb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/3-wic/saved_weights \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/copa \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/4-copa \
+   --per_device_train_batch_size $BSZ \
+   --per_device_eval_batch_size $EVAL_BSZ \
+   --gradient_accumulation_steps $GA \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order4_t5_small_specroute \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_copa \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --mlp_hidden_dim 100 \
+   --model_name specroute \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   $FP16_FLAG
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/4-copa/checkpoint*
+
+sleep 5
+
+if [ "$GPU_MODE" = "t4_2gpu" ]; then
+    BSZ=16; GA=1; EVAL_BSZ=256
+elif [ "$GPU_MODE" = "t4_1gpu" ]; then
+    BSZ=32; GA=1; EVAL_BSZ=256
+else
+    BSZ=64; GA=1; EVAL_BSZ=512
+fi
+
+CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
+   --do_train \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/1-mnli/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/2-cb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/3-wic/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/4-copa/saved_weights \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/qqp \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/5-qqp \
+   --per_device_train_batch_size $BSZ \
+   --per_device_eval_batch_size $EVAL_BSZ \
+   --gradient_accumulation_steps $GA \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order4_t5_small_specroute \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_qqp \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --mlp_hidden_dim 100 \
+   --model_name specroute \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   $FP16_FLAG
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/5-qqp/checkpoint*
+
+sleep 5
+
+if [ "$GPU_MODE" = "t4_2gpu" ]; then
+    BSZ=16; GA=1; EVAL_BSZ=256
+elif [ "$GPU_MODE" = "t4_1gpu" ]; then
+    BSZ=32; GA=1; EVAL_BSZ=256
+else
+    BSZ=64; GA=1; EVAL_BSZ=512
+fi
+
+CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
+   --do_train \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/1-mnli/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/2-cb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/3-wic/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/4-copa/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/5-qqp/saved_weights \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/boolq \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/6-boolq \
+   --per_device_train_batch_size $BSZ \
+   --per_device_eval_batch_size $EVAL_BSZ \
+   --gradient_accumulation_steps $GA \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order4_t5_small_specroute \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_boolq \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --mlp_hidden_dim 100 \
+   --model_name specroute \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   $FP16_FLAG
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/6-boolq/checkpoint*
+
+sleep 5
+
+if [ "$GPU_MODE" = "t4_2gpu" ]; then
+    BSZ=16; GA=1; EVAL_BSZ=256
+elif [ "$GPU_MODE" = "t4_1gpu" ]; then
+    BSZ=32; GA=1; EVAL_BSZ=256
+else
+    BSZ=64; GA=1; EVAL_BSZ=512
+fi
+
+CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
+   --do_train \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/1-mnli/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/2-cb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/3-wic/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/4-copa/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/5-qqp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/6-boolq/saved_weights \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/rte \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/7-rte \
+   --per_device_train_batch_size $BSZ \
+   --per_device_eval_batch_size $EVAL_BSZ \
+   --gradient_accumulation_steps $GA \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order4_t5_small_specroute \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_rte \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --mlp_hidden_dim 100 \
+   --model_name specroute \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   $FP16_FLAG
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/7-rte/checkpoint*
+
+sleep 5
+
+if [ "$GPU_MODE" = "t4_2gpu" ]; then
+    BSZ=16; GA=1; EVAL_BSZ=256
+elif [ "$GPU_MODE" = "t4_1gpu" ]; then
+    BSZ=32; GA=1; EVAL_BSZ=256
+else
+    BSZ=64; GA=1; EVAL_BSZ=512
+fi
+
+CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
+   --do_train \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/1-mnli/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/2-cb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/3-wic/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/4-copa/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/5-qqp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/6-boolq/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/7-rte/saved_weights \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/imdb \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/8-imdb \
+   --per_device_train_batch_size $BSZ \
+   --per_device_eval_batch_size $EVAL_BSZ \
+   --gradient_accumulation_steps $GA \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order4_t5_small_specroute \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_imdb \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --mlp_hidden_dim 100 \
+   --model_name specroute \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   $FP16_FLAG
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/8-imdb/checkpoint*
+
+sleep 5
+
+if [ "$GPU_MODE" = "t4_2gpu" ]; then
+    BSZ=16; GA=1; EVAL_BSZ=256
+elif [ "$GPU_MODE" = "t4_1gpu" ]; then
+    BSZ=32; GA=1; EVAL_BSZ=256
+else
+    BSZ=64; GA=1; EVAL_BSZ=512
+fi
+
+CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
+   --do_train \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/1-mnli/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/2-cb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/3-wic/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/4-copa/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/5-qqp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/6-boolq/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/8-imdb/saved_weights \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/yelp \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/9-yelp \
+   --per_device_train_batch_size $BSZ \
+   --per_device_eval_batch_size $EVAL_BSZ \
+   --gradient_accumulation_steps $GA \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order4_t5_small_specroute \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_yelp \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --mlp_hidden_dim 100 \
+   --model_name specroute \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   $FP16_FLAG
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/9-yelp/checkpoint*
+
+sleep 5
+
+if [ "$GPU_MODE" = "t4_2gpu" ]; then
+    BSZ=16; GA=1; EVAL_BSZ=256
+elif [ "$GPU_MODE" = "t4_1gpu" ]; then
+    BSZ=32; GA=1; EVAL_BSZ=256
+else
+    BSZ=64; GA=1; EVAL_BSZ=512
+fi
+
+CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
+   --do_train \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/1-mnli/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/2-cb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/3-wic/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/4-copa/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/5-qqp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/6-boolq/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/8-imdb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/9-yelp/saved_weights \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/amazon \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/10-amazon \
+   --per_device_train_batch_size $BSZ \
+   --per_device_eval_batch_size $EVAL_BSZ \
+   --gradient_accumulation_steps $GA \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order4_t5_small_specroute \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_amazon \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --mlp_hidden_dim 100 \
+   --model_name specroute \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   $FP16_FLAG
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/10-amazon/checkpoint*
+
+sleep 5
+
+if [ "$GPU_MODE" = "t4_2gpu" ]; then
+    BSZ=16; GA=1; EVAL_BSZ=256
+elif [ "$GPU_MODE" = "t4_1gpu" ]; then
+    BSZ=32; GA=1; EVAL_BSZ=256
+else
+    BSZ=64; GA=1; EVAL_BSZ=512
+fi
+
+CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
+   --do_train \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/1-mnli/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/2-cb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/3-wic/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/4-copa/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/5-qqp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/6-boolq/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/8-imdb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/9-yelp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/10-amazon/saved_weights \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/sst2 \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/11-sst2 \
+   --per_device_train_batch_size $BSZ \
+   --per_device_eval_batch_size $EVAL_BSZ \
+   --gradient_accumulation_steps $GA \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order4_t5_small_specroute \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_sst2 \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --mlp_hidden_dim 100 \
+   --model_name specroute \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   $FP16_FLAG
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/11-sst2/checkpoint*
+
+sleep 5
+
+if [ "$GPU_MODE" = "t4_2gpu" ]; then
+    BSZ=16; GA=1; EVAL_BSZ=256
+elif [ "$GPU_MODE" = "t4_1gpu" ]; then
+    BSZ=32; GA=1; EVAL_BSZ=256
+else
+    BSZ=64; GA=1; EVAL_BSZ=512
+fi
+
+CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
+   --do_train \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/1-mnli/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/2-cb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/3-wic/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/4-copa/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/5-qqp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/6-boolq/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/8-imdb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/9-yelp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/10-amazon/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/11-sst2/saved_weights \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/dbpedia \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/12-dbpedia \
+   --per_device_train_batch_size $BSZ \
+   --per_device_eval_batch_size $EVAL_BSZ \
+   --gradient_accumulation_steps $GA \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order4_t5_small_specroute \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_dbpedia \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --mlp_hidden_dim 100 \
+   --model_name specroute \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   $FP16_FLAG
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/12-dbpedia/checkpoint*
+
+sleep 5
+
+if [ "$GPU_MODE" = "t4_2gpu" ]; then
+    BSZ=16; GA=1; EVAL_BSZ=256
+elif [ "$GPU_MODE" = "t4_1gpu" ]; then
+    BSZ=32; GA=1; EVAL_BSZ=256
+else
+    BSZ=64; GA=1; EVAL_BSZ=512
+fi
+
+CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
+   --do_train \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/1-mnli/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/2-cb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/3-wic/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/4-copa/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/5-qqp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/6-boolq/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/8-imdb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/9-yelp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/10-amazon/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/11-sst2/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/12-dbpedia/saved_weights \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/agnews \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/13-agnews \
+   --per_device_train_batch_size $BSZ \
+   --per_device_eval_batch_size $EVAL_BSZ \
+   --gradient_accumulation_steps $GA \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order4_t5_small_specroute \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_agnews \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --mlp_hidden_dim 100 \
+   --model_name specroute \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   $FP16_FLAG
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/13-agnews/checkpoint*
+
+sleep 5
+
+if [ "$GPU_MODE" = "t4_2gpu" ]; then
+    BSZ=16; GA=1; EVAL_BSZ=256
+elif [ "$GPU_MODE" = "t4_1gpu" ]; then
+    BSZ=32; GA=1; EVAL_BSZ=256
+else
+    BSZ=64; GA=1; EVAL_BSZ=512
+fi
+
+CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
+   --do_train \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/1-mnli/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/2-cb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/3-wic/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/4-copa/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/5-qqp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/6-boolq/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/8-imdb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/9-yelp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/10-amazon/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/11-sst2/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/12-dbpedia/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/13-agnews/saved_weights \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/multirc \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/14-multirc \
+   --per_device_train_batch_size $BSZ \
+   --per_device_eval_batch_size $EVAL_BSZ \
+   --gradient_accumulation_steps $GA \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order4_t5_small_specroute \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_multirc \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --mlp_hidden_dim 100 \
+   --model_name specroute \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   $FP16_FLAG
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/14-multirc/checkpoint*
+
+sleep 5
+
+if [ "$GPU_MODE" = "t4_2gpu" ]; then
+    BSZ=16; GA=1; EVAL_BSZ=256
+elif [ "$GPU_MODE" = "t4_1gpu" ]; then
+    BSZ=32; GA=1; EVAL_BSZ=256
+else
+    BSZ=64; GA=1; EVAL_BSZ=512
+fi
+
+CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
+   --do_train \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/1-mnli/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/2-cb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/3-wic/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/4-copa/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/5-qqp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/6-boolq/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/8-imdb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/9-yelp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/10-amazon/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/11-sst2/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/12-dbpedia/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/13-agnews/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/14-multirc/saved_weights \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/yahoo \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/15-yahoo \
+   --per_device_train_batch_size $BSZ \
+   --per_device_eval_batch_size $EVAL_BSZ \
+   --gradient_accumulation_steps $GA \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order4_t5_small_specroute \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_yahoo \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --mlp_hidden_dim 100 \
+   --model_name specroute \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   $FP16_FLAG
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_specroute/outputs/15-yahoo/checkpoint*
+
+sleep 5

improve_gainlora/T5_small/gen_script_superni_order1_t5_small_gainlora_inflora.sh

@@ -0,0 +1,744 @@
+#!/bin/bash
+#SBATCH -J cl                           
+#SBATCH -o cl-%j.out                       
+#SBATCH -p compute 
+#SBATCH -N 1                           
+#SBATCH -t 20:00:00   
+#SBATCH --mem 128G 
+#SBATCH --gres=gpu:a100-sxm4-80gb:1  
+
+export CUDA_DEVICE_ORDER="PCI_BUS_ID"
+
+port=$(shuf -i25000-30000 -n1)  
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task1572_samsum_summary \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/1-task1572_samsum_summary \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --load_best_model_at_end \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/1-task1572_samsum_summary/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/1-task1572_samsum_summary/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/1-task1572_samsum_summary/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task363_sst2_polarity_classification \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/2-task363_sst2_polarity_classification \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task363_sst2_polarity_classification \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/2-task363_sst2_polarity_classification/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/2-task363_sst2_polarity_classification/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/2-task363_sst2_polarity_classification/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task1290_xsum_summarization \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/3-task1290_xsum_summarization \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task1290_xsum_summarization \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995 
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/3-task1290_xsum_summarization/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/3-task1290_xsum_summarization/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/3-task1290_xsum_summarization/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task181_outcome_extraction \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/4-task181_outcome_extraction \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task181_outcome_extraction \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/4-task181_outcome_extraction/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/4-task181_outcome_extraction/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/4-task181_outcome_extraction/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task002_quoref_answer_generation \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/5-task002_quoref_answer_generation \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task002_quoref_answer_generation \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/5-task002_quoref_answer_generation/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/5-task002_quoref_answer_generation/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/5-task002_quoref_answer_generation/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task1510_evalution_relation_extraction \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/6-task1510_evalution_relation_extraction \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task1510_evalution_relation_extraction \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/6-task1510_evalution_relation_extraction/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/5-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/6-task1510_evalution_relation_extraction/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/6-task1510_evalution_relation_extraction/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task639_multi_woz_user_utterance_generation \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/7-task639_multi_woz_user_utterance_generation \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task639_multi_woz_user_utterance_generation \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/7-task639_multi_woz_user_utterance_generation/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/5-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/6-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/7-task639_multi_woz_user_utterance_generation/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/7-task639_multi_woz_user_utterance_generation/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task1729_personachat_generate_next \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/8-task1729_personachat_generate_next \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task1729_personachat_generate_next \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/8-task1729_personachat_generate_next/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/5-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/6-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/7-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/8-task1729_personachat_generate_next/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/8-task1729_personachat_generate_next/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task073_commonsenseqa_answer_generation \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/9-task073_commonsenseqa_answer_generation \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task073_commonsenseqa_answer_generation \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/9-task073_commonsenseqa_answer_generation/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/5-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/6-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/7-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/8-task1729_personachat_generate_next/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/9-task073_commonsenseqa_answer_generation/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/9-task073_commonsenseqa_answer_generation/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task1590_diplomacy_text_generation \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/10-task1590_diplomacy_text_generation \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task1590_diplomacy_text_generation \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/10-task1590_diplomacy_text_generation/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/5-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/6-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/7-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/8-task1729_personachat_generate_next/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/9-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/10-task1590_diplomacy_text_generation/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/10-task1590_diplomacy_text_generation/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task748_glucose_reverse_cause_event_detection \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/11-task748_glucose_reverse_cause_event_detection \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task748_glucose_reverse_cause_event_detection \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/11-task748_glucose_reverse_cause_event_detection/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/5-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/6-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/7-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/8-task1729_personachat_generate_next/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/9-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/10-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/11-task748_glucose_reverse_cause_event_detection/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/11-task748_glucose_reverse_cause_event_detection/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task511_reddit_tifu_long_text_summarization \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/12-task511_reddit_tifu_long_text_summarization \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task511_reddit_tifu_long_text_summarization \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/12-task511_reddit_tifu_long_text_summarization/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/5-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/6-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/7-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/8-task1729_personachat_generate_next/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/9-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/10-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/11-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/12-task511_reddit_tifu_long_text_summarization/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/12-task511_reddit_tifu_long_text_summarization/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task591_sciq_answer_generation \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/13-task591_sciq_answer_generation \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task591_sciq_answer_generation \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/13-task591_sciq_answer_generation/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/5-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/6-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/7-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/8-task1729_personachat_generate_next/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/9-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/10-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/11-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/12-task511_reddit_tifu_long_text_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/13-task591_sciq_answer_generation/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/13-task591_sciq_answer_generation/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task1687_sentiment140_classification \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/14-task1687_sentiment140_classification \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task1687_sentiment140_classification \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/14-task1687_sentiment140_classification/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/5-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/6-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/7-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/8-task1729_personachat_generate_next/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/9-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/10-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/11-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/12-task511_reddit_tifu_long_text_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/13-task591_sciq_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/14-task1687_sentiment140_classification/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/14-task1687_sentiment140_classification/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task875_emotion_classification \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/15-task875_emotion_classification \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task875_emotion_classification \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+python score.py gen_script_superni_order1_t5_small_gainlora_inflora gen_script_superni_order1_t5_small_gainlora_inflora

improve_gainlora/T5_small/gen_script_superni_order1_t5_small_inflora.sh

@@ -0,0 +1,713 @@
+#!/bin/bash
+#SBATCH -J cl                           
+#SBATCH -o cl-%j.out                       
+#SBATCH -p compute 
+#SBATCH -N 1                           
+#SBATCH -t 20:00:00   
+#SBATCH --mem 128G 
+#SBATCH --gres=gpu:a100-sxm4-80gb:1  
+
+export CUDA_DEVICE_ORDER="PCI_BUS_ID"
+
+port=$(shuf -i25000-30000 -n1)  
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task1572_samsum_summary \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/1-task1572_samsum_summary \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --load_best_model_at_end \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/1-task1572_samsum_summary/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/1-task1572_samsum_summary/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/1-task1572_samsum_summary/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task363_sst2_polarity_classification \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/2-task363_sst2_polarity_classification \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task363_sst2_polarity_classification \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/2-task363_sst2_polarity_classification/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/2-task363_sst2_polarity_classification/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/2-task363_sst2_polarity_classification/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task1290_xsum_summarization \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/3-task1290_xsum_summarization \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task1290_xsum_summarization \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995 
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/3-task1290_xsum_summarization/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/3-task1290_xsum_summarization/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/3-task1290_xsum_summarization/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task181_outcome_extraction \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/4-task181_outcome_extraction \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task181_outcome_extraction \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/4-task181_outcome_extraction/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/4-task181_outcome_extraction/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/4-task181_outcome_extraction/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task002_quoref_answer_generation \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/5-task002_quoref_answer_generation \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task002_quoref_answer_generation \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/5-task002_quoref_answer_generation/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/5-task002_quoref_answer_generation/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/5-task002_quoref_answer_generation/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task1510_evalution_relation_extraction \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/6-task1510_evalution_relation_extraction \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task1510_evalution_relation_extraction \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/6-task1510_evalution_relation_extraction/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/5-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/6-task1510_evalution_relation_extraction/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/6-task1510_evalution_relation_extraction/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task639_multi_woz_user_utterance_generation \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/7-task639_multi_woz_user_utterance_generation \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task639_multi_woz_user_utterance_generation \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/7-task639_multi_woz_user_utterance_generation/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/5-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/6-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/7-task639_multi_woz_user_utterance_generation/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/7-task639_multi_woz_user_utterance_generation/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task1729_personachat_generate_next \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/8-task1729_personachat_generate_next \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task1729_personachat_generate_next \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/8-task1729_personachat_generate_next/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/5-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/6-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/7-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/8-task1729_personachat_generate_next/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/8-task1729_personachat_generate_next/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task073_commonsenseqa_answer_generation \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/9-task073_commonsenseqa_answer_generation \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task073_commonsenseqa_answer_generation \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/9-task073_commonsenseqa_answer_generation/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/5-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/6-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/7-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/8-task1729_personachat_generate_next/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/9-task073_commonsenseqa_answer_generation/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/9-task073_commonsenseqa_answer_generation/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task1590_diplomacy_text_generation \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/10-task1590_diplomacy_text_generation \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task1590_diplomacy_text_generation \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/10-task1590_diplomacy_text_generation/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/5-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/6-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/7-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/8-task1729_personachat_generate_next/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/9-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/10-task1590_diplomacy_text_generation/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/10-task1590_diplomacy_text_generation/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task748_glucose_reverse_cause_event_detection \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/11-task748_glucose_reverse_cause_event_detection \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task748_glucose_reverse_cause_event_detection \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/11-task748_glucose_reverse_cause_event_detection/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/5-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/6-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/7-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/8-task1729_personachat_generate_next/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/9-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/10-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/11-task748_glucose_reverse_cause_event_detection/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/11-task748_glucose_reverse_cause_event_detection/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task511_reddit_tifu_long_text_summarization \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/12-task511_reddit_tifu_long_text_summarization \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task511_reddit_tifu_long_text_summarization \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/12-task511_reddit_tifu_long_text_summarization/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/5-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/6-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/7-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/8-task1729_personachat_generate_next/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/9-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/10-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/11-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/12-task511_reddit_tifu_long_text_summarization/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/12-task511_reddit_tifu_long_text_summarization/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task591_sciq_answer_generation \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/13-task591_sciq_answer_generation \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task591_sciq_answer_generation \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/13-task591_sciq_answer_generation/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/5-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/6-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/7-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/8-task1729_personachat_generate_next/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/9-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/10-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/11-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/12-task511_reddit_tifu_long_text_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/13-task591_sciq_answer_generation/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/13-task591_sciq_answer_generation/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task1687_sentiment140_classification \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/14-task1687_sentiment140_classification \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task1687_sentiment140_classification \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/14-task1687_sentiment140_classification/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/5-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/6-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/7-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/8-task1729_personachat_generate_next/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/9-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/10-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/11-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/12-task511_reddit_tifu_long_text_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/13-task591_sciq_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/14-task1687_sentiment140_classification/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/14-task1687_sentiment140_classification/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task875_emotion_classification \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/15-task875_emotion_classification \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task875_emotion_classification \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+python score.py gen_script_superni_order1_t5_small_inflora gen_script_superni_order1_t5_small_inflora

improve_gainlora/T5_small/gen_script_superni_order1_t5_small_specroute.sh

@@ -0,0 +1,821 @@
+#!/bin/bash
+#SBATCH -J cl
+#SBATCH -o cl-%j.out
+#SBATCH -p compute
+#SBATCH -N 1
+#SBATCH -t 20:00:00
+#SBATCH --mem 128G
+#SBATCH --gres=gpu:2
+
+export CUDA_DEVICE_ORDER="PCI_BUS_ID"
+
+port=$(shuf -i25000-30000 -n1)
+
+# ============================================================
+# Auto-detect GPU count and type for optimal parallelism
+# ============================================================
+NUM_GPUS=$(nvidia-smi -L 2>/dev/null | wc -l)
+GPU_MEM=$(nvidia-smi --query-gpu=memory.total --format=csv,noheader,nounits 2>/dev/null | head -1)
+
+if [ -z "$GPU_MEM" ]; then
+    echo "ERROR: No GPU detected!"
+    exit 1
+fi
+
+# Determine GPU type
+if [ "$GPU_MEM" -lt 20000 ]; then
+    IS_T4=1
+    echo "[GPU] Detected T4 GPUs (${GPU_MEM}MB VRAM each)"
+else
+    IS_T4=0
+    echo "[GPU] Detected high-memory GPUs (${GPU_MEM}MB VRAM each)"
+fi
+
+# Determine parallelism strategy
+# NOTE: T5 models trained in bfloat16 produce NaN with fp16 (overflow).
+# T4 GPUs do not support bf16. Use fp32 + gradient_checkpointing instead.
+if [ "$IS_T4" -eq 1 ] && [ "$NUM_GPUS" -ge 2 ]; then
+    GPU_MODE="t4_2gpu"
+    GPU_IDS="0,1"
+    FP16_FLAG=""
+    echo "[GPU] Strategy: 2x T4 DataParallel + fp32 + gradient_checkpointing"
+elif [ "$IS_T4" -eq 1 ]; then
+    GPU_MODE="t4_1gpu"
+    GPU_IDS="${1:-0}"
+    FP16_FLAG=""
+    echo "[GPU] Strategy: 1x T4 + fp32 + gradient_checkpointing"
+else
+    GPU_MODE="a100"
+    GPU_IDS="${1:-0}"
+    FP16_FLAG=""
+    echo "[GPU] Strategy: A100 (single GPU, fp32)"
+fi
+
+echo "[GPU] Using CUDA_VISIBLE_DEVICES=$GPU_IDS"
+echo "============================================================"
+echo ""
+
+if [ "$GPU_MODE" = "t4_2gpu" ]; then
+    BSZ=8;  GA=2; EVAL_BSZ=16
+elif [ "$GPU_MODE" = "t4_1gpu" ]; then
+    BSZ=16; GA=2; EVAL_BSZ=16
+else
+    BSZ=32; GA=1; EVAL_BSZ=32
+fi
+
+CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task1572_samsum_summary \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/1-task1572_samsum_summary \
+   --per_device_train_batch_size $BSZ \
+   --per_device_eval_batch_size $EVAL_BSZ \
+   --gradient_accumulation_steps $GA \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_specroute \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_safetensors false \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --run_single True \
+   --data_replay_freq -1 \
+   --mlp_hidden_dim 100 \
+   --model_name specroute \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   $FP16_FLAG
+
+if [ "$GPU_MODE" = "t4_2gpu" ]; then
+    BSZ=8;  GA=2; EVAL_BSZ=16
+elif [ "$GPU_MODE" = "t4_1gpu" ]; then
+    BSZ=16; GA=2; EVAL_BSZ=16
+else
+    BSZ=32; GA=1; EVAL_BSZ=32
+fi
+
+CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/1-task1572_samsum_summary/saved_weights \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task363_sst2_polarity_classification \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/2-task363_sst2_polarity_classification \
+   --per_device_train_batch_size $BSZ \
+   --per_device_eval_batch_size $EVAL_BSZ \
+   --gradient_accumulation_steps $GA \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_specroute \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task363_sst2_polarity_classification \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_safetensors false \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --mlp_hidden_dim 100 \
+   --model_name specroute \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   $FP16_FLAG
+
+if [ "$GPU_MODE" = "t4_2gpu" ]; then
+    BSZ=8;  GA=2; EVAL_BSZ=16
+elif [ "$GPU_MODE" = "t4_1gpu" ]; then
+    BSZ=16; GA=2; EVAL_BSZ=16
+else
+    BSZ=32; GA=1; EVAL_BSZ=32
+fi
+
+CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/2-task363_sst2_polarity_classification/saved_weights \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task1290_xsum_summarization \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/3-task1290_xsum_summarization \
+   --per_device_train_batch_size $BSZ \
+   --per_device_eval_batch_size $EVAL_BSZ \
+   --gradient_accumulation_steps $GA \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_specroute \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task1290_xsum_summarization \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_safetensors false \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --mlp_hidden_dim 100 \
+   --model_name specroute \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   $FP16_FLAG
+
+if [ "$GPU_MODE" = "t4_2gpu" ]; then
+    BSZ=8;  GA=2; EVAL_BSZ=16
+elif [ "$GPU_MODE" = "t4_1gpu" ]; then
+    BSZ=16; GA=2; EVAL_BSZ=16
+else
+    BSZ=32; GA=1; EVAL_BSZ=32
+fi
+
+CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/3-task1290_xsum_summarization/saved_weights \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task181_outcome_extraction \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/4-task181_outcome_extraction \
+   --per_device_train_batch_size $BSZ \
+   --per_device_eval_batch_size $EVAL_BSZ \
+   --gradient_accumulation_steps $GA \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_specroute \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task181_outcome_extraction \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_safetensors false \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --mlp_hidden_dim 100 \
+   --model_name specroute \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   $FP16_FLAG
+
+if [ "$GPU_MODE" = "t4_2gpu" ]; then
+    BSZ=8;  GA=2; EVAL_BSZ=16
+elif [ "$GPU_MODE" = "t4_1gpu" ]; then
+    BSZ=16; GA=2; EVAL_BSZ=16
+else
+    BSZ=32; GA=1; EVAL_BSZ=32
+fi
+
+CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/4-task181_outcome_extraction/saved_weights \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task002_quoref_answer_generation \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/5-task002_quoref_answer_generation \
+   --per_device_train_batch_size $BSZ \
+   --per_device_eval_batch_size $EVAL_BSZ \
+   --gradient_accumulation_steps $GA \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_specroute \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task002_quoref_answer_generation \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_safetensors false \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --mlp_hidden_dim 100 \
+   --model_name specroute \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   $FP16_FLAG
+
+if [ "$GPU_MODE" = "t4_2gpu" ]; then
+    BSZ=8;  GA=2; EVAL_BSZ=16
+elif [ "$GPU_MODE" = "t4_1gpu" ]; then
+    BSZ=16; GA=2; EVAL_BSZ=16
+else
+    BSZ=32; GA=1; EVAL_BSZ=32
+fi
+
+CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/5-task002_quoref_answer_generation/saved_weights \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task1510_evalution_relation_extraction \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/6-task1510_evalution_relation_extraction \
+   --per_device_train_batch_size $BSZ \
+   --per_device_eval_batch_size $EVAL_BSZ \
+   --gradient_accumulation_steps $GA \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_specroute \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task1510_evalution_relation_extraction \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_safetensors false \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --mlp_hidden_dim 100 \
+   --model_name specroute \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   $FP16_FLAG
+
+if [ "$GPU_MODE" = "t4_2gpu" ]; then
+    BSZ=8;  GA=2; EVAL_BSZ=16
+elif [ "$GPU_MODE" = "t4_1gpu" ]; then
+    BSZ=16; GA=2; EVAL_BSZ=16
+else
+    BSZ=32; GA=1; EVAL_BSZ=32
+fi
+
+CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/5-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/6-task1510_evalution_relation_extraction/saved_weights \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task639_multi_woz_user_utterance_generation \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/7-task639_multi_woz_user_utterance_generation \
+   --per_device_train_batch_size $BSZ \
+   --per_device_eval_batch_size $EVAL_BSZ \
+   --gradient_accumulation_steps $GA \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_specroute \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task639_multi_woz_user_utterance_generation \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_safetensors false \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --mlp_hidden_dim 100 \
+   --model_name specroute \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   $FP16_FLAG
+
+if [ "$GPU_MODE" = "t4_2gpu" ]; then
+    BSZ=8;  GA=2; EVAL_BSZ=16
+elif [ "$GPU_MODE" = "t4_1gpu" ]; then
+    BSZ=16; GA=2; EVAL_BSZ=16
+else
+    BSZ=32; GA=1; EVAL_BSZ=32
+fi
+
+CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/5-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/6-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/7-task639_multi_woz_user_utterance_generation/saved_weights \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task1729_personachat_generate_next \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/8-task1729_personachat_generate_next \
+   --per_device_train_batch_size $BSZ \
+   --per_device_eval_batch_size $EVAL_BSZ \
+   --gradient_accumulation_steps $GA \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_specroute \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task1729_personachat_generate_next \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_safetensors false \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --mlp_hidden_dim 100 \
+   --model_name specroute \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   $FP16_FLAG
+
+if [ "$GPU_MODE" = "t4_2gpu" ]; then
+    BSZ=8;  GA=2; EVAL_BSZ=16
+elif [ "$GPU_MODE" = "t4_1gpu" ]; then
+    BSZ=16; GA=2; EVAL_BSZ=16
+else
+    BSZ=32; GA=1; EVAL_BSZ=32
+fi
+
+CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/5-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/6-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/7-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/8-task1729_personachat_generate_next/saved_weights \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task073_commonsenseqa_answer_generation \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/9-task073_commonsenseqa_answer_generation \
+   --per_device_train_batch_size $BSZ \
+   --per_device_eval_batch_size $EVAL_BSZ \
+   --gradient_accumulation_steps $GA \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_specroute \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task073_commonsenseqa_answer_generation \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_safetensors false \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --mlp_hidden_dim 100 \
+   --model_name specroute \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   $FP16_FLAG
+
+if [ "$GPU_MODE" = "t4_2gpu" ]; then
+    BSZ=8;  GA=2; EVAL_BSZ=16
+elif [ "$GPU_MODE" = "t4_1gpu" ]; then
+    BSZ=16; GA=2; EVAL_BSZ=16
+else
+    BSZ=32; GA=1; EVAL_BSZ=32
+fi
+
+CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/5-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/6-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/7-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/8-task1729_personachat_generate_next/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/9-task073_commonsenseqa_answer_generation/saved_weights \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task1590_diplomacy_text_generation \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/10-task1590_diplomacy_text_generation \
+   --per_device_train_batch_size $BSZ \
+   --per_device_eval_batch_size $EVAL_BSZ \
+   --gradient_accumulation_steps $GA \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_specroute \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task1590_diplomacy_text_generation \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_safetensors false \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --mlp_hidden_dim 100 \
+   --model_name specroute \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   $FP16_FLAG
+
+if [ "$GPU_MODE" = "t4_2gpu" ]; then
+    BSZ=8;  GA=2; EVAL_BSZ=16
+elif [ "$GPU_MODE" = "t4_1gpu" ]; then
+    BSZ=16; GA=2; EVAL_BSZ=16
+else
+    BSZ=32; GA=1; EVAL_BSZ=32
+fi
+
+CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/5-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/6-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/7-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/8-task1729_personachat_generate_next/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/9-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/10-task1590_diplomacy_text_generation/saved_weights \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task748_glucose_reverse_cause_event_detection \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/11-task748_glucose_reverse_cause_event_detection \
+   --per_device_train_batch_size $BSZ \
+   --per_device_eval_batch_size $EVAL_BSZ \
+   --gradient_accumulation_steps $GA \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_specroute \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task748_glucose_reverse_cause_event_detection \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_safetensors false \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --mlp_hidden_dim 100 \
+   --model_name specroute \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   $FP16_FLAG
+
+if [ "$GPU_MODE" = "t4_2gpu" ]; then
+    BSZ=8;  GA=2; EVAL_BSZ=16
+elif [ "$GPU_MODE" = "t4_1gpu" ]; then
+    BSZ=16; GA=2; EVAL_BSZ=16
+else
+    BSZ=32; GA=1; EVAL_BSZ=32
+fi
+
+CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/5-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/6-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/7-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/8-task1729_personachat_generate_next/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/9-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/10-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/11-task748_glucose_reverse_cause_event_detection/saved_weights \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task511_reddit_tifu_long_text_summarization \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/12-task511_reddit_tifu_long_text_summarization \
+   --per_device_train_batch_size $BSZ \
+   --per_device_eval_batch_size $EVAL_BSZ \
+   --gradient_accumulation_steps $GA \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_specroute \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task511_reddit_tifu_long_text_summarization \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_safetensors false \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --mlp_hidden_dim 100 \
+   --model_name specroute \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   $FP16_FLAG
+
+if [ "$GPU_MODE" = "t4_2gpu" ]; then
+    BSZ=8;  GA=2; EVAL_BSZ=16
+elif [ "$GPU_MODE" = "t4_1gpu" ]; then
+    BSZ=16; GA=2; EVAL_BSZ=16
+else
+    BSZ=32; GA=1; EVAL_BSZ=32
+fi
+
+CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/5-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/6-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/7-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/8-task1729_personachat_generate_next/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/9-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/10-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/11-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/12-task511_reddit_tifu_long_text_summarization/saved_weights \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task591_sciq_answer_generation \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/13-task591_sciq_answer_generation \
+   --per_device_train_batch_size $BSZ \
+   --per_device_eval_batch_size $EVAL_BSZ \
+   --gradient_accumulation_steps $GA \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_specroute \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task591_sciq_answer_generation \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_safetensors false \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --mlp_hidden_dim 100 \
+   --model_name specroute \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   $FP16_FLAG
+
+if [ "$GPU_MODE" = "t4_2gpu" ]; then
+    BSZ=8;  GA=2; EVAL_BSZ=16
+elif [ "$GPU_MODE" = "t4_1gpu" ]; then
+    BSZ=16; GA=2; EVAL_BSZ=16
+else
+    BSZ=32; GA=1; EVAL_BSZ=32
+fi
+
+CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/5-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/6-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/7-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/8-task1729_personachat_generate_next/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/9-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/10-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/11-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/12-task511_reddit_tifu_long_text_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/13-task591_sciq_answer_generation/saved_weights \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task1687_sentiment140_classification \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/14-task1687_sentiment140_classification \
+   --per_device_train_batch_size $BSZ \
+   --per_device_eval_batch_size $EVAL_BSZ \
+   --gradient_accumulation_steps $GA \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_specroute \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task1687_sentiment140_classification \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_safetensors false \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --mlp_hidden_dim 100 \
+   --model_name specroute \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   $FP16_FLAG
+
+if [ "$GPU_MODE" = "t4_2gpu" ]; then
+    BSZ=8;  GA=2; EVAL_BSZ=16
+elif [ "$GPU_MODE" = "t4_1gpu" ]; then
+    BSZ=16; GA=2; EVAL_BSZ=16
+else
+    BSZ=32; GA=1; EVAL_BSZ=32
+fi
+
+CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/5-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/6-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/7-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/8-task1729_personachat_generate_next/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/9-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/10-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/11-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/12-task511_reddit_tifu_long_text_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/13-task591_sciq_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/14-task1687_sentiment140_classification/saved_weights \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task875_emotion_classification \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_specroute/outputs/15-task875_emotion_classification \
+   --per_device_train_batch_size $BSZ \
+   --per_device_eval_batch_size $EVAL_BSZ \
+   --gradient_accumulation_steps $GA \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_specroute \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task875_emotion_classification \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_safetensors false \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --mlp_hidden_dim 100 \
+   --model_name specroute \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   $FP16_FLAG

improve_gainlora/T5_small/gen_script_superni_order2_t5_small_gainlora_inflora.sh

@@ -0,0 +1,743 @@
+#!/bin/bash
+#SBATCH -J cl                           
+#SBATCH -o cl-%j.out                       
+#SBATCH -p compute 
+#SBATCH -N 1                           
+#SBATCH -t 20:00:00   
+#SBATCH --mem 128G 
+#SBATCH --gres=gpu:a100-sxm4-80gb:1  
+
+export CUDA_DEVICE_ORDER="PCI_BUS_ID"
+
+port=$(shuf -i25000-30000 -n1)  
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task748_glucose_reverse_cause_event_detection \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/1-task748_glucose_reverse_cause_event_detection \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --load_best_model_at_end \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --model_name gainlora_inflora \
+   --mlp_hidden_dim 100 \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task073_commonsenseqa_answer_generation \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/2-task073_commonsenseqa_answer_generation \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task073_commonsenseqa_answer_generation \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --model_name gainlora_inflora \
+   --mlp_hidden_dim 100 \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/2-task073_commonsenseqa_answer_generation/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/2-task073_commonsenseqa_answer_generation/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/2-task073_commonsenseqa_answer_generation/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task1590_diplomacy_text_generation \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/3-task1590_diplomacy_text_generation \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task1590_diplomacy_text_generation \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --model_name gainlora_inflora \
+   --mlp_hidden_dim 100 \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/3-task1590_diplomacy_text_generation/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/3-task1590_diplomacy_text_generation/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/3-task1590_diplomacy_text_generation/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task639_multi_woz_user_utterance_generation \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/4-task639_multi_woz_user_utterance_generation \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task639_multi_woz_user_utterance_generation \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --model_name gainlora_inflora \
+   --mlp_hidden_dim 100 \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/3-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task1572_samsum_summary \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/5-task1572_samsum_summary \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task1572_samsum_summary \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --model_name gainlora_inflora \
+   --mlp_hidden_dim 100 \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/5-task1572_samsum_summary/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/3-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/5-task1572_samsum_summary/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/5-task1572_samsum_summary/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task1687_sentiment140_classification \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/6-task1687_sentiment140_classification \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task1687_sentiment140_classification \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --model_name gainlora_inflora \
+   --mlp_hidden_dim 100 \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/6-task1687_sentiment140_classification/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/3-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/5-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/6-task1687_sentiment140_classification/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/6-task1687_sentiment140_classification/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task591_sciq_answer_generation \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/7-task591_sciq_answer_generation \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task591_sciq_answer_generation \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --model_name gainlora_inflora \
+   --mlp_hidden_dim 100 \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/7-task591_sciq_answer_generation/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/3-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/5-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/6-task1687_sentiment140_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/7-task591_sciq_answer_generation/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/7-task591_sciq_answer_generation/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task363_sst2_polarity_classification \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/8-task363_sst2_polarity_classification \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task363_sst2_polarity_classification \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --model_name gainlora_inflora \
+   --mlp_hidden_dim 100 \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/8-task363_sst2_polarity_classification/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/3-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/5-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/6-task1687_sentiment140_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/7-task591_sciq_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/8-task363_sst2_polarity_classification/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/8-task363_sst2_polarity_classification/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task1510_evalution_relation_extraction \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/9-task1510_evalution_relation_extraction \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task1510_evalution_relation_extraction \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --model_name gainlora_inflora \
+   --mlp_hidden_dim 100 \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/9-task1510_evalution_relation_extraction/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/3-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/5-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/6-task1687_sentiment140_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/7-task591_sciq_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/8-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/9-task1510_evalution_relation_extraction/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/9-task1510_evalution_relation_extraction/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task1729_personachat_generate_next \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/10-task1729_personachat_generate_next \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task1729_personachat_generate_next \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --model_name gainlora_inflora \
+   --mlp_hidden_dim 100 \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/10-task1729_personachat_generate_next/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/3-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/5-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/6-task1687_sentiment140_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/7-task591_sciq_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/8-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/9-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/10-task1729_personachat_generate_next/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/10-task1729_personachat_generate_next/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task181_outcome_extraction \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/11-task181_outcome_extraction \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task181_outcome_extraction \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --model_name gainlora_inflora \
+   --mlp_hidden_dim 100 \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/11-task181_outcome_extraction/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/3-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/5-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/6-task1687_sentiment140_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/7-task591_sciq_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/8-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/9-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/10-task1729_personachat_generate_next/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/11-task181_outcome_extraction/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/11-task181_outcome_extraction/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task511_reddit_tifu_long_text_summarization \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/12-task511_reddit_tifu_long_text_summarization \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task511_reddit_tifu_long_text_summarization \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --model_name gainlora_inflora \
+   --mlp_hidden_dim 100 \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/12-task511_reddit_tifu_long_text_summarization/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/3-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/5-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/6-task1687_sentiment140_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/7-task591_sciq_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/8-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/9-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/10-task1729_personachat_generate_next/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/11-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/12-task511_reddit_tifu_long_text_summarization/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/12-task511_reddit_tifu_long_text_summarization/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task002_quoref_answer_generation \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/13-task002_quoref_answer_generation \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task002_quoref_answer_generation \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --model_name gainlora_inflora \
+   --mlp_hidden_dim 100 \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/13-task002_quoref_answer_generation/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/3-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/5-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/6-task1687_sentiment140_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/7-task591_sciq_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/8-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/9-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/10-task1729_personachat_generate_next/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/11-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/12-task511_reddit_tifu_long_text_summarization/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/13-task002_quoref_answer_generation/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/13-task002_quoref_answer_generation/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task1290_xsum_summarization \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/14-task1290_xsum_summarization \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task1290_xsum_summarization \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --model_name gainlora_inflora \
+   --mlp_hidden_dim 100 \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/14-task1290_xsum_summarization/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/3-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/5-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/6-task1687_sentiment140_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/7-task591_sciq_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/8-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/9-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/10-task1729_personachat_generate_next/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/11-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/12-task511_reddit_tifu_long_text_summarization/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/13-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/14-task1290_xsum_summarization/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/14-task1290_xsum_summarization/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task875_emotion_classification \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/15-task875_emotion_classification \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task875_emotion_classification \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --model_name gainlora_inflora \
+   --mlp_hidden_dim 100 \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1
+
+python score.py gen_script_superni_order2_t5_small_gainlora_inflora gen_script_superni_order2_t5_small_gainlora_inflora

improve_gainlora/T5_small/gen_script_superni_order2_t5_small_inflora.sh

@@ -0,0 +1,713 @@
+#!/bin/bash
+#SBATCH -J cl                           
+#SBATCH -o cl-%j.out                       
+#SBATCH -p compute 
+#SBATCH -N 1                           
+#SBATCH -t 20:00:00   
+#SBATCH --mem 128G 
+#SBATCH --gres=gpu:a100-sxm4-80gb:1  
+
+export CUDA_DEVICE_ORDER="PCI_BUS_ID"
+
+port=$(shuf -i25000-30000 -n1)  
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task748_glucose_reverse_cause_event_detection \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/1-task748_glucose_reverse_cause_event_detection \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --load_best_model_at_end \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --model_name inflora \
+   --threshold 0.995 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task073_commonsenseqa_answer_generation \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/2-task073_commonsenseqa_answer_generation \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task073_commonsenseqa_answer_generation \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --model_name inflora \
+   --threshold 0.995 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/2-task073_commonsenseqa_answer_generation/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/2-task073_commonsenseqa_answer_generation/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/2-task073_commonsenseqa_answer_generation/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task1590_diplomacy_text_generation \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/3-task1590_diplomacy_text_generation \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task1590_diplomacy_text_generation \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --model_name inflora \
+   --threshold 0.995 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/3-task1590_diplomacy_text_generation/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/3-task1590_diplomacy_text_generation/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/3-task1590_diplomacy_text_generation/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task639_multi_woz_user_utterance_generation \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/4-task639_multi_woz_user_utterance_generation \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task639_multi_woz_user_utterance_generation \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --model_name inflora \
+   --threshold 0.995 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/3-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task1572_samsum_summary \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/5-task1572_samsum_summary \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task1572_samsum_summary \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --model_name inflora \
+   --threshold 0.995 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/5-task1572_samsum_summary/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/3-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/5-task1572_samsum_summary/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/5-task1572_samsum_summary/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task1687_sentiment140_classification \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/6-task1687_sentiment140_classification \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task1687_sentiment140_classification \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --model_name inflora \
+   --threshold 0.995 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/6-task1687_sentiment140_classification/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/3-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/5-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/6-task1687_sentiment140_classification/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/6-task1687_sentiment140_classification/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task591_sciq_answer_generation \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/7-task591_sciq_answer_generation \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task591_sciq_answer_generation \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --model_name inflora \
+   --threshold 0.995 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/7-task591_sciq_answer_generation/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/3-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/5-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/6-task1687_sentiment140_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/7-task591_sciq_answer_generation/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/7-task591_sciq_answer_generation/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task363_sst2_polarity_classification \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/8-task363_sst2_polarity_classification \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task363_sst2_polarity_classification \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --model_name inflora \
+   --threshold 0.995 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/8-task363_sst2_polarity_classification/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/3-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/5-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/6-task1687_sentiment140_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/7-task591_sciq_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/8-task363_sst2_polarity_classification/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/8-task363_sst2_polarity_classification/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task1510_evalution_relation_extraction \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/9-task1510_evalution_relation_extraction \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task1510_evalution_relation_extraction \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --model_name inflora \
+   --threshold 0.995 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/9-task1510_evalution_relation_extraction/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/3-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/5-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/6-task1687_sentiment140_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/7-task591_sciq_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/8-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/9-task1510_evalution_relation_extraction/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/9-task1510_evalution_relation_extraction/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task1729_personachat_generate_next \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/10-task1729_personachat_generate_next \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task1729_personachat_generate_next \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --model_name inflora \
+   --threshold 0.995 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/10-task1729_personachat_generate_next/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/3-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/5-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/6-task1687_sentiment140_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/7-task591_sciq_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/8-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/9-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/10-task1729_personachat_generate_next/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/10-task1729_personachat_generate_next/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task181_outcome_extraction \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/11-task181_outcome_extraction \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task181_outcome_extraction \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --model_name inflora \
+   --threshold 0.995 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/11-task181_outcome_extraction/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/3-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/5-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/6-task1687_sentiment140_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/7-task591_sciq_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/8-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/9-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/10-task1729_personachat_generate_next/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/11-task181_outcome_extraction/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/11-task181_outcome_extraction/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task511_reddit_tifu_long_text_summarization \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/12-task511_reddit_tifu_long_text_summarization \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task511_reddit_tifu_long_text_summarization \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --model_name inflora \
+   --threshold 0.995 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/12-task511_reddit_tifu_long_text_summarization/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/3-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/5-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/6-task1687_sentiment140_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/7-task591_sciq_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/8-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/9-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/10-task1729_personachat_generate_next/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/11-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/12-task511_reddit_tifu_long_text_summarization/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/12-task511_reddit_tifu_long_text_summarization/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task002_quoref_answer_generation \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/13-task002_quoref_answer_generation \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task002_quoref_answer_generation \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --model_name inflora \
+   --threshold 0.995 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/13-task002_quoref_answer_generation/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/3-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/5-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/6-task1687_sentiment140_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/7-task591_sciq_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/8-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/9-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/10-task1729_personachat_generate_next/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/11-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/12-task511_reddit_tifu_long_text_summarization/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/13-task002_quoref_answer_generation/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/13-task002_quoref_answer_generation/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task1290_xsum_summarization \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/14-task1290_xsum_summarization \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task1290_xsum_summarization \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --model_name inflora \
+   --threshold 0.995 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/14-task1290_xsum_summarization/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/3-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/5-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/6-task1687_sentiment140_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/7-task591_sciq_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/8-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/9-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/10-task1729_personachat_generate_next/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/11-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/12-task511_reddit_tifu_long_text_summarization/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/13-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/14-task1290_xsum_summarization/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/14-task1290_xsum_summarization/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task875_emotion_classification \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/15-task875_emotion_classification \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task875_emotion_classification \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --model_name inflora \
+   --threshold 0.995 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1
+
+python score.py gen_script_superni_order2_t5_small_inflora gen_script_superni_order2_t5_small_inflora

improve_gainlora/T5_small/gen_script_superni_order2_t5_small_specroute.sh

@@ -0,0 +1,804 @@
+#!/bin/bash
+#SBATCH -J cl
+#SBATCH -o cl-%j.out
+#SBATCH -p compute
+#SBATCH -N 1
+#SBATCH -t 20:00:00
+#SBATCH --mem 128G
+#SBATCH --gres=gpu:2
+
+export CUDA_DEVICE_ORDER="PCI_BUS_ID"
+
+port=$(shuf -i25000-30000 -n1)
+
+# ============================================================
+# Auto-detect GPU count and type for optimal parallelism
+# ============================================================
+NUM_GPUS=$(nvidia-smi -L 2>/dev/null | wc -l)
+GPU_MEM=$(nvidia-smi --query-gpu=memory.total --format=csv,noheader,nounits 2>/dev/null | head -1)
+
+if [ -z "$GPU_MEM" ]; then
+    echo "ERROR: No GPU detected!"
+    exit 1
+fi
+
+# Determine GPU type
+if [ "$GPU_MEM" -lt 20000 ]; then
+    IS_T4=1
+    echo "[GPU] Detected T4 GPUs (${GPU_MEM}MB VRAM each)"
+else
+    IS_T4=0
+    echo "[GPU] Detected high-memory GPUs (${GPU_MEM}MB VRAM each)"
+fi
+
+# Determine parallelism strategy
+if [ "$IS_T4" -eq 1 ] && [ "$NUM_GPUS" -ge 2 ]; then
+    GPU_MODE="t4_2gpu"
+    GPU_IDS="0,1"
+    FP16_FLAG=""
+    echo "[GPU] Strategy: 2x T4 DataParallel + fp32 + gradient_checkpointing"
+elif [ "$IS_T4" -eq 1 ]; then
+    GPU_MODE="t4_1gpu"
+    GPU_IDS="${1:-0}"
+    FP16_FLAG=""
+    echo "[GPU] Strategy: 1x T4 + fp32 + gradient_checkpointing"
+else
+    GPU_MODE="a100"
+    GPU_IDS="${1:-0}"
+    FP16_FLAG=""
+    echo "[GPU] Strategy: A100 (single GPU, fp32)"
+fi
+
+echo "[GPU] Using CUDA_VISIBLE_DEVICES=$GPU_IDS"
+echo "============================================================"
+echo ""
+
+if [ "$GPU_MODE" = "t4_2gpu" ]; then
+    BSZ=8;  GA=2; EVAL_BSZ=16
+elif [ "$GPU_MODE" = "t4_1gpu" ]; then
+    BSZ=16; GA=2; EVAL_BSZ=16
+else
+    BSZ=32; GA=1; EVAL_BSZ=32
+fi
+
+CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task875_emotion_classification,task002_quoref_answer_generation,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1572_samsum_summary,task511_reddit_tifu_long_text_summarization,task1290_xsum_summarization,task639_multi_woz_user_utterance_generation,task1510_evalution_relation_extraction,task181_outcome_extraction,task1729_personachat_generate_next,task1590_diplomacy_text_generation \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task748_glucose_reverse_cause_event_detection \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/1-task748_glucose_reverse_cause_event_detection \
+   --per_device_train_batch_size $BSZ \
+   --per_device_eval_batch_size $EVAL_BSZ \
+   --gradient_accumulation_steps $GA \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_specroute \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --run_single True \
+   --data_replay_freq -1 \
+   --mlp_hidden_dim 100 \
+   --model_name specroute \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   $FP16_FLAG
+
+if [ "$GPU_MODE" = "t4_2gpu" ]; then
+    BSZ=8;  GA=2; EVAL_BSZ=16
+elif [ "$GPU_MODE" = "t4_1gpu" ]; then
+    BSZ=16; GA=2; EVAL_BSZ=16
+else
+    BSZ=32; GA=1; EVAL_BSZ=32
+fi
+
+CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task875_emotion_classification,task002_quoref_answer_generation,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1572_samsum_summary,task511_reddit_tifu_long_text_summarization,task1290_xsum_summarization,task639_multi_woz_user_utterance_generation,task1510_evalution_relation_extraction,task181_outcome_extraction,task1729_personachat_generate_next,task1590_diplomacy_text_generation \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task073_commonsenseqa_answer_generation \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/2-task073_commonsenseqa_answer_generation \
+   --per_device_train_batch_size $BSZ \
+   --per_device_eval_batch_size $EVAL_BSZ \
+   --gradient_accumulation_steps $GA \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_specroute \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task073_commonsenseqa_answer_generation \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --mlp_hidden_dim 100 \
+   --model_name specroute \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   $FP16_FLAG
+
+if [ "$GPU_MODE" = "t4_2gpu" ]; then
+    BSZ=8;  GA=2; EVAL_BSZ=16
+elif [ "$GPU_MODE" = "t4_1gpu" ]; then
+    BSZ=16; GA=2; EVAL_BSZ=16
+else
+    BSZ=32; GA=1; EVAL_BSZ=32
+fi
+
+CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/2-task073_commonsenseqa_answer_generation/saved_weights \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task875_emotion_classification,task002_quoref_answer_generation,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1572_samsum_summary,task511_reddit_tifu_long_text_summarization,task1290_xsum_summarization,task639_multi_woz_user_utterance_generation,task1510_evalution_relation_extraction,task181_outcome_extraction,task1729_personachat_generate_next,task1590_diplomacy_text_generation \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task875_emotion_classification \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/3-task875_emotion_classification \
+   --per_device_train_batch_size $BSZ \
+   --per_device_eval_batch_size $EVAL_BSZ \
+   --gradient_accumulation_steps $GA \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_specroute \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task875_emotion_classification \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --mlp_hidden_dim 100 \
+   --model_name specroute \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   $FP16_FLAG
+
+if [ "$GPU_MODE" = "t4_2gpu" ]; then
+    BSZ=8;  GA=2; EVAL_BSZ=16
+elif [ "$GPU_MODE" = "t4_1gpu" ]; then
+    BSZ=16; GA=2; EVAL_BSZ=16
+else
+    BSZ=32; GA=1; EVAL_BSZ=32
+fi
+
+CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/3-task875_emotion_classification/saved_weights \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task875_emotion_classification,task002_quoref_answer_generation,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1572_samsum_summary,task511_reddit_tifu_long_text_summarization,task1290_xsum_summarization,task639_multi_woz_user_utterance_generation,task1510_evalution_relation_extraction,task181_outcome_extraction,task1729_personachat_generate_next,task1590_diplomacy_text_generation \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task002_quoref_answer_generation \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/4-task002_quoref_answer_generation \
+   --per_device_train_batch_size $BSZ \
+   --per_device_eval_batch_size $EVAL_BSZ \
+   --gradient_accumulation_steps $GA \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_specroute \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task002_quoref_answer_generation \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --mlp_hidden_dim 100 \
+   --model_name specroute \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   $FP16_FLAG
+
+if [ "$GPU_MODE" = "t4_2gpu" ]; then
+    BSZ=8;  GA=2; EVAL_BSZ=16
+elif [ "$GPU_MODE" = "t4_1gpu" ]; then
+    BSZ=16; GA=2; EVAL_BSZ=16
+else
+    BSZ=32; GA=1; EVAL_BSZ=32
+fi
+
+CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/3-task875_emotion_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/4-task002_quoref_answer_generation/saved_weights \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task875_emotion_classification,task002_quoref_answer_generation,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1572_samsum_summary,task511_reddit_tifu_long_text_summarization,task1290_xsum_summarization,task639_multi_woz_user_utterance_generation,task1510_evalution_relation_extraction,task181_outcome_extraction,task1729_personachat_generate_next,task1590_diplomacy_text_generation \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task1687_sentiment140_classification \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/5-task1687_sentiment140_classification \
+   --per_device_train_batch_size $BSZ \
+   --per_device_eval_batch_size $EVAL_BSZ \
+   --gradient_accumulation_steps $GA \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_specroute \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task1687_sentiment140_classification \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --mlp_hidden_dim 100 \
+   --model_name specroute \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   $FP16_FLAG
+
+if [ "$GPU_MODE" = "t4_2gpu" ]; then
+    BSZ=8;  GA=2; EVAL_BSZ=16
+elif [ "$GPU_MODE" = "t4_1gpu" ]; then
+    BSZ=16; GA=2; EVAL_BSZ=16
+else
+    BSZ=32; GA=1; EVAL_BSZ=32
+fi
+
+CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/3-task875_emotion_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/4-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/5-task1687_sentiment140_classification/saved_weights \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task875_emotion_classification,task002_quoref_answer_generation,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1572_samsum_summary,task511_reddit_tifu_long_text_summarization,task1290_xsum_summarization,task639_multi_woz_user_utterance_generation,task1510_evalution_relation_extraction,task181_outcome_extraction,task1729_personachat_generate_next,task1590_diplomacy_text_generation \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task591_sciq_answer_generation \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/6-task591_sciq_answer_generation \
+   --per_device_train_batch_size $BSZ \
+   --per_device_eval_batch_size $EVAL_BSZ \
+   --gradient_accumulation_steps $GA \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_specroute \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task591_sciq_answer_generation \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --mlp_hidden_dim 100 \
+   --model_name specroute \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   $FP16_FLAG
+
+if [ "$GPU_MODE" = "t4_2gpu" ]; then
+    BSZ=8;  GA=2; EVAL_BSZ=16
+elif [ "$GPU_MODE" = "t4_1gpu" ]; then
+    BSZ=16; GA=2; EVAL_BSZ=16
+else
+    BSZ=32; GA=1; EVAL_BSZ=32
+fi
+
+CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/3-task875_emotion_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/4-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/5-task1687_sentiment140_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/6-task591_sciq_answer_generation/saved_weights \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task875_emotion_classification,task002_quoref_answer_generation,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1572_samsum_summary,task511_reddit_tifu_long_text_summarization,task1290_xsum_summarization,task639_multi_woz_user_utterance_generation,task1510_evalution_relation_extraction,task181_outcome_extraction,task1729_personachat_generate_next,task1590_diplomacy_text_generation \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task363_sst2_polarity_classification \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/7-task363_sst2_polarity_classification \
+   --per_device_train_batch_size $BSZ \
+   --per_device_eval_batch_size $EVAL_BSZ \
+   --gradient_accumulation_steps $GA \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_specroute \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task363_sst2_polarity_classification \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --mlp_hidden_dim 100 \
+   --model_name specroute \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   $FP16_FLAG
+
+if [ "$GPU_MODE" = "t4_2gpu" ]; then
+    BSZ=8;  GA=2; EVAL_BSZ=16
+elif [ "$GPU_MODE" = "t4_1gpu" ]; then
+    BSZ=16; GA=2; EVAL_BSZ=16
+else
+    BSZ=32; GA=1; EVAL_BSZ=32
+fi
+
+CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/3-task875_emotion_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/4-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/5-task1687_sentiment140_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/6-task591_sciq_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/7-task363_sst2_polarity_classification/saved_weights \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task875_emotion_classification,task002_quoref_answer_generation,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1572_samsum_summary,task511_reddit_tifu_long_text_summarization,task1290_xsum_summarization,task639_multi_woz_user_utterance_generation,task1510_evalution_relation_extraction,task181_outcome_extraction,task1729_personachat_generate_next,task1590_diplomacy_text_generation \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task1572_samsum_summary \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/8-task1572_samsum_summary \
+   --per_device_train_batch_size $BSZ \
+   --per_device_eval_batch_size $EVAL_BSZ \
+   --gradient_accumulation_steps $GA \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_specroute \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task1572_samsum_summary \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --mlp_hidden_dim 100 \
+   --model_name specroute \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   $FP16_FLAG
+
+if [ "$GPU_MODE" = "t4_2gpu" ]; then
+    BSZ=8;  GA=2; EVAL_BSZ=16
+elif [ "$GPU_MODE" = "t4_1gpu" ]; then
+    BSZ=16; GA=2; EVAL_BSZ=16
+else
+    BSZ=32; GA=1; EVAL_BSZ=32
+fi
+
+CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/3-task875_emotion_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/4-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/5-task1687_sentiment140_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/6-task591_sciq_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/7-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/8-task1572_samsum_summary/saved_weights \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task875_emotion_classification,task002_quoref_answer_generation,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1572_samsum_summary,task511_reddit_tifu_long_text_summarization,task1290_xsum_summarization,task639_multi_woz_user_utterance_generation,task1510_evalution_relation_extraction,task181_outcome_extraction,task1729_personachat_generate_next,task1590_diplomacy_text_generation \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task511_reddit_tifu_long_text_summarization \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/9-task511_reddit_tifu_long_text_summarization \
+   --per_device_train_batch_size $BSZ \
+   --per_device_eval_batch_size $EVAL_BSZ \
+   --gradient_accumulation_steps $GA \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_specroute \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task511_reddit_tifu_long_text_summarization \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --mlp_hidden_dim 100 \
+   --model_name specroute \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   $FP16_FLAG
+
+if [ "$GPU_MODE" = "t4_2gpu" ]; then
+    BSZ=8;  GA=2; EVAL_BSZ=16
+elif [ "$GPU_MODE" = "t4_1gpu" ]; then
+    BSZ=16; GA=2; EVAL_BSZ=16
+else
+    BSZ=32; GA=1; EVAL_BSZ=32
+fi
+
+CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/3-task875_emotion_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/4-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/5-task1687_sentiment140_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/6-task591_sciq_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/7-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/8-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/9-task511_reddit_tifu_long_text_summarization/saved_weights \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task875_emotion_classification,task002_quoref_answer_generation,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1572_samsum_summary,task511_reddit_tifu_long_text_summarization,task1290_xsum_summarization,task639_multi_woz_user_utterance_generation,task1510_evalution_relation_extraction,task181_outcome_extraction,task1729_personachat_generate_next,task1590_diplomacy_text_generation \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task1290_xsum_summarization \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/10-task1290_xsum_summarization \
+   --per_device_train_batch_size $BSZ \
+   --per_device_eval_batch_size $EVAL_BSZ \
+   --gradient_accumulation_steps $GA \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_specroute \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task1290_xsum_summarization \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --mlp_hidden_dim 100 \
+   --model_name specroute \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   $FP16_FLAG
+
+if [ "$GPU_MODE" = "t4_2gpu" ]; then
+    BSZ=8;  GA=2; EVAL_BSZ=16
+elif [ "$GPU_MODE" = "t4_1gpu" ]; then
+    BSZ=16; GA=2; EVAL_BSZ=16
+else
+    BSZ=32; GA=1; EVAL_BSZ=32
+fi
+
+CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/3-task875_emotion_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/4-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/5-task1687_sentiment140_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/6-task591_sciq_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/7-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/8-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/9-task511_reddit_tifu_long_text_summarization/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/10-task1290_xsum_summarization/saved_weights \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task875_emotion_classification,task002_quoref_answer_generation,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1572_samsum_summary,task511_reddit_tifu_long_text_summarization,task1290_xsum_summarization,task639_multi_woz_user_utterance_generation,task1510_evalution_relation_extraction,task181_outcome_extraction,task1729_personachat_generate_next,task1590_diplomacy_text_generation \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task639_multi_woz_user_utterance_generation \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/11-task639_multi_woz_user_utterance_generation \
+   --per_device_train_batch_size $BSZ \
+   --per_device_eval_batch_size $EVAL_BSZ \
+   --gradient_accumulation_steps $GA \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_specroute \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task639_multi_woz_user_utterance_generation \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --mlp_hidden_dim 100 \
+   --model_name specroute \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   $FP16_FLAG
+
+if [ "$GPU_MODE" = "t4_2gpu" ]; then
+    BSZ=8;  GA=2; EVAL_BSZ=16
+elif [ "$GPU_MODE" = "t4_1gpu" ]; then
+    BSZ=16; GA=2; EVAL_BSZ=16
+else
+    BSZ=32; GA=1; EVAL_BSZ=32
+fi
+
+CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/3-task875_emotion_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/4-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/5-task1687_sentiment140_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/6-task591_sciq_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/7-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/8-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/9-task511_reddit_tifu_long_text_summarization/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/10-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/11-task639_multi_woz_user_utterance_generation/saved_weights \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task875_emotion_classification,task002_quoref_answer_generation,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1572_samsum_summary,task511_reddit_tifu_long_text_summarization,task1290_xsum_summarization,task639_multi_woz_user_utterance_generation,task1510_evalution_relation_extraction,task181_outcome_extraction,task1729_personachat_generate_next,task1590_diplomacy_text_generation \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task1510_evalution_relation_extraction \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/12-task1510_evalution_relation_extraction \
+   --per_device_train_batch_size $BSZ \
+   --per_device_eval_batch_size $EVAL_BSZ \
+   --gradient_accumulation_steps $GA \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_specroute \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task1510_evalution_relation_extraction \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --mlp_hidden_dim 100 \
+   --model_name specroute \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   $FP16_FLAG
+
+if [ "$GPU_MODE" = "t4_2gpu" ]; then
+    BSZ=8;  GA=2; EVAL_BSZ=16
+elif [ "$GPU_MODE" = "t4_1gpu" ]; then
+    BSZ=16; GA=2; EVAL_BSZ=16
+else
+    BSZ=32; GA=1; EVAL_BSZ=32
+fi
+
+CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/3-task875_emotion_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/4-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/5-task1687_sentiment140_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/6-task591_sciq_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/7-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/8-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/9-task511_reddit_tifu_long_text_summarization/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/10-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/11-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/12-task1510_evalution_relation_extraction/saved_weights \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task875_emotion_classification,task002_quoref_answer_generation,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1572_samsum_summary,task511_reddit_tifu_long_text_summarization,task1290_xsum_summarization,task639_multi_woz_user_utterance_generation,task1510_evalution_relation_extraction,task181_outcome_extraction,task1729_personachat_generate_next,task1590_diplomacy_text_generation \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task181_outcome_extraction \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/13-task181_outcome_extraction \
+   --per_device_train_batch_size $BSZ \
+   --per_device_eval_batch_size $EVAL_BSZ \
+   --gradient_accumulation_steps $GA \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_specroute \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task181_outcome_extraction \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --mlp_hidden_dim 100 \
+   --model_name specroute \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   $FP16_FLAG
+
+if [ "$GPU_MODE" = "t4_2gpu" ]; then
+    BSZ=8;  GA=2; EVAL_BSZ=16
+elif [ "$GPU_MODE" = "t4_1gpu" ]; then
+    BSZ=16; GA=2; EVAL_BSZ=16
+else
+    BSZ=32; GA=1; EVAL_BSZ=32
+fi
+
+CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/3-task875_emotion_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/4-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/5-task1687_sentiment140_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/6-task591_sciq_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/7-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/8-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/9-task511_reddit_tifu_long_text_summarization/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/10-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/11-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/12-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/13-task181_outcome_extraction/saved_weights \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task875_emotion_classification,task002_quoref_answer_generation,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1572_samsum_summary,task511_reddit_tifu_long_text_summarization,task1290_xsum_summarization,task639_multi_woz_user_utterance_generation,task1510_evalution_relation_extraction,task181_outcome_extraction,task1729_personachat_generate_next,task1590_diplomacy_text_generation \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task1729_personachat_generate_next \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/14-task1729_personachat_generate_next \
+   --per_device_train_batch_size $BSZ \
+   --per_device_eval_batch_size $EVAL_BSZ \
+   --gradient_accumulation_steps $GA \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_specroute \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task1729_personachat_generate_next \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --mlp_hidden_dim 100 \
+   --model_name specroute \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   $FP16_FLAG
+
+if [ "$GPU_MODE" = "t4_2gpu" ]; then
+    BSZ=8;  GA=2; EVAL_BSZ=16
+elif [ "$GPU_MODE" = "t4_1gpu" ]; then
+    BSZ=16; GA=2; EVAL_BSZ=16
+else
+    BSZ=32; GA=1; EVAL_BSZ=32
+fi
+
+CUDA_VISIBLE_DEVICES=$GPU_IDS python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/3-task875_emotion_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/4-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/5-task1687_sentiment140_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/6-task591_sciq_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/7-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/8-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/9-task511_reddit_tifu_long_text_summarization/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/10-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/11-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/12-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/13-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/14-task1729_personachat_generate_next/saved_weights \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task875_emotion_classification,task002_quoref_answer_generation,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1572_samsum_summary,task511_reddit_tifu_long_text_summarization,task1290_xsum_summarization,task639_multi_woz_user_utterance_generation,task1510_evalution_relation_extraction,task181_outcome_extraction,task1729_personachat_generate_next,task1590_diplomacy_text_generation \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task1590_diplomacy_text_generation \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_specroute/outputs/15-task1590_diplomacy_text_generation \
+   --per_device_train_batch_size $BSZ \
+   --per_device_eval_batch_size $EVAL_BSZ \
+   --gradient_accumulation_steps $GA \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_specroute \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task1590_diplomacy_text_generation \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --mlp_hidden_dim 100 \
+   --model_name specroute \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   $FP16_FLAG

root_gainlora/T5_small/gen_script_long_order3_t5_small_gainlora_inflora.sh

@@ -0,0 +1,763 @@
+#!/bin/bash
+#SBATCH -J cl                           
+#SBATCH -o cl-%j.out                       
+#SBATCH -p compute 
+#SBATCH -N 1                           
+#SBATCH -t 20:00:00   
+#SBATCH --mem 128G 
+#SBATCH --gres=gpu:a100-sxm4-80gb:1  
+
+export CUDA_DEVICE_ORDER="PCI_BUS_ID"
+
+port=$(shuf -i25000-30000 -n1)  
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/yelp \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/1-yelp \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/1-yelp/checkpoint*
+
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/1-yelp/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/1-yelp/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/1-yelp/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/amazon \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/2-amazon \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_amazon \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/2-amazon/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/2-amazon/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/1-yelp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/2-amazon/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/2-amazon/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/mnli \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/3-mnli \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_mnli \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/3-mnli/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/3-mnli/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/1-yelp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/2-amazon/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/3-mnli/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/3-mnli/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/cb \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/4-cb \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_cb \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/4-cb/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/4-cb/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/1-yelp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/2-amazon/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/3-mnli/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/4-cb/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/4-cb/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/copa \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/5-copa \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_copa \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/5-copa/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/5-copa/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/1-yelp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/2-amazon/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/3-mnli/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/4-cb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/5-copa/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/5-copa/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/qqp \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/6-qqp \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_qqp \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/6-qqp/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/6-qqp/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/1-yelp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/2-amazon/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/3-mnli/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/4-cb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/5-copa/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/6-qqp/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/6-qqp/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/rte \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/7-rte \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_rte \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/7-rte/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/7-rte/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/1-yelp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/2-amazon/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/3-mnli/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/4-cb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/5-copa/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/6-qqp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/7-rte/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/7-rte/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/imdb \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/8-imdb \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_imdb \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/8-imdb/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/8-imdb/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/1-yelp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/2-amazon/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/3-mnli/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/4-cb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/5-copa/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/6-qqp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/8-imdb/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/8-imdb/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/sst2 \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/9-sst2 \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_sst2 \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/9-sst2/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/9-sst2/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/1-yelp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/2-amazon/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/3-mnli/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/4-cb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/5-copa/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/6-qqp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/8-imdb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/9-sst2/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/9-sst2/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/dbpedia \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/10-dbpedia \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_dbpedia \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/10-dbpedia/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/10-dbpedia/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/1-yelp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/2-amazon/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/3-mnli/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/4-cb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/5-copa/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/6-qqp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/8-imdb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/9-sst2/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/10-dbpedia/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/10-dbpedia/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/agnews \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/11-agnews \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_agnews \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/11-agnews/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/11-agnews/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/1-yelp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/2-amazon/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/3-mnli/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/4-cb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/5-copa/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/6-qqp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/8-imdb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/9-sst2/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/10-dbpedia/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/11-agnews/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/11-agnews/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/yahoo \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/12-yahoo \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_yahoo \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/12-yahoo/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/12-yahoo/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/1-yelp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/2-amazon/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/3-mnli/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/4-cb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/5-copa/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/6-qqp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/8-imdb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/9-sst2/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/10-dbpedia/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/11-agnews/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/12-yahoo/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/12-yahoo/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/multirc \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/13-multirc \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_multirc \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/13-multirc/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/13-multirc/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/1-yelp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/2-amazon/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/3-mnli/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/4-cb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/5-copa/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/6-qqp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/8-imdb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/9-sst2/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/10-dbpedia/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/11-agnews/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/12-yahoo/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/13-multirc/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/13-multirc/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/boolq \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/14-boolq \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_boolq \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/14-boolq/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/14-boolq/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/1-yelp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/2-amazon/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/3-mnli/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/4-cb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/5-copa/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/6-qqp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/8-imdb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/9-sst2/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/10-dbpedia/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/11-agnews/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/12-yahoo/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/13-multirc/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/14-boolq/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/14-boolq/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/wic \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/15-wic \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_wic \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_gainlora_inflora/outputs/15-wic/checkpoint*
+   
+sleep 5
+
+CUDA_VISIBLE_DEVICES=$1 python score.py gen_script_long_order3_t5_small_gainlora_inflora gen_script_long_order3_t5_small_gainlora_inflora
+
+
+

root_gainlora/T5_small/gen_script_long_order3_t5_small_inflora.sh

@@ -0,0 +1,744 @@
+#!/bin/bash
+#SBATCH -J cl                           
+#SBATCH -o cl-%j.out                       
+#SBATCH -p compute 
+#SBATCH -N 1                           
+#SBATCH -t 20:00:00   
+#SBATCH --mem 128G 
+#SBATCH --gres=gpu:a100-sxm4-80gb:1  
+
+export CUDA_DEVICE_ORDER="PCI_BUS_ID"
+
+port=$(shuf -i25000-30000 -n1)  
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/yelp \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/1-yelp \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --model_name inflora \
+   --threshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/1-yelp/checkpoint*
+
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/1-yelp/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/1-yelp/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/1-yelp/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/amazon \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/2-amazon \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_amazon \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/2-amazon/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/2-amazon/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/1-yelp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/2-amazon/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/2-amazon/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/mnli \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/3-mnli \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_mnli \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/3-mnli/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/3-mnli/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/1-yelp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/2-amazon/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/3-mnli/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/3-mnli/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/cb \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/4-cb \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_cb \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/4-cb/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/4-cb/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/1-yelp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/2-amazon/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/3-mnli/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/4-cb/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/4-cb/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/copa \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/5-copa \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_copa \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/5-copa/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/5-copa/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/1-yelp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/2-amazon/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/3-mnli/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/4-cb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/5-copa/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/5-copa/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/qqp \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/6-qqp \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_qqp \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/6-qqp/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/6-qqp/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/1-yelp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/2-amazon/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/3-mnli/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/4-cb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/5-copa/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/6-qqp/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/6-qqp/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/rte \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/7-rte \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_rte \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/7-rte/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/7-rte/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/1-yelp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/2-amazon/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/3-mnli/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/4-cb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/5-copa/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/6-qqp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/7-rte/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/7-rte/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/imdb \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/8-imdb \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_imdb \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/8-imdb/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/8-imdb/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/1-yelp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/2-amazon/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/3-mnli/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/4-cb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/5-copa/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/6-qqp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/8-imdb/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/8-imdb/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/sst2 \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/9-sst2 \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_sst2 \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/9-sst2/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/9-sst2/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/1-yelp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/2-amazon/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/3-mnli/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/4-cb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/5-copa/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/6-qqp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/8-imdb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/9-sst2/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/9-sst2/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/dbpedia \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/10-dbpedia \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_dbpedia \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/10-dbpedia/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/10-dbpedia/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/1-yelp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/2-amazon/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/3-mnli/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/4-cb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/5-copa/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/6-qqp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/8-imdb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/9-sst2/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/10-dbpedia/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/10-dbpedia/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/agnews \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/11-agnews \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_agnews \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/11-agnews/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/11-agnews/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/1-yelp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/2-amazon/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/3-mnli/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/4-cb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/5-copa/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/6-qqp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/8-imdb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/9-sst2/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/10-dbpedia/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/11-agnews/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/11-agnews/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/yahoo \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/12-yahoo \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_yahoo \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/12-yahoo/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/12-yahoo/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/1-yelp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/2-amazon/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/3-mnli/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/4-cb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/5-copa/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/6-qqp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/8-imdb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/9-sst2/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/10-dbpedia/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/11-agnews/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/12-yahoo/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/12-yahoo/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/multirc \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/13-multirc \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_multirc \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/13-multirc/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/13-multirc/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/1-yelp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/2-amazon/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/3-mnli/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/4-cb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/5-copa/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/6-qqp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/8-imdb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/9-sst2/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/10-dbpedia/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/11-agnews/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/12-yahoo/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/13-multirc/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/13-multirc/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/boolq \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/14-boolq \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_boolq \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/14-boolq/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/14-boolq/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/1-yelp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/2-amazon/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/3-mnli/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/4-cb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/5-copa/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/6-qqp/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/8-imdb/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/9-sst2/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/10-dbpedia/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/11-agnews/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/12-yahoo/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/13-multirc/saved_weights,logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/14-boolq/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/14-boolq/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order yelp,amazon,mnli,cb,copa,qqp,rte,imdb,sst2,dbpedia,agnews,yahoo,multirc,boolq,wic \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order3_t5_small_configs/wic \
+   --output_dir logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/15-wic \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order3_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_wic \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order3_t5_small_inflora/outputs/15-wic/checkpoint*
+   
+sleep 5
+
+CUDA_VISIBLE_DEVICES=$1 python score.py gen_script_long_order3_t5_small_inflora gen_script_long_order3_t5_small_inflora

root_gainlora/T5_small/gen_script_long_order4_t5_small_gainlora_inflora.sh

@@ -0,0 +1,774 @@
+#!/bin/bash
+#SBATCH -J cl                           
+#SBATCH -o cl-%j.out                       
+#SBATCH -p compute 
+#SBATCH -N 1                           
+#SBATCH -t 20:00:00   
+#SBATCH --mem 128G 
+#SBATCH --gres=gpu:a100-sxm4-80gb:1  
+
+export CUDA_DEVICE_ORDER="PCI_BUS_ID"
+
+port=$(shuf -i25000-30000 -n1)  
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/mnli \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/1-mnli \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order4_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/1-mnli/checkpoint*
+
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/1-mnli/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/1-mnli/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/1-mnli/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/cb \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/2-cb \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10\
+   --run_name gen_script_long_order4_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_cb \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/2-cb/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/2-cb/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/1-mnli/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/2-cb/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/2-cb/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/wic \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/3-wic \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10\
+   --run_name gen_script_long_order4_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_wic \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/3-wic/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/3-wic/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/1-mnli/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/2-cb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/3-wic/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/3-wic/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/copa \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/4-copa \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10\
+   --run_name gen_script_long_order4_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_copa \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/4-copa/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/4-copa/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/1-mnli/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/2-cb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/3-wic/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/4-copa/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/4-copa/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/qqp \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/5-qqp \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10\
+   --run_name gen_script_long_order4_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_qqp \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/5-qqp/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/5-qqp/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/1-mnli/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/2-cb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/3-wic/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/4-copa/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/5-qqp/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/5-qqp/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/boolq \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/6-boolq \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10\
+   --run_name gen_script_long_order4_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_boolq \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/6-boolq/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/6-boolq/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/1-mnli/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/2-cb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/3-wic/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/4-copa/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/5-qqp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/6-boolq/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/6-boolq/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/rte \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/7-rte \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10\
+   --run_name gen_script_long_order4_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_rte \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/7-rte/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/7-rte/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/1-mnli/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/2-cb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/3-wic/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/4-copa/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/5-qqp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/6-boolq/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/7-rte/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/7-rte/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/imdb \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/8-imdb \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10\
+   --run_name gen_script_long_order4_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_imdb \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/8-imdb/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/8-imdb/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/1-mnli/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/2-cb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/3-wic/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/4-copa/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/5-qqp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/6-boolq/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/8-imdb/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/8-imdb/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/yelp \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/9-yelp \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10\
+   --run_name gen_script_long_order4_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_yelp \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/9-yelp/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/9-yelp/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/1-mnli/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/2-cb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/3-wic/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/4-copa/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/5-qqp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/6-boolq/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/8-imdb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/9-yelp/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/9-yelp/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/amazon \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/10-amazon \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10\
+   --run_name gen_script_long_order4_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_amazon \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/10-amazon/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/10-amazon/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/1-mnli/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/2-cb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/3-wic/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/4-copa/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/5-qqp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/6-boolq/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/8-imdb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/9-yelp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/10-amazon/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/10-amazon/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/sst2 \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/11-sst2 \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10\
+   --run_name gen_script_long_order4_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_sst2 \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/11-sst2/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/11-sst2/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/1-mnli/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/2-cb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/3-wic/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/4-copa/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/5-qqp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/6-boolq/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/8-imdb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/9-yelp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/10-amazon/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/11-sst2/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/11-sst2/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/dbpedia \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/12-dbpedia \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10\
+   --run_name gen_script_long_order4_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_dbpedia \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/12-dbpedia/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/12-dbpedia/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/1-mnli/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/2-cb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/3-wic/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/4-copa/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/5-qqp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/6-boolq/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/8-imdb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/9-yelp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/10-amazon/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/11-sst2/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/12-dbpedia/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/12-dbpedia/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/agnews \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/13-agnews \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10\
+   --run_name gen_script_long_order4_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_agnews \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/13-agnews/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/13-agnews/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/1-mnli/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/2-cb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/3-wic/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/4-copa/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/5-qqp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/6-boolq/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/8-imdb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/9-yelp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/10-amazon/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/11-sst2/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/12-dbpedia/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/13-agnews/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/13-agnews/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/multirc \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/14-multirc \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10\
+   --run_name gen_script_long_order4_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_multirc \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/14-multirc/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/14-multirc/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/1-mnli/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/2-cb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/3-wic/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/4-copa/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/5-qqp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/6-boolq/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/8-imdb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/9-yelp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/10-amazon/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/11-sst2/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/12-dbpedia/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/13-agnews/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/14-multirc/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/14-multirc/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/yahoo \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/15-yahoo \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10\
+   --run_name gen_script_long_order4_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_yahoo \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_gainlora_inflora/outputs/15-yahoo/checkpoint*
+   
+sleep 5
+
+CUDA_VISIBLE_DEVICES=$1 python score.py gen_script_long_order4_t5_small_gainlora_inflora gen_script_long_order4_t5_small_gainlora_inflora

root_gainlora/T5_small/gen_script_long_order4_t5_small_inflora.sh

@@ -0,0 +1,744 @@
+#!/bin/bash
+#SBATCH -J cl                           
+#SBATCH -o cl-%j.out                       
+#SBATCH -p compute 
+#SBATCH -N 1                           
+#SBATCH -t 20:00:00   
+#SBATCH --mem 128G 
+#SBATCH --gres=gpu:a100-sxm4-80gb:1  
+
+export CUDA_DEVICE_ORDER="PCI_BUS_ID"
+
+port=$(shuf -i25000-30000 -n1)  
+
+CUDA_VISIBLE_DEVICES=1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/mnli \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/1-mnli \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10 \
+   --run_name gen_script_long_order4_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --model_name inflora \
+   --threshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/1-mnli/checkpoint*
+
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/1-mnli/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/1-mnli/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/1-mnli/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/cb \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/2-cb \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10\
+   --run_name gen_script_long_order4_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_cb \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/2-cb/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/2-cb/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/1-mnli/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/2-cb/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/2-cb/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/wic \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/3-wic \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10\
+   --run_name gen_script_long_order4_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_wic \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/3-wic/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/3-wic/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/1-mnli/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/2-cb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/3-wic/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/3-wic/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/copa \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/4-copa \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10\
+   --run_name gen_script_long_order4_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_copa \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/4-copa/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/4-copa/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/1-mnli/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/2-cb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/3-wic/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/4-copa/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/4-copa/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/qqp \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/5-qqp \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10\
+   --run_name gen_script_long_order4_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_qqp \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/5-qqp/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/5-qqp/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/1-mnli/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/2-cb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/3-wic/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/4-copa/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/5-qqp/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/5-qqp/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/boolq \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/6-boolq \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10\
+   --run_name gen_script_long_order4_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_boolq \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/6-boolq/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/6-boolq/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/1-mnli/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/2-cb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/3-wic/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/4-copa/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/5-qqp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/6-boolq/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/6-boolq/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/rte \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/7-rte \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10\
+   --run_name gen_script_long_order4_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_rte \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/7-rte/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/7-rte/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/1-mnli/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/2-cb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/3-wic/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/4-copa/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/5-qqp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/6-boolq/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/7-rte/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/7-rte/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/imdb \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/8-imdb \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10\
+   --run_name gen_script_long_order4_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_imdb \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/8-imdb/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/8-imdb/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/1-mnli/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/2-cb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/3-wic/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/4-copa/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/5-qqp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/6-boolq/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/8-imdb/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/8-imdb/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/yelp \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/9-yelp \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10\
+   --run_name gen_script_long_order4_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_yelp \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/9-yelp/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/9-yelp/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/1-mnli/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/2-cb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/3-wic/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/4-copa/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/5-qqp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/6-boolq/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/8-imdb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/9-yelp/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/9-yelp/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/amazon \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/10-amazon \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10\
+   --run_name gen_script_long_order4_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_amazon \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/10-amazon/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/10-amazon/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/1-mnli/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/2-cb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/3-wic/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/4-copa/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/5-qqp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/6-boolq/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/8-imdb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/9-yelp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/10-amazon/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/10-amazon/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/sst2 \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/11-sst2 \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10\
+   --run_name gen_script_long_order4_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_sst2 \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/11-sst2/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/11-sst2/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/1-mnli/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/2-cb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/3-wic/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/4-copa/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/5-qqp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/6-boolq/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/8-imdb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/9-yelp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/10-amazon/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/11-sst2/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/11-sst2/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/dbpedia \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/12-dbpedia \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10\
+   --run_name gen_script_long_order4_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_dbpedia \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/12-dbpedia/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/12-dbpedia/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/1-mnli/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/2-cb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/3-wic/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/4-copa/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/5-qqp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/6-boolq/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/8-imdb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/9-yelp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/10-amazon/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/11-sst2/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/12-dbpedia/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/12-dbpedia/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/agnews \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/13-agnews \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10\
+   --run_name gen_script_long_order4_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_agnews \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/13-agnews/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/13-agnews/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/1-mnli/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/2-cb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/3-wic/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/4-copa/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/5-qqp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/6-boolq/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/8-imdb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/9-yelp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/10-amazon/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/11-sst2/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/12-dbpedia/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/13-agnews/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/13-agnews/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/multirc \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/14-multirc \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10\
+   --run_name gen_script_long_order4_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_multirc \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/14-multirc/checkpoint*
+   
+sleep 5
+
+
+CUDA_VISIBLE_DEVICES=1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/14-multirc/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/1-mnli/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/2-cb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/3-wic/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/4-copa/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/5-qqp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/6-boolq/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/7-rte/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/8-imdb/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/9-yelp/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/10-amazon/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/11-sst2/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/12-dbpedia/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/13-agnews/saved_weights,logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/14-multirc/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/14-multirc/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order mnli,cb,wic,copa,qqp,boolq,rte,imdb,yelp,amazon,sst2,dbpedia,agnews,multirc,yahoo \
+   --gen_data_dir generated_data/lora_gen_long_t5 \
+   --task_config_dir configs/gen_script_long_order4_t5_small_configs/yahoo \
+   --output_dir logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/15-yahoo \
+   --per_device_train_batch_size 32 \
+   --per_device_eval_batch_size 256 \
+   --gradient_accumulation_steps 1 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 10\
+   --run_name gen_script_long_order4_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_exact_match_for_yahoo \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 8 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --data_replay_freq -1 \
+   --kl_ratio 0.1 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+rm -rf logs_and_outputs/gen_script_long_order4_t5_small_inflora/outputs/15-yahoo/checkpoint*
+   
+sleep 5
+
+CUDA_VISIBLE_DEVICES=1 python score.py gen_script_long_order4_t5_small_inflora gen_script_long_order4_t5_small_inflora

root_gainlora/T5_small/gen_script_superni_order1_t5_small_gainlora_inflora.sh

@@ -0,0 +1,744 @@
+#!/bin/bash
+#SBATCH -J cl                           
+#SBATCH -o cl-%j.out                       
+#SBATCH -p compute 
+#SBATCH -N 1                           
+#SBATCH -t 20:00:00   
+#SBATCH --mem 128G 
+#SBATCH --gres=gpu:a100-sxm4-80gb:1  
+
+export CUDA_DEVICE_ORDER="PCI_BUS_ID"
+
+port=$(shuf -i25000-30000 -n1)  
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task1572_samsum_summary \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/1-task1572_samsum_summary \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --load_best_model_at_end \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/1-task1572_samsum_summary/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/1-task1572_samsum_summary/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/1-task1572_samsum_summary/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task363_sst2_polarity_classification \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/2-task363_sst2_polarity_classification \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task363_sst2_polarity_classification \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/2-task363_sst2_polarity_classification/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/2-task363_sst2_polarity_classification/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/2-task363_sst2_polarity_classification/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task1290_xsum_summarization \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/3-task1290_xsum_summarization \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task1290_xsum_summarization \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995 
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/3-task1290_xsum_summarization/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/3-task1290_xsum_summarization/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/3-task1290_xsum_summarization/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task181_outcome_extraction \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/4-task181_outcome_extraction \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task181_outcome_extraction \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/4-task181_outcome_extraction/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/4-task181_outcome_extraction/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/4-task181_outcome_extraction/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task002_quoref_answer_generation \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/5-task002_quoref_answer_generation \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task002_quoref_answer_generation \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/5-task002_quoref_answer_generation/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/5-task002_quoref_answer_generation/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/5-task002_quoref_answer_generation/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task1510_evalution_relation_extraction \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/6-task1510_evalution_relation_extraction \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task1510_evalution_relation_extraction \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/6-task1510_evalution_relation_extraction/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/5-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/6-task1510_evalution_relation_extraction/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/6-task1510_evalution_relation_extraction/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task639_multi_woz_user_utterance_generation \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/7-task639_multi_woz_user_utterance_generation \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task639_multi_woz_user_utterance_generation \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/7-task639_multi_woz_user_utterance_generation/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/5-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/6-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/7-task639_multi_woz_user_utterance_generation/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/7-task639_multi_woz_user_utterance_generation/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task1729_personachat_generate_next \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/8-task1729_personachat_generate_next \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task1729_personachat_generate_next \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/8-task1729_personachat_generate_next/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/5-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/6-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/7-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/8-task1729_personachat_generate_next/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/8-task1729_personachat_generate_next/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task073_commonsenseqa_answer_generation \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/9-task073_commonsenseqa_answer_generation \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task073_commonsenseqa_answer_generation \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/9-task073_commonsenseqa_answer_generation/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/5-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/6-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/7-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/8-task1729_personachat_generate_next/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/9-task073_commonsenseqa_answer_generation/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/9-task073_commonsenseqa_answer_generation/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task1590_diplomacy_text_generation \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/10-task1590_diplomacy_text_generation \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task1590_diplomacy_text_generation \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/10-task1590_diplomacy_text_generation/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/5-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/6-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/7-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/8-task1729_personachat_generate_next/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/9-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/10-task1590_diplomacy_text_generation/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/10-task1590_diplomacy_text_generation/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task748_glucose_reverse_cause_event_detection \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/11-task748_glucose_reverse_cause_event_detection \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task748_glucose_reverse_cause_event_detection \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/11-task748_glucose_reverse_cause_event_detection/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/5-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/6-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/7-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/8-task1729_personachat_generate_next/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/9-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/10-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/11-task748_glucose_reverse_cause_event_detection/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/11-task748_glucose_reverse_cause_event_detection/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task511_reddit_tifu_long_text_summarization \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/12-task511_reddit_tifu_long_text_summarization \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task511_reddit_tifu_long_text_summarization \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/12-task511_reddit_tifu_long_text_summarization/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/5-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/6-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/7-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/8-task1729_personachat_generate_next/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/9-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/10-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/11-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/12-task511_reddit_tifu_long_text_summarization/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/12-task511_reddit_tifu_long_text_summarization/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task591_sciq_answer_generation \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/13-task591_sciq_answer_generation \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task591_sciq_answer_generation \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/13-task591_sciq_answer_generation/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/5-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/6-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/7-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/8-task1729_personachat_generate_next/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/9-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/10-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/11-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/12-task511_reddit_tifu_long_text_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/13-task591_sciq_answer_generation/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/13-task591_sciq_answer_generation/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task1687_sentiment140_classification \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/14-task1687_sentiment140_classification \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task1687_sentiment140_classification \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/14-task1687_sentiment140_classification/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/5-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/6-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/7-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/8-task1729_personachat_generate_next/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/9-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/10-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/11-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/12-task511_reddit_tifu_long_text_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/13-task591_sciq_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/14-task1687_sentiment140_classification/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/14-task1687_sentiment140_classification/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task875_emotion_classification \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_gainlora_inflora/outputs/15-task875_emotion_classification \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task875_emotion_classification \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1 \
+   --mlp_hidden_dim 100 \
+   --model_name gainlora_inflora \
+   --threshold 0.995 \
+   --transthreshold 0.995
+
+python score.py gen_script_superni_order1_t5_small_gainlora_inflora gen_script_superni_order1_t5_small_gainlora_inflora

root_gainlora/T5_small/gen_script_superni_order1_t5_small_inflora.sh

@@ -0,0 +1,713 @@
+#!/bin/bash
+#SBATCH -J cl                           
+#SBATCH -o cl-%j.out                       
+#SBATCH -p compute 
+#SBATCH -N 1                           
+#SBATCH -t 20:00:00   
+#SBATCH --mem 128G 
+#SBATCH --gres=gpu:a100-sxm4-80gb:1  
+
+export CUDA_DEVICE_ORDER="PCI_BUS_ID"
+
+port=$(shuf -i25000-30000 -n1)  
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task1572_samsum_summary \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/1-task1572_samsum_summary \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --load_best_model_at_end \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/1-task1572_samsum_summary/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/1-task1572_samsum_summary/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/1-task1572_samsum_summary/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task363_sst2_polarity_classification \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/2-task363_sst2_polarity_classification \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task363_sst2_polarity_classification \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/2-task363_sst2_polarity_classification/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/2-task363_sst2_polarity_classification/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/2-task363_sst2_polarity_classification/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task1290_xsum_summarization \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/3-task1290_xsum_summarization \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task1290_xsum_summarization \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995 
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/3-task1290_xsum_summarization/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/3-task1290_xsum_summarization/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/3-task1290_xsum_summarization/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task181_outcome_extraction \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/4-task181_outcome_extraction \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task181_outcome_extraction \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/4-task181_outcome_extraction/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/4-task181_outcome_extraction/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/4-task181_outcome_extraction/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task002_quoref_answer_generation \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/5-task002_quoref_answer_generation \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task002_quoref_answer_generation \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/5-task002_quoref_answer_generation/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/5-task002_quoref_answer_generation/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/5-task002_quoref_answer_generation/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task1510_evalution_relation_extraction \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/6-task1510_evalution_relation_extraction \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task1510_evalution_relation_extraction \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/6-task1510_evalution_relation_extraction/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/5-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/6-task1510_evalution_relation_extraction/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/6-task1510_evalution_relation_extraction/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task639_multi_woz_user_utterance_generation \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/7-task639_multi_woz_user_utterance_generation \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task639_multi_woz_user_utterance_generation \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/7-task639_multi_woz_user_utterance_generation/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/5-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/6-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/7-task639_multi_woz_user_utterance_generation/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/7-task639_multi_woz_user_utterance_generation/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task1729_personachat_generate_next \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/8-task1729_personachat_generate_next \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task1729_personachat_generate_next \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/8-task1729_personachat_generate_next/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/5-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/6-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/7-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/8-task1729_personachat_generate_next/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/8-task1729_personachat_generate_next/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task073_commonsenseqa_answer_generation \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/9-task073_commonsenseqa_answer_generation \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task073_commonsenseqa_answer_generation \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/9-task073_commonsenseqa_answer_generation/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/5-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/6-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/7-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/8-task1729_personachat_generate_next/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/9-task073_commonsenseqa_answer_generation/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/9-task073_commonsenseqa_answer_generation/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task1590_diplomacy_text_generation \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/10-task1590_diplomacy_text_generation \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task1590_diplomacy_text_generation \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/10-task1590_diplomacy_text_generation/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/5-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/6-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/7-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/8-task1729_personachat_generate_next/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/9-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/10-task1590_diplomacy_text_generation/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/10-task1590_diplomacy_text_generation/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task748_glucose_reverse_cause_event_detection \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/11-task748_glucose_reverse_cause_event_detection \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task748_glucose_reverse_cause_event_detection \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/11-task748_glucose_reverse_cause_event_detection/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/5-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/6-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/7-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/8-task1729_personachat_generate_next/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/9-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/10-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/11-task748_glucose_reverse_cause_event_detection/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/11-task748_glucose_reverse_cause_event_detection/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task511_reddit_tifu_long_text_summarization \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/12-task511_reddit_tifu_long_text_summarization \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task511_reddit_tifu_long_text_summarization \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/12-task511_reddit_tifu_long_text_summarization/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/5-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/6-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/7-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/8-task1729_personachat_generate_next/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/9-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/10-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/11-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/12-task511_reddit_tifu_long_text_summarization/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/12-task511_reddit_tifu_long_text_summarization/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task591_sciq_answer_generation \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/13-task591_sciq_answer_generation \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task591_sciq_answer_generation \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/13-task591_sciq_answer_generation/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/5-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/6-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/7-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/8-task1729_personachat_generate_next/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/9-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/10-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/11-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/12-task511_reddit_tifu_long_text_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/13-task591_sciq_answer_generation/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/13-task591_sciq_answer_generation/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task1687_sentiment140_classification \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/14-task1687_sentiment140_classification \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task1687_sentiment140_classification \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/14-task1687_sentiment140_classification/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/1-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/2-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/3-task1290_xsum_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/4-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/5-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/6-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/7-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/8-task1729_personachat_generate_next/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/9-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/10-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/11-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/12-task511_reddit_tifu_long_text_summarization/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/13-task591_sciq_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/14-task1687_sentiment140_classification/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/14-task1687_sentiment140_classification/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task1572_samsum_summary,task363_sst2_polarity_classification,task1290_xsum_summarization,task181_outcome_extraction,task002_quoref_answer_generation,task1510_evalution_relation_extraction,task639_multi_woz_user_utterance_generation,task1729_personachat_generate_next,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task748_glucose_reverse_cause_event_detection,task511_reddit_tifu_long_text_summarization,task591_sciq_answer_generation,task1687_sentiment140_classification,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order1_t5_small_configs/task875_emotion_classification \
+   --output_dir logs_and_outputs/gen_script_superni_order1_t5_small_inflora/outputs/15-task875_emotion_classification \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order1_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task875_emotion_classification \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1 \
+   --model_name inflora \
+   --threshold 0.995
+
+python score.py gen_script_superni_order1_t5_small_inflora gen_script_superni_order1_t5_small_inflora

root_gainlora/T5_small/gen_script_superni_order2_t5_small_gainlora_inflora.sh

@@ -0,0 +1,743 @@
+#!/bin/bash
+#SBATCH -J cl                           
+#SBATCH -o cl-%j.out                       
+#SBATCH -p compute 
+#SBATCH -N 1                           
+#SBATCH -t 20:00:00   
+#SBATCH --mem 128G 
+#SBATCH --gres=gpu:a100-sxm4-80gb:1  
+
+export CUDA_DEVICE_ORDER="PCI_BUS_ID"
+
+port=$(shuf -i25000-30000 -n1)  
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task748_glucose_reverse_cause_event_detection \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/1-task748_glucose_reverse_cause_event_detection \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --load_best_model_at_end \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --model_name gainlora_inflora \
+   --mlp_hidden_dim 100 \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task073_commonsenseqa_answer_generation \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/2-task073_commonsenseqa_answer_generation \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task073_commonsenseqa_answer_generation \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --model_name gainlora_inflora \
+   --mlp_hidden_dim 100 \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/2-task073_commonsenseqa_answer_generation/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/2-task073_commonsenseqa_answer_generation/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/2-task073_commonsenseqa_answer_generation/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task1590_diplomacy_text_generation \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/3-task1590_diplomacy_text_generation \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task1590_diplomacy_text_generation \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --model_name gainlora_inflora \
+   --mlp_hidden_dim 100 \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/3-task1590_diplomacy_text_generation/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/3-task1590_diplomacy_text_generation/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/3-task1590_diplomacy_text_generation/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task639_multi_woz_user_utterance_generation \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/4-task639_multi_woz_user_utterance_generation \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task639_multi_woz_user_utterance_generation \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --model_name gainlora_inflora \
+   --mlp_hidden_dim 100 \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/3-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task1572_samsum_summary \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/5-task1572_samsum_summary \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task1572_samsum_summary \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --model_name gainlora_inflora \
+   --mlp_hidden_dim 100 \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/5-task1572_samsum_summary/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/3-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/5-task1572_samsum_summary/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/5-task1572_samsum_summary/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task1687_sentiment140_classification \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/6-task1687_sentiment140_classification \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task1687_sentiment140_classification \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --model_name gainlora_inflora \
+   --mlp_hidden_dim 100 \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/6-task1687_sentiment140_classification/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/3-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/5-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/6-task1687_sentiment140_classification/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/6-task1687_sentiment140_classification/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task591_sciq_answer_generation \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/7-task591_sciq_answer_generation \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task591_sciq_answer_generation \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --model_name gainlora_inflora \
+   --mlp_hidden_dim 100 \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/7-task591_sciq_answer_generation/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/3-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/5-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/6-task1687_sentiment140_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/7-task591_sciq_answer_generation/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/7-task591_sciq_answer_generation/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task363_sst2_polarity_classification \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/8-task363_sst2_polarity_classification \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task363_sst2_polarity_classification \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --model_name gainlora_inflora \
+   --mlp_hidden_dim 100 \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/8-task363_sst2_polarity_classification/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/3-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/5-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/6-task1687_sentiment140_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/7-task591_sciq_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/8-task363_sst2_polarity_classification/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/8-task363_sst2_polarity_classification/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task1510_evalution_relation_extraction \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/9-task1510_evalution_relation_extraction \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task1510_evalution_relation_extraction \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --model_name gainlora_inflora \
+   --mlp_hidden_dim 100 \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/9-task1510_evalution_relation_extraction/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/3-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/5-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/6-task1687_sentiment140_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/7-task591_sciq_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/8-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/9-task1510_evalution_relation_extraction/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/9-task1510_evalution_relation_extraction/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task1729_personachat_generate_next \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/10-task1729_personachat_generate_next \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task1729_personachat_generate_next \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --model_name gainlora_inflora \
+   --mlp_hidden_dim 100 \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/10-task1729_personachat_generate_next/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/3-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/5-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/6-task1687_sentiment140_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/7-task591_sciq_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/8-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/9-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/10-task1729_personachat_generate_next/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/10-task1729_personachat_generate_next/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task181_outcome_extraction \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/11-task181_outcome_extraction \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task181_outcome_extraction \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --model_name gainlora_inflora \
+   --mlp_hidden_dim 100 \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/11-task181_outcome_extraction/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/3-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/5-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/6-task1687_sentiment140_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/7-task591_sciq_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/8-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/9-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/10-task1729_personachat_generate_next/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/11-task181_outcome_extraction/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/11-task181_outcome_extraction/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task511_reddit_tifu_long_text_summarization \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/12-task511_reddit_tifu_long_text_summarization \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task511_reddit_tifu_long_text_summarization \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --model_name gainlora_inflora \
+   --mlp_hidden_dim 100 \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/12-task511_reddit_tifu_long_text_summarization/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/3-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/5-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/6-task1687_sentiment140_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/7-task591_sciq_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/8-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/9-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/10-task1729_personachat_generate_next/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/11-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/12-task511_reddit_tifu_long_text_summarization/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/12-task511_reddit_tifu_long_text_summarization/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task002_quoref_answer_generation \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/13-task002_quoref_answer_generation \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task002_quoref_answer_generation \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --model_name gainlora_inflora \
+   --mlp_hidden_dim 100 \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/13-task002_quoref_answer_generation/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/3-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/5-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/6-task1687_sentiment140_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/7-task591_sciq_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/8-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/9-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/10-task1729_personachat_generate_next/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/11-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/12-task511_reddit_tifu_long_text_summarization/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/13-task002_quoref_answer_generation/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/13-task002_quoref_answer_generation/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task1290_xsum_summarization \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/14-task1290_xsum_summarization \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task1290_xsum_summarization \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --model_name gainlora_inflora \
+   --mlp_hidden_dim 100 \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/14-task1290_xsum_summarization/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/3-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/5-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/6-task1687_sentiment140_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/7-task591_sciq_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/8-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/9-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/10-task1729_personachat_generate_next/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/11-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/12-task511_reddit_tifu_long_text_summarization/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/13-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/14-task1290_xsum_summarization/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/14-task1290_xsum_summarization/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task875_emotion_classification \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_gainlora_inflora/outputs/15-task875_emotion_classification \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_gainlora_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task875_emotion_classification \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --model_name gainlora_inflora \
+   --mlp_hidden_dim 100 \
+   --threshold 0.995 \
+   --transthreshold 0.995 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1
+
+python score.py gen_script_superni_order2_t5_small_gainlora_inflora gen_script_superni_order2_t5_small_gainlora_inflora

root_gainlora/T5_small/gen_script_superni_order2_t5_small_inflora.sh

@@ -0,0 +1,713 @@
+#!/bin/bash
+#SBATCH -J cl                           
+#SBATCH -o cl-%j.out                       
+#SBATCH -p compute 
+#SBATCH -N 1                           
+#SBATCH -t 20:00:00   
+#SBATCH --mem 128G 
+#SBATCH --gres=gpu:a100-sxm4-80gb:1  
+
+export CUDA_DEVICE_ORDER="PCI_BUS_ID"
+
+port=$(shuf -i25000-30000 -n1)  
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task748_glucose_reverse_cause_event_detection \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/1-task748_glucose_reverse_cause_event_detection \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --load_best_model_at_end \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --model_name inflora \
+   --threshold 0.995 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task073_commonsenseqa_answer_generation \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/2-task073_commonsenseqa_answer_generation \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task073_commonsenseqa_answer_generation \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --model_name inflora \
+   --threshold 0.995 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/2-task073_commonsenseqa_answer_generation/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/2-task073_commonsenseqa_answer_generation/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/2-task073_commonsenseqa_answer_generation/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task1590_diplomacy_text_generation \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/3-task1590_diplomacy_text_generation \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task1590_diplomacy_text_generation \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --model_name inflora \
+   --threshold 0.995 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/3-task1590_diplomacy_text_generation/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/3-task1590_diplomacy_text_generation/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/3-task1590_diplomacy_text_generation/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task639_multi_woz_user_utterance_generation \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/4-task639_multi_woz_user_utterance_generation \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task639_multi_woz_user_utterance_generation \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --model_name inflora \
+   --threshold 0.995 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/3-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task1572_samsum_summary \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/5-task1572_samsum_summary \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task1572_samsum_summary \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --model_name inflora \
+   --threshold 0.995 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/5-task1572_samsum_summary/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/3-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/5-task1572_samsum_summary/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/5-task1572_samsum_summary/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task1687_sentiment140_classification \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/6-task1687_sentiment140_classification \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task1687_sentiment140_classification \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --model_name inflora \
+   --threshold 0.995 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/6-task1687_sentiment140_classification/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/3-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/5-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/6-task1687_sentiment140_classification/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/6-task1687_sentiment140_classification/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task591_sciq_answer_generation \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/7-task591_sciq_answer_generation \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task591_sciq_answer_generation \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --model_name inflora \
+   --threshold 0.995 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/7-task591_sciq_answer_generation/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/3-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/5-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/6-task1687_sentiment140_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/7-task591_sciq_answer_generation/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/7-task591_sciq_answer_generation/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task363_sst2_polarity_classification \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/8-task363_sst2_polarity_classification \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task363_sst2_polarity_classification \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --model_name inflora \
+   --threshold 0.995 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/8-task363_sst2_polarity_classification/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/3-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/5-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/6-task1687_sentiment140_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/7-task591_sciq_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/8-task363_sst2_polarity_classification/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/8-task363_sst2_polarity_classification/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task1510_evalution_relation_extraction \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/9-task1510_evalution_relation_extraction \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task1510_evalution_relation_extraction \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --model_name inflora \
+   --threshold 0.995 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/9-task1510_evalution_relation_extraction/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/3-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/5-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/6-task1687_sentiment140_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/7-task591_sciq_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/8-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/9-task1510_evalution_relation_extraction/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/9-task1510_evalution_relation_extraction/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task1729_personachat_generate_next \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/10-task1729_personachat_generate_next \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task1729_personachat_generate_next \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --model_name inflora \
+   --threshold 0.995 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/10-task1729_personachat_generate_next/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/3-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/5-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/6-task1687_sentiment140_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/7-task591_sciq_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/8-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/9-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/10-task1729_personachat_generate_next/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/10-task1729_personachat_generate_next/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task181_outcome_extraction \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/11-task181_outcome_extraction \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task181_outcome_extraction \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --model_name inflora \
+   --threshold 0.995 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/11-task181_outcome_extraction/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/3-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/5-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/6-task1687_sentiment140_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/7-task591_sciq_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/8-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/9-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/10-task1729_personachat_generate_next/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/11-task181_outcome_extraction/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/11-task181_outcome_extraction/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task511_reddit_tifu_long_text_summarization \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/12-task511_reddit_tifu_long_text_summarization \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task511_reddit_tifu_long_text_summarization \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --model_name inflora \
+   --threshold 0.995 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/12-task511_reddit_tifu_long_text_summarization/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/3-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/5-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/6-task1687_sentiment140_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/7-task591_sciq_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/8-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/9-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/10-task1729_personachat_generate_next/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/11-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/12-task511_reddit_tifu_long_text_summarization/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/12-task511_reddit_tifu_long_text_summarization/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task002_quoref_answer_generation \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/13-task002_quoref_answer_generation \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task002_quoref_answer_generation \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --model_name inflora \
+   --threshold 0.995 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/13-task002_quoref_answer_generation/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/3-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/5-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/6-task1687_sentiment140_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/7-task591_sciq_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/8-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/9-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/10-task1729_personachat_generate_next/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/11-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/12-task511_reddit_tifu_long_text_summarization/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/13-task002_quoref_answer_generation/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/13-task002_quoref_answer_generation/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task1290_xsum_summarization \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/14-task1290_xsum_summarization \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task1290_xsum_summarization \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --model_name inflora \
+   --threshold 0.995 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1
+
+
+CUDA_VISIBLE_DEVICES=$1 python src/run_t5.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path $2 \
+   --load_checkpoint_from logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/14-task1290_xsum_summarization/saved_weights/trans_input.pt \
+   --previous_lora_path logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/1-task748_glucose_reverse_cause_event_detection/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/2-task073_commonsenseqa_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/3-task1590_diplomacy_text_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/4-task639_multi_woz_user_utterance_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/5-task1572_samsum_summary/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/6-task1687_sentiment140_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/7-task591_sciq_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/8-task363_sst2_polarity_classification/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/9-task1510_evalution_relation_extraction/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/10-task1729_personachat_generate_next/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/11-task181_outcome_extraction/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/12-task511_reddit_tifu_long_text_summarization/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/13-task002_quoref_answer_generation/saved_weights,logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/14-task1290_xsum_summarization/saved_weights \
+   --previous_prompt_key_path logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/14-task1290_xsum_summarization/saved_weights/prompts_keys_till_now.pt \
+   --data_dir CL_Benchmark \
+   --task_order task748_glucose_reverse_cause_event_detection,task073_commonsenseqa_answer_generation,task1590_diplomacy_text_generation,task639_multi_woz_user_utterance_generation,task1572_samsum_summary,task1687_sentiment140_classification,task591_sciq_answer_generation,task363_sst2_polarity_classification,task1510_evalution_relation_extraction,task1729_personachat_generate_next,task181_outcome_extraction,task511_reddit_tifu_long_text_summarization,task002_quoref_answer_generation,task1290_xsum_summarization,task875_emotion_classification \
+   --gen_data_dir generated_data/lora_gen_superni_t5 \
+   --task_config_dir configs/gen_script_superni_order2_t5_small_configs/task875_emotion_classification \
+   --output_dir logs_and_outputs/gen_script_superni_order2_t5_small_inflora/outputs/15-task875_emotion_classification \
+   --per_device_train_batch_size 16 \
+   --per_device_eval_batch_size 8 \
+   --gradient_accumulation_steps 2 \
+   --learning_rate 0.0003 \
+   --num_train_epochs 100 \
+   --run_name gen_script_superni_order2_t5_small_inflora \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 10 \
+   --metric_for_best_model eval_rougeL_for_task875_emotion_classification \
+   --evaluation_strategy steps \
+   --save_strategy steps \
+   --save_total_limit 1 \
+   --load_best_model_at_end \
+   --lora_r 4 \
+   --lora_alpha 32 \
+   --lora_dropout 0.0 \
+   --add_instruction_replay \
+   --data_replay_freq -1 \
+   --replay_after_n_epoch 0 \
+   --model_name inflora \
+   --threshold 0.995 \
+   --kl_ratio 0.5 \
+   --attn_temperature 1
+
+python score.py gen_script_superni_order2_t5_small_inflora gen_script_superni_order2_t5_small_inflora