Upload folder using huggingface_hub

viscop_qwen2.5_7b_EgoExo4D-EGOvideos_train-VisCoP_projector-LLM_LoRA_CorrectedOptimizer/README.md

@@ -0,0 +1,9 @@
+---
+library_name: peft
+---
+## Training procedure
+
+### Framework versions
+
+
+- PEFT 0.4.0

viscop_qwen2.5_7b_EgoExo4D-EGOvideos_train-VisCoP_projector-LLM_LoRA_CorrectedOptimizer/adapter_config.json

@@ -0,0 +1,18 @@
+{
+  "auto_mapping": null,
+  "base_model_name_or_path": "/home/dreilly1/Projects/VisCoP/base_vlm/videollama3-image_7b_local",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "lora_alpha": 16,
+  "lora_dropout": 0.05,
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 64,
+  "revision": null,
+  "target_modules": ".*model\\.layers\\..*\\.(gate_proj|o_proj|q_proj|v_proj|up_proj|down_proj|k_proj)$",
+  "task_type": "CAUSAL_LM"
+}

viscop_qwen2.5_7b_EgoExo4D-EGOvideos_train-VisCoP_projector-LLM_LoRA_CorrectedOptimizer/adapter_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:54b0d449306b48fc0193eeb2dec4280c5d91eb1180bc00faaf1e134edd727cbe
+size 323097578

viscop_qwen2.5_7b_EgoExo4D-EGOvideos_train-VisCoP_projector-LLM_LoRA_CorrectedOptimizer/config.json

@@ -0,0 +1,132 @@
+{
+  "_attn_implementation_autoset": true,
+  "_name_or_path": "/home/dreilly1/Projects/VisCoP/base_vlm/videollama3-image_7b_local",
+  "architectures": [
+    "Videollama3Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "auto_map": {
+    "AutoConfig": "DAMO-NLP-SG/VideoLLaMA3-7B-Image--configuration_videollama3.Videollama3Qwen2Config",
+    "AutoModelForCausalLM": "DAMO-NLP-SG/VideoLLaMA3-7B-Image--modeling_videollama3.Videollama3Qwen2ForCausalLM"
+  },
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 3584,
+  "image_aspect_ratio": "square",
+  "image_size": -1,
+  "image_token_index": 151665,
+  "image_token_length": 1,
+  "include_visual_probes": true,
+  "include_visual_tokens": true,
+  "initializer_range": 0.02,
+  "interaction_module": "cross_attention",
+  "interaction_module_layers": null,
+  "intermediate_size": 18944,
+  "is_alignment": false,
+  "llm_lr": 1e-05,
+  "max_frames": 180,
+  "max_position_embeddings": 32768,
+  "max_window_layers": 28,
+  "mm_hidden_size": 1152,
+  "mm_projector_lr": 1e-05,
+  "mm_projector_type": "mlp2x_gelu",
+  "mm_vision_encoder": "DAMO-NLP-SG/SigLIP-NaViT",
+  "mm_vision_select_feature": "patch",
+  "mm_vision_select_layer": -1,
+  "model_type": "viscop_qwen2",
+  "num_attention_heads": 28,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 4,
+  "num_visual_probes": 16,
+  "probe_token_index": 151668,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "tokenizer_model_max_length": 16384,
+  "tokenizer_padding_side": "right",
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.46.3",
+  "use_cache": true,
+  "use_mm_proj": true,
+  "use_sliding_window": false,
+  "use_token_compression": false,
+  "vision_encoder": "DAMO-NLP-SG/SigLIP-NaViT",
+  "vision_encoder_config": {
+    "_attn_implementation_autoset": false,
+    "_name_or_path": "",
+    "add_cross_attention": false,
+    "architectures": null,
+    "attention_dropout": 0.0,
+    "bad_words_ids": null,
+    "begin_suppress_tokens": null,
+    "bos_token_id": null,
+    "chunk_size_feed_forward": 0,
+    "cross_attention_hidden_size": null,
+    "decoder_start_token_id": null,
+    "diversity_penalty": 0.0,
+    "do_sample": false,
+    "early_stopping": false,
+    "encoder_no_repeat_ngram_size": 0,
+    "eos_token_id": null,
+    "exponential_decay_length_penalty": null,
+    "finetuning_task": null,
+    "forced_bos_token_id": null,
+    "forced_eos_token_id": null,
+    "hidden_act": "gelu_pytorch_tanh",
+    "hidden_size": 1152,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "intermediate_size": 4304,
+    "is_decoder": false,
+    "is_encoder_decoder": false,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "layer_norm_eps": 1e-06,
+    "length_penalty": 1.0,
+    "max_length": 20,
+    "min_length": 0,
+    "model_type": "videollama3_vision_encoder",
+    "no_repeat_ngram_size": 0,
+    "num_attention_heads": 16,
+    "num_beam_groups": 1,
+    "num_beams": 1,
+    "num_channels": 3,
+    "num_hidden_layers": 27,
+    "num_return_sequences": 1,
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "output_scores": false,
+    "pad_token_id": null,
+    "patch_size": 14,
+    "prefix": null,
+    "problem_type": null,
+    "pruned_heads": {},
+    "remove_invalid_values": false,
+    "repetition_penalty": 1.0,
+    "return_dict": true,
+    "return_dict_in_generate": false,
+    "sep_token_id": null,
+    "suppress_tokens": null,
+    "task_specific_params": null,
+    "temperature": 1.0,
+    "tf_legacy_loss": false,
+    "tie_encoder_decoder": false,
+    "tie_word_embeddings": true,
+    "tokenizer_class": null,
+    "top_k": 50,
+    "top_p": 1.0,
+    "torch_dtype": null,
+    "torchscript": false,
+    "typical_p": 1.0,
+    "use_bfloat16": false
+  },
+  "vision_encoder_lr": null,
+  "vocab_size": 152064
+}

viscop_qwen2.5_7b_EgoExo4D-EGOvideos_train-VisCoP_projector-LLM_LoRA_CorrectedOptimizer/non_lora_trainables.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf012cd5517edf0873ba621833c3bb639f70191e9f9d733ff69e0bcdd71628be
+size 354950700

viscop_qwen2.5_7b_EgoExo4D-EGOvideos_train-VisCoP_projector-LLM_LoRA_CorrectedOptimizer/train_viscop.sh

@@ -0,0 +1,124 @@
+#!/bin/bash
+# Environment Variables
+ARG_WORLD_SIZE=${1:-1}
+ARG_NPROC_PER_NODE=${2:-8}
+
+if [[ -v MASTER_ADDR_PASSED ]]; then
+    ARG_MASTER_ADDR=$MASTER_ADDR_PASSED # passed via slurm submission script
+else
+    ARG_MASTER_ADDR=127.0.0.1 # for dev environments
+fi
+ARG_MASTER_PORT=12355
+# ARG_RANK=$SLURM_NODEID
+ARG_RANK=0
+
+# Multiple conditions
+if [ ! -n "$WORLD_SIZE" ] || [ ! -n "$NPROC_PER_NODE" ]; then
+    WORLD_SIZE=$ARG_WORLD_SIZE
+    NPROC_PER_NODE=$ARG_NPROC_PER_NODE
+fi
+
+if [ ! -n "$MASTER_ADDR" ] || [ ! -n "$MASTER_PORT" ] || [ ! -n "$RANK" ]; then
+    MASTER_ADDR=$ARG_MASTER_ADDR
+    MASTER_PORT=$ARG_MASTER_PORT
+    RANK=$ARG_RANK
+fi
+
+echo "MASTER_ADDR: $MASTER_ADDR. MASTER_PORT: $MASTER_PORT. RANK: $RANK"
+echo "WORLD_SIZE: $WORLD_SIZE"
+echo "NPROC_PER_NODE: $NPROC_PER_NODE"
+
+# Training Arguments
+GLOBAL_BATCH_SIZE=128 # aka effective batch size
+LOCAL_BATCH_SIZE=8 # batch size per GPU
+GRADIENT_ACCUMULATION_STEPS=$[$GLOBAL_BATCH_SIZE/($WORLD_SIZE*$NPROC_PER_NODE*$LOCAL_BATCH_SIZE)]
+echo $GRADIENT_ACCUMULATION_STEPS
+
+INIT_MODEL=/home/dreilly1/Projects/VisCoP/base_vlm/videollama3-image_7b_local # path to base VLM (for ViSCoP we use VideoLLaMA3 as the base VLM)
+
+NUM_DATA_WORKERS=8
+NUM_TRAIN_EPOCHS=3
+LORA_TRAINING=True
+
+# ViSCoP Arguments
+NUM_VISUAL_PROBES=16
+INTERACTION_MODULE_POS=all
+PASS_PROBES_TO_LLM=True
+PASS_VIS_FEATURES_TO_LLM=True
+
+# Logging Arguments
+export WANDB_PROJECT=sony26_mm_viscop
+REPORT_TO=wandb
+OUTP_DIR=work_dirs/egoexo
+RUN_NAME=viscop_qwen2.5_7b_EgoExo4D-EGOvideos_train-VisCoP_projector-LLM_LoRA_CorrectedOptimizer
+
+# Data Arguments
+# DATA_DIR=/home/dreilly1/Projects/paired_egoexo_dl/paired_videos_DATA/
+DATA_DIR=/data/dreilly1/EgoExo4D_symlink/
+TRAINING_JSON="/home/dreilly1/Projects/viscop_sony/training_jsons/train-instr_viscop_egoview_ALLDATA.json"
+
+# if [[ $TRAINING_JSON == *"egoview"* ]]; then
+#     MAX_FRAMES=40 # use 40 frames for training on ego
+# else
+#     MAX_FRAMES=180
+# fi
+MAX_FRAMES=180
+
+# Optional Arguments. Set TESTING to 1 to quickly test the training script without logging or data workers, useful for debugging
+TESTING=0
+if [ $TESTING -eq 1 ]; then
+    NUM_DATA_WORKERS=0
+    REPORT_TO=none
+    RUN_NAME=TESTING
+fi
+
+mkdir -p "${OUTP_DIR}/${RUN_NAME}/"
+cp "$0" "${OUTP_DIR}/${RUN_NAME}/"
+
+torchrun --nnodes $WORLD_SIZE \
+    --nproc_per_node $NPROC_PER_NODE \
+    --master_addr=$MASTER_ADDR \
+    --master_port=$MASTER_PORT \
+    --node_rank $RANK \
+    viscop/train_viscop.py \
+    --interaction_module_layers $INTERACTION_MODULE_POS \
+    --lora_enable $LORA_TRAINING \
+    --num_train_epochs $NUM_TRAIN_EPOCHS \
+    --deepspeed scripts/zero2.json \
+    --model_type viscop_qwen2 \
+    --model_path $INIT_MODEL \
+    --vision_encoder DAMO-NLP-SG/SigLIP-NaViT \
+    --mm_projector_type mlp2x_gelu \
+    --data_path $TRAINING_JSON \
+    --data_folder $DATA_DIR \
+    --image_merge_size 2 \
+    --video_merge_size 2 \
+    --fps 1 \
+    --max_frames $MAX_FRAMES \
+    --model_max_length 16384 \
+    --mm_max_length 10240 \
+    --bf16 True \
+    --tf32 True \
+    --fp16 False \
+    --output_dir ${OUTP_DIR}/${RUN_NAME} \
+    --per_device_train_batch_size $LOCAL_BATCH_SIZE \
+    --per_device_eval_batch_size 2 \
+    --gradient_accumulation_steps $GRADIENT_ACCUMULATION_STEPS \
+    --evaluation_strategy "no" \
+    --save_strategy "no" \
+    --save_steps 5000 \
+    --save_total_limit 1 \
+    --mm_projector_lr 1e-5 \
+    --llm_lr 1e-5 \
+    --weight_decay 0. \
+    --warmup_ratio 0.03 \
+    --lr_scheduler_type "cosine" \
+    --logging_steps 1 \
+    --gradient_checkpointing True \
+    --dataloader_num_workers $NUM_DATA_WORKERS \
+    --report_to $REPORT_TO \
+    --run_name $RUN_NAME \
+    --dataset_cache_dir /home/dreilly1/.cache/viscop_datasetcache \
+    --include_visual_tokens $PASS_VIS_FEATURES_TO_LLM \
+    --include_visual_probes $PASS_PROBES_TO_LLM \
+    --num_visual_probes $NUM_VISUAL_PROBES