config.json

{
  "_comment_cca": "--- Cross-Layer Consistency Alignment (CCA) ---",
  "_comment_spera": "--- Frequency-domain Representation Alignment (SPERA) ---",
  "_comment_spara": "--- Spatial-domain Representation Alignment (SPARA) ---",
  "_comment_ms": "--- Multi-scale Configuration for SPARA ---",
  "_comment_prompt": "--- Vision Prompt Module ---",
  "_comment_shared": "--- Shared Configuration ---",
  "_name_or_path": "./checkpoints/SSVAL-7B-Instruct",
  "architectures": [
    "Qwen2ForCausalLM"
  ],
  "attention_dropout": 0.0,
  "bos_token_id": 151643,
  "cca_weight": 0.15,
  "cpd_target_layers": [
    14
  ],
  "data": {
    "data_path": "./data/llava-665k/llava_v1_5_mix665k.json",
    "image_aspect_ratio": "pad",
    "image_folder": "./data/llava-665k",
    "lazy_preprocess": true
  },
  "early_layer": 2,
  "enable_cca_loss": false,
  "enable_spera_loss": true,
  "enable_spara_loss": true,
  "enable_vision_prompts": true,
  "eos_token_id": 151645,
  "spera_gamma_high": 1.0,
  "spera_gamma_low": 1.0,
  "spera_weight": 0.15,
  "freeze_mm_mlp_adapter": false,
  "hidden_act": "silu",
  "hidden_size": 3584,
  "spara_alpha": 1.0,
  "spara_beta": 0.3,
  "spara_weight": 0.3,
  "image_aspect_ratio": "pad",
  "initializer_range": 0.02,
  "intermediate_size": 18944,
  "late_layer": 26,
  "max_position_embeddings": 32768,
  "max_window_layers": 28,
  "mm_hidden_size": 1024,
  "mm_patch_merge_type": "flat",
  "mm_projector_lr": null,
  "mm_projector_type": "mlp2x_gelu",
  "mm_use_im_patch_token": false,
  "mm_use_im_start_end": false,
  "mm_vision_select_feature": "patch",
  "mm_vision_select_layer": -2,
  "mm_vision_tower": "./checkpoints/clip-vit-large-patch14-336",
  "model": {
    "cpd": {
      "ms_scales": [
        1,
        2,
        4
      ],
      "ms_weights": [
        1.0,
        1.0,
        1.0
      ]
    },
    "mm_projector_type": "mlp2x_gelu",
    "mm_use_im_patch_token": false,
    "mm_use_im_start_end": false,
    "mm_vision_select_layer": -2,
    "model_name_or_path": "./checkpoints/Qwen2.5-7B-Instruct",
    "pretrain_mm_mlp_adapter": "./checkpoints/llava-v1.5-qwen2-pretrain/mm_projector.bin",
    "version": "qwen_2",
    "vision_tower": "./checkpoints/clip-vit-large-patch14-336"
  },
  "model_type": "llava_qwen2",
  "ms_scales": [
    1,
    2,
    4
  ],
  "ms_weights": [
    1.0,
    1.0,
    1.0
  ],
  "num_attention_heads": 28,
  "num_hidden_layers": 28,
  "num_key_value_heads": 4,
  "num_vfm_prompts": 4,
  "num_vision_prompts": 4,
  "only_coco": false,
  "pretraining_tp": 1,
  "projector_dim": 2048,
  "rms_norm_eps": 1e-06,
  "rope_theta": 1000000.0,
  "sliding_window": 131072,
  "tensor_parallel_size": 1,
  "tie_word_embeddings": false,
  "tokenizer_model_max_length": 2048,
  "tokenizer_padding_side": "right",
  "torch_dtype": "bfloat16",
  "training": {
    "bf16": true,
    "dataloader_num_workers": 4,
    "ddp_find_unused_parameters": true,
    "deepspeed": "configs/zero2.json",
    "evaluation_strategy": "no",
    "gradient_accumulation_steps": 4,
    "gradient_checkpointing": false,
    "group_by_modality_length": true,
    "learning_rate": 0.0002,
    "logging_steps": 1,
    "lora_alpha": 128,
    "lora_dropout": 0.05,
    "lora_enable": true,
    "lora_r": 64,
    "lr_scheduler_type": "cosine",
    "model_max_length": 2048,
    "num_train_epochs": 1,
    "output_dir": "./result/llava-qwen2.5-7b-SSVAL",
    "per_device_eval_batch_size": 2,
    "per_device_train_batch_size": 2,
    "report_to": "wandb",
    "run_name": "cpd-llava-qwen2.5-7b-lora",
    "save_steps": 5000,
    "save_strategy": "steps",
    "save_total_limit": 1,
    "tf32": true,
    "warmup_ratio": 0.03,
    "weight_decay": 0.0
  },
  "transformers_version": "4.37.2",
  "tune_mm_mlp_adapter": false,
  "use_cache": true,
  "use_cca_projection": false,
  "use_mm_proj": true,
  "use_sliding_window": false,
  "vfm_target": "dinov2-vit-b",
  "vocab_size": 152064,
  "vra_loss": false,
  "z_dim": 768
}