Upload model.config.json

model.config.json

@@ -0,0 +1,384 @@
+{
+    "__config_type__": "robo_orchard_lab.models.monodream.model:MonoDreamConfig",
+    "class_type": "robo_orchard_lab.models.monodream.model:MonoDream",
+    "llava_llama_config": {
+        "Ubit": 100,
+        "_attn_implementation_autoset": true,
+        "architectures": [
+            "LlavaLlamaModel"
+        ],
+        "babit": "E5M2",
+        "bobit": "E5M2",
+        "bwbit": "E5M2",
+        "chat_template": null,
+        "col_blocksize": -1,
+        "col_blocksize_optimizer": 128,
+        "draw_distribution_backward": false,
+        "draw_distribution_forward": false,
+        "drop_path_rate": 0.0,
+        "dynamic_s2": false,
+        "epsilon": 1e-10,
+        "epsilon_optimizer": 1e-15,
+        "fabit": "E4M3",
+        "first_order_bit": null,
+        "first_order_quant_type": null,
+        "fobit": "E4M3",
+        "fps": 0.0,
+        "fwbit": "E4M3",
+        "group_size": -1,
+        "hidden_size": 1536,
+        "image_aspect_ratio": "dynamic",
+        "image_encoder": {
+            "_target_": "llava.model.encoders.BasicImageEncoder"
+        },
+        "interpolate_mode": "linear",
+        "llm_cfg": {
+            "_attn_implementation_autoset": false,
+            "add_cross_attention": false,
+            "architectures": [
+                "Qwen2ForCausalLM"
+            ],
+            "attention_dropout": 0.0,
+            "bad_words_ids": null,
+            "begin_suppress_tokens": null,
+            "bos_token_id": 151643,
+            "chunk_size_feed_forward": 0,
+            "cross_attention_hidden_size": null,
+            "decoder_start_token_id": null,
+            "diversity_penalty": 0.0,
+            "do_sample": false,
+            "early_stopping": false,
+            "encoder_no_repeat_ngram_size": 0,
+            "eos_token_id": 151645,
+            "exponential_decay_length_penalty": null,
+            "finetuning_task": null,
+            "forced_bos_token_id": null,
+            "forced_eos_token_id": null,
+            "hidden_act": "silu",
+            "hidden_size": 1536,
+            "id2label": {
+                "0": "LABEL_0",
+                "1": "LABEL_1"
+            },
+            "initializer_range": 0.02,
+            "intermediate_size": 8960,
+            "is_decoder": false,
+            "is_encoder_decoder": false,
+            "label2id": {
+                "LABEL_0": 0,
+                "LABEL_1": 1
+            },
+            "length_penalty": 1.0,
+            "max_length": 20,
+            "max_position_embeddings": 32768,
+            "max_window_layers": 28,
+            "min_length": 0,
+            "model_max_length": 4096,
+            "model_type": "qwen2",
+            "no_repeat_ngram_size": 0,
+            "num_attention_heads": 12,
+            "num_beam_groups": 1,
+            "num_beams": 1,
+            "num_hidden_layers": 28,
+            "num_key_value_heads": 2,
+            "num_return_sequences": 1,
+            "output_attentions": false,
+            "output_hidden_states": false,
+            "output_scores": false,
+            "pad_token_id": null,
+            "prefix": null,
+            "problem_type": null,
+            "pruned_heads": {},
+            "remove_invalid_values": false,
+            "repetition_penalty": 1.0,
+            "return_dict": true,
+            "return_dict_in_generate": false,
+            "rms_norm_eps": 1e-6,
+            "rope_scaling": null,
+            "rope_theta": 1000000.0,
+            "sep_token_id": null,
+            "sliding_window": null,
+            "suppress_tokens": null,
+            "task_specific_params": null,
+            "temperature": 1.0,
+            "tf_legacy_loss": false,
+            "tie_encoder_decoder": false,
+            "tie_word_embeddings": true,
+            "tokenizer_class": null,
+            "tokenizer_model_max_length": 4096,
+            "tokenizer_padding_side": "right",
+            "top_k": 50,
+            "top_p": 1.0,
+            "torch_dtype": "bfloat16",
+            "torchscript": false,
+            "typical_p": 1.0,
+            "use_bfloat16": false,
+            "use_cache": false,
+            "use_sliding_window": false,
+            "vocab_size": 151651
+        },
+        "max_tiles": 12,
+        "min_blockunit_col": 4,
+        "min_blockunit_row": 4,
+        "min_tiles": 1,
+        "mlp_path": null,
+        "mm_hidden_size": 1152,
+        "mm_projector": "mlp_downsample_3x3_fix",
+        "mm_projector_cfg": {
+            "_attn_implementation_autoset": false,
+            "add_cross_attention": false,
+            "architectures": [
+                "MultimodalProjector"
+            ],
+            "bad_words_ids": null,
+            "begin_suppress_tokens": null,
+            "bos_token_id": null,
+            "chunk_size_feed_forward": 0,
+            "cross_attention_hidden_size": null,
+            "decoder_start_token_id": null,
+            "diversity_penalty": 0.0,
+            "do_sample": false,
+            "early_stopping": false,
+            "encoder_no_repeat_ngram_size": 0,
+            "eos_token_id": null,
+            "exponential_decay_length_penalty": null,
+            "finetuning_task": null,
+            "forced_bos_token_id": null,
+            "forced_eos_token_id": null,
+            "id2label": {
+                "0": "LABEL_0",
+                "1": "LABEL_1"
+            },
+            "is_decoder": false,
+            "is_encoder_decoder": false,
+            "label2id": {
+                "LABEL_0": 0,
+                "LABEL_1": 1
+            },
+            "length_penalty": 1.0,
+            "max_length": 20,
+            "min_length": 0,
+            "mm_projector_type": "mlp_downsample_3x3_fix",
+            "model_type": "v2l_projector",
+            "no_repeat_ngram_size": 0,
+            "num_beam_groups": 1,
+            "num_beams": 1,
+            "num_return_sequences": 1,
+            "output_attentions": false,
+            "output_hidden_states": false,
+            "output_scores": false,
+            "pad_token_id": null,
+            "prefix": null,
+            "problem_type": null,
+            "pruned_heads": {},
+            "remove_invalid_values": false,
+            "repetition_penalty": 1.0,
+            "return_dict": true,
+            "return_dict_in_generate": false,
+            "sep_token_id": null,
+            "suppress_tokens": null,
+            "task_specific_params": null,
+            "temperature": 1.0,
+            "tf_legacy_loss": false,
+            "tie_encoder_decoder": false,
+            "tie_word_embeddings": true,
+            "tokenizer_class": null,
+            "top_k": 50,
+            "top_p": 1.0,
+            "torch_dtype": "bfloat16",
+            "torchscript": false,
+            "typical_p": 1.0,
+            "use_bfloat16": false
+        },
+        "mm_projector_lr": null,
+        "mm_use_im_patch_token": false,
+        "mm_use_im_start_end": false,
+        "mm_vision_select_feature": "cls_patch",
+        "mm_vision_select_layer": -2,
+        "model_dtype": "torch.bfloat16",
+        "model_type": "llava_llama",
+        "num_time_tokens": 0,
+        "num_video_frames": 8,
+        "pad_block": false,
+        "pad_to_multiple_of": 0,
+        "qchoice": "none",
+        "quantize_model": false,
+        "refine_attn_blocksize": false,
+        "refine_col_blocksize": 4,
+        "refine_ln_blocksize": false,
+        "refine_ln_blocksize_but_only_backward": false,
+        "refine_ln_blocksize_but_only_forward": false,
+        "refine_ln_pertoken": false,
+        "refine_mlp_blocksize": false,
+        "refine_residual_fp": false,
+        "refine_row_blocksize": 4,
+        "row_blocksize": -1,
+        "row_blocksize_optimizer": 1,
+        "s2": false,
+        "s2_max_split_size": 336,
+        "s2_resize_output_to_scale_idx": 0,
+        "s2_scales": "336,672,1008",
+        "second_order_bit": null,
+        "second_order_quant_type": null,
+        "soft_ce_std": 1.0,
+        "symm": true,
+        "time_token_format": "<t{t}>",
+        "time_token_ids": [],
+        "transformers_version": "4.46.0",
+        "tune_language_model": true,
+        "tune_mm_projector": true,
+        "tune_vision_tower": true,
+        "use_quantize_optimizer": false,
+        "version": "auto",
+        "video_encoder": {
+            "_target_": "llava.model.encoders.BasicVideoEncoder"
+        },
+        "vision_resolution": -1,
+        "vision_tower": "Efficient-Large-Model/paligemma-siglip-so400m-patch14-448",
+        "vision_tower_cfg": {
+            "_attn_implementation_autoset": false,
+            "add_cross_attention": false,
+            "architectures": [
+                "SiglipVisionModel"
+            ],
+            "attention_dropout": 0.0,
+            "bad_words_ids": null,
+            "begin_suppress_tokens": null,
+            "bos_token_id": null,
+            "chunk_size_feed_forward": 0,
+            "cross_attention_hidden_size": null,
+            "decoder_start_token_id": null,
+            "diversity_penalty": 0.0,
+            "do_sample": false,
+            "early_stopping": false,
+            "encoder_no_repeat_ngram_size": 0,
+            "eos_token_id": null,
+            "exponential_decay_length_penalty": null,
+            "finetuning_task": null,
+            "forced_bos_token_id": null,
+            "forced_eos_token_id": null,
+            "hidden_act": "gelu_pytorch_tanh",
+            "hidden_size": 1152,
+            "id2label": {
+                "0": "LABEL_0",
+                "1": "LABEL_1"
+            },
+            "image_size": 448,
+            "intermediate_size": 4304,
+            "is_decoder": false,
+            "is_encoder_decoder": false,
+            "label2id": {
+                "LABEL_0": 0,
+                "LABEL_1": 1
+            },
+            "layer_norm_eps": 1e-6,
+            "length_penalty": 1.0,
+            "max_length": 20,
+            "min_length": 0,
+            "model_type": "siglip_vision_model",
+            "no_repeat_ngram_size": 0,
+            "num_attention_heads": 16,
+            "num_beam_groups": 1,
+            "num_beams": 1,
+            "num_channels": 3,
+            "num_hidden_layers": 27,
+            "num_image_tokens": 256,
+            "num_return_sequences": 1,
+            "output_attentions": false,
+            "output_hidden_states": false,
+            "output_scores": false,
+            "pad_token_id": null,
+            "patch_size": 14,
+            "prefix": null,
+            "problem_type": null,
+            "projection_dim": 2048,
+            "projector_hidden_act": "gelu_fast",
+            "pruned_heads": {},
+            "remove_invalid_values": false,
+            "repetition_penalty": 1.0,
+            "return_dict": true,
+            "return_dict_in_generate": false,
+            "sep_token_id": null,
+            "suppress_tokens": null,
+            "task_specific_params": null,
+            "temperature": 1.0,
+            "tf_legacy_loss": false,
+            "tie_encoder_decoder": false,
+            "tie_word_embeddings": true,
+            "tokenizer_class": null,
+            "top_k": 50,
+            "top_p": 1.0,
+            "torch_dtype": "bfloat16",
+            "torchscript": false,
+            "typical_p": 1.0,
+            "use_bfloat16": false,
+            "vision_use_head": false
+        },
+        "vision_tower_lr": null,
+        "weight_memory_efficient": true
+    },
+    "projector_config": {
+        "architectures": [
+            "MultimodalProjector"
+        ],
+        "mm_projector_type": "mlp_downsample_3x3_fix",
+        "model_type": "v2l_projector",
+        "torch_dtype": "bfloat16",
+        "transformers_version": "4.46.0"
+    },
+    "vision_config": {
+        "architectures": [
+            "SiglipVisionModel"
+        ],
+        "attention_dropout": 0.0,
+        "hidden_act": "gelu_pytorch_tanh",
+        "hidden_size": 1152,
+        "image_size": 448,
+        "intermediate_size": 4304,
+        "layer_norm_eps": 1e-6,
+        "model_type": "siglip_vision_model",
+        "num_attention_heads": 16,
+        "num_channels": 3,
+        "num_hidden_layers": 27,
+        "num_image_tokens": 256,
+        "patch_size": 14,
+        "projection_dim": 2048,
+        "projector_hidden_act": "gelu_fast",
+        "torch_dtype": "bfloat16",
+        "transformers_version": "4.46.0",
+        "vision_use_head": false
+    },
+    "llm_config": {
+        "architectures": [
+            "Qwen2ForCausalLM"
+        ],
+        "attention_dropout": 0.0,
+        "bos_token_id": 151643,
+        "eos_token_id": 151645,
+        "hidden_act": "silu",
+        "hidden_size": 1536,
+        "initializer_range": 0.02,
+        "intermediate_size": 8960,
+        "max_position_embeddings": 32768,
+        "max_window_layers": 28,
+        "model_max_length": 4096,
+        "model_type": "qwen2",
+        "num_attention_heads": 12,
+        "num_hidden_layers": 28,
+        "num_key_value_heads": 2,
+        "rms_norm_eps": 1e-6,
+        "rope_scaling": null,
+        "rope_theta": 1000000.0,
+        "sliding_window": null,
+        "tie_word_embeddings": true,
+        "tokenizer_model_max_length": 4096,
+        "tokenizer_padding_side": "right",
+        "torch_dtype": "bfloat16",
+        "transformers_version": "4.46.0",
+        "use_cache": false,
+        "use_sliding_window": false,
+        "vocab_size": 151651
+    },
+    "tokenizer_path": "/mnt/data3/shuo03.wang/model/aux_think_2B/llm",
+    "image_processor_path": "/mnt/data3/shuo03.wang/model/aux_think_2B/vision_tower/preprocessor_config.json"
+}