Upload folder using huggingface_hub
Browse files- .claude/settings.local.json +9 -0
- __pycache__/configuration_internvideo3.cpython-313.pyc +0 -0
- __pycache__/processing_internvideo3.cpython-313.pyc +0 -0
- added_tokens.json +28 -0
- chat_template.jinja +120 -0
- config.json +110 -0
- configuration_internvideo3.py +300 -0
- generation_config.json +13 -0
- inference_demo.py +183 -0
- merges.txt +0 -0
- model-language-0001-others-save_rank0.safetensors +3 -0
- model-language-0002-others-save_rank0.safetensors +3 -0
- model-language-0003-others-save_rank0.safetensors +3 -0
- model-language-0004-others-save_rank0.safetensors +3 -0
- model-language-0005-others-save_rank0.safetensors +3 -0
- model-projector-0001-others-save_rank0.safetensors +3 -0
- model-vision-0001-others-save_rank0.safetensors +3 -0
- model.safetensors.index.json +737 -0
- modeling_internvideo3.py +0 -0
- modeling_internvideo3_xslinear.py +1551 -0
- preprocessor_config.json +42 -0
- processing_internvideo3.py +237 -0
- special_tokens_map.json +31 -0
- tokenizer.json +0 -0
- tokenizer_config.json +239 -0
- video_preprocessor_config.json +21 -0
- video_processing_internvideo3.py +235 -0
- vocab.json +0 -0
.claude/settings.local.json
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"permissions": {
|
| 3 |
+
"allow": [
|
| 4 |
+
"Bash(python:*)",
|
| 5 |
+
"Bash(python3:*)",
|
| 6 |
+
"Bash(pip show:*)"
|
| 7 |
+
]
|
| 8 |
+
}
|
| 9 |
+
}
|
__pycache__/configuration_internvideo3.cpython-313.pyc
ADDED
|
Binary file (13.8 kB). View file
|
|
|
__pycache__/processing_internvideo3.cpython-313.pyc
ADDED
|
Binary file (11.3 kB). View file
|
|
|
added_tokens.json
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"</think>": 151668,
|
| 3 |
+
"</tool_call>": 151658,
|
| 4 |
+
"</tool_response>": 151666,
|
| 5 |
+
"<think>": 151667,
|
| 6 |
+
"<tool_call>": 151657,
|
| 7 |
+
"<tool_response>": 151665,
|
| 8 |
+
"<|box_end|>": 151649,
|
| 9 |
+
"<|box_start|>": 151648,
|
| 10 |
+
"<|endoftext|>": 151643,
|
| 11 |
+
"<|file_sep|>": 151664,
|
| 12 |
+
"<|fim_middle|>": 151660,
|
| 13 |
+
"<|fim_pad|>": 151662,
|
| 14 |
+
"<|fim_prefix|>": 151659,
|
| 15 |
+
"<|fim_suffix|>": 151661,
|
| 16 |
+
"<|im_end|>": 151645,
|
| 17 |
+
"<|im_start|>": 151644,
|
| 18 |
+
"<|image_pad|>": 151655,
|
| 19 |
+
"<|object_ref_end|>": 151647,
|
| 20 |
+
"<|object_ref_start|>": 151646,
|
| 21 |
+
"<|quad_end|>": 151651,
|
| 22 |
+
"<|quad_start|>": 151650,
|
| 23 |
+
"<|repo_name|>": 151663,
|
| 24 |
+
"<|video_pad|>": 151656,
|
| 25 |
+
"<|vision_end|>": 151653,
|
| 26 |
+
"<|vision_pad|>": 151654,
|
| 27 |
+
"<|vision_start|>": 151652
|
| 28 |
+
}
|
chat_template.jinja
ADDED
|
@@ -0,0 +1,120 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{%- if tools %}
|
| 2 |
+
{{- '<|im_start|>system\n' }}
|
| 3 |
+
{%- if messages[0].role == 'system' %}
|
| 4 |
+
{%- if messages[0].content is string %}
|
| 5 |
+
{{- messages[0].content }}
|
| 6 |
+
{%- else %}
|
| 7 |
+
{%- for content in messages[0].content %}
|
| 8 |
+
{%- if 'text' in content %}
|
| 9 |
+
{{- content.text }}
|
| 10 |
+
{%- endif %}
|
| 11 |
+
{%- endfor %}
|
| 12 |
+
{%- endif %}
|
| 13 |
+
{{- '\n\n' }}
|
| 14 |
+
{%- endif %}
|
| 15 |
+
{{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
|
| 16 |
+
{%- for tool in tools %}
|
| 17 |
+
{{- "\n" }}
|
| 18 |
+
{{- tool | tojson }}
|
| 19 |
+
{%- endfor %}
|
| 20 |
+
{{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
|
| 21 |
+
{%- else %}
|
| 22 |
+
{%- if messages[0].role == 'system' %}
|
| 23 |
+
{{- '<|im_start|>system\n' }}
|
| 24 |
+
{%- if messages[0].content is string %}
|
| 25 |
+
{{- messages[0].content }}
|
| 26 |
+
{%- else %}
|
| 27 |
+
{%- for content in messages[0].content %}
|
| 28 |
+
{%- if 'text' in content %}
|
| 29 |
+
{{- content.text }}
|
| 30 |
+
{%- endif %}
|
| 31 |
+
{%- endfor %}
|
| 32 |
+
{%- endif %}
|
| 33 |
+
{{- '<|im_end|>\n' }}
|
| 34 |
+
{%- endif %}
|
| 35 |
+
{%- endif %}
|
| 36 |
+
{%- set image_count = namespace(value=0) %}
|
| 37 |
+
{%- set video_count = namespace(value=0) %}
|
| 38 |
+
{%- for message in messages %}
|
| 39 |
+
{%- if message.role == "user" %}
|
| 40 |
+
{{- '<|im_start|>' + message.role + '\n' }}
|
| 41 |
+
{%- if message.content is string %}
|
| 42 |
+
{{- message.content }}
|
| 43 |
+
{%- else %}
|
| 44 |
+
{%- for content in message.content %}
|
| 45 |
+
{%- if content.type == 'image' or 'image' in content or 'image_url' in content %}
|
| 46 |
+
{%- set image_count.value = image_count.value + 1 %}
|
| 47 |
+
{%- if add_vision_id %}Picture {{ image_count.value }}: {% endif -%}
|
| 48 |
+
<|vision_start|><|image_pad|><|vision_end|>
|
| 49 |
+
{%- elif content.type == 'video' or 'video' in content %}
|
| 50 |
+
{%- set video_count.value = video_count.value + 1 %}
|
| 51 |
+
{%- if add_vision_id %}Video {{ video_count.value }}: {% endif -%}
|
| 52 |
+
<|vision_start|><|video_pad|><|vision_end|>
|
| 53 |
+
{%- elif 'text' in content %}
|
| 54 |
+
{{- content.text }}
|
| 55 |
+
{%- endif %}
|
| 56 |
+
{%- endfor %}
|
| 57 |
+
{%- endif %}
|
| 58 |
+
{{- '<|im_end|>\n' }}
|
| 59 |
+
{%- elif message.role == "assistant" %}
|
| 60 |
+
{{- '<|im_start|>' + message.role + '\n' }}
|
| 61 |
+
{%- if message.content is string %}
|
| 62 |
+
{{- message.content }}
|
| 63 |
+
{%- else %}
|
| 64 |
+
{%- for content_item in message.content %}
|
| 65 |
+
{%- if 'text' in content_item %}
|
| 66 |
+
{{- content_item.text }}
|
| 67 |
+
{%- endif %}
|
| 68 |
+
{%- endfor %}
|
| 69 |
+
{%- endif %}
|
| 70 |
+
{%- if message.tool_calls %}
|
| 71 |
+
{%- for tool_call in message.tool_calls %}
|
| 72 |
+
{%- if (loop.first and message.content) or (not loop.first) %}
|
| 73 |
+
{{- '\n' }}
|
| 74 |
+
{%- endif %}
|
| 75 |
+
{%- if tool_call.function %}
|
| 76 |
+
{%- set tool_call = tool_call.function %}
|
| 77 |
+
{%- endif %}
|
| 78 |
+
{{- '<tool_call>\n{"name": "' }}
|
| 79 |
+
{{- tool_call.name }}
|
| 80 |
+
{{- '", "arguments": ' }}
|
| 81 |
+
{%- if tool_call.arguments is string %}
|
| 82 |
+
{{- tool_call.arguments }}
|
| 83 |
+
{%- else %}
|
| 84 |
+
{{- tool_call.arguments | tojson }}
|
| 85 |
+
{%- endif %}
|
| 86 |
+
{{- '}\n</tool_call>' }}
|
| 87 |
+
{%- endfor %}
|
| 88 |
+
{%- endif %}
|
| 89 |
+
{{- '<|im_end|>\n' }}
|
| 90 |
+
{%- elif message.role == "tool" %}
|
| 91 |
+
{%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
|
| 92 |
+
{{- '<|im_start|>user' }}
|
| 93 |
+
{%- endif %}
|
| 94 |
+
{{- '\n<tool_response>\n' }}
|
| 95 |
+
{%- if message.content is string %}
|
| 96 |
+
{{- message.content }}
|
| 97 |
+
{%- else %}
|
| 98 |
+
{%- for content in message.content %}
|
| 99 |
+
{%- if content.type == 'image' or 'image' in content or 'image_url' in content %}
|
| 100 |
+
{%- set image_count.value = image_count.value + 1 %}
|
| 101 |
+
{%- if add_vision_id %}Picture {{ image_count.value }}: {% endif -%}
|
| 102 |
+
<|vision_start|><|image_pad|><|vision_end|>
|
| 103 |
+
{%- elif content.type == 'video' or 'video' in content %}
|
| 104 |
+
{%- set video_count.value = video_count.value + 1 %}
|
| 105 |
+
{%- if add_vision_id %}Video {{ video_count.value }}: {% endif -%}
|
| 106 |
+
<|vision_start|><|video_pad|><|vision_end|>
|
| 107 |
+
{%- elif 'text' in content %}
|
| 108 |
+
{{- content.text }}
|
| 109 |
+
{%- endif %}
|
| 110 |
+
{%- endfor %}
|
| 111 |
+
{%- endif %}
|
| 112 |
+
{{- '\n</tool_response>' }}
|
| 113 |
+
{%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
|
| 114 |
+
{{- '<|im_end|>\n' }}
|
| 115 |
+
{%- endif %}
|
| 116 |
+
{%- endif %}
|
| 117 |
+
{%- endfor %}
|
| 118 |
+
{%- if add_generation_prompt %}
|
| 119 |
+
{{- '<|im_start|>assistant\n' }}
|
| 120 |
+
{%- endif %}
|
config.json
ADDED
|
@@ -0,0 +1,110 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"architectures": [
|
| 3 |
+
"InternVideo3ForConditionalGeneration"
|
| 4 |
+
],
|
| 5 |
+
"auto_map": {
|
| 6 |
+
"AutoModel": "modeling_internvideo3.InternVideo3Model",
|
| 7 |
+
"AutoModelForCausalLM": "modeling_internvideo3.InternVideo3ForConditionalGeneration",
|
| 8 |
+
"AutoConfig": "configuration_internvideo3.InternVideo3Config",
|
| 9 |
+
"AutoProcessor": "processing_internvideo3.InternVideo3Processor"
|
| 10 |
+
},
|
| 11 |
+
"dtype": "bfloat16",
|
| 12 |
+
"image_token_id": 151655,
|
| 13 |
+
"model_type": "internvideo3",
|
| 14 |
+
"text_config": {
|
| 15 |
+
"attention_bias": true,
|
| 16 |
+
"attention_dropout": 0.0,
|
| 17 |
+
"bos_token_id": 151643,
|
| 18 |
+
"dtype": "bfloat16",
|
| 19 |
+
"eos_token_id": 151645,
|
| 20 |
+
"head_dim": 128,
|
| 21 |
+
"hidden_act": "silu",
|
| 22 |
+
"hidden_size": 4096,
|
| 23 |
+
"initializer_range": 0.02,
|
| 24 |
+
"intermediate_size": 12288,
|
| 25 |
+
"max_position_embeddings": 262144,
|
| 26 |
+
"model_type": "internvideo3_text",
|
| 27 |
+
"num_attention_heads": 32,
|
| 28 |
+
"num_hidden_layers": 36,
|
| 29 |
+
"num_key_value_heads": 32,
|
| 30 |
+
"rms_norm_eps": 1e-06,
|
| 31 |
+
"rope_scaling": {
|
| 32 |
+
"mrope_interleaved": true,
|
| 33 |
+
"mrope_section": [
|
| 34 |
+
24,
|
| 35 |
+
20,
|
| 36 |
+
20
|
| 37 |
+
],
|
| 38 |
+
"rope_type": "default"
|
| 39 |
+
},
|
| 40 |
+
"rope_theta": 5000000,
|
| 41 |
+
"use_cache": true,
|
| 42 |
+
"vocab_size": 151936,
|
| 43 |
+
"qk_rope_head_dim": 128,
|
| 44 |
+
"qk_nope_head_dim": 128,
|
| 45 |
+
"v_head_dim": 128,
|
| 46 |
+
"q_lora_rank": null,
|
| 47 |
+
"rope_interleave": true,
|
| 48 |
+
"kv_lora_rank_list": [
|
| 49 |
+
896,
|
| 50 |
+
896,
|
| 51 |
+
896,
|
| 52 |
+
896,
|
| 53 |
+
896,
|
| 54 |
+
896,
|
| 55 |
+
896,
|
| 56 |
+
896,
|
| 57 |
+
896,
|
| 58 |
+
896,
|
| 59 |
+
896,
|
| 60 |
+
896,
|
| 61 |
+
896,
|
| 62 |
+
896,
|
| 63 |
+
896,
|
| 64 |
+
896,
|
| 65 |
+
896,
|
| 66 |
+
896,
|
| 67 |
+
896,
|
| 68 |
+
896,
|
| 69 |
+
896,
|
| 70 |
+
896,
|
| 71 |
+
896,
|
| 72 |
+
896,
|
| 73 |
+
896,
|
| 74 |
+
896,
|
| 75 |
+
896,
|
| 76 |
+
896,
|
| 77 |
+
896,
|
| 78 |
+
896,
|
| 79 |
+
896,
|
| 80 |
+
896,
|
| 81 |
+
896,
|
| 82 |
+
896,
|
| 83 |
+
896,
|
| 84 |
+
896
|
| 85 |
+
],
|
| 86 |
+
"qk_latent_layernorm": false
|
| 87 |
+
},
|
| 88 |
+
"tie_word_embeddings": false,
|
| 89 |
+
"transformers_version": "4.57.3",
|
| 90 |
+
"video_token_id": 151656,
|
| 91 |
+
"vision_config": {
|
| 92 |
+
"deepstack_visual_indexes": [],
|
| 93 |
+
"depth": 27,
|
| 94 |
+
"dtype": "bfloat16",
|
| 95 |
+
"hidden_act": "gelu_pytorch_tanh",
|
| 96 |
+
"hidden_size": 1152,
|
| 97 |
+
"in_channels": 3,
|
| 98 |
+
"initializer_range": 0.02,
|
| 99 |
+
"intermediate_size": 4304,
|
| 100 |
+
"model_type": "internvideo3",
|
| 101 |
+
"num_heads": 16,
|
| 102 |
+
"num_position_embeddings": 2304,
|
| 103 |
+
"out_hidden_size": 4096,
|
| 104 |
+
"patch_size": 16,
|
| 105 |
+
"spatial_merge_size": 2,
|
| 106 |
+
"temporal_patch_size": 2
|
| 107 |
+
},
|
| 108 |
+
"vision_end_token_id": 151653,
|
| 109 |
+
"vision_start_token_id": 151652
|
| 110 |
+
}
|
configuration_internvideo3.py
ADDED
|
@@ -0,0 +1,300 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# coding=utf-8
|
| 2 |
+
# Copyright 2025 The InternVideo Team. All rights reserved.
|
| 3 |
+
#
|
| 4 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 5 |
+
# you may not use this file except in compliance with the License.
|
| 6 |
+
# You may obtain a copy of the License at
|
| 7 |
+
#
|
| 8 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 9 |
+
#
|
| 10 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 11 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 12 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 13 |
+
# See the License for the specific language governing permissions and
|
| 14 |
+
# limitations under the License.
|
| 15 |
+
from transformers.configuration_utils import PretrainedConfig
|
| 16 |
+
from transformers.modeling_rope_utils import rope_config_validation
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
class InternVideo3VisionConfig(PretrainedConfig):
|
| 21 |
+
model_type = "internvideo3"
|
| 22 |
+
base_config_key = "vision_config"
|
| 23 |
+
|
| 24 |
+
def __init__(
|
| 25 |
+
self,
|
| 26 |
+
depth=27,
|
| 27 |
+
hidden_size=1152,
|
| 28 |
+
hidden_act="gelu_pytorch_tanh",
|
| 29 |
+
intermediate_size=4304,
|
| 30 |
+
num_heads=16,
|
| 31 |
+
in_channels=3,
|
| 32 |
+
patch_size=16,
|
| 33 |
+
spatial_merge_size=2,
|
| 34 |
+
temporal_patch_size=2,
|
| 35 |
+
out_hidden_size=3584,
|
| 36 |
+
num_position_embeddings=2304,
|
| 37 |
+
deepstack_visual_indexes=[8, 16, 24],
|
| 38 |
+
initializer_range=0.02,
|
| 39 |
+
**kwargs,
|
| 40 |
+
):
|
| 41 |
+
super().__init__(**kwargs)
|
| 42 |
+
|
| 43 |
+
self.depth = depth
|
| 44 |
+
self.hidden_size = hidden_size
|
| 45 |
+
self.hidden_act = hidden_act
|
| 46 |
+
self.intermediate_size = intermediate_size
|
| 47 |
+
self.num_heads = num_heads
|
| 48 |
+
self.in_channels = in_channels
|
| 49 |
+
self.patch_size = patch_size
|
| 50 |
+
self.spatial_merge_size = spatial_merge_size
|
| 51 |
+
self.temporal_patch_size = temporal_patch_size
|
| 52 |
+
self.out_hidden_size = out_hidden_size
|
| 53 |
+
self.num_position_embeddings = num_position_embeddings
|
| 54 |
+
self.initializer_range = initializer_range
|
| 55 |
+
self.deepstack_visual_indexes = deepstack_visual_indexes
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
class InternVideo3TextConfig(PretrainedConfig):
|
| 59 |
+
r"""
|
| 60 |
+
This is the configuration class to store the configuration of a [`InternVideo3TextModel`]. It is used to instantiate a
|
| 61 |
+
Qwen3-VL model according to the specified arguments, defining the model architecture. Instantiating a configuration
|
| 62 |
+
with the defaults will yield a similar configuration to that of
|
| 63 |
+
Qwen3-VL-4B-Instruct [Qwen/Qwen3-VL-4B-Instruct](https://huggingface.co/Qwen/Qwen3-VL-4B-Instruct).
|
| 64 |
+
|
| 65 |
+
Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
|
| 66 |
+
documentation from [`PretrainedConfig`] for more information.
|
| 67 |
+
|
| 68 |
+
Args:
|
| 69 |
+
vocab_size (`int`, *optional*, defaults to 151936):
|
| 70 |
+
Vocabulary size of the InternVideo3 model. Defines the number of different tokens that can be represented by the
|
| 71 |
+
`inputs_ids` passed when calling [`InternVideo3Model`]
|
| 72 |
+
hidden_size (`int`, *optional*, defaults to 4096):
|
| 73 |
+
Dimension of the hidden representations.
|
| 74 |
+
intermediate_size (`int`, *optional*, defaults to 22016):
|
| 75 |
+
Dimension of the MLP representations.
|
| 76 |
+
num_hidden_layers (`int`, *optional*, defaults to 32):
|
| 77 |
+
Number of hidden layers in the Transformer encoder.
|
| 78 |
+
num_attention_heads (`int`, *optional*, defaults to 32):
|
| 79 |
+
Number of attention heads for each attention layer in the Transformer encoder.
|
| 80 |
+
num_key_value_heads (`int`, *optional*, defaults to 32):
|
| 81 |
+
This is the number of key_value heads that should be used to implement Grouped Query Attention. If
|
| 82 |
+
`num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
|
| 83 |
+
`num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used. When
|
| 84 |
+
converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
|
| 85 |
+
by meanpooling all the original heads within that group. For more details, check out [this
|
| 86 |
+
paper](https://huggingface.co/papers/2305.13245). If it is not specified, will default to `32`.
|
| 87 |
+
head_dim (`int`, *optional*, defaults to 128):
|
| 88 |
+
The dimension of the head. If not specified, will default to `hidden_size // num_attention_heads`.
|
| 89 |
+
hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
|
| 90 |
+
The non-linear activation function (function or string) in the decoder.
|
| 91 |
+
max_position_embeddings (`int`, *optional*, defaults to 128000):
|
| 92 |
+
The maximum sequence length that this model might ever be used with.
|
| 93 |
+
initializer_range (`float`, *optional*, defaults to 0.02):
|
| 94 |
+
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
|
| 95 |
+
rms_norm_eps (`float`, *optional*, defaults to 1e-06):
|
| 96 |
+
The epsilon used by the rms normalization layers.
|
| 97 |
+
use_cache (`bool`, *optional*, defaults to `True`):
|
| 98 |
+
Whether or not the model should return the last key/values attentions (not used by all models). Only
|
| 99 |
+
relevant if `config.is_decoder=True`.
|
| 100 |
+
tie_word_embeddings (`bool`, *optional*, defaults to `False`):
|
| 101 |
+
Whether the model's input and output word embeddings should be tied.
|
| 102 |
+
rope_theta (`float`, *optional*, defaults to 5000000.0):
|
| 103 |
+
The base period of the RoPE embeddings.
|
| 104 |
+
rope_scaling (`Dict`, *optional*):
|
| 105 |
+
Dictionary containing the scaling configuration for the RoPE embeddings. NOTE: if you apply new rope type
|
| 106 |
+
and you expect the model to work on longer `max_position_embeddings`, we recommend you to update this value
|
| 107 |
+
accordingly.
|
| 108 |
+
Expected contents:
|
| 109 |
+
`rope_type` (`str`):
|
| 110 |
+
The sub-variant of RoPE to use. Can be one of ['default', 'linear', 'dynamic', 'yarn', 'longrope',
|
| 111 |
+
'llama3'], with 'default' being the original RoPE implementation.
|
| 112 |
+
`factor` (`float`, *optional*):
|
| 113 |
+
Used with all rope types except 'default'. The scaling factor to apply to the RoPE embeddings. In
|
| 114 |
+
most scaling types, a `factor` of x will enable the model to handle sequences of length x *
|
| 115 |
+
original maximum pre-trained length.
|
| 116 |
+
`original_max_position_embeddings` (`int`, *optional*):
|
| 117 |
+
Used with 'dynamic', 'longrope' and 'llama3'. The original max position embeddings used during
|
| 118 |
+
pretraining.
|
| 119 |
+
`attention_factor` (`float`, *optional*):
|
| 120 |
+
Used with 'yarn' and 'longrope'. The scaling factor to be applied on the attention
|
| 121 |
+
computation. If unspecified, it defaults to value recommended by the implementation, using the
|
| 122 |
+
`factor` field to infer the suggested value.
|
| 123 |
+
`beta_fast` (`float`, *optional*):
|
| 124 |
+
Only used with 'yarn'. Parameter to set the boundary for extrapolation (only) in the linear
|
| 125 |
+
ramp function. If unspecified, it defaults to 32.
|
| 126 |
+
`beta_slow` (`float`, *optional*):
|
| 127 |
+
Only used with 'yarn'. Parameter to set the boundary for interpolation (only) in the linear
|
| 128 |
+
ramp function. If unspecified, it defaults to 1.
|
| 129 |
+
`short_factor` (`list[float]`, *optional*):
|
| 130 |
+
Only used with 'longrope'. The scaling factor to be applied to short contexts (<
|
| 131 |
+
`original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden
|
| 132 |
+
size divided by the number of attention heads divided by 2
|
| 133 |
+
`long_factor` (`list[float]`, *optional*):
|
| 134 |
+
Only used with 'longrope'. The scaling factor to be applied to long contexts (<
|
| 135 |
+
`original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden
|
| 136 |
+
size divided by the number of attention heads divided by 2
|
| 137 |
+
`low_freq_factor` (`float`, *optional*):
|
| 138 |
+
Only used with 'llama3'. Scaling factor applied to low frequency components of the RoPE
|
| 139 |
+
`high_freq_factor` (`float`, *optional*):
|
| 140 |
+
Only used with 'llama3'. Scaling factor applied to high frequency components of the RoPE
|
| 141 |
+
attention_bias (`bool`, defaults to `False`, *optional*, defaults to `False`):
|
| 142 |
+
Whether to use a bias in the query, key, value and output projection layers during self-attention.
|
| 143 |
+
attention_dropout (`float`, *optional*, defaults to 0.0):
|
| 144 |
+
The dropout ratio for the attention probabilities.
|
| 145 |
+
|
| 146 |
+
```python
|
| 147 |
+
>>> from transformers import InternVideo3TextModel, InternVideo3TextConfig
|
| 148 |
+
|
| 149 |
+
>>> # Initializing a InternVideo3 style configuration
|
| 150 |
+
>>> configuration = InternVideo3TextConfig()
|
| 151 |
+
|
| 152 |
+
>>> # Initializing a model from the Qwen3-VL-7B style configuration
|
| 153 |
+
>>> model = InternVideo3TextModel(configuration)
|
| 154 |
+
|
| 155 |
+
>>> # Accessing the model configuration
|
| 156 |
+
>>> configuration = model.config
|
| 157 |
+
```"""
|
| 158 |
+
|
| 159 |
+
model_type = "internvideo3_text"
|
| 160 |
+
base_config_key = "text_config"
|
| 161 |
+
|
| 162 |
+
def __init__(
|
| 163 |
+
self,
|
| 164 |
+
vocab_size=151936,
|
| 165 |
+
hidden_size=4096,
|
| 166 |
+
intermediate_size=22016,
|
| 167 |
+
num_hidden_layers=32,
|
| 168 |
+
num_attention_heads=32,
|
| 169 |
+
num_key_value_heads=32,
|
| 170 |
+
head_dim=128,
|
| 171 |
+
hidden_act="silu",
|
| 172 |
+
max_position_embeddings=128000,
|
| 173 |
+
initializer_range=0.02,
|
| 174 |
+
rms_norm_eps=1e-6,
|
| 175 |
+
use_cache=True,
|
| 176 |
+
tie_word_embeddings=False,
|
| 177 |
+
rope_theta=5000000.0,
|
| 178 |
+
rope_scaling=None,
|
| 179 |
+
attention_bias=False,
|
| 180 |
+
attention_dropout=0.0,
|
| 181 |
+
kv_lora_rank=512,
|
| 182 |
+
kv_lora_rank_list=None,
|
| 183 |
+
q_lora_rank=None,
|
| 184 |
+
qk_rope_head_dim=64,
|
| 185 |
+
qk_nope_head_dim=128,
|
| 186 |
+
v_head_dim=128,
|
| 187 |
+
qk_latent_layernorm=True,
|
| 188 |
+
rope_interleave=True,
|
| 189 |
+
**kwargs,
|
| 190 |
+
):
|
| 191 |
+
self.vocab_size = vocab_size
|
| 192 |
+
self.max_position_embeddings = max_position_embeddings
|
| 193 |
+
self.hidden_size = hidden_size
|
| 194 |
+
self.intermediate_size = intermediate_size
|
| 195 |
+
self.num_hidden_layers = num_hidden_layers
|
| 196 |
+
self.num_attention_heads = num_attention_heads
|
| 197 |
+
|
| 198 |
+
# for backward compatibility
|
| 199 |
+
if num_key_value_heads is None:
|
| 200 |
+
num_key_value_heads = num_attention_heads
|
| 201 |
+
|
| 202 |
+
self.num_key_value_heads = num_key_value_heads
|
| 203 |
+
self.head_dim = head_dim
|
| 204 |
+
self.hidden_act = hidden_act
|
| 205 |
+
self.initializer_range = initializer_range
|
| 206 |
+
self.rms_norm_eps = rms_norm_eps
|
| 207 |
+
self.use_cache = use_cache
|
| 208 |
+
self.rope_theta = rope_theta
|
| 209 |
+
self.rope_scaling = rope_scaling
|
| 210 |
+
self.attention_bias = attention_bias
|
| 211 |
+
self.attention_dropout = attention_dropout
|
| 212 |
+
|
| 213 |
+
rope_config_validation(self, ignore_keys={"mrope_section", "mrope_interleaved"})
|
| 214 |
+
|
| 215 |
+
super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs)
|
| 216 |
+
|
| 217 |
+
self.kv_lora_rank = kv_lora_rank
|
| 218 |
+
self.kv_lora_rank_list = kv_lora_rank_list
|
| 219 |
+
self.q_lora_rank = q_lora_rank
|
| 220 |
+
self.qk_rope_head_dim = qk_rope_head_dim
|
| 221 |
+
self.qk_nope_head_dim = qk_nope_head_dim
|
| 222 |
+
self.qk_head_dim = qk_rope_head_dim + qk_nope_head_dim
|
| 223 |
+
self.v_head_dim = v_head_dim
|
| 224 |
+
self.qk_latent_layernorm = qk_latent_layernorm
|
| 225 |
+
self.rope_interleave = rope_interleave
|
| 226 |
+
|
| 227 |
+
|
| 228 |
+
class InternVideo3Config(PretrainedConfig):
|
| 229 |
+
r"""
|
| 230 |
+
This is the configuration class to store the configuration of a [`InternVideo3Model`]. It is used to instantiate a
|
| 231 |
+
Qwen3-VL model according to the specified arguments, defining the model architecture. Instantiating a configuration
|
| 232 |
+
with the defaults will yield a similar configuration to that of
|
| 233 |
+
Qwen3-VL-4B-Instruct [Qwen/Qwen3-VL-4B-Instruct](https://huggingface.co/Qwen/Qwen3-VL-4B-Instruct).
|
| 234 |
+
|
| 235 |
+
Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
|
| 236 |
+
documentation from [`PretrainedConfig`] for more information.
|
| 237 |
+
|
| 238 |
+
|
| 239 |
+
Args:
|
| 240 |
+
text_config (`Union[PreTrainedConfig, dict]`, *optional*, defaults to `InternVideo3TextConfig`):
|
| 241 |
+
The config object or dictionary of the text backbone.
|
| 242 |
+
vision_config (`Union[PreTrainedConfig, dict]`, *optional*, defaults to `InternVideo3VisionConfig`):
|
| 243 |
+
The config object or dictionary of the vision backbone.
|
| 244 |
+
image_token_id (`int`, *optional*, defaults to 151655):
|
| 245 |
+
The image token index to encode the image prompt.
|
| 246 |
+
video_token_id (`int`, *optional*, defaults to 151656):
|
| 247 |
+
The video token index to encode the image prompt.
|
| 248 |
+
vision_start_token_id (`int`, *optional*, defaults to 151652):
|
| 249 |
+
The start token index to encode the image prompt.
|
| 250 |
+
vision_end_token_id (`int`, *optional*, defaults to 151653):
|
| 251 |
+
The end token index to encode the image prompt.
|
| 252 |
+
tie_word_embeddings (`bool`, *optional*, defaults to `False`):
|
| 253 |
+
Whether to tie the word embeddings.
|
| 254 |
+
|
| 255 |
+
```python
|
| 256 |
+
>>> from transformers import InternVideo3ForConditionalGeneration, InternVideo3Config
|
| 257 |
+
|
| 258 |
+
>>> # Initializing a Qwen3-VL style configuration
|
| 259 |
+
>>> configuration = InternVideo3Config()
|
| 260 |
+
|
| 261 |
+
>>> # Initializing a model from the Qwen3-VL-4B style configuration
|
| 262 |
+
>>> model = InternVideo3ForConditionalGeneration(configuration)
|
| 263 |
+
|
| 264 |
+
>>> # Accessing the model configuration
|
| 265 |
+
>>> configuration = model.config
|
| 266 |
+
```"""
|
| 267 |
+
|
| 268 |
+
model_type = "internvideo3"
|
| 269 |
+
sub_configs = {"vision_config": InternVideo3VisionConfig, "text_config": InternVideo3TextConfig}
|
| 270 |
+
keys_to_ignore_at_inference = ["past_key_values"]
|
| 271 |
+
|
| 272 |
+
def __init__(
|
| 273 |
+
self,
|
| 274 |
+
text_config=None,
|
| 275 |
+
vision_config=None,
|
| 276 |
+
image_token_id=151655,
|
| 277 |
+
video_token_id=151656,
|
| 278 |
+
vision_start_token_id=151652,
|
| 279 |
+
vision_end_token_id=151653,
|
| 280 |
+
tie_word_embeddings=False,
|
| 281 |
+
**kwargs,
|
| 282 |
+
):
|
| 283 |
+
if isinstance(vision_config, dict):
|
| 284 |
+
self.vision_config = self.sub_configs["vision_config"](**vision_config)
|
| 285 |
+
elif vision_config is None:
|
| 286 |
+
self.vision_config = self.sub_configs["vision_config"]()
|
| 287 |
+
|
| 288 |
+
if isinstance(text_config, dict):
|
| 289 |
+
self.text_config = self.sub_configs["text_config"](**text_config)
|
| 290 |
+
elif text_config is None:
|
| 291 |
+
self.text_config = self.sub_configs["text_config"]()
|
| 292 |
+
|
| 293 |
+
self.image_token_id = image_token_id
|
| 294 |
+
self.video_token_id = video_token_id
|
| 295 |
+
self.vision_start_token_id = vision_start_token_id
|
| 296 |
+
self.vision_end_token_id = vision_end_token_id
|
| 297 |
+
super().__init__(**kwargs, tie_word_embeddings=tie_word_embeddings)
|
| 298 |
+
|
| 299 |
+
|
| 300 |
+
__all__ = ["InternVideo3Config", "InternVideo3TextConfig"]
|
generation_config.json
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"bos_token_id": 151643,
|
| 3 |
+
"do_sample": true,
|
| 4 |
+
"eos_token_id": [
|
| 5 |
+
151645,
|
| 6 |
+
151643
|
| 7 |
+
],
|
| 8 |
+
"pad_token_id": 151643,
|
| 9 |
+
"temperature": 0.7,
|
| 10 |
+
"top_k": 20,
|
| 11 |
+
"top_p": 0.8,
|
| 12 |
+
"transformers_version": "4.57.3"
|
| 13 |
+
}
|
inference_demo.py
ADDED
|
@@ -0,0 +1,183 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
InternVideo3-8B-Instruct Inference Demo
|
| 3 |
+
|
| 4 |
+
Supports:
|
| 5 |
+
- Text-only conversation
|
| 6 |
+
- Video understanding
|
| 7 |
+
- Image understanding
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
import torch
|
| 11 |
+
import time
|
| 12 |
+
from transformers import AutoModelForCausalLM, AutoProcessor
|
| 13 |
+
from qwen_vl_utils import process_vision_info
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
# ============ Model Loading ============
|
| 17 |
+
model_path = "/mnt/shared-storage-user/yanziang/HF_toupload/InternVideo3-8B-Instruct" # current directory
|
| 18 |
+
|
| 19 |
+
model = AutoModelForCausalLM.from_pretrained(
|
| 20 |
+
model_path,
|
| 21 |
+
dtype=torch.bfloat16,
|
| 22 |
+
attn_implementation="sdpa",
|
| 23 |
+
device_map="cuda:0",
|
| 24 |
+
trust_remote_code=True,
|
| 25 |
+
)
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
processor = AutoProcessor.from_pretrained(
|
| 30 |
+
model_path,
|
| 31 |
+
trust_remote_code=True,
|
| 32 |
+
)
|
| 33 |
+
# processor = AutoProcessor.from_pretrained("/mnt/shared-storage-user/sfteval/sfteval_models/Qwen3-VL-8B-Instruct/",trust_remote_code=True)
|
| 34 |
+
# ============ Example 1: Text-only ============
|
| 35 |
+
print("=" * 50)
|
| 36 |
+
print("Example 1: Text-only")
|
| 37 |
+
print("=" * 50)
|
| 38 |
+
|
| 39 |
+
messages = [
|
| 40 |
+
{
|
| 41 |
+
"role": "user",
|
| 42 |
+
"content": [
|
| 43 |
+
{"type": "text", "text": "Please introduce yourself."},
|
| 44 |
+
],
|
| 45 |
+
}
|
| 46 |
+
]
|
| 47 |
+
|
| 48 |
+
text = processor.apply_chat_template(
|
| 49 |
+
messages,
|
| 50 |
+
tokenize=False,
|
| 51 |
+
add_generation_prompt=True,
|
| 52 |
+
enable_thinking=True,
|
| 53 |
+
)
|
| 54 |
+
|
| 55 |
+
inputs = processor(text=text, images=None, videos=None, do_resize=False, return_tensors="pt")
|
| 56 |
+
inputs = inputs.to(model.device)
|
| 57 |
+
|
| 58 |
+
start_time = time.time()
|
| 59 |
+
gen_output = model.generate(
|
| 60 |
+
**inputs,
|
| 61 |
+
max_new_tokens=1024,
|
| 62 |
+
use_cache=True,
|
| 63 |
+
)
|
| 64 |
+
|
| 65 |
+
generated_ids_trimmed = [
|
| 66 |
+
out_ids[len(in_ids):] for in_ids, out_ids in zip(inputs.input_ids, gen_output)
|
| 67 |
+
]
|
| 68 |
+
output_text = processor.batch_decode(
|
| 69 |
+
generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False
|
| 70 |
+
)
|
| 71 |
+
print(output_text[0])
|
| 72 |
+
print(f"Time taken: {time.time() - start_time:.2f}s\n")
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
# ============ Example 2: Video Understanding ============
|
| 76 |
+
print("=" * 50)
|
| 77 |
+
print("Example 2: Video Understanding")
|
| 78 |
+
print("=" * 50)
|
| 79 |
+
|
| 80 |
+
video_path = "/mnt/shared-storage-user/yanziang/space_woaudio.mp4" # Replace with your video path
|
| 81 |
+
|
| 82 |
+
fps = 1
|
| 83 |
+
min_pixels = 128 * 32 * 32
|
| 84 |
+
max_pixels = 128 * 32 * 32
|
| 85 |
+
|
| 86 |
+
messages = [
|
| 87 |
+
{
|
| 88 |
+
"role": "user",
|
| 89 |
+
"content": [
|
| 90 |
+
{
|
| 91 |
+
"type": "video",
|
| 92 |
+
"video": video_path,
|
| 93 |
+
"fps": fps,
|
| 94 |
+
},
|
| 95 |
+
{"type": "text", "text": "Please describe this video in detail."},
|
| 96 |
+
],
|
| 97 |
+
}
|
| 98 |
+
]
|
| 99 |
+
|
| 100 |
+
processor.video_processor.size = {"longest_edge": max_pixels * 512, "shortest_edge": min_pixels * 32}
|
| 101 |
+
|
| 102 |
+
inputs = processor.apply_chat_template(
|
| 103 |
+
messages,
|
| 104 |
+
tokenize=True,
|
| 105 |
+
add_generation_prompt=True,
|
| 106 |
+
return_dict=True,
|
| 107 |
+
fps=fps,
|
| 108 |
+
return_tensors="pt",
|
| 109 |
+
)
|
| 110 |
+
inputs = inputs.to(model.device)
|
| 111 |
+
|
| 112 |
+
start_time = time.time()
|
| 113 |
+
gen_output = model.generate(
|
| 114 |
+
**inputs,
|
| 115 |
+
max_new_tokens=1024,
|
| 116 |
+
use_cache=True,
|
| 117 |
+
)
|
| 118 |
+
|
| 119 |
+
generated_ids_trimmed = [
|
| 120 |
+
out_ids[len(in_ids):] for in_ids, out_ids in zip(inputs.input_ids, gen_output)
|
| 121 |
+
]
|
| 122 |
+
output_text = processor.batch_decode(
|
| 123 |
+
generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False
|
| 124 |
+
)
|
| 125 |
+
print(output_text[0])
|
| 126 |
+
print(f"Time taken: {time.time() - start_time:.2f}s\n")
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
# # ============ Example 3: Image Understanding ============
|
| 130 |
+
# print("=" * 50)
|
| 131 |
+
# print("Example 3: Image Understanding")
|
| 132 |
+
# print("=" * 50)
|
| 133 |
+
|
| 134 |
+
# image_path = "/mnt/shared-storage-user/yanziang/demo.jpeg" # Replace with your image path
|
| 135 |
+
|
| 136 |
+
# messages = [
|
| 137 |
+
# {
|
| 138 |
+
# "role": "user",
|
| 139 |
+
# "content": [
|
| 140 |
+
# {
|
| 141 |
+
# "type": "image",
|
| 142 |
+
# "image": image_path,
|
| 143 |
+
# },
|
| 144 |
+
# {"type": "text", "text": "Please describe this image in detail."},
|
| 145 |
+
# ],
|
| 146 |
+
# }
|
| 147 |
+
# ]
|
| 148 |
+
|
| 149 |
+
# text = processor.apply_chat_template(
|
| 150 |
+
# messages,
|
| 151 |
+
# tokenize=False,
|
| 152 |
+
# add_generation_prompt=True,
|
| 153 |
+
# enable_thinking=True,
|
| 154 |
+
# )
|
| 155 |
+
|
| 156 |
+
# images, videos, video_kwargs = process_vision_info(
|
| 157 |
+
# messages, image_patch_size=16, return_video_kwargs=True, return_video_metadata=True
|
| 158 |
+
# )
|
| 159 |
+
|
| 160 |
+
# inputs = processor(
|
| 161 |
+
# text=text,
|
| 162 |
+
# images=images,
|
| 163 |
+
# videos=None,
|
| 164 |
+
# do_resize=False,
|
| 165 |
+
# return_tensors="pt",
|
| 166 |
+
# )
|
| 167 |
+
# inputs = inputs.to(model.device)
|
| 168 |
+
|
| 169 |
+
# start_time = time.time()
|
| 170 |
+
# gen_output = model.generate(
|
| 171 |
+
# **inputs,
|
| 172 |
+
# max_new_tokens=1024,
|
| 173 |
+
# use_cache=True,
|
| 174 |
+
# )
|
| 175 |
+
|
| 176 |
+
# generated_ids_trimmed = [
|
| 177 |
+
# out_ids[len(in_ids):] for in_ids, out_ids in zip(inputs.input_ids, gen_output)
|
| 178 |
+
# ]
|
| 179 |
+
# output_text = processor.batch_decode(
|
| 180 |
+
# generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False
|
| 181 |
+
# )
|
| 182 |
+
# print(output_text[0])
|
| 183 |
+
# print(f"Time taken: {time.time() - start_time:.2f}s\n")
|
merges.txt
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
model-language-0001-others-save_rank0.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:914ec6c9f5ede61c3dd4694c6e8522b4db4e4e71b6b0675a3e49dc19d3754650
|
| 3 |
+
size 4292100488
|
model-language-0002-others-save_rank0.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:36262c85b72728b543eb0c482956ecb2d81fe3c41493531ee8e2f7434cdf0128
|
| 3 |
+
size 4280652416
|
model-language-0003-others-save_rank0.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a7b034b33c7704fdad971cbb80e2121857ee05664bdccfa55f39c3e83f7b04a7
|
| 3 |
+
size 4291135880
|
model-language-0004-others-save_rank0.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:4dea96bbcf19ea509c78e1ab521bc4dd9592aff5635f137f4e7ef01d43245187
|
| 3 |
+
size 3708071968
|
model-language-0005-others-save_rank0.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:2a6fdae561d4d1ebd1410a962c7f702d01b83a18a37b91089baf763281d985a7
|
| 3 |
+
size 1244659840
|
model-projector-0001-others-save_rank0.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:123db5dd1a0a768fb4e877bc80b5b8faf6e056aff6df6e4ee8235edaf92f6cc0
|
| 3 |
+
size 80238712
|
model-vision-0001-others-save_rank0.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:166a8a730fc7b51832200f937a4d79aea2c3a43401154046e9ecb6ee0b6fef1d
|
| 3 |
+
size 831819240
|
model.safetensors.index.json
ADDED
|
@@ -0,0 +1,737 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"weight_map": {
|
| 3 |
+
"model.language_model.norm.weight": "model-language-0001-others-save_rank0.safetensors",
|
| 4 |
+
"lm_head.weight": "model-language-0001-others-save_rank0.safetensors",
|
| 5 |
+
"model.language_model.layers.0.self_attn.q_proj.weight": "model-language-0001-others-save_rank0.safetensors",
|
| 6 |
+
"model.language_model.layers.0.self_attn.q_proj.bias": "model-language-0001-others-save_rank0.safetensors",
|
| 7 |
+
"model.language_model.layers.0.self_attn.kv_a_proj_with_mqa.weight": "model-language-0001-others-save_rank0.safetensors",
|
| 8 |
+
"model.language_model.layers.0.self_attn.kv_a_proj_with_mqa.bias": "model-language-0001-others-save_rank0.safetensors",
|
| 9 |
+
"model.language_model.layers.0.self_attn.kv_b_proj.weight": "model-language-0001-others-save_rank0.safetensors",
|
| 10 |
+
"model.language_model.layers.0.self_attn.o_proj.weight": "model-language-0001-others-save_rank0.safetensors",
|
| 11 |
+
"model.language_model.layers.0.mlp.gate_proj.weight": "model-language-0001-others-save_rank0.safetensors",
|
| 12 |
+
"model.language_model.layers.0.mlp.up_proj.weight": "model-language-0001-others-save_rank0.safetensors",
|
| 13 |
+
"model.language_model.layers.0.mlp.down_proj.weight": "model-language-0001-others-save_rank0.safetensors",
|
| 14 |
+
"model.language_model.layers.0.input_layernorm.weight": "model-language-0001-others-save_rank0.safetensors",
|
| 15 |
+
"model.language_model.layers.0.post_attention_layernorm.weight": "model-language-0001-others-save_rank0.safetensors",
|
| 16 |
+
"model.language_model.layers.1.self_attn.q_proj.weight": "model-language-0001-others-save_rank0.safetensors",
|
| 17 |
+
"model.language_model.layers.1.self_attn.q_proj.bias": "model-language-0001-others-save_rank0.safetensors",
|
| 18 |
+
"model.language_model.layers.1.self_attn.kv_a_proj_with_mqa.weight": "model-language-0001-others-save_rank0.safetensors",
|
| 19 |
+
"model.language_model.layers.1.self_attn.kv_a_proj_with_mqa.bias": "model-language-0001-others-save_rank0.safetensors",
|
| 20 |
+
"model.language_model.layers.1.self_attn.kv_b_proj.weight": "model-language-0001-others-save_rank0.safetensors",
|
| 21 |
+
"model.language_model.layers.1.self_attn.o_proj.weight": "model-language-0001-others-save_rank0.safetensors",
|
| 22 |
+
"model.language_model.layers.1.mlp.gate_proj.weight": "model-language-0001-others-save_rank0.safetensors",
|
| 23 |
+
"model.language_model.layers.1.mlp.up_proj.weight": "model-language-0001-others-save_rank0.safetensors",
|
| 24 |
+
"model.language_model.layers.1.mlp.down_proj.weight": "model-language-0001-others-save_rank0.safetensors",
|
| 25 |
+
"model.language_model.layers.1.input_layernorm.weight": "model-language-0001-others-save_rank0.safetensors",
|
| 26 |
+
"model.language_model.layers.1.post_attention_layernorm.weight": "model-language-0001-others-save_rank0.safetensors",
|
| 27 |
+
"model.language_model.layers.2.self_attn.q_proj.weight": "model-language-0001-others-save_rank0.safetensors",
|
| 28 |
+
"model.language_model.layers.2.self_attn.q_proj.bias": "model-language-0001-others-save_rank0.safetensors",
|
| 29 |
+
"model.language_model.layers.2.self_attn.kv_a_proj_with_mqa.weight": "model-language-0001-others-save_rank0.safetensors",
|
| 30 |
+
"model.language_model.layers.2.self_attn.kv_a_proj_with_mqa.bias": "model-language-0001-others-save_rank0.safetensors",
|
| 31 |
+
"model.language_model.layers.2.self_attn.kv_b_proj.weight": "model-language-0001-others-save_rank0.safetensors",
|
| 32 |
+
"model.language_model.layers.2.self_attn.o_proj.weight": "model-language-0001-others-save_rank0.safetensors",
|
| 33 |
+
"model.language_model.layers.2.mlp.gate_proj.weight": "model-language-0001-others-save_rank0.safetensors",
|
| 34 |
+
"model.language_model.layers.2.mlp.up_proj.weight": "model-language-0001-others-save_rank0.safetensors",
|
| 35 |
+
"model.language_model.layers.2.mlp.down_proj.weight": "model-language-0001-others-save_rank0.safetensors",
|
| 36 |
+
"model.language_model.layers.2.input_layernorm.weight": "model-language-0001-others-save_rank0.safetensors",
|
| 37 |
+
"model.language_model.layers.2.post_attention_layernorm.weight": "model-language-0001-others-save_rank0.safetensors",
|
| 38 |
+
"model.language_model.layers.3.self_attn.q_proj.weight": "model-language-0001-others-save_rank0.safetensors",
|
| 39 |
+
"model.language_model.layers.3.self_attn.q_proj.bias": "model-language-0001-others-save_rank0.safetensors",
|
| 40 |
+
"model.language_model.layers.3.self_attn.kv_a_proj_with_mqa.weight": "model-language-0001-others-save_rank0.safetensors",
|
| 41 |
+
"model.language_model.layers.3.self_attn.kv_a_proj_with_mqa.bias": "model-language-0001-others-save_rank0.safetensors",
|
| 42 |
+
"model.language_model.layers.3.self_attn.kv_b_proj.weight": "model-language-0001-others-save_rank0.safetensors",
|
| 43 |
+
"model.language_model.layers.3.self_attn.o_proj.weight": "model-language-0001-others-save_rank0.safetensors",
|
| 44 |
+
"model.language_model.layers.3.mlp.gate_proj.weight": "model-language-0001-others-save_rank0.safetensors",
|
| 45 |
+
"model.language_model.layers.3.mlp.up_proj.weight": "model-language-0001-others-save_rank0.safetensors",
|
| 46 |
+
"model.language_model.layers.3.mlp.down_proj.weight": "model-language-0001-others-save_rank0.safetensors",
|
| 47 |
+
"model.language_model.layers.3.input_layernorm.weight": "model-language-0001-others-save_rank0.safetensors",
|
| 48 |
+
"model.language_model.layers.3.post_attention_layernorm.weight": "model-language-0001-others-save_rank0.safetensors",
|
| 49 |
+
"model.language_model.layers.4.self_attn.q_proj.weight": "model-language-0001-others-save_rank0.safetensors",
|
| 50 |
+
"model.language_model.layers.4.self_attn.q_proj.bias": "model-language-0001-others-save_rank0.safetensors",
|
| 51 |
+
"model.language_model.layers.4.self_attn.kv_a_proj_with_mqa.weight": "model-language-0001-others-save_rank0.safetensors",
|
| 52 |
+
"model.language_model.layers.4.self_attn.kv_a_proj_with_mqa.bias": "model-language-0001-others-save_rank0.safetensors",
|
| 53 |
+
"model.language_model.layers.4.self_attn.kv_b_proj.weight": "model-language-0001-others-save_rank0.safetensors",
|
| 54 |
+
"model.language_model.layers.4.self_attn.o_proj.weight": "model-language-0001-others-save_rank0.safetensors",
|
| 55 |
+
"model.language_model.layers.4.mlp.gate_proj.weight": "model-language-0001-others-save_rank0.safetensors",
|
| 56 |
+
"model.language_model.layers.4.mlp.up_proj.weight": "model-language-0001-others-save_rank0.safetensors",
|
| 57 |
+
"model.language_model.layers.4.mlp.down_proj.weight": "model-language-0001-others-save_rank0.safetensors",
|
| 58 |
+
"model.language_model.layers.4.input_layernorm.weight": "model-language-0001-others-save_rank0.safetensors",
|
| 59 |
+
"model.language_model.layers.4.post_attention_layernorm.weight": "model-language-0001-others-save_rank0.safetensors",
|
| 60 |
+
"model.language_model.layers.5.self_attn.q_proj.weight": "model-language-0001-others-save_rank0.safetensors",
|
| 61 |
+
"model.language_model.layers.5.self_attn.q_proj.bias": "model-language-0001-others-save_rank0.safetensors",
|
| 62 |
+
"model.language_model.layers.5.self_attn.kv_a_proj_with_mqa.weight": "model-language-0001-others-save_rank0.safetensors",
|
| 63 |
+
"model.language_model.layers.5.self_attn.kv_a_proj_with_mqa.bias": "model-language-0001-others-save_rank0.safetensors",
|
| 64 |
+
"model.language_model.layers.5.self_attn.kv_b_proj.weight": "model-language-0001-others-save_rank0.safetensors",
|
| 65 |
+
"model.language_model.layers.5.self_attn.o_proj.weight": "model-language-0001-others-save_rank0.safetensors",
|
| 66 |
+
"model.language_model.layers.5.mlp.gate_proj.weight": "model-language-0001-others-save_rank0.safetensors",
|
| 67 |
+
"model.language_model.layers.5.mlp.up_proj.weight": "model-language-0001-others-save_rank0.safetensors",
|
| 68 |
+
"model.language_model.layers.5.mlp.down_proj.weight": "model-language-0001-others-save_rank0.safetensors",
|
| 69 |
+
"model.language_model.layers.5.input_layernorm.weight": "model-language-0001-others-save_rank0.safetensors",
|
| 70 |
+
"model.language_model.layers.5.post_attention_layernorm.weight": "model-language-0001-others-save_rank0.safetensors",
|
| 71 |
+
"model.language_model.layers.6.self_attn.q_proj.weight": "model-language-0001-others-save_rank0.safetensors",
|
| 72 |
+
"model.language_model.layers.6.self_attn.q_proj.bias": "model-language-0001-others-save_rank0.safetensors",
|
| 73 |
+
"model.language_model.layers.6.self_attn.kv_a_proj_with_mqa.weight": "model-language-0001-others-save_rank0.safetensors",
|
| 74 |
+
"model.language_model.layers.6.self_attn.kv_a_proj_with_mqa.bias": "model-language-0001-others-save_rank0.safetensors",
|
| 75 |
+
"model.language_model.layers.6.self_attn.kv_b_proj.weight": "model-language-0001-others-save_rank0.safetensors",
|
| 76 |
+
"model.language_model.layers.6.self_attn.o_proj.weight": "model-language-0001-others-save_rank0.safetensors",
|
| 77 |
+
"model.language_model.layers.6.mlp.gate_proj.weight": "model-language-0001-others-save_rank0.safetensors",
|
| 78 |
+
"model.language_model.layers.6.mlp.up_proj.weight": "model-language-0001-others-save_rank0.safetensors",
|
| 79 |
+
"model.language_model.layers.6.mlp.down_proj.weight": "model-language-0001-others-save_rank0.safetensors",
|
| 80 |
+
"model.language_model.layers.6.input_layernorm.weight": "model-language-0001-others-save_rank0.safetensors",
|
| 81 |
+
"model.language_model.layers.6.post_attention_layernorm.weight": "model-language-0001-others-save_rank0.safetensors",
|
| 82 |
+
"model.language_model.layers.7.self_attn.q_proj.weight": "model-language-0001-others-save_rank0.safetensors",
|
| 83 |
+
"model.language_model.layers.7.self_attn.q_proj.bias": "model-language-0001-others-save_rank0.safetensors",
|
| 84 |
+
"model.language_model.layers.7.self_attn.kv_a_proj_with_mqa.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 85 |
+
"model.language_model.layers.7.self_attn.kv_a_proj_with_mqa.bias": "model-language-0002-others-save_rank0.safetensors",
|
| 86 |
+
"model.language_model.layers.7.self_attn.kv_b_proj.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 87 |
+
"model.language_model.layers.7.self_attn.o_proj.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 88 |
+
"model.language_model.layers.7.mlp.gate_proj.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 89 |
+
"model.language_model.layers.7.mlp.up_proj.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 90 |
+
"model.language_model.layers.7.mlp.down_proj.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 91 |
+
"model.language_model.layers.7.input_layernorm.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 92 |
+
"model.language_model.layers.7.post_attention_layernorm.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 93 |
+
"model.language_model.layers.8.self_attn.q_proj.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 94 |
+
"model.language_model.layers.8.self_attn.q_proj.bias": "model-language-0002-others-save_rank0.safetensors",
|
| 95 |
+
"model.language_model.layers.8.self_attn.kv_a_proj_with_mqa.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 96 |
+
"model.language_model.layers.8.self_attn.kv_a_proj_with_mqa.bias": "model-language-0002-others-save_rank0.safetensors",
|
| 97 |
+
"model.language_model.layers.8.self_attn.kv_b_proj.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 98 |
+
"model.language_model.layers.8.self_attn.o_proj.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 99 |
+
"model.language_model.layers.8.mlp.gate_proj.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 100 |
+
"model.language_model.layers.8.mlp.up_proj.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 101 |
+
"model.language_model.layers.8.mlp.down_proj.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 102 |
+
"model.language_model.layers.8.input_layernorm.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 103 |
+
"model.language_model.layers.8.post_attention_layernorm.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 104 |
+
"model.language_model.layers.9.self_attn.q_proj.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 105 |
+
"model.language_model.layers.9.self_attn.q_proj.bias": "model-language-0002-others-save_rank0.safetensors",
|
| 106 |
+
"model.language_model.layers.9.self_attn.kv_a_proj_with_mqa.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 107 |
+
"model.language_model.layers.9.self_attn.kv_a_proj_with_mqa.bias": "model-language-0002-others-save_rank0.safetensors",
|
| 108 |
+
"model.language_model.layers.9.self_attn.kv_b_proj.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 109 |
+
"model.language_model.layers.9.self_attn.o_proj.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 110 |
+
"model.language_model.layers.9.mlp.gate_proj.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 111 |
+
"model.language_model.layers.9.mlp.up_proj.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 112 |
+
"model.language_model.layers.9.mlp.down_proj.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 113 |
+
"model.language_model.layers.9.input_layernorm.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 114 |
+
"model.language_model.layers.9.post_attention_layernorm.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 115 |
+
"model.language_model.layers.10.self_attn.q_proj.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 116 |
+
"model.language_model.layers.10.self_attn.q_proj.bias": "model-language-0002-others-save_rank0.safetensors",
|
| 117 |
+
"model.language_model.layers.10.self_attn.kv_a_proj_with_mqa.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 118 |
+
"model.language_model.layers.10.self_attn.kv_a_proj_with_mqa.bias": "model-language-0002-others-save_rank0.safetensors",
|
| 119 |
+
"model.language_model.layers.10.self_attn.kv_b_proj.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 120 |
+
"model.language_model.layers.10.self_attn.o_proj.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 121 |
+
"model.language_model.layers.10.mlp.gate_proj.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 122 |
+
"model.language_model.layers.10.mlp.up_proj.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 123 |
+
"model.language_model.layers.10.mlp.down_proj.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 124 |
+
"model.language_model.layers.10.input_layernorm.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 125 |
+
"model.language_model.layers.10.post_attention_layernorm.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 126 |
+
"model.language_model.layers.11.self_attn.q_proj.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 127 |
+
"model.language_model.layers.11.self_attn.q_proj.bias": "model-language-0002-others-save_rank0.safetensors",
|
| 128 |
+
"model.language_model.layers.11.self_attn.kv_a_proj_with_mqa.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 129 |
+
"model.language_model.layers.11.self_attn.kv_a_proj_with_mqa.bias": "model-language-0002-others-save_rank0.safetensors",
|
| 130 |
+
"model.language_model.layers.11.self_attn.kv_b_proj.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 131 |
+
"model.language_model.layers.11.self_attn.o_proj.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 132 |
+
"model.language_model.layers.11.mlp.gate_proj.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 133 |
+
"model.language_model.layers.11.mlp.up_proj.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 134 |
+
"model.language_model.layers.11.mlp.down_proj.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 135 |
+
"model.language_model.layers.11.input_layernorm.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 136 |
+
"model.language_model.layers.11.post_attention_layernorm.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 137 |
+
"model.language_model.layers.12.self_attn.q_proj.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 138 |
+
"model.language_model.layers.12.self_attn.q_proj.bias": "model-language-0002-others-save_rank0.safetensors",
|
| 139 |
+
"model.language_model.layers.12.self_attn.kv_a_proj_with_mqa.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 140 |
+
"model.language_model.layers.12.self_attn.kv_a_proj_with_mqa.bias": "model-language-0002-others-save_rank0.safetensors",
|
| 141 |
+
"model.language_model.layers.12.self_attn.kv_b_proj.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 142 |
+
"model.language_model.layers.12.self_attn.o_proj.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 143 |
+
"model.language_model.layers.12.mlp.gate_proj.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 144 |
+
"model.language_model.layers.12.mlp.up_proj.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 145 |
+
"model.language_model.layers.12.mlp.down_proj.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 146 |
+
"model.language_model.layers.12.input_layernorm.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 147 |
+
"model.language_model.layers.12.post_attention_layernorm.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 148 |
+
"model.language_model.layers.13.self_attn.q_proj.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 149 |
+
"model.language_model.layers.13.self_attn.q_proj.bias": "model-language-0002-others-save_rank0.safetensors",
|
| 150 |
+
"model.language_model.layers.13.self_attn.kv_a_proj_with_mqa.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 151 |
+
"model.language_model.layers.13.self_attn.kv_a_proj_with_mqa.bias": "model-language-0002-others-save_rank0.safetensors",
|
| 152 |
+
"model.language_model.layers.13.self_attn.kv_b_proj.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 153 |
+
"model.language_model.layers.13.self_attn.o_proj.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 154 |
+
"model.language_model.layers.13.mlp.gate_proj.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 155 |
+
"model.language_model.layers.13.mlp.up_proj.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 156 |
+
"model.language_model.layers.13.mlp.down_proj.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 157 |
+
"model.language_model.layers.13.input_layernorm.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 158 |
+
"model.language_model.layers.13.post_attention_layernorm.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 159 |
+
"model.language_model.layers.14.self_attn.q_proj.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 160 |
+
"model.language_model.layers.14.self_attn.q_proj.bias": "model-language-0002-others-save_rank0.safetensors",
|
| 161 |
+
"model.language_model.layers.14.self_attn.kv_a_proj_with_mqa.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 162 |
+
"model.language_model.layers.14.self_attn.kv_a_proj_with_mqa.bias": "model-language-0002-others-save_rank0.safetensors",
|
| 163 |
+
"model.language_model.layers.14.self_attn.kv_b_proj.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 164 |
+
"model.language_model.layers.14.self_attn.o_proj.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 165 |
+
"model.language_model.layers.14.mlp.gate_proj.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 166 |
+
"model.language_model.layers.14.mlp.up_proj.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 167 |
+
"model.language_model.layers.14.mlp.down_proj.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 168 |
+
"model.language_model.layers.14.input_layernorm.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 169 |
+
"model.language_model.layers.14.post_attention_layernorm.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 170 |
+
"model.language_model.layers.15.self_attn.q_proj.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 171 |
+
"model.language_model.layers.15.self_attn.q_proj.bias": "model-language-0002-others-save_rank0.safetensors",
|
| 172 |
+
"model.language_model.layers.15.self_attn.kv_a_proj_with_mqa.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 173 |
+
"model.language_model.layers.15.self_attn.kv_a_proj_with_mqa.bias": "model-language-0002-others-save_rank0.safetensors",
|
| 174 |
+
"model.language_model.layers.15.self_attn.kv_b_proj.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 175 |
+
"model.language_model.layers.15.self_attn.o_proj.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 176 |
+
"model.language_model.layers.15.mlp.gate_proj.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 177 |
+
"model.language_model.layers.15.mlp.up_proj.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 178 |
+
"model.language_model.layers.15.mlp.down_proj.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 179 |
+
"model.language_model.layers.15.input_layernorm.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 180 |
+
"model.language_model.layers.15.post_attention_layernorm.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 181 |
+
"model.language_model.layers.16.self_attn.q_proj.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 182 |
+
"model.language_model.layers.16.self_attn.q_proj.bias": "model-language-0002-others-save_rank0.safetensors",
|
| 183 |
+
"model.language_model.layers.16.self_attn.kv_a_proj_with_mqa.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 184 |
+
"model.language_model.layers.16.self_attn.kv_a_proj_with_mqa.bias": "model-language-0002-others-save_rank0.safetensors",
|
| 185 |
+
"model.language_model.layers.16.self_attn.kv_b_proj.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 186 |
+
"model.language_model.layers.16.self_attn.o_proj.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 187 |
+
"model.language_model.layers.16.mlp.gate_proj.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 188 |
+
"model.language_model.layers.16.mlp.up_proj.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 189 |
+
"model.language_model.layers.16.mlp.down_proj.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 190 |
+
"model.language_model.layers.16.input_layernorm.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 191 |
+
"model.language_model.layers.16.post_attention_layernorm.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 192 |
+
"model.language_model.layers.17.self_attn.q_proj.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 193 |
+
"model.language_model.layers.17.self_attn.q_proj.bias": "model-language-0002-others-save_rank0.safetensors",
|
| 194 |
+
"model.language_model.layers.17.self_attn.kv_a_proj_with_mqa.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 195 |
+
"model.language_model.layers.17.self_attn.kv_a_proj_with_mqa.bias": "model-language-0002-others-save_rank0.safetensors",
|
| 196 |
+
"model.language_model.layers.17.self_attn.kv_b_proj.weight": "model-language-0002-others-save_rank0.safetensors",
|
| 197 |
+
"model.language_model.layers.17.self_attn.o_proj.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 198 |
+
"model.language_model.layers.17.mlp.gate_proj.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 199 |
+
"model.language_model.layers.17.mlp.up_proj.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 200 |
+
"model.language_model.layers.17.mlp.down_proj.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 201 |
+
"model.language_model.layers.17.input_layernorm.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 202 |
+
"model.language_model.layers.17.post_attention_layernorm.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 203 |
+
"model.language_model.layers.18.self_attn.q_proj.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 204 |
+
"model.language_model.layers.18.self_attn.q_proj.bias": "model-language-0003-others-save_rank0.safetensors",
|
| 205 |
+
"model.language_model.layers.18.self_attn.kv_a_proj_with_mqa.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 206 |
+
"model.language_model.layers.18.self_attn.kv_a_proj_with_mqa.bias": "model-language-0003-others-save_rank0.safetensors",
|
| 207 |
+
"model.language_model.layers.18.self_attn.kv_b_proj.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 208 |
+
"model.language_model.layers.18.self_attn.o_proj.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 209 |
+
"model.language_model.layers.18.mlp.gate_proj.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 210 |
+
"model.language_model.layers.18.mlp.up_proj.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 211 |
+
"model.language_model.layers.18.mlp.down_proj.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 212 |
+
"model.language_model.layers.18.input_layernorm.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 213 |
+
"model.language_model.layers.18.post_attention_layernorm.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 214 |
+
"model.language_model.layers.19.self_attn.q_proj.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 215 |
+
"model.language_model.layers.19.self_attn.q_proj.bias": "model-language-0003-others-save_rank0.safetensors",
|
| 216 |
+
"model.language_model.layers.19.self_attn.kv_a_proj_with_mqa.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 217 |
+
"model.language_model.layers.19.self_attn.kv_a_proj_with_mqa.bias": "model-language-0003-others-save_rank0.safetensors",
|
| 218 |
+
"model.language_model.layers.19.self_attn.kv_b_proj.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 219 |
+
"model.language_model.layers.19.self_attn.o_proj.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 220 |
+
"model.language_model.layers.19.mlp.gate_proj.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 221 |
+
"model.language_model.layers.19.mlp.up_proj.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 222 |
+
"model.language_model.layers.19.mlp.down_proj.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 223 |
+
"model.language_model.layers.19.input_layernorm.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 224 |
+
"model.language_model.layers.19.post_attention_layernorm.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 225 |
+
"model.language_model.layers.20.self_attn.q_proj.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 226 |
+
"model.language_model.layers.20.self_attn.q_proj.bias": "model-language-0003-others-save_rank0.safetensors",
|
| 227 |
+
"model.language_model.layers.20.self_attn.kv_a_proj_with_mqa.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 228 |
+
"model.language_model.layers.20.self_attn.kv_a_proj_with_mqa.bias": "model-language-0003-others-save_rank0.safetensors",
|
| 229 |
+
"model.language_model.layers.20.self_attn.kv_b_proj.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 230 |
+
"model.language_model.layers.20.self_attn.o_proj.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 231 |
+
"model.language_model.layers.20.mlp.gate_proj.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 232 |
+
"model.language_model.layers.20.mlp.up_proj.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 233 |
+
"model.language_model.layers.20.mlp.down_proj.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 234 |
+
"model.language_model.layers.20.input_layernorm.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 235 |
+
"model.language_model.layers.20.post_attention_layernorm.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 236 |
+
"model.language_model.layers.21.self_attn.q_proj.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 237 |
+
"model.language_model.layers.21.self_attn.q_proj.bias": "model-language-0003-others-save_rank0.safetensors",
|
| 238 |
+
"model.language_model.layers.21.self_attn.kv_a_proj_with_mqa.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 239 |
+
"model.language_model.layers.21.self_attn.kv_a_proj_with_mqa.bias": "model-language-0003-others-save_rank0.safetensors",
|
| 240 |
+
"model.language_model.layers.21.self_attn.kv_b_proj.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 241 |
+
"model.language_model.layers.21.self_attn.o_proj.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 242 |
+
"model.language_model.layers.21.mlp.gate_proj.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 243 |
+
"model.language_model.layers.21.mlp.up_proj.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 244 |
+
"model.language_model.layers.21.mlp.down_proj.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 245 |
+
"model.language_model.layers.21.input_layernorm.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 246 |
+
"model.language_model.layers.21.post_attention_layernorm.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 247 |
+
"model.language_model.layers.22.self_attn.q_proj.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 248 |
+
"model.language_model.layers.22.self_attn.q_proj.bias": "model-language-0003-others-save_rank0.safetensors",
|
| 249 |
+
"model.language_model.layers.22.self_attn.kv_a_proj_with_mqa.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 250 |
+
"model.language_model.layers.22.self_attn.kv_a_proj_with_mqa.bias": "model-language-0003-others-save_rank0.safetensors",
|
| 251 |
+
"model.language_model.layers.22.self_attn.kv_b_proj.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 252 |
+
"model.language_model.layers.22.self_attn.o_proj.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 253 |
+
"model.language_model.layers.22.mlp.gate_proj.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 254 |
+
"model.language_model.layers.22.mlp.up_proj.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 255 |
+
"model.language_model.layers.22.mlp.down_proj.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 256 |
+
"model.language_model.layers.22.input_layernorm.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 257 |
+
"model.language_model.layers.22.post_attention_layernorm.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 258 |
+
"model.language_model.layers.23.self_attn.q_proj.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 259 |
+
"model.language_model.layers.23.self_attn.q_proj.bias": "model-language-0003-others-save_rank0.safetensors",
|
| 260 |
+
"model.language_model.layers.23.self_attn.kv_a_proj_with_mqa.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 261 |
+
"model.language_model.layers.23.self_attn.kv_a_proj_with_mqa.bias": "model-language-0003-others-save_rank0.safetensors",
|
| 262 |
+
"model.language_model.layers.23.self_attn.kv_b_proj.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 263 |
+
"model.language_model.layers.23.self_attn.o_proj.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 264 |
+
"model.language_model.layers.23.mlp.gate_proj.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 265 |
+
"model.language_model.layers.23.mlp.up_proj.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 266 |
+
"model.language_model.layers.23.mlp.down_proj.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 267 |
+
"model.language_model.layers.23.input_layernorm.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 268 |
+
"model.language_model.layers.23.post_attention_layernorm.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 269 |
+
"model.language_model.layers.24.self_attn.q_proj.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 270 |
+
"model.language_model.layers.24.self_attn.q_proj.bias": "model-language-0003-others-save_rank0.safetensors",
|
| 271 |
+
"model.language_model.layers.24.self_attn.kv_a_proj_with_mqa.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 272 |
+
"model.language_model.layers.24.self_attn.kv_a_proj_with_mqa.bias": "model-language-0003-others-save_rank0.safetensors",
|
| 273 |
+
"model.language_model.layers.24.self_attn.kv_b_proj.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 274 |
+
"model.language_model.layers.24.self_attn.o_proj.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 275 |
+
"model.language_model.layers.24.mlp.gate_proj.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 276 |
+
"model.language_model.layers.24.mlp.up_proj.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 277 |
+
"model.language_model.layers.24.mlp.down_proj.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 278 |
+
"model.language_model.layers.24.input_layernorm.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 279 |
+
"model.language_model.layers.24.post_attention_layernorm.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 280 |
+
"model.language_model.layers.25.self_attn.q_proj.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 281 |
+
"model.language_model.layers.25.self_attn.q_proj.bias": "model-language-0003-others-save_rank0.safetensors",
|
| 282 |
+
"model.language_model.layers.25.self_attn.kv_a_proj_with_mqa.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 283 |
+
"model.language_model.layers.25.self_attn.kv_a_proj_with_mqa.bias": "model-language-0003-others-save_rank0.safetensors",
|
| 284 |
+
"model.language_model.layers.25.self_attn.kv_b_proj.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 285 |
+
"model.language_model.layers.25.self_attn.o_proj.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 286 |
+
"model.language_model.layers.25.mlp.gate_proj.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 287 |
+
"model.language_model.layers.25.mlp.up_proj.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 288 |
+
"model.language_model.layers.25.mlp.down_proj.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 289 |
+
"model.language_model.layers.25.input_layernorm.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 290 |
+
"model.language_model.layers.25.post_attention_layernorm.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 291 |
+
"model.language_model.layers.26.self_attn.q_proj.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 292 |
+
"model.language_model.layers.26.self_attn.q_proj.bias": "model-language-0003-others-save_rank0.safetensors",
|
| 293 |
+
"model.language_model.layers.26.self_attn.kv_a_proj_with_mqa.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 294 |
+
"model.language_model.layers.26.self_attn.kv_a_proj_with_mqa.bias": "model-language-0003-others-save_rank0.safetensors",
|
| 295 |
+
"model.language_model.layers.26.self_attn.kv_b_proj.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 296 |
+
"model.language_model.layers.26.self_attn.o_proj.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 297 |
+
"model.language_model.layers.26.mlp.gate_proj.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 298 |
+
"model.language_model.layers.26.mlp.up_proj.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 299 |
+
"model.language_model.layers.26.mlp.down_proj.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 300 |
+
"model.language_model.layers.26.input_layernorm.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 301 |
+
"model.language_model.layers.26.post_attention_layernorm.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 302 |
+
"model.language_model.layers.27.self_attn.q_proj.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 303 |
+
"model.language_model.layers.27.self_attn.q_proj.bias": "model-language-0003-others-save_rank0.safetensors",
|
| 304 |
+
"model.language_model.layers.27.self_attn.kv_a_proj_with_mqa.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 305 |
+
"model.language_model.layers.27.self_attn.kv_a_proj_with_mqa.bias": "model-language-0003-others-save_rank0.safetensors",
|
| 306 |
+
"model.language_model.layers.27.self_attn.kv_b_proj.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 307 |
+
"model.language_model.layers.27.self_attn.o_proj.weight": "model-language-0003-others-save_rank0.safetensors",
|
| 308 |
+
"model.language_model.layers.27.mlp.gate_proj.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 309 |
+
"model.language_model.layers.27.mlp.up_proj.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 310 |
+
"model.language_model.layers.27.mlp.down_proj.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 311 |
+
"model.language_model.layers.27.input_layernorm.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 312 |
+
"model.language_model.layers.27.post_attention_layernorm.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 313 |
+
"model.language_model.layers.28.self_attn.q_proj.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 314 |
+
"model.language_model.layers.28.self_attn.q_proj.bias": "model-language-0004-others-save_rank0.safetensors",
|
| 315 |
+
"model.language_model.layers.28.self_attn.kv_a_proj_with_mqa.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 316 |
+
"model.language_model.layers.28.self_attn.kv_a_proj_with_mqa.bias": "model-language-0004-others-save_rank0.safetensors",
|
| 317 |
+
"model.language_model.layers.28.self_attn.kv_b_proj.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 318 |
+
"model.language_model.layers.28.self_attn.o_proj.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 319 |
+
"model.language_model.layers.28.mlp.gate_proj.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 320 |
+
"model.language_model.layers.28.mlp.up_proj.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 321 |
+
"model.language_model.layers.28.mlp.down_proj.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 322 |
+
"model.language_model.layers.28.input_layernorm.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 323 |
+
"model.language_model.layers.28.post_attention_layernorm.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 324 |
+
"model.language_model.layers.29.self_attn.q_proj.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 325 |
+
"model.language_model.layers.29.self_attn.q_proj.bias": "model-language-0004-others-save_rank0.safetensors",
|
| 326 |
+
"model.language_model.layers.29.self_attn.kv_a_proj_with_mqa.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 327 |
+
"model.language_model.layers.29.self_attn.kv_a_proj_with_mqa.bias": "model-language-0004-others-save_rank0.safetensors",
|
| 328 |
+
"model.language_model.layers.29.self_attn.kv_b_proj.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 329 |
+
"model.language_model.layers.29.self_attn.o_proj.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 330 |
+
"model.language_model.layers.29.mlp.gate_proj.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 331 |
+
"model.language_model.layers.29.mlp.up_proj.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 332 |
+
"model.language_model.layers.29.mlp.down_proj.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 333 |
+
"model.language_model.layers.29.input_layernorm.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 334 |
+
"model.language_model.layers.29.post_attention_layernorm.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 335 |
+
"model.language_model.layers.30.self_attn.q_proj.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 336 |
+
"model.language_model.layers.30.self_attn.q_proj.bias": "model-language-0004-others-save_rank0.safetensors",
|
| 337 |
+
"model.language_model.layers.30.self_attn.kv_a_proj_with_mqa.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 338 |
+
"model.language_model.layers.30.self_attn.kv_a_proj_with_mqa.bias": "model-language-0004-others-save_rank0.safetensors",
|
| 339 |
+
"model.language_model.layers.30.self_attn.kv_b_proj.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 340 |
+
"model.language_model.layers.30.self_attn.o_proj.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 341 |
+
"model.language_model.layers.30.mlp.gate_proj.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 342 |
+
"model.language_model.layers.30.mlp.up_proj.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 343 |
+
"model.language_model.layers.30.mlp.down_proj.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 344 |
+
"model.language_model.layers.30.input_layernorm.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 345 |
+
"model.language_model.layers.30.post_attention_layernorm.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 346 |
+
"model.language_model.layers.31.self_attn.q_proj.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 347 |
+
"model.language_model.layers.31.self_attn.q_proj.bias": "model-language-0004-others-save_rank0.safetensors",
|
| 348 |
+
"model.language_model.layers.31.self_attn.kv_a_proj_with_mqa.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 349 |
+
"model.language_model.layers.31.self_attn.kv_a_proj_with_mqa.bias": "model-language-0004-others-save_rank0.safetensors",
|
| 350 |
+
"model.language_model.layers.31.self_attn.kv_b_proj.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 351 |
+
"model.language_model.layers.31.self_attn.o_proj.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 352 |
+
"model.language_model.layers.31.mlp.gate_proj.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 353 |
+
"model.language_model.layers.31.mlp.up_proj.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 354 |
+
"model.language_model.layers.31.mlp.down_proj.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 355 |
+
"model.language_model.layers.31.input_layernorm.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 356 |
+
"model.language_model.layers.31.post_attention_layernorm.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 357 |
+
"model.language_model.layers.32.self_attn.q_proj.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 358 |
+
"model.language_model.layers.32.self_attn.q_proj.bias": "model-language-0004-others-save_rank0.safetensors",
|
| 359 |
+
"model.language_model.layers.32.self_attn.kv_a_proj_with_mqa.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 360 |
+
"model.language_model.layers.32.self_attn.kv_a_proj_with_mqa.bias": "model-language-0004-others-save_rank0.safetensors",
|
| 361 |
+
"model.language_model.layers.32.self_attn.kv_b_proj.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 362 |
+
"model.language_model.layers.32.self_attn.o_proj.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 363 |
+
"model.language_model.layers.32.mlp.gate_proj.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 364 |
+
"model.language_model.layers.32.mlp.up_proj.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 365 |
+
"model.language_model.layers.32.mlp.down_proj.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 366 |
+
"model.language_model.layers.32.input_layernorm.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 367 |
+
"model.language_model.layers.32.post_attention_layernorm.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 368 |
+
"model.language_model.layers.33.self_attn.q_proj.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 369 |
+
"model.language_model.layers.33.self_attn.q_proj.bias": "model-language-0004-others-save_rank0.safetensors",
|
| 370 |
+
"model.language_model.layers.33.self_attn.kv_a_proj_with_mqa.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 371 |
+
"model.language_model.layers.33.self_attn.kv_a_proj_with_mqa.bias": "model-language-0004-others-save_rank0.safetensors",
|
| 372 |
+
"model.language_model.layers.33.self_attn.kv_b_proj.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 373 |
+
"model.language_model.layers.33.self_attn.o_proj.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 374 |
+
"model.language_model.layers.33.mlp.gate_proj.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 375 |
+
"model.language_model.layers.33.mlp.up_proj.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 376 |
+
"model.language_model.layers.33.mlp.down_proj.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 377 |
+
"model.language_model.layers.33.input_layernorm.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 378 |
+
"model.language_model.layers.33.post_attention_layernorm.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 379 |
+
"model.language_model.layers.34.self_attn.q_proj.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 380 |
+
"model.language_model.layers.34.self_attn.q_proj.bias": "model-language-0004-others-save_rank0.safetensors",
|
| 381 |
+
"model.language_model.layers.34.self_attn.kv_a_proj_with_mqa.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 382 |
+
"model.language_model.layers.34.self_attn.kv_a_proj_with_mqa.bias": "model-language-0004-others-save_rank0.safetensors",
|
| 383 |
+
"model.language_model.layers.34.self_attn.kv_b_proj.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 384 |
+
"model.language_model.layers.34.self_attn.o_proj.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 385 |
+
"model.language_model.layers.34.mlp.gate_proj.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 386 |
+
"model.language_model.layers.34.mlp.up_proj.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 387 |
+
"model.language_model.layers.34.mlp.down_proj.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 388 |
+
"model.language_model.layers.34.input_layernorm.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 389 |
+
"model.language_model.layers.34.post_attention_layernorm.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 390 |
+
"model.language_model.layers.35.self_attn.q_proj.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 391 |
+
"model.language_model.layers.35.self_attn.q_proj.bias": "model-language-0004-others-save_rank0.safetensors",
|
| 392 |
+
"model.language_model.layers.35.self_attn.kv_a_proj_with_mqa.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 393 |
+
"model.language_model.layers.35.self_attn.kv_a_proj_with_mqa.bias": "model-language-0004-others-save_rank0.safetensors",
|
| 394 |
+
"model.language_model.layers.35.self_attn.kv_b_proj.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 395 |
+
"model.language_model.layers.35.self_attn.o_proj.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 396 |
+
"model.language_model.layers.35.mlp.gate_proj.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 397 |
+
"model.language_model.layers.35.mlp.up_proj.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 398 |
+
"model.language_model.layers.35.mlp.down_proj.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 399 |
+
"model.language_model.layers.35.input_layernorm.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 400 |
+
"model.language_model.layers.35.post_attention_layernorm.weight": "model-language-0004-others-save_rank0.safetensors",
|
| 401 |
+
"model.language_model.embed_tokens.weight": "model-language-0005-others-save_rank0.safetensors",
|
| 402 |
+
"model.visual.patch_embed.proj.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 403 |
+
"model.visual.patch_embed.proj.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 404 |
+
"model.visual.pos_embed.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 405 |
+
"model.visual.blocks.0.norm1.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 406 |
+
"model.visual.blocks.0.norm1.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 407 |
+
"model.visual.blocks.0.norm2.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 408 |
+
"model.visual.blocks.0.norm2.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 409 |
+
"model.visual.blocks.0.attn.qkv.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 410 |
+
"model.visual.blocks.0.attn.qkv.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 411 |
+
"model.visual.blocks.0.attn.proj.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 412 |
+
"model.visual.blocks.0.attn.proj.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 413 |
+
"model.visual.blocks.0.mlp.linear_fc1.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 414 |
+
"model.visual.blocks.0.mlp.linear_fc1.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 415 |
+
"model.visual.blocks.0.mlp.linear_fc2.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 416 |
+
"model.visual.blocks.0.mlp.linear_fc2.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 417 |
+
"model.visual.blocks.1.norm1.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 418 |
+
"model.visual.blocks.1.norm1.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 419 |
+
"model.visual.blocks.1.norm2.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 420 |
+
"model.visual.blocks.1.norm2.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 421 |
+
"model.visual.blocks.1.attn.qkv.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 422 |
+
"model.visual.blocks.1.attn.qkv.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 423 |
+
"model.visual.blocks.1.attn.proj.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 424 |
+
"model.visual.blocks.1.attn.proj.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 425 |
+
"model.visual.blocks.1.mlp.linear_fc1.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 426 |
+
"model.visual.blocks.1.mlp.linear_fc1.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 427 |
+
"model.visual.blocks.1.mlp.linear_fc2.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 428 |
+
"model.visual.blocks.1.mlp.linear_fc2.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 429 |
+
"model.visual.blocks.2.norm1.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 430 |
+
"model.visual.blocks.2.norm1.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 431 |
+
"model.visual.blocks.2.norm2.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 432 |
+
"model.visual.blocks.2.norm2.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 433 |
+
"model.visual.blocks.2.attn.qkv.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 434 |
+
"model.visual.blocks.2.attn.qkv.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 435 |
+
"model.visual.blocks.2.attn.proj.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 436 |
+
"model.visual.blocks.2.attn.proj.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 437 |
+
"model.visual.blocks.2.mlp.linear_fc1.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 438 |
+
"model.visual.blocks.2.mlp.linear_fc1.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 439 |
+
"model.visual.blocks.2.mlp.linear_fc2.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 440 |
+
"model.visual.blocks.2.mlp.linear_fc2.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 441 |
+
"model.visual.blocks.3.norm1.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 442 |
+
"model.visual.blocks.3.norm1.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 443 |
+
"model.visual.blocks.3.norm2.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 444 |
+
"model.visual.blocks.3.norm2.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 445 |
+
"model.visual.blocks.3.attn.qkv.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 446 |
+
"model.visual.blocks.3.attn.qkv.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 447 |
+
"model.visual.blocks.3.attn.proj.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 448 |
+
"model.visual.blocks.3.attn.proj.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 449 |
+
"model.visual.blocks.3.mlp.linear_fc1.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 450 |
+
"model.visual.blocks.3.mlp.linear_fc1.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 451 |
+
"model.visual.blocks.3.mlp.linear_fc2.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 452 |
+
"model.visual.blocks.3.mlp.linear_fc2.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 453 |
+
"model.visual.blocks.4.norm1.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 454 |
+
"model.visual.blocks.4.norm1.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 455 |
+
"model.visual.blocks.4.norm2.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 456 |
+
"model.visual.blocks.4.norm2.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 457 |
+
"model.visual.blocks.4.attn.qkv.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 458 |
+
"model.visual.blocks.4.attn.qkv.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 459 |
+
"model.visual.blocks.4.attn.proj.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 460 |
+
"model.visual.blocks.4.attn.proj.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 461 |
+
"model.visual.blocks.4.mlp.linear_fc1.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 462 |
+
"model.visual.blocks.4.mlp.linear_fc1.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 463 |
+
"model.visual.blocks.4.mlp.linear_fc2.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 464 |
+
"model.visual.blocks.4.mlp.linear_fc2.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 465 |
+
"model.visual.blocks.5.norm1.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 466 |
+
"model.visual.blocks.5.norm1.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 467 |
+
"model.visual.blocks.5.norm2.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 468 |
+
"model.visual.blocks.5.norm2.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 469 |
+
"model.visual.blocks.5.attn.qkv.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 470 |
+
"model.visual.blocks.5.attn.qkv.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 471 |
+
"model.visual.blocks.5.attn.proj.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 472 |
+
"model.visual.blocks.5.attn.proj.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 473 |
+
"model.visual.blocks.5.mlp.linear_fc1.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 474 |
+
"model.visual.blocks.5.mlp.linear_fc1.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 475 |
+
"model.visual.blocks.5.mlp.linear_fc2.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 476 |
+
"model.visual.blocks.5.mlp.linear_fc2.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 477 |
+
"model.visual.blocks.6.norm1.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 478 |
+
"model.visual.blocks.6.norm1.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 479 |
+
"model.visual.blocks.6.norm2.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 480 |
+
"model.visual.blocks.6.norm2.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 481 |
+
"model.visual.blocks.6.attn.qkv.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 482 |
+
"model.visual.blocks.6.attn.qkv.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 483 |
+
"model.visual.blocks.6.attn.proj.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 484 |
+
"model.visual.blocks.6.attn.proj.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 485 |
+
"model.visual.blocks.6.mlp.linear_fc1.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 486 |
+
"model.visual.blocks.6.mlp.linear_fc1.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 487 |
+
"model.visual.blocks.6.mlp.linear_fc2.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 488 |
+
"model.visual.blocks.6.mlp.linear_fc2.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 489 |
+
"model.visual.blocks.7.norm1.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 490 |
+
"model.visual.blocks.7.norm1.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 491 |
+
"model.visual.blocks.7.norm2.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 492 |
+
"model.visual.blocks.7.norm2.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 493 |
+
"model.visual.blocks.7.attn.qkv.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 494 |
+
"model.visual.blocks.7.attn.qkv.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 495 |
+
"model.visual.blocks.7.attn.proj.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 496 |
+
"model.visual.blocks.7.attn.proj.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 497 |
+
"model.visual.blocks.7.mlp.linear_fc1.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 498 |
+
"model.visual.blocks.7.mlp.linear_fc1.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 499 |
+
"model.visual.blocks.7.mlp.linear_fc2.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 500 |
+
"model.visual.blocks.7.mlp.linear_fc2.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 501 |
+
"model.visual.blocks.8.norm1.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 502 |
+
"model.visual.blocks.8.norm1.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 503 |
+
"model.visual.blocks.8.norm2.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 504 |
+
"model.visual.blocks.8.norm2.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 505 |
+
"model.visual.blocks.8.attn.qkv.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 506 |
+
"model.visual.blocks.8.attn.qkv.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 507 |
+
"model.visual.blocks.8.attn.proj.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 508 |
+
"model.visual.blocks.8.attn.proj.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 509 |
+
"model.visual.blocks.8.mlp.linear_fc1.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 510 |
+
"model.visual.blocks.8.mlp.linear_fc1.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 511 |
+
"model.visual.blocks.8.mlp.linear_fc2.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 512 |
+
"model.visual.blocks.8.mlp.linear_fc2.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 513 |
+
"model.visual.blocks.9.norm1.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 514 |
+
"model.visual.blocks.9.norm1.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 515 |
+
"model.visual.blocks.9.norm2.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 516 |
+
"model.visual.blocks.9.norm2.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 517 |
+
"model.visual.blocks.9.attn.qkv.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 518 |
+
"model.visual.blocks.9.attn.qkv.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 519 |
+
"model.visual.blocks.9.attn.proj.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 520 |
+
"model.visual.blocks.9.attn.proj.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 521 |
+
"model.visual.blocks.9.mlp.linear_fc1.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 522 |
+
"model.visual.blocks.9.mlp.linear_fc1.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 523 |
+
"model.visual.blocks.9.mlp.linear_fc2.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 524 |
+
"model.visual.blocks.9.mlp.linear_fc2.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 525 |
+
"model.visual.blocks.10.norm1.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 526 |
+
"model.visual.blocks.10.norm1.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 527 |
+
"model.visual.blocks.10.norm2.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 528 |
+
"model.visual.blocks.10.norm2.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 529 |
+
"model.visual.blocks.10.attn.qkv.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 530 |
+
"model.visual.blocks.10.attn.qkv.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 531 |
+
"model.visual.blocks.10.attn.proj.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 532 |
+
"model.visual.blocks.10.attn.proj.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 533 |
+
"model.visual.blocks.10.mlp.linear_fc1.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 534 |
+
"model.visual.blocks.10.mlp.linear_fc1.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 535 |
+
"model.visual.blocks.10.mlp.linear_fc2.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 536 |
+
"model.visual.blocks.10.mlp.linear_fc2.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 537 |
+
"model.visual.blocks.11.norm1.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 538 |
+
"model.visual.blocks.11.norm1.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 539 |
+
"model.visual.blocks.11.norm2.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 540 |
+
"model.visual.blocks.11.norm2.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 541 |
+
"model.visual.blocks.11.attn.qkv.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 542 |
+
"model.visual.blocks.11.attn.qkv.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 543 |
+
"model.visual.blocks.11.attn.proj.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 544 |
+
"model.visual.blocks.11.attn.proj.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 545 |
+
"model.visual.blocks.11.mlp.linear_fc1.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 546 |
+
"model.visual.blocks.11.mlp.linear_fc1.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 547 |
+
"model.visual.blocks.11.mlp.linear_fc2.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 548 |
+
"model.visual.blocks.11.mlp.linear_fc2.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 549 |
+
"model.visual.blocks.12.norm1.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 550 |
+
"model.visual.blocks.12.norm1.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 551 |
+
"model.visual.blocks.12.norm2.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 552 |
+
"model.visual.blocks.12.norm2.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 553 |
+
"model.visual.blocks.12.attn.qkv.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 554 |
+
"model.visual.blocks.12.attn.qkv.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 555 |
+
"model.visual.blocks.12.attn.proj.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 556 |
+
"model.visual.blocks.12.attn.proj.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 557 |
+
"model.visual.blocks.12.mlp.linear_fc1.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 558 |
+
"model.visual.blocks.12.mlp.linear_fc1.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 559 |
+
"model.visual.blocks.12.mlp.linear_fc2.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 560 |
+
"model.visual.blocks.12.mlp.linear_fc2.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 561 |
+
"model.visual.blocks.13.norm1.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 562 |
+
"model.visual.blocks.13.norm1.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 563 |
+
"model.visual.blocks.13.norm2.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 564 |
+
"model.visual.blocks.13.norm2.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 565 |
+
"model.visual.blocks.13.attn.qkv.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 566 |
+
"model.visual.blocks.13.attn.qkv.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 567 |
+
"model.visual.blocks.13.attn.proj.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 568 |
+
"model.visual.blocks.13.attn.proj.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 569 |
+
"model.visual.blocks.13.mlp.linear_fc1.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 570 |
+
"model.visual.blocks.13.mlp.linear_fc1.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 571 |
+
"model.visual.blocks.13.mlp.linear_fc2.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 572 |
+
"model.visual.blocks.13.mlp.linear_fc2.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 573 |
+
"model.visual.blocks.14.norm1.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 574 |
+
"model.visual.blocks.14.norm1.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 575 |
+
"model.visual.blocks.14.norm2.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 576 |
+
"model.visual.blocks.14.norm2.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 577 |
+
"model.visual.blocks.14.attn.qkv.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 578 |
+
"model.visual.blocks.14.attn.qkv.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 579 |
+
"model.visual.blocks.14.attn.proj.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 580 |
+
"model.visual.blocks.14.attn.proj.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 581 |
+
"model.visual.blocks.14.mlp.linear_fc1.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 582 |
+
"model.visual.blocks.14.mlp.linear_fc1.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 583 |
+
"model.visual.blocks.14.mlp.linear_fc2.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 584 |
+
"model.visual.blocks.14.mlp.linear_fc2.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 585 |
+
"model.visual.blocks.15.norm1.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 586 |
+
"model.visual.blocks.15.norm1.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 587 |
+
"model.visual.blocks.15.norm2.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 588 |
+
"model.visual.blocks.15.norm2.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 589 |
+
"model.visual.blocks.15.attn.qkv.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 590 |
+
"model.visual.blocks.15.attn.qkv.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 591 |
+
"model.visual.blocks.15.attn.proj.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 592 |
+
"model.visual.blocks.15.attn.proj.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 593 |
+
"model.visual.blocks.15.mlp.linear_fc1.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 594 |
+
"model.visual.blocks.15.mlp.linear_fc1.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 595 |
+
"model.visual.blocks.15.mlp.linear_fc2.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 596 |
+
"model.visual.blocks.15.mlp.linear_fc2.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 597 |
+
"model.visual.blocks.16.norm1.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 598 |
+
"model.visual.blocks.16.norm1.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 599 |
+
"model.visual.blocks.16.norm2.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 600 |
+
"model.visual.blocks.16.norm2.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 601 |
+
"model.visual.blocks.16.attn.qkv.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 602 |
+
"model.visual.blocks.16.attn.qkv.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 603 |
+
"model.visual.blocks.16.attn.proj.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 604 |
+
"model.visual.blocks.16.attn.proj.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 605 |
+
"model.visual.blocks.16.mlp.linear_fc1.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 606 |
+
"model.visual.blocks.16.mlp.linear_fc1.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 607 |
+
"model.visual.blocks.16.mlp.linear_fc2.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 608 |
+
"model.visual.blocks.16.mlp.linear_fc2.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 609 |
+
"model.visual.blocks.17.norm1.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 610 |
+
"model.visual.blocks.17.norm1.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 611 |
+
"model.visual.blocks.17.norm2.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 612 |
+
"model.visual.blocks.17.norm2.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 613 |
+
"model.visual.blocks.17.attn.qkv.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 614 |
+
"model.visual.blocks.17.attn.qkv.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 615 |
+
"model.visual.blocks.17.attn.proj.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 616 |
+
"model.visual.blocks.17.attn.proj.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 617 |
+
"model.visual.blocks.17.mlp.linear_fc1.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 618 |
+
"model.visual.blocks.17.mlp.linear_fc1.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 619 |
+
"model.visual.blocks.17.mlp.linear_fc2.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 620 |
+
"model.visual.blocks.17.mlp.linear_fc2.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 621 |
+
"model.visual.blocks.18.norm1.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 622 |
+
"model.visual.blocks.18.norm1.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 623 |
+
"model.visual.blocks.18.norm2.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 624 |
+
"model.visual.blocks.18.norm2.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 625 |
+
"model.visual.blocks.18.attn.qkv.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 626 |
+
"model.visual.blocks.18.attn.qkv.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 627 |
+
"model.visual.blocks.18.attn.proj.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 628 |
+
"model.visual.blocks.18.attn.proj.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 629 |
+
"model.visual.blocks.18.mlp.linear_fc1.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 630 |
+
"model.visual.blocks.18.mlp.linear_fc1.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 631 |
+
"model.visual.blocks.18.mlp.linear_fc2.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 632 |
+
"model.visual.blocks.18.mlp.linear_fc2.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 633 |
+
"model.visual.blocks.19.norm1.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 634 |
+
"model.visual.blocks.19.norm1.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 635 |
+
"model.visual.blocks.19.norm2.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 636 |
+
"model.visual.blocks.19.norm2.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 637 |
+
"model.visual.blocks.19.attn.qkv.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 638 |
+
"model.visual.blocks.19.attn.qkv.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 639 |
+
"model.visual.blocks.19.attn.proj.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 640 |
+
"model.visual.blocks.19.attn.proj.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 641 |
+
"model.visual.blocks.19.mlp.linear_fc1.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 642 |
+
"model.visual.blocks.19.mlp.linear_fc1.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 643 |
+
"model.visual.blocks.19.mlp.linear_fc2.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 644 |
+
"model.visual.blocks.19.mlp.linear_fc2.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 645 |
+
"model.visual.blocks.20.norm1.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 646 |
+
"model.visual.blocks.20.norm1.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 647 |
+
"model.visual.blocks.20.norm2.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 648 |
+
"model.visual.blocks.20.norm2.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 649 |
+
"model.visual.blocks.20.attn.qkv.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 650 |
+
"model.visual.blocks.20.attn.qkv.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 651 |
+
"model.visual.blocks.20.attn.proj.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 652 |
+
"model.visual.blocks.20.attn.proj.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 653 |
+
"model.visual.blocks.20.mlp.linear_fc1.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 654 |
+
"model.visual.blocks.20.mlp.linear_fc1.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 655 |
+
"model.visual.blocks.20.mlp.linear_fc2.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 656 |
+
"model.visual.blocks.20.mlp.linear_fc2.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 657 |
+
"model.visual.blocks.21.norm1.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 658 |
+
"model.visual.blocks.21.norm1.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 659 |
+
"model.visual.blocks.21.norm2.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 660 |
+
"model.visual.blocks.21.norm2.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 661 |
+
"model.visual.blocks.21.attn.qkv.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 662 |
+
"model.visual.blocks.21.attn.qkv.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 663 |
+
"model.visual.blocks.21.attn.proj.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 664 |
+
"model.visual.blocks.21.attn.proj.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 665 |
+
"model.visual.blocks.21.mlp.linear_fc1.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 666 |
+
"model.visual.blocks.21.mlp.linear_fc1.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 667 |
+
"model.visual.blocks.21.mlp.linear_fc2.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 668 |
+
"model.visual.blocks.21.mlp.linear_fc2.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 669 |
+
"model.visual.blocks.22.norm1.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 670 |
+
"model.visual.blocks.22.norm1.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 671 |
+
"model.visual.blocks.22.norm2.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 672 |
+
"model.visual.blocks.22.norm2.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 673 |
+
"model.visual.blocks.22.attn.qkv.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 674 |
+
"model.visual.blocks.22.attn.qkv.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 675 |
+
"model.visual.blocks.22.attn.proj.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 676 |
+
"model.visual.blocks.22.attn.proj.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 677 |
+
"model.visual.blocks.22.mlp.linear_fc1.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 678 |
+
"model.visual.blocks.22.mlp.linear_fc1.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 679 |
+
"model.visual.blocks.22.mlp.linear_fc2.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 680 |
+
"model.visual.blocks.22.mlp.linear_fc2.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 681 |
+
"model.visual.blocks.23.norm1.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 682 |
+
"model.visual.blocks.23.norm1.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 683 |
+
"model.visual.blocks.23.norm2.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 684 |
+
"model.visual.blocks.23.norm2.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 685 |
+
"model.visual.blocks.23.attn.qkv.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 686 |
+
"model.visual.blocks.23.attn.qkv.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 687 |
+
"model.visual.blocks.23.attn.proj.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 688 |
+
"model.visual.blocks.23.attn.proj.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 689 |
+
"model.visual.blocks.23.mlp.linear_fc1.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 690 |
+
"model.visual.blocks.23.mlp.linear_fc1.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 691 |
+
"model.visual.blocks.23.mlp.linear_fc2.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 692 |
+
"model.visual.blocks.23.mlp.linear_fc2.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 693 |
+
"model.visual.blocks.24.norm1.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 694 |
+
"model.visual.blocks.24.norm1.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 695 |
+
"model.visual.blocks.24.norm2.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 696 |
+
"model.visual.blocks.24.norm2.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 697 |
+
"model.visual.blocks.24.attn.qkv.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 698 |
+
"model.visual.blocks.24.attn.qkv.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 699 |
+
"model.visual.blocks.24.attn.proj.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 700 |
+
"model.visual.blocks.24.attn.proj.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 701 |
+
"model.visual.blocks.24.mlp.linear_fc1.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 702 |
+
"model.visual.blocks.24.mlp.linear_fc1.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 703 |
+
"model.visual.blocks.24.mlp.linear_fc2.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 704 |
+
"model.visual.blocks.24.mlp.linear_fc2.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 705 |
+
"model.visual.blocks.25.norm1.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 706 |
+
"model.visual.blocks.25.norm1.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 707 |
+
"model.visual.blocks.25.norm2.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 708 |
+
"model.visual.blocks.25.norm2.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 709 |
+
"model.visual.blocks.25.attn.qkv.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 710 |
+
"model.visual.blocks.25.attn.qkv.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 711 |
+
"model.visual.blocks.25.attn.proj.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 712 |
+
"model.visual.blocks.25.attn.proj.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 713 |
+
"model.visual.blocks.25.mlp.linear_fc1.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 714 |
+
"model.visual.blocks.25.mlp.linear_fc1.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 715 |
+
"model.visual.blocks.25.mlp.linear_fc2.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 716 |
+
"model.visual.blocks.25.mlp.linear_fc2.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 717 |
+
"model.visual.blocks.26.norm1.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 718 |
+
"model.visual.blocks.26.norm1.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 719 |
+
"model.visual.blocks.26.norm2.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 720 |
+
"model.visual.blocks.26.norm2.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 721 |
+
"model.visual.blocks.26.attn.qkv.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 722 |
+
"model.visual.blocks.26.attn.qkv.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 723 |
+
"model.visual.blocks.26.attn.proj.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 724 |
+
"model.visual.blocks.26.attn.proj.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 725 |
+
"model.visual.blocks.26.mlp.linear_fc1.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 726 |
+
"model.visual.blocks.26.mlp.linear_fc1.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 727 |
+
"model.visual.blocks.26.mlp.linear_fc2.weight": "model-vision-0001-others-save_rank0.safetensors",
|
| 728 |
+
"model.visual.blocks.26.mlp.linear_fc2.bias": "model-vision-0001-others-save_rank0.safetensors",
|
| 729 |
+
"model.visual.merger.norm.weight": "model-projector-0001-others-save_rank0.safetensors",
|
| 730 |
+
"model.visual.merger.norm.bias": "model-projector-0001-others-save_rank0.safetensors",
|
| 731 |
+
"model.visual.merger.linear_fc1.weight": "model-projector-0001-others-save_rank0.safetensors",
|
| 732 |
+
"model.visual.merger.linear_fc1.bias": "model-projector-0001-others-save_rank0.safetensors",
|
| 733 |
+
"model.visual.merger.linear_fc2.weight": "model-projector-0001-others-save_rank0.safetensors",
|
| 734 |
+
"model.visual.merger.linear_fc2.bias": "model-projector-0001-others-save_rank0.safetensors"
|
| 735 |
+
},
|
| 736 |
+
"metadata": {}
|
| 737 |
+
}
|
modeling_internvideo3.py
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
modeling_internvideo3_xslinear.py
ADDED
|
@@ -0,0 +1,1551 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# coding=utf-8
|
| 2 |
+
# Copyright 2025 The InternVideo Team. All rights reserved.
|
| 3 |
+
#
|
| 4 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 5 |
+
# you may not use this file except in compliance with the License.
|
| 6 |
+
# You may obtain a copy of the License at
|
| 7 |
+
#
|
| 8 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 9 |
+
#
|
| 10 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 11 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 12 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 13 |
+
# See the License for the specific language governing permissions and
|
| 14 |
+
# limitations under the License.
|
| 15 |
+
|
| 16 |
+
from dataclasses import dataclass
|
| 17 |
+
from typing import Any, Callable, Optional, Union
|
| 18 |
+
|
| 19 |
+
import torch
|
| 20 |
+
import torch.nn as nn
|
| 21 |
+
import torch.nn.functional as F
|
| 22 |
+
|
| 23 |
+
from transformers.activations import ACT2FN
|
| 24 |
+
from transformers.cache_utils import Cache, DynamicCache
|
| 25 |
+
from transformers.generation import GenerationMixin
|
| 26 |
+
from transformers.integrations import use_kernel_forward_from_hub
|
| 27 |
+
from transformers.masking_utils import create_causal_mask
|
| 28 |
+
from transformers.modeling_flash_attention_utils import FlashAttentionKwargs
|
| 29 |
+
from transformers.modeling_layers import GradientCheckpointingLayer
|
| 30 |
+
from transformers.modeling_outputs import BaseModelOutputWithPast, ModelOutput
|
| 31 |
+
from transformers.modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update
|
| 32 |
+
from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel
|
| 33 |
+
from transformers.processing_utils import Unpack
|
| 34 |
+
from transformers.utils import TransformersKwargs, auto_docstring, is_torchdynamo_compiling
|
| 35 |
+
from transformers.utils.deprecation import deprecate_kwarg
|
| 36 |
+
from transformers.utils.generic import check_model_inputs
|
| 37 |
+
from .configuration_internvideo3 import InternVideo3Config, InternVideo3TextConfig, InternVideo3VisionConfig
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
class InternVideo3VisionMLP(nn.Module):
|
| 41 |
+
def __init__(self, config):
|
| 42 |
+
super().__init__()
|
| 43 |
+
self.hidden_size = config.hidden_size
|
| 44 |
+
self.intermediate_size = config.intermediate_size
|
| 45 |
+
self.linear_fc1 = nn.Linear(self.hidden_size, self.intermediate_size, bias=True)
|
| 46 |
+
self.linear_fc2 = nn.Linear(self.intermediate_size, self.hidden_size, bias=True)
|
| 47 |
+
self.act_fn = ACT2FN[config.hidden_act]
|
| 48 |
+
|
| 49 |
+
def forward(self, hidden_state):
|
| 50 |
+
return self.linear_fc2(self.act_fn(self.linear_fc1(hidden_state)))
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
class InternVideo3VisionPatchEmbed(nn.Module):
|
| 54 |
+
def __init__(self, config) -> None:
|
| 55 |
+
super().__init__()
|
| 56 |
+
self.patch_size = config.patch_size
|
| 57 |
+
self.temporal_patch_size = config.temporal_patch_size
|
| 58 |
+
self.in_channels = config.in_channels
|
| 59 |
+
self.embed_dim = config.hidden_size
|
| 60 |
+
|
| 61 |
+
kernel_size = [self.temporal_patch_size, self.patch_size, self.patch_size]
|
| 62 |
+
self.proj = nn.Conv3d(self.in_channels, self.embed_dim, kernel_size=kernel_size, stride=kernel_size, bias=True)
|
| 63 |
+
|
| 64 |
+
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
| 65 |
+
target_dtype = self.proj.weight.dtype
|
| 66 |
+
hidden_states = hidden_states.view(
|
| 67 |
+
-1, self.in_channels, self.temporal_patch_size, self.patch_size, self.patch_size
|
| 68 |
+
)
|
| 69 |
+
hidden_states = self.proj(hidden_states.to(dtype=target_dtype)).view(-1, self.embed_dim)
|
| 70 |
+
return hidden_states
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
class InternVideo3VisionRotaryEmbedding(nn.Module):
|
| 74 |
+
inv_freq: torch.Tensor # fix linting for `register_buffer`
|
| 75 |
+
|
| 76 |
+
def __init__(self, dim: int, theta: float = 10000.0) -> None:
|
| 77 |
+
super().__init__()
|
| 78 |
+
inv_freq = 1.0 / (theta ** (torch.arange(0, dim, 2, dtype=torch.float) / dim))
|
| 79 |
+
self.register_buffer("inv_freq", inv_freq, persistent=False)
|
| 80 |
+
|
| 81 |
+
def forward(self, seqlen: int) -> torch.Tensor:
|
| 82 |
+
seq = torch.arange(seqlen, device=self.inv_freq.device, dtype=self.inv_freq.dtype)
|
| 83 |
+
freqs = torch.outer(seq, self.inv_freq)
|
| 84 |
+
return freqs
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
class InternVideo3VisionPatchMerger(nn.Module):
|
| 88 |
+
def __init__(self, config: InternVideo3VisionConfig, use_postshuffle_norm=False) -> None:
|
| 89 |
+
super().__init__()
|
| 90 |
+
self.hidden_size = config.hidden_size * (config.spatial_merge_size**2)
|
| 91 |
+
self.use_postshuffle_norm = use_postshuffle_norm
|
| 92 |
+
self.norm = nn.LayerNorm(self.hidden_size if use_postshuffle_norm else config.hidden_size, eps=1e-6)
|
| 93 |
+
self.linear_fc1 = nn.Linear(self.hidden_size, self.hidden_size)
|
| 94 |
+
self.act_fn = nn.GELU()
|
| 95 |
+
self.linear_fc2 = nn.Linear(self.hidden_size, config.out_hidden_size)
|
| 96 |
+
|
| 97 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 98 |
+
x = self.norm(x.view(-1, self.hidden_size) if self.use_postshuffle_norm else x).view(-1, self.hidden_size)
|
| 99 |
+
x = self.linear_fc2(self.act_fn(self.linear_fc1(x)))
|
| 100 |
+
return x
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
def rotate_half(x):
|
| 104 |
+
"""Rotates half the hidden dims of the input."""
|
| 105 |
+
x1 = x[..., : x.shape[-1] // 2]
|
| 106 |
+
x2 = x[..., x.shape[-1] // 2 :]
|
| 107 |
+
return torch.cat((-x2, x1), dim=-1)
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
def apply_rotary_pos_emb_vision(
|
| 111 |
+
q: torch.Tensor, k: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor
|
| 112 |
+
) -> tuple[torch.Tensor, torch.Tensor]:
|
| 113 |
+
orig_q_dtype = q.dtype
|
| 114 |
+
orig_k_dtype = k.dtype
|
| 115 |
+
q, k = q.float(), k.float()
|
| 116 |
+
cos, sin = cos.unsqueeze(-2).float(), sin.unsqueeze(-2).float()
|
| 117 |
+
q_embed = (q * cos) + (rotate_half(q) * sin)
|
| 118 |
+
k_embed = (k * cos) + (rotate_half(k) * sin)
|
| 119 |
+
q_embed = q_embed.to(orig_q_dtype)
|
| 120 |
+
k_embed = k_embed.to(orig_k_dtype)
|
| 121 |
+
return q_embed, k_embed
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:
|
| 125 |
+
"""
|
| 126 |
+
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
|
| 127 |
+
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
|
| 128 |
+
"""
|
| 129 |
+
batch, num_key_value_heads, slen, head_dim = hidden_states.shape
|
| 130 |
+
if n_rep == 1:
|
| 131 |
+
return hidden_states
|
| 132 |
+
hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim)
|
| 133 |
+
return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim)
|
| 134 |
+
|
| 135 |
+
|
| 136 |
+
def eager_attention_forward(
|
| 137 |
+
module: nn.Module,
|
| 138 |
+
query: torch.Tensor,
|
| 139 |
+
key: torch.Tensor,
|
| 140 |
+
value: torch.Tensor,
|
| 141 |
+
attention_mask: Optional[torch.Tensor],
|
| 142 |
+
scaling: float,
|
| 143 |
+
dropout: float = 0.0,
|
| 144 |
+
**kwargs: Unpack[TransformersKwargs],
|
| 145 |
+
):
|
| 146 |
+
key_states = repeat_kv(key, module.num_key_value_groups)
|
| 147 |
+
value_states = repeat_kv(value, module.num_key_value_groups)
|
| 148 |
+
|
| 149 |
+
attn_weights = torch.matmul(query, key_states.transpose(2, 3)) * scaling
|
| 150 |
+
if attention_mask is not None:
|
| 151 |
+
causal_mask = attention_mask[:, :, :, : key_states.shape[-2]]
|
| 152 |
+
attn_weights = attn_weights + causal_mask
|
| 153 |
+
|
| 154 |
+
attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query.dtype)
|
| 155 |
+
attn_weights = nn.functional.dropout(attn_weights, p=dropout, training=module.training)
|
| 156 |
+
attn_output = torch.matmul(attn_weights, value_states)
|
| 157 |
+
attn_output = attn_output.transpose(1, 2).contiguous()
|
| 158 |
+
|
| 159 |
+
return attn_output, attn_weights
|
| 160 |
+
|
| 161 |
+
|
| 162 |
+
class InternVideo3VisionAttention(nn.Module):
|
| 163 |
+
def __init__(self, config: InternVideo3VisionConfig) -> None:
|
| 164 |
+
super().__init__()
|
| 165 |
+
self.dim = config.hidden_size
|
| 166 |
+
self.num_heads = config.num_heads
|
| 167 |
+
self.head_dim = self.dim // self.num_heads
|
| 168 |
+
self.num_key_value_groups = 1 # needed for eager attention
|
| 169 |
+
self.qkv = nn.Linear(self.dim, self.dim * 3, bias=True)
|
| 170 |
+
self.proj = nn.Linear(self.dim, self.dim)
|
| 171 |
+
self.scaling = self.head_dim**-0.5
|
| 172 |
+
self.config = config
|
| 173 |
+
self.attention_dropout = 0.0
|
| 174 |
+
self.is_causal = False
|
| 175 |
+
|
| 176 |
+
def forward(
|
| 177 |
+
self,
|
| 178 |
+
hidden_states: torch.Tensor,
|
| 179 |
+
cu_seqlens: torch.Tensor,
|
| 180 |
+
rotary_pos_emb: Optional[torch.Tensor] = None,
|
| 181 |
+
position_embeddings: Optional[tuple[torch.Tensor, torch.Tensor]] = None,
|
| 182 |
+
**kwargs,
|
| 183 |
+
) -> torch.Tensor:
|
| 184 |
+
seq_length = hidden_states.shape[0]
|
| 185 |
+
query_states, key_states, value_states = (
|
| 186 |
+
self.qkv(hidden_states).reshape(seq_length, 3, self.num_heads, -1).permute(1, 0, 2, 3).unbind(0)
|
| 187 |
+
)
|
| 188 |
+
cos, sin = position_embeddings
|
| 189 |
+
query_states, key_states = apply_rotary_pos_emb_vision(query_states, key_states, cos, sin)
|
| 190 |
+
|
| 191 |
+
query_states = query_states.transpose(0, 1).unsqueeze(0)
|
| 192 |
+
key_states = key_states.transpose(0, 1).unsqueeze(0)
|
| 193 |
+
value_states = value_states.transpose(0, 1).unsqueeze(0)
|
| 194 |
+
|
| 195 |
+
attention_interface: Callable = eager_attention_forward
|
| 196 |
+
if self.config._attn_implementation != "eager":
|
| 197 |
+
attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation]
|
| 198 |
+
|
| 199 |
+
if self.config._attn_implementation == "flash_attention_2":
|
| 200 |
+
# Flash Attention 2: Use cu_seqlens for variable length attention
|
| 201 |
+
max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max()
|
| 202 |
+
attn_output, _ = attention_interface(
|
| 203 |
+
self,
|
| 204 |
+
query_states,
|
| 205 |
+
key_states,
|
| 206 |
+
value_states,
|
| 207 |
+
attention_mask=None,
|
| 208 |
+
scaling=self.scaling,
|
| 209 |
+
dropout=0.0 if not self.training else self.attention_dropout,
|
| 210 |
+
cu_seq_lens_q=cu_seqlens,
|
| 211 |
+
cu_seq_lens_k=cu_seqlens,
|
| 212 |
+
max_length_q=max_seqlen,
|
| 213 |
+
max_length_k=max_seqlen,
|
| 214 |
+
is_causal=False,
|
| 215 |
+
**kwargs,
|
| 216 |
+
)
|
| 217 |
+
else:
|
| 218 |
+
# Other implementations: Process each chunk separately
|
| 219 |
+
lengths = cu_seqlens[1:] - cu_seqlens[:-1]
|
| 220 |
+
splits = [
|
| 221 |
+
torch.split(tensor, lengths.tolist(), dim=2) for tensor in (query_states, key_states, value_states)
|
| 222 |
+
]
|
| 223 |
+
|
| 224 |
+
attn_outputs = [
|
| 225 |
+
attention_interface(
|
| 226 |
+
self,
|
| 227 |
+
q,
|
| 228 |
+
k,
|
| 229 |
+
v,
|
| 230 |
+
attention_mask=None,
|
| 231 |
+
scaling=self.scaling,
|
| 232 |
+
dropout=0.0 if not self.training else self.attention_dropout,
|
| 233 |
+
is_causal=False,
|
| 234 |
+
**kwargs,
|
| 235 |
+
)[0]
|
| 236 |
+
for q, k, v in zip(*splits)
|
| 237 |
+
]
|
| 238 |
+
attn_output = torch.cat(attn_outputs, dim=1)
|
| 239 |
+
|
| 240 |
+
attn_output = attn_output.reshape(seq_length, -1).contiguous()
|
| 241 |
+
attn_output = self.proj(attn_output)
|
| 242 |
+
return attn_output
|
| 243 |
+
|
| 244 |
+
|
| 245 |
+
class InternVideo3VisionBlock(GradientCheckpointingLayer):
|
| 246 |
+
def __init__(self, config, attn_implementation: str = "sdpa") -> None:
|
| 247 |
+
super().__init__()
|
| 248 |
+
self.norm1 = nn.LayerNorm(config.hidden_size, eps=1e-6)
|
| 249 |
+
self.norm2 = nn.LayerNorm(config.hidden_size, eps=1e-6)
|
| 250 |
+
self.attn = InternVideo3VisionAttention(config=config)
|
| 251 |
+
self.mlp = InternVideo3VisionMLP(config=config)
|
| 252 |
+
|
| 253 |
+
def forward(
|
| 254 |
+
self,
|
| 255 |
+
hidden_states: torch.Tensor,
|
| 256 |
+
cu_seqlens: torch.Tensor,
|
| 257 |
+
rotary_pos_emb: Optional[torch.Tensor] = None,
|
| 258 |
+
position_embeddings: Optional[tuple[torch.Tensor, torch.Tensor]] = None,
|
| 259 |
+
**kwargs,
|
| 260 |
+
) -> torch.Tensor:
|
| 261 |
+
hidden_states = hidden_states + self.attn(
|
| 262 |
+
self.norm1(hidden_states),
|
| 263 |
+
cu_seqlens=cu_seqlens,
|
| 264 |
+
rotary_pos_emb=rotary_pos_emb,
|
| 265 |
+
position_embeddings=position_embeddings,
|
| 266 |
+
**kwargs,
|
| 267 |
+
)
|
| 268 |
+
hidden_states = hidden_states + self.mlp(self.norm2(hidden_states))
|
| 269 |
+
return hidden_states
|
| 270 |
+
|
| 271 |
+
|
| 272 |
+
class InternVideo3TextRotaryEmbedding(nn.Module):
|
| 273 |
+
inv_freq: torch.Tensor # fix linting for `register_buffer`
|
| 274 |
+
|
| 275 |
+
def __init__(self, config: InternVideo3TextConfig, device=None):
|
| 276 |
+
super().__init__()
|
| 277 |
+
if hasattr(config, "rope_scaling") and config.rope_scaling is not None:
|
| 278 |
+
self.rope_type = config.rope_scaling.get("rope_type", "default")
|
| 279 |
+
else:
|
| 280 |
+
self.rope_type = "default"
|
| 281 |
+
self.max_seq_len_cached = config.max_position_embeddings
|
| 282 |
+
self.original_max_seq_len = config.max_position_embeddings
|
| 283 |
+
|
| 284 |
+
self.config = config
|
| 285 |
+
self.rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type]
|
| 286 |
+
|
| 287 |
+
inv_freq, self.attention_scaling = self.rope_init_fn(self.config, device)
|
| 288 |
+
self.register_buffer("inv_freq", inv_freq, persistent=False)
|
| 289 |
+
self.original_inv_freq = self.inv_freq
|
| 290 |
+
|
| 291 |
+
self.mrope_section = config.rope_scaling.get("mrope_section", [24, 20, 20])
|
| 292 |
+
|
| 293 |
+
def apply_interleaved_mrope(self, freqs, mrope_section):
|
| 294 |
+
"""Apply interleaved MRoPE to 3D rotary embeddings.
|
| 295 |
+
Reorganizes frequency layout from chunked [TTT...HHH...WWW] to
|
| 296 |
+
interleaved [THTHWHTHW...TT], preserving frequency continuity.
|
| 297 |
+
args:
|
| 298 |
+
x: (3, bs, seq_len, head_dim // 2)
|
| 299 |
+
mrope_section: (3,)
|
| 300 |
+
returns:
|
| 301 |
+
x_t: (bs, seq_len, head_dim // 2)
|
| 302 |
+
"""
|
| 303 |
+
freqs_t = freqs[0] # just overwrite the first dimension T
|
| 304 |
+
for dim, offset in enumerate((1, 2), start=1): # H, W
|
| 305 |
+
length = mrope_section[dim] * 3
|
| 306 |
+
idx = slice(offset, length, 3)
|
| 307 |
+
freqs_t[..., idx] = freqs[dim, ..., idx]
|
| 308 |
+
return freqs_t
|
| 309 |
+
|
| 310 |
+
@torch.no_grad()
|
| 311 |
+
@dynamic_rope_update # power user: used with advanced RoPE types (e.g. dynamic rope)
|
| 312 |
+
def forward(self, x, position_ids):
|
| 313 |
+
# In contrast to other models, InternVideo3 has different position ids for the grids
|
| 314 |
+
# So we expand the inv_freq to shape (3, ...)
|
| 315 |
+
if position_ids.ndim == 2:
|
| 316 |
+
position_ids = position_ids[None, ...].expand(3, position_ids.shape[0], -1)
|
| 317 |
+
inv_freq_expanded = self.inv_freq[None, None, :, None].float().expand(3, position_ids.shape[1], -1, 1)
|
| 318 |
+
position_ids_expanded = position_ids[:, :, None, :].float() # shape (3, bs, 1, positions)
|
| 319 |
+
|
| 320 |
+
device_type = x.device.type if isinstance(x.device.type, str) and x.device.type != "mps" else "cpu"
|
| 321 |
+
with torch.autocast(device_type=device_type, enabled=False): # Force float32
|
| 322 |
+
freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(2, 3)
|
| 323 |
+
freqs = self.apply_interleaved_mrope(freqs, self.mrope_section)
|
| 324 |
+
emb = torch.cat((freqs, freqs), dim=-1)
|
| 325 |
+
cos = emb.cos() * self.attention_scaling
|
| 326 |
+
sin = emb.sin() * self.attention_scaling
|
| 327 |
+
|
| 328 |
+
return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype)
|
| 329 |
+
|
| 330 |
+
|
| 331 |
+
@use_kernel_forward_from_hub("RMSNorm")
|
| 332 |
+
class InternVideo3TextRMSNorm(nn.Module):
|
| 333 |
+
def __init__(self, hidden_size, eps: float = 1e-6) -> None:
|
| 334 |
+
"""
|
| 335 |
+
InternVideo3TextRMSNorm is equivalent to T5LayerNorm
|
| 336 |
+
"""
|
| 337 |
+
super().__init__()
|
| 338 |
+
self.weight = nn.Parameter(torch.ones(hidden_size))
|
| 339 |
+
self.variance_epsilon = eps
|
| 340 |
+
|
| 341 |
+
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
| 342 |
+
input_dtype = hidden_states.dtype
|
| 343 |
+
hidden_states = hidden_states.to(torch.float32)
|
| 344 |
+
variance = hidden_states.pow(2).mean(-1, keepdim=True)
|
| 345 |
+
hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
|
| 346 |
+
return self.weight * hidden_states.to(input_dtype)
|
| 347 |
+
|
| 348 |
+
def extra_repr(self):
|
| 349 |
+
return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}"
|
| 350 |
+
|
| 351 |
+
|
| 352 |
+
def apply_rotary_pos_emb(q, k, cos, sin, position_ids=None, unsqueeze_dim=1):
|
| 353 |
+
"""Applies Rotary Position Embedding to the query and key tensors.
|
| 354 |
+
|
| 355 |
+
Args:
|
| 356 |
+
q (`torch.Tensor`): The query tensor.
|
| 357 |
+
k (`torch.Tensor`): The key tensor.
|
| 358 |
+
cos (`torch.Tensor`): The cosine part of the rotary embedding.
|
| 359 |
+
sin (`torch.Tensor`): The sine part of the rotary embedding.
|
| 360 |
+
position_ids (`torch.Tensor`, *optional*):
|
| 361 |
+
Deprecated and unused.
|
| 362 |
+
unsqueeze_dim (`int`, *optional*, defaults to 1):
|
| 363 |
+
The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
|
| 364 |
+
sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
|
| 365 |
+
that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
|
| 366 |
+
k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
|
| 367 |
+
cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
|
| 368 |
+
the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
|
| 369 |
+
Returns:
|
| 370 |
+
`tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
|
| 371 |
+
"""
|
| 372 |
+
cos = cos.unsqueeze(unsqueeze_dim)
|
| 373 |
+
sin = sin.unsqueeze(unsqueeze_dim)
|
| 374 |
+
q_embed = (q * cos) + (rotate_half(q) * sin)
|
| 375 |
+
k_embed = (k * cos) + (rotate_half(k) * sin)
|
| 376 |
+
return q_embed, k_embed
|
| 377 |
+
|
| 378 |
+
|
| 379 |
+
class InternVideo3TextAttention(nn.Module):
|
| 380 |
+
"""Multi-headed attention from 'Attention Is All You Need' paper"""
|
| 381 |
+
|
| 382 |
+
def __init__(self, config: InternVideo3TextConfig, layer_idx: int):
|
| 383 |
+
super().__init__()
|
| 384 |
+
self.config = config
|
| 385 |
+
self.layer_idx = layer_idx
|
| 386 |
+
self.head_dim = getattr(config, "head_dim", config.hidden_size // config.num_attention_heads)
|
| 387 |
+
self.num_key_value_groups = config.num_attention_heads // config.num_key_value_heads
|
| 388 |
+
self.scaling = self.head_dim**-0.5
|
| 389 |
+
self.attention_dropout = config.attention_dropout
|
| 390 |
+
self.is_causal = True
|
| 391 |
+
|
| 392 |
+
self.q_proj = nn.Linear(
|
| 393 |
+
config.hidden_size, config.num_attention_heads * self.head_dim, bias=config.attention_bias
|
| 394 |
+
)
|
| 395 |
+
self.k_proj = nn.Linear(
|
| 396 |
+
config.hidden_size, config.num_key_value_heads * self.head_dim, bias=config.attention_bias
|
| 397 |
+
)
|
| 398 |
+
self.v_proj = nn.Linear(
|
| 399 |
+
config.hidden_size, config.num_key_value_heads * self.head_dim, bias=config.attention_bias
|
| 400 |
+
)
|
| 401 |
+
self.o_proj = nn.Linear(
|
| 402 |
+
config.num_attention_heads * self.head_dim, config.hidden_size, bias=False
|
| 403 |
+
)
|
| 404 |
+
|
| 405 |
+
@deprecate_kwarg("past_key_value", new_name="past_key_values", version="4.58")
|
| 406 |
+
def forward(
|
| 407 |
+
self,
|
| 408 |
+
hidden_states: torch.Tensor,
|
| 409 |
+
position_embeddings: tuple[torch.Tensor, torch.Tensor],
|
| 410 |
+
attention_mask: Optional[torch.Tensor],
|
| 411 |
+
past_key_values: Optional[Cache] = None,
|
| 412 |
+
cache_position: Optional[torch.LongTensor] = None,
|
| 413 |
+
**kwargs: Unpack[FlashAttentionKwargs],
|
| 414 |
+
) -> tuple[torch.Tensor, Optional[torch.Tensor]]:
|
| 415 |
+
input_shape = hidden_states.shape[:-1]
|
| 416 |
+
hidden_shape = (*input_shape, -1, self.head_dim)
|
| 417 |
+
|
| 418 |
+
query_states = self.q_proj(hidden_states).view(hidden_shape).transpose(1, 2)
|
| 419 |
+
key_states = self.k_proj(hidden_states).view(hidden_shape).transpose(1, 2)
|
| 420 |
+
value_states = self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2)
|
| 421 |
+
|
| 422 |
+
cos, sin = position_embeddings
|
| 423 |
+
query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin)
|
| 424 |
+
|
| 425 |
+
if past_key_values is not None:
|
| 426 |
+
# sin and cos are specific to RoPE models; cache_position needed for the static cache
|
| 427 |
+
cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position}
|
| 428 |
+
key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs)
|
| 429 |
+
|
| 430 |
+
attention_interface: Callable = eager_attention_forward
|
| 431 |
+
if self.config._attn_implementation != "eager":
|
| 432 |
+
attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation]
|
| 433 |
+
|
| 434 |
+
attn_output, attn_weights = attention_interface(
|
| 435 |
+
self,
|
| 436 |
+
query_states,
|
| 437 |
+
key_states,
|
| 438 |
+
value_states,
|
| 439 |
+
attention_mask,
|
| 440 |
+
dropout=0.0 if not self.training else self.attention_dropout,
|
| 441 |
+
scaling=self.scaling,
|
| 442 |
+
**kwargs,
|
| 443 |
+
)
|
| 444 |
+
|
| 445 |
+
attn_output = attn_output.reshape(*input_shape, -1).contiguous()
|
| 446 |
+
attn_output = self.o_proj(attn_output)
|
| 447 |
+
return attn_output, attn_weights
|
| 448 |
+
|
| 449 |
+
|
| 450 |
+
class InternVideo3TextMLP(nn.Module):
|
| 451 |
+
def __init__(self, config):
|
| 452 |
+
super().__init__()
|
| 453 |
+
self.config = config
|
| 454 |
+
self.hidden_size = config.hidden_size
|
| 455 |
+
self.intermediate_size = config.intermediate_size
|
| 456 |
+
self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)
|
| 457 |
+
self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)
|
| 458 |
+
self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=False)
|
| 459 |
+
self.act_fn = ACT2FN[config.hidden_act]
|
| 460 |
+
|
| 461 |
+
def forward(self, x):
|
| 462 |
+
down_proj = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))
|
| 463 |
+
return down_proj
|
| 464 |
+
|
| 465 |
+
|
| 466 |
+
class InternVideo3TextDecoderLayer(GradientCheckpointingLayer):
|
| 467 |
+
def __init__(self, config: InternVideo3TextConfig, layer_idx: int):
|
| 468 |
+
super().__init__()
|
| 469 |
+
self.hidden_size = config.hidden_size
|
| 470 |
+
|
| 471 |
+
self.self_attn = InternVideo3TextAttention(config=config, layer_idx=layer_idx)
|
| 472 |
+
|
| 473 |
+
self.mlp = InternVideo3TextMLP(config)
|
| 474 |
+
self.input_layernorm = InternVideo3TextRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
| 475 |
+
self.post_attention_layernorm = InternVideo3TextRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
| 476 |
+
|
| 477 |
+
@deprecate_kwarg("past_key_value", new_name="past_key_values", version="4.58")
|
| 478 |
+
def forward(
|
| 479 |
+
self,
|
| 480 |
+
hidden_states: torch.Tensor,
|
| 481 |
+
position_embeddings: tuple[torch.Tensor, torch.Tensor],
|
| 482 |
+
attention_mask: Optional[torch.Tensor] = None,
|
| 483 |
+
position_ids: Optional[torch.LongTensor] = None,
|
| 484 |
+
past_key_values: Optional[Cache] = None,
|
| 485 |
+
use_cache: Optional[bool] = False,
|
| 486 |
+
cache_position: Optional[torch.LongTensor] = None,
|
| 487 |
+
**kwargs: Unpack[TransformersKwargs],
|
| 488 |
+
) -> torch.Tensor:
|
| 489 |
+
residual = hidden_states
|
| 490 |
+
hidden_states = self.input_layernorm(hidden_states)
|
| 491 |
+
# Self Attention
|
| 492 |
+
hidden_states, _ = self.self_attn(
|
| 493 |
+
hidden_states=hidden_states,
|
| 494 |
+
attention_mask=attention_mask,
|
| 495 |
+
position_ids=position_ids,
|
| 496 |
+
past_key_values=past_key_values,
|
| 497 |
+
use_cache=use_cache,
|
| 498 |
+
cache_position=cache_position,
|
| 499 |
+
position_embeddings=position_embeddings,
|
| 500 |
+
**kwargs,
|
| 501 |
+
)
|
| 502 |
+
hidden_states = residual + hidden_states
|
| 503 |
+
|
| 504 |
+
# Fully Connected
|
| 505 |
+
residual = hidden_states
|
| 506 |
+
hidden_states = self.post_attention_layernorm(hidden_states)
|
| 507 |
+
hidden_states = self.mlp(hidden_states)
|
| 508 |
+
hidden_states = residual + hidden_states
|
| 509 |
+
return hidden_states
|
| 510 |
+
|
| 511 |
+
|
| 512 |
+
@dataclass
|
| 513 |
+
@auto_docstring(
|
| 514 |
+
custom_intro="""
|
| 515 |
+
Base class for Llava outputs, with hidden states and attentions.
|
| 516 |
+
"""
|
| 517 |
+
)
|
| 518 |
+
class InternVideo3ModelOutputWithPast(ModelOutput):
|
| 519 |
+
r"""
|
| 520 |
+
past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
|
| 521 |
+
It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).
|
| 522 |
+
|
| 523 |
+
Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
|
| 524 |
+
`past_key_values` input) to speed up sequential decoding.
|
| 525 |
+
rope_deltas (`torch.LongTensor` of shape `(batch_size, )`, *optional*):
|
| 526 |
+
The rope index difference between sequence length and multimodal rope.
|
| 527 |
+
"""
|
| 528 |
+
|
| 529 |
+
last_hidden_state: Optional[torch.FloatTensor] = None
|
| 530 |
+
past_key_values: Optional[Cache] = None
|
| 531 |
+
hidden_states: Optional[tuple[torch.FloatTensor]] = None
|
| 532 |
+
attentions: Optional[tuple[torch.FloatTensor]] = None
|
| 533 |
+
rope_deltas: Optional[torch.LongTensor] = None
|
| 534 |
+
|
| 535 |
+
|
| 536 |
+
@auto_docstring
|
| 537 |
+
class InternVideo3PreTrainedModel(PreTrainedModel):
|
| 538 |
+
config: InternVideo3Config
|
| 539 |
+
base_model_prefix = "model"
|
| 540 |
+
supports_gradient_checkpointing = True
|
| 541 |
+
_no_split_modules = ["InternVideo3TextDecoderLayer", "InternVideo3VisionBlock"]
|
| 542 |
+
_skip_keys_device_placement = "past_key_values"
|
| 543 |
+
_supports_flash_attn = True
|
| 544 |
+
_supports_sdpa = True
|
| 545 |
+
|
| 546 |
+
_can_compile_fullgraph = True
|
| 547 |
+
_supports_attention_backend = True
|
| 548 |
+
_can_record_outputs = {
|
| 549 |
+
"hidden_states": InternVideo3TextDecoderLayer,
|
| 550 |
+
"attentions": InternVideo3TextAttention,
|
| 551 |
+
}
|
| 552 |
+
|
| 553 |
+
|
| 554 |
+
class InternVideo3VisionModel(InternVideo3PreTrainedModel):
|
| 555 |
+
config: InternVideo3VisionConfig
|
| 556 |
+
_no_split_modules = ["InternVideo3VisionBlock"]
|
| 557 |
+
|
| 558 |
+
def __init__(self, config, *inputs, **kwargs) -> None:
|
| 559 |
+
super().__init__(config, *inputs, **kwargs)
|
| 560 |
+
self.spatial_merge_size = config.spatial_merge_size
|
| 561 |
+
self.patch_size = config.patch_size
|
| 562 |
+
self.spatial_merge_unit = self.spatial_merge_size * self.spatial_merge_size
|
| 563 |
+
|
| 564 |
+
self.patch_embed = InternVideo3VisionPatchEmbed(
|
| 565 |
+
config=config,
|
| 566 |
+
)
|
| 567 |
+
|
| 568 |
+
self.pos_embed = nn.Embedding(config.num_position_embeddings, config.hidden_size)
|
| 569 |
+
self.num_grid_per_side = int(config.num_position_embeddings**0.5)
|
| 570 |
+
|
| 571 |
+
head_dim = config.hidden_size // config.num_heads
|
| 572 |
+
self.rotary_pos_emb = InternVideo3VisionRotaryEmbedding(head_dim // 2)
|
| 573 |
+
|
| 574 |
+
self.blocks = nn.ModuleList([InternVideo3VisionBlock(config) for _ in range(config.depth)])
|
| 575 |
+
self.merger = InternVideo3VisionPatchMerger(
|
| 576 |
+
config=config,
|
| 577 |
+
use_postshuffle_norm=False,
|
| 578 |
+
)
|
| 579 |
+
|
| 580 |
+
self.deepstack_visual_indexes = config.deepstack_visual_indexes
|
| 581 |
+
self.deepstack_merger_list = nn.ModuleList(
|
| 582 |
+
[
|
| 583 |
+
InternVideo3VisionPatchMerger(
|
| 584 |
+
config=config,
|
| 585 |
+
use_postshuffle_norm=True,
|
| 586 |
+
)
|
| 587 |
+
for _ in range(len(config.deepstack_visual_indexes))
|
| 588 |
+
]
|
| 589 |
+
)
|
| 590 |
+
|
| 591 |
+
self.gradient_checkpointing = False
|
| 592 |
+
|
| 593 |
+
def rot_pos_emb(self, grid_thw: torch.Tensor) -> torch.Tensor:
|
| 594 |
+
merge_size = self.spatial_merge_size
|
| 595 |
+
|
| 596 |
+
max_hw = int(grid_thw[:, 1:].max().item())
|
| 597 |
+
freq_table = self.rotary_pos_emb(max_hw) # (max_hw, dim // 2)
|
| 598 |
+
device = freq_table.device
|
| 599 |
+
|
| 600 |
+
total_tokens = int(torch.prod(grid_thw, dim=1).sum().item())
|
| 601 |
+
pos_ids = torch.empty((total_tokens, 2), dtype=torch.long, device=device)
|
| 602 |
+
|
| 603 |
+
offset = 0
|
| 604 |
+
for num_frames, height, width in grid_thw:
|
| 605 |
+
merged_h, merged_w = height // merge_size, width // merge_size
|
| 606 |
+
|
| 607 |
+
block_rows = torch.arange(merged_h, device=device) # block row indices
|
| 608 |
+
block_cols = torch.arange(merged_w, device=device) # block col indices
|
| 609 |
+
intra_row = torch.arange(merge_size, device=device) # intra-block row offsets
|
| 610 |
+
intra_col = torch.arange(merge_size, device=device) # intra-block col offsets
|
| 611 |
+
|
| 612 |
+
# Compute full-resolution positions
|
| 613 |
+
row_idx = block_rows[:, None, None, None] * merge_size + intra_row[None, None, :, None]
|
| 614 |
+
col_idx = block_cols[None, :, None, None] * merge_size + intra_col[None, None, None, :]
|
| 615 |
+
|
| 616 |
+
row_idx = row_idx.expand(merged_h, merged_w, merge_size, merge_size).reshape(-1)
|
| 617 |
+
col_idx = col_idx.expand(merged_h, merged_w, merge_size, merge_size).reshape(-1)
|
| 618 |
+
|
| 619 |
+
coords = torch.stack((row_idx, col_idx), dim=-1)
|
| 620 |
+
|
| 621 |
+
if num_frames > 1:
|
| 622 |
+
coords = coords.repeat(num_frames, 1)
|
| 623 |
+
|
| 624 |
+
num_tokens = coords.shape[0]
|
| 625 |
+
pos_ids[offset : offset + num_tokens] = coords
|
| 626 |
+
offset += num_tokens
|
| 627 |
+
|
| 628 |
+
embeddings = freq_table[pos_ids] # lookup rotary embeddings
|
| 629 |
+
embeddings = embeddings.flatten(1)
|
| 630 |
+
return embeddings
|
| 631 |
+
|
| 632 |
+
def fast_pos_embed_interpolate(self, grid_thw):
|
| 633 |
+
grid_ts, grid_hs, grid_ws = grid_thw[:, 0], grid_thw[:, 1], grid_thw[:, 2]
|
| 634 |
+
|
| 635 |
+
idx_list = [[] for _ in range(4)]
|
| 636 |
+
weight_list = [[] for _ in range(4)]
|
| 637 |
+
|
| 638 |
+
for t, h, w in zip(grid_ts, grid_hs, grid_ws):
|
| 639 |
+
h_idxs = torch.linspace(0, self.num_grid_per_side - 1, h)
|
| 640 |
+
w_idxs = torch.linspace(0, self.num_grid_per_side - 1, w)
|
| 641 |
+
|
| 642 |
+
h_idxs_floor = h_idxs.int()
|
| 643 |
+
w_idxs_floor = w_idxs.int()
|
| 644 |
+
h_idxs_ceil = (h_idxs.int() + 1).clip(max=self.num_grid_per_side - 1)
|
| 645 |
+
w_idxs_ceil = (w_idxs.int() + 1).clip(max=self.num_grid_per_side - 1)
|
| 646 |
+
|
| 647 |
+
dh = h_idxs - h_idxs_floor
|
| 648 |
+
dw = w_idxs - w_idxs_floor
|
| 649 |
+
|
| 650 |
+
base_h = h_idxs_floor * self.num_grid_per_side
|
| 651 |
+
base_h_ceil = h_idxs_ceil * self.num_grid_per_side
|
| 652 |
+
|
| 653 |
+
indices = [
|
| 654 |
+
(base_h[None].T + w_idxs_floor[None]).flatten(),
|
| 655 |
+
(base_h[None].T + w_idxs_ceil[None]).flatten(),
|
| 656 |
+
(base_h_ceil[None].T + w_idxs_floor[None]).flatten(),
|
| 657 |
+
(base_h_ceil[None].T + w_idxs_ceil[None]).flatten(),
|
| 658 |
+
]
|
| 659 |
+
|
| 660 |
+
weights = [
|
| 661 |
+
((1 - dh)[None].T * (1 - dw)[None]).flatten(),
|
| 662 |
+
((1 - dh)[None].T * dw[None]).flatten(),
|
| 663 |
+
(dh[None].T * (1 - dw)[None]).flatten(),
|
| 664 |
+
(dh[None].T * dw[None]).flatten(),
|
| 665 |
+
]
|
| 666 |
+
|
| 667 |
+
for i in range(4):
|
| 668 |
+
idx_list[i].extend(indices[i].tolist())
|
| 669 |
+
weight_list[i].extend(weights[i].tolist())
|
| 670 |
+
|
| 671 |
+
idx_tensor = torch.tensor(idx_list, dtype=torch.long, device=self.pos_embed.weight.device)
|
| 672 |
+
weight_tensor = torch.tensor(
|
| 673 |
+
weight_list, dtype=self.pos_embed.weight.dtype, device=self.pos_embed.weight.device
|
| 674 |
+
)
|
| 675 |
+
pos_embeds = self.pos_embed(idx_tensor) * weight_tensor[:, :, None]
|
| 676 |
+
patch_pos_embeds = pos_embeds[0] + pos_embeds[1] + pos_embeds[2] + pos_embeds[3]
|
| 677 |
+
|
| 678 |
+
patch_pos_embeds = patch_pos_embeds.split([h * w for h, w in zip(grid_hs, grid_ws)])
|
| 679 |
+
|
| 680 |
+
patch_pos_embeds_permute = []
|
| 681 |
+
merge_size = self.config.spatial_merge_size
|
| 682 |
+
for pos_embed, t, h, w in zip(patch_pos_embeds, grid_ts, grid_hs, grid_ws):
|
| 683 |
+
pos_embed = pos_embed.repeat(t, 1)
|
| 684 |
+
pos_embed = (
|
| 685 |
+
pos_embed.view(t, h // merge_size, merge_size, w // merge_size, merge_size, -1)
|
| 686 |
+
.permute(0, 1, 3, 2, 4, 5)
|
| 687 |
+
.flatten(0, 4)
|
| 688 |
+
)
|
| 689 |
+
patch_pos_embeds_permute.append(pos_embed)
|
| 690 |
+
patch_pos_embeds = torch.cat(patch_pos_embeds_permute)
|
| 691 |
+
return patch_pos_embeds
|
| 692 |
+
|
| 693 |
+
def forward(self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, **kwargs) -> torch.Tensor:
|
| 694 |
+
"""
|
| 695 |
+
Args:
|
| 696 |
+
hidden_states (`torch.Tensor` of shape `(seq_len, hidden_size)`):
|
| 697 |
+
The final hidden states of the model.
|
| 698 |
+
grid_thw (`torch.Tensor` of shape `(num_images_or_videos, 3)`):
|
| 699 |
+
The temporal, height and width of feature shape of each image in LLM.
|
| 700 |
+
|
| 701 |
+
Returns:
|
| 702 |
+
`torch.Tensor`: hidden_states.
|
| 703 |
+
"""
|
| 704 |
+
hidden_states = self.patch_embed(hidden_states)
|
| 705 |
+
|
| 706 |
+
pos_embeds = self.fast_pos_embed_interpolate(grid_thw)
|
| 707 |
+
hidden_states = hidden_states + pos_embeds
|
| 708 |
+
|
| 709 |
+
rotary_pos_emb = self.rot_pos_emb(grid_thw)
|
| 710 |
+
|
| 711 |
+
seq_len, _ = hidden_states.size()
|
| 712 |
+
hidden_states = hidden_states.reshape(seq_len, -1)
|
| 713 |
+
rotary_pos_emb = rotary_pos_emb.reshape(seq_len, -1)
|
| 714 |
+
emb = torch.cat((rotary_pos_emb, rotary_pos_emb), dim=-1)
|
| 715 |
+
position_embeddings = (emb.cos(), emb.sin())
|
| 716 |
+
|
| 717 |
+
cu_seqlens = torch.repeat_interleave(grid_thw[:, 1] * grid_thw[:, 2], grid_thw[:, 0]).cumsum(
|
| 718 |
+
dim=0,
|
| 719 |
+
# Select dtype based on the following factors:
|
| 720 |
+
# - FA2 requires that cu_seqlens_q must have dtype int32
|
| 721 |
+
# - torch.onnx.export requires that cu_seqlens_q must have same dtype as grid_thw
|
| 722 |
+
# See https://github.com/huggingface/transformers/pull/34852 for more information
|
| 723 |
+
dtype=grid_thw.dtype if torch.jit.is_tracing() else torch.int32,
|
| 724 |
+
)
|
| 725 |
+
cu_seqlens = F.pad(cu_seqlens, (1, 0), value=0)
|
| 726 |
+
|
| 727 |
+
deepstack_feature_lists = []
|
| 728 |
+
for layer_num, blk in enumerate(self.blocks):
|
| 729 |
+
hidden_states = blk(
|
| 730 |
+
hidden_states,
|
| 731 |
+
cu_seqlens=cu_seqlens,
|
| 732 |
+
position_embeddings=position_embeddings,
|
| 733 |
+
**kwargs,
|
| 734 |
+
)
|
| 735 |
+
if layer_num in self.deepstack_visual_indexes:
|
| 736 |
+
deepstack_feature = self.deepstack_merger_list[self.deepstack_visual_indexes.index(layer_num)](
|
| 737 |
+
hidden_states
|
| 738 |
+
)
|
| 739 |
+
deepstack_feature_lists.append(deepstack_feature)
|
| 740 |
+
|
| 741 |
+
hidden_states = self.merger(hidden_states)
|
| 742 |
+
|
| 743 |
+
return hidden_states, deepstack_feature_lists
|
| 744 |
+
|
| 745 |
+
|
| 746 |
+
@auto_docstring(
|
| 747 |
+
custom_intro=(
|
| 748 |
+
"Text part of InternVideo3, "
|
| 749 |
+
"not a pure text-only model, as DeepStack integrates visual features into the early hidden states."
|
| 750 |
+
)
|
| 751 |
+
)
|
| 752 |
+
class InternVideo3TextModel(InternVideo3PreTrainedModel):
|
| 753 |
+
config: InternVideo3TextConfig
|
| 754 |
+
_no_split_modules = ["InternVideo3TextDecoderLayer"]
|
| 755 |
+
|
| 756 |
+
def __init__(self, config: InternVideo3TextConfig):
|
| 757 |
+
super().__init__(config)
|
| 758 |
+
self.padding_idx = config.pad_token_id
|
| 759 |
+
self.vocab_size = config.vocab_size
|
| 760 |
+
|
| 761 |
+
self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx)
|
| 762 |
+
self.layers = nn.ModuleList(
|
| 763 |
+
[InternVideo3TextDecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)]
|
| 764 |
+
)
|
| 765 |
+
self.norm = InternVideo3TextRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
| 766 |
+
self.rotary_emb = InternVideo3TextRotaryEmbedding(config=config)
|
| 767 |
+
self.gradient_checkpointing = False
|
| 768 |
+
|
| 769 |
+
# Initialize weights and apply final processing
|
| 770 |
+
self.post_init()
|
| 771 |
+
|
| 772 |
+
@check_model_inputs()
|
| 773 |
+
@auto_docstring
|
| 774 |
+
def forward(
|
| 775 |
+
self,
|
| 776 |
+
input_ids: Optional[torch.LongTensor] = None,
|
| 777 |
+
attention_mask: Optional[torch.Tensor] = None,
|
| 778 |
+
position_ids: Optional[torch.LongTensor] = None,
|
| 779 |
+
past_key_values: Optional[Cache] = None,
|
| 780 |
+
inputs_embeds: Optional[torch.FloatTensor] = None,
|
| 781 |
+
use_cache: Optional[bool] = None,
|
| 782 |
+
cache_position: Optional[torch.LongTensor] = None,
|
| 783 |
+
# args for deepstack
|
| 784 |
+
visual_pos_masks: Optional[torch.Tensor] = None,
|
| 785 |
+
deepstack_visual_embeds: Optional[list[torch.Tensor]] = None,
|
| 786 |
+
**kwargs: Unpack[FlashAttentionKwargs],
|
| 787 |
+
) -> Union[tuple, BaseModelOutputWithPast]:
|
| 788 |
+
r"""
|
| 789 |
+
visual_pos_masks (`torch.Tensor` of shape `(batch_size, seqlen)`, *optional*):
|
| 790 |
+
The mask of the visual positions.
|
| 791 |
+
deepstack_visual_embeds (`list[torch.Tensor]`, *optional*):
|
| 792 |
+
The deepstack visual embeddings. The shape is (num_layers, visual_seqlen, embed_dim).
|
| 793 |
+
The feature is extracted from the different visual encoder layers, and fed to the decoder
|
| 794 |
+
hidden states. It's from the paper DeepStack(https://arxiv.org/abs/2406.04334).
|
| 795 |
+
"""
|
| 796 |
+
if (input_ids is None) ^ (inputs_embeds is not None):
|
| 797 |
+
raise ValueError("You must specify exactly one of input_ids or inputs_embeds")
|
| 798 |
+
|
| 799 |
+
# torch.jit.trace() doesn't support cache objects in the output
|
| 800 |
+
if use_cache and past_key_values is None and not torch.jit.is_tracing():
|
| 801 |
+
past_key_values = DynamicCache(config=self.config)
|
| 802 |
+
|
| 803 |
+
if inputs_embeds is None:
|
| 804 |
+
inputs_embeds = self.embed_tokens(input_ids)
|
| 805 |
+
|
| 806 |
+
if cache_position is None:
|
| 807 |
+
past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0
|
| 808 |
+
cache_position = torch.arange(
|
| 809 |
+
past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device
|
| 810 |
+
)
|
| 811 |
+
|
| 812 |
+
# the hard coded `3` is for temporal, height and width.
|
| 813 |
+
if position_ids is None:
|
| 814 |
+
position_ids = cache_position.view(1, 1, -1).expand(3, inputs_embeds.shape[0], -1)
|
| 815 |
+
elif position_ids.ndim == 2:
|
| 816 |
+
position_ids = position_ids[None, ...].expand(3, position_ids.shape[0], -1)
|
| 817 |
+
|
| 818 |
+
if position_ids.ndim == 3 and position_ids.shape[0] == 4:
|
| 819 |
+
text_position_ids = position_ids[0]
|
| 820 |
+
position_ids = position_ids[1:]
|
| 821 |
+
else:
|
| 822 |
+
text_position_ids = position_ids[0]
|
| 823 |
+
|
| 824 |
+
attention_mask = create_causal_mask(
|
| 825 |
+
config=self.config,
|
| 826 |
+
input_embeds=inputs_embeds,
|
| 827 |
+
attention_mask=attention_mask,
|
| 828 |
+
cache_position=cache_position,
|
| 829 |
+
past_key_values=past_key_values,
|
| 830 |
+
position_ids=text_position_ids,
|
| 831 |
+
)
|
| 832 |
+
|
| 833 |
+
hidden_states = inputs_embeds
|
| 834 |
+
|
| 835 |
+
# create position embeddings to be shared across the decoder layers
|
| 836 |
+
position_embeddings = self.rotary_emb(hidden_states, position_ids)
|
| 837 |
+
|
| 838 |
+
# decoder layers
|
| 839 |
+
for layer_idx, decoder_layer in enumerate(self.layers):
|
| 840 |
+
layer_outputs = decoder_layer(
|
| 841 |
+
hidden_states,
|
| 842 |
+
attention_mask=attention_mask,
|
| 843 |
+
position_ids=text_position_ids,
|
| 844 |
+
past_key_values=past_key_values,
|
| 845 |
+
cache_position=cache_position,
|
| 846 |
+
position_embeddings=position_embeddings,
|
| 847 |
+
**kwargs,
|
| 848 |
+
)
|
| 849 |
+
hidden_states = layer_outputs
|
| 850 |
+
|
| 851 |
+
# add visual features to the hidden states of first several layers
|
| 852 |
+
if deepstack_visual_embeds is not None and layer_idx in range(len(deepstack_visual_embeds)):
|
| 853 |
+
hidden_states = self._deepstack_process(
|
| 854 |
+
hidden_states,
|
| 855 |
+
visual_pos_masks,
|
| 856 |
+
deepstack_visual_embeds[layer_idx],
|
| 857 |
+
)
|
| 858 |
+
|
| 859 |
+
hidden_states = self.norm(hidden_states)
|
| 860 |
+
|
| 861 |
+
return BaseModelOutputWithPast(
|
| 862 |
+
last_hidden_state=hidden_states,
|
| 863 |
+
past_key_values=past_key_values,
|
| 864 |
+
)
|
| 865 |
+
|
| 866 |
+
def _deepstack_process(
|
| 867 |
+
self, hidden_states: torch.Tensor, visual_pos_masks: torch.Tensor, visual_embeds: torch.Tensor
|
| 868 |
+
):
|
| 869 |
+
visual_pos_masks = visual_pos_masks.to(hidden_states.device)
|
| 870 |
+
visual_embeds = visual_embeds.to(hidden_states.device, hidden_states.dtype)
|
| 871 |
+
local_this = hidden_states[visual_pos_masks, :].clone() + visual_embeds
|
| 872 |
+
hidden_states[visual_pos_masks, :] = local_this
|
| 873 |
+
return hidden_states
|
| 874 |
+
|
| 875 |
+
|
| 876 |
+
@auto_docstring
|
| 877 |
+
class InternVideo3Model(InternVideo3PreTrainedModel):
|
| 878 |
+
base_model_prefix = ""
|
| 879 |
+
_checkpoint_conversion_mapping = {}
|
| 880 |
+
# Reference: fix gemma3 grad acc #37208
|
| 881 |
+
accepts_loss_kwargs = False
|
| 882 |
+
config: InternVideo3Config
|
| 883 |
+
_no_split_modules = ["InternVideo3TextDecoderLayer", "InternVideo3VisionBlock"]
|
| 884 |
+
|
| 885 |
+
def __init__(self, config):
|
| 886 |
+
super().__init__(config)
|
| 887 |
+
self.visual = InternVideo3VisionModel._from_config(config.vision_config)
|
| 888 |
+
self.language_model = InternVideo3TextModel._from_config(config.text_config)
|
| 889 |
+
self.rope_deltas = None # cache rope_deltas here
|
| 890 |
+
|
| 891 |
+
# Initialize weights and apply final processing
|
| 892 |
+
self.post_init()
|
| 893 |
+
|
| 894 |
+
def get_input_embeddings(self):
|
| 895 |
+
return self.language_model.get_input_embeddings()
|
| 896 |
+
|
| 897 |
+
def set_input_embeddings(self, value):
|
| 898 |
+
self.language_model.set_input_embeddings(value)
|
| 899 |
+
|
| 900 |
+
def set_decoder(self, decoder):
|
| 901 |
+
self.language_model = decoder
|
| 902 |
+
|
| 903 |
+
def get_decoder(self):
|
| 904 |
+
return self.language_model
|
| 905 |
+
|
| 906 |
+
def get_rope_index(
|
| 907 |
+
self,
|
| 908 |
+
input_ids: Optional[torch.LongTensor] = None,
|
| 909 |
+
image_grid_thw: Optional[torch.LongTensor] = None,
|
| 910 |
+
video_grid_thw: Optional[torch.LongTensor] = None,
|
| 911 |
+
attention_mask: Optional[torch.Tensor] = None,
|
| 912 |
+
) -> tuple[torch.Tensor, torch.Tensor]:
|
| 913 |
+
"""Different from the original implementation, InternVideo3 use timestamps rather than absolute time position ids."""
|
| 914 |
+
|
| 915 |
+
# Since we use timestamps to seperate videos, like <t1> <vision_start> <frame1> <vision_end> <t2> <vision_start> <frame2> <vision_end>, the video_grid_thw should also be split
|
| 916 |
+
if video_grid_thw is not None:
|
| 917 |
+
video_grid_thw = torch.repeat_interleave(video_grid_thw, video_grid_thw[:, 0], dim=0)
|
| 918 |
+
video_grid_thw[:, 0] = 1
|
| 919 |
+
|
| 920 |
+
spatial_merge_size = self.config.vision_config.spatial_merge_size
|
| 921 |
+
image_token_id = self.config.image_token_id
|
| 922 |
+
video_token_id = self.config.video_token_id
|
| 923 |
+
vision_start_token_id = self.config.vision_start_token_id
|
| 924 |
+
mrope_position_deltas = []
|
| 925 |
+
if input_ids is not None and (image_grid_thw is not None or video_grid_thw is not None):
|
| 926 |
+
total_input_ids = input_ids
|
| 927 |
+
if attention_mask is None:
|
| 928 |
+
attention_mask = torch.ones_like(total_input_ids)
|
| 929 |
+
position_ids = torch.ones(
|
| 930 |
+
3,
|
| 931 |
+
input_ids.shape[0],
|
| 932 |
+
input_ids.shape[1],
|
| 933 |
+
dtype=input_ids.dtype,
|
| 934 |
+
device=input_ids.device,
|
| 935 |
+
)
|
| 936 |
+
image_index, video_index = 0, 0
|
| 937 |
+
attention_mask = attention_mask.to(total_input_ids.device)
|
| 938 |
+
for i, input_ids in enumerate(total_input_ids):
|
| 939 |
+
input_ids = input_ids[attention_mask[i] == 1]
|
| 940 |
+
image_nums, video_nums = 0, 0
|
| 941 |
+
vision_start_indices = torch.argwhere(input_ids == vision_start_token_id).squeeze(1)
|
| 942 |
+
vision_tokens = input_ids[vision_start_indices + 1]
|
| 943 |
+
image_nums = (vision_tokens == image_token_id).sum()
|
| 944 |
+
video_nums = (vision_tokens == video_token_id).sum()
|
| 945 |
+
input_tokens = input_ids.tolist()
|
| 946 |
+
llm_pos_ids_list: list = []
|
| 947 |
+
st = 0
|
| 948 |
+
remain_images, remain_videos = image_nums, video_nums
|
| 949 |
+
for _ in range(image_nums + video_nums):
|
| 950 |
+
if image_token_id in input_tokens and remain_images > 0:
|
| 951 |
+
ed_image = input_tokens.index(image_token_id, st)
|
| 952 |
+
else:
|
| 953 |
+
ed_image = len(input_tokens) + 1
|
| 954 |
+
if video_token_id in input_tokens and remain_videos > 0:
|
| 955 |
+
ed_video = input_tokens.index(video_token_id, st)
|
| 956 |
+
else:
|
| 957 |
+
ed_video = len(input_tokens) + 1
|
| 958 |
+
if ed_image < ed_video:
|
| 959 |
+
t, h, w = (
|
| 960 |
+
image_grid_thw[image_index][0],
|
| 961 |
+
image_grid_thw[image_index][1],
|
| 962 |
+
image_grid_thw[image_index][2],
|
| 963 |
+
)
|
| 964 |
+
image_index += 1
|
| 965 |
+
remain_images -= 1
|
| 966 |
+
ed = ed_image
|
| 967 |
+
|
| 968 |
+
else:
|
| 969 |
+
t, h, w = (
|
| 970 |
+
video_grid_thw[video_index][0],
|
| 971 |
+
video_grid_thw[video_index][1],
|
| 972 |
+
video_grid_thw[video_index][2],
|
| 973 |
+
)
|
| 974 |
+
video_index += 1
|
| 975 |
+
remain_videos -= 1
|
| 976 |
+
ed = ed_video
|
| 977 |
+
llm_grid_t, llm_grid_h, llm_grid_w = (
|
| 978 |
+
t.item(),
|
| 979 |
+
h.item() // spatial_merge_size,
|
| 980 |
+
w.item() // spatial_merge_size,
|
| 981 |
+
)
|
| 982 |
+
text_len = ed - st
|
| 983 |
+
|
| 984 |
+
st_idx = llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0
|
| 985 |
+
llm_pos_ids_list.append(torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx)
|
| 986 |
+
|
| 987 |
+
# t_index is always 0 because llm_grid_t is always 1 (we use timestamps to encode the temporal information for videos)
|
| 988 |
+
t_index = torch.arange(llm_grid_t).view(-1, 1).expand(-1, llm_grid_h * llm_grid_w).flatten()
|
| 989 |
+
h_index = torch.arange(llm_grid_h).view(1, -1, 1).expand(llm_grid_t, -1, llm_grid_w).flatten()
|
| 990 |
+
w_index = torch.arange(llm_grid_w).view(1, 1, -1).expand(llm_grid_t, llm_grid_h, -1).flatten()
|
| 991 |
+
llm_pos_ids_list.append(torch.stack([t_index, h_index, w_index]) + text_len + st_idx)
|
| 992 |
+
st = ed + llm_grid_t * llm_grid_h * llm_grid_w
|
| 993 |
+
|
| 994 |
+
if st < len(input_tokens):
|
| 995 |
+
st_idx = llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0
|
| 996 |
+
text_len = len(input_tokens) - st
|
| 997 |
+
llm_pos_ids_list.append(torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx)
|
| 998 |
+
|
| 999 |
+
llm_positions = torch.cat(llm_pos_ids_list, dim=1).reshape(3, -1)
|
| 1000 |
+
position_ids[..., i, attention_mask[i] == 1] = llm_positions.to(position_ids.device)
|
| 1001 |
+
mrope_position_deltas.append(llm_positions.max() + 1 - len(total_input_ids[i]))
|
| 1002 |
+
mrope_position_deltas = torch.tensor(mrope_position_deltas, device=input_ids.device).unsqueeze(1)
|
| 1003 |
+
return position_ids, mrope_position_deltas
|
| 1004 |
+
else:
|
| 1005 |
+
if attention_mask is not None:
|
| 1006 |
+
position_ids = attention_mask.long().cumsum(-1) - 1
|
| 1007 |
+
position_ids.masked_fill_(attention_mask == 0, 1)
|
| 1008 |
+
position_ids = position_ids.unsqueeze(0).expand(3, -1, -1).to(attention_mask.device)
|
| 1009 |
+
max_position_ids = position_ids.max(0, keepdim=False)[0].max(-1, keepdim=True)[0]
|
| 1010 |
+
mrope_position_deltas = max_position_ids + 1 - attention_mask.shape[-1]
|
| 1011 |
+
else:
|
| 1012 |
+
position_ids = (
|
| 1013 |
+
torch.arange(input_ids.shape[1], device=input_ids.device)
|
| 1014 |
+
.view(1, 1, -1)
|
| 1015 |
+
.expand(3, input_ids.shape[0], -1)
|
| 1016 |
+
)
|
| 1017 |
+
mrope_position_deltas = torch.zeros(
|
| 1018 |
+
[input_ids.shape[0], 1],
|
| 1019 |
+
device=input_ids.device,
|
| 1020 |
+
dtype=input_ids.dtype,
|
| 1021 |
+
)
|
| 1022 |
+
|
| 1023 |
+
return position_ids, mrope_position_deltas
|
| 1024 |
+
|
| 1025 |
+
def get_video_features(
|
| 1026 |
+
self, pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None
|
| 1027 |
+
):
|
| 1028 |
+
"""
|
| 1029 |
+
Encodes videos into continuous embeddings that can be forwarded to the language model. The deepstack visual features are also returned.
|
| 1030 |
+
|
| 1031 |
+
Args:
|
| 1032 |
+
pixel_values_videos (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`):
|
| 1033 |
+
The tensors corresponding to the input videos.
|
| 1034 |
+
video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*):
|
| 1035 |
+
The temporal, height and width of feature shape of each video in LLM.
|
| 1036 |
+
"""
|
| 1037 |
+
# Same implementation as for images
|
| 1038 |
+
return self.get_image_features(pixel_values_videos, video_grid_thw)
|
| 1039 |
+
|
| 1040 |
+
def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None):
|
| 1041 |
+
"""
|
| 1042 |
+
Encodes images into continuous embeddings that can be forwarded to the language model. The deepstack visual features are also returned.
|
| 1043 |
+
|
| 1044 |
+
Args:
|
| 1045 |
+
pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`):
|
| 1046 |
+
The tensors corresponding to the input images.
|
| 1047 |
+
image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*):
|
| 1048 |
+
The temporal, height and width of feature shape of each image in LLM.
|
| 1049 |
+
"""
|
| 1050 |
+
pixel_values = pixel_values.type(self.visual.dtype)
|
| 1051 |
+
image_embeds, deepstack_image_embeds = self.visual(pixel_values, grid_thw=image_grid_thw)
|
| 1052 |
+
split_sizes = (image_grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist()
|
| 1053 |
+
image_embeds = torch.split(image_embeds, split_sizes)
|
| 1054 |
+
return image_embeds, deepstack_image_embeds
|
| 1055 |
+
|
| 1056 |
+
def get_placeholder_mask(
|
| 1057 |
+
self,
|
| 1058 |
+
input_ids: torch.LongTensor,
|
| 1059 |
+
inputs_embeds: torch.FloatTensor,
|
| 1060 |
+
image_features: Optional[torch.FloatTensor] = None,
|
| 1061 |
+
video_features: Optional[torch.FloatTensor] = None,
|
| 1062 |
+
):
|
| 1063 |
+
"""
|
| 1064 |
+
Obtains multimodal placeholder mask from `input_ids` or `inputs_embeds`, and checks that the placeholder token count is
|
| 1065 |
+
equal to the length of multimodal features. If the lengths are different, an error is raised.
|
| 1066 |
+
"""
|
| 1067 |
+
if input_ids is None:
|
| 1068 |
+
special_image_mask = inputs_embeds == self.get_input_embeddings()(
|
| 1069 |
+
torch.tensor(self.config.image_token_id, dtype=torch.long, device=inputs_embeds.device)
|
| 1070 |
+
)
|
| 1071 |
+
special_image_mask = special_image_mask.all(-1)
|
| 1072 |
+
special_video_mask = inputs_embeds == self.get_input_embeddings()(
|
| 1073 |
+
torch.tensor(self.config.video_token_id, dtype=torch.long, device=inputs_embeds.device)
|
| 1074 |
+
)
|
| 1075 |
+
special_video_mask = special_video_mask.all(-1)
|
| 1076 |
+
else:
|
| 1077 |
+
special_image_mask = input_ids == self.config.image_token_id
|
| 1078 |
+
special_video_mask = input_ids == self.config.video_token_id
|
| 1079 |
+
|
| 1080 |
+
n_image_tokens = special_image_mask.sum()
|
| 1081 |
+
special_image_mask = special_image_mask.unsqueeze(-1).expand_as(inputs_embeds).to(inputs_embeds.device)
|
| 1082 |
+
if image_features is not None and inputs_embeds[special_image_mask].numel() != image_features.numel():
|
| 1083 |
+
raise ValueError(
|
| 1084 |
+
f"Image features and image tokens do not match: tokens: {n_image_tokens}, features {image_features.shape[0]}"
|
| 1085 |
+
)
|
| 1086 |
+
|
| 1087 |
+
n_video_tokens = special_video_mask.sum()
|
| 1088 |
+
special_video_mask = special_video_mask.unsqueeze(-1).expand_as(inputs_embeds).to(inputs_embeds.device)
|
| 1089 |
+
if video_features is not None and inputs_embeds[special_video_mask].numel() != video_features.numel():
|
| 1090 |
+
raise ValueError(
|
| 1091 |
+
f"Videos features and video tokens do not match: tokens: {n_video_tokens}, features {video_features.shape[0]}"
|
| 1092 |
+
)
|
| 1093 |
+
|
| 1094 |
+
return special_image_mask, special_video_mask
|
| 1095 |
+
|
| 1096 |
+
@auto_docstring
|
| 1097 |
+
@check_model_inputs()
|
| 1098 |
+
def forward(
|
| 1099 |
+
self,
|
| 1100 |
+
input_ids: torch.LongTensor = None,
|
| 1101 |
+
attention_mask: Optional[torch.Tensor] = None,
|
| 1102 |
+
position_ids: Optional[torch.LongTensor] = None,
|
| 1103 |
+
past_key_values: Optional[Cache] = None,
|
| 1104 |
+
inputs_embeds: Optional[torch.FloatTensor] = None,
|
| 1105 |
+
pixel_values: Optional[torch.Tensor] = None,
|
| 1106 |
+
pixel_values_videos: Optional[torch.FloatTensor] = None,
|
| 1107 |
+
image_grid_thw: Optional[torch.LongTensor] = None,
|
| 1108 |
+
video_grid_thw: Optional[torch.LongTensor] = None,
|
| 1109 |
+
cache_position: Optional[torch.LongTensor] = None,
|
| 1110 |
+
**kwargs: Unpack[TransformersKwargs],
|
| 1111 |
+
) -> Union[tuple, InternVideo3ModelOutputWithPast]:
|
| 1112 |
+
r"""
|
| 1113 |
+
image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*):
|
| 1114 |
+
The temporal, height and width of feature shape of each image in LLM.
|
| 1115 |
+
video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*):
|
| 1116 |
+
The temporal, height and width of feature shape of each video in LLM.
|
| 1117 |
+
"""
|
| 1118 |
+
if (input_ids is None) ^ (inputs_embeds is not None):
|
| 1119 |
+
raise ValueError("You must specify exactly one of input_ids or inputs_embeds")
|
| 1120 |
+
|
| 1121 |
+
if inputs_embeds is None:
|
| 1122 |
+
inputs_embeds = self.get_input_embeddings()(input_ids)
|
| 1123 |
+
|
| 1124 |
+
image_mask = None
|
| 1125 |
+
video_mask = None
|
| 1126 |
+
|
| 1127 |
+
if pixel_values is not None:
|
| 1128 |
+
image_embeds, deepstack_image_embeds = self.get_image_features(pixel_values, image_grid_thw)
|
| 1129 |
+
image_embeds = torch.cat(image_embeds, dim=0).to(inputs_embeds.device, inputs_embeds.dtype)
|
| 1130 |
+
image_mask, _ = self.get_placeholder_mask(
|
| 1131 |
+
input_ids, inputs_embeds=inputs_embeds, image_features=image_embeds
|
| 1132 |
+
)
|
| 1133 |
+
inputs_embeds = inputs_embeds.masked_scatter(image_mask, image_embeds)
|
| 1134 |
+
|
| 1135 |
+
if pixel_values_videos is not None:
|
| 1136 |
+
video_embeds, deepstack_video_embeds = self.get_video_features(pixel_values_videos, video_grid_thw)
|
| 1137 |
+
video_embeds = torch.cat(video_embeds, dim=0).to(inputs_embeds.device, inputs_embeds.dtype)
|
| 1138 |
+
_, video_mask = self.get_placeholder_mask(
|
| 1139 |
+
input_ids, inputs_embeds=inputs_embeds, video_features=video_embeds
|
| 1140 |
+
)
|
| 1141 |
+
inputs_embeds = inputs_embeds.masked_scatter(video_mask, video_embeds)
|
| 1142 |
+
|
| 1143 |
+
visual_pos_masks = None
|
| 1144 |
+
deepstack_visual_embeds = None
|
| 1145 |
+
if image_mask is not None and video_mask is not None:
|
| 1146 |
+
# aggregate visual_pos_masks and deepstack_visual_embeds
|
| 1147 |
+
image_mask = image_mask[..., 0]
|
| 1148 |
+
video_mask = video_mask[..., 0]
|
| 1149 |
+
visual_pos_masks = image_mask | video_mask
|
| 1150 |
+
deepstack_visual_embeds = []
|
| 1151 |
+
image_mask_joint = image_mask[visual_pos_masks]
|
| 1152 |
+
video_mask_joint = video_mask[visual_pos_masks]
|
| 1153 |
+
for img_embed, vid_embed in zip(deepstack_image_embeds, deepstack_video_embeds):
|
| 1154 |
+
embed_joint = img_embed.new_zeros(visual_pos_masks.sum(), img_embed.shape[-1]).to(img_embed.device)
|
| 1155 |
+
embed_joint[image_mask_joint, :] = img_embed
|
| 1156 |
+
embed_joint[video_mask_joint, :] = vid_embed
|
| 1157 |
+
deepstack_visual_embeds.append(embed_joint)
|
| 1158 |
+
elif image_mask is not None:
|
| 1159 |
+
image_mask = image_mask[..., 0]
|
| 1160 |
+
visual_pos_masks = image_mask
|
| 1161 |
+
deepstack_visual_embeds = deepstack_image_embeds
|
| 1162 |
+
elif video_mask is not None:
|
| 1163 |
+
video_mask = video_mask[..., 0]
|
| 1164 |
+
visual_pos_masks = video_mask
|
| 1165 |
+
deepstack_visual_embeds = deepstack_video_embeds
|
| 1166 |
+
|
| 1167 |
+
if position_ids is None:
|
| 1168 |
+
attention_mask_tensor = (
|
| 1169 |
+
attention_mask if not isinstance(attention_mask, dict) else attention_mask["full_attention"]
|
| 1170 |
+
)
|
| 1171 |
+
if attention_mask_tensor is not None and attention_mask_tensor.ndim == 4:
|
| 1172 |
+
attention_mask_tensor = torch.diagonal(attention_mask_tensor[:, 0], dim1=1, dim2=2)
|
| 1173 |
+
# Only apply conversion for floating point tensors (inverted masks)
|
| 1174 |
+
if attention_mask_tensor.dtype.is_floating_point:
|
| 1175 |
+
attention_mask_tensor = attention_mask_tensor / torch.finfo(attention_mask_tensor.dtype).min
|
| 1176 |
+
attention_mask_tensor = (1.0 - attention_mask_tensor).int()
|
| 1177 |
+
|
| 1178 |
+
# Calculate RoPE index once per generation in the pre-fill stage only.
|
| 1179 |
+
# When compiling, we can't check tensor values thus we check only input length
|
| 1180 |
+
# It is safe to assume that `length!=1` means we're in pre-fill because compiled
|
| 1181 |
+
# models currently cannot do asssisted decoding
|
| 1182 |
+
prefill_compiled_stage = is_torchdynamo_compiling() and (
|
| 1183 |
+
(input_ids is not None and input_ids.shape[1] != 1)
|
| 1184 |
+
or (inputs_embeds is not None and inputs_embeds.shape[1] != 1)
|
| 1185 |
+
)
|
| 1186 |
+
prefill_noncompiled_stage = not is_torchdynamo_compiling() and (
|
| 1187 |
+
(cache_position is not None and cache_position[0] == 0)
|
| 1188 |
+
or (past_key_values is None or past_key_values.get_seq_length() == 0)
|
| 1189 |
+
)
|
| 1190 |
+
if (prefill_compiled_stage or prefill_noncompiled_stage) or self.rope_deltas is None:
|
| 1191 |
+
position_ids, rope_deltas = self.get_rope_index(
|
| 1192 |
+
input_ids,
|
| 1193 |
+
image_grid_thw,
|
| 1194 |
+
video_grid_thw,
|
| 1195 |
+
attention_mask=attention_mask_tensor,
|
| 1196 |
+
)
|
| 1197 |
+
self.rope_deltas = rope_deltas
|
| 1198 |
+
# then use the prev pre-calculated rope-deltas to get the correct position ids
|
| 1199 |
+
else:
|
| 1200 |
+
batch_size, seq_length, _ = inputs_embeds.shape
|
| 1201 |
+
delta = (
|
| 1202 |
+
(cache_position[0] + self.rope_deltas).to(inputs_embeds.device)
|
| 1203 |
+
if cache_position is not None
|
| 1204 |
+
else 0
|
| 1205 |
+
)
|
| 1206 |
+
position_ids = torch.arange(seq_length, device=inputs_embeds.device)
|
| 1207 |
+
position_ids = position_ids.view(1, -1).expand(batch_size, -1)
|
| 1208 |
+
if cache_position is not None: # otherwise `deltas` is an int `0`
|
| 1209 |
+
delta = delta.repeat_interleave(batch_size // delta.shape[0], dim=0)
|
| 1210 |
+
position_ids = position_ids.add(delta)
|
| 1211 |
+
position_ids = position_ids.unsqueeze(0).expand(3, -1, -1)
|
| 1212 |
+
|
| 1213 |
+
outputs = self.language_model(
|
| 1214 |
+
input_ids=None,
|
| 1215 |
+
position_ids=position_ids,
|
| 1216 |
+
attention_mask=attention_mask,
|
| 1217 |
+
past_key_values=past_key_values,
|
| 1218 |
+
inputs_embeds=inputs_embeds,
|
| 1219 |
+
cache_position=cache_position,
|
| 1220 |
+
visual_pos_masks=visual_pos_masks,
|
| 1221 |
+
deepstack_visual_embeds=deepstack_visual_embeds,
|
| 1222 |
+
**kwargs,
|
| 1223 |
+
)
|
| 1224 |
+
|
| 1225 |
+
return InternVideo3ModelOutputWithPast(
|
| 1226 |
+
last_hidden_state=outputs.last_hidden_state,
|
| 1227 |
+
past_key_values=outputs.past_key_values,
|
| 1228 |
+
rope_deltas=self.rope_deltas,
|
| 1229 |
+
)
|
| 1230 |
+
|
| 1231 |
+
|
| 1232 |
+
@dataclass
|
| 1233 |
+
@auto_docstring(
|
| 1234 |
+
custom_intro="""
|
| 1235 |
+
Base class for InternVideo3 causal language model (or autoregressive) outputs.
|
| 1236 |
+
"""
|
| 1237 |
+
)
|
| 1238 |
+
class InternVideo3CausalLMOutputWithPast(ModelOutput):
|
| 1239 |
+
r"""
|
| 1240 |
+
loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
|
| 1241 |
+
Language modeling loss (for next-token prediction).
|
| 1242 |
+
logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
|
| 1243 |
+
Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
|
| 1244 |
+
past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
|
| 1245 |
+
It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).
|
| 1246 |
+
|
| 1247 |
+
Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
|
| 1248 |
+
`past_key_values` input) to speed up sequential decoding.
|
| 1249 |
+
rope_deltas (`torch.LongTensor` of shape `(batch_size, )`, *optional*):
|
| 1250 |
+
The rope index difference between sequence length and multimodal rope.
|
| 1251 |
+
"""
|
| 1252 |
+
|
| 1253 |
+
loss: Optional[torch.FloatTensor] = None
|
| 1254 |
+
logits: Optional[torch.FloatTensor] = None
|
| 1255 |
+
past_key_values: Optional[Cache] = None
|
| 1256 |
+
hidden_states: Optional[tuple[torch.FloatTensor]] = None
|
| 1257 |
+
attentions: Optional[tuple[torch.FloatTensor]] = None
|
| 1258 |
+
rope_deltas: Optional[torch.LongTensor] = None
|
| 1259 |
+
|
| 1260 |
+
|
| 1261 |
+
class InternVideo3ForConditionalGeneration(InternVideo3PreTrainedModel, GenerationMixin):
|
| 1262 |
+
_checkpoint_conversion_mapping = {}
|
| 1263 |
+
_tied_weights_keys = ["lm_head.weight"]
|
| 1264 |
+
# Reference: fix gemma3 grad acc #37208
|
| 1265 |
+
accepts_loss_kwargs = False
|
| 1266 |
+
config: InternVideo3Config
|
| 1267 |
+
|
| 1268 |
+
def __init__(self, config):
|
| 1269 |
+
super().__init__(config)
|
| 1270 |
+
self.model = InternVideo3Model(config)
|
| 1271 |
+
self.lm_head = nn.Linear(config.text_config.hidden_size, config.text_config.vocab_size, bias=False)
|
| 1272 |
+
|
| 1273 |
+
self.post_init()
|
| 1274 |
+
|
| 1275 |
+
def get_input_embeddings(self):
|
| 1276 |
+
return self.model.get_input_embeddings()
|
| 1277 |
+
|
| 1278 |
+
def set_input_embeddings(self, value):
|
| 1279 |
+
self.model.set_input_embeddings(value)
|
| 1280 |
+
|
| 1281 |
+
def set_decoder(self, decoder):
|
| 1282 |
+
self.model.set_decoder(decoder)
|
| 1283 |
+
|
| 1284 |
+
def get_decoder(self):
|
| 1285 |
+
return self.model.get_decoder()
|
| 1286 |
+
|
| 1287 |
+
def get_video_features(
|
| 1288 |
+
self, pixel_values_videos: torch.FloatTensor, video_grid_thw: Optional[torch.LongTensor] = None
|
| 1289 |
+
):
|
| 1290 |
+
return self.model.get_video_features(pixel_values_videos, video_grid_thw)
|
| 1291 |
+
|
| 1292 |
+
def get_image_features(self, pixel_values: torch.FloatTensor, image_grid_thw: Optional[torch.LongTensor] = None):
|
| 1293 |
+
return self.model.get_image_features(pixel_values, image_grid_thw)
|
| 1294 |
+
|
| 1295 |
+
# Make modules available through conditional class for BC
|
| 1296 |
+
@property
|
| 1297 |
+
def language_model(self):
|
| 1298 |
+
return self.model.language_model
|
| 1299 |
+
|
| 1300 |
+
@property
|
| 1301 |
+
def visual(self):
|
| 1302 |
+
return self.model.visual
|
| 1303 |
+
|
| 1304 |
+
@check_model_inputs()
|
| 1305 |
+
def forward(
|
| 1306 |
+
self,
|
| 1307 |
+
input_ids: torch.LongTensor = None,
|
| 1308 |
+
attention_mask: Optional[torch.Tensor] = None,
|
| 1309 |
+
position_ids: Optional[torch.LongTensor] = None,
|
| 1310 |
+
past_key_values: Optional[Cache] = None,
|
| 1311 |
+
inputs_embeds: Optional[torch.FloatTensor] = None,
|
| 1312 |
+
labels: Optional[torch.LongTensor] = None,
|
| 1313 |
+
pixel_values: Optional[torch.Tensor] = None,
|
| 1314 |
+
pixel_values_videos: Optional[torch.FloatTensor] = None,
|
| 1315 |
+
image_grid_thw: Optional[torch.LongTensor] = None,
|
| 1316 |
+
video_grid_thw: Optional[torch.LongTensor] = None,
|
| 1317 |
+
cache_position: Optional[torch.LongTensor] = None,
|
| 1318 |
+
logits_to_keep: Union[int, torch.Tensor] = 0,
|
| 1319 |
+
**kwargs: Unpack[TransformersKwargs],
|
| 1320 |
+
) -> Union[tuple, InternVideo3CausalLMOutputWithPast]:
|
| 1321 |
+
r"""
|
| 1322 |
+
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
|
| 1323 |
+
Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
|
| 1324 |
+
config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
|
| 1325 |
+
(masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
|
| 1326 |
+
image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*):
|
| 1327 |
+
The temporal, height and width of feature shape of each image in LLM.
|
| 1328 |
+
video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*):
|
| 1329 |
+
The temporal, height and width of feature shape of each video in LLM.
|
| 1330 |
+
|
| 1331 |
+
Example:
|
| 1332 |
+
TODO: Add example
|
| 1333 |
+
"""
|
| 1334 |
+
outputs = self.model(
|
| 1335 |
+
input_ids=input_ids,
|
| 1336 |
+
pixel_values=pixel_values,
|
| 1337 |
+
pixel_values_videos=pixel_values_videos,
|
| 1338 |
+
image_grid_thw=image_grid_thw,
|
| 1339 |
+
video_grid_thw=video_grid_thw,
|
| 1340 |
+
position_ids=position_ids,
|
| 1341 |
+
attention_mask=attention_mask,
|
| 1342 |
+
past_key_values=past_key_values,
|
| 1343 |
+
inputs_embeds=inputs_embeds,
|
| 1344 |
+
cache_position=cache_position,
|
| 1345 |
+
**kwargs,
|
| 1346 |
+
)
|
| 1347 |
+
|
| 1348 |
+
hidden_states = outputs[0]
|
| 1349 |
+
|
| 1350 |
+
# Only compute necessary logits, and do not upcast them to float if we are not computing the loss
|
| 1351 |
+
slice_indices = slice(-logits_to_keep, None) if isinstance(logits_to_keep, int) else logits_to_keep
|
| 1352 |
+
logits = self.lm_head(hidden_states[:, slice_indices, :])
|
| 1353 |
+
|
| 1354 |
+
loss = None
|
| 1355 |
+
if labels is not None:
|
| 1356 |
+
loss = self.loss_function(logits=logits, labels=labels, vocab_size=self.config.text_config.vocab_size)
|
| 1357 |
+
|
| 1358 |
+
return InternVideo3CausalLMOutputWithPast(
|
| 1359 |
+
loss=loss,
|
| 1360 |
+
logits=logits,
|
| 1361 |
+
past_key_values=outputs.past_key_values,
|
| 1362 |
+
rope_deltas=outputs.rope_deltas,
|
| 1363 |
+
)
|
| 1364 |
+
|
| 1365 |
+
def prepare_inputs_for_generation(
|
| 1366 |
+
self,
|
| 1367 |
+
input_ids,
|
| 1368 |
+
past_key_values=None,
|
| 1369 |
+
attention_mask=None,
|
| 1370 |
+
inputs_embeds=None,
|
| 1371 |
+
cache_position=None,
|
| 1372 |
+
position_ids=None,
|
| 1373 |
+
use_cache=True,
|
| 1374 |
+
pixel_values=None,
|
| 1375 |
+
pixel_values_videos=None,
|
| 1376 |
+
image_grid_thw=None,
|
| 1377 |
+
video_grid_thw=None,
|
| 1378 |
+
**kwargs,
|
| 1379 |
+
):
|
| 1380 |
+
# Overwritten -- in specific circumstances we don't want to forward image inputs to the model
|
| 1381 |
+
|
| 1382 |
+
model_inputs = super().prepare_inputs_for_generation(
|
| 1383 |
+
input_ids,
|
| 1384 |
+
past_key_values=past_key_values,
|
| 1385 |
+
attention_mask=attention_mask,
|
| 1386 |
+
inputs_embeds=inputs_embeds,
|
| 1387 |
+
cache_position=cache_position,
|
| 1388 |
+
position_ids=position_ids,
|
| 1389 |
+
pixel_values=pixel_values,
|
| 1390 |
+
pixel_values_videos=pixel_values_videos,
|
| 1391 |
+
image_grid_thw=image_grid_thw,
|
| 1392 |
+
video_grid_thw=video_grid_thw,
|
| 1393 |
+
use_cache=use_cache,
|
| 1394 |
+
**kwargs,
|
| 1395 |
+
)
|
| 1396 |
+
|
| 1397 |
+
# InternVideo3 position_ids are prepareed with rope_deltas in forward
|
| 1398 |
+
model_inputs["position_ids"] = None
|
| 1399 |
+
|
| 1400 |
+
if cache_position[0] != 0:
|
| 1401 |
+
model_inputs["pixel_values"] = None
|
| 1402 |
+
model_inputs["pixel_values_videos"] = None
|
| 1403 |
+
|
| 1404 |
+
return model_inputs
|
| 1405 |
+
|
| 1406 |
+
def _get_image_nums_and_video_nums(
|
| 1407 |
+
self,
|
| 1408 |
+
input_ids: Optional[torch.LongTensor],
|
| 1409 |
+
inputs_embeds: Optional[torch.Tensor] = None,
|
| 1410 |
+
) -> tuple[torch.Tensor, torch.Tensor]:
|
| 1411 |
+
"""
|
| 1412 |
+
Get the number of images and videos for each sample to calculate the separation length of the sample tensor.
|
| 1413 |
+
These parameters are not passed through the processor to avoid unpredictable impacts from interface modifications.
|
| 1414 |
+
|
| 1415 |
+
Args:
|
| 1416 |
+
input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
|
| 1417 |
+
Indices of input sequence tokens in the vocabulary.
|
| 1418 |
+
|
| 1419 |
+
Returns:
|
| 1420 |
+
image_nums (`torch.LongTensor` of shape `(batch_size, num_images_sample)`)
|
| 1421 |
+
video_nums (`torch.LongTensor` of shape `(batch_size, num_videos_sample)`)
|
| 1422 |
+
"""
|
| 1423 |
+
image_token_id = self.config.image_token_id
|
| 1424 |
+
video_token_id = self.config.video_token_id
|
| 1425 |
+
vision_start_token_id = self.config.vision_start_token_id
|
| 1426 |
+
|
| 1427 |
+
if inputs_embeds is not None:
|
| 1428 |
+
vision_start_mask = (
|
| 1429 |
+
inputs_embeds
|
| 1430 |
+
== self.get_input_embeddings()(
|
| 1431 |
+
torch.tensor(vision_start_token_id, dtype=torch.long, device=inputs_embeds.device)
|
| 1432 |
+
)
|
| 1433 |
+
)[..., 0]
|
| 1434 |
+
image_mask = (
|
| 1435 |
+
inputs_embeds
|
| 1436 |
+
== self.get_input_embeddings()(
|
| 1437 |
+
torch.tensor(image_token_id, dtype=torch.long, device=inputs_embeds.device)
|
| 1438 |
+
)
|
| 1439 |
+
)[..., 0]
|
| 1440 |
+
video_mask = (
|
| 1441 |
+
inputs_embeds
|
| 1442 |
+
== self.get_input_embeddings()(
|
| 1443 |
+
torch.tensor(video_token_id, dtype=torch.long, device=inputs_embeds.device)
|
| 1444 |
+
)
|
| 1445 |
+
)[..., 0]
|
| 1446 |
+
else:
|
| 1447 |
+
vision_start_mask = input_ids == vision_start_token_id
|
| 1448 |
+
image_mask = input_ids == image_token_id
|
| 1449 |
+
video_mask = input_ids == video_token_id
|
| 1450 |
+
|
| 1451 |
+
vision_first_mask = torch.roll(vision_start_mask, shifts=1, dims=1)
|
| 1452 |
+
image_nums = torch.sum(vision_first_mask & image_mask, dim=1)
|
| 1453 |
+
video_nums = torch.sum(vision_first_mask & video_mask, dim=1)
|
| 1454 |
+
|
| 1455 |
+
return image_nums, video_nums
|
| 1456 |
+
|
| 1457 |
+
def _expand_inputs_for_generation(
|
| 1458 |
+
self,
|
| 1459 |
+
expand_size: int = 1,
|
| 1460 |
+
is_encoder_decoder: bool = False,
|
| 1461 |
+
input_ids: Optional[torch.LongTensor] = None,
|
| 1462 |
+
**model_kwargs,
|
| 1463 |
+
) -> tuple[torch.LongTensor, dict[str, Any]]:
|
| 1464 |
+
# Overwritten -- Support for expanding tensors without a batch size dimension
|
| 1465 |
+
# e.g., pixel_values, image_grid_thw, pixel_values_videos, video_grid_thw, second_per_grid_t
|
| 1466 |
+
# pixel_values.shape[0] is sum(seqlen_images for samples)
|
| 1467 |
+
# image_grid_thw.shape[0] is sum(num_images for samples)
|
| 1468 |
+
|
| 1469 |
+
if expand_size == 1:
|
| 1470 |
+
return input_ids, model_kwargs
|
| 1471 |
+
|
| 1472 |
+
visual_keys = ["pixel_values", "image_grid_thw", "pixel_values_videos", "video_grid_thw", "second_per_grid_ts"]
|
| 1473 |
+
|
| 1474 |
+
def _expand_dict_for_generation_visual(dict_to_expand):
|
| 1475 |
+
image_grid_thw = model_kwargs.get("image_grid_thw", None)
|
| 1476 |
+
video_grid_thw = model_kwargs.get("video_grid_thw", None)
|
| 1477 |
+
image_nums, video_nums = self._get_image_nums_and_video_nums(
|
| 1478 |
+
input_ids, inputs_embeds=model_kwargs.get("inputs_embeds", None)
|
| 1479 |
+
)
|
| 1480 |
+
|
| 1481 |
+
def _repeat_interleave_samples(x, lengths, repeat_times):
|
| 1482 |
+
samples = torch.split(x, lengths)
|
| 1483 |
+
repeat_args = [repeat_times] + [1] * (x.dim() - 1)
|
| 1484 |
+
result = torch.cat([sample.repeat(*repeat_args) for sample in samples], dim=0)
|
| 1485 |
+
return result
|
| 1486 |
+
|
| 1487 |
+
for key in dict_to_expand:
|
| 1488 |
+
if key == "pixel_values":
|
| 1489 |
+
# split images into samples
|
| 1490 |
+
samples = torch.split(image_grid_thw, list(image_nums))
|
| 1491 |
+
# compute the sequence length of images for each sample
|
| 1492 |
+
lengths = [torch.prod(sample, dim=1).sum() for sample in samples]
|
| 1493 |
+
dict_to_expand[key] = _repeat_interleave_samples(
|
| 1494 |
+
dict_to_expand[key], lengths=lengths, repeat_times=expand_size
|
| 1495 |
+
)
|
| 1496 |
+
elif key == "image_grid_thw":
|
| 1497 |
+
# get the num of images for each sample
|
| 1498 |
+
lengths = list(image_nums)
|
| 1499 |
+
dict_to_expand[key] = _repeat_interleave_samples(
|
| 1500 |
+
dict_to_expand[key], lengths=lengths, repeat_times=expand_size
|
| 1501 |
+
)
|
| 1502 |
+
elif key == "pixel_values_videos":
|
| 1503 |
+
samples = torch.split(video_grid_thw, list(video_nums))
|
| 1504 |
+
lengths = [torch.prod(sample, dim=1).sum() for sample in samples]
|
| 1505 |
+
dict_to_expand[key] = _repeat_interleave_samples(
|
| 1506 |
+
dict_to_expand[key], lengths=lengths, repeat_times=expand_size
|
| 1507 |
+
)
|
| 1508 |
+
elif key == "video_grid_thw":
|
| 1509 |
+
lengths = list(video_nums)
|
| 1510 |
+
dict_to_expand[key] = _repeat_interleave_samples(
|
| 1511 |
+
dict_to_expand[key], lengths=lengths, repeat_times=expand_size
|
| 1512 |
+
)
|
| 1513 |
+
elif key == "second_per_grid_ts":
|
| 1514 |
+
dict_to_expand[key] = _repeat_interleave_samples(
|
| 1515 |
+
dict_to_expand[key], lengths=list(video_nums), repeat_times=expand_size
|
| 1516 |
+
)
|
| 1517 |
+
return dict_to_expand
|
| 1518 |
+
|
| 1519 |
+
def _expand_dict_for_generation(dict_to_expand):
|
| 1520 |
+
for key in dict_to_expand:
|
| 1521 |
+
if (
|
| 1522 |
+
key != "cache_position"
|
| 1523 |
+
and dict_to_expand[key] is not None
|
| 1524 |
+
and isinstance(dict_to_expand[key], torch.Tensor)
|
| 1525 |
+
and key not in visual_keys
|
| 1526 |
+
):
|
| 1527 |
+
dict_to_expand[key] = dict_to_expand[key].repeat_interleave(expand_size, dim=0)
|
| 1528 |
+
return dict_to_expand
|
| 1529 |
+
|
| 1530 |
+
model_kwargs = _expand_dict_for_generation_visual(model_kwargs)
|
| 1531 |
+
|
| 1532 |
+
if input_ids is not None:
|
| 1533 |
+
input_ids = input_ids.repeat_interleave(expand_size, dim=0)
|
| 1534 |
+
|
| 1535 |
+
model_kwargs = _expand_dict_for_generation(model_kwargs)
|
| 1536 |
+
|
| 1537 |
+
if is_encoder_decoder:
|
| 1538 |
+
if model_kwargs.get("encoder_outputs") is None:
|
| 1539 |
+
raise ValueError("If `is_encoder_decoder` is True, make sure that `encoder_outputs` is defined.")
|
| 1540 |
+
model_kwargs["encoder_outputs"] = _expand_dict_for_generation(model_kwargs["encoder_outputs"])
|
| 1541 |
+
|
| 1542 |
+
return input_ids, model_kwargs
|
| 1543 |
+
|
| 1544 |
+
|
| 1545 |
+
__all__ = [
|
| 1546 |
+
"InternVideo3VisionModel",
|
| 1547 |
+
"InternVideo3ForConditionalGeneration",
|
| 1548 |
+
"InternVideo3Model",
|
| 1549 |
+
"InternVideo3PreTrainedModel",
|
| 1550 |
+
"InternVideo3TextModel",
|
| 1551 |
+
]
|
preprocessor_config.json
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"crop_size": null,
|
| 3 |
+
"data_format": "channels_first",
|
| 4 |
+
"default_to_square": true,
|
| 5 |
+
"device": null,
|
| 6 |
+
"disable_grouping": null,
|
| 7 |
+
"do_center_crop": null,
|
| 8 |
+
"do_convert_rgb": true,
|
| 9 |
+
"do_normalize": true,
|
| 10 |
+
"do_pad": null,
|
| 11 |
+
"do_rescale": true,
|
| 12 |
+
"do_resize": true,
|
| 13 |
+
"image_mean": [
|
| 14 |
+
0.5,
|
| 15 |
+
0.5,
|
| 16 |
+
0.5
|
| 17 |
+
],
|
| 18 |
+
"image_processor_type": "Qwen2VLImageProcessorFast",
|
| 19 |
+
"image_std": [
|
| 20 |
+
0.5,
|
| 21 |
+
0.5,
|
| 22 |
+
0.5
|
| 23 |
+
],
|
| 24 |
+
"input_data_format": null,
|
| 25 |
+
"max_pixels": null,
|
| 26 |
+
"merge_size": 2,
|
| 27 |
+
"min_pixels": null,
|
| 28 |
+
"pad_size": null,
|
| 29 |
+
"patch_size": 16,
|
| 30 |
+
"processor_class": "InternVideo3Processor",
|
| 31 |
+
"auto_map": {
|
| 32 |
+
"AutoProcessor": "processing_internvideo3.InternVideo3Processor"
|
| 33 |
+
},
|
| 34 |
+
"resample": 3,
|
| 35 |
+
"rescale_factor": 0.00392156862745098,
|
| 36 |
+
"return_tensors": null,
|
| 37 |
+
"size": {
|
| 38 |
+
"longest_edge": 16777216,
|
| 39 |
+
"shortest_edge": 65536
|
| 40 |
+
},
|
| 41 |
+
"temporal_patch_size": 2
|
| 42 |
+
}
|
processing_internvideo3.py
ADDED
|
@@ -0,0 +1,237 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# coding=utf-8
|
| 2 |
+
# Copyright 2025 The InternVideo Team. All rights reserved.
|
| 3 |
+
#
|
| 4 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 5 |
+
# you may not use this file except in compliance with the License.
|
| 6 |
+
# You may obtain a copy of the License at
|
| 7 |
+
#
|
| 8 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 9 |
+
#
|
| 10 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 11 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 12 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 13 |
+
# See the License for the specific language governing permissions and
|
| 14 |
+
# limitations under the License.
|
| 15 |
+
"""Processor class for InternVideo3."""
|
| 16 |
+
|
| 17 |
+
from typing import Optional, Union
|
| 18 |
+
|
| 19 |
+
import numpy as np
|
| 20 |
+
|
| 21 |
+
from transformers.feature_extraction_utils import BatchFeature
|
| 22 |
+
from transformers.image_utils import ImageInput
|
| 23 |
+
from transformers.processing_utils import ImagesKwargs, ProcessingKwargs, ProcessorMixin, Unpack, VideosKwargs
|
| 24 |
+
from transformers.tokenization_utils_base import PreTokenizedInput, TextInput
|
| 25 |
+
from transformers.utils import logging
|
| 26 |
+
from transformers.video_utils import VideoInput
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
logger = logging.get_logger(__name__)
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
class InternVideo3VideosProcessorKwargs(VideosKwargs, total=False):
|
| 33 |
+
pass
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
class InternVideo3ImagesKwargs(ImagesKwargs):
|
| 37 |
+
min_pixels: Optional[int]
|
| 38 |
+
max_pixels: Optional[int]
|
| 39 |
+
patch_size: Optional[int]
|
| 40 |
+
temporal_patch_size: Optional[int]
|
| 41 |
+
merge_size: Optional[int]
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
class InternVideo3ProcessorKwargs(ProcessingKwargs, total=False):
|
| 45 |
+
images_kwargs: InternVideo3ImagesKwargs
|
| 46 |
+
videos_kwargs: InternVideo3VideosProcessorKwargs
|
| 47 |
+
_defaults = {
|
| 48 |
+
"text_kwargs": {
|
| 49 |
+
"padding": False,
|
| 50 |
+
"return_token_type_ids": False,
|
| 51 |
+
},
|
| 52 |
+
"videos_kwargs": {"return_metadata": True},
|
| 53 |
+
}
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
class InternVideo3Processor(ProcessorMixin):
|
| 57 |
+
r"""
|
| 58 |
+
Constructs an InternVideo3 processor which wraps an image processor, a video processor,
|
| 59 |
+
and a tokenizer into a single processor.
|
| 60 |
+
|
| 61 |
+
Args:
|
| 62 |
+
image_processor: The image processor.
|
| 63 |
+
tokenizer: The tokenizer.
|
| 64 |
+
video_processor: The video processor.
|
| 65 |
+
chat_template (`str`, *optional*): A Jinja template for chat formatting.
|
| 66 |
+
"""
|
| 67 |
+
|
| 68 |
+
attributes = ["image_processor", "tokenizer", "video_processor"]
|
| 69 |
+
image_processor_class = "AutoImageProcessor"
|
| 70 |
+
video_processor_class = "AutoVideoProcessor"
|
| 71 |
+
tokenizer_class = ("Qwen2Tokenizer", "Qwen2TokenizerFast")
|
| 72 |
+
|
| 73 |
+
def __init__(self, image_processor=None, tokenizer=None, video_processor=None, chat_template=None, **kwargs):
|
| 74 |
+
super().__init__(image_processor, tokenizer, video_processor, chat_template=chat_template)
|
| 75 |
+
self.image_token = "<|image_pad|>" if not hasattr(tokenizer, "image_token") else tokenizer.image_token
|
| 76 |
+
self.video_token = "<|video_pad|>" if not hasattr(tokenizer, "video_token") else tokenizer.video_token
|
| 77 |
+
self.image_token_id = (
|
| 78 |
+
tokenizer.image_token_id
|
| 79 |
+
if getattr(tokenizer, "image_token_id", None)
|
| 80 |
+
else tokenizer.convert_tokens_to_ids(self.image_token)
|
| 81 |
+
)
|
| 82 |
+
self.video_token_id = (
|
| 83 |
+
tokenizer.video_token_id
|
| 84 |
+
if getattr(tokenizer, "video_token_id", None)
|
| 85 |
+
else tokenizer.convert_tokens_to_ids(self.video_token)
|
| 86 |
+
)
|
| 87 |
+
self.vision_start_token = (
|
| 88 |
+
"<|vision_start|>" if not hasattr(tokenizer, "vision_start_token") else tokenizer.vision_start_token
|
| 89 |
+
)
|
| 90 |
+
self.vision_end_token = (
|
| 91 |
+
"<|vision_end|>" if not hasattr(tokenizer, "vision_end_token") else tokenizer.vision_end_token
|
| 92 |
+
)
|
| 93 |
+
self.vision_start_token_id = (
|
| 94 |
+
tokenizer.vision_start_token_id
|
| 95 |
+
if getattr(tokenizer, "vision_start_token_id", None)
|
| 96 |
+
else tokenizer.convert_tokens_to_ids(self.vision_start_token)
|
| 97 |
+
)
|
| 98 |
+
self.vision_end_token_id = (
|
| 99 |
+
tokenizer.vision_end_token_id
|
| 100 |
+
if getattr(tokenizer, "vision_end_token_id", None)
|
| 101 |
+
else tokenizer.convert_tokens_to_ids(self.vision_end_token)
|
| 102 |
+
)
|
| 103 |
+
|
| 104 |
+
def __call__(
|
| 105 |
+
self,
|
| 106 |
+
images: ImageInput = None,
|
| 107 |
+
text: Union[TextInput, PreTokenizedInput, list[TextInput], list[PreTokenizedInput]] = None,
|
| 108 |
+
videos: VideoInput = None,
|
| 109 |
+
**kwargs: Unpack[InternVideo3ProcessorKwargs],
|
| 110 |
+
) -> BatchFeature:
|
| 111 |
+
"""
|
| 112 |
+
Main method to prepare inputs for the model.
|
| 113 |
+
|
| 114 |
+
Args:
|
| 115 |
+
images: The image or batch of images to be prepared.
|
| 116 |
+
text: The sequence or batch of sequences to be encoded.
|
| 117 |
+
videos: The video or batch of videos to be prepared.
|
| 118 |
+
return_tensors: If set, will return tensors of a particular framework.
|
| 119 |
+
|
| 120 |
+
Returns:
|
| 121 |
+
[`BatchFeature`]: A [`BatchFeature`] with the following fields:
|
| 122 |
+
- **input_ids** -- Token ids to be fed to a model.
|
| 123 |
+
- **attention_mask** -- Attention mask.
|
| 124 |
+
- **pixel_values** -- Pixel values for images.
|
| 125 |
+
- **pixel_values_videos** -- Pixel values for videos.
|
| 126 |
+
- **image_grid_thw** -- Image 3D grid dimensions.
|
| 127 |
+
- **video_grid_thw** -- Video 3D grid dimensions.
|
| 128 |
+
"""
|
| 129 |
+
output_kwargs = self._merge_kwargs(
|
| 130 |
+
InternVideo3ProcessorKwargs,
|
| 131 |
+
tokenizer_init_kwargs=self.tokenizer.init_kwargs,
|
| 132 |
+
**kwargs,
|
| 133 |
+
)
|
| 134 |
+
|
| 135 |
+
if images is not None:
|
| 136 |
+
image_inputs = self.image_processor(images=images, **output_kwargs["images_kwargs"])
|
| 137 |
+
image_grid_thw = image_inputs["image_grid_thw"]
|
| 138 |
+
else:
|
| 139 |
+
image_inputs = {}
|
| 140 |
+
image_grid_thw = None
|
| 141 |
+
|
| 142 |
+
if videos is not None:
|
| 143 |
+
videos_inputs = self.video_processor(videos=videos, **output_kwargs["videos_kwargs"])
|
| 144 |
+
video_grid_thw = videos_inputs["video_grid_thw"]
|
| 145 |
+
# If user has not requested video metadata, pop it
|
| 146 |
+
if "return_metadata" not in kwargs:
|
| 147 |
+
video_metadata = videos_inputs.pop("video_metadata", None)
|
| 148 |
+
else:
|
| 149 |
+
video_metadata = videos_inputs.get("video_metadata", None)
|
| 150 |
+
video_grid_thw = videos_inputs["video_grid_thw"]
|
| 151 |
+
else:
|
| 152 |
+
videos_inputs = {}
|
| 153 |
+
video_grid_thw = None
|
| 154 |
+
video_metadata = None
|
| 155 |
+
|
| 156 |
+
if not isinstance(text, list):
|
| 157 |
+
text = [text]
|
| 158 |
+
|
| 159 |
+
text = text.copy()
|
| 160 |
+
if image_grid_thw is not None:
|
| 161 |
+
merge_length = self.image_processor.merge_size**2
|
| 162 |
+
index = 0
|
| 163 |
+
for i in range(len(text)):
|
| 164 |
+
while self.image_token in text[i]:
|
| 165 |
+
num_image_tokens = image_grid_thw[index].prod() // merge_length
|
| 166 |
+
text[i] = text[i].replace(self.image_token, "<|placeholder|>" * num_image_tokens, 1)
|
| 167 |
+
index += 1
|
| 168 |
+
text[i] = text[i].replace("<|placeholder|>", self.image_token)
|
| 169 |
+
|
| 170 |
+
if video_grid_thw is not None:
|
| 171 |
+
merge_length = self.video_processor.merge_size**2
|
| 172 |
+
index = 0
|
| 173 |
+
for i in range(len(text)):
|
| 174 |
+
while self.video_token in text[i]:
|
| 175 |
+
metadata = video_metadata[index] if video_metadata else None
|
| 176 |
+
if metadata is not None:
|
| 177 |
+
if metadata.fps is None:
|
| 178 |
+
logger.warning_once(
|
| 179 |
+
"InternVideo3 requires frame timestamps to construct prompts, but the `fps` of the input video could not be inferred. "
|
| 180 |
+
"Defaulting to `fps=24`. Please provide `video_metadata` for more accurate results."
|
| 181 |
+
)
|
| 182 |
+
metadata.fps = 24
|
| 183 |
+
|
| 184 |
+
curr_timestamp = self._calculate_timestamps(
|
| 185 |
+
metadata.frames_indices,
|
| 186 |
+
metadata.fps,
|
| 187 |
+
self.video_processor.merge_size,
|
| 188 |
+
)
|
| 189 |
+
|
| 190 |
+
video_placeholder = ""
|
| 191 |
+
frame_seqlen = video_grid_thw[index][1:].prod() // merge_length
|
| 192 |
+
for frame_idx in range(video_grid_thw[index][0]):
|
| 193 |
+
curr_time = curr_timestamp[frame_idx]
|
| 194 |
+
video_placeholder += f"<{curr_time:.1f} seconds>"
|
| 195 |
+
video_placeholder += (
|
| 196 |
+
self.vision_start_token + "<|placeholder|>" * frame_seqlen + self.vision_end_token
|
| 197 |
+
)
|
| 198 |
+
if f"{self.vision_start_token}{self.video_token}{self.vision_end_token}" in text[i]:
|
| 199 |
+
text[i] = text[i].replace(
|
| 200 |
+
f"{self.vision_start_token}{self.video_token}{self.vision_end_token}", video_placeholder, 1
|
| 201 |
+
)
|
| 202 |
+
else:
|
| 203 |
+
text[i] = text[i].replace(self.video_token, video_placeholder, 1)
|
| 204 |
+
else:
|
| 205 |
+
num_video_tokens = video_grid_thw[index].prod() // merge_length
|
| 206 |
+
text[i] = text[i].replace(self.video_token, "<|placeholder|>" * num_video_tokens, 1)
|
| 207 |
+
index += 1
|
| 208 |
+
text[i] = text[i].replace("<|placeholder|>", self.video_token)
|
| 209 |
+
|
| 210 |
+
return_tensors = output_kwargs["text_kwargs"].pop("return_tensors", None)
|
| 211 |
+
text_inputs = self.tokenizer(text, **output_kwargs["text_kwargs"])
|
| 212 |
+
|
| 213 |
+
return BatchFeature(data={**text_inputs, **image_inputs, **videos_inputs}, tensor_type=return_tensors)
|
| 214 |
+
|
| 215 |
+
def _calculate_timestamps(self, indices: Union[list[int], np.ndarray], video_fps: float, merge_size: int = 2):
|
| 216 |
+
if not isinstance(indices, list):
|
| 217 |
+
indices = indices.tolist()
|
| 218 |
+
if len(indices) % merge_size != 0:
|
| 219 |
+
indices.extend(indices[-1] for _ in range(merge_size - len(indices) % merge_size))
|
| 220 |
+
timestamps = [idx / video_fps for idx in indices]
|
| 221 |
+
timestamps = [
|
| 222 |
+
(timestamps[i] + timestamps[i + merge_size - 1]) / 2 for i in range(0, len(timestamps), merge_size)
|
| 223 |
+
]
|
| 224 |
+
return timestamps
|
| 225 |
+
|
| 226 |
+
def post_process_image_text_to_text(
|
| 227 |
+
self, generated_outputs, skip_special_tokens=True, clean_up_tokenization_spaces=False, **kwargs
|
| 228 |
+
):
|
| 229 |
+
return self.tokenizer.batch_decode(
|
| 230 |
+
generated_outputs,
|
| 231 |
+
skip_special_tokens=skip_special_tokens,
|
| 232 |
+
clean_up_tokenization_spaces=clean_up_tokenization_spaces,
|
| 233 |
+
**kwargs,
|
| 234 |
+
)
|
| 235 |
+
|
| 236 |
+
|
| 237 |
+
__all__ = ["InternVideo3Processor"]
|
special_tokens_map.json
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"additional_special_tokens": [
|
| 3 |
+
"<|im_start|>",
|
| 4 |
+
"<|im_end|>",
|
| 5 |
+
"<|object_ref_start|>",
|
| 6 |
+
"<|object_ref_end|>",
|
| 7 |
+
"<|box_start|>",
|
| 8 |
+
"<|box_end|>",
|
| 9 |
+
"<|quad_start|>",
|
| 10 |
+
"<|quad_end|>",
|
| 11 |
+
"<|vision_start|>",
|
| 12 |
+
"<|vision_end|>",
|
| 13 |
+
"<|vision_pad|>",
|
| 14 |
+
"<|image_pad|>",
|
| 15 |
+
"<|video_pad|>"
|
| 16 |
+
],
|
| 17 |
+
"eos_token": {
|
| 18 |
+
"content": "<|im_end|>",
|
| 19 |
+
"lstrip": false,
|
| 20 |
+
"normalized": false,
|
| 21 |
+
"rstrip": false,
|
| 22 |
+
"single_word": false
|
| 23 |
+
},
|
| 24 |
+
"pad_token": {
|
| 25 |
+
"content": "<|endoftext|>",
|
| 26 |
+
"lstrip": false,
|
| 27 |
+
"normalized": false,
|
| 28 |
+
"rstrip": false,
|
| 29 |
+
"single_word": false
|
| 30 |
+
}
|
| 31 |
+
}
|
tokenizer.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
tokenizer_config.json
ADDED
|
@@ -0,0 +1,239 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"add_bos_token": false,
|
| 3 |
+
"add_prefix_space": false,
|
| 4 |
+
"added_tokens_decoder": {
|
| 5 |
+
"151643": {
|
| 6 |
+
"content": "<|endoftext|>",
|
| 7 |
+
"lstrip": false,
|
| 8 |
+
"normalized": false,
|
| 9 |
+
"rstrip": false,
|
| 10 |
+
"single_word": false,
|
| 11 |
+
"special": true
|
| 12 |
+
},
|
| 13 |
+
"151644": {
|
| 14 |
+
"content": "<|im_start|>",
|
| 15 |
+
"lstrip": false,
|
| 16 |
+
"normalized": false,
|
| 17 |
+
"rstrip": false,
|
| 18 |
+
"single_word": false,
|
| 19 |
+
"special": true
|
| 20 |
+
},
|
| 21 |
+
"151645": {
|
| 22 |
+
"content": "<|im_end|>",
|
| 23 |
+
"lstrip": false,
|
| 24 |
+
"normalized": false,
|
| 25 |
+
"rstrip": false,
|
| 26 |
+
"single_word": false,
|
| 27 |
+
"special": true
|
| 28 |
+
},
|
| 29 |
+
"151646": {
|
| 30 |
+
"content": "<|object_ref_start|>",
|
| 31 |
+
"lstrip": false,
|
| 32 |
+
"normalized": false,
|
| 33 |
+
"rstrip": false,
|
| 34 |
+
"single_word": false,
|
| 35 |
+
"special": true
|
| 36 |
+
},
|
| 37 |
+
"151647": {
|
| 38 |
+
"content": "<|object_ref_end|>",
|
| 39 |
+
"lstrip": false,
|
| 40 |
+
"normalized": false,
|
| 41 |
+
"rstrip": false,
|
| 42 |
+
"single_word": false,
|
| 43 |
+
"special": true
|
| 44 |
+
},
|
| 45 |
+
"151648": {
|
| 46 |
+
"content": "<|box_start|>",
|
| 47 |
+
"lstrip": false,
|
| 48 |
+
"normalized": false,
|
| 49 |
+
"rstrip": false,
|
| 50 |
+
"single_word": false,
|
| 51 |
+
"special": true
|
| 52 |
+
},
|
| 53 |
+
"151649": {
|
| 54 |
+
"content": "<|box_end|>",
|
| 55 |
+
"lstrip": false,
|
| 56 |
+
"normalized": false,
|
| 57 |
+
"rstrip": false,
|
| 58 |
+
"single_word": false,
|
| 59 |
+
"special": true
|
| 60 |
+
},
|
| 61 |
+
"151650": {
|
| 62 |
+
"content": "<|quad_start|>",
|
| 63 |
+
"lstrip": false,
|
| 64 |
+
"normalized": false,
|
| 65 |
+
"rstrip": false,
|
| 66 |
+
"single_word": false,
|
| 67 |
+
"special": true
|
| 68 |
+
},
|
| 69 |
+
"151651": {
|
| 70 |
+
"content": "<|quad_end|>",
|
| 71 |
+
"lstrip": false,
|
| 72 |
+
"normalized": false,
|
| 73 |
+
"rstrip": false,
|
| 74 |
+
"single_word": false,
|
| 75 |
+
"special": true
|
| 76 |
+
},
|
| 77 |
+
"151652": {
|
| 78 |
+
"content": "<|vision_start|>",
|
| 79 |
+
"lstrip": false,
|
| 80 |
+
"normalized": false,
|
| 81 |
+
"rstrip": false,
|
| 82 |
+
"single_word": false,
|
| 83 |
+
"special": true
|
| 84 |
+
},
|
| 85 |
+
"151653": {
|
| 86 |
+
"content": "<|vision_end|>",
|
| 87 |
+
"lstrip": false,
|
| 88 |
+
"normalized": false,
|
| 89 |
+
"rstrip": false,
|
| 90 |
+
"single_word": false,
|
| 91 |
+
"special": true
|
| 92 |
+
},
|
| 93 |
+
"151654": {
|
| 94 |
+
"content": "<|vision_pad|>",
|
| 95 |
+
"lstrip": false,
|
| 96 |
+
"normalized": false,
|
| 97 |
+
"rstrip": false,
|
| 98 |
+
"single_word": false,
|
| 99 |
+
"special": true
|
| 100 |
+
},
|
| 101 |
+
"151655": {
|
| 102 |
+
"content": "<|image_pad|>",
|
| 103 |
+
"lstrip": false,
|
| 104 |
+
"normalized": false,
|
| 105 |
+
"rstrip": false,
|
| 106 |
+
"single_word": false,
|
| 107 |
+
"special": true
|
| 108 |
+
},
|
| 109 |
+
"151656": {
|
| 110 |
+
"content": "<|video_pad|>",
|
| 111 |
+
"lstrip": false,
|
| 112 |
+
"normalized": false,
|
| 113 |
+
"rstrip": false,
|
| 114 |
+
"single_word": false,
|
| 115 |
+
"special": true
|
| 116 |
+
},
|
| 117 |
+
"151657": {
|
| 118 |
+
"content": "<tool_call>",
|
| 119 |
+
"lstrip": false,
|
| 120 |
+
"normalized": false,
|
| 121 |
+
"rstrip": false,
|
| 122 |
+
"single_word": false,
|
| 123 |
+
"special": false
|
| 124 |
+
},
|
| 125 |
+
"151658": {
|
| 126 |
+
"content": "</tool_call>",
|
| 127 |
+
"lstrip": false,
|
| 128 |
+
"normalized": false,
|
| 129 |
+
"rstrip": false,
|
| 130 |
+
"single_word": false,
|
| 131 |
+
"special": false
|
| 132 |
+
},
|
| 133 |
+
"151659": {
|
| 134 |
+
"content": "<|fim_prefix|>",
|
| 135 |
+
"lstrip": false,
|
| 136 |
+
"normalized": false,
|
| 137 |
+
"rstrip": false,
|
| 138 |
+
"single_word": false,
|
| 139 |
+
"special": false
|
| 140 |
+
},
|
| 141 |
+
"151660": {
|
| 142 |
+
"content": "<|fim_middle|>",
|
| 143 |
+
"lstrip": false,
|
| 144 |
+
"normalized": false,
|
| 145 |
+
"rstrip": false,
|
| 146 |
+
"single_word": false,
|
| 147 |
+
"special": false
|
| 148 |
+
},
|
| 149 |
+
"151661": {
|
| 150 |
+
"content": "<|fim_suffix|>",
|
| 151 |
+
"lstrip": false,
|
| 152 |
+
"normalized": false,
|
| 153 |
+
"rstrip": false,
|
| 154 |
+
"single_word": false,
|
| 155 |
+
"special": false
|
| 156 |
+
},
|
| 157 |
+
"151662": {
|
| 158 |
+
"content": "<|fim_pad|>",
|
| 159 |
+
"lstrip": false,
|
| 160 |
+
"normalized": false,
|
| 161 |
+
"rstrip": false,
|
| 162 |
+
"single_word": false,
|
| 163 |
+
"special": false
|
| 164 |
+
},
|
| 165 |
+
"151663": {
|
| 166 |
+
"content": "<|repo_name|>",
|
| 167 |
+
"lstrip": false,
|
| 168 |
+
"normalized": false,
|
| 169 |
+
"rstrip": false,
|
| 170 |
+
"single_word": false,
|
| 171 |
+
"special": false
|
| 172 |
+
},
|
| 173 |
+
"151664": {
|
| 174 |
+
"content": "<|file_sep|>",
|
| 175 |
+
"lstrip": false,
|
| 176 |
+
"normalized": false,
|
| 177 |
+
"rstrip": false,
|
| 178 |
+
"single_word": false,
|
| 179 |
+
"special": false
|
| 180 |
+
},
|
| 181 |
+
"151665": {
|
| 182 |
+
"content": "<tool_response>",
|
| 183 |
+
"lstrip": false,
|
| 184 |
+
"normalized": false,
|
| 185 |
+
"rstrip": false,
|
| 186 |
+
"single_word": false,
|
| 187 |
+
"special": false
|
| 188 |
+
},
|
| 189 |
+
"151666": {
|
| 190 |
+
"content": "</tool_response>",
|
| 191 |
+
"lstrip": false,
|
| 192 |
+
"normalized": false,
|
| 193 |
+
"rstrip": false,
|
| 194 |
+
"single_word": false,
|
| 195 |
+
"special": false
|
| 196 |
+
},
|
| 197 |
+
"151667": {
|
| 198 |
+
"content": "<think>",
|
| 199 |
+
"lstrip": false,
|
| 200 |
+
"normalized": false,
|
| 201 |
+
"rstrip": false,
|
| 202 |
+
"single_word": false,
|
| 203 |
+
"special": false
|
| 204 |
+
},
|
| 205 |
+
"151668": {
|
| 206 |
+
"content": "</think>",
|
| 207 |
+
"lstrip": false,
|
| 208 |
+
"normalized": false,
|
| 209 |
+
"rstrip": false,
|
| 210 |
+
"single_word": false,
|
| 211 |
+
"special": false
|
| 212 |
+
}
|
| 213 |
+
},
|
| 214 |
+
"additional_special_tokens": [
|
| 215 |
+
"<|im_start|>",
|
| 216 |
+
"<|im_end|>",
|
| 217 |
+
"<|object_ref_start|>",
|
| 218 |
+
"<|object_ref_end|>",
|
| 219 |
+
"<|box_start|>",
|
| 220 |
+
"<|box_end|>",
|
| 221 |
+
"<|quad_start|>",
|
| 222 |
+
"<|quad_end|>",
|
| 223 |
+
"<|vision_start|>",
|
| 224 |
+
"<|vision_end|>",
|
| 225 |
+
"<|vision_pad|>",
|
| 226 |
+
"<|image_pad|>",
|
| 227 |
+
"<|video_pad|>"
|
| 228 |
+
],
|
| 229 |
+
"bos_token": null,
|
| 230 |
+
"chat_template": "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].role == 'system' %}\n {%- if messages[0].content is string %}\n {{- messages[0].content }}\n {%- else %}\n {%- for content in messages[0].content %}\n {%- if 'text' in content %}\n {{- content.text }}\n {%- endif %}\n {%- endfor %}\n {%- endif %}\n {{- '\\n\\n' }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].content is string %}\n {{- messages[0].content }}\n {%- else %}\n {%- for content in messages[0].content %}\n {%- if 'text' in content %}\n {{- content.text }}\n {%- endif %}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- set image_count = namespace(value=0) %}\n{%- set video_count = namespace(value=0) %}\n{%- for message in messages %}\n {%- if message.role == \"user\" %}\n {{- '<|im_start|>' + message.role + '\\n' }}\n {%- if message.content is string %}\n {{- message.content }}\n {%- else %}\n {%- for content in message.content %}\n {%- if content.type == 'image' or 'image' in content or 'image_url' in content %}\n {%- set image_count.value = image_count.value + 1 %}\n {%- if add_vision_id %}Picture {{ image_count.value }}: {% endif -%}\n <|vision_start|><|image_pad|><|vision_end|>\n {%- elif content.type == 'video' or 'video' in content %}\n {%- set video_count.value = video_count.value + 1 %}\n {%- if add_vision_id %}Video {{ video_count.value }}: {% endif -%}\n <|vision_start|><|video_pad|><|vision_end|>\n {%- elif 'text' in content %}\n {{- content.text }}\n {%- endif %}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"assistant\" %}\n {{- '<|im_start|>' + message.role + '\\n' }}\n {%- if message.content is string %}\n {{- message.content }}\n {%- else %}\n {%- for content_item in message.content %}\n {%- if 'text' in content_item %}\n {{- content_item.text }}\n {%- endif %}\n {%- endfor %}\n {%- endif %}\n {%- if message.tool_calls %}\n {%- for tool_call in message.tool_calls %}\n {%- if (loop.first and message.content) or (not loop.first) %}\n {{- '\\n' }}\n {%- endif %}\n {%- if tool_call.function %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '<tool_call>\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {%- if tool_call.arguments is string %}\n {{- tool_call.arguments }}\n {%- else %}\n {{- tool_call.arguments | tojson }}\n {%- endif %}\n {{- '}\\n</tool_call>' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n<tool_response>\\n' }}\n {%- if message.content is string %}\n {{- message.content }}\n {%- else %}\n {%- for content in message.content %}\n {%- if content.type == 'image' or 'image' in content or 'image_url' in content %}\n {%- set image_count.value = image_count.value + 1 %}\n {%- if add_vision_id %}Picture {{ image_count.value }}: {% endif -%}\n <|vision_start|><|image_pad|><|vision_end|>\n {%- elif content.type == 'video' or 'video' in content %}\n {%- set video_count.value = video_count.value + 1 %}\n {%- if add_vision_id %}Video {{ video_count.value }}: {% endif -%}\n <|vision_start|><|video_pad|><|vision_end|>\n {%- elif 'text' in content %}\n {{- content.text }}\n {%- endif %}\n {%- endfor %}\n {%- endif %}\n {{- '\\n</tool_response>' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
|
| 231 |
+
"clean_up_tokenization_spaces": false,
|
| 232 |
+
"eos_token": "<|im_end|>",
|
| 233 |
+
"errors": "replace",
|
| 234 |
+
"model_max_length": 262144,
|
| 235 |
+
"pad_token": "<|endoftext|>",
|
| 236 |
+
"split_special_tokens": false,
|
| 237 |
+
"tokenizer_class": "Qwen2Tokenizer",
|
| 238 |
+
"unk_token": null
|
| 239 |
+
}
|
video_preprocessor_config.json
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"size": {
|
| 3 |
+
"longest_edge": 25165824,
|
| 4 |
+
"shortest_edge": 4096
|
| 5 |
+
},
|
| 6 |
+
"patch_size": 16,
|
| 7 |
+
"temporal_patch_size": 2,
|
| 8 |
+
"merge_size": 2,
|
| 9 |
+
"image_mean": [
|
| 10 |
+
0.5,
|
| 11 |
+
0.5,
|
| 12 |
+
0.5
|
| 13 |
+
],
|
| 14 |
+
"image_std": [
|
| 15 |
+
0.5,
|
| 16 |
+
0.5,
|
| 17 |
+
0.5
|
| 18 |
+
],
|
| 19 |
+
"processor_class": "Qwen3VLProcessor",
|
| 20 |
+
"video_processor_type": "Qwen3VLVideoProcessor"
|
| 21 |
+
}
|
video_processing_internvideo3.py
ADDED
|
@@ -0,0 +1,235 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# coding=utf-8
|
| 2 |
+
# Copyright 2025 The InternVideo Team. All rights reserved.
|
| 3 |
+
#
|
| 4 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 5 |
+
# you may not use this file except in compliance with the License.
|
| 6 |
+
# You may obtain a copy of the License at
|
| 7 |
+
#
|
| 8 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 9 |
+
#
|
| 10 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 11 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 12 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 13 |
+
# See the License for the specific language governing permissions and
|
| 14 |
+
# limitations under the License.
|
| 15 |
+
"""Video processor class for InternVideo3."""
|
| 16 |
+
|
| 17 |
+
import math
|
| 18 |
+
from typing import Optional, Union
|
| 19 |
+
|
| 20 |
+
import numpy as np
|
| 21 |
+
import torch
|
| 22 |
+
|
| 23 |
+
from transformers.feature_extraction_utils import BatchFeature
|
| 24 |
+
from transformers.image_utils import ChannelDimension, PILImageResampling, SizeDict, get_image_size
|
| 25 |
+
from transformers.processing_utils import Unpack, VideosKwargs
|
| 26 |
+
from transformers.utils import TensorType, logging
|
| 27 |
+
from transformers.video_processing_utils import BaseVideoProcessor
|
| 28 |
+
from transformers.video_utils import VideoMetadata, group_videos_by_shape, reorder_videos
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
logger = logging.get_logger(__name__)
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def smart_resize(
|
| 35 |
+
num_frames: int,
|
| 36 |
+
height: int,
|
| 37 |
+
width: int,
|
| 38 |
+
temporal_factor: int = 2,
|
| 39 |
+
factor: int = 32,
|
| 40 |
+
min_pixels: int = 128 * 128,
|
| 41 |
+
max_pixels: int = 16 * 16 * 2 * 2 * 2 * 6144,
|
| 42 |
+
):
|
| 43 |
+
if num_frames < temporal_factor:
|
| 44 |
+
raise ValueError(f"t:{num_frames} must be larger than temporal_factor:{temporal_factor}")
|
| 45 |
+
if height < factor or width < factor:
|
| 46 |
+
raise ValueError(f"height:{height} or width:{width} must be larger than factor:{factor}")
|
| 47 |
+
elif max(height, width) / min(height, width) > 200:
|
| 48 |
+
raise ValueError(
|
| 49 |
+
f"absolute aspect ratio must be smaller than 200, got {max(height, width) / min(height, width)}"
|
| 50 |
+
)
|
| 51 |
+
h_bar = round(height / factor) * factor
|
| 52 |
+
w_bar = round(width / factor) * factor
|
| 53 |
+
t_bar = round(num_frames / temporal_factor) * temporal_factor
|
| 54 |
+
|
| 55 |
+
if t_bar * h_bar * w_bar > max_pixels:
|
| 56 |
+
beta = math.sqrt((num_frames * height * width) / max_pixels)
|
| 57 |
+
h_bar = max(factor, math.floor(height / beta / factor) * factor)
|
| 58 |
+
w_bar = max(factor, math.floor(width / beta / factor) * factor)
|
| 59 |
+
elif t_bar * h_bar * w_bar < min_pixels:
|
| 60 |
+
beta = math.sqrt(min_pixels / (num_frames * height * width))
|
| 61 |
+
h_bar = math.ceil(height * beta / factor) * factor
|
| 62 |
+
w_bar = math.ceil(width * beta / factor) * factor
|
| 63 |
+
|
| 64 |
+
return h_bar, w_bar
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
class InternVideo3VideoProcessorInitKwargs(VideosKwargs):
|
| 68 |
+
patch_size: Optional[int]
|
| 69 |
+
temporal_patch_size: Optional[int]
|
| 70 |
+
merge_size: Optional[int]
|
| 71 |
+
min_frames: Optional[int]
|
| 72 |
+
max_frames: Optional[int]
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
class InternVideo3VideoProcessor(BaseVideoProcessor):
|
| 76 |
+
resample = PILImageResampling.BICUBIC
|
| 77 |
+
size = {"shortest_edge": 128 * 32 * 32, "longest_edge": 32 * 32 * 768}
|
| 78 |
+
image_mean = [0.5, 0.5, 0.5]
|
| 79 |
+
image_std = [0.5, 0.5, 0.5]
|
| 80 |
+
do_resize = True
|
| 81 |
+
do_rescale = True
|
| 82 |
+
do_normalize = True
|
| 83 |
+
do_convert_rgb = True
|
| 84 |
+
patch_size = 16
|
| 85 |
+
temporal_patch_size = 2
|
| 86 |
+
merge_size = 2
|
| 87 |
+
fps = 2
|
| 88 |
+
min_frames = 4
|
| 89 |
+
max_frames = 768
|
| 90 |
+
do_sample_frames = True
|
| 91 |
+
valid_kwargs = InternVideo3VideoProcessorInitKwargs
|
| 92 |
+
model_input_names = ["pixel_values_videos", "video_grid_thw"]
|
| 93 |
+
|
| 94 |
+
def __init__(self, **kwargs: Unpack[InternVideo3VideoProcessorInitKwargs]):
|
| 95 |
+
super().__init__(**kwargs)
|
| 96 |
+
if self.size is not None and (
|
| 97 |
+
self.size.get("shortest_edge", None) is None or self.size.get("longest_edge", None) is None
|
| 98 |
+
):
|
| 99 |
+
raise ValueError("size must contain 'shortest_edge' and 'longest_edge' keys.")
|
| 100 |
+
|
| 101 |
+
def _further_process_kwargs(
|
| 102 |
+
self,
|
| 103 |
+
size: Optional[SizeDict] = None,
|
| 104 |
+
**kwargs,
|
| 105 |
+
) -> dict:
|
| 106 |
+
if size is not None and ("shortest_edge" not in size or "longest_edge" not in size):
|
| 107 |
+
raise ValueError("size must contain 'shortest_edge' and 'longest_edge' keys.")
|
| 108 |
+
return super()._further_process_kwargs(size=size, **kwargs)
|
| 109 |
+
|
| 110 |
+
def sample_frames(
|
| 111 |
+
self,
|
| 112 |
+
metadata: VideoMetadata,
|
| 113 |
+
num_frames: Optional[int] = None,
|
| 114 |
+
fps: Optional[Union[int, float]] = None,
|
| 115 |
+
**kwargs,
|
| 116 |
+
):
|
| 117 |
+
if fps is not None and num_frames is not None:
|
| 118 |
+
raise ValueError("`num_frames` and `fps` are mutually exclusive arguments, please use only one!")
|
| 119 |
+
|
| 120 |
+
total_num_frames = metadata.total_num_frames
|
| 121 |
+
fps = fps if fps is not None else self.fps
|
| 122 |
+
|
| 123 |
+
if num_frames is None and fps is not None:
|
| 124 |
+
if metadata.fps is None:
|
| 125 |
+
metadata.fps = 24
|
| 126 |
+
logger.warning_once(
|
| 127 |
+
"Asked to sample `fps` frames per second but no video metadata was provided. "
|
| 128 |
+
"Defaulting to `fps=24`. Please provide `video_metadata` for more accurate results."
|
| 129 |
+
)
|
| 130 |
+
num_frames = int(total_num_frames / metadata.fps * fps)
|
| 131 |
+
num_frames = min(min(max(num_frames, self.min_frames), self.max_frames), total_num_frames)
|
| 132 |
+
|
| 133 |
+
if num_frames is None:
|
| 134 |
+
num_frames = min(max(total_num_frames, self.min_frames), self.max_frames)
|
| 135 |
+
|
| 136 |
+
indices = np.linspace(0, total_num_frames - 1, num_frames).round().astype(int)
|
| 137 |
+
return indices
|
| 138 |
+
|
| 139 |
+
def _preprocess(
|
| 140 |
+
self,
|
| 141 |
+
videos: list[torch.Tensor],
|
| 142 |
+
do_convert_rgb: bool = True,
|
| 143 |
+
do_resize: bool = True,
|
| 144 |
+
size: Optional[SizeDict] = None,
|
| 145 |
+
interpolation: PILImageResampling = PILImageResampling.BICUBIC,
|
| 146 |
+
do_rescale: bool = True,
|
| 147 |
+
rescale_factor: float = 1 / 255.0,
|
| 148 |
+
do_normalize: bool = True,
|
| 149 |
+
image_mean: Optional[Union[float, list[float]]] = None,
|
| 150 |
+
image_std: Optional[Union[float, list[float]]] = None,
|
| 151 |
+
patch_size: Optional[int] = None,
|
| 152 |
+
temporal_patch_size: Optional[int] = None,
|
| 153 |
+
merge_size: Optional[int] = None,
|
| 154 |
+
return_tensors: Optional[Union[str, TensorType]] = None,
|
| 155 |
+
**kwargs,
|
| 156 |
+
):
|
| 157 |
+
grouped_videos, grouped_videos_index = group_videos_by_shape(videos)
|
| 158 |
+
resized_videos_grouped = {}
|
| 159 |
+
|
| 160 |
+
for shape, stacked_videos in grouped_videos.items():
|
| 161 |
+
B, T, C, H, W = stacked_videos.shape
|
| 162 |
+
num_frames, height, width = T, H, W
|
| 163 |
+
if do_resize:
|
| 164 |
+
resized_height, resized_width = smart_resize(
|
| 165 |
+
num_frames=num_frames,
|
| 166 |
+
height=height,
|
| 167 |
+
width=width,
|
| 168 |
+
temporal_factor=temporal_patch_size,
|
| 169 |
+
factor=patch_size * merge_size,
|
| 170 |
+
min_pixels=size.shortest_edge,
|
| 171 |
+
max_pixels=size.longest_edge,
|
| 172 |
+
)
|
| 173 |
+
stacked_videos = stacked_videos.view(B * T, C, H, W)
|
| 174 |
+
stacked_videos = self.resize(
|
| 175 |
+
stacked_videos,
|
| 176 |
+
size=SizeDict(height=resized_height, width=resized_width),
|
| 177 |
+
interpolation=interpolation,
|
| 178 |
+
)
|
| 179 |
+
stacked_videos = stacked_videos.view(B, T, C, resized_height, resized_width)
|
| 180 |
+
resized_videos_grouped[shape] = stacked_videos
|
| 181 |
+
resized_videos = reorder_videos(resized_videos_grouped, grouped_videos_index)
|
| 182 |
+
|
| 183 |
+
grouped_videos, grouped_videos_index = group_videos_by_shape(resized_videos)
|
| 184 |
+
processed_videos_grouped = {}
|
| 185 |
+
processed_grids = {}
|
| 186 |
+
for shape, stacked_videos in grouped_videos.items():
|
| 187 |
+
resized_height, resized_width = get_image_size(stacked_videos[0], channel_dim=ChannelDimension.FIRST)
|
| 188 |
+
|
| 189 |
+
stacked_videos = self.rescale_and_normalize(
|
| 190 |
+
stacked_videos, do_rescale, rescale_factor, do_normalize, image_mean, image_std
|
| 191 |
+
)
|
| 192 |
+
patches = stacked_videos
|
| 193 |
+
|
| 194 |
+
if patches.shape[1] % temporal_patch_size != 0:
|
| 195 |
+
repeats = patches[:, -1:].repeat(1, temporal_patch_size - 1, 1, 1, 1)
|
| 196 |
+
patches = torch.cat([patches, repeats], dim=1)
|
| 197 |
+
batch_size, grid_t, channel = patches.shape[:3]
|
| 198 |
+
grid_t = grid_t // temporal_patch_size
|
| 199 |
+
grid_h, grid_w = resized_height // patch_size, resized_width // patch_size
|
| 200 |
+
|
| 201 |
+
patches = patches.view(
|
| 202 |
+
batch_size,
|
| 203 |
+
grid_t,
|
| 204 |
+
temporal_patch_size,
|
| 205 |
+
channel,
|
| 206 |
+
grid_h // merge_size,
|
| 207 |
+
merge_size,
|
| 208 |
+
patch_size,
|
| 209 |
+
grid_w // merge_size,
|
| 210 |
+
merge_size,
|
| 211 |
+
patch_size,
|
| 212 |
+
)
|
| 213 |
+
patches = patches.permute(0, 1, 4, 7, 5, 8, 3, 2, 6, 9)
|
| 214 |
+
flatten_patches = patches.reshape(
|
| 215 |
+
batch_size,
|
| 216 |
+
grid_t * grid_h * grid_w,
|
| 217 |
+
channel * temporal_patch_size * patch_size * patch_size,
|
| 218 |
+
)
|
| 219 |
+
|
| 220 |
+
processed_videos_grouped[shape] = flatten_patches
|
| 221 |
+
processed_grids[shape] = [[grid_t, grid_h, grid_w]] * batch_size
|
| 222 |
+
|
| 223 |
+
processed_videos = reorder_videos(processed_videos_grouped, grouped_videos_index)
|
| 224 |
+
processed_grids = reorder_videos(processed_grids, grouped_videos_index)
|
| 225 |
+
pixel_values_videos = torch.cat(processed_videos, dim=0)
|
| 226 |
+
video_grid_thw = torch.tensor(processed_grids)
|
| 227 |
+
data = {
|
| 228 |
+
"pixel_values_videos": pixel_values_videos,
|
| 229 |
+
"video_grid_thw": video_grid_thw,
|
| 230 |
+
}
|
| 231 |
+
|
| 232 |
+
return BatchFeature(data=data, tensor_type=return_tensors)
|
| 233 |
+
|
| 234 |
+
|
| 235 |
+
__all__ = ["InternVideo3VideoProcessor"]
|
vocab.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|