Commit ·
21b4e01
0
Parent(s):
Duplicate from jingyaogong/minimind-3v
Browse filesCo-authored-by: jingyaogong <jingyaogong@users.noreply.huggingface.co>
- .gitattributes +41 -0
- README.md +699 -0
- README_en.md +716 -0
- chat_template.jinja +85 -0
- config.json +42 -0
- generation_config.json +6 -0
- images/VLM-structure-moe.jpg +3 -0
- images/VLM-structure.jpg +3 -0
- images/llava-structure.png +3 -0
- images/logo.png +3 -0
- images/minimind-3v.gif +3 -0
- images/minimind-v-input.jpg +3 -0
- images/pretrain_loss.jpg +0 -0
- images/sft_loss.jpg +0 -0
- model_minimind.py +279 -0
- model_vlm.py +155 -0
- pytorch_model.bin +3 -0
- special_tokens_map.json +52 -0
- tokenizer.json +0 -0
- tokenizer_config.json +335 -0
.gitattributes
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
*.7z filter=lfs diff=lfs merge=lfs -text
|
| 2 |
+
*.arrow filter=lfs diff=lfs merge=lfs -text
|
| 3 |
+
*.bin filter=lfs diff=lfs merge=lfs -text
|
| 4 |
+
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
| 5 |
+
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
| 6 |
+
*.ftz filter=lfs diff=lfs merge=lfs -text
|
| 7 |
+
*.gz filter=lfs diff=lfs merge=lfs -text
|
| 8 |
+
*.h5 filter=lfs diff=lfs merge=lfs -text
|
| 9 |
+
*.joblib filter=lfs diff=lfs merge=lfs -text
|
| 10 |
+
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
| 11 |
+
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
| 12 |
+
*.model filter=lfs diff=lfs merge=lfs -text
|
| 13 |
+
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
| 14 |
+
*.npy filter=lfs diff=lfs merge=lfs -text
|
| 15 |
+
*.npz filter=lfs diff=lfs merge=lfs -text
|
| 16 |
+
*.onnx filter=lfs diff=lfs merge=lfs -text
|
| 17 |
+
*.ot filter=lfs diff=lfs merge=lfs -text
|
| 18 |
+
*.parquet filter=lfs diff=lfs merge=lfs -text
|
| 19 |
+
*.pb filter=lfs diff=lfs merge=lfs -text
|
| 20 |
+
*.pickle filter=lfs diff=lfs merge=lfs -text
|
| 21 |
+
*.pkl filter=lfs diff=lfs merge=lfs -text
|
| 22 |
+
*.pt filter=lfs diff=lfs merge=lfs -text
|
| 23 |
+
*.pth filter=lfs diff=lfs merge=lfs -text
|
| 24 |
+
*.rar filter=lfs diff=lfs merge=lfs -text
|
| 25 |
+
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
| 26 |
+
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
| 27 |
+
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
| 28 |
+
*.tar filter=lfs diff=lfs merge=lfs -text
|
| 29 |
+
*.tflite filter=lfs diff=lfs merge=lfs -text
|
| 30 |
+
*.tgz filter=lfs diff=lfs merge=lfs -text
|
| 31 |
+
*.wasm filter=lfs diff=lfs merge=lfs -text
|
| 32 |
+
*.xz filter=lfs diff=lfs merge=lfs -text
|
| 33 |
+
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
+
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
+
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
images/llava-structure.png filter=lfs diff=lfs merge=lfs -text
|
| 37 |
+
images/logo.png filter=lfs diff=lfs merge=lfs -text
|
| 38 |
+
images/minimind-3v.gif filter=lfs diff=lfs merge=lfs -text
|
| 39 |
+
images/minimind-v-input.jpg filter=lfs diff=lfs merge=lfs -text
|
| 40 |
+
images/VLM-structure-moe.jpg filter=lfs diff=lfs merge=lfs -text
|
| 41 |
+
images/VLM-structure.jpg filter=lfs diff=lfs merge=lfs -text
|
README.md
ADDED
|
@@ -0,0 +1,699 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
license: apache-2.0
|
| 3 |
+
datasets:
|
| 4 |
+
- jingyaogong/minimind-v_dataset
|
| 5 |
+
language:
|
| 6 |
+
- zh
|
| 7 |
+
- en
|
| 8 |
+
pipeline_tag: image-text-to-text
|
| 9 |
+
---
|
| 10 |
+
<div align="center">
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
</div>
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
<div align="center">
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
[](https://github.com/jingyaogong/minimind-v/stargazers)
|
| 21 |
+
[](LICENSE)
|
| 22 |
+
[](https://github.com/jingyaogong/minimind-v/commits/master)
|
| 23 |
+
[](https://github.com/jingyaogong/minimind-v/pulls)
|
| 24 |
+
[](https://huggingface.co/collections/jingyaogong/minimind-v-67000833fb60b3a2e1f3597d)
|
| 25 |
+
|
| 26 |
+
</div>
|
| 27 |
+
|
| 28 |
+
<div align="center">
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
</div>
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
<div align="center">
|
| 36 |
+
<h3>"大道至简"</h3>
|
| 37 |
+
</div>
|
| 38 |
+
|
| 39 |
+
<div align="center">
|
| 40 |
+
|
| 41 |
+
中文 | [English](./README_en.md)
|
| 42 |
+
|
| 43 |
+
</div>
|
| 44 |
+
|
| 45 |
+
* 此项目旨在从0开始，仅用1.3块钱成本 + 1小时！即可训练出67M参数的超小多模态视觉语言模型**MiniMind-V**。
|
| 46 |
+
* **MiniMind-V**最小版本体积仅为 GPT3 的约 $\frac{1}{2600}$，力求做到个人GPU也可快速推理甚至训练。
|
| 47 |
+
* **MiniMind-V**是[MiniMind](https://github.com/jingyaogong/minimind)纯语言模型的视觉能力额外拓展。
|
| 48 |
+
* 项目同时包含了VLM大模型的极简结构、数据集清洗、预训练(Pretrain)、监督微调(SFT)等全过程代码。
|
| 49 |
+
* 这不仅是一个开源VLM模型的最小实现，也是入门视觉语言模型的简明教程。
|
| 50 |
+
* 希望此项目能为所有人提供一个抛砖引玉的示例，一起感受创造的乐趣！推动更广泛AI社区的进步！
|
| 51 |
+
|
| 52 |
+
> 为防止误解，“1小时” 基于NVIDIA 3090硬件设备（单卡）测试`1 epoch`，“1.3块钱” 指GPU服务器租用成本。
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
<div align="center">
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
[🔗🤖在线体验](https://www.modelscope.cn/studios/gongjy/MiniMind-V) | [🔗🎞️视频介绍](https://www.bilibili.com/video/BV1Sh1vYBEzY)
|
| 61 |
+
|
| 62 |
+
</div>
|
| 63 |
+
|
| 64 |
+
# 📌 项目介绍
|
| 65 |
+
|
| 66 |
+
“用乐高拼出一架飞机，远比坐在头等舱里飞行更让人兴奋！”
|
| 67 |
+
构建VLM范式的多模态大模型是否真的如想象中那样复杂？它的代码实现到底如何？
|
| 68 |
+
训练过程究竟难不难？那么现在，探索它们的答案，一起感受创造的乐趣吧！
|
| 69 |
+
|
| 70 |
+
> [!TIP]
|
| 71 |
+
> （截至2026-02-15）MiniMind-V 系列已完成了以下型号模型训练，最小仅需67M (0.067B)，即可具备识图和对话的能力！
|
| 72 |
+
|
| 73 |
+
| 模型 (大小) | 推理占用 | release |
|
| 74 |
+
|---------------------------|--------|------------|
|
| 75 |
+
| minimind-3v-moe (201M-A67M) | 1.0 GB | 2026.04.01 |
|
| 76 |
+
| minimind-3v (67M) | 0.5 GB | 2026.04.01 |
|
| 77 |
+
| MiniMind2-V (104M) | 1.1 GB | 2025.02.20 |
|
| 78 |
+
| MiniMind2-Small-V (26M) | 0.6 GB | 2025.02.20 |
|
| 79 |
+
| minimind-v-v1-small (27M) | 0.6 GB | 2024.10.04 |
|
| 80 |
+
| minimind-v-v1 (109M) | 1.1 GB | 2024.10.04 |
|
| 81 |
+
|
| 82 |
+
#### 👉 更新日志
|
| 83 |
+
|
| 84 |
+
<details close>
|
| 85 |
+
<summary> <b>2026-02-15</b> </summary>
|
| 86 |
+
|
| 87 |
+
- 新增 minimind-3v (67M) 和 minimind-3v-moe (201M-A67M) 模型
|
| 88 |
+
- 统一使用768+8架构，支持dense和moe两种模式
|
| 89 |
+
- 数据集格式更新为parquet，新增LLaVA-SFT-665K数据源
|
| 90 |
+
- 更新tokenizer，图像占位符改为`<|image_pad|>`
|
| 91 |
+
|
| 92 |
+
</details>
|
| 93 |
+
|
| 94 |
+
<details close>
|
| 95 |
+
<summary> <b>2025-10-24</b> </summary>
|
| 96 |
+
|
| 97 |
+
- bug修复：模型权重不对应
|
| 98 |
+
- 适配[「minimind-1024更新」](https://github.com/jingyaogong/minimind)
|
| 99 |
+
- 代码重构：训练和评估脚本规范化
|
| 100 |
+
- 新增完整的断点续训支持
|
| 101 |
+
|
| 102 |
+
</details>
|
| 103 |
+
|
| 104 |
+
<details close>
|
| 105 |
+
<summary> <b>2025-04-27</b> </summary>
|
| 106 |
+
|
| 107 |
+
- 兼容性更新
|
| 108 |
+
- 适配[「minimind仓库新特性」](https://github.com/jingyaogong/minimind/issues/370)
|
| 109 |
+
- 规范化部分代码
|
| 110 |
+
|
| 111 |
+
</details>
|
| 112 |
+
|
| 113 |
+
<details close>
|
| 114 |
+
|
| 115 |
+
<summary> <b>More...</b> </summary>
|
| 116 |
+
|
| 117 |
+
**2025-02-20**
|
| 118 |
+
|
| 119 |
+
- MiniMind2-V伴随MiniMind2同步更新
|
| 120 |
+
- 大幅减少所有冗余代码，规范代码格式
|
| 121 |
+
- 大幅精简模型冗余结构
|
| 122 |
+
- 更新数据集格式，拓展新的SFT数据集
|
| 123 |
+
- 比前代VLM更优秀的效果！
|
| 124 |
+
|
| 125 |
+
**2024-10-05**
|
| 126 |
+
|
| 127 |
+
- MiniMind-V如期而至，首次开源
|
| 128 |
+
|
| 129 |
+
</details>
|
| 130 |
+
|
| 131 |
+
# 📌 快速开始
|
| 132 |
+
|
| 133 |
+
<details style="color:rgb(128,128,128)">
|
| 134 |
+
<summary>分享本人的软硬件配置（仅供参考）</summary>
|
| 135 |
+
|
| 136 |
+
* CPU: Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz
|
| 137 |
+
* RAM: 128 GB
|
| 138 |
+
* GPU: NVIDIA GeForce RTX 3090(24GB) * 8
|
| 139 |
+
* Ubuntu==20.04
|
| 140 |
+
* CUDA==12.2
|
| 141 |
+
* Python==3.10.16
|
| 142 |
+
* [requirements.txt](./requirements.txt)
|
| 143 |
+
|
| 144 |
+
</details>
|
| 145 |
+
|
| 146 |
+
### 第0步
|
| 147 |
+
|
| 148 |
+
```bash
|
| 149 |
+
# 克隆代码仓库
|
| 150 |
+
git clone https://github.com/jingyaogong/minimind-v
|
| 151 |
+
```
|
| 152 |
+
|
| 153 |
+
```bash
|
| 154 |
+
# 下载siglip2模型到 ./model 目录下
|
| 155 |
+
git clone https://huggingface.co/jingyaogong/siglip2-base-p16-ve
|
| 156 |
+
# or
|
| 157 |
+
git clone https://modelscope.cn/models/gongjy/siglip2-base-p16-ve
|
| 158 |
+
```
|
| 159 |
+
|
| 160 |
+
```bash
|
| 161 |
+
# 下载minimind语言模型权重到 ./out 目录下（作为训练VLM的基座语言模型）
|
| 162 |
+
# HuggingFace
|
| 163 |
+
https://huggingface.co/jingyaogong/minimind-3v-pytorch/blob/main/llm_768.pth
|
| 164 |
+
# 国内源
|
| 165 |
+
https://modelscope.cn/models/gongjy/minimind-3v-pytorch/resolve/master/llm_768.pth
|
| 166 |
+
```
|
| 167 |
+
|
| 168 |
+
## Ⅰ 测试已有模型效果
|
| 169 |
+
|
| 170 |
+
### 1' 环境准备
|
| 171 |
+
|
| 172 |
+
```bash
|
| 173 |
+
pip install -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple
|
| 174 |
+
```
|
| 175 |
+
|
| 176 |
+
### 2' 下载模型
|
| 177 |
+
|
| 178 |
+
```bash
|
| 179 |
+
git clone https://huggingface.co/jingyaogong/minimind-3v
|
| 180 |
+
```
|
| 181 |
+
|
| 182 |
+
### 3' 命令行问答
|
| 183 |
+
|
| 184 |
+
```bash
|
| 185 |
+
# load_from='model': 加载原生PyTorch权重, load_from='其他路径': 加载transformers格式
|
| 186 |
+
python eval_vlm.py --load_from model --weight sft_vlm
|
| 187 |
+
|
| 188 |
+
# 或使用transformers格式模型
|
| 189 |
+
python eval_vlm.py --load_from minimind-3v
|
| 190 |
+
```
|
| 191 |
+
|
| 192 |
+
### 4' 启动WebUI（可选）
|
| 193 |
+
|
| 194 |
+
```bash
|
| 195 |
+
# ⚠️ 须先将 transformers 格式模型文件夹复制到 ./scripts/ 目录下（例如：cp -r minimind-3v ./scripts/minimind-3v），web_demo_vlm 脚本会自动扫描该目录下包含权重文件的子文件夹，如不存在则报错
|
| 196 |
+
cd scripts && python web_demo_vlm.py
|
| 197 |
+
```
|
| 198 |
+
|
| 199 |
+
## Ⅱ 从0开始自己训练
|
| 200 |
+
|
| 201 |
+
### 1' 环境准备
|
| 202 |
+
|
| 203 |
+
```bash
|
| 204 |
+
pip install -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple
|
| 205 |
+
```
|
| 206 |
+
|
| 207 |
+
<details style="color:rgb(128,128,128)">
|
| 208 |
+
<summary>注：提前测试Torch是否可用cuda</summary>
|
| 209 |
+
|
| 210 |
+
```bash
|
| 211 |
+
import torch
|
| 212 |
+
print(torch.cuda.is_available())
|
| 213 |
+
```
|
| 214 |
+
|
| 215 |
+
如果不可用，请自行去[torch_stable](https://download.pytorch.org/whl/torch_stable.html)
|
| 216 |
+
下载whl文件安装。参考[链接](https://blog.csdn.net/weixin_45456738/article/details/141029610?ops_request_misc=&request_id=&biz_id=102&utm_term=%E5%AE%89%E8%A3%85torch&utm_medium=distribute.pc_search_result.none-task-blog-2~all~sobaiduweb~default-2-141029610.nonecase&spm=1018.2226.3001.4187)
|
| 217 |
+
|
| 218 |
+
</details>
|
| 219 |
+
|
| 220 |
+
### 2' 数据下载
|
| 221 |
+
|
| 222 |
+
从下文提供的[数据集链接](https://huggingface.co/datasets/jingyaogong/minimind-v_dataset)
|
| 223 |
+
下载所需内容并放到`./dataset`下。
|
| 224 |
+
|
| 225 |
+
<details style="color:rgb(128,128,128)">
|
| 226 |
+
<summary>注：数据集须知</summary>
|
| 227 |
+
|
| 228 |
+
【注1】之前需解压50万零碎的图像文件可能非常慢。2025-12-27起，数据集格式统一为Parquet，图文一体化存储，体积更小，无需解压，加载更快。
|
| 229 |
+
|
| 230 |
+
【注2】Parquet是列式存储格式，支持高效压缩和快速读取。如果你对它感到陌生，可以预览数据内容，在`dataset/`目录下执行`python lm_dataset.py`可视化前5条图文对
|
| 231 |
+
|
| 232 |
+
Pretrain数据：
|
| 233 |
+
```bash
|
| 234 |
+
wget https://hf-mirror.com/datasets/jingyaogong/minimind-v_dataset/resolve/main/pretrain_i2t.parquet
|
| 235 |
+
```
|
| 236 |
+
|
| 237 |
+
SFT数据：
|
| 238 |
+
```bash
|
| 239 |
+
wget https://hf-mirror.com/datasets/jingyaogong/minimind-v_dataset/resolve/main/sft_i2t.parquet
|
| 240 |
+
```
|
| 241 |
+
|
| 242 |
+
建议预留~2GB空间存放数据集，若无多余空间存放pretrain数据，可尝试跳过pretrain训练步骤直接进行sft训练。
|
| 243 |
+
|
| 244 |
+
</details>
|
| 245 |
+
|
| 246 |
+
### 3' 开始训练
|
| 247 |
+
|
| 248 |
+
**3.1 预训练（学图像描述）**
|
| 249 |
+
|
| 250 |
+
```bash
|
| 251 |
+
# 基础训练命令（从LLM权重开始，仅训练vision_proj）
|
| 252 |
+
python train_pretrain_vlm.py --epochs 4 --from_weight llm
|
| 253 |
+
```
|
| 254 |
+
|
| 255 |
+
> 执行预训练，得到 `pretrain_vlm_*.pth` 作为预训练的输出权重（其中*为模型的dimension，默认为768）
|
| 256 |
+
|
| 257 |
+
|
| 258 |
+
**3.2 监督微调（学看图对话方式）**
|
| 259 |
+
|
| 260 |
+
```bash
|
| 261 |
+
# 基础训练命令（从预训练权重开始，全参数微调）
|
| 262 |
+
python train_sft_vlm.py --epochs 2 --from_weight pretrain_vlm
|
| 263 |
+
```
|
| 264 |
+
|
| 265 |
+
> 执行监督微调，得到 `sft_vlm_*.pth` 作为指令微调的输出权重
|
| 266 |
+
|
| 267 |
+
<details style="color:rgb(128,128,128)">
|
| 268 |
+
<summary>注：训练须知</summary>
|
| 269 |
+
|
| 270 |
+
**训练特性：**
|
| 271 |
+
- 支持断点续训：添加`--from_resume 1`参数可从上次中断处继续训练
|
| 272 |
+
- 支持GPU数量变化：续训时GPU数量改变会自动转换step
|
| 273 |
+
- 原子性保存：使用临时文件+替换机制，防止保存过程中断导致权重损坏
|
| 274 |
+
- 每次保存同时生成`out/**.pth`（模型权重）和`checkpoints/**_resume.pth`（训练状态）文件
|
| 275 |
+
|
| 276 |
+
```bash
|
| 277 |
+
# 训练中断后，使用相同命令并添加 --from_resume 1
|
| 278 |
+
python train_sft_vlm.py --epochs 4 --from_resume 1
|
| 279 |
+
```
|
| 280 |
+
|
| 281 |
+
**参数说明：**
|
| 282 |
+
- `--from_weight`: 基础权重名称（llm, pretrain_vlm, none等）
|
| 283 |
+
- `--save_weight`: 保存权重的前缀名
|
| 284 |
+
- `--from_resume`: 是否续训（0=从头开始，1=从检查点继续）
|
| 285 |
+
- `--freeze_llm`: 是否冻结LLM参数（仅pretrain使用）
|
| 286 |
+
- 更多可直接参考代码
|
| 287 |
+
|
| 288 |
+
</details>
|
| 289 |
+
|
| 290 |
+
|
| 291 |
+
---
|
| 292 |
+
|
| 293 |
+
### 4' 测试模型效果
|
| 294 |
+
|
| 295 |
+
确保需要测试的模型`*.pth`文件位于`./out/`目录下。
|
| 296 |
+
也可以直接去[此处](https://huggingface.co/jingyaogong/minimind-3v-pytorch)下载使用我训练的`*.pth`文件。
|
| 297 |
+
|
| 298 |
+
```bash
|
| 299 |
+
# 测试SFT模型（默认）
|
| 300 |
+
python eval_vlm.py --weight sft_vlm
|
| 301 |
+
|
| 302 |
+
# 测试Pretrain模型
|
| 303 |
+
python eval_vlm.py --weight pretrain_vlm
|
| 304 |
+
```
|
| 305 |
+
|
| 306 |
+
---
|
| 307 |
+
|
| 308 |
+
> [!TIP]
|
| 309 |
+
> 训练脚本均为Pytorch原生框架，均支持多卡加速，假设你的设备有N (N＞1) 张显卡：
|
| 310 |
+
|
| 311 |
+
单机N卡启动训练方式 (DDP, 支持多机多卡集群)
|
| 312 |
+
|
| 313 |
+
```bash
|
| 314 |
+
torchrun --nproc_per_node N train_xxx.py
|
| 315 |
+
```
|
| 316 |
+
|
| 317 |
+
<details style="color:rgb(128,128,128)">
|
| 318 |
+
<summary>注：其它须知</summary>
|
| 319 |
+
|
| 320 |
+
<del>
|
| 321 |
+
单机N卡启动训练 (DeepSpeed)
|
| 322 |
+
|
| 323 |
+
```bash
|
| 324 |
+
deepspeed --master_port 29500 --num_gpus=N train_xxx.py
|
| 325 |
+
```
|
| 326 |
+
</del>
|
| 327 |
+
|
| 328 |
+
可根据需要开启wandb记录训练过程
|
| 329 |
+
|
| 330 |
+
```bash
|
| 331 |
+
# 需要登录: wandb login
|
| 332 |
+
torchrun --nproc_per_node N train_xxx.py --use_wandb
|
| 333 |
+
# and
|
| 334 |
+
python train_xxx.py --use_wandb
|
| 335 |
+
```
|
| 336 |
+
|
| 337 |
+
通过添加`--use_wandb`参数，可以记录训练过程，训练完成后，可以在wandb网站上查看训练过程。通过修改`wandb_project`
|
| 338 |
+
和`wandb_run_name`参数，可以指定项目名称和运行名称。
|
| 339 |
+
|
| 340 |
+
【注】：25年6月后，国内网络环境无法直连WandB，MiniMind项目默认转为使用[SwanLab](https://swanlab.cn/)作为训练可视化工具（完全兼容WandB API），即`import wandb`改为`import swanlab as wandb`即可，其他均无需改动。
|
| 341 |
+
|
| 342 |
+
</details>
|
| 343 |
+
|
| 344 |
+
# 📌 模型细节
|
| 345 |
+
|
| 346 |
+
MiniMind-V (VLM)的基座语言模型MiniMind (LLM)来自孪生项目[minimind](https://github.com/jingyaogong/minimind)，
|
| 347 |
+
具体的模型结构、训练细节、原理、测试效果等均可移步[minimind](https://github.com/jingyaogong/minimind)项目查阅。
|
| 348 |
+
此处为减少冗余，省略讨论LLM的相关部分，默认您已对MiniMind (LLM)的细节有基本的了解。
|
| 349 |
+
|
| 350 |
+
> 即使您不太了解LLM的细节，也可参考“快速开始”流程训练一个MiniMind-V，
|
| 351 |
+
> 这并不受到影响，仓库致力于最低成本的开箱即用！
|
| 352 |
+
|
| 353 |
+
MiniMind-V的结构仅增加Visual Encoder和特征投影两个子模块，增加模态混合分支，以支持多种模态信息的输入：
|
| 354 |
+
|
| 355 |
+
|
| 356 |
+
|
| 357 |
+
|
| 358 |
+
<details>
|
| 359 |
+
<summary> 【重要】一些有趣的思考 </summary>
|
| 360 |
+
|
| 361 |
+
此处不妨展开想一想两个问题：
|
| 362 |
+
|
| 363 |
+
* 什么叫做**L**arge **L**anguage **M**odel (LLM)？
|
| 364 |
+
* 什么叫做多模态模型？
|
| 365 |
+
|
| 366 |
+
[这篇文章](https://www.jiqizhixin.com/articles/2024-09-15-3)完美吻合本人的想法：
|
| 367 |
+
大语言模型（LLM）名字虽然带有语言二字，但它们其实与语言关系不大，这只是历史问题，更确切的名字应该是自回归 Transformer
|
| 368 |
+
或者其他。LLM 更多是一种统计建模的通用技术，它们主要通过自回归 Transformer 来模拟 token 流，而这些 token
|
| 369 |
+
可以代表文本、图片、音频、动作选择、甚至是分子等任何东西。
|
| 370 |
+
因此，只要能将问题转化为模拟一系列离散 token 的流程，理论上都可以应用 LLM 来解决。
|
| 371 |
+
实际上，随着大型语言模型技术栈的日益成熟，我们可能会看到越来越多的问题被纳入这种建模范式。也就是说，问题固定在使用 LLM
|
| 372 |
+
进行『下一个 token 的预测』，只是每个领域中 token 的用途和含义有所不同。
|
| 373 |
+
|
| 374 |
+
[ZJU-LiXi老师](https://person.zju.edu.cn/xilics#694283)同样谈及过类似观点（原话大意如下）：
|
| 375 |
+
文本、视频、语音、动作等在人类看来属于「多模态」信号，但所谓的「模态」其实只是人类在信息存储方式上的一种分类概念。
|
| 376 |
+
就像`.txt`和`.png`文件，虽然在视觉呈现和高级表现形式上有所不同，但它们本质上并没有根本区别。
|
| 377 |
+
之所以出现「多模态」这个概念，仅仅是因为人类在不同的感知层面上对这些信号的分类需求。
|
| 378 |
+
然而，对于机器来说，无论信号来自何种「模态」，最终它们都只是以一串二进制的「单模态」数字序列来呈现。
|
| 379 |
+
机器并不会区分这些信号的模态来源，而只是处理和分析这些序列背后所承载的信息内容。
|
| 380 |
+
|
| 381 |
+
个人认为**G**enerative **P**retrained **T**ransformer (GPT) 比 **L**arge **L**anguage **M**odel (LLM)更为贴切，
|
| 382 |
+
因此本人表达上更习惯用"GPT"去代表LLM/VLM/类GPT架构的系列模型，而非为了蹭OpenAI的热度。
|
| 383 |
+
|
| 384 |
+
至此，我们可以用一句话总结GPT的所作所为：
|
| 385 |
+
|
| 386 |
+
GPT模型根据现有token预测输出下一个下下一个下下下一个token ...，直到模型输出结束符；此处的"token"其实并不需要一定是文本！
|
| 387 |
+
|
| 388 |
+
```text
|
| 389 |
+
> 对于LLM模型，如果需要理解"图片"，我们只要把"图片"作为对一种特殊的从来没见过的"外国语言"，通过"外语词典"翻译后即可作为特殊的语言输入LLM
|
| 390 |
+
> 对于LLM模型，如果需要理解"音频"，我们只要把"音频"作为对一种特殊的从来没见过的"外国语言"，通过"外语词典"翻译后即可作为特殊的语言输入LLM
|
| 391 |
+
> ...
|
| 392 |
+
```
|
| 393 |
+
|
| 394 |
+
<u>**为了得到MiniMind-V，我们只需要完成这2件事即可：**</u>
|
| 395 |
+
|
| 396 |
+
1. 借助擅长翻译图片的 **"外语词典"** ，把图片从 **"外国语言"** 翻译为模型便于理解的 **"LLM语言"**
|
| 397 |
+
2. 训练微调LLM，使其和 **"外语词典"** 度过磨合期，从而更好的理解图片
|
| 398 |
+
|
| 399 |
+
"外语词典" 称之为Visual Encoder模型。
|
| 400 |
+
和LlaVA、Qwen-VL等视觉语言模型类似，MiniMind-V当前选用开源SigLIP2系列模型作为Visual Encoder。
|
| 401 |
+
具体使用[siglip2-base-p16-ve](https://huggingface.co/jingyaogong/siglip2-base-p16-ve)，
|
| 402 |
+
一种基于 ViT-B/16 架构的Visual Encoder用于描述图像文本信息。
|
| 403 |
+
当前使用的 SigLIP2 NaFlex 视觉编码器会根据预处理结果生成最多256个patch token作为encoder编码层的输入，
|
| 404 |
+
最终产生1×768维的嵌入向量用于和文本对计算误差。
|
| 405 |
+
我们并不需要最终嵌入表示，因此只取encoder层的输出，也就是VIT核心主干的输出特征即可。
|
| 406 |
+
它拿到前一层256×768大小的特征，通过reshape将每4个相邻token拼接为1个（256×768 → 64×3072），再经过2层MLP（Linear→GELU→Linear）投影到LLM的隐藏维度，最终作为64个visual token输入MiniMind-V。
|
| 407 |
+
与LLM的结合在获取图像encoder特征后，一方面需要把视觉特征对齐到LLM的文本token维度，
|
| 408 |
+
另一方面，要将图像特征映射到与文本embedding相同的空间，即文本token和原生的视觉token需要磨合并不能直接地一视同仁，
|
| 409 |
+
可以称之为跨模态的特征对齐。
|
| 410 |
+
|
| 411 |
+
[LlaVA-1](https://arxiv.org/pdf/2304.08485)使用简单的线性变换完成对齐，[LlaVA-1.5](https://arxiv.org/pdf/2310.03744)升级为2层MLP，MiniMind-V采用与LlaVA-1.5相同的MLP Projection方案，并结合reshape进行token压缩。
|
| 412 |
+
|
| 413 |
+
|
| 414 |
+
|
| 415 |
+
MiniMind-V的主要结构已介绍完毕。
|
| 416 |
+
|
| 417 |
+
</details>
|
| 418 |
+
|
| 419 |
+
|
| 420 |
+
---
|
| 421 |
+
|
| 422 |
+
下面，我们简单讨论MiniMind-V的外部输入输出的变化。
|
| 423 |
+
|
| 424 |
+
VLM的输入依然是一段文本，其中包含特殊的`<image>`占位符。
|
| 425 |
+
在计算文本嵌入后，可以将图像编码器生成的向量投影到该占位符对应的嵌入部分，替换掉原先的占位符embedding。
|
| 426 |
+
例如：
|
| 427 |
+
|
| 428 |
+
```text
|
| 429 |
+
<image>\n这个图像中有什么内容？
|
| 430 |
+
```
|
| 431 |
+
|
| 432 |
+
在`minimind-v`中，使用64个`<|image_pad|>`组成的占位符代替图像（SigLIP2输出的256个patch特征经reshape+MLP压缩为64个token），因此`minimind-v`的prompt为：
|
| 433 |
+
|
| 434 |
+
```text
|
| 435 |
+
<|image_pad|><|image_pad|>...<|image_pad|>(×64)\n这个图片描述的是什么内容？
|
| 436 |
+
```
|
| 437 |
+
|
| 438 |
+
计算完embedding和projection，用视觉特征替换掉对应占位符的embedding后，整个计算过程到输出则和LLM部分没有差异。
|
| 439 |
+
|
| 440 |
+
|
| 441 |
+
|
| 442 |
+
至此，`MiniMind-V`的所有细节呈现完毕，VLM模型子类继承自`MiniMind`，仅做**最小**变更而产生，核心算法改动`< 50行`，迁移难度极低，和`LlaVA`等模型具体实现存在区别，但思路一致。
|
| 443 |
+
|
| 444 |
+
# 📌 实验
|
| 445 |
+
|
| 446 |
+
## Ⅰ 数据集
|
| 447 |
+
|
| 448 |
+
原始来源：
|
| 449 |
+
- [Chinese-LLaVA-Vision](https://huggingface.co/datasets/LinkSoul/Chinese-LLaVA-Vision-Instructions)：包含约57万张预训练图像，来自CC-3M和COCO 2014
|
| 450 |
+
- [llava-en-zh-300k](https://huggingface.co/datasets/BUAADreamer/llava-en-zh-300k)：包含300k条指令微调数据和15万张图像
|
| 451 |
+
- [LLaVA-SFT-665K](https://huggingface.co/datasets/csuhan/LLaVA-SFT-665K)：包含665k条指令微调数据
|
| 452 |
+
|
| 453 |
+
其中部分为中文数据，部分为英文数据。问答内容经过翻译，对中文支持更友好，进一步经过整理并`resize`（pretrain分辨率128×128，sft分辨率160×160）。
|
| 454 |
+
|
| 455 |
+
(pretrain_i2t.parquet) 预训练数据集格式：
|
| 456 |
+
|
| 457 |
+
```text
|
| 458 |
+
列名: conversations (json string), image_bytes (binary), image_names (string)
|
| 459 |
+
|
| 460 |
+
conversations 示例:
|
| 461 |
+
[
|
| 462 |
+
{"role": "user", "content": "提供给定图像的简要描述。\n<image>"},
|
| 463 |
+
{"role": "assistant", "content": "橄榄油是自由使用的健康成分。"}
|
| 464 |
+
]
|
| 465 |
+
image_bytes: <图像二进制数据>
|
| 466 |
+
```
|
| 467 |
+
|
| 468 |
+
(sft_i2t.parquet) 单图指令微调数据集格式：
|
| 469 |
+
|
| 470 |
+
```text
|
| 471 |
+
列名: conversations (json string), image_bytes (binary), image_names (string)
|
| 472 |
+
|
| 473 |
+
conversations 示例:
|
| 474 |
+
[
|
| 475 |
+
{"role": "user", "content": "闹钟的位置对睡眠质量有什么影响？<image>"},
|
| 476 |
+
{"role": "assistant", "content": "把数字闹钟放在床头柜..."}
|
| 477 |
+
]
|
| 478 |
+
image_bytes: <图像二进制数据>
|
| 479 |
+
```
|
| 480 |
+
|
| 481 |
+
> 注：sft_i2t.parquet 共约 58 万条数据，其中约 23.6 万条为含图对话（i2t），约 34.6 万条为纯文本对话（t2t），后者用于保持模型的基础语言能力。
|
| 482 |
+
|
| 483 |
+
数据集下载地址：([ModelScope](https://www.modelscope.cn/datasets/gongjy/minimind-v_dataset) | [HuggingFace](https://huggingface.co/datasets/jingyaogong/minimind-v_dataset))
|
| 484 |
+
|
| 485 |
+
## Ⅱ 训练
|
| 486 |
+
|
| 487 |
+
训练分为两个阶段，均冻结Visual Encoder梯度，仅训练Projection和LLM部分。
|
| 488 |
+
训练基于LLM预训练权重初始化，支持DDP多卡训练、混合精度（bfloat16）、torch.compile加速和swanlab日志记录。
|
| 489 |
+
|
| 490 |
+
> train_pretrain_vlm
|
| 491 |
+
|
| 492 |
+
预训练阶段从约113万条图文描述数据中学习图片的通用知识（如鹿是鹿，狗是狗）。
|
| 493 |
+
此阶段采用较高学习率（1e-4），最大序列长度360，冻结LLM主体参数，仅设置Projection和LLM的第0层可学习，
|
| 494 |
+
目的是让模型快速建立视觉特征到语言空间的基础映射，同时避免破坏LLM已有的语言能力。
|
| 495 |
+
|
| 496 |
+
> train_sft_vlm
|
| 497 |
+
|
| 498 |
+
指令微���阶段从约58万条数据中学习真实问答格式，其中约23.6万条为图文多轮对话，约34.6万条为纯文本对话（用于保持LLM基础能力）。
|
| 499 |
+
此阶段采用较低学习率（1e-5~1e-6），最大序列长度768，解冻Projection和LLM全部参数进行全量微调，
|
| 500 |
+
使模型学会根据图片内容进行多轮对话，并通过混入的纯文本数据缓解灾难性遗忘。
|
| 501 |
+
|
| 502 |
+
> 训练时间和Loss走势（仅供参考）
|
| 503 |
+
|
| 504 |
+
Pretrain [768+8] (dense & moe)
|
| 505 |
+
|
| 506 |
+
|
| 507 |
+
SFT [768+8] (dense & moe)
|
| 508 |
+
|
| 509 |
+
|
| 510 |
+
## Ⅲ 模型权重
|
| 511 |
+
|
| 512 |
+
| 模型格式 | ModelScope | HuggingFace |
|
| 513 |
+
|---|---|---|
|
| 514 |
+
| 原生PyTorch (`*.pth`) | [minimind-3v-pytorch](https://www.modelscope.cn/models/gongjy/minimind-3v-pytorch) | [minimind-3v-pytorch](https://huggingface.co/jingyaogong/minimind-3v-pytorch) |
|
| 515 |
+
| Transformers 格式 | [minimind-v collection](https://modelscope.cn/collections/MiniMind-V-42b841dde22d41) | [minimind-v collection](https://huggingface.co/collections/jingyaogong/minimind-v-67000833fb60b3a2e1f3597d) |
|
| 516 |
+
|
| 517 |
+
> 注：Transformers版本均为单图指令微调后的`MiniMind-V`模型
|
| 518 |
+
|
| 519 |
+
# 📌 评估
|
| 520 |
+
|
| 521 |
+
### 效果测试
|
| 522 |
+
|
| 523 |
+
#### 单图对话
|
| 524 |
+
|
| 525 |
+
<table>
|
| 526 |
+
<thead>
|
| 527 |
+
<tr>
|
| 528 |
+
<th>image</th>
|
| 529 |
+
<th>minimind-3v</th>
|
| 530 |
+
<th>minimind-3v-moe</th>
|
| 531 |
+
</tr>
|
| 532 |
+
</thead>
|
| 533 |
+
<tbody>
|
| 534 |
+
<tr>
|
| 535 |
+
<td>
|
| 536 |
+
<img src="./dataset/eval_images/airplane-flying-blue-sky.jpg" alt="airplane">
|
| 537 |
+
|
| 538 |
+
</td>
|
| 539 |
+
<td>在这幅图片中，一架白色的飞机正降落在一片广阔的天空中。天空中飘浮着许多流线型的建筑物，这些建筑物散落在天空中。除了主要的飞机外，还有两辆汽车在场景中各处停放，包括一辆巴士和一辆小型汽车。这架飞机似乎停在地面上，表明它们正在进行商业活动。</td>
|
| 540 |
+
<td>在这张照片中，有一架大型的飞机正在降落，这表明它是为这架飞机而设计的。此外，它停在云层之下，这表明它在移动。天空中有云朵，暗示着这架飞机正在空中飞行。整个场景的背景显示出一种宁静祥和的气氛，暗示这架飞机正在飞行中作为一个机会来进行外交活动或与其他客机接触。</td>
|
| 541 |
+
</tr>
|
| 542 |
+
<tr>
|
| 543 |
+
<td>
|
| 544 |
+
<img src="./dataset/eval_images/birthday-cake-candles-table.jpg" alt="birthday-cake">
|
| 545 |
+
|
| 546 |
+
</td>
|
| 547 |
+
<td>图片显示了一张大生日蛋糕，上面刻有各种生日和装饰物的生日和一束蜡烛。这些甜蜜而独特的气味使其成为这个特殊场合的焦点。蛋糕被放在餐桌上，有蛋糕放在上面，很可能是放在桌上，用蜡烛加上蜡烛来营造出诱人的场景。此外，这张生日蛋糕还配上了蜡烛作为装饰物，进一步丰富了整个生日的氛围。</td>
|
| 548 |
+
<td>这张照片捕捉到了一个生日聚会，各种美味可口的生日蛋糕和一把勺子放在那里。它被放置在一个充满生气的蛋糕盒子里，里面装着各种各样的草莓糖霜糖浆。蛋糕上有一个大号的蜡烛，给人一种温馨和诱人的图案。周围有很多小点心，比如小心地蜡烛和糖霜，让整个蛋糕看起来更加令人放松和诱人。</td>
|
| 549 |
+
</tr>
|
| 550 |
+
<tr>
|
| 551 |
+
<td>
|
| 552 |
+
<img src="./dataset/eval_images/pizza-on-wooden-board.jpg" alt="pizza">
|
| 553 |
+
|
| 554 |
+
</td>
|
| 555 |
+
<td>在这张图片中，有一个装饰着奶酪的比萨饼和一片新鲜的青绿野餐毯。这看起来像是一块沙拉，给人一种清新、诱人、美味的享受体验。披萨的大小和大小暗示着一种随意和随意的用餐体验。画面中，一群人围坐在一块砖块上，其中一些则分散在桌子上，周围摆放着不同种类的青绿野餐毯。此外，桌子上还放着一块披萨片，上面摆放着各种各样的青绿野餐毯，营造出一种轻松愉快的氛围。</td>
|
| 556 |
+
<td>图中，比萨饼在木桌上。它被切成了比萨饼的大小，而且披萨放置在木桌上。比萨有很多配料，包括奶酪和各种酱汁。披萨有多种口味，包括经典的牛排式和意大利式，还有一些加在比萨表面。比萨上有很多新鲜水果，如西红柿和莫吉托，以及奶酪，使这幅图片更加丰富和有吸引力。</td>
|
| 557 |
+
</tr>
|
| 558 |
+
<tr>
|
| 559 |
+
<td>
|
| 560 |
+
<img src="./dataset/eval_images/red-sports-car-road.jpg" alt="red-car">
|
| 561 |
+
|
| 562 |
+
</td>
|
| 563 |
+
<td>在这幅图片中，一辆白色的马车停泊在一条红色的路牌上。这个车辆位于一条绿油油的道路上，很可能是一个购物中心或高速公路。在这辆车的后部，可以看到一个绿色的马车停泊在路上，这是典型的户外场所。这辆马车可能是为了娱乐或观赏车辆而停放。</td>
|
| 564 |
+
<td>画面显示了一辆红色高性能赛车，停在一辆大型汽车后面。这辆车可以看到车身、汽车、汽车及汽车停在里面。它似乎是一辆大型红色汽车，有各种大小的汽车，表明它可能是汽车制造商生产的。此外，车辆周围的环境暗示了一种户外环境，因为一辆汽车也出现在场景中。</td>
|
| 565 |
+
</tr>
|
| 566 |
+
<tr>
|
| 567 |
+
<td>
|
| 568 |
+
<img src="./dataset/eval_images/row-of-colorful-houses.jpg" alt="colorful-houses">
|
| 569 |
+
|
| 570 |
+
</td>
|
| 571 |
+
<td>画面中，一座蓝色白色的大房子位于一条城市街道上，为这个地区增添了一丝自然与奇思妙想。墙上挂着一盏交通灯，为整个场景增添了特色和引人入胜。</td>
|
| 572 |
+
<td>画面中，一座蓝色房子旁边有很多小花瓶。这表明这座房子里可能正在营运一些小型小型花盆或盆栽植物。一些人聚集在房子周围，可能正在享受户外生活中的某种乐趣或美景。有些人站在画面中，而其他人则散布于场景中。总体而言，这幅场景捕捉到了一个美丽而令人愉快的场景，展示出该房子里一个令人放松和宁静的环境。</td>
|
| 573 |
+
</tr>
|
| 574 |
+
<tr>
|
| 575 |
+
<td>
|
| 576 |
+
<img src="./dataset/eval_images/snow-mountain-lake-view.jpg" alt="snow-mountain">
|
| 577 |
+
|
| 578 |
+
</td>
|
| 579 |
+
<td>在这幅图片中，有一个高高的山，它看起来像是大森林中的一片高山。天空中闪烁着不同颜色的星星，给画面增加了一抹红点。天空中有两朵高大的树，树木高高地挂在一起，暗示着森林中的宁静与自然之美。在画面的中心，可以看到一棵高大的松树，树干上覆盖着一层薄薄的苔藓。这种高高的松树与周围的大山构成了一个有趣而引人入胜的背景，为这片自然之美增添了一丝神秘色彩。</td>
|
| 580 |
+
<td>这幅图片展示了一个令人印象深刻的宁静湖面。湖水从天上飘浮着，暗示着一个令人平静和放松的水面。湖边上有几匹高大、形状各异的景象，它们在湖面上显得格外美丽。此外，在湖的边缘，有一座巨大的高山，为整个湖景增添了几分神秘色彩。湖景中的天空也被描述得如画，给整个画面增添了一种纯净和宁静的气氛。</td>
|
| 581 |
+
</tr>
|
| 582 |
+
<tr>
|
| 583 |
+
<td>
|
| 584 |
+
<img src="./dataset/eval_images/street-food-hotpot-table.jpg" alt="street-food">
|
| 585 |
+
|
| 586 |
+
</td>
|
| 587 |
+
<td>图中，一大群人聚集在一张大餐桌旁欣赏着烤肉和热带水果，其中一人站在一碗里摆放着各种各样的盘子。桌子上放着几个碗，上面摆满了肉类和其他配料。有些放在盘子里，其他的则放在桌子上。在这张餐桌周围，有几个盘子，其中两个放着一杯酒，另一个放在靠近餐桌的左侧。</td>
|
| 588 |
+
<td>画面中，一群人聚集在一家大餐馆里，享受着一顿饭。这家餐厅的菜单上有一些生菜和猪肉，但它们已经被切成了四份，上面还有一个碗。他们拿着烤肉准备食用。在背景中，有几个瓶子在场景中。还有一把勺子位于桌子左侧，使盘子看起来更吸引人。一盆盆栽植物放在桌子左侧上方，为空间增添了一抹绿色。场景中的其他元素包括一个碗，里面放着葡萄酒和两根葡萄酒。</td>
|
| 589 |
+
</tr>
|
| 590 |
+
<tr>
|
| 591 |
+
<td>
|
| 592 |
+
<img src="./dataset/eval_images/three-kittens-basket.jpg" alt="kittens">
|
| 593 |
+
|
| 594 |
+
</td>
|
| 595 |
+
<td>图中，一只棕色的小灰猫正坐在篮子里。这只猫身上戴着一顶棕色的帽子，很可能是一个戴帽子的男人。在篮子里，一只棕色的紫色小灰猫正沿着篮子里去休息。这些小猫似乎也在享受这份温暖，但它们似乎并没有完全放过来。此外，背景中还可以看到一把剪刀。这把剪刀看起来是专门为小猫设计的，它可以用作家庭相册或礼物。在篮子的侧边，有几条篮子，其中一条是最亮的，另一条则是最暗的。在篮子中可以看到一只棕色小灰猫，而另一条则是更暗的。</td>
|
| 596 |
+
<td>在这张照片中，一只小猫坐在篮子里，紧挨着它坐在篮子里的那块木篮上。猫的身体上有九条纹毛发。这个场景描绘了它们之间的亲密关系，展示了它们在一起度过时光的不同场合。画面中，一群大的猫坐在篮子里，其中一只小猫也被描述为小猫，这可能表明他们正在享受与猫互动、与它们的互动或一起度过愉快时光。</td>
|
| 597 |
+
</tr>
|
| 598 |
+
<tr>
|
| 599 |
+
<td>
|
| 600 |
+
<img src="./dataset/eval_images/tropical-beach-palm-tree.jpg" alt="tropical-beach">
|
| 601 |
+
|
| 602 |
+
</td>
|
| 603 |
+
<td>在这张图片中，沙滩上有很多椅子，还有一些人站着，可以看到一把遮阳伞。虽然它看起来很大，但却没有任何特别的设计。沙滩上有许多椅子，表明这是一家餐馆或者服务员办公室。其中最引人注目的是一张海边椅子，椅子上放着一只热带海滩椅。这个椅子非常适合放松身心、享受海滩时光。此外，还有一些椅子和其他人在场景中，可能是为了放置食物或其他用途。靠近椅子的椅子表示该位置可供使用的其他人使用，也许也有一人在靠近那个椅子的地方。</td>
|
| 604 |
+
<td>图片显示了一个美丽的海滩场景，有很多椅子散布在天然的棕榈树上。其中一个椅子靠近海滩，而另一个则较小。沙滩上有两把椅子，其中一把靠近中间，另一把则稍微偏左，还有一些则在靠近边缘处。在海边的海滩周围，你可以看到几个人坐在海边的沙滩上，有的靠近海水中，还有一张沙滩椅。其中一张椅子靠近海滩，另一张椅子靠近海边。此外，还可以看到几只遮阳伞，为沙滩上的躺椅提供了遮阴。</td>
|
| 605 |
+
</tr>
|
| 606 |
+
<tr>
|
| 607 |
+
<td>
|
| 608 |
+
<img src="./dataset/eval_images/yellow-school-bus-road.jpg" alt="school-bus">
|
| 609 |
+
|
| 610 |
+
</td>
|
| 611 |
+
<td>画面中，一辆蓝色的黄色公共汽车正从一辆黄色公共汽车驶过，在道路上停泊着。这辆公共汽车看起来是在一个黄色的黄色高速公路上。图中有几个人，其中一些靠近前景，而另一些则靠后一些，但都没有看到。在黄色公共汽车附近，可以看到一辆停在路边，那辆停在路边。此外，还有两辆不同方向的巴士，一辆靠近前景，另一辆靠近前景，另一辆稍微靠后一些。</td>
|
| 612 |
+
<td>画面中，一辆黄色和黄色相间的黄色和蓝色交叉路口的蓝色公共汽车正在一条通往路缘上的红色公交车站。有几辆公共汽车正停在路边，它们离一排车道很近。在背景中，可以看到一些长凳，它们在城市里交叉起来。一个长凳位于图中最左侧，而另一个则稍微靠后一点，为画面增添了一些城市特色。整个场景中有很多人和车辆散落在场景各处，包括黄色和蓝色的交叉路口。整个场景给人一种忙碌和迷茫的感觉，这也突显了公共汽车在市区中的存在和目的。</td>
|
| 613 |
+
</tr>
|
| 614 |
+
</tbody>
|
| 615 |
+
</table>
|
| 616 |
+
|
| 617 |
+
### 效果小结：
|
| 618 |
+
|
| 619 |
+
两个模型均能识别图像主体（飞机、蛋糕、汽车、海滩等），但普遍存在重复表述和幻觉细节。受限于模型和数据规模，整体处于"能看懂大意、细节不准"的阶段。
|
| 620 |
+
|
| 621 |
+
视觉信号对于LLM视作一种特殊的外语，因此"学习外语"的能力高低，很大程度上取决于LLM的能力。LLM性能越强，对应的VLM越强，此时效果增益会很明显。
|
| 622 |
+
|
| 623 |
+
#### 未来值得改进的方面：
|
| 624 |
+
|
| 625 |
+
```text
|
| 626 |
+
> 可引入动态分辨率和Tile-based编码（如LLaVA-NeXT），突破固定分辨率限制。
|
| 627 |
+
> Visual Encoder可升级为更强的视觉编码器，获取更细粒度的图像特征。
|
| 628 |
+
> 拓展多图理解、视频理解和视觉定位（Visual Grounding）能力。
|
| 629 |
+
> ...
|
| 630 |
+
```
|
| 631 |
+
|
| 632 |
+
# 📌 致谢
|
| 633 |
+
|
| 634 |
+
> [!TIP]
|
| 635 |
+
> 如果您觉得 `MiniMind-V`对您有所帮助，可以在 GitHub 上加一个⭐<br/>
|
| 636 |
+
> 水平有限难免存在未���的纰漏，欢迎所有人在Issues交流指正或提交PR改进项目<br/>
|
| 637 |
+
> 您的支持就是持续改进项目的动力，谢谢！
|
| 638 |
+
|
| 639 |
+
## 🤝[贡献者](https://github.com/jingyaogong/minimind-v/graphs/contributors)
|
| 640 |
+
|
| 641 |
+
<a href="https://github.com/jingyaogong/minimind-v/graphs/contributors">
|
| 642 |
+
<img width="200" src="https://contrib.rocks/image?repo=jingyaogong/minimind-v" />
|
| 643 |
+
</a>
|
| 644 |
+
|
| 645 |
+
## 😊鸣谢
|
| 646 |
+
|
| 647 |
+
<a href="https://github.com/xinyanghuang7"><b>@xinyanghuang7</b></a>: <a href="https://github.com/xinyanghuang7/minimind-v/tree/hxy">多图vlm分支</a> | <a href="https://github.com/jingyaogong/minimind-v/tree/32cf4c5c01337231fd907b92d513de8945594263">仓库截至此版本提供</a>
|
| 648 |
+
|
| 649 |
+
<details close>
|
| 650 |
+
<summary> <b>参考链接 & 感谢以下优秀的论文或项目</b> </summary>
|
| 651 |
+
|
| 652 |
+
- 排名不分任何先后顺序
|
| 653 |
+
- [LlaVA](https://arxiv.org/pdf/2304.08485)
|
| 654 |
+
- [LlaVA-VL](https://arxiv.org/pdf/2310.03744)
|
| 655 |
+
- [Chinese-LLaVA-Vision-Instructions](https://huggingface.co/datasets/LinkSoul/Chinese-LLaVA-Vision-Instructions)
|
| 656 |
+
|
| 657 |
+
</details>
|
| 658 |
+
|
| 659 |
+
## 🫶支持者
|
| 660 |
+
|
| 661 |
+
<a href="https://github.com/jingyaogong/minimind-v/stargazers">
|
| 662 |
+
<picture>
|
| 663 |
+
<source media="(prefers-color-scheme: dark)" srcset="https://reporoster.com/stars/dark/jingyaogong/minimind-v"/>
|
| 664 |
+
<source media="(prefers-color-scheme: light)" srcset="https://reporoster.com/stars/jingyaogong/minimind-v"/>
|
| 665 |
+
<img alt="github contribution grid snake animation" src="https://reporoster.com/stars/jingyaogong/minimind-v"/>
|
| 666 |
+
</picture>
|
| 667 |
+
</a>
|
| 668 |
+
|
| 669 |
+
<a href="https://github.com/jingyaogong/minimind-v/network/members">
|
| 670 |
+
<picture>
|
| 671 |
+
<source media="(prefers-color-scheme: dark)" srcset="https://reporoster.com/forks/dark/jingyaogong/minimind-v"/>
|
| 672 |
+
<source media="(prefers-color-scheme: light)" srcset="https://reporoster.com/forks/jingyaogong/minimind-v"/>
|
| 673 |
+
<img alt="github contribution grid snake animation" src="https://reporoster.com/forks/jingyaogong/minimind-v"/>
|
| 674 |
+
</picture>
|
| 675 |
+
</a>
|
| 676 |
+
|
| 677 |
+
<picture>
|
| 678 |
+
<source media="(prefers-color-scheme: dark)" srcset="https://api.star-history.com/svg?repos=jingyaogong/minimind-v&type=Date&theme=dark"/>
|
| 679 |
+
<source media="(prefers-color-scheme: light)" srcset="https://api.star-history.com/svg?repos=jingyaogong/minimind-v&type=Date"/>
|
| 680 |
+
<img alt="Star History Chart" src="https://api.star-history.com/svg?repos=jingyaogong/minimind-v&type=Date"/>
|
| 681 |
+
</picture>
|
| 682 |
+
|
| 683 |
+
# 🎓 引用
|
| 684 |
+
|
| 685 |
+
如果您觉得 MiniMind-V 对您的研究或工作有所帮助，请引用：
|
| 686 |
+
|
| 687 |
+
```bibtex
|
| 688 |
+
@misc{minimind-v,
|
| 689 |
+
title = {MiniMind-V: Train a Tiny VLM from Scratch},
|
| 690 |
+
author = {Jingyao Gong},
|
| 691 |
+
year = {2024},
|
| 692 |
+
url = {https://github.com/jingyaogong/minimind-v},
|
| 693 |
+
note = {GitHub repository, accessed 2026}
|
| 694 |
+
}
|
| 695 |
+
```
|
| 696 |
+
|
| 697 |
+
# 📜 许可协议
|
| 698 |
+
|
| 699 |
+
本仓库遵循 [Apache-2.0 License](LICENSE) 开源协议。
|
README_en.md
ADDED
|
@@ -0,0 +1,716 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
<div align="center">
|
| 2 |
+
|
| 3 |
+
|
| 4 |
+
|
| 5 |
+
</div>
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
<div align="center">
|
| 9 |
+
|
| 10 |
+
[](https://github.com/jingyaogong/minimind-v/stargazers)
|
| 11 |
+
[](LICENSE)
|
| 12 |
+
[](https://github.com/jingyaogong/minimind-v/commits/master)
|
| 13 |
+
[](https://github.com/jingyaogong/minimind-v/pulls)
|
| 14 |
+
[](https://huggingface.co/collections/jingyaogong/minimind-v-67000833fb60b3a2e1f3597d)
|
| 15 |
+
|
| 16 |
+
</div>
|
| 17 |
+
|
| 18 |
+
<div align="center">
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
</div>
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
<div align="center">
|
| 26 |
+
<h3>"The Greatest Path is the Simplest"</h3>
|
| 27 |
+
</div>
|
| 28 |
+
|
| 29 |
+
<div align="center">
|
| 30 |
+
|
| 31 |
+
[中文](./README.md) | English
|
| 32 |
+
|
| 33 |
+
</div>
|
| 34 |
+
|
| 35 |
+
* This project aims to train a super-small multimodal vision-language model, **MiniMind-V**, with just a cost of 1.3 RMB
|
| 36 |
+
and 1 hours of work, starting from scratch!
|
| 37 |
+
* The smallest version of **MiniMind-V** is only about $\frac{1}{2600}$ the size of GPT-3, designed to enable fast
|
| 38 |
+
inference and even training on personal GPUs.
|
| 39 |
+
* **MiniMind-V** is an extension of the visual capabilities of the [MiniMind](https://github.com/jingyaogong/minimind)
|
| 40 |
+
pure language model.
|
| 41 |
+
* The project includes full code for the minimalist structure of large VLM models, dataset cleaning, pretraining, and
|
| 42 |
+
supervised fine-tuning (SFT).
|
| 43 |
+
* This is not only the smallest implementation of an open-source VLM model but also a concise tutorial for beginners in
|
| 44 |
+
vision-language models.
|
| 45 |
+
* The hope is that this project can provide a useful example to inspire others and share the joy of creation, helping to
|
| 46 |
+
drive progress in the wider AI community!
|
| 47 |
+
|
| 48 |
+
> To avoid misunderstandings, the "1 hours" is based on testing (`1 epoch`) with an NVIDIA 3090 hardware device (single GPU), and
|
| 49 |
+
> the "1.3 RMB" refers to GPU server rental costs.
|
| 50 |
+
|
| 51 |
+
<div align="center">
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
[🔗🤖 Online Experience](https://www.modelscope.cn/studios/gongjy/MiniMind-V) | [🔗🎞️ Video Introduction](https://www.bilibili.com/video/BV1Sh1vYBEzY)
|
| 56 |
+
|
| 57 |
+
</div>
|
| 58 |
+
|
| 59 |
+
# 📌 Introduction
|
| 60 |
+
|
| 61 |
+
“Building a plane with Legos is much more exciting than flying in first class!”
|
| 62 |
+
Is it really as complex as imagined to build a VLM-based multimodal large model? How is the code implementation done?
|
| 63 |
+
Is the training process difficult? Now, let's explore the answers and feel the joy of creation together!
|
| 64 |
+
|
| 65 |
+
> [!TIP]
|
| 66 |
+
> (As of 2026-02-15) The MiniMind-V series has completed the training of the following model versions, with the smallest
|
| 67 |
+
> requiring only 67M (0.067B) parameters, capable of both image recognition and conversation!
|
| 68 |
+
|
| 69 |
+
| Model (Size) | Inference Memory | Release |
|
| 70 |
+
|---------------------------|------------------|------------|
|
| 71 |
+
| minimind-3v-moe (201M-A67M) | 1.0 GB | 2026.04.01 |
|
| 72 |
+
| minimind-3v (67M) | 0.5 GB | 2026.04.01 |
|
| 73 |
+
| MiniMind2-V (104M) | 1.1 GB | 2025.02.20 |
|
| 74 |
+
| MiniMind2-Small-V (26M) | 0.6 GB | 2025.02.20 |
|
| 75 |
+
| minimind-v-v1-small (27M) | 0.6 GB | 2024.10.04 |
|
| 76 |
+
| minimind-v-v1 (109M) | 1.1 GB | 2024.10.04 |
|
| 77 |
+
|
| 78 |
+
### 👉**Recent Updates**
|
| 79 |
+
|
| 80 |
+
<details close>
|
| 81 |
+
<summary> <b>2026-02-15</b> </summary>
|
| 82 |
+
|
| 83 |
+
- Added minimind-3v (67M) and minimind-3v-moe (201M-A67M) models
|
| 84 |
+
- Unified 768+8 architecture, supporting both dense and moe modes
|
| 85 |
+
- Dataset format updated to parquet, added LLaVA-SFT-665K data source
|
| 86 |
+
- Updated tokenizer, image placeholder changed to `<|image_pad|>`
|
| 87 |
+
|
| 88 |
+
</details>
|
| 89 |
+
|
| 90 |
+
<details close>
|
| 91 |
+
<summary> <b>2025-10-24</b> </summary>
|
| 92 |
+
|
| 93 |
+
- Bug fix: model weights mismatch
|
| 94 |
+
- Adapted to ["minimind-1024 update"](https://github.com/jingyaogong/minimind)
|
| 95 |
+
- Code refactoring: training and evaluation scripts standardized
|
| 96 |
+
- Added complete checkpoint resumption support
|
| 97 |
+
|
| 98 |
+
</details>
|
| 99 |
+
|
| 100 |
+
<details close>
|
| 101 |
+
<summary> <b>2025-04-27</b> </summary>
|
| 102 |
+
|
| 103 |
+
- Compatibility updates
|
| 104 |
+
- Adapted to the new feature in the "minimind" repository
|
| 105 |
+
- Standardized parts of the code
|
| 106 |
+
|
| 107 |
+
</details>
|
| 108 |
+
|
| 109 |
+
<details close>
|
| 110 |
+
|
| 111 |
+
<summary> <b>More...</b> </summary>
|
| 112 |
+
|
| 113 |
+
**2025-02-20**
|
| 114 |
+
|
| 115 |
+
- MiniMind2-V updated alongside MiniMind2
|
| 116 |
+
- Significant reduction of all redundant code, standardized code format
|
| 117 |
+
- Major simplification of the model's redundant structure
|
| 118 |
+
- Updated dataset format, expanded with new SFT datasets
|
| 119 |
+
- Better performance than the previous VLM version!
|
| 120 |
+
|
| 121 |
+
**2024-10-05**
|
| 122 |
+
|
| 123 |
+
- MiniMind-V released on schedule, first open-source release
|
| 124 |
+
|
| 125 |
+
</details>
|
| 126 |
+
|
| 127 |
+
# 📌 Quick Start
|
| 128 |
+
|
| 129 |
+
<details style="color:rgb(128,128,128)">
|
| 130 |
+
<summary>Sharing my hardware and software configuration (for reference only)</summary>
|
| 131 |
+
|
| 132 |
+
* CPU: Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz
|
| 133 |
+
* RAM: 128 GB
|
| 134 |
+
* GPU: NVIDIA GeForce RTX 3090(24GB) * 8
|
| 135 |
+
* Ubuntu==20.04
|
| 136 |
+
* CUDA==12.2
|
| 137 |
+
* Python==3.10.16
|
| 138 |
+
* [requirements.txt](./requirements.txt)
|
| 139 |
+
|
| 140 |
+
</details>
|
| 141 |
+
|
| 142 |
+
### Step 0
|
| 143 |
+
|
| 144 |
+
```bash
|
| 145 |
+
# Clone the code repository
|
| 146 |
+
git clone https://github.com/jingyaogong/minimind-v
|
| 147 |
+
```
|
| 148 |
+
|
| 149 |
+
```bash
|
| 150 |
+
# Download the siglip2 model to the ./model directory
|
| 151 |
+
git clone https://huggingface.co/jingyaogong/siglip2-base-p16-ve
|
| 152 |
+
# or
|
| 153 |
+
git clone https://modelscope.cn/models/gongjy/siglip2-base-p16-ve
|
| 154 |
+
```
|
| 155 |
+
|
| 156 |
+
```bash
|
| 157 |
+
# Download the minimind language model to the ./out directory (as the base language model for training VLM):
|
| 158 |
+
# HuggingFace
|
| 159 |
+
https://huggingface.co/jingyaogong/minimind-3v-pytorch/blob/main/llm_768.pth
|
| 160 |
+
# Domestic source
|
| 161 |
+
https://modelscope.cn/models/gongjy/minimind-3v-pytorch/resolve/master/llm_768.pth
|
| 162 |
+
```
|
| 163 |
+
|
| 164 |
+
|
| 165 |
+
## Ⅰ Test an existing model's performance
|
| 166 |
+
|
| 167 |
+
### 1' Environment Preparation
|
| 168 |
+
|
| 169 |
+
```bash
|
| 170 |
+
pip install -r requirements.txt -i https://mirrors.aliyun.com/pypi/simple
|
| 171 |
+
```
|
| 172 |
+
|
| 173 |
+
### 2' Download the model
|
| 174 |
+
|
| 175 |
+
```bash
|
| 176 |
+
git clone https://huggingface.co/jingyaogong/minimind-3v
|
| 177 |
+
```
|
| 178 |
+
|
| 179 |
+
### 3' Command-line Q&A
|
| 180 |
+
|
| 181 |
+
```bash
|
| 182 |
+
# load_from='model': load native PyTorch weights, load_from='other path': load transformers format
|
| 183 |
+
python eval_vlm.py --load_from model --weight sft_vlm
|
| 184 |
+
|
| 185 |
+
# Or use transformers format model
|
| 186 |
+
python eval_vlm.py --load_from minimind-3v
|
| 187 |
+
```
|
| 188 |
+
|
| 189 |
+
### 4' Or start the WebUI
|
| 190 |
+
|
| 191 |
+
```bash
|
| 192 |
+
# ⚠️ You must first copy the transformers model folder to the ./scripts/ directory (e.g.: cp -r minimind-3v ./scripts/minimind-3v). The web_demo_vlm script will automatically scan subdirectories containing weight files; it will report an error if none are found.
|
| 193 |
+
cd scripts && python web_demo_vlm.py
|
| 194 |
+
```
|
| 195 |
+
|
| 196 |
+
## Ⅱ Train from scratch
|
| 197 |
+
|
| 198 |
+
### 1' Environment Preparation
|
| 199 |
+
|
| 200 |
+
```bash
|
| 201 |
+
pip install -r requirements.txt -i https://mirrors.aliyun.com/pypi/simple
|
| 202 |
+
```
|
| 203 |
+
|
| 204 |
+
<details style="color:rgb(128,128,128)">
|
| 205 |
+
<summary>Note: Test if Torch can use CUDA</summary>
|
| 206 |
+
|
| 207 |
+
```bash
|
| 208 |
+
import torch
|
| 209 |
+
print(torch.cuda.is_available())
|
| 210 |
+
```
|
| 211 |
+
|
| 212 |
+
If unavailable, download the whl file from [torch_stable](https://download.pytorch.org/whl/torch_stable.html) for
|
| 213 |
+
installation. Refer
|
| 214 |
+
to [this link](https://blog.csdn.net/weixin_45456738/article/details/141029610?ops_request_misc=&request_id=&biz_id=102&utm_term=%E5%AE%89%E8%A3%85torch&utm_medium=distribute.pc_search_result.none-task-blog-2~all~sobaiduweb~default-2-141029610.nonecase&spm=1018.2226.3001.4187)
|
| 215 |
+
for help.
|
| 216 |
+
|
| 217 |
+
</details>
|
| 218 |
+
|
| 219 |
+
### 2' Download Data
|
| 220 |
+
|
| 221 |
+
Download the required content from the [dataset link](https://huggingface.co/datasets/jingyaogong/minimind-v_dataset)
|
| 222 |
+
and place it under `./dataset`.
|
| 223 |
+
|
| 224 |
+
<details style="color:rgb(128,128,128)">
|
| 225 |
+
<summary>Note: Dataset Details</summary>
|
| 226 |
+
|
| 227 |
+
**[Note 1]** Previously, extracting 500k fragmented image files could be very slow. From 2025-12-27, dataset format is unified to Parquet with image-text integrated storage, smaller size, no decompression needed, faster loading.
|
| 228 |
+
|
| 229 |
+
**[Note 2]** Parquet is a columnar storage format supporting efficient compression and fast reading. To preview data content, run `python lm_dataset.py` in the `dataset/` directory to visualize the first 5 image-text pairs.
|
| 230 |
+
|
| 231 |
+
Pretrain data:
|
| 232 |
+
```bash
|
| 233 |
+
wget https://hf-mirror.com/datasets/jingyaogong/minimind-v_dataset/resolve/main/pretrain_i2t.parquet
|
| 234 |
+
```
|
| 235 |
+
|
| 236 |
+
SFT data:
|
| 237 |
+
```bash
|
| 238 |
+
wget https://hf-mirror.com/datasets/jingyaogong/minimind-v_dataset/resolve/main/sft_i2t.parquet
|
| 239 |
+
```
|
| 240 |
+
|
| 241 |
+
Please reserve about ~2GB of space for the dataset. If there is insufficient space for pretrain data, you can try skipping the pretrain training step and proceed directly to SFT training.
|
| 242 |
+
|
| 243 |
+
</details>
|
| 244 |
+
|
| 245 |
+
### 3' Start Training
|
| 246 |
+
|
| 247 |
+
**3.1 Pretraining (Learning image description)**
|
| 248 |
+
|
| 249 |
+
```bash
|
| 250 |
+
# Basic training command (start from LLM weights, train vision_proj only)
|
| 251 |
+
python train_pretrain_vlm.py --epochs 4 --from_weight llm
|
| 252 |
+
```
|
| 253 |
+
|
| 254 |
+
> Run pretraining to get `pretrain_vlm_*.pth` as the pretrained model's output weights (* represents the model
|
| 255 |
+
> dimension, default is 768).
|
| 256 |
+
|
| 257 |
+
**3.2 Supervised Fine-Tuning (Learning image-caption dialogue style)**
|
| 258 |
+
|
| 259 |
+
```bash
|
| 260 |
+
# Basic training command (start from pretrain weights, full parameter fine-tuning)
|
| 261 |
+
python train_sft_vlm.py --epochs 2 --from_weight pretrain_vlm
|
| 262 |
+
```
|
| 263 |
+
|
| 264 |
+
> Perform supervised fine-tuning to get `sft_vlm_*.pth` as the output weights for the fine-tuned model.
|
| 265 |
+
|
| 266 |
+
<details style="color:rgb(128,128,128)">
|
| 267 |
+
<summary>Note: Training Details</summary>
|
| 268 |
+
|
| 269 |
+
**Training Features:**
|
| 270 |
+
- Support checkpoint resumption: add `--from_resume 1` parameter to continue from last interruption
|
| 271 |
+
- Support GPU count changes: automatically convert steps when GPU count changes during resumption
|
| 272 |
+
- Atomic saving: use temporary file + replacement mechanism to prevent weight corruption from interruption
|
| 273 |
+
- Each save generates `out/**.pth` (model weights) and `checkpoints/**_resume.pth` (training state) files
|
| 274 |
+
|
| 275 |
+
```bash
|
| 276 |
+
# To resume training after interruption, use the same command and add --from_resume 1
|
| 277 |
+
python train_sft_vlm.py --epochs 4 --from_resume 1
|
| 278 |
+
```
|
| 279 |
+
|
| 280 |
+
**Parameter Description:**
|
| 281 |
+
- `--from_weight`: base weight name (llm, pretrain_vlm, none, etc.)
|
| 282 |
+
- `--save_weight`: save weight prefix name
|
| 283 |
+
- `--from_resume`: whether to resume training (0=start from scratch, 1=continue from checkpoint)
|
| 284 |
+
- `--freeze_llm`: whether to freeze LLM parameters (pretrain use only)
|
| 285 |
+
- More details can be found in the code
|
| 286 |
+
|
| 287 |
+
</details>
|
| 288 |
+
|
| 289 |
+
---
|
| 290 |
+
|
| 291 |
+
### 4' Test the Model's Performance
|
| 292 |
+
|
| 293 |
+
Ensure that the model `*.pth` file you want to test is located in the `./out/` directory.
|
| 294 |
+
You can also directly download the pre-trained `*.pth` file
|
| 295 |
+
from [here](https://huggingface.co/jingyaogong/minimind-3v-pytorch).
|
| 296 |
+
|
| 297 |
+
```bash
|
| 298 |
+
# Test SFT model (default)
|
| 299 |
+
python eval_vlm.py --weight sft_vlm
|
| 300 |
+
|
| 301 |
+
# Test Pretrain model
|
| 302 |
+
python eval_vlm.py --weight pretrain_vlm
|
| 303 |
+
```
|
| 304 |
+
|
| 305 |
+
---
|
| 306 |
+
|
| 307 |
+
> [!TIP]
|
| 308 |
+
> The training scripts are based on PyTorch's native framework and support multi-card acceleration. If your device has
|
| 309 |
+
> N (N>1) GPUs:
|
| 310 |
+
|
| 311 |
+
Single-machine N-card training method (DDP, supports multi-machine multi-card cluster)
|
| 312 |
+
|
| 313 |
+
```bash
|
| 314 |
+
torchrun --nproc_per_node N train_xxx.py
|
| 315 |
+
```
|
| 316 |
+
|
| 317 |
+
<details style="color:rgb(128,128,128)">
|
| 318 |
+
<summary>Note: Other Details</summary>
|
| 319 |
+
|
| 320 |
+
Single-machine N-card training (DeepSpeed)
|
| 321 |
+
|
| 322 |
+
```bash
|
| 323 |
+
deepspeed --master_port 29500 --num_gpus=N train_xxx.py
|
| 324 |
+
```
|
| 325 |
+
|
| 326 |
+
You can enable wandb logging during training:
|
| 327 |
+
|
| 328 |
+
```bash
|
| 329 |
+
# You need to log in: wandb login
|
| 330 |
+
torchrun --nproc_per_node N train_xxx.py --use_wandb
|
| 331 |
+
# and
|
| 332 |
+
python train_xxx.py --use_wandb
|
| 333 |
+
```
|
| 334 |
+
|
| 335 |
+
By adding the `--use_wandb` parameter, you can log the training process, and after training is complete, you can view
|
| 336 |
+
the process on the wandb website. You can specify the project name and run name by modifying the `wandb_project`
|
| 337 |
+
and `wandb_run_name` parameters.
|
| 338 |
+
|
| 339 |
+
[Note]: After June 2025, the domestic network environment cannot directly connect to WandB. The MiniMind project by default switches to using [SwanLab](https://swanlab.cn/) as the training visualization tool (fully compatible with WandB API), that is, just change `import wandb` to `import swanlab as wandb`, no other changes are needed.
|
| 340 |
+
|
| 341 |
+
</details>
|
| 342 |
+
|
| 343 |
+
# 📌 VLM Detail
|
| 344 |
+
|
| 345 |
+
The base language model of MiniMind-V (VLM), MiniMind (LLM), comes from the twin
|
| 346 |
+
project [minimind](https://github.com/jingyaogong/minimind). For detailed information on the model structure, training
|
| 347 |
+
specifics, principles, and testing results, please refer to the [minimind](https://github.com/jingyaogong/minimind)
|
| 348 |
+
project. To reduce redundancy, the discussion on LLM-related topics is omitted here, assuming you have a basic
|
| 349 |
+
understanding of MiniMind (LLM).
|
| 350 |
+
|
| 351 |
+
> Even if you are not very familiar with the details of LLMs, you can still follow the "Quick Start" guide to train a
|
| 352 |
+
> MiniMind-V, as it remains unaffected and the repository focuses on the lowest cost for out-of-the-box use!
|
| 353 |
+
|
| 354 |
+
MiniMind-V's structure adds two submodules, a Visual Encoder and a feature projection, with a modality-mixing branch to
|
| 355 |
+
support inputs from multiple modalities:
|
| 356 |
+
|
| 357 |
+
|
| 358 |
+
|
| 359 |
+
|
| 360 |
+
<details>
|
| 361 |
+
<summary> [Important] Some Interesting Thoughts </summary>
|
| 362 |
+
|
| 363 |
+
Let's take a moment to think about two questions:
|
| 364 |
+
|
| 365 |
+
* What is a **Large Language Model (LLM)**?
|
| 366 |
+
* What is a multimodal model?
|
| 367 |
+
|
| 368 |
+
[This article](https://www.jiqizhixin.com/articles/2024-09-15-3) perfectly aligns with my thoughts:
|
| 369 |
+
Although the name "large language model" (LLM) contains the word "language," they are actually not closely related to
|
| 370 |
+
language; this is just a historical issue. A more accurate name would be self-regressive Transformer or something else.
|
| 371 |
+
LLMs are more of a general statistical modeling technology, mainly using a self-regressive Transformer to simulate token
|
| 372 |
+
flows. These tokens can represent text, images, audio, action choices, and even molecules—anything, really.
|
| 373 |
+
Therefore, as long as the problem can be converted into a process of simulating a series of discrete tokens, LLM can
|
| 374 |
+
theoretically solve it. In fact, with the increasing maturity of large language model technologies, we may see more and
|
| 375 |
+
more problems falling under this modeling paradigm. In other words, the problem is fixed in using LLM to "predict the
|
| 376 |
+
next token," but the role and meaning of the tokens differ in each domain.
|
| 377 |
+
|
| 378 |
+
[ZJU-LiXi](https://person.zju.edu.cn/xilics#694283) has also mentioned a similar viewpoint (roughly stated below):
|
| 379 |
+
Text, video, audio, actions, etc., are considered "multimodal" signals in human perception, but the term "modality" is
|
| 380 |
+
essentially just a classification concept based on how humans store information. Just like `.txt` and `.png` files,
|
| 381 |
+
though they differ in visual presentation and higher-level forms, they are fundamentally the same. The concept of "
|
| 382 |
+
multimodal" arose simply because humans need to categorize these signals based on different sensory dimensions.
|
| 383 |
+
However, for machines, regardless of the signal's "modality," they are ultimately presented as a sequence of binary "
|
| 384 |
+
monomodal" numbers. Machines do not differentiate the origin of these signals; they just process and analyze the
|
| 385 |
+
information contained within these sequences.
|
| 386 |
+
|
| 387 |
+
Personally, I think **Generative Pretrained Transformer (GPT)** is a more fitting term than **Large Language Model (LLM)
|
| 388 |
+
**, and I prefer to use "GPT" to represent models in the LLM/VLM/GPT-like architecture series rather than to ride on
|
| 389 |
+
OpenAI's coattails.
|
| 390 |
+
|
| 391 |
+
To summarize what GPTs do in one sentence:
|
| 392 |
+
|
| 393 |
+
A GPT model predicts the next, next-next, next-next-next token, etc., based on the current token... until the model
|
| 394 |
+
outputs the end token; here, the "token" doesn’t necessarily have to be text!
|
| 395 |
+
|
| 396 |
+
```text
|
| 397 |
+
> For an LLM model, if we need to understand an "image," we just treat the "image" as a special "foreign language" that has never been encountered before, and translate it into the "LLM language" via a "foreign language dictionary."
|
| 398 |
+
> For an LLM model, if we need to understand "audio," we just treat "audio" as a special "foreign language" that has never been encountered before, and translate it into the "LLM language" via a "foreign language dictionary."
|
| 399 |
+
> ...
|
| 400 |
+
```
|
| 401 |
+
|
| 402 |
+
<u>**To obtain MiniMind-V, we only need to do these 2 things:**</u>
|
| 403 |
+
|
| 404 |
+
1. Use the **"foreign language dictionary"** that is good at translating images, to translate the image from the **"
|
| 405 |
+
foreign language"** into a model-understandable **"LLM language."**
|
| 406 |
+
2. Fine-tune the LLM so that it and the **"foreign language dictionary"** go through a period of adaptation, thereby
|
| 407 |
+
better understanding images.
|
| 408 |
+
|
| 409 |
+
The "foreign language dictionary" is referred to as the Visual Encoder model.
|
| 410 |
+
Like LlaVA, Qwen-VL, and other visual language models, MiniMind-V now uses the open-source SigLIP2 series models as the
|
| 411 |
+
Visual Encoder.
|
| 412 |
+
Specifically, we use [siglip2-base-p16-ve](https://huggingface.co/jingyaogong/siglip2-base-p16-ve), a Visual
|
| 413 |
+
Encoder based on the ViT-B/16 architecture for describing image-text information.
|
| 414 |
+
The current SigLIP2 NaFlex vision encoder generates up to 256 patch tokens from the processor output as the input to the
|
| 415 |
+
encoder layer, which produces a 1×768 dimensional embedding vector for calculating error with the text.
|
| 416 |
+
We don’t need the final embedding representation, so we only take the output from the encoder layer, which is the output
|
| 417 |
+
feature from the core ViT backbone.
|
| 418 |
+
It receives 256×768 features from the previous layer, which are then reshaped by concatenating every 4 adjacent tokens into 1 (256×768 → 64×3072), then projected to the LLM's hidden dimension via a 2-layer MLP (Linear→GELU→Linear), resulting in 64 visual tokens input into MiniMind-V.
|
| 419 |
+
After obtaining the image encoder features, the integration with the LLM requires aligning the visual features to the LLM's text token dimension, and mapping the image features into the same space as text embeddings. In other
|
| 420 |
+
words, the image features and native visual tokens cannot be directly treated the same; they require cross-modal feature
|
| 421 |
+
alignment.
|
| 422 |
+
|
| 423 |
+
[LlaVA-1](https://arxiv.org/pdf/2304.08485) achieves good alignment with a simple linear transformation, [LlaVA-1.5](https://arxiv.org/pdf/2310.03744) upgrades to a 2-layer MLP. MiniMind-V adopts the same MLP Projection approach as LlaVA-1.5, combined with reshape for token compression.
|
| 424 |
+
|
| 425 |
+
|
| 426 |
+
|
| 427 |
+
With that, the internal structural changes of MiniMind-V are now fully presented.
|
| 428 |
+
|
| 429 |
+
</details>
|
| 430 |
+
|
| 431 |
+
|
| 432 |
+
---
|
| 433 |
+
|
| 434 |
+
Next, let's briefly discuss the changes in the external input and output of MiniMind-V.
|
| 435 |
+
|
| 436 |
+
The input to the VLM is still a segment of text containing special `<image>` placeholders.
|
| 437 |
+
After computing the text embedding, the vector generated by the image encoder can be projected onto the corresponding
|
| 438 |
+
embedding part of the placeholder, replacing the original placeholder embedding.
|
| 439 |
+
For example:
|
| 440 |
+
|
| 441 |
+
```text
|
| 442 |
+
<image>\nWhat is in this image?
|
| 443 |
+
```
|
| 444 |
+
|
| 445 |
+
In `minimind-v`, the image is replaced by 64 `<|image_pad|>` tokens as placeholder (the 256 SigLIP2 patch features are compressed to 64 tokens via reshape+MLP),
|
| 446 |
+
thus the `minimind-v` prompt becomes:
|
| 447 |
+
|
| 448 |
+
```text
|
| 449 |
+
<|image_pad|><|image_pad|>...<|image_pad|>(×64)\nWhat is this image describing?
|
| 450 |
+
```
|
| 451 |
+
|
| 452 |
+
After calculating the embedding and projection, the vision features replace the corresponding placeholder embeddings, and the rest of the computation is identical to the LLM part.
|
| 453 |
+
|
| 454 |
+
|
| 455 |
+
|
| 456 |
+
At this point, all the details of `MiniMind-V` have been presented. The VLM model subclass inherits from `MiniMind` with only **minimal** changes, core algorithm modifications `< 50 lines`, very low migration difficulty. The specific implementation may differ from `LlaVA` and similar models, but the overall idea is consistent.
|
| 457 |
+
|
| 458 |
+
# 📌 Experiment
|
| 459 |
+
|
| 460 |
+
## Ⅰ Dataset
|
| 461 |
+
|
| 462 |
+
Original Source:
|
| 463 |
+
- [Chinese-LLaVA-Vision](https://huggingface.co/datasets/LinkSoul/Chinese-LLaVA-Vision-Instructions): Contains approximately 570,000 pre-trained images from CC-3M and COCO 2014
|
| 464 |
+
- [llava-en-zh-300k](https://huggingface.co/datasets/BUAADreamer/llava-en-zh-300k): Contains 300k instruction fine-tuning data and 150k images
|
| 465 |
+
- [LLaVA-SFT-665K](https://huggingface.co/datasets/csuhan/LLaVA-SFT-665K): Contains 665k instruction fine-tuning data
|
| 466 |
+
|
| 467 |
+
The dataset contains both Chinese and English data. The Q&A content has been translated, with better support for Chinese, further organized and resized (pretrain resolution 128×128, sft resolution 160×160).
|
| 468 |
+
|
| 469 |
+
(pretrain_i2t.parquet) Pre-training dataset format:
|
| 470 |
+
|
| 471 |
+
```text
|
| 472 |
+
Columns: conversations (json string), image_bytes (binary), image_names (string)
|
| 473 |
+
|
| 474 |
+
conversations example:
|
| 475 |
+
[
|
| 476 |
+
{"role": "user", "content": "Provide a brief description of the given image.\n<image>"},
|
| 477 |
+
{"role": "assistant", "content": "Olive oil is a healthy ingredient for free use."}
|
| 478 |
+
]
|
| 479 |
+
image_bytes: <binary image data>
|
| 480 |
+
```
|
| 481 |
+
|
| 482 |
+
(sft_i2t.parquet) Single image instruction fine-tuning dataset format:
|
| 483 |
+
|
| 484 |
+
```text
|
| 485 |
+
Columns: conversations (json string), image_bytes (binary), image_names (string)
|
| 486 |
+
|
| 487 |
+
conversations example:
|
| 488 |
+
[
|
| 489 |
+
{"role": "user", "content": "What impact does the location of the alarm clock have on sleep quality?<image>"},
|
| 490 |
+
{"role": "assistant", "content": "Place the digital alarm clock on the nightstand..."}
|
| 491 |
+
]
|
| 492 |
+
image_bytes: <binary image data>
|
| 493 |
+
```
|
| 494 |
+
|
| 495 |
+
> Note: sft_i2t.parquet contains ~580K samples in total, of which ~236K are image-text conversations (i2t) and ~346K are pure text conversations (t2t). The latter is used to preserve the model's base language capabilities.
|
| 496 |
+
|
| 497 |
+
Dataset download
|
| 498 |
+
link: ([ModelScope](https://www.modelscope.cn/datasets/gongjy/minimind-v_dataset) | [HuggingFace](https://huggingface.co/datasets/jingyaogong/minimind-v_dataset))
|
| 499 |
+
|
| 500 |
+
## Ⅱ Training
|
| 501 |
+
|
| 502 |
+
Training is divided into two stages, both freezing the Visual Encoder gradients and only training the Projection and LLM parts.
|
| 503 |
+
Training is initialized from LLM pre-trained weights, with support for DDP multi-GPU training, mixed precision (bfloat16), torch.compile acceleration, and swanlab logging.
|
| 504 |
+
|
| 505 |
+
> train_pretrain_vlm
|
| 506 |
+
|
| 507 |
+
The pre-training stage learns general image knowledge from ~1.13M image-text description pairs (e.g., a deer is a deer, a dog is a dog).
|
| 508 |
+
This stage uses a higher learning rate (1e-4), max sequence length of 360, freezes the LLM main parameters, and only sets the Projection and LLM's layer 0 as learnable,
|
| 509 |
+
aiming to quickly establish a basic mapping from visual features to the language space while avoiding damage to the LLM's existing language capabilities.
|
| 510 |
+
|
| 511 |
+
> train_sft_vlm
|
| 512 |
+
|
| 513 |
+
The instruction fine-tuning stage learns real Q&A formats from ~580K samples, of which ~236K are image-text multi-turn conversations and ~346K are pure text conversations (to preserve LLM base capabilities).
|
| 514 |
+
This stage uses a lower learning rate (1e-5~1e-6), max sequence length of 768, unfreezes all Projection and LLM parameters for full fine-tuning,
|
| 515 |
+
enabling the model to conduct multi-turn conversations based on image content, while mitigating catastrophic forgetting through the mixed-in pure text data.
|
| 516 |
+
|
| 517 |
+
> Training Time and Loss Trend (for reference only)
|
| 518 |
+
|
| 519 |
+
Pretrain [768+8] (dense & moe)
|
| 520 |
+
|
| 521 |
+
|
| 522 |
+
SFT [768+8] (dense & moe)
|
| 523 |
+
|
| 524 |
+
|
| 525 |
+
## Ⅲ Model Weights
|
| 526 |
+
|
| 527 |
+
| Format | ModelScope | HuggingFace |
|
| 528 |
+
|---|---|---|
|
| 529 |
+
| Native PyTorch (`*.pth`) | [minimind-3v-pytorch](https://www.modelscope.cn/models/gongjy/minimind-3v-pytorch) | [minimind-3v-pytorch](https://huggingface.co/jingyaogong/minimind-3v-pytorch) |
|
| 530 |
+
| Transformers | [minimind-v collection](https://modelscope.cn/collections/MiniMind-V-42b841dde22d41) | [minimind-v collection](https://huggingface.co/collections/jingyaogong/minimind-v-67000833fb60b3a2e1f3597d) |
|
| 531 |
+
|
| 532 |
+
> Note: The Transformers version is the `MiniMind-V` model after single-image instruction fine-tuning
|
| 533 |
+
|
| 534 |
+
# 📌 Test
|
| 535 |
+
|
| 536 |
+
### Effect Test
|
| 537 |
+
|
| 538 |
+
#### Single Image Dialogue
|
| 539 |
+
|
| 540 |
+
<table>
|
| 541 |
+
<thead>
|
| 542 |
+
<tr>
|
| 543 |
+
<th>image</th>
|
| 544 |
+
<th>minimind-3v</th>
|
| 545 |
+
<th>minimind-3v-moe</th>
|
| 546 |
+
</tr>
|
| 547 |
+
</thead>
|
| 548 |
+
<tbody>
|
| 549 |
+
<tr>
|
| 550 |
+
<td>
|
| 551 |
+
<img src="./dataset/eval_images/airplane-flying-blue-sky.jpg" alt="airplane">
|
| 552 |
+
|
| 553 |
+
</td>
|
| 554 |
+
<td>The image features a white and black airplane parked on a grassy terrain. The layers of a building are likely to be filled with air traffic control, such as the building's tall building, the large building, or the overall pavement. There are also two airplanes parked in the background. The airplanes are displayed on a board, and the airplane is flying through the air while the black and white airplane is parked on the ground.</td>
|
| 555 |
+
<td>The image features a large jetliner with a large airplane sitting on the ground. It is likely an airplane or an aircraft, possibly a flight jet or a runway.</td>
|
| 556 |
+
</tr>
|
| 557 |
+
<tr>
|
| 558 |
+
<td>
|
| 559 |
+
<img src="./dataset/eval_images/birthday-cake-candles-table.jpg" alt="birthday-cake">
|
| 560 |
+
|
| 561 |
+
</td>
|
| 562 |
+
<td>The image features a white cake in an old-fashioned cake with cake placed on it. It is surrounded by a few cooked ingredients, including the wedding cake.</td>
|
| 563 |
+
<td>The image features a white cake in an old-fashioned cake with cake placed on it. It is surrounded by a few cooked ingredients, including the wedding cake.</td>
|
| 564 |
+
</tr>
|
| 565 |
+
<tr>
|
| 566 |
+
<td>
|
| 567 |
+
<img src="./dataset/eval_images/pizza-on-wooden-board.jpg" alt="pizza">
|
| 568 |
+
|
| 569 |
+
</td>
|
| 570 |
+
<td>The image depicts a delicious pizza pizza with fresh toppings, which are likely present in a slice of pizza. The pizza is perfectly crispy, as it has a crispy crust and slightly crispy, making it a delightful pizza presentation. The pizza is filled with fresh toppings, adding to the crispy crust. The pizza is also a bit scrambled, as it has a fresh topping, while the crispy crust is cooked with a pizza pan. The pizza is likely to be a pizza with its crunchy texture and flavorfully. The pizza is also a popular choice for pizza with others, and it is a filling and crispy crust.</td>
|
| 571 |
+
<td>The image features a scenic burning pizza in a pasta-style board, surrounded by a wooden cabinet, a garner topping for cheese and vegetables. There is a small cabinet nearby, with a pan-familt, a pizza flatter topping. The pizza is situated on the left side of the board, with a bowl of olives placed on top of the side.</td>
|
| 572 |
+
</tr>
|
| 573 |
+
<tr>
|
| 574 |
+
<td>
|
| 575 |
+
<img src="./dataset/eval_images/red-sports-car-road.jpg" alt="red-car">
|
| 576 |
+
|
| 577 |
+
</td>
|
| 578 |
+
<td>The image shows a yellow yellow and red turf coastline on a street, with a couple of cars and a yellow and red traffic lights in the background.</td>
|
| 579 |
+
<td>The image features a couple of female vanity van lying on a car. The car is situated on the ground, surrounded by a wet furniture. The couple is seen in the middle of the car, possibly observing the scene.</td>
|
| 580 |
+
</tr>
|
| 581 |
+
<tr>
|
| 582 |
+
<td>
|
| 583 |
+
<img src="./dataset/eval_images/row-of-colorful-houses.jpg" alt="colorful-houses">
|
| 584 |
+
|
| 585 |
+
</td>
|
| 586 |
+
<td>IoT tree is a holiday that is often associated with Christmas culture, history, and celebration. The tree has a unique black and white striped pattern, which features a sweet treat, a budget-friendly chocolate cake with brown spots. The cake is burnt and has a balcony with a sweet treat, with the rich, vibrant colors of the striped pattern. The tree has a rich and burnt color, while the rich and vibrant colors are visually appealing. The tree has a striped pattern, which adds to the overall atmosphere of the image.</td>
|
| 587 |
+
<td>The image is a colorful scene of a colorful vintage house, with a large pink roof of the wall and a red color scheme. The colorful vintage house has a pink color, and the color scheme appears to be fine, with a red color scheme.</td>
|
| 588 |
+
</tr>
|
| 589 |
+
<tr>
|
| 590 |
+
<td>
|
| 591 |
+
<img src="./dataset/eval_images/snow-mountain-lake-view.jpg" alt="snow-mountain">
|
| 592 |
+
|
| 593 |
+
</td>
|
| 594 |
+
<td>The image features a snowy mountain surrounded by a large, dense mountains. The large body of water is quite dense, with the body of water being hot and the water is sandy. The tall trees are swirls, and the tall trees are standing on the snow-covered ground. The body of water is also sink, creating a savanna-like appearance.</td>
|
| 595 |
+
<td>The image is an image of a large mountain visible in a lake, surrounded by the idyllic mountains and the mountains. It appears to be a blanket in the ocean, with the river and idylis watching the sea. The mountains are lined with fresh sand and waves, adding a sense of tranquility to the scene.</td>
|
| 596 |
+
</tr>
|
| 597 |
+
<tr>
|
| 598 |
+
<td>
|
| 599 |
+
<img src="./dataset/eval_images/street-food-hotpot-table.jpg" alt="street-food">
|
| 600 |
+
|
| 601 |
+
</td>
|
| 602 |
+
<td>The image shows a variety of cooking options, including baking meat, cupcakes, and spinach. The baked vegetables are rich in a variety of flavors, including grilled, sautéed, and baked vegetables. The presence of a baked vegetable with a variety of vegetables in different parts suggests a variety of options, including baking, cooking, and baking. The cooking process is highly recommended, with a mix of vegetables and baking times, making it an ideal choice for those who prefer a variety of cooking options. The cooking process is also highly compatible, with a variety of flavors and textures enjoying the cooking process.</td>
|
| 603 |
+
<td>The image features a variety of freshwater salads, glasses, and coworkers displayed on a table. There is a mix of freshwater ingredients, likely a bun, which can be seen in the menu. The freshwater ingredients are placed on the table, and there is a portion of the coworkers displayed in the middle of the room. There is also a bowl filled with various ingredients.</td>
|
| 604 |
+
</tr>
|
| 605 |
+
<tr>
|
| 606 |
+
<td>
|
| 607 |
+
<img src="./dataset/eval_images/three-kittens-basket.jpg" alt="kittens">
|
| 608 |
+
|
| 609 |
+
</td>
|
| 610 |
+
<td>The image features a table filled with people standing together, casing bars, and a pair of brown cats. The table is filled with pink and white cats.</td>
|
| 611 |
+
<td>The image is a black and white detail of a miscellaneous bunch of toys, which is likely to be a part of a group or a similar artistic field.</td>
|
| 612 |
+
</tr>
|
| 613 |
+
<tr>
|
| 614 |
+
<td>
|
| 615 |
+
<img src="./dataset/eval_images/tropical-beach-palm-tree.jpg" alt="tropical-beach">
|
| 616 |
+
|
| 617 |
+
</td>
|
| 618 |
+
<td>The image features a brown wooden coat.</td>
|
| 619 |
+
<td>The image shows a sandy beach with an umbrella on top of a chair, providing a visual appeal for people to sit on.</td>
|
| 620 |
+
</tr>
|
| 621 |
+
<tr>
|
| 622 |
+
<td>
|
| 623 |
+
<img src="./dataset/eval_images/yellow-school-bus-road.jpg" alt="school-bus">
|
| 624 |
+
|
| 625 |
+
</td>
|
| 626 |
+
<td>The image displays a group of people sitting on the bus. They are waiting to be cautious and attentive to their feet, which indicates they are likely to be cautious and followed by the bus.</td>
|
| 627 |
+
<td>The image features a large collection of school buses, a brick-and-middle bus, and a stack of cars visible in the background. The school bus is situated next to a school bus, and there are several people watching the bus. The school bus is visible in the background, with one person standing behind the other, while the other person is watching the bus. The bus is positioned behind the school bus, creating a seamless and dynamic visual effect.</td>
|
| 628 |
+
</tr>
|
| 629 |
+
</tbody>
|
| 630 |
+
</table>
|
| 631 |
+
|
| 632 |
+
### Effect Summary:
|
| 633 |
+
|
| 634 |
+
Both models can identify image subjects (airplane, cake, car, beach, etc.), but commonly exhibit repetitive expressions and hallucinated details. Limited by model and data scale, the overall performance is at a stage of "understanding the gist but inaccurate on details".
|
| 635 |
+
|
| 636 |
+
Visual signals are treated as a special foreign language by LLMs, so the "language learning" ability highly depends on the LLM's capacity. The stronger the LLM, the more powerful the corresponding VLM, and the performance boost becomes significant.
|
| 637 |
+
|
| 638 |
+
#### Future Areas for Improvement:
|
| 639 |
+
|
| 640 |
+
```text
|
| 641 |
+
> Introduce dynamic resolution and Tile-based encoding (like LLaVA-NeXT) to break through the fixed resolution limit.
|
| 642 |
+
> Visual Encoder could be upgraded to stronger vision encoders for finer-grained image features.
|
| 643 |
+
> Extend multi-image understanding, video understanding, and Visual Grounding capabilities.
|
| 644 |
+
> ...
|
| 645 |
+
```
|
| 646 |
+
|
| 647 |
+
# 📌 Acknowledge
|
| 648 |
+
|
| 649 |
+
> [!TIP]
|
| 650 |
+
> If you find `MiniMind-V` helpful, please consider giving it a ⭐ on GitHub. <br/>
|
| 651 |
+
> Given the limited expertise, there may be unknown issues, and we welcome everyone to discuss, correct, or submit PRs
|
| 652 |
+
> to improve the project in Issues. <br/>
|
| 653 |
+
> Your support is the driving force behind continuous improvements to the project. Thank you!
|
| 654 |
+
|
| 655 |
+
## 🤝 [Contributors](https://github.com/jingyaogong/minimind-v/graphs/contributors)
|
| 656 |
+
|
| 657 |
+
<a href="https://github.com/jingyaogong/minimind-v/graphs/contributors">
|
| 658 |
+
<img width="200" src="https://contrib.rocks/image?repo=jingyaogong/minimind-v" />
|
| 659 |
+
</a>
|
| 660 |
+
|
| 661 |
+
## 😊 Acknowledgments
|
| 662 |
+
|
| 663 |
+
<a href="https://github.com/xinyanghuang7"><b>@xinyanghuang7</b></a>: <a href="https://github.com/xinyanghuang7/minimind-v/tree/hxy">Multi-image VLM branch</a> | <a href="https://github.com/jingyaogong/minimind-v/tree/32cf4c5c01337231fd907b92d513de8945594263">Repository provided up to this version</a>
|
| 664 |
+
|
| 665 |
+
<details close>
|
| 666 |
+
<summary> <b>Reference Links & Thanks to the following excellent papers or projects</b> </summary>
|
| 667 |
+
|
| 668 |
+
- No particular order
|
| 669 |
+
- [LlaVA](https://arxiv.org/pdf/2304.08485)
|
| 670 |
+
- [LlaVA-VL](https://arxiv.org/pdf/2310.03744)
|
| 671 |
+
- [Chinese-LLaVA-Vision-Instructions](https://huggingface.co/datasets/LinkSoul/Chinese-LLaVA-Vision-Instructions)
|
| 672 |
+
|
| 673 |
+
</details>
|
| 674 |
+
|
| 675 |
+
## 🫶Supporter
|
| 676 |
+
|
| 677 |
+
<a href="https://github.com/jingyaogong/minimind-v/stargazers">
|
| 678 |
+
<picture>
|
| 679 |
+
<source media="(prefers-color-scheme: dark)" srcset="https://reporoster.com/stars/dark/jingyaogong/minimind-v"/>
|
| 680 |
+
<source media="(prefers-color-scheme: light)" srcset="https://reporoster.com/stars/jingyaogong/minimind-v"/>
|
| 681 |
+
<img alt="github contribution grid snake animation" src="https://reporoster.com/stars/jingyaogong/minimind-v"/>
|
| 682 |
+
</picture>
|
| 683 |
+
</a>
|
| 684 |
+
|
| 685 |
+
<a href="https://github.com/jingyaogong/minimind-v/network/members">
|
| 686 |
+
<picture>
|
| 687 |
+
<source media="(prefers-color-scheme: dark)" srcset="https://reporoster.com/forks/dark/jingyaogong/minimind-v"/>
|
| 688 |
+
<source media="(prefers-color-scheme: light)" srcset="https://reporoster.com/forks/jingyaogong/minimind-v"/>
|
| 689 |
+
<img alt="github contribution grid snake animation" src="https://reporoster.com/forks/jingyaogong/minimind-v"/>
|
| 690 |
+
</picture>
|
| 691 |
+
</a>
|
| 692 |
+
|
| 693 |
+
<picture>
|
| 694 |
+
<source media="(prefers-color-scheme: dark)" srcset="https://api.star-history.com/svg?repos=jingyaogong/minimind-v&type=Date&theme=dark"/>
|
| 695 |
+
<source media="(prefers-color-scheme: light)" srcset="https://api.star-history.com/svg?repos=jingyaogong/minimind-v&type=Date"/>
|
| 696 |
+
<img alt="Star History Chart" src="https://api.star-history.com/svg?repos=jingyaogong/minimind-v&type=Date"/>
|
| 697 |
+
</picture>
|
| 698 |
+
|
| 699 |
+
# 🎓 Citation
|
| 700 |
+
|
| 701 |
+
If you find MiniMind-V helpful in your research or work, please cite:
|
| 702 |
+
|
| 703 |
+
```bibtex
|
| 704 |
+
@misc{minimind-v,
|
| 705 |
+
title = {MiniMind-V: Train a Tiny VLM from Scratch},
|
| 706 |
+
author = {Jingyao Gong},
|
| 707 |
+
year = {2024},
|
| 708 |
+
url = {https://github.com/jingyaogong/minimind-v},
|
| 709 |
+
note = {GitHub repository, accessed 2026}
|
| 710 |
+
}
|
| 711 |
+
```
|
| 712 |
+
|
| 713 |
+
# 📜 License
|
| 714 |
+
|
| 715 |
+
This repository is licensed under the [Apache-2.0 License](LICENSE).
|
| 716 |
+
|
chat_template.jinja
ADDED
|
@@ -0,0 +1,85 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{%- if tools %}
|
| 2 |
+
{{- '<|im_start|>system\n' }}
|
| 3 |
+
{%- if messages[0].role == 'system' %}
|
| 4 |
+
{{- messages[0].content + '\n\n' }}
|
| 5 |
+
{%- endif %}
|
| 6 |
+
{{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
|
| 7 |
+
{%- for tool in tools %}
|
| 8 |
+
{{- "\n" }}
|
| 9 |
+
{{- tool | tojson }}
|
| 10 |
+
{%- endfor %}
|
| 11 |
+
{{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
|
| 12 |
+
{%- else %}
|
| 13 |
+
{%- if messages[0].role == 'system' %}
|
| 14 |
+
{{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
|
| 15 |
+
{%- endif %}
|
| 16 |
+
{%- endif %}
|
| 17 |
+
{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
|
| 18 |
+
{%- for message in messages[::-1] %}
|
| 19 |
+
{%- set index = (messages|length - 1) - loop.index0 %}
|
| 20 |
+
{%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
|
| 21 |
+
{%- set ns.multi_step_tool = false %}
|
| 22 |
+
{%- set ns.last_query_index = index %}
|
| 23 |
+
{%- endif %}
|
| 24 |
+
{%- endfor %}
|
| 25 |
+
{%- for message in messages %}
|
| 26 |
+
{%- if message.content is string %}
|
| 27 |
+
{%- set content = message.content %}
|
| 28 |
+
{%- else %}
|
| 29 |
+
{%- set content = '' %}
|
| 30 |
+
{%- endif %}
|
| 31 |
+
{%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
|
| 32 |
+
{{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
|
| 33 |
+
{%- elif message.role == "assistant" %}
|
| 34 |
+
{%- set reasoning_content = '' %}
|
| 35 |
+
{%- if message.reasoning_content is string %}
|
| 36 |
+
{%- set reasoning_content = message.reasoning_content %}
|
| 37 |
+
{%- else %}
|
| 38 |
+
{%- if '</think>' in content %}
|
| 39 |
+
{%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
|
| 40 |
+
{%- set content = content.split('</think>')[-1].lstrip('\n') %}
|
| 41 |
+
{%- endif %}
|
| 42 |
+
{%- endif %}
|
| 43 |
+
{%- if true %}
|
| 44 |
+
{{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
|
| 45 |
+
{%- endif %}
|
| 46 |
+
{%- if message.tool_calls %}
|
| 47 |
+
{%- for tool_call in message.tool_calls %}
|
| 48 |
+
{%- if (loop.first and content) or (not loop.first) %}
|
| 49 |
+
{{- '\n' }}
|
| 50 |
+
{%- endif %}
|
| 51 |
+
{%- if tool_call.function %}
|
| 52 |
+
{%- set tool_call = tool_call.function %}
|
| 53 |
+
{%- endif %}
|
| 54 |
+
{{- '<tool_call>\n{"name": "' }}
|
| 55 |
+
{{- tool_call.name }}
|
| 56 |
+
{{- '", "arguments": ' }}
|
| 57 |
+
{%- if tool_call.arguments is string %}
|
| 58 |
+
{{- tool_call.arguments }}
|
| 59 |
+
{%- else %}
|
| 60 |
+
{{- tool_call.arguments | tojson }}
|
| 61 |
+
{%- endif %}
|
| 62 |
+
{{- '}\n</tool_call>' }}
|
| 63 |
+
{%- endfor %}
|
| 64 |
+
{%- endif %}
|
| 65 |
+
{{- '<|im_end|>\n' }}
|
| 66 |
+
{%- elif message.role == "tool" %}
|
| 67 |
+
{%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
|
| 68 |
+
{{- '<|im_start|>user' }}
|
| 69 |
+
{%- endif %}
|
| 70 |
+
{{- '\n<tool_response>\n' }}
|
| 71 |
+
{{- content }}
|
| 72 |
+
{{- '\n</tool_response>' }}
|
| 73 |
+
{%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
|
| 74 |
+
{{- '<|im_end|>\n' }}
|
| 75 |
+
{%- endif %}
|
| 76 |
+
{%- endif %}
|
| 77 |
+
{%- endfor %}
|
| 78 |
+
{%- if add_generation_prompt %}
|
| 79 |
+
{{- '<|im_start|>assistant\n' }}
|
| 80 |
+
{%- if open_thinking is defined and open_thinking is true %}
|
| 81 |
+
{{- '<think>\n' }}
|
| 82 |
+
{%- else %}
|
| 83 |
+
{{- '<think>\n\n</think>\n\n' }}
|
| 84 |
+
{%- endif %}
|
| 85 |
+
{%- endif %}
|
config.json
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"architectures": [
|
| 3 |
+
"MiniMindVLM"
|
| 4 |
+
],
|
| 5 |
+
"auto_map": {
|
| 6 |
+
"AutoConfig": "model_vlm.VLMConfig",
|
| 7 |
+
"AutoModelForCausalLM": "model_vlm.MiniMindVLM"
|
| 8 |
+
},
|
| 9 |
+
"bos_token_id": 1,
|
| 10 |
+
"dropout": 0.0,
|
| 11 |
+
"dtype": "bfloat16",
|
| 12 |
+
"eos_token_id": 2,
|
| 13 |
+
"flash_attn": true,
|
| 14 |
+
"head_dim": 96,
|
| 15 |
+
"hidden_act": "silu",
|
| 16 |
+
"hidden_size": 768,
|
| 17 |
+
"image_hidden_size": 768,
|
| 18 |
+
"image_ids": [
|
| 19 |
+
12
|
| 20 |
+
],
|
| 21 |
+
"image_special_token": "<|image_pad|>",
|
| 22 |
+
"image_token_len": 64,
|
| 23 |
+
"inference_rope_scaling": false,
|
| 24 |
+
"intermediate_size": 2432,
|
| 25 |
+
"max_position_embeddings": 32768,
|
| 26 |
+
"max_seq_len": 8192,
|
| 27 |
+
"model_type": "minimind-v",
|
| 28 |
+
"moe_intermediate_size": 2432,
|
| 29 |
+
"norm_topk_prob": true,
|
| 30 |
+
"num_attention_heads": 8,
|
| 31 |
+
"num_experts": 4,
|
| 32 |
+
"num_experts_per_tok": 1,
|
| 33 |
+
"num_hidden_layers": 8,
|
| 34 |
+
"num_key_value_heads": 4,
|
| 35 |
+
"rms_norm_eps": 1e-06,
|
| 36 |
+
"rope_scaling": null,
|
| 37 |
+
"rope_theta": 1000000.0,
|
| 38 |
+
"router_aux_loss_coef": 0.0005,
|
| 39 |
+
"transformers_version": "4.57.6",
|
| 40 |
+
"use_moe": false,
|
| 41 |
+
"vocab_size": 6400
|
| 42 |
+
}
|
generation_config.json
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"_from_model_config": true,
|
| 3 |
+
"bos_token_id": 1,
|
| 4 |
+
"eos_token_id": 2,
|
| 5 |
+
"transformers_version": "4.57.6"
|
| 6 |
+
}
|
images/VLM-structure-moe.jpg
ADDED
|
Git LFS Details
|
images/VLM-structure.jpg
ADDED
|
Git LFS Details
|
images/llava-structure.png
ADDED
|
Git LFS Details
|
images/logo.png
ADDED
|
Git LFS Details
|
images/minimind-3v.gif
ADDED
|
Git LFS Details
|
images/minimind-v-input.jpg
ADDED
|
Git LFS Details
|
images/pretrain_loss.jpg
ADDED
|
images/sft_loss.jpg
ADDED
|
model_minimind.py
ADDED
|
@@ -0,0 +1,279 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import math, torch, torch.nn.functional as F
|
| 2 |
+
from torch import nn
|
| 3 |
+
from transformers.activations import ACT2FN
|
| 4 |
+
from transformers import PreTrainedModel, GenerationMixin, PretrainedConfig
|
| 5 |
+
from transformers.modeling_outputs import MoeCausalLMOutputWithPast
|
| 6 |
+
|
| 7 |
+
# 🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏
|
| 8 |
+
# MiniMind Config
|
| 9 |
+
# 🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏
|
| 10 |
+
class MiniMindConfig(PretrainedConfig):
|
| 11 |
+
model_type = "minimind"
|
| 12 |
+
def __init__(self, hidden_size=768, num_hidden_layers=8, use_moe=False, **kwargs):
|
| 13 |
+
super().__init__(**kwargs)
|
| 14 |
+
self.hidden_size = hidden_size
|
| 15 |
+
self.num_hidden_layers = num_hidden_layers
|
| 16 |
+
self.use_moe = use_moe
|
| 17 |
+
self.dropout = kwargs.get("dropout", 0.0)
|
| 18 |
+
self.vocab_size = kwargs.get("vocab_size", 6400)
|
| 19 |
+
self.bos_token_id = kwargs.get("bos_token_id", 1)
|
| 20 |
+
self.eos_token_id = kwargs.get("eos_token_id", 2)
|
| 21 |
+
self.flash_attn = kwargs.get("flash_attn", True)
|
| 22 |
+
self.num_attention_heads = kwargs.get("num_attention_heads", 8)
|
| 23 |
+
self.num_key_value_heads = kwargs.get("num_key_value_heads", 4)
|
| 24 |
+
self.head_dim = kwargs.get("head_dim", self.hidden_size // self.num_attention_heads)
|
| 25 |
+
self.hidden_act = kwargs.get("hidden_act", 'silu')
|
| 26 |
+
self.intermediate_size = kwargs.get("intermediate_size", math.ceil(hidden_size * math.pi / 64) * 64)
|
| 27 |
+
self.max_position_embeddings = kwargs.get("max_position_embeddings", 32768)
|
| 28 |
+
self.rms_norm_eps = kwargs.get("rms_norm_eps", 1e-6)
|
| 29 |
+
self.rope_theta = kwargs.get("rope_theta", 1e6)
|
| 30 |
+
self.inference_rope_scaling = kwargs.get("inference_rope_scaling", False)
|
| 31 |
+
self.rope_scaling = {
|
| 32 |
+
"beta_fast": 32,
|
| 33 |
+
"beta_slow": 1,
|
| 34 |
+
"factor": 16,
|
| 35 |
+
"original_max_position_embeddings": 2048,
|
| 36 |
+
"attention_factor": 1.0,
|
| 37 |
+
"type": "yarn"
|
| 38 |
+
} if self.inference_rope_scaling else None
|
| 39 |
+
### MoE specific configs (ignored if use_moe = False)
|
| 40 |
+
self.num_experts = kwargs.get("num_experts", 4)
|
| 41 |
+
self.num_experts_per_tok = kwargs.get("num_experts_per_tok", 1)
|
| 42 |
+
self.moe_intermediate_size = kwargs.get("moe_intermediate_size", self.intermediate_size)
|
| 43 |
+
self.norm_topk_prob = kwargs.get("norm_topk_prob", True)
|
| 44 |
+
self.router_aux_loss_coef = kwargs.get("router_aux_loss_coef", 5e-4)
|
| 45 |
+
|
| 46 |
+
# 🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏
|
| 47 |
+
# MiniMind Model
|
| 48 |
+
# 🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏
|
| 49 |
+
class RMSNorm(torch.nn.Module):
|
| 50 |
+
def __init__(self, dim: int, eps: float = 1e-5):
|
| 51 |
+
super().__init__()
|
| 52 |
+
self.eps = eps
|
| 53 |
+
self.weight = nn.Parameter(torch.ones(dim))
|
| 54 |
+
|
| 55 |
+
def norm(self, x):
|
| 56 |
+
return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps)
|
| 57 |
+
|
| 58 |
+
def forward(self, x):
|
| 59 |
+
return (self.weight * self.norm(x.float())).type_as(x)
|
| 60 |
+
|
| 61 |
+
def precompute_freqs_cis(dim: int, end: int = int(32 * 1024), rope_base: float = 1e6, rope_scaling: dict = None):
|
| 62 |
+
freqs, attn_factor = 1.0 / (rope_base ** (torch.arange(0, dim, 2)[: (dim // 2)].float() / dim)), 1.0
|
| 63 |
+
if rope_scaling is not None: # YaRN: f'(i) = f(i)((1-γ) + γ/s), where γ∈[0,1] is linear ramp
|
| 64 |
+
orig_max, factor, beta_fast, beta_slow, attn_factor = (
|
| 65 |
+
rope_scaling.get("original_max_position_embeddings", 2048), rope_scaling.get("factor", 16),
|
| 66 |
+
rope_scaling.get("beta_fast", 32.0), rope_scaling.get("beta_slow", 1.0), rope_scaling.get("attention_factor", 1.0)
|
| 67 |
+
)
|
| 68 |
+
if end / orig_max > 1.0:
|
| 69 |
+
inv_dim = lambda b: (dim * math.log(orig_max / (b * 2 * math.pi))) / (2 * math.log(rope_base))
|
| 70 |
+
low, high = max(math.floor(inv_dim(beta_fast)), 0), min(math.ceil(inv_dim(beta_slow)), dim // 2 - 1)
|
| 71 |
+
ramp = torch.clamp((torch.arange(dim // 2, device=freqs.device).float() - low) / max(high - low, 0.001), 0, 1)
|
| 72 |
+
freqs = freqs * (1 - ramp + ramp / factor)
|
| 73 |
+
t = torch.arange(end, device=freqs.device)
|
| 74 |
+
freqs = torch.outer(t, freqs).float()
|
| 75 |
+
freqs_cos = torch.cat([torch.cos(freqs), torch.cos(freqs)], dim=-1) * attn_factor
|
| 76 |
+
freqs_sin = torch.cat([torch.sin(freqs), torch.sin(freqs)], dim=-1) * attn_factor
|
| 77 |
+
return freqs_cos, freqs_sin
|
| 78 |
+
|
| 79 |
+
def apply_rotary_pos_emb(q, k, cos, sin, unsqueeze_dim=1):
|
| 80 |
+
def rotate_half(x): return torch.cat((-x[..., x.shape[-1] // 2:], x[..., : x.shape[-1] // 2]), dim=-1)
|
| 81 |
+
q_embed = ((q * cos.unsqueeze(unsqueeze_dim)) + (rotate_half(q) * sin.unsqueeze(unsqueeze_dim))).to(q.dtype)
|
| 82 |
+
k_embed = ((k * cos.unsqueeze(unsqueeze_dim)) + (rotate_half(k) * sin.unsqueeze(unsqueeze_dim))).to(k.dtype)
|
| 83 |
+
return q_embed, k_embed
|
| 84 |
+
|
| 85 |
+
def repeat_kv(x: torch.Tensor, n_rep: int) -> torch.Tensor:
|
| 86 |
+
bs, slen, num_key_value_heads, head_dim = x.shape
|
| 87 |
+
if n_rep == 1: return x
|
| 88 |
+
return (x[:, :, :, None, :].expand(bs, slen, num_key_value_heads, n_rep, head_dim).reshape(bs, slen, num_key_value_heads * n_rep, head_dim))
|
| 89 |
+
|
| 90 |
+
class Attention(nn.Module):
|
| 91 |
+
def __init__(self, config: MiniMindConfig):
|
| 92 |
+
super().__init__()
|
| 93 |
+
self.num_key_value_heads = config.num_attention_heads if config.num_key_value_heads is None else config.num_key_value_heads
|
| 94 |
+
self.n_local_heads = config.num_attention_heads
|
| 95 |
+
self.n_local_kv_heads = self.num_key_value_heads
|
| 96 |
+
self.n_rep = self.n_local_heads // self.n_local_kv_heads
|
| 97 |
+
self.head_dim = config.head_dim
|
| 98 |
+
self.q_proj = nn.Linear(config.hidden_size, config.num_attention_heads * self.head_dim, bias=False)
|
| 99 |
+
self.k_proj = nn.Linear(config.hidden_size, self.num_key_value_heads * self.head_dim, bias=False)
|
| 100 |
+
self.v_proj = nn.Linear(config.hidden_size, self.num_key_value_heads * self.head_dim, bias=False)
|
| 101 |
+
self.o_proj = nn.Linear(config.num_attention_heads * self.head_dim, config.hidden_size, bias=False)
|
| 102 |
+
self.q_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
| 103 |
+
self.k_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
| 104 |
+
self.attn_dropout = nn.Dropout(config.dropout)
|
| 105 |
+
self.resid_dropout = nn.Dropout(config.dropout)
|
| 106 |
+
self.dropout = config.dropout
|
| 107 |
+
self.flash = hasattr(torch.nn.functional, 'scaled_dot_product_attention') and config.flash_attn
|
| 108 |
+
|
| 109 |
+
def forward(self, x, position_embeddings, past_key_value=None, use_cache=False, attention_mask=None):
|
| 110 |
+
bsz, seq_len, _ = x.shape
|
| 111 |
+
xq, xk, xv = self.q_proj(x), self.k_proj(x), self.v_proj(x)
|
| 112 |
+
xq = xq.view(bsz, seq_len, self.n_local_heads, self.head_dim)
|
| 113 |
+
xk = xk.view(bsz, seq_len, self.n_local_kv_heads, self.head_dim)
|
| 114 |
+
xv = xv.view(bsz, seq_len, self.n_local_kv_heads, self.head_dim)
|
| 115 |
+
xq, xk = self.q_norm(xq), self.k_norm(xk)
|
| 116 |
+
cos, sin = position_embeddings
|
| 117 |
+
xq, xk = apply_rotary_pos_emb(xq, xk, cos, sin)
|
| 118 |
+
if past_key_value is not None:
|
| 119 |
+
xk = torch.cat([past_key_value[0], xk], dim=1)
|
| 120 |
+
xv = torch.cat([past_key_value[1], xv], dim=1)
|
| 121 |
+
past_kv = (xk, xv) if use_cache else None
|
| 122 |
+
xq, xk, xv = (xq.transpose(1, 2), repeat_kv(xk, self.n_rep).transpose(1, 2), repeat_kv(xv, self.n_rep).transpose(1, 2))
|
| 123 |
+
if self.flash and (seq_len > 1) and (past_key_value is None) and (attention_mask is None or torch.all(attention_mask == 1)):
|
| 124 |
+
output = F.scaled_dot_product_attention(xq, xk, xv, dropout_p=self.dropout if self.training else 0.0, is_causal=True)
|
| 125 |
+
else:
|
| 126 |
+
scores = (xq @ xk.transpose(-2, -1)) / math.sqrt(self.head_dim)
|
| 127 |
+
scores[:, :, :, -seq_len:] += torch.full((seq_len, seq_len), float("-inf"), device=scores.device).triu(1)
|
| 128 |
+
if attention_mask is not None: scores += (1.0 - attention_mask.unsqueeze(1).unsqueeze(2)) * -1e9
|
| 129 |
+
output = self.attn_dropout(F.softmax(scores.float(), dim=-1).type_as(xq)) @ xv
|
| 130 |
+
output = output.transpose(1, 2).reshape(bsz, seq_len, -1)
|
| 131 |
+
output = self.resid_dropout(self.o_proj(output))
|
| 132 |
+
return output, past_kv
|
| 133 |
+
|
| 134 |
+
class FeedForward(nn.Module):
|
| 135 |
+
def __init__(self, config: MiniMindConfig, intermediate_size: int = None):
|
| 136 |
+
super().__init__()
|
| 137 |
+
intermediate_size = intermediate_size or config.intermediate_size
|
| 138 |
+
self.gate_proj = nn.Linear(config.hidden_size, intermediate_size, bias=False)
|
| 139 |
+
self.down_proj = nn.Linear(intermediate_size, config.hidden_size, bias=False)
|
| 140 |
+
self.up_proj = nn.Linear(config.hidden_size, intermediate_size, bias=False)
|
| 141 |
+
self.act_fn = ACT2FN[config.hidden_act]
|
| 142 |
+
|
| 143 |
+
def forward(self, x):
|
| 144 |
+
return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))
|
| 145 |
+
|
| 146 |
+
class MOEFeedForward(nn.Module):
|
| 147 |
+
def __init__(self, config: MiniMindConfig):
|
| 148 |
+
super().__init__()
|
| 149 |
+
self.config = config
|
| 150 |
+
self.gate = nn.Linear(config.hidden_size, config.num_experts, bias=False)
|
| 151 |
+
self.experts = nn.ModuleList([FeedForward(config, intermediate_size=config.moe_intermediate_size) for _ in range(config.num_experts)])
|
| 152 |
+
self.act_fn = ACT2FN[config.hidden_act]
|
| 153 |
+
|
| 154 |
+
def forward(self, x):
|
| 155 |
+
batch_size, seq_len, hidden_dim = x.shape
|
| 156 |
+
x_flat = x.view(-1, hidden_dim)
|
| 157 |
+
scores = F.softmax(self.gate(x_flat), dim=-1)
|
| 158 |
+
topk_weight, topk_idx = torch.topk(scores, k=self.config.num_experts_per_tok, dim=-1, sorted=False)
|
| 159 |
+
if self.config.norm_topk_prob: topk_weight = topk_weight / (topk_weight.sum(dim=-1, keepdim=True) + 1e-20)
|
| 160 |
+
y = torch.zeros_like(x_flat)
|
| 161 |
+
for i, expert in enumerate(self.experts):
|
| 162 |
+
mask = (topk_idx == i)
|
| 163 |
+
if mask.any():
|
| 164 |
+
token_idx = mask.any(dim=-1).nonzero().flatten()
|
| 165 |
+
weight = topk_weight[mask].view(-1, 1)
|
| 166 |
+
y.index_add_(0, token_idx, (expert(x_flat[token_idx]) * weight).to(y.dtype))
|
| 167 |
+
elif self.training:
|
| 168 |
+
y[0, 0] += 0 * sum(p.sum() for p in expert.parameters())
|
| 169 |
+
if self.training and self.config.router_aux_loss_coef > 0:
|
| 170 |
+
load = F.one_hot(topk_idx, self.config.num_experts).float().mean(0)
|
| 171 |
+
self.aux_loss = (load * scores.mean(0)).sum() * self.config.num_experts * self.config.router_aux_loss_coef
|
| 172 |
+
else:
|
| 173 |
+
self.aux_loss = scores.new_zeros(1).squeeze()
|
| 174 |
+
return y.view(batch_size, seq_len, hidden_dim)
|
| 175 |
+
|
| 176 |
+
class MiniMindBlock(nn.Module):
|
| 177 |
+
def __init__(self, layer_id: int, config: MiniMindConfig):
|
| 178 |
+
super().__init__()
|
| 179 |
+
self.self_attn = Attention(config)
|
| 180 |
+
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
| 181 |
+
self.post_attention_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
| 182 |
+
self.mlp = FeedForward(config) if not config.use_moe else MOEFeedForward(config)
|
| 183 |
+
|
| 184 |
+
def forward(self, hidden_states, position_embeddings, past_key_value=None, use_cache=False, attention_mask=None):
|
| 185 |
+
residual = hidden_states
|
| 186 |
+
hidden_states, present_key_value = self.self_attn(
|
| 187 |
+
self.input_layernorm(hidden_states), position_embeddings,
|
| 188 |
+
past_key_value, use_cache, attention_mask
|
| 189 |
+
)
|
| 190 |
+
hidden_states += residual
|
| 191 |
+
hidden_states = hidden_states + self.mlp(self.post_attention_layernorm(hidden_states))
|
| 192 |
+
return hidden_states, present_key_value
|
| 193 |
+
|
| 194 |
+
class MiniMindModel(nn.Module):
|
| 195 |
+
def __init__(self, config: MiniMindConfig):
|
| 196 |
+
super().__init__()
|
| 197 |
+
self.config = config
|
| 198 |
+
self.vocab_size, self.num_hidden_layers = config.vocab_size, config.num_hidden_layers
|
| 199 |
+
self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size)
|
| 200 |
+
self.dropout = nn.Dropout(config.dropout)
|
| 201 |
+
self.layers = nn.ModuleList([MiniMindBlock(l, config) for l in range(self.num_hidden_layers)])
|
| 202 |
+
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
| 203 |
+
freqs_cos, freqs_sin = precompute_freqs_cis(dim=config.head_dim, end=config.max_position_embeddings, rope_base=config.rope_theta, rope_scaling=config.rope_scaling)
|
| 204 |
+
self.register_buffer("freqs_cos", freqs_cos, persistent=False)
|
| 205 |
+
self.register_buffer("freqs_sin", freqs_sin, persistent=False)
|
| 206 |
+
|
| 207 |
+
def forward(self, input_ids, attention_mask=None, past_key_values=None, use_cache=False, **kwargs):
|
| 208 |
+
batch_size, seq_length = input_ids.shape
|
| 209 |
+
if hasattr(past_key_values, 'layers'): past_key_values = None
|
| 210 |
+
past_key_values = past_key_values or [None] * len(self.layers)
|
| 211 |
+
start_pos = past_key_values[0][0].shape[1] if past_key_values[0] is not None else 0
|
| 212 |
+
hidden_states = self.dropout(self.embed_tokens(input_ids))
|
| 213 |
+
position_embeddings = (self.freqs_cos[start_pos:start_pos + seq_length], self.freqs_sin[start_pos:start_pos + seq_length])
|
| 214 |
+
presents = []
|
| 215 |
+
for layer, past_key_value in zip(self.layers, past_key_values):
|
| 216 |
+
hidden_states, present = layer(
|
| 217 |
+
hidden_states,
|
| 218 |
+
position_embeddings,
|
| 219 |
+
past_key_value=past_key_value,
|
| 220 |
+
use_cache=use_cache,
|
| 221 |
+
attention_mask=attention_mask
|
| 222 |
+
)
|
| 223 |
+
presents.append(present)
|
| 224 |
+
hidden_states = self.norm(hidden_states)
|
| 225 |
+
aux_loss = sum([l.mlp.aux_loss for l in self.layers if isinstance(l.mlp, MOEFeedForward)], hidden_states.new_zeros(1).squeeze())
|
| 226 |
+
return hidden_states, presents, aux_loss
|
| 227 |
+
|
| 228 |
+
class MiniMindForCausalLM(PreTrainedModel, GenerationMixin):
|
| 229 |
+
config_class = MiniMindConfig
|
| 230 |
+
def __init__(self, config: MiniMindConfig = None):
|
| 231 |
+
self.config = config or MiniMindConfig()
|
| 232 |
+
super().__init__(self.config)
|
| 233 |
+
self.model = MiniMindModel(self.config)
|
| 234 |
+
self.lm_head = nn.Linear(self.config.hidden_size, self.config.vocab_size, bias=False)
|
| 235 |
+
self.model.embed_tokens.weight = self.lm_head.weight
|
| 236 |
+
|
| 237 |
+
def forward(self, input_ids, attention_mask=None, past_key_values=None, use_cache=False, logits_to_keep=0, labels=None, **kwargs):
|
| 238 |
+
hidden_states, past_key_values, aux_loss = self.model(input_ids, attention_mask, past_key_values, use_cache, **kwargs)
|
| 239 |
+
slice_indices = slice(-logits_to_keep, None) if isinstance(logits_to_keep, int) else logits_to_keep
|
| 240 |
+
logits = self.lm_head(hidden_states[:, slice_indices, :])
|
| 241 |
+
loss = None
|
| 242 |
+
if labels is not None:
|
| 243 |
+
x, y = logits[..., :-1, :].contiguous(), labels[..., 1:].contiguous()
|
| 244 |
+
loss = F.cross_entropy(x.view(-1, x.size(-1)), y.view(-1), ignore_index=-100)
|
| 245 |
+
return MoeCausalLMOutputWithPast(loss=loss, aux_loss=aux_loss, logits=logits, past_key_values=past_key_values, hidden_states=hidden_states)
|
| 246 |
+
|
| 247 |
+
# https://github.com/jingyaogong/minimind/discussions/611
|
| 248 |
+
@torch.inference_mode()
|
| 249 |
+
def generate(self, inputs=None, attention_mask=None, max_new_tokens=8192, temperature=0.85, top_p=0.85, top_k=50, eos_token_id=2, streamer=None, use_cache=True, num_return_sequences=1, do_sample=True, repetition_penalty=1.0, **kwargs):
|
| 250 |
+
input_ids = kwargs.pop("input_ids", inputs).repeat(num_return_sequences, 1)
|
| 251 |
+
attention_mask = attention_mask.repeat(num_return_sequences, 1) if attention_mask is not None else None
|
| 252 |
+
past_key_values = kwargs.pop("past_key_values", None)
|
| 253 |
+
finished = torch.zeros(input_ids.shape[0], dtype=torch.bool, device=input_ids.device)
|
| 254 |
+
if streamer: streamer.put(input_ids.cpu())
|
| 255 |
+
for _ in range(max_new_tokens):
|
| 256 |
+
past_len = past_key_values[0][0].shape[1] if past_key_values else 0
|
| 257 |
+
outputs = self.forward(input_ids[:, past_len:], attention_mask, past_key_values, use_cache=use_cache, **kwargs)
|
| 258 |
+
attention_mask = torch.cat([attention_mask, attention_mask.new_ones(attention_mask.shape[0], 1)], -1) if attention_mask is not None else None
|
| 259 |
+
logits = outputs.logits[:, -1, :] / temperature
|
| 260 |
+
if repetition_penalty != 1.0:
|
| 261 |
+
for i in range(input_ids.shape[0]): logits[i, torch.unique(input_ids[i])] /= repetition_penalty
|
| 262 |
+
if top_k > 0:
|
| 263 |
+
logits[logits < torch.topk(logits, top_k)[0][..., -1, None]] = -float('inf')
|
| 264 |
+
if top_p < 1.0:
|
| 265 |
+
sorted_logits, sorted_indices = torch.sort(logits, descending=True)
|
| 266 |
+
mask = torch.cumsum(torch.softmax(sorted_logits, dim=-1), dim=-1) > top_p
|
| 267 |
+
mask[..., 1:], mask[..., 0] = mask[..., :-1].clone(), 0
|
| 268 |
+
logits[mask.scatter(1, sorted_indices, mask)] = -float('inf')
|
| 269 |
+
next_token = torch.multinomial(torch.softmax(logits, dim=-1), num_samples=1) if do_sample else torch.argmax(logits, dim=-1, keepdim=True)
|
| 270 |
+
if eos_token_id is not None: next_token = torch.where(finished.unsqueeze(-1), next_token.new_full((next_token.shape[0], 1), eos_token_id), next_token)
|
| 271 |
+
input_ids = torch.cat([input_ids, next_token], dim=-1)
|
| 272 |
+
past_key_values = outputs.past_key_values if use_cache else None
|
| 273 |
+
if streamer: streamer.put(next_token.cpu())
|
| 274 |
+
if eos_token_id is not None:
|
| 275 |
+
finished |= next_token.squeeze(-1).eq(eos_token_id)
|
| 276 |
+
if finished.all(): break
|
| 277 |
+
if streamer: streamer.end()
|
| 278 |
+
if kwargs.get("return_kv"): return {'generated_ids': input_ids, 'past_kv': past_key_values}
|
| 279 |
+
return input_ids
|
model_vlm.py
ADDED
|
@@ -0,0 +1,155 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import torch
|
| 3 |
+
import warnings
|
| 4 |
+
from .model_minimind import *
|
| 5 |
+
from typing import Optional, Tuple, List, Union
|
| 6 |
+
from torch import nn
|
| 7 |
+
from transformers import Siglip2ImageProcessor, Siglip2VisionModel
|
| 8 |
+
from transformers.modeling_outputs import MoeCausalLMOutputWithPast
|
| 9 |
+
|
| 10 |
+
warnings.filterwarnings('ignore')
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
class VLMConfig(MiniMindConfig):
|
| 14 |
+
model_type = "minimind-v"
|
| 15 |
+
|
| 16 |
+
def __init__(self, image_special_token='<|image_pad|>', image_ids=[12], **kwargs):
|
| 17 |
+
self.image_special_token = image_special_token
|
| 18 |
+
self.image_ids = image_ids
|
| 19 |
+
self.image_hidden_size = kwargs.get("image_hidden_size", 768)
|
| 20 |
+
self.image_token_len = kwargs.get("image_token_len", 64)
|
| 21 |
+
super().__init__(**kwargs)
|
| 22 |
+
|
| 23 |
+
class MMVisionProjector(nn.Module):
|
| 24 |
+
def __init__(self, in_dim, out_dim, source_tokens=256, target_tokens=64):
|
| 25 |
+
super().__init__()
|
| 26 |
+
self.target_tokens = target_tokens
|
| 27 |
+
self.merge = source_tokens // target_tokens
|
| 28 |
+
self.mlp = nn.Sequential(
|
| 29 |
+
nn.Linear(in_dim * self.merge, out_dim),
|
| 30 |
+
nn.GELU(),
|
| 31 |
+
nn.Linear(out_dim, out_dim),
|
| 32 |
+
)
|
| 33 |
+
def forward(self, x):
|
| 34 |
+
b, n, d = x.shape
|
| 35 |
+
x = x.reshape(b, self.target_tokens, d * self.merge)
|
| 36 |
+
return self.mlp(x)
|
| 37 |
+
|
| 38 |
+
# 继承自语言模型
|
| 39 |
+
class MiniMindVLM(MiniMindForCausalLM):
|
| 40 |
+
config_class = VLMConfig
|
| 41 |
+
|
| 42 |
+
def __init__(self, config: VLMConfig = None, vision_model_path="./model/siglip2-base-p16-ve"):
|
| 43 |
+
self.config = config or VLMConfig()
|
| 44 |
+
super().__init__(self.config)
|
| 45 |
+
self.vision_encoder, self.processor = self.__class__.get_vision_model(vision_model_path)
|
| 46 |
+
self.vision_proj = MMVisionProjector(self.config.image_hidden_size, self.config.hidden_size, target_tokens=self.config.image_token_len)
|
| 47 |
+
|
| 48 |
+
@staticmethod
|
| 49 |
+
def get_vision_model(model_path: str):
|
| 50 |
+
from transformers import logging as hf_logging
|
| 51 |
+
hf_logging.set_verbosity_error()
|
| 52 |
+
if not os.path.exists(model_path):
|
| 53 |
+
return None, None
|
| 54 |
+
model = Siglip2VisionModel.from_pretrained(model_path)
|
| 55 |
+
processor = Siglip2ImageProcessor.from_pretrained(model_path)
|
| 56 |
+
# 冻结 vision_encoder 的所有参数
|
| 57 |
+
for param in model.parameters():
|
| 58 |
+
param.requires_grad = False
|
| 59 |
+
return model.eval(), processor
|
| 60 |
+
|
| 61 |
+
@staticmethod
|
| 62 |
+
def image2tensor(image, processor):
|
| 63 |
+
if image.mode in ['RGBA', 'LA']: image = image.convert('RGB')
|
| 64 |
+
inputs = processor(images=image, return_tensors="pt")
|
| 65 |
+
return inputs
|
| 66 |
+
|
| 67 |
+
@staticmethod
|
| 68 |
+
def get_image_embeddings(image_inputs, vision_model):
|
| 69 |
+
if hasattr(image_inputs, 'keys'):
|
| 70 |
+
image_inputs = {k: v.squeeze(1) if v.ndim > 2 and v.shape[1] == 1 else v for k, v in image_inputs.items()}
|
| 71 |
+
with torch.no_grad():
|
| 72 |
+
outputs = vision_model(**image_inputs)
|
| 73 |
+
return outputs.last_hidden_state
|
| 74 |
+
|
| 75 |
+
@torch.compiler.disable
|
| 76 |
+
def count_vision_proj(self, tokens, h, vision_tensors=None, seqlen=512):
|
| 77 |
+
if vision_tensors is None or not self.config.image_ids:
|
| 78 |
+
return h
|
| 79 |
+
marker, vf = self.config.image_ids[0], vision_tensors
|
| 80 |
+
if vf.dim() == 3:
|
| 81 |
+
vf = vf.unsqueeze(1)
|
| 82 |
+
out = []
|
| 83 |
+
for b in range(h.size(0)):
|
| 84 |
+
hb, seq, k, i = h[b], tokens[b].tolist(), 0, 0
|
| 85 |
+
while i < len(seq):
|
| 86 |
+
if seq[i] == marker:
|
| 87 |
+
start = i
|
| 88 |
+
while i < len(seq) and seq[i] == marker:
|
| 89 |
+
i += 1
|
| 90 |
+
if k < vf.size(1):
|
| 91 |
+
hb = torch.cat((hb[:start], vf[b][k][:i - start], hb[i:]), dim=0)[:seqlen]
|
| 92 |
+
k += 1
|
| 93 |
+
else:
|
| 94 |
+
i += 1
|
| 95 |
+
out.append(hb)
|
| 96 |
+
return torch.stack(out)
|
| 97 |
+
|
| 98 |
+
def forward(self,
|
| 99 |
+
input_ids: Optional[torch.Tensor] = None,
|
| 100 |
+
attention_mask: Optional[torch.Tensor] = None,
|
| 101 |
+
past_key_values: Optional[List[Tuple[torch.Tensor, torch.Tensor]]] = None,
|
| 102 |
+
use_cache: bool = False,
|
| 103 |
+
logits_to_keep: Union[int, torch.Tensor] = 0,
|
| 104 |
+
labels: Optional[torch.Tensor] = None,
|
| 105 |
+
pixel_values: Optional[torch.FloatTensor] = None,
|
| 106 |
+
**args):
|
| 107 |
+
batch_size, seq_length = input_ids.shape
|
| 108 |
+
if hasattr(past_key_values, 'layers'): past_key_values = None
|
| 109 |
+
past_key_values = past_key_values or [None] * len(self.model.layers)
|
| 110 |
+
start_pos = past_key_values[0][0].shape[1] if past_key_values[0] is not None else 0
|
| 111 |
+
|
| 112 |
+
hidden_states = self.model.dropout(self.model.embed_tokens(input_ids))
|
| 113 |
+
|
| 114 |
+
if pixel_values is not None and start_pos == 0:
|
| 115 |
+
if hasattr(pixel_values, 'keys'):
|
| 116 |
+
img_emb = MiniMindVLM.get_image_embeddings(pixel_values, self.vision_encoder)
|
| 117 |
+
vision_tensors = self.vision_proj(img_emb)
|
| 118 |
+
else:
|
| 119 |
+
if len(pixel_values.shape) == 6:
|
| 120 |
+
pixel_values = pixel_values.squeeze(2)
|
| 121 |
+
bs, num, c, im_h, im_w = pixel_values.shape
|
| 122 |
+
stack_dim = 1 if bs > 1 else 0
|
| 123 |
+
vision_tensors = torch.stack([self.vision_proj(MiniMindVLM.get_image_embeddings(pixel_values[:, i, :, :, :], self.vision_encoder)) for i in range(num)], dim=stack_dim)
|
| 124 |
+
hidden_states = self.count_vision_proj(tokens=input_ids, h=hidden_states, vision_tensors=vision_tensors, seqlen=input_ids.shape[1])
|
| 125 |
+
|
| 126 |
+
position_embeddings = (
|
| 127 |
+
self.model.freqs_cos[start_pos:start_pos + seq_length],
|
| 128 |
+
self.model.freqs_sin[start_pos:start_pos + seq_length]
|
| 129 |
+
)
|
| 130 |
+
|
| 131 |
+
presents = []
|
| 132 |
+
for layer_idx, (layer, past_key_value) in enumerate(zip(self.model.layers, past_key_values)):
|
| 133 |
+
hidden_states, present = layer(
|
| 134 |
+
hidden_states,
|
| 135 |
+
position_embeddings,
|
| 136 |
+
past_key_value=past_key_value,
|
| 137 |
+
use_cache=use_cache,
|
| 138 |
+
attention_mask=attention_mask
|
| 139 |
+
)
|
| 140 |
+
presents.append(present)
|
| 141 |
+
|
| 142 |
+
hidden_states = self.model.norm(hidden_states)
|
| 143 |
+
|
| 144 |
+
aux_loss = sum([l.mlp.aux_loss for l in self.model.layers if isinstance(l.mlp, MOEFeedForward)], hidden_states.new_zeros(1).squeeze())
|
| 145 |
+
slice_indices = slice(-logits_to_keep, None) if isinstance(logits_to_keep, int) else logits_to_keep
|
| 146 |
+
logits = self.lm_head(hidden_states[:, slice_indices, :])
|
| 147 |
+
|
| 148 |
+
loss = None
|
| 149 |
+
if labels is not None:
|
| 150 |
+
shift_logits = logits[..., :-1, :].contiguous()
|
| 151 |
+
shift_labels = labels[..., 1:].contiguous()
|
| 152 |
+
loss = F.cross_entropy(shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1), ignore_index=-100)
|
| 153 |
+
|
| 154 |
+
output = MoeCausalLMOutputWithPast(loss=loss, aux_loss=aux_loss, logits=logits, past_key_values=presents, hidden_states=hidden_states)
|
| 155 |
+
return output
|
pytorch_model.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:05bdaaa8c3dae0ef58d0e0afb4ba8089d3674cafc85f706fe7859d332c1cbb18
|
| 3 |
+
size 133756040
|
special_tokens_map.json
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"additional_special_tokens": [
|
| 3 |
+
"<|im_start|>",
|
| 4 |
+
"<|im_end|>",
|
| 5 |
+
"<|object_ref_start|>",
|
| 6 |
+
"<|object_ref_end|>",
|
| 7 |
+
"<|box_start|>",
|
| 8 |
+
"<|box_end|>",
|
| 9 |
+
"<|quad_start|>",
|
| 10 |
+
"<|quad_end|>",
|
| 11 |
+
"<|vision_start|>",
|
| 12 |
+
"<|vision_end|>",
|
| 13 |
+
"<|vision_pad|>",
|
| 14 |
+
"<|image_pad|>",
|
| 15 |
+
"<|video_pad|>",
|
| 16 |
+
"<|audio_start|>",
|
| 17 |
+
"<|audio_end|>",
|
| 18 |
+
"<|audio_pad|>",
|
| 19 |
+
"<tts_pad>",
|
| 20 |
+
"<tts_text_bos>",
|
| 21 |
+
"<tts_text_eod>",
|
| 22 |
+
"<tts_text_bos_single>"
|
| 23 |
+
],
|
| 24 |
+
"bos_token": {
|
| 25 |
+
"content": "<|im_start|>",
|
| 26 |
+
"lstrip": false,
|
| 27 |
+
"normalized": false,
|
| 28 |
+
"rstrip": false,
|
| 29 |
+
"single_word": false
|
| 30 |
+
},
|
| 31 |
+
"eos_token": {
|
| 32 |
+
"content": "<|im_end|>",
|
| 33 |
+
"lstrip": false,
|
| 34 |
+
"normalized": false,
|
| 35 |
+
"rstrip": false,
|
| 36 |
+
"single_word": false
|
| 37 |
+
},
|
| 38 |
+
"pad_token": {
|
| 39 |
+
"content": "<|endoftext|>",
|
| 40 |
+
"lstrip": false,
|
| 41 |
+
"normalized": false,
|
| 42 |
+
"rstrip": false,
|
| 43 |
+
"single_word": false
|
| 44 |
+
},
|
| 45 |
+
"unk_token": {
|
| 46 |
+
"content": "<|endoftext|>",
|
| 47 |
+
"lstrip": false,
|
| 48 |
+
"normalized": false,
|
| 49 |
+
"rstrip": false,
|
| 50 |
+
"single_word": false
|
| 51 |
+
}
|
| 52 |
+
}
|
tokenizer.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
tokenizer_config.json
ADDED
|
@@ -0,0 +1,335 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"add_bos_token": false,
|
| 3 |
+
"add_eos_token": false,
|
| 4 |
+
"add_prefix_space": false,
|
| 5 |
+
"added_tokens_decoder": {
|
| 6 |
+
"0": {
|
| 7 |
+
"content": "<|endoftext|>",
|
| 8 |
+
"lstrip": false,
|
| 9 |
+
"normalized": false,
|
| 10 |
+
"rstrip": false,
|
| 11 |
+
"single_word": false,
|
| 12 |
+
"special": true
|
| 13 |
+
},
|
| 14 |
+
"1": {
|
| 15 |
+
"content": "<|im_start|>",
|
| 16 |
+
"lstrip": false,
|
| 17 |
+
"normalized": false,
|
| 18 |
+
"rstrip": false,
|
| 19 |
+
"single_word": false,
|
| 20 |
+
"special": true
|
| 21 |
+
},
|
| 22 |
+
"2": {
|
| 23 |
+
"content": "<|im_end|>",
|
| 24 |
+
"lstrip": false,
|
| 25 |
+
"normalized": false,
|
| 26 |
+
"rstrip": false,
|
| 27 |
+
"single_word": false,
|
| 28 |
+
"special": true
|
| 29 |
+
},
|
| 30 |
+
"3": {
|
| 31 |
+
"content": "<|object_ref_start|>",
|
| 32 |
+
"lstrip": false,
|
| 33 |
+
"normalized": false,
|
| 34 |
+
"rstrip": false,
|
| 35 |
+
"single_word": false,
|
| 36 |
+
"special": true
|
| 37 |
+
},
|
| 38 |
+
"4": {
|
| 39 |
+
"content": "<|object_ref_end|>",
|
| 40 |
+
"lstrip": false,
|
| 41 |
+
"normalized": false,
|
| 42 |
+
"rstrip": false,
|
| 43 |
+
"single_word": false,
|
| 44 |
+
"special": true
|
| 45 |
+
},
|
| 46 |
+
"5": {
|
| 47 |
+
"content": "<|box_start|>",
|
| 48 |
+
"lstrip": false,
|
| 49 |
+
"normalized": false,
|
| 50 |
+
"rstrip": false,
|
| 51 |
+
"single_word": false,
|
| 52 |
+
"special": true
|
| 53 |
+
},
|
| 54 |
+
"6": {
|
| 55 |
+
"content": "<|box_end|>",
|
| 56 |
+
"lstrip": false,
|
| 57 |
+
"normalized": false,
|
| 58 |
+
"rstrip": false,
|
| 59 |
+
"single_word": false,
|
| 60 |
+
"special": true
|
| 61 |
+
},
|
| 62 |
+
"7": {
|
| 63 |
+
"content": "<|quad_start|>",
|
| 64 |
+
"lstrip": false,
|
| 65 |
+
"normalized": false,
|
| 66 |
+
"rstrip": false,
|
| 67 |
+
"single_word": false,
|
| 68 |
+
"special": true
|
| 69 |
+
},
|
| 70 |
+
"8": {
|
| 71 |
+
"content": "<|quad_end|>",
|
| 72 |
+
"lstrip": false,
|
| 73 |
+
"normalized": false,
|
| 74 |
+
"rstrip": false,
|
| 75 |
+
"single_word": false,
|
| 76 |
+
"special": true
|
| 77 |
+
},
|
| 78 |
+
"9": {
|
| 79 |
+
"content": "<|vision_start|>",
|
| 80 |
+
"lstrip": false,
|
| 81 |
+
"normalized": false,
|
| 82 |
+
"rstrip": false,
|
| 83 |
+
"single_word": false,
|
| 84 |
+
"special": true
|
| 85 |
+
},
|
| 86 |
+
"10": {
|
| 87 |
+
"content": "<|vision_end|>",
|
| 88 |
+
"lstrip": false,
|
| 89 |
+
"normalized": false,
|
| 90 |
+
"rstrip": false,
|
| 91 |
+
"single_word": false,
|
| 92 |
+
"special": true
|
| 93 |
+
},
|
| 94 |
+
"11": {
|
| 95 |
+
"content": "<|vision_pad|>",
|
| 96 |
+
"lstrip": false,
|
| 97 |
+
"normalized": false,
|
| 98 |
+
"rstrip": false,
|
| 99 |
+
"single_word": false,
|
| 100 |
+
"special": true
|
| 101 |
+
},
|
| 102 |
+
"12": {
|
| 103 |
+
"content": "<|image_pad|>",
|
| 104 |
+
"lstrip": false,
|
| 105 |
+
"normalized": false,
|
| 106 |
+
"rstrip": false,
|
| 107 |
+
"single_word": false,
|
| 108 |
+
"special": true
|
| 109 |
+
},
|
| 110 |
+
"13": {
|
| 111 |
+
"content": "<|video_pad|>",
|
| 112 |
+
"lstrip": false,
|
| 113 |
+
"normalized": false,
|
| 114 |
+
"rstrip": false,
|
| 115 |
+
"single_word": false,
|
| 116 |
+
"special": true
|
| 117 |
+
},
|
| 118 |
+
"14": {
|
| 119 |
+
"content": "<|audio_start|>",
|
| 120 |
+
"lstrip": false,
|
| 121 |
+
"normalized": false,
|
| 122 |
+
"rstrip": false,
|
| 123 |
+
"single_word": false,
|
| 124 |
+
"special": true
|
| 125 |
+
},
|
| 126 |
+
"15": {
|
| 127 |
+
"content": "<|audio_end|>",
|
| 128 |
+
"lstrip": false,
|
| 129 |
+
"normalized": false,
|
| 130 |
+
"rstrip": false,
|
| 131 |
+
"single_word": false,
|
| 132 |
+
"special": true
|
| 133 |
+
},
|
| 134 |
+
"16": {
|
| 135 |
+
"content": "<|audio_pad|>",
|
| 136 |
+
"lstrip": false,
|
| 137 |
+
"normalized": false,
|
| 138 |
+
"rstrip": false,
|
| 139 |
+
"single_word": false,
|
| 140 |
+
"special": true
|
| 141 |
+
},
|
| 142 |
+
"17": {
|
| 143 |
+
"content": "<tts_pad>",
|
| 144 |
+
"lstrip": false,
|
| 145 |
+
"normalized": false,
|
| 146 |
+
"rstrip": false,
|
| 147 |
+
"single_word": false,
|
| 148 |
+
"special": true
|
| 149 |
+
},
|
| 150 |
+
"18": {
|
| 151 |
+
"content": "<tts_text_bos>",
|
| 152 |
+
"lstrip": false,
|
| 153 |
+
"normalized": false,
|
| 154 |
+
"rstrip": false,
|
| 155 |
+
"single_word": false,
|
| 156 |
+
"special": true
|
| 157 |
+
},
|
| 158 |
+
"19": {
|
| 159 |
+
"content": "<tts_text_eod>",
|
| 160 |
+
"lstrip": false,
|
| 161 |
+
"normalized": false,
|
| 162 |
+
"rstrip": false,
|
| 163 |
+
"single_word": false,
|
| 164 |
+
"special": true
|
| 165 |
+
},
|
| 166 |
+
"20": {
|
| 167 |
+
"content": "<tts_text_bos_single>",
|
| 168 |
+
"lstrip": false,
|
| 169 |
+
"normalized": false,
|
| 170 |
+
"rstrip": false,
|
| 171 |
+
"single_word": false,
|
| 172 |
+
"special": true
|
| 173 |
+
},
|
| 174 |
+
"21": {
|
| 175 |
+
"content": "<tool_call>",
|
| 176 |
+
"lstrip": false,
|
| 177 |
+
"normalized": false,
|
| 178 |
+
"rstrip": false,
|
| 179 |
+
"single_word": false,
|
| 180 |
+
"special": false
|
| 181 |
+
},
|
| 182 |
+
"22": {
|
| 183 |
+
"content": "</tool_call>",
|
| 184 |
+
"lstrip": false,
|
| 185 |
+
"normalized": false,
|
| 186 |
+
"rstrip": false,
|
| 187 |
+
"single_word": false,
|
| 188 |
+
"special": false
|
| 189 |
+
},
|
| 190 |
+
"23": {
|
| 191 |
+
"content": "<tool_response>",
|
| 192 |
+
"lstrip": false,
|
| 193 |
+
"normalized": false,
|
| 194 |
+
"rstrip": false,
|
| 195 |
+
"single_word": false,
|
| 196 |
+
"special": false
|
| 197 |
+
},
|
| 198 |
+
"24": {
|
| 199 |
+
"content": "</tool_response>",
|
| 200 |
+
"lstrip": false,
|
| 201 |
+
"normalized": false,
|
| 202 |
+
"rstrip": false,
|
| 203 |
+
"single_word": false,
|
| 204 |
+
"special": false
|
| 205 |
+
},
|
| 206 |
+
"25": {
|
| 207 |
+
"content": "<think>",
|
| 208 |
+
"lstrip": false,
|
| 209 |
+
"normalized": false,
|
| 210 |
+
"rstrip": false,
|
| 211 |
+
"single_word": false,
|
| 212 |
+
"special": false
|
| 213 |
+
},
|
| 214 |
+
"26": {
|
| 215 |
+
"content": "</think>",
|
| 216 |
+
"lstrip": false,
|
| 217 |
+
"normalized": false,
|
| 218 |
+
"rstrip": false,
|
| 219 |
+
"single_word": false,
|
| 220 |
+
"special": false
|
| 221 |
+
},
|
| 222 |
+
"27": {
|
| 223 |
+
"content": "<|buffer1|>",
|
| 224 |
+
"lstrip": false,
|
| 225 |
+
"normalized": false,
|
| 226 |
+
"rstrip": false,
|
| 227 |
+
"single_word": false,
|
| 228 |
+
"special": false
|
| 229 |
+
},
|
| 230 |
+
"28": {
|
| 231 |
+
"content": "<|buffer2|>",
|
| 232 |
+
"lstrip": false,
|
| 233 |
+
"normalized": false,
|
| 234 |
+
"rstrip": false,
|
| 235 |
+
"single_word": false,
|
| 236 |
+
"special": false
|
| 237 |
+
},
|
| 238 |
+
"29": {
|
| 239 |
+
"content": "<|buffer3|>",
|
| 240 |
+
"lstrip": false,
|
| 241 |
+
"normalized": false,
|
| 242 |
+
"rstrip": false,
|
| 243 |
+
"single_word": false,
|
| 244 |
+
"special": false
|
| 245 |
+
},
|
| 246 |
+
"30": {
|
| 247 |
+
"content": "<|buffer4|>",
|
| 248 |
+
"lstrip": false,
|
| 249 |
+
"normalized": false,
|
| 250 |
+
"rstrip": false,
|
| 251 |
+
"single_word": false,
|
| 252 |
+
"special": false
|
| 253 |
+
},
|
| 254 |
+
"31": {
|
| 255 |
+
"content": "<|buffer5|>",
|
| 256 |
+
"lstrip": false,
|
| 257 |
+
"normalized": false,
|
| 258 |
+
"rstrip": false,
|
| 259 |
+
"single_word": false,
|
| 260 |
+
"special": false
|
| 261 |
+
},
|
| 262 |
+
"32": {
|
| 263 |
+
"content": "<|buffer6|>",
|
| 264 |
+
"lstrip": false,
|
| 265 |
+
"normalized": false,
|
| 266 |
+
"rstrip": false,
|
| 267 |
+
"single_word": false,
|
| 268 |
+
"special": false
|
| 269 |
+
},
|
| 270 |
+
"33": {
|
| 271 |
+
"content": "<|buffer7|>",
|
| 272 |
+
"lstrip": false,
|
| 273 |
+
"normalized": false,
|
| 274 |
+
"rstrip": false,
|
| 275 |
+
"single_word": false,
|
| 276 |
+
"special": false
|
| 277 |
+
},
|
| 278 |
+
"34": {
|
| 279 |
+
"content": "<|buffer8|>",
|
| 280 |
+
"lstrip": false,
|
| 281 |
+
"normalized": false,
|
| 282 |
+
"rstrip": false,
|
| 283 |
+
"single_word": false,
|
| 284 |
+
"special": false
|
| 285 |
+
},
|
| 286 |
+
"35": {
|
| 287 |
+
"content": "<|buffer9|>",
|
| 288 |
+
"lstrip": false,
|
| 289 |
+
"normalized": false,
|
| 290 |
+
"rstrip": false,
|
| 291 |
+
"single_word": false,
|
| 292 |
+
"special": false
|
| 293 |
+
}
|
| 294 |
+
},
|
| 295 |
+
"additional_special_tokens": [
|
| 296 |
+
"<|im_start|>",
|
| 297 |
+
"<|im_end|>",
|
| 298 |
+
"<|object_ref_start|>",
|
| 299 |
+
"<|object_ref_end|>",
|
| 300 |
+
"<|box_start|>",
|
| 301 |
+
"<|box_end|>",
|
| 302 |
+
"<|quad_start|>",
|
| 303 |
+
"<|quad_end|>",
|
| 304 |
+
"<|vision_start|>",
|
| 305 |
+
"<|vision_end|>",
|
| 306 |
+
"<|vision_pad|>",
|
| 307 |
+
"<|image_pad|>",
|
| 308 |
+
"<|video_pad|>",
|
| 309 |
+
"<|audio_start|>",
|
| 310 |
+
"<|audio_end|>",
|
| 311 |
+
"<|audio_pad|>",
|
| 312 |
+
"<tts_pad>",
|
| 313 |
+
"<tts_text_bos>",
|
| 314 |
+
"<tts_text_eod>",
|
| 315 |
+
"<tts_text_bos_single>"
|
| 316 |
+
],
|
| 317 |
+
"audio_bos_token": "<|audio_start|>",
|
| 318 |
+
"audio_eos_token": "<|audio_end|>",
|
| 319 |
+
"audio_token": "<|audio_pad|>",
|
| 320 |
+
"bos_token": "<|im_start|>",
|
| 321 |
+
"clean_up_tokenization_spaces": false,
|
| 322 |
+
"eos_token": "<|im_end|>",
|
| 323 |
+
"extra_special_tokens": {},
|
| 324 |
+
"image_token": "<|image_pad|>",
|
| 325 |
+
"legacy": true,
|
| 326 |
+
"model_max_length": 131072,
|
| 327 |
+
"pad_token": "<|endoftext|>",
|
| 328 |
+
"sp_model_kwargs": {},
|
| 329 |
+
"spaces_between_special_tokens": false,
|
| 330 |
+
"tokenizer_class": "PreTrainedTokenizerFast",
|
| 331 |
+
"unk_token": "<|endoftext|>",
|
| 332 |
+
"video_token": "<|video_pad|>",
|
| 333 |
+
"vision_bos_token": "<|vision_start|>",
|
| 334 |
+
"vision_eos_token": "<|vision_end|>"
|
| 335 |
+
}
|