fix compatibility issue for transformers 4.46+

README.md

@@ -13,11 +13,12 @@ tags:
   - custom_code
 base_model: OpenGVLab/InternVL2-40B
 base_model_relation: quantized
+new_version: OpenGVLab/InternVL2_5-40B-AWQ
 ---
 
 # InternVL2-40B-AWQ
 
-[\[📂 GitHub\]](https://github.com/OpenGVLab/InternVL)  [\[🆕 Blog\]](https://internvl.github.io/blog/)  [\[📜 InternVL 1.0 Paper\]](https://arxiv.org/abs/2312.14238)  [\[📜 InternVL 1.5 Report\]](https://arxiv.org/abs/2404.16821)
+[\[📂 GitHub\]](https://github.com/OpenGVLab/InternVL)  [\[🆕 Blog\]](https://internvl.github.io/blog/)  [\[📜 InternVL 1.0\]](https://arxiv.org/abs/2312.14238)  [\[📜 InternVL 1.5\]](https://arxiv.org/abs/2404.16821) [\[📜 Mini-InternVL\]](https://arxiv.org/abs/2410.16261)
 
 [\[🗨️ Chat Demo\]](https://internvl.opengvlab.com/)  [\[🤗 HF Demo\]](https://huggingface.co/spaces/OpenGVLab/InternVL)  [\[🚀 Quick Start\]](#quick-start)  [\[📖 中文解读\]](https://zhuanlan.zhihu.com/p/706547971)  [\[📖 Documents\]](https://internvl.readthedocs.io/en/latest/)
 
@@ -42,7 +43,7 @@ LMDeploy supports the following NVIDIA GPU for W4A16 inference:
 Before proceeding with the quantization and inference, please ensure that lmdeploy is installed.
 
 ```shell
-pip install lmdeploy==0.5.3
+pip install lmdeploy>=0.5.3
 ```
 
 This article comprises the following sections:
@@ -123,6 +124,12 @@ This project is released under the MIT license, while InternLM2 is licensed unde
 If you find this project useful in your research, please consider citing:
 
 ```BibTeX
+@article{gao2024mini,
+  title={Mini-internvl: A flexible-transfer pocket multimodal model with 5\% parameters and 90\% performance},
+  author={Gao, Zhangwei and Chen, Zhe and Cui, Erfei and Ren, Yiming and Wang, Weiyun and Zhu, Jinguo and Tian, Hao and Ye, Shenglong and He, Junjun and Zhu, Xizhou and others},
+  journal={arXiv preprint arXiv:2410.16261},
+  year={2024}
+}
 @article{chen2023internvl,
   title={InternVL: Scaling up Vision Foundation Models and Aligning for Generic Visual-Linguistic Tasks},
   author={Chen, Zhe and Wu, Jiannan and Wang, Wenhai and Su, Weijie and Chen, Guo and Xing, Sen and Zhong, Muyan and Zhang, Qinglong and Zhu, Xizhou and Lu, Lewei and Li, Bin and Luo, Ping and Lu, Tong and Qiao, Yu and Dai, Jifeng},

config.json

@@ -17,9 +17,9 @@
     "architectures": [
       "LlamaForCausalLM"
     ],
+    "attn_implementation": "eager",
     "attention_bias": false,
     "attention_dropout": 0.0,
-    "attn_implementation": "eager",
     "bad_words_ids": null,
     "begin_suppress_tokens": null,
     "bos_token_id": 1,

configuration_internvl_chat.py

@@ -38,11 +38,11 @@ class InternVLChatConfig(PretrainedConfig):
         super().__init__(**kwargs)
 
         if vision_config is None:
-            vision_config = {}
+            vision_config = {'architectures': ['InternVisionModel']}
             logger.info('vision_config is None. Initializing the InternVisionConfig with default values.')
 
         if llm_config is None:
-            llm_config = {}
+            llm_config = {'architectures': ['LlamaForCausalLM']}
             logger.info('llm_config is None. Initializing the LlamaConfig config with default values (`LlamaConfig`).')
 
         self.vision_config = InternVisionConfig(**vision_config)

modeling_intern_vit.py

@@ -3,6 +3,7 @@
 # Copyright (c) 2024 OpenGVLab
 # Licensed under The MIT License [see LICENSE for details]
 # --------------------------------------------------------
+
 from typing import Optional, Tuple, Union
 
 import torch