Upload folder using huggingface_hub

README.md

@@ -112,6 +112,7 @@ model = AutoModel.from_pretrained(
     path,
     torch_dtype=torch.bfloat16,
     low_cpu_mem_usage=True,
+    use_flash_attn=True,
     trust_remote_code=True).eval().cuda()
 ```
 
@@ -126,6 +127,7 @@ model = AutoModel.from_pretrained(
     torch_dtype=torch.bfloat16,
     load_in_8bit=True,
     low_cpu_mem_usage=True,
+    use_flash_attn=True,
     trust_remote_code=True).eval()
 ```
 
@@ -172,6 +174,7 @@ model = AutoModel.from_pretrained(
     path,
     torch_dtype=torch.bfloat16,
     low_cpu_mem_usage=True,
+    use_flash_attn=True,
     trust_remote_code=True,
     device_map=device_map).eval()
 ```
@@ -189,6 +192,7 @@ model = AutoModel.from_pretrained(
     path,
     torch_dtype=torch.bfloat16,
     low_cpu_mem_usage=True,
+    use_flash_attn=True,
     trust_remote_code=True).eval().cuda()
 tokenizer = AutoTokenizer.from_pretrained(path, trust_remote_code=True, use_fast=False)
 
@@ -216,6 +220,7 @@ model = AutoModel.from_pretrained(
     path,
     torch_dtype=torch.bfloat16,
     low_cpu_mem_usage=True,
+    use_flash_attn=True,
     trust_remote_code=True).eval().cuda()
 tokenizer = AutoTokenizer.from_pretrained(path, trust_remote_code=True, use_fast=False)
 
@@ -242,6 +247,7 @@ model = AutoModel.from_pretrained(
     path,
     torch_dtype=torch.bfloat16,
     low_cpu_mem_usage=True,
+    use_flash_attn=True,
     trust_remote_code=True).eval().cuda()
 tokenizer = AutoTokenizer.from_pretrained(path, trust_remote_code=True, use_fast=False)
 
@@ -275,6 +281,7 @@ model = AutoModel.from_pretrained(
     path,
     torch_dtype=torch.bfloat16,
     low_cpu_mem_usage=True,
+    use_flash_attn=True,
     trust_remote_code=True).eval().cuda()
 tokenizer = AutoTokenizer.from_pretrained(path, trust_remote_code=True, use_fast=False)
 
@@ -313,6 +320,7 @@ model = AutoModel.from_pretrained(
     path,
     torch_dtype=torch.bfloat16,
     low_cpu_mem_usage=True,
+    use_flash_attn=True,
     trust_remote_code=True).eval().cuda()
 tokenizer = AutoTokenizer.from_pretrained(path, trust_remote_code=True, use_fast=False)
 
@@ -350,6 +358,7 @@ model = AutoModel.from_pretrained(
     path,
     torch_dtype=torch.bfloat16,
     low_cpu_mem_usage=True,
+    use_flash_attn=True,
     trust_remote_code=True).eval().cuda()
 tokenizer = AutoTokenizer.from_pretrained(path, trust_remote_code=True, use_fast=False)
 
@@ -420,6 +429,7 @@ model = AutoModel.from_pretrained(
     path,
     torch_dtype=torch.bfloat16,
     low_cpu_mem_usage=True,
+    use_flash_attn=True,
     trust_remote_code=True).eval().cuda()
 tokenizer = AutoTokenizer.from_pretrained(path, trust_remote_code=True, use_fast=False)
 

modeling_internvl_chat.py

@@ -17,7 +17,7 @@ from transformers.utils import ModelOutput, logging
 
 from .configuration_internvl_chat import InternVLChatConfig
 from .conversation import get_conv_template
-from .modeling_intern_vit import InternVisionModel
+from .modeling_intern_vit import InternVisionModel, has_flash_attn
 
 logger = logging.get_logger(__name__)
 
@@ -48,6 +48,7 @@ class InternVLChatModel(PreTrainedModel):
         self.num_image_token = int((image_size // patch_size) ** 2 * (config.downsample_ratio ** 2))
         self.downsample_ratio = config.downsample_ratio
         self.ps_version = config.ps_version
+        use_flash_attn = use_flash_attn if has_flash_attn else False
         config.vision_config.use_flash_attn = True if use_flash_attn else False
         config.llm_config._attn_implementation = 'flash_attention_2' if use_flash_attn else 'eager'