Upload folder using huggingface_hub

README.md

@@ -57,6 +57,8 @@ Limitations: Although we have made efforts to ensure the safety of the model dur
 
 We provide an example code to run Mini-InternVL-Chat-4B-V1-5 using `transformers`.
 
+We also welcome you to experience the InternVL2 series models in our [online demo](https://internvl.opengvlab.com/). Currently, due to the limited GPU resources with public IP addresses, we can only deploy models up to a maximum of 26B. We will expand soon and deploy larger models to the online demo.
+
 > Please use transformers==4.37.2 to ensure the model works normally.
 
 ```python
@@ -301,7 +303,150 @@ print(f'Assistant: {response}')
 
 ### LMDeploy
 
-> Warning: This model is not yet supported by LMDeploy.
+LMDeploy is a toolkit for compressing, deploying, and serving LLM, developed by the MMRazor and MMDeploy teams.
+
+```sh
+pip install lmdeploy
+```
+
+LMDeploy abstracts the complex inference process of multi-modal Vision-Language Models (VLM) into an easy-to-use pipeline, similar to the Large Language Model (LLM) inference pipeline.
+
+#### A 'Hello, world' example
+
+```python
+from lmdeploy import pipeline, PytorchEngineConfig, ChatTemplateConfig
+from lmdeploy.vl import load_image
+
+model = 'OpenGVLab/Mini-InternVL-Chat-4B-V1-5'
+image = load_image('https://raw.githubusercontent.com/open-mmlab/mmdeploy/main/tests/data/tiger.jpeg')
+chat_template_config = ChatTemplateConfig('internvl-phi3')
+pipe = pipeline(model, chat_template_config=chat_template_config,
+                backend_config=PytorchEngineConfig(session_len=8192))
+response = pipe(('describe this image', image))
+print(response.text)
+```
+
+If `ImportError` occurs while executing this case, please install the required dependency packages as prompted.
+
+#### Multi-images inference
+
+When dealing with multiple images, you can put them all in one list. Keep in mind that multiple images will lead to a higher number of input tokens, and as a result, the size of the context window typically needs to be increased.
+
+> Warning: Due to the scarcity of multi-image conversation data, the performance on multi-image tasks may be unstable, and it may require multiple attempts to achieve satisfactory results.
+
+```python
+from lmdeploy import pipeline, PytorchEngineConfig, ChatTemplateConfig
+from lmdeploy.vl import load_image
+from lmdeploy.vl.constants import IMAGE_TOKEN
+
+model = 'OpenGVLab/Mini-InternVL-Chat-4B-V1-5'
+chat_template_config = ChatTemplateConfig('internvl-phi3')
+pipe = pipeline(model, chat_template_config=chat_template_config,
+                backend_config=PytorchEngineConfig(session_len=8192))
+
+image_urls=[
+    'https://raw.githubusercontent.com/open-mmlab/mmdeploy/main/demo/resources/human-pose.jpg',
+    'https://raw.githubusercontent.com/open-mmlab/mmdeploy/main/demo/resources/det.jpg'
+]
+
+images = [load_image(img_url) for img_url in image_urls]
+# Numbering images improves multi-image conversations
+response = pipe((f'Image-1: {IMAGE_TOKEN}\nImage-2: {IMAGE_TOKEN}\ndescribe these two images', images))
+print(response.text)
+```
+
+#### Batch prompts inference
+
+Conducting inference with batch prompts is quite straightforward; just place them within a list structure:
+
+```python
+from lmdeploy import pipeline, PytorchEngineConfig, ChatTemplateConfig
+from lmdeploy.vl import load_image
+
+model = 'OpenGVLab/Mini-InternVL-Chat-4B-V1-5'
+chat_template_config = ChatTemplateConfig('internvl-phi3')
+pipe = pipeline(model, chat_template_config=chat_template_config,
+                backend_config=PytorchEngineConfig(session_len=8192))
+
+image_urls=[
+    "https://raw.githubusercontent.com/open-mmlab/mmdeploy/main/demo/resources/human-pose.jpg",
+    "https://raw.githubusercontent.com/open-mmlab/mmdeploy/main/demo/resources/det.jpg"
+]
+prompts = [('describe this image', load_image(img_url)) for img_url in image_urls]
+response = pipe(prompts)
+print(response)
+```
+
+#### Multi-turn conversation
+
+There are two ways to do the multi-turn conversations with the pipeline. One is to construct messages according to the format of OpenAI and use above introduced method, the other is to use the `pipeline.chat` interface.
+
+```python
+from lmdeploy import pipeline, PytorchEngineConfig, ChatTemplateConfig, GenerationConfig
+from lmdeploy.vl import load_image
+
+model = 'OpenGVLab/Mini-InternVL-Chat-4B-V1-5'
+chat_template_config = ChatTemplateConfig('internvl-phi3')
+pipe = pipeline(model, chat_template_config=chat_template_config,
+                backend_config=PytorchEngineConfig(session_len=8192))
+
+image = load_image('https://raw.githubusercontent.com/open-mmlab/mmdeploy/main/demo/resources/human-pose.jpg')
+gen_config = GenerationConfig(top_k=40, top_p=0.8, temperature=0.8)
+sess = pipe.chat(('describe this image', image), gen_config=gen_config)
+print(sess.response.text)
+sess = pipe.chat('What is the woman doing?', session=sess, gen_config=gen_config)
+print(sess.response.text)
+```
+
+#### Service
+
+LMDeploy's `api_server` enables models to be easily packed into services with a single command. The provided RESTful APIs are compatible with OpenAI's interfaces. Below are an example of service startup:
+
+```shell
+lmdeploy serve api_server OpenGVLab/Mini-InternVL-Chat-4B-V1-5 --model-name Mini-InternVL-Chat-4B-V1-5 --backend pytorch --server-port 23333
+```
+
+To use the OpenAI-style interface, you need to install OpenAI:
+
+```shell
+pip install openai
+```
+
+Then, use the code below to make the API call:
+
+```python
+from openai import OpenAI
+
+client = OpenAI(api_key='YOUR_API_KEY', base_url='http://0.0.0.0:23333/v1')
+model_name = client.models.list().data[0].id
+response = client.chat.completions.create(
+    model="Mini-InternVL-Chat-4B-V1-5",
+    messages=[{
+        'role':
+        'user',
+        'content': [{
+            'type': 'text',
+            'text': 'describe this image',
+        }, {
+            'type': 'image_url',
+            'image_url': {
+                'url':
+                'https://modelscope.oss-cn-beijing.aliyuncs.com/resource/tiger.jpeg',
+            },
+        }],
+    }],
+    temperature=0.8,
+    top_p=0.8)
+print(response)
+```
+
+### vLLM
+
+TODO
+
+### Ollama
+
+TODO
 
 ## License
 

config.json

@@ -193,7 +193,7 @@
     "tie_word_embeddings": false,
     "tokenizer_class": null,
     "top_k": 50,
-    "top_p": null,
+    "top_p": 1.0,
     "torch_dtype": "bfloat16",
     "torchscript": false,
     "transformers_version": "4.37.2",

configuration_intern_vit.py

@@ -1,6 +1,6 @@
 # --------------------------------------------------------
 # InternVL
-# Copyright (c) 2023 OpenGVLab
+# Copyright (c) 2024 OpenGVLab
 # Licensed under The MIT License [see LICENSE for details]
 # --------------------------------------------------------
 import os

configuration_internvl_chat.py

@@ -1,6 +1,6 @@
 # --------------------------------------------------------
 # InternVL
-# Copyright (c) 2023 OpenGVLab
+# Copyright (c) 2024 OpenGVLab
 # Licensed under The MIT License [see LICENSE for details]
 # --------------------------------------------------------
 

modeling_intern_vit.py

@@ -1,6 +1,6 @@
 # --------------------------------------------------------
 # InternVL
-# Copyright (c) 2023 OpenGVLab
+# Copyright (c) 2024 OpenGVLab
 # Licensed under The MIT License [see LICENSE for details]
 # --------------------------------------------------------
 from typing import Optional, Tuple, Union

modeling_phi3.py

@@ -53,6 +53,7 @@ try:
                                          unpad_input)
 
     _flash_supports_window_size = 'window_size' in list(inspect.signature(flash_attn_func).parameters)
+    has_flash_attn = True
 except ImportError as error:
     logger.warning(
         f'`flash-attention` package not found, consider installing for better performance: {error}.'
@@ -61,6 +62,7 @@ except ImportError as error:
         logger.warning(
             "Current `flash-attenton` does not support `window_size`. Either upgrade or use `attn_implementation='eager'`."
         )
+    has_flash_attn = False
 
 _CHECKPOINT_FOR_DOC = 'microsoft/Phi-3-mini-4k-instruct'
 _CONFIG_FOR_DOC = 'Phi3Config'
@@ -937,6 +939,12 @@ class Phi3PreTrainedModel(PreTrainedModel):
 
     _version = '0.0.5'
 
+    def __init__(self, config: Phi3Config):
+        if not has_flash_attn:
+            config._attn_implementation = 'eager'
+            print('Warning: Flash attention is not available, using eager attention instead.')
+        super().__init__(config)
+
     def _init_weights(self, module):
         std = self.config.initializer_range
         if isinstance(module, nn.Linear):
@@ -1042,6 +1050,7 @@ class Phi3Model(Phi3PreTrainedModel):
             [Phi3DecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)]
         )
         self._attn_implementation = config._attn_implementation
+
         self.norm = Phi3RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
 
         self.gradient_checkpointing = False