Create README.md

README.md

@@ -0,0 +1,224 @@
+---
+datasets:
+- Lin-Chen/ShareGPT4V
+pipeline_tag: image-text-to-text
+library_name: xtuner
+---
+
+<div align="center">
+  <img src="https://github.com/InternLM/lmdeploy/assets/36994684/0cf8d00f-e86b-40ba-9b54-dc8f1bc6c8d8" width="600"/>
+
+
+[![Generic badge](https://img.shields.io/badge/GitHub-%20XTuner-black.svg)](https://github.com/InternLM/xtuner)
+
+
+</div>
+
+## Model
+
+llava-phi-3-mini is a LLaVA model fine-tuned from [microsoft/Phi-3-mini-4k-instruct](https://huggingface.co/microsoft/Phi-3-mini-4k-instruct) and [CLIP-ViT-Large-patch14-336](https://huggingface.co/openai/clip-vit-large-patch14-336) with [ShareGPT4V-PT](https://huggingface.co/datasets/Lin-Chen/ShareGPT4V) and [InternVL-SFT](https://github.com/OpenGVLab/InternVL/tree/main/internvl_chat#prepare-training-datasets) by [XTuner](https://github.com/InternLM/xtuner).
+
+**Note: This model is in official LLaVA format. The models in xtuner LLaVA format and HuggingFace LLaVA format can be found on [xtuner/llava-phi-3-mini-xtuner](https://huggingface.co/xtuner/llava-phi-3-mini-xtuner) and [xtuner/llava-phi-3-mini-hf](https://huggingface.co/xtuner/llava-phi-3-mini-hf).**
+
+
+## Details
+
+| Model                 | Visual      Encoder | Projector | Resolution |   Pretraining Strategy | Fine-tuning      Strategy |      Pretrain     Dataset |    Fine-tune     Dataset |
+| :-------------------- | ------------------: | --------: | ---------: | ---------------------: | ------------------------: | ------------------------: | -----------------------: |
+| LLaVA-v1.5-7B         |              CLIP-L |       MLP |        336 | Frozen LLM, Frozen ViT |      Full LLM, Frozen ViT |       LLaVA-PT     (558K) |     LLaVA-Mix     (665K) |
+| LLaVA-Llama-3-8B      |              CLIP-L |       MLP |        336 | Frozen LLM, Frozen ViT |        Full LLM, LoRA ViT |       LLaVA-PT     (558K) |     LLaVA-Mix     (665K) |
+| LLaVA-Llama-3-8B-v1.1 |              CLIP-L |       MLP |        336 | Frozen LLM, Frozen ViT |        Full LLM, LoRA ViT | ShareGPT4V-PT     (1246K) | InternVL-SFT     (1268K) |
+| LLaVA-Phi-3-mini      |              CLIP-L |       MLP |        336 | Frozen LLM, Frozen ViT |        Full LLM, Full ViT | ShareGPT4V-PT     (1246K) | InternVL-SFT     (1268K) |
+
+## Results
+
+
+## Quickstart
+
+### Chat with LLaVA official library
+
+1. Install official LLaVA library
+
+```bash
+pip install git+https://github.com/haotian-liu/LLaVA.git
+```
+
+2. Chat with below script
+
+<details> 
+    <summary>cli.py</summary>
+
+```python
+import argparse
+from io import BytesIO
+
+import requests
+import torch
+from llava.constants import DEFAULT_IMAGE_TOKEN, IMAGE_TOKEN_INDEX
+from llava.conversation import Conversation, SeparatorStyle
+from llava.mm_utils import process_images, tokenizer_image_token
+from llava.model import LlavaLlamaForCausalLM
+from PIL import Image
+from transformers import (AutoTokenizer, BitsAndBytesConfig, StoppingCriteria,
+                          StoppingCriteriaList, TextStreamer)
+
+
+def load_image(image_file):
+    if image_file.startswith('http://') or image_file.startswith('https://'):
+        response = requests.get(image_file)
+        image = Image.open(BytesIO(response.content)).convert('RGB')
+    else:
+        image = Image.open(image_file).convert('RGB')
+    return image
+
+
+class StopWordStoppingCriteria(StoppingCriteria):
+    """StopWord stopping criteria."""
+
+    def __init__(self, tokenizer, stop_word):
+        self.tokenizer = tokenizer
+        self.stop_word = stop_word
+        self.length = len(self.stop_word)
+
+    def __call__(self, input_ids, *args, **kwargs) -> bool:
+        cur_text = self.tokenizer.decode(input_ids[0])
+        cur_text = cur_text.replace('\r', '').replace('\n', '')
+        return cur_text[-self.length:] == self.stop_word
+
+
+def get_stop_criteria(tokenizer, stop_words=[]):
+    stop_criteria = StoppingCriteriaList()
+    for word in stop_words:
+        stop_criteria.append(StopWordStoppingCriteria(tokenizer, word))
+    return stop_criteria
+
+
+def main(args):
+    kwargs = {'device_map': args.device}
+    if args.load_8bit:
+        kwargs['load_in_8bit'] = True
+    elif args.load_4bit:
+        kwargs['load_in_4bit'] = True
+        kwargs['quantization_config'] = BitsAndBytesConfig(
+            load_in_4bit=True,
+            bnb_4bit_compute_dtype=torch.float16,
+            bnb_4bit_use_double_quant=True,
+            bnb_4bit_quant_type='nf4')
+    else:
+        kwargs['torch_dtype'] = torch.float16
+
+    tokenizer = AutoTokenizer.from_pretrained(args.model_path)
+    model = LlavaLlamaForCausalLM.from_pretrained(
+        args.model_path, low_cpu_mem_usage=True, **kwargs)
+    vision_tower = model.get_vision_tower()
+    if not vision_tower.is_loaded:
+        vision_tower.load_model(device_map=args.device)
+    image_processor = vision_tower.image_processor
+
+    conv = Conversation(
+        system=system='<|start_header_id|>system<|end_header_id|>\n\nAnswer the questions.',
+        roles=('<|start_header_id|>user<|end_header_id|>\n\n',
+               '<|start_header_id|>assistant<|end_header_id|>\n\n'),
+        messages=[],
+        offset=0,
+        sep_style=SeparatorStyle.MPT,
+        sep='<|eot_id|>',
+    )
+    roles = conv.roles
+
+    image = load_image(args.image_file)
+    image_size = image.size
+    image_tensor = process_images([image], image_processor, model.config)
+
+    if type(image_tensor) is list:
+        image_tensor = [
+            image.to(model.device, dtype=torch.float16)
+            for image in image_tensor
+        ]
+    else:
+        image_tensor = image_tensor.to(model.device, dtype=torch.float16)
+
+    while True:
+        try:
+            inp = input(f'{roles[0]}: ')
+        except EOFError:
+            inp = ''
+        if not inp:
+            print('exit...')
+            break
+
+        print(f'{roles[1]}: ', end='')
+
+        if image is not None:
+            inp = DEFAULT_IMAGE_TOKEN + '\n' + inp
+            image = None
+
+        conv.append_message(conv.roles[0], inp)
+        conv.append_message(conv.roles[1], None)
+        prompt = conv.get_prompt()
+
+        input_ids = tokenizer_image_token(
+            prompt, tokenizer, IMAGE_TOKEN_INDEX,
+            return_tensors='pt').unsqueeze(0).to(model.device)
+        stop_criteria = get_stop_criteria(
+            tokenizer=tokenizer, stop_words=[conv.sep])
+
+        streamer = TextStreamer(
+            tokenizer, skip_prompt=True, skip_special_tokens=True)
+
+        with torch.inference_mode():
+            output_ids = model.generate(
+                input_ids,
+                images=image_tensor,
+                image_sizes=[image_size],
+                do_sample=True if args.temperature > 0 else False,
+                temperature=args.temperature,
+                max_new_tokens=args.max_new_tokens,
+                streamer=streamer,
+                stopping_criteria=stop_criteria,
+                use_cache=True)
+
+        outputs = tokenizer.decode(output_ids[0]).strip()
+        conv.messages[-1][-1] = outputs
+
+        if args.debug:
+            print('\n', {'prompt': prompt, 'outputs': outputs}, '\n')
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '--model-path', type=str, default='xtuner/llava-llama-3-8b-v1_1-hf')
+    parser.add_argument('--image-file', type=str, required=True)
+    parser.add_argument('--device', type=str, default='auto')
+    parser.add_argument('--temperature', type=float, default=0.2)
+    parser.add_argument('--max-new-tokens', type=int, default=512)
+    parser.add_argument('--load-8bit', action='store_true')
+    parser.add_argument('--load-4bit', action='store_true')
+    parser.add_argument('--debug', action='store_true')
+    args = parser.parse_args()
+    main(args)
+```
+
+</details>
+
+```
+# example
+python ./cli.py  --model-path xtuner/llava-phi-3-mini --image-file https://raw.githubusercontent.com/open-mmlab/mmdeploy/main/tests/data/tiger.jpeg  --load-4bit
+```
+
+
+### Reproduction
+
+Please refer to [docs](https://github.com/InternLM/xtuner/tree/main/xtuner/configs/llava/phi3_mini_4k_instruct_clip_vit_large_p14_336#readme).
+
+## Citation
+
+```bibtex
+@misc{2023xtuner,
+    title={XTuner: A Toolkit for Efficiently Fine-tuning LLM},
+    author={XTuner Contributors},
+    howpublished = {\url{https://github.com/InternLM/xtuner}},
+    year={2023}
+}
+```