update

.dockerignore

@@ -1,21 +0,0 @@
-# The .dockerignore file excludes files from the container build process.
-#
-# https://docs.docker.com/engine/reference/builder/#dockerignore-file
-
-# Exclude Git files
-.git
-.github
-.gitignore
-
-# Exclude Python cache files
-__pycache__
-.mypy_cache
-.pytest_cache
-.ruff_cache
-
-# Exclude Python virtual environment
-/venv
-
-# Exclude some weights
-/openai
-/liuhaotian

.editorconfig

@@ -1,18 +0,0 @@
-root = true
-
-# Unix-style newlines with a newline ending every file
-[*]
-end_of_line = lf
-insert_final_newline = true
-trim_trailing_whitespace = true
-charset = utf-8
-
-# 4 space indentation
-[*.{py,json}]
-indent_style = space
-indent_size = 4
-
-# 2 space indentation
-[*.{md,sh,yaml,yml}]
-indent_style = space
-indent_size = 2

.gitignore

@@ -28,8 +28,4 @@ ckpts*
 .ipynb_checkpoints
 *.ipynb
 
-# DevContainer
-!.devcontainer/*
-
-# Demo
-serve_images/
+*.log

README.md

@@ -1,13 +1,9 @@
 ---
-title: EgoLlava
-emoji: 👀
-colorFrom: gray
+title: LLaVA
+emoji: 🔥
+colorFrom: purple
 colorTo: gray
 sdk: gradio
-sdk_version: 5.17.1
-app_file: app.py
-pinned: false
-short_description: Demonstration of EgoLlava
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+sdk_version: 3.36.1
+app_port: 7860
+---

app.py

@@ -23,9 +23,9 @@ logger = build_logger("gradio_web_server", "gradio_web_server.log")
 
 headers = {"User-Agent": "LLaVA Client"}
 
-no_change_btn = gr.update()
-enable_btn = gr.update(interactive=True)
-disable_btn = gr.update(interactive=False)
+no_change_btn = gr.Button.update()
+enable_btn = gr.Button.update(interactive=True)
+disable_btn = gr.Button.update(interactive=False)
 
 priority = {
     "vicuna-13b": "aaaaaaa",
@@ -339,13 +339,17 @@ def http_bot(
 title_markdown = """
 # 🌋 LLaVA: Large Language and Vision Assistant
 [[Project Page]](https://llava-vl.github.io) [[Paper]](https://arxiv.org/abs/2304.08485) [[Code]](https://github.com/haotian-liu/LLaVA) [[Model]](https://github.com/haotian-liu/LLaVA/blob/main/docs/MODEL_ZOO.md)
+
 ONLY WORKS WITH GPU!
+
 You can load the model with 4-bit or 8-bit quantization to make it fit in smaller hardwares. Setting the environment variable `bits` to control the quantization.
 *Note: 8-bit seems to be slower than both 4-bit/16-bit. Although it has enough VRAM to support 8-bit, until we figure out the inference speed issue, we recommend 4-bit for A10G for the best efficiency.*
+
 Recommended configurations:
 | Hardware          | T4-Small (16G)  | A10G-Small (24G) | A100-Large (40G) |
 |-------------------|-----------------|------------------|------------------|
 | **Bits**          | 4 (default)     | 4                | 16               |
+
 """
 
 tos_markdown = """
@@ -363,9 +367,11 @@ The service is a research preview intended for non-commercial use only, subject
 """
 
 block_css = """
+
 #buttons button {
     min-width: min(120px,100%);
 }
+
 """
 
 
@@ -590,7 +596,7 @@ def get_args():
 def start_demo(args):
     demo = build_demo(args.embed)
     demo.queue(
-        default_concurrency_limit=args.concurrency_count, status_update_rate=10, api_open=False
+        concurrency_count=args.concurrency_count, status_update_rate=10, api_open=False
     ).launch(server_name=args.host, server_port=args.port, share=args.share)
 
 
@@ -598,7 +604,7 @@ if __name__ == "__main__":
     args = get_args()
     logger.info(f"args: {args}")
 
-    model_path = "rogerxi/Spatial-LLaVA-7B"
+    model_path = "liuhaotian/llava-v1.5-13b"
     bits = int(os.getenv("bits", 8))
 
     controller_proc = start_controller()

cog.yaml

@@ -1,37 +0,0 @@
-# Configuration for Cog ⚙️
-# Reference: https://github.com/replicate/cog/blob/main/docs/yaml.md
-
-build:
-  gpu: true
-
-  python_version: "3.11"
-
-  python_packages:
-    - "torch==2.0.1"
-    - "accelerate==0.21.0"
-    - "bitsandbytes==0.41.0"
-    - "deepspeed==0.9.5"
-    - "einops-exts==0.0.4"
-    - "einops==0.6.1"
-    - "gradio==3.35.2"
-    - "gradio_client==0.2.9"
-    - "httpx==0.24.0"
-    - "markdown2==2.4.10"
-    - "numpy==1.26.0"
-    - "peft==0.4.0"
-    - "scikit-learn==1.2.2"
-    - "sentencepiece==0.1.99"
-    - "shortuuid==1.0.11"
-    - "timm==0.6.13"
-    - "tokenizers==0.13.3"
-    - "torch==2.0.1"
-    - "torchvision==0.15.2"
-    - "transformers==4.31.0"
-    - "wandb==0.15.12"
-    - "wavedrom==2.0.3.post3"
-    - "Pygments==2.16.1"
-  run:
-    - curl -o /usr/local/bin/pget -L "https://github.com/replicate/pget/releases/download/v0.0.3/pget" && chmod +x /usr/local/bin/pget
-
-# predict.py defines how predictions are run on your model
-predict: "predict.py:Predictor"

data/prompts/complex_reasoning/000_caps.txt

@@ -0,0 +1,18 @@
+A man wearing multiple neck ties making a goofy face.
+A man in a white shirt wearing very many ties.
+a man with ties on poses for a picture
+A man wearing multiple ties on his neck.
+A young man smiles while wearing several ties.
+
+tie: [0.574, 0.298, 0.752, 0.704]
+tie: [0.464, 0.339, 0.639, 0.789]
+tie: [0.349, 0.363, 0.563, 0.732]
+tie: [0.259, 0.255, 0.668, 0.805]
+person: [0.019, 0.065, 0.962, 0.988]
+person: [0.0, 0.24, 0.214, 1.0]
+tie: [0.316, 0.778, 0.443, 0.867]
+tie: [0.386, 0.707, 0.496, 0.801]
+tie: [0.251, 0.354, 0.402, 0.747]
+tie: [0.44, 0.318, 0.63, 0.397]
+tie: [0.566, 0.33, 0.655, 0.543]
+tie: [0.25, 0.409, 0.359, 0.556]

data/prompts/complex_reasoning/000_conv.txt

@@ -0,0 +1,5 @@
+Question:
+What is unusual about this photo?
+===
+Answer:
+In the photo, the man is wearing a total of ten ties around his neck. This is considered unusual because it is not common for someone to wear multiple ties at once. In a typical setting, a person would wear only one tie. The man's goofy facial expression also adds to the unconventional and humorous nature of the image.

data/prompts/complex_reasoning/001_caps.txt

@@ -0,0 +1,18 @@
+A group of people standing outside of a black vehicle with various luggage.
+Luggage surrounds a vehicle in an underground parking area
+People try to fit all of their luggage in an SUV.
+The sport utility vehicle is parked in the public garage, being packed for a trip
+Some people with luggage near a van that is transporting it.
+
+person: [0.681, 0.242, 0.774, 0.694]
+person: [0.63, 0.222, 0.686, 0.516]
+person: [0.444, 0.233, 0.487, 0.34]
+backpack: [0.384, 0.696, 0.485, 0.914]
+backpack: [0.755, 0.413, 0.846, 0.692]
+suitcase: [0.758, 0.413, 0.845, 0.69]
+suitcase: [0.1, 0.497, 0.173, 0.579]
+bicycle: [0.282, 0.363, 0.327, 0.442]
+car: [0.786, 0.25, 0.848, 0.322]
+car: [0.783, 0.27, 0.827, 0.335]
+car: [0.86, 0.254, 0.891, 0.3]
+car: [0.261, 0.101, 0.787, 0.626]

data/prompts/complex_reasoning/001_conv.txt

@@ -0,0 +1,5 @@
+Question:
+What challenges do these people face?
+===
+Answer:
+In the image, a group of people is standing outside a black SUV in a parking area, surrounded by various pieces of luggage, including suitcases and backpacks. They are facing the challenge of fitting all their luggage into the black SUV. There are multiple suitcases and backpacks to be packed, which suggests that the group has a significant amount of belongings to accommodate. They might have to strategize and arrange the luggage efficiently to ensure that everything fits properly into the vehicle. Additionally, they need to consider the comfort of the passengers and visibility while driving, so the placement of the luggage must not obstruct the driver's view or make the passengers uncomfortable during the trip.

data/prompts/complex_reasoning/002_caps.txt

@@ -0,0 +1,7 @@
+There is a movie theater that displays the show times above the doors.
+A red fire hydrant is deep in the snow.
+The fire hydrant is in the snow near a recently plowed sidewalk.
+This city has had a very hard winter with snow.
+A hotel for dogs in the snow in winter.
+
+fire hydrant: [0.326, 0.612, 0.426, 0.72]

data/prompts/complex_reasoning/002_conv.txt

@@ -0,0 +1,5 @@
+Question:
+What challenges might this city face?
+===
+Answer:
+The city faces challenges due to the harsh winter conditions and heavy snowfall. In the image, a red fire hydrant is almost buried deep in the snow, which indicates the significant amount of snow the city has experienced. This can lead to various challenges such as difficulties in transportation, increased risk of accidents, and disruptions to daily life. For example, the recently plowed sidewalk near the fire hydrant shows that the city has to constantly clear snow from roads and sidewalks to maintain access and safety for pedestrians and vehicles. Moreover, emergency services, like firefighters, might face challenges accessing crucial equipment, such as fire hydrants, during emergencies due to the snow accumulation. This highlights the importance of effective snow management strategies and preparedness in such cities to minimize the impact of harsh winter conditions on residents and essential services.

data/prompts/complex_reasoning/system_message.txt

@@ -0,0 +1,10 @@
+You are an AI visual assistant that can analyze a single image. You receive five sentences, each describing the same image you are observing. In addition, specific object locations within the image are given, along with detailed coordinates. These coordinates are in the form of bounding boxes, represented as (x1, y1, x2, y2) with floating numbers ranging from 0 to 1. These values correspond to the top left x, top left y, bottom right x, and bottom right y.
+
+The task is to use the provided caption and bounding box information, create a plausible question about the image, and provide the answer in detail.
+
+Create complex questions beyond describing the scene.
+To answer such questions, one should require first understanding the visual content, then based on the background knowledge or reasoning, either explain why the things are happening that way, or provide guides and help to user's request.  Make the question challenging by not including the visual content details in the question so that the user needs to reason about that first.
+
+Instead of directly mentioning the bounding box coordinates, utilize this data to explain the scene using natural language. Include details like object counts, position of the objects, relative position between the objects.  
+
+When using the information from the caption and coordinates, directly explain the scene, and do not mention that the information source is the caption or the bounding box.  Always answer as if you are directly looking at the image.

data/prompts/conversation/000_caps.txt

@@ -0,0 +1,5 @@
+There is a movie theater that displays the show times above the doors.
+A red fire hydrant is deep in the snow.
+The fire hydrant is in the snow near a recently plowed sidewalk.
+This city has had a very hard winter with snow.
+A hotel for dogs in the snow in winter.

data/prompts/conversation/000_conv.txt

@@ -0,0 +1,29 @@
+Question:
+What color is the fire hydrant in the image?
+===
+Answer:
+The fire hydrant in the image is red.
+===
+Question:
+Is the fire hydrant fully covered by snow or partially visible?
+===
+Answer:
+The fire hydrant is partially visible, with a significant portion of it covered by the deep snow.
+===
+Question:
+What is the condition of the sidewalk near the fire hydrant?
+===
+Answer:
+The sidewalk near the fire hydrant has been recently plowed, so it is relatively clear of snow compared to the surrounding area.
+===
+Question:
+What does the image suggest about the winter season in the city?
+===
+Answer:
+The image suggests that the winter season in the city has been quite harsh and challenging. The significant snow accumulation seen in the image indicates that the city has experienced heavy snowfall, which can lead to various difficulties and disruptions for its residents. 
+
+For instance, the deep snow covering the fire hydrant could potentially hinder its accessibility in case of emergencies, which poses safety concerns. The recently plowed sidewalk implies that the city's maintenance crews have been actively working to keep the walkways clear and safe for pedestrians, but the sheer amount of snowfall might make it difficult to keep up with the continuous snow removal efforts. 
+
+Furthermore, such extreme winter conditions can impact transportation, leading to traffic issues, delays in public transit, and increased risks of accidents due to slippery roads. It can also cause problems for businesses and schools, as people might face challenges commuting to work or attending classes. Additionally, the heavy snow can put extra strain on infrastructure, such as roofs and power lines, increasing the likelihood of structural damage or power outages. 
+
+In conclusion, the image of the red fire hydrant deep in the snow and the recently plowed sidewalk suggest that the city has faced a particularly severe winter season, with substantial snowfall that has likely caused various challenges and disruptions for its residents and infrastructure.

data/prompts/conversation/001_caps.txt

@@ -0,0 +1,5 @@
+A man is skiing in the open snow covered hills
+A skier is making his way into the snow.
+A skier on the foothills of a huge mountain range.
+A skier looks at mountains while standing near a trail sign.
+a single person skiing by an area with a lot of bushes

data/prompts/conversation/001_conv.txt

@@ -0,0 +1,37 @@
+Question:
+What is the skier doing?
+===
+Answer:
+The skier is skiing in the snow-covered hills, making his way through the snow and enjoying the beautiful mountain scenery. They are also standing near a trail sign, which indicates their location on the mountain.
+===
+Question:
+Are there any other people or skiers visible in the image?
+===
+Answer:
+No, there is only a single person skiing in the image, and they are surrounded by an area with a lot of bushes.
+===
+Question:
+What kind of skiing is the person doing? Cross-country or downhill?
+===
+Answer:
+Based on what I can see in the photo, it is not possible to definitively determine whether the skier is doing cross-country or downhill skiing. However, given the presence of a trail sign and the fact that they are skiing in the foothills, it is more likely that the skier is engaged in cross-country skiing. Downhill skiing typically takes place on more steep slopes and involves ski lifts, whereas cross-country skiing often occurs on relatively flatter terrain with marked trails.
+===
+Question:
+Can you explain cross-country and downhill skiing a bit more?
+===
+Answer:
+Cross-country skiing and downhill skiing are two distinct types of skiing, each with its own techniques, equipment, and terrain preferences. 
+
+1. Cross-Country Skiing:
+Also known as Nordic skiing, cross-country skiing involves traversing relatively flat terrain or gentle slopes using skis and poles. It is a popular recreational and competitive sport that focuses on endurance, balance, and aerobic fitness. Cross-country skiers use their own locomotion to move across snow-covered terrain, rather than relying on ski lifts or other forms of assistance. 
+There are two main techniques in cross-country skiing:
+- Classic: This technique involves skiing in parallel tracks, using a forward-and-backward gliding motion while keeping the skis parallel to each other. The skier uses their poles to propel themselves forward.
+- Skate skiing: This technique is similar to ice skating, where the skier pushes off one ski at an angle and glides on the other. It requires a smooth, groomed surface and is typically faster than the classic technique. 
+Cross-country ski equipment consists of lightweight skis, boots, bindings, and poles. The boots are more flexible than those used in downhill skiing, allowing for better ankle movement and control. 
+
+2. Downhill Skiing:
+Also known as alpine skiing, downhill skiing involves descending slopes at high speeds using skis and poles for balance and control. This sport is more focused on speed, technique, and navigating challenging terrain, including steep slopes, moguls, and even jumps. 
+Downhill skiing can be further categorized into several disciplines, such as slalom, giant slalom, super-G, and downhill racing. Each discipline has its own set of rules, courses, and ski equipment. 
+Downhill ski equipment includes heavier and stiffer skis, boots, bindings, and poles than those used in cross-country skiing. The boots are more rigid to provide better support and control during high-speed descents and sharp turns. 
+
+In summary, cross-country skiing is an endurance-based sport that involves traveling across flat or gently sloping terrain, while downhill skiing is focused on speed and technique as skiers navigate steeper slopes and challenging terrain. Both sports require specialized equipment and techniques, but they offer different experiences and challenges to participants.

data/prompts/conversation/system_message.txt

@@ -0,0 +1,12 @@
+You are an AI visual assistant, and you are seeing a single image. What you see are provided with five sentences, describing the same image you are looking at. Answer all questions as you are seeing the image.
+
+Design a conversation between you and a person asking about this photo. The answers should be in a tone that a visual AI assistant is seeing the image and answering the question.
+Ask diverse questions and give corresponding answers.
+
+Include questions asking about the visual content of the image, including the object types, counting the objects, object actions, object locations, relative positions between objects, etc. Only include questions that have definite answers:
+(1) one can see the content in the image that the question asks about and can answer confidently;
+(2) one can determine confidently from the image that it is not in the image.
+Do not ask any question that cannot be answered confidently.
+
+Also include complex questions that are relevant to the content in the image, for example, asking about background knowledge of the objects in the image, asking to discuss about events happening in the image, etc. Again, do not ask about uncertain details.
+Provide detailed answers when answering complex questions. For example, give detailed examples or reasoning steps to make the content more convincing and well-organized.  You can include multiple paragraphs if necessary.

data/prompts/detail_description/000_caps.txt

@@ -0,0 +1,18 @@
+A harbor filled with lots of boats next to a building.
+A bicycle parked in front of several boats at a dock.
+A red bicycle in front of a line of docked white yachts
+A bike sits before boats which sit before a long building.
+A bicycle is a convenient means of land transportation when you live on a boat.
+
+bicycle: [0.287, 0.641, 0.507, 0.874]
+bicycle: [0.566, 0.667, 0.63, 0.731]
+boat: [0.318, 0.579, 0.575, 0.724]
+boat: [0.704, 0.607, 0.818, 0.727]
+boat: [0.818, 0.601, 0.942, 0.744]
+boat: [0.002, 0.53, 0.243, 0.71]
+boat: [0.541, 0.611, 0.668, 0.731]
+person: [0.778, 0.527, 0.797, 0.57]
+cup: [0.708, 0.733, 0.724, 0.758]
+boat: [0.236, 0.532, 0.404, 0.64]
+boat: [0.81, 0.632, 0.836, 0.676]
+boat: [0.957, 0.526, 1.0, 0.752]

data/prompts/detail_description/000_conv.txt

@@ -0,0 +1,3 @@
+It is a harbor filled with numerous boats of various sizes docked next to a long building. Among the boats, there are a few white yachts lined up, standing out from the rest. There is a red bicycle prominently parked in front of the line of docked boats, serving as a convenient means of land transportation for those living on the boats. Another bicycle can be seen further back in the scene, near the middle of the harbor.
+
+A person is visible near the right side of the harbor, possibly enjoying the view or attending to their boat. Additionally, there is a cup placed on a surface near the middle of the scene.

data/prompts/detail_description/001_caps.txt

@@ -0,0 +1,18 @@
+A group of people standing outside of a black vehicle with various luggage.
+Luggage surrounds a vehicle in an underground parking area
+People try to fit all of their luggage in an SUV.
+The sport utility vehicle is parked in the public garage, being packed for a trip
+Some people with luggage near a van that is transporting it.
+
+person: [0.681, 0.242, 0.774, 0.694]
+person: [0.63, 0.222, 0.686, 0.516]
+person: [0.444, 0.233, 0.487, 0.34]
+backpack: [0.384, 0.696, 0.485, 0.914]
+backpack: [0.755, 0.413, 0.846, 0.692]
+suitcase: [0.758, 0.413, 0.845, 0.69]
+suitcase: [0.1, 0.497, 0.173, 0.579]
+bicycle: [0.282, 0.363, 0.327, 0.442]
+car: [0.786, 0.25, 0.848, 0.322]
+car: [0.783, 0.27, 0.827, 0.335]
+car: [0.86, 0.254, 0.891, 0.3]
+car: [0.261, 0.101, 0.787, 0.626]

data/prompts/detail_description/001_conv.txt

@@ -0,0 +1,5 @@
+The image is an underground parking area with a black sport utility vehicle (SUV) parked. There are three people in the scene, with one person standing closer to the left side of the vehicle, another person in the middle, and the third person on the right side. They are all working together to pack their luggage into the SUV for a trip. 
+
+In the parking area, various luggage items are scattered around the vehicle. There are two backpacks, one located near the left rear wheel and the other closer to the right side of the vehicle. Additionally, there are two suitcases, one on the right side of the car and another further away near the center of the parking area. A bicycle can also be seen on the left side of the vehicle. 
+
+Other cars are parked around the main SUV, with one car positioned behind it and slightly to the left, another behind and slightly to the right, and the third car further behind on the right side.

data/prompts/detail_description/002_caps.txt

@@ -0,0 +1,15 @@
+A man holds a Wii-mote above his head while another looks on.
+A guy and his friend are playing Nintendo Wii.
+A young man is holding a video game remote over his head.
+two men standing in a room while one plays with a wii mote
+Some guys standing and playing a video game.
+
+couch: [0.697, 0.759, 0.995, 1.0]
+dining table: [0.426, 0.755, 1.0, 0.987]
+person: [0.082, 0.252, 0.342, 1.0]
+person: [0.399, 0.085, 0.742, 0.982]
+remote: [0.477, 0.135, 0.516, 0.187]
+sink: [0.016, 0.501, 0.063, 0.52]
+potted plant: [0.798, 0.384, 0.888, 0.645]
+refrigerator: [0.305, 0.389, 0.414, 0.547]
+chair: [0.72, 0.509, 0.858, 0.725]

data/prompts/detail_description/002_conv.txt

@@ -0,0 +1,3 @@
+The image shows two men standing in a room, engaged in playing a video game on a Nintendo Wii console. One of the men is holding a Wii remote above his head with enthusiasm, while the other man looks on, likely enjoying the friendly competition. 
+
+The room appears to be a living space with a couch located in the background and a dining table nearby. A potted plant can be seen placed close to the couch, and a chair is situated in the middle of the room. The room also features a kitchen area with a sink and a refrigerator visible in the background.

data/prompts/detail_description/system_message.txt

@@ -0,0 +1,7 @@
+You are an AI visual assistant that can analyze a single image. You receive five sentences, each describing the same image you are observing. In addition, specific object locations within the image are given, along with detailed coordinates. These coordinates are in the form of bounding boxes, represented as (x1, y1, x2, y2) with floating numbers ranging from 0 to 1. These values correspond to the top left x, top left y, bottom right x, and bottom right y.
+
+Using the provided caption and bounding box information, describe the scene in a detailed manner.
+
+Instead of directly mentioning the bounding box coordinates, utilize this data to explain the scene using natural language. Include details like object counts, position of the objects, relative position between the objects.
+
+When using the information from the caption and coordinates, directly explain the scene, and do not mention that the information source is the caption or the bounding box.  Always answer as if you are directly looking at the image.

docs/Customize_Component.md

@@ -0,0 +1,20 @@
+# Customize Components in LLaVA
+
+This is an initial guide on how to replace the LLMs, visual encoders, etc. with your choice of components.
+
+## LLM
+
+It is quite simple to swap out LLaMA to any other LLMs.  You can refer to our implementation of [`llava_llama.py`](https://raw.githubusercontent.com/haotian-liu/LLaVA/main/llava/model/language_model/llava_llama.py) for an example of how to replace the LLM.
+
+Although it may seem that it still needs ~100 lines of code, most of them are copied from the original `llama.py` from HF.  The only part that is different is to insert some lines for processing the multimodal inputs.
+
+In `forward` function, you can see that we call `self.prepare_inputs_labels_for_multimodal` to process the multimodal inputs.  This function is defined in `LlavaMetaForCausalLM` and you just need to insert it into the `forward` function of your LLM.
+
+In `prepare_inputs_for_generation` function, you can see that we add `images` to the `model_inputs`.  This is because we need to pass the images to the LLM during generation.
+
+These are basically all the changes you need to make to replace the LLM.
+
+## Visual Encoder
+
+You can check out [`clip_encoder.py`](https://github.com/haotian-liu/LLaVA/blob/main/llava/model/multimodal_encoder/clip_encoder.py) on how we implement the CLIP visual encoder.
+

docs/Data.md

@@ -0,0 +1,29 @@
+## Data
+
+| Data file name | Size |
+| --- | ---: |
+| [llava_instruct_150k.json](https://huggingface.co/datasets/liuhaotian/LLaVA-Instruct-150K/blob/main/llava_instruct_150k.json) | 229 MB |
+| [llava_instruct_80k.json](https://huggingface.co/datasets/liuhaotian/LLaVA-Instruct-150K/blob/main/llava_instruct_80k.json) | 229 MB |
+| [conversation_58k.json](https://huggingface.co/datasets/liuhaotian/LLaVA-Instruct-150K/blob/main/conversation_58k.json) | 126 MB |
+| [detail_23k.json](https://huggingface.co/datasets/liuhaotian/LLaVA-Instruct-150K/blob/main/detail_23k.json) | 20.5 MB |
+| [complex_reasoning_77k.json](https://huggingface.co/datasets/liuhaotian/LLaVA-Instruct-150K/blob/main/complex_reasoning_77k.json) | 79.6 MB |
+
+### Pretraining Dataset
+The pretraining dataset used in this release is a subset of CC-3M dataset, filtered with a more balanced concept coverage distribution.  Please see [here](https://huggingface.co/datasets/liuhaotian/LLaVA-CC3M-Pretrain-595K) for a detailed description of the dataset structure and how to download the images.
+
+If you already have CC-3M dataset on your disk, the image names follow this format: `GCC_train_000000000.jpg`.  You may edit the `image` field correspondingly if necessary.
+
+| Data | Chat File | Meta Data | Size |
+| --- |  --- |  --- | ---: |
+| CC-3M Concept-balanced 595K | [chat.json](https://huggingface.co/datasets/liuhaotian/LLaVA-CC3M-Pretrain-595K/blob/main/chat.json) | [metadata.json](https://huggingface.co/datasets/liuhaotian/LLaVA-CC3M-Pretrain-595K/blob/main/metadata.json) | 211 MB
+| LAION/CC/SBU BLIP-Caption Concept-balanced 558K | [blip_laion_cc_sbu_558k.json](https://huggingface.co/datasets/liuhaotian/LLaVA-Pretrain/blob/main/blip_laion_cc_sbu_558k.json) | [metadata.json](#) | 181 MB
+
+**Important notice**: Upon the request from the community, as ~15% images of the original CC-3M dataset are no longer accessible, we upload [`images.zip`](https://huggingface.co/datasets/liuhaotian/LLaVA-CC3M-Pretrain-595K/blob/main/images.zip) for better reproducing our work in research community. It must not be used for any other purposes. The use of these images must comply with the CC-3M license. This may be taken down at any time when requested by the original CC-3M dataset owner or owners of the referenced images.
+
+### GPT-4 Prompts
+
+We provide our prompts and few-shot samples for GPT-4 queries, to better facilitate research in this domain.  Please check out the [`prompts`](playground/data/prompts) folder for three kinds of questions: conversation, detail description, and complex reasoning.
+
+They are organized in a format of `system_message.txt` for system message, pairs of `abc_caps.txt` for few-shot sample user input, and `abc_conv.txt` for few-shot sample reference output.
+
+Note that you may find them in different format. For example, `conversation` is in `jsonl`, and detail description is answer-only.  The selected format in our preliminary experiments works slightly better than a limited set of alternatives that we tried: `jsonl`, more natural format, answer-only.  If interested, you may try other variants or conduct more careful study in this.  Contributions are welcomed!

docs/LLaVA_Bench.md

@@ -0,0 +1,31 @@
+# LLaVA-Bench [[Download](https://huggingface.co/datasets/liuhaotian/llava-bench-in-the-wild)]
+
+**-Introduction-**  Large commercial multimodal chatbots have been released in this week, including 
+- [Multimodal Bing-Chat by Microsoft](https://blogs.bing.com/search/july-2023/Bing-Chat-Enterprise-announced,-multimodal-Visual-Search-rolling-out-to-Bing-Chat) (July 18, 2023) 
+- [Multimodal Bard by Google](https://bard.google.com/). 
+
+These chatbots are presumably supported by proprietary large multimodal models (LMM). Compared with the open-source LMM such as LLaVA, proprietary LMM represent the scaling success upperbound of the current SoTA techniques. They share the goal of developing multimodal chatbots that follow human intents to complete various daily-life visual tasks in the wild. While it remains less unexplored how to evaluate multimodal chat ability, it provides useful feedback to study open-source LMMs against the commercial multimodal chatbots. In addition to the *LLaVA-Bench (COCO)* dataset we used to develop the early versions of LLaVA, we are releasing  [*LLaVA-Bench (In-the-Wild)*](https://huggingface.co/datasets/liuhaotian/llava-bench-in-the-wild) to the community for the public use.
+
+## LLaVA-Bench (In-the-Wild *[Ongoing work]*)
+
+To evaluate the model's capability in more challenging tasks and generalizability to novel domains, we collect a diverse set of 24 images with 60 questions in total, including indoor and outdoor scenes, memes, paintings, sketches, etc, and associate each image with a highly-detailed and manually-curated description and a proper selection of questions. Such design also assesses the model's robustness to different prompts. In this release, we also categorize questions into three categories: conversation (simple QA), detailed description, and complex reasoning. We continue to expand and improve the diversity of the LLaVA-Bench (In-the-Wild).  We manually query Bing-Chat and Bard to get the responses. 
+
+### Results
+
+The score is measured by comparing against a reference answer generated by text-only GPT-4. It is generated by feeding the question, along with the ground truth image annotations as the context. A text-only GPT-4 evaluator rates both answers. We query GPT-4 by putting the reference answer first, and then the answer generated by the candidate model. We upload images at their original resolution to Bard and Bing-Chat to obtain the results.
+
+| Approach       | Conversation | Detail | Reasoning | Overall |
+|----------------|--------------|--------|-----------|---------|
+| Bard-0718      | 83.7         | 69.7   | 78.7      | 77.8    |
+| Bing-Chat-0629 | 59.6         | 52.2   | 90.1      | 71.5    |
+| LLaVA-13B-v1-336px-0719 (beam=1) | 64.3         | 55.9   | 81.7      | 70.1    |
+| LLaVA-13B-v1-336px-0719 (beam=5) | 68.4         | 59.9   | 84.3      | 73.5    |
+
+Note that Bard sometimes refuses to answer questions about images containing humans, and Bing-Chat blurs the human faces in the images. We also provide the benchmark score for the subset without humans.
+
+| Approach       | Conversation | Detail | Reasoning | Overall |
+|----------------|--------------|--------|-----------|---------|
+| Bard-0718      | 94.9         | 74.3   | 84.3      | 84.6    |
+| Bing-Chat-0629 | 55.8         | 53.6   | 93.5      | 72.6    |
+| LLaVA-13B-v1-336px-0719 (beam=1) | 62.2         | 56.4   | 82.2      | 70.0    |
+| LLaVA-13B-v1-336px-0719 (beam=5) | 65.6         | 61.7   | 85.0      | 73.6    |

docs/LLaVA_from_LLaMA2.md

@@ -0,0 +1,29 @@
+# LLaVA (based on Llama 2 LLM, Preview)
+
+*NOTE: This is a technical preview. We are still running hyperparameter search, and will release the final model soon.  If you'd like to contribute to this, please contact us.*
+
+:llama: **-Introduction-** [Llama 2 is an open-source LLM released by Meta AI](https://about.fb.com/news/2023/07/llama-2/) today (July 18, 2023). Compared with its early version [Llama 1](https://ai.meta.com/blog/large-language-model-llama-meta-ai/), Llama 2 is more favored in ***stronger language performance***, ***longer context window***, and importantly ***commercially usable***! While Llama 2 is changing the LLM market landscape in the language space, its multimodal ability remains unknown. We quickly develop the LLaVA variant based on the latest Llama 2 checkpoints, and release it to the community for the public use.
+
+You need to apply for and download the lastest Llama 2 checkpoints to start your own training (apply [here](https://ai.meta.com/resources/models-and-libraries/llama-downloads/))
+
+
+## Training
+
+Please checkout [`pretrain.sh`](https://github.com/haotian-liu/LLaVA/blob/main/scripts/pretrain.sh), [`finetune.sh`](https://github.com/haotian-liu/LLaVA/blob/main/scripts/finetune.sh), [`finetune_lora.sh`](https://github.com/haotian-liu/LLaVA/blob/main/scripts/finetune_lora.sh).
+
+## LLaVA (based on Llama 2), What is different? 
+
+:volcano: How is the new LLaVA based on Llama 2 different from Llama 1? The comparisons of the training process are described:
+- **Pre-training**. The pre-trained base LLM is changed from Llama 1 to Llama 2
+- **Language instruction-tuning**. The previous LLaVA model starts with Vicuna, which is instruct tuned on ShareGPT data from Llama 1; The new LLaVA model starts with Llama 2 Chat, which is an instruct tuned checkpoint on dialogue data from Llama 2.
+- **Multimodal instruction-tuning**. The same LLaVA-Lighting process is applied.
+
+
+### Results
+
+- Llama 2 is better at following the instructions of role playing; Llama 2 fails in following the instructions of translation
+- The quantitative evaluation on [LLaVA-Bench](https://github.com/haotian-liu/LLaVA/blob/main/docs/LLaVA_Bench.md) demonstrates on-par performance between Llama 2 and Llama 1 in LLaVA's multimodal chat ability.
+
+
+<img src="../images/llava_example_cmp.png" width="100%">
+

docs/LoRA.md

@@ -0,0 +1,46 @@
+# LLaVA (LoRA, Preview)
+
+NOTE: This is a technical preview, and is not yet ready for production use. We are still running hyperparameter search for the LoRA model, and will release the final model soon.  If you'd like to contribute to this, please contact us.
+
+You need latest code base for LoRA support (instructions [here](https://github.com/haotian-liu/LLaVA#upgrade-to-latest-code-base))
+
+## Demo (Web UI)
+
+Please execute each of the command below one by one (after the previous one has finished).  The commands are the same as launching other demos except for an additional `--model-base` flag to specify the base model to use. Please make sure the base model corresponds to the LoRA checkpoint that you are using.  For this technical preview, you need Vicuna v1.1 (7B) checkpoint (if you do not have that already, follow the instructions [here](https://github.com/lm-sys/FastChat#vicuna-weights)).
+
+#### Launch a controller
+```Shell
+python -m llava.serve.controller --host 0.0.0.0 --port 10000
+```
+
+#### Launch a gradio web server.
+```Shell
+python -m llava.serve.gradio_web_server --controller http://localhost:10000 --model-list-mode reload
+```
+You just launched the Gradio web interface. Now, you can open the web interface with the URL printed on the screen. You may notice that there is no model in the model list. Do not worry, as we have not launched any model worker yet. It will be automatically updated when you launch a model worker.
+
+#### Launch a model worker
+```Shell
+python -m llava.serve.model_worker --host 0.0.0.0 --controller http://localhost:10000 --port 40000 --worker http://localhost:40000 --model-path liuhaotian/llava-vicuna-7b-v1.1-lcs_558k-instruct_80k_3e-lora-preview-alpha --model-base /path/to/vicuna-v1.1
+```
+Wait until the process finishes loading the model and you see "Uvicorn running on ...".  Now, refresh your Gradio web UI, and you will see the model you just launched in the model list.
+
+You can launch as many workers as you want, and compare between different model checkpoints in the same Gradio interface. Please keep the `--controller` the same, and modify the `--port` and `--worker` to a different port number for each worker.
+
+
+## Training
+
+Please see sample training scripts for [LoRA](https://github.com/haotian-liu/LLaVA/blob/main/scripts/finetune_lora.sh) and [QLoRA](https://github.com/haotian-liu/LLaVA/blob/main/scripts/finetune_qlora.sh).
+
+We provide sample DeepSpeed configs, [`zero3.json`](https://github.com/haotian-liu/LLaVA/blob/main/scripts/zero3.json) is more like PyTorch FSDP, and [`zero3_offload.json`](https://github.com/haotian-liu/LLaVA/blob/main/scripts/zero3_offload.json) can further save memory consumption by offloading parameters to CPU. `zero3.json` is usually faster than `zero3_offload.json` but requires more GPU memory, therefore, we recommend trying `zero3.json` first, and if you run out of GPU memory, try `zero3_offload.json`. You can also tweak the `per_device_train_batch_size` and `gradient_accumulation_steps` in the config to save memory, and just to make sure that `per_device_train_batch_size` and `gradient_accumulation_steps` remains the same.
+
+If you are having issues with ZeRO-3 configs, and there are enough VRAM, you may try [`zero2.json`](https://github.com/haotian-liu/LLaVA/blob/main/scripts/zero2.json). This consumes slightly more memory than ZeRO-3, and behaves more similar to PyTorch FSDP, while still supporting parameter-efficient tuning.
+
+## Create Merged Checkpoints
+
+```Shell
+python scripts/merge_lora_weights.py \
+    --model-path /path/to/lora_model \
+    --model-base /path/to/base_model \
+    --save-model-path /path/to/merge_model
+```

docs/MODEL_ZOO.md

@@ -0,0 +1,136 @@
+# Model Zoo
+
+**To Use LLaVA-1.5 checkpoints, your llava package version must be newer than 1.1.0. [Instructions](https://github.com/haotian-liu/LLaVA#upgrade-to-latest-code-base) on how to upgrade.**
+
+If you are interested in including any other details in Model Zoo, please open an issue :)
+
+The model weights below are *merged* weights. You do not need to apply delta. The usage of LLaVA checkpoints should comply with the base LLM's model license: [LLaMA](https://github.com/facebookresearch/llama/blob/main/MODEL_CARD.md).
+
+## LLaVA-v1.5
+
+| Version | Size | Schedule | Checkpoint | VQAv2 | GQA | VizWiz | SQA | T-VQA | POPE | MME | MM-Bench | MM-Bench-CN | SEED | LLaVA-Bench-Wild | MM-Vet |
+|----------|----------|-----------|-----------|---|---|---|---|---|---|---|---|---|---|---|---|
+| LLaVA-1.5 | 7B | full_ft-1e | [liuhaotian/llava-v1.5-7b](https://huggingface.co/liuhaotian/llava-v1.5-7b) | 78.5 | 62.0 | 50.0 | 66.8 | 58.2 | 85.9 | 1510.7 | 64.3 | 58.3 | 58.6 | 63.4 | 30.5 |
+| LLaVA-1.5 | 13B | full_ft-1e | [liuhaotian/llava-v1.5-13b](https://huggingface.co/liuhaotian/llava-v1.5-13b) | 80.0 | 63.3 | 53.6 | 71.6 | 61.3 | 85.9 | 1531.3 | 67.7 | 63.6 | 61.6 | 70.7 | 35.4 |
+| LLaVA-1.5 | 7B | lora-1e | coming soon |
+| LLaVA-1.5 | 13B | lora-1e | coming soon |
+
+<p align="center">
+  <img src="../images/llava_v1_5_radar.jpg" width="500px"> <br>
+  LLaVA-1.5 achieves SoTA performance across 11 benchmarks.
+</p>
+
+
+## LLaVA-v1
+
+*Note: We recommend using the most capable LLaVA-v1.5 series above for the best performance.*
+
+| Base LLM | Vision Encoder | Pretrain Data | Pretraining schedule | Finetuning Data | Finetuning schedule | LLaVA-Bench-Conv | LLaVA-Bench-Detail | LLaVA-Bench-Complex | LLaVA-Bench-Overall | Download |
+|----------|----------------|---------------|----------------------|-----------------|--------------------|------------------|--------------------|---------------------|---------------------|---------------------|
+| Vicuna-13B-v1.3 | CLIP-L-336px | LCS-558K | 1e | LLaVA-Instruct-80K | proj-1e, lora-1e | 64.3 | 55.9 | 81.7 | 70.1 | [LoRA](https://huggingface.co/liuhaotian/llava-v1-0719-336px-lora-vicuna-13b-v1.3) [LoRA-Merged](https://huggingface.co/liuhaotian/llava-v1-0719-336px-lora-merge-vicuna-13b-v1.3) |
+| LLaMA-2-13B-Chat | CLIP-L | LCS-558K | 1e | LLaVA-Instruct-80K | full_ft-1e | 56.7 | 58.6 | 80.0 | 67.9 | [ckpt](https://huggingface.co/liuhaotian/llava-llama-2-13b-chat-lightning-preview) |
+| LLaMA-2-7B-Chat | CLIP-L | LCS-558K | 1e | LLaVA-Instruct-80K | lora-1e | 51.2 | 58.9 | 71.6 | 62.8 | [LoRA](https://huggingface.co/liuhaotian/llava-llama-2-7b-chat-lightning-lora-preview) |
+
+
+## Projector weights
+
+The model weights below are projector weights we have pretrained. You can use these projector weights for visual instruction tuning. We'll add more projector weights into model zoo very soon.
+
+**NOTE**: These projector weights are only compatible with the `llava>=1.0.0`, please check out the latest code base if your local code version is below `v1.0.0`.
+
+**NOTE**: When you use our pretrained projector for visual instruction tuning, it is very important to **use the same base LLM and vision encoder** as the one we used for pretraining the projector. Otherwise, the performance will be very bad.
+
+When using these projector weights to instruction tune your LMM, please make sure that these options are correctly set as follows,
+
+```Shell
+--mm_use_im_start_end False
+--mm_use_im_patch_token False
+```
+
+| Base LLM | Vision Encoder | Projection | Pretrain Data | Pretraining schedule | Download |
+|----------|----------------|---------------|----------------------|----------|----------|
+| Vicuna-13B-v1.5 | CLIP-L-336px | MLP-2x | LCS-558K | 1e | [projector](https://huggingface.co/liuhaotian/llava-v1.5-mlp2x-336px-pretrain-vicuna-13b-v1.5) |
+| Vicuna-7B-v1.5 | CLIP-L-336px | MLP-2x | LCS-558K | 1e | [projector](https://huggingface.co/liuhaotian/llava-v1.5-mlp2x-336px-pretrain-vicuna-7b-v1.5) |
+| LLaMA-2-13B-Chat | CLIP-L-336px | Linear | LCS-558K | 1e | [projector](https://huggingface.co/liuhaotian/llava-336px-pretrain-llama-2-13b-chat) |
+| LLaMA-2-7B-Chat | CLIP-L-336px | Linear | LCS-558K | 1e | [projector](https://huggingface.co/liuhaotian/llava-336px-pretrain-llama-2-7b-chat) |
+| LLaMA-2-13B-Chat | CLIP-L | Linear | LCS-558K | 1e | [projector](https://huggingface.co/liuhaotian/llava-pretrain-llama-2-13b-chat) |
+| LLaMA-2-7B-Chat | CLIP-L | Linear | LCS-558K | 1e | [projector](https://huggingface.co/liuhaotian/llava-pretrain-llama-2-7b-chat) |
+| Vicuna-13B-v1.3 | CLIP-L-336px | Linear | LCS-558K | 1e | [projector](https://huggingface.co/liuhaotian/llava-336px-pretrain-vicuna-13b-v1.3) |
+| Vicuna-7B-v1.3 | CLIP-L-336px | Linear | LCS-558K | 1e | [projector](https://huggingface.co/liuhaotian/llava-336px-pretrain-vicuna-7b-v1.3) |
+| Vicuna-13B-v1.3 | CLIP-L | Linear | LCS-558K | 1e | [projector](https://huggingface.co/liuhaotian/llava-pretrain-vicuna-13b-v1.3) |
+| Vicuna-7B-v1.3 | CLIP-L | Linear | LCS-558K | 1e | [projector](https://huggingface.co/liuhaotian/llava-pretrain-vicuna-7b-v1.3) |
+
+
+## Science QA Checkpoints
+
+| Base LLM | Vision Encoder | Pretrain Data | Pretraining schedule | Finetuning Data | Finetuning schedule | Download |
+|----------|----------------|---------------|----------------------|-----------------|--------------------|---------------------|
+| Vicuna-13B-v1.3 | CLIP-L | LCS-558K | 1e | ScienceQA | full_ft-12e | [ckpt](https://huggingface.co/liuhaotian/llava-lcs558k-scienceqa-vicuna-13b-v1.3) |
+
+
+## Legacy Models (merged weights)
+
+The model weights below are *merged* weights. You do not need to apply delta. The usage of LLaVA checkpoints should comply with the base LLM's model license.
+
+| Base LLM | Vision Encoder | Pretrain Data | Pretraining schedule | Finetuning Data | Finetuning schedule | Download |
+|----------|----------------|---------------|----------------------|-----------------|--------------------|------------------|
+| MPT-7B-Chat | CLIP-L | LCS-558K | 1e | LLaVA-Instruct-80K | full_ft-1e | [preview](https://huggingface.co/liuhaotian/LLaVA-Lightning-MPT-7B-preview) |
+
+
+## Legacy Models (delta weights)
+
+The model weights below are *delta* weights. The usage of LLaVA checkpoints should comply with the base LLM's model license: [LLaMA](https://github.com/facebookresearch/llama/blob/main/MODEL_CARD.md).
+
+You can add our delta to the original LLaMA weights to obtain the LLaVA weights.
+
+Instructions:
+
+1. Get the original LLaMA weights in the huggingface format by following the instructions [here](https://huggingface.co/docs/transformers/main/model_doc/llama).
+2. Use the following scripts to get LLaVA weights by applying our delta. It will automatically download delta weights from our Hugging Face account. In the script below, we use the delta weights of [`liuhaotian/LLaVA-7b-delta-v0`](https://huggingface.co/liuhaotian/LLaVA-7b-delta-v0) as an example. It can be adapted for other delta weights by changing the `--delta` argument (and base/target accordingly).
+
+```bash
+python3 -m llava.model.apply_delta \
+    --base /path/to/llama-7b \
+    --target /output/path/to/LLaVA-7B-v0 \
+    --delta liuhaotian/LLaVA-7b-delta-v0
+```
+
+| Base LLM | Vision Encoder | Pretrain Data | Pretraining schedule | Finetuning Data | Finetuning schedule | Download |
+|----------|----------------|---------------|----------------------|-----------------|--------------------|------------------|
+| Vicuna-13B-v1.1 | CLIP-L | CC-595K | 1e | LLaVA-Instruct-158K | full_ft-3e | [delta-weights](https://huggingface.co/liuhaotian/LLaVA-13b-delta-v1-1) |
+| Vicuna-7B-v1.1 | CLIP-L | LCS-558K | 1e | LLaVA-Instruct-80K | full_ft-1e | [delta-weights](https://huggingface.co/liuhaotian/LLaVA-Lightning-7B-delta-v1-1) |
+| Vicuna-13B-v0 | CLIP-L | CC-595K | 1e | LLaVA-Instruct-158K | full_ft-3e | [delta-weights](https://huggingface.co/liuhaotian/LLaVA-13b-delta-v0) |
+| Vicuna-13B-v0 | CLIP-L | CC-595K | 1e | ScienceQA | full_ft-12e | [delta-weights](https://huggingface.co/liuhaotian/LLaVA-13b-delta-v0-science_qa) |
+| Vicuna-7B-v0 | CLIP-L | CC-595K | 1e | LLaVA-Instruct-158K | full_ft-3e | [delta-weights](https://huggingface.co/liuhaotian/LLaVA-7b-delta-v0) |
+
+
+
+## Legacy Projector weights
+
+The following projector weights are deprecated, and the support for them may be removed in the future. They do not support zero-shot inference. Please use the projector weights in the [table above](#projector-weights) if possible.
+
+**NOTE**: When you use our pretrained projector for visual instruction tuning, it is very important to **use the same base LLM and vision encoder** as the one we used for pretraining the projector. Otherwise, the performance will be very bad.
+
+When using these projector weights to instruction tune your LMM, please make sure that these options are correctly set as follows,
+
+```Shell
+--mm_use_im_start_end True
+--mm_use_im_patch_token False
+```
+
+| Base LLM | Vision Encoder | Pretrain Data | Pretraining schedule | Download |
+|----------|----------------|---------------|----------------------|----------|
+| Vicuna-7B-v1.1 | CLIP-L | LCS-558K | 1e | [projector](https://huggingface.co/liuhaotian/LLaVA-Pretrained-Projectors/blob/main/LLaVA-7b-pretrain-projector-v1-1-LCS-558K-blip_caption.bin) |
+| Vicuna-13B-v0 | CLIP-L | CC-595K | 1e | [projector](https://huggingface.co/liuhaotian/LLaVA-Pretrained-Projectors/blob/main/LLaVA-13b-pretrain-projector-v0-CC3M-595K-original_caption.bin) |
+| Vicuna-7B-v0 | CLIP-L | CC-595K | 1e | [projector](https://huggingface.co/liuhaotian/LLaVA-Pretrained-Projectors/blob/main/LLaVA-7b-pretrain-projector-v0-CC3M-595K-original_caption.bin) |
+
+When using these projector weights to instruction tune your LMM, please make sure that these options are correctly set as follows,
+
+```Shell
+--mm_use_im_start_end False
+--mm_use_im_patch_token False
+```
+
+| Base LLM | Vision Encoder | Pretrain Data | Pretraining schedule | Download |
+|----------|----------------|---------------|----------------------|----------|
+| Vicuna-13B-v0 | CLIP-L | CC-595K | 1e | [projector](https://huggingface.co/liuhaotian/LLaVA-Pretrained-Projectors/blob/main/LLaVA-13b-pretrain-projector-v0-CC3M-595K-original_caption-no_im_token.bin) |

docs/ScienceQA.md

@@ -0,0 +1,53 @@
+### ScienceQA
+
+#### Prepare Data
+1. Please see ScienceQA [repo](https://github.com/lupantech/ScienceQA) for setting up the dataset.
+2. Generate ScienceQA dataset for LLaVA conversation-style format.
+
+```Shell
+python scripts/convert_sqa_to_llava.py \
+    convert_to_llava \
+    --base-dir /path/to/ScienceQA/data/scienceqa \
+    --prompt-format "QCM-LEA" \
+    --split {train,val,minival,test,minitest}
+```
+
+#### Training
+
+1. Pretraining
+
+You can download our pretrained projector weights from our [Model Zoo](), or train your own projector weights using [`pretrain.sh`](https://github.com/haotian-liu/LLaVA/blob/main/scripts/pretrain.sh).
+
+2. Finetuning
+
+See [`finetune_sqa.sh`](https://github.com/haotian-liu/LLaVA/blob/main/scripts/finetune_sqa.sh).
+
+#### Evaluation
+
+1. Multiple-GPU inference
+You may evaluate this with multiple GPUs, and concatenate the generated jsonl files.  Please refer to our script for [batch evaluation](https://github.com/haotian-liu/LLaVA/blob/main/scripts/sqa_eval_batch.sh) and [results gathering](https://github.com/haotian-liu/LLaVA/blob/main/scripts/sqa_eval_gather.sh).
+
+2. Single-GPU inference
+
+(a) Generate LLaVA responses on ScienceQA dataset
+
+```Shell
+python -m llava.eval.model_vqa_science \
+    --model-path liuhaotian/llava-lcs558k-scienceqa-vicuna-13b-v1.3 \
+    --question-file /path/to/ScienceQA/data/scienceqa/llava_test_QCM-LEA.json \
+    --image-folder /path/to/ScienceQA/data/scienceqa/images/test \
+    --answers-file vqa/results/ScienceQA/test_llava-13b.jsonl \
+    --conv-mode llava_v1
+```
+
+(b) Evaluate the generated responses
+
+```Shell
+python eval_science_qa.py \
+    --base-dir /path/to/ScienceQA/data/scienceqa \
+    --result-file vqa/results/ScienceQA/test_llava-13b.jsonl \
+    --output-file vqa/results/ScienceQA/test_llava-13b_output.json \
+    --output-result vqa/results/ScienceQA/test_llava-13b_result.json \
+```
+
+For reference, we attach our prediction file [`test_sqa_llava_lcs_558k_sqa_12e_vicuna_v1_3_13b.json`](https://github.com/haotian-liu/LLaVA/blob/main/llava/eval/table/results/test_sqa_llava_lcs_558k_sqa_12e_vicuna_v1_3_13b.json) and [`test_sqa_llava_13b_v0.json`](https://github.com/haotian-liu/LLaVA/blob/main/llava/eval/table/results/test_sqa_llava_13b_v0.json) for comparison when reproducing our results, as well as for further analysis in detail.

llava/__init__.py

@@ -1 +1 @@
-from llava.model.language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+from .model import LlavaLlamaForCausalLM

llava/constants.py

@@ -10,4 +10,3 @@ DEFAULT_IMAGE_TOKEN = "<image>"
 DEFAULT_IMAGE_PATCH_TOKEN = "<im_patch>"
 DEFAULT_IM_START_TOKEN = "<im_start>"
 DEFAULT_IM_END_TOKEN = "<im_end>"
-IMAGE_PLACEHOLDER = "<image-placeholder>"

llava/conversation.py

@@ -1,9 +1,6 @@
 import dataclasses
 from enum import auto, Enum
 from typing import List, Tuple
-import base64
-from io import BytesIO
-from PIL import Image
 
 
 class SeparatorStyle(Enum):
@@ -71,7 +68,7 @@ class Conversation:
                 else:
                     ret += role
         elif self.sep_style == SeparatorStyle.LLAMA_2:
-            wrap_sys = lambda msg: f"<<SYS>>\n{msg}\n<</SYS>>\n\n" if len(msg) > 0 else msg
+            wrap_sys = lambda msg: f"<<SYS>>\n{msg}\n<</SYS>>\n\n"
             wrap_inst = lambda msg: f"[INST] {msg} [/INST]"
             ret = ""
 
@@ -109,54 +106,54 @@ class Conversation:
     def append_message(self, role, message):
         self.messages.append([role, message])
 
-    def process_image(self, image, image_process_mode, return_pil=False, image_format='PNG', max_len=1344, min_len=672):
-        if image_process_mode == "Pad":
-            def expand2square(pil_img, background_color=(122, 116, 104)):
-                width, height = pil_img.size
-                if width == height:
-                    return pil_img
-                elif width > height:
-                    result = Image.new(pil_img.mode, (width, width), background_color)
-                    result.paste(pil_img, (0, (width - height) // 2))
-                    return result
-                else:
-                    result = Image.new(pil_img.mode, (height, height), background_color)
-                    result.paste(pil_img, ((height - width) // 2, 0))
-                    return result
-            image = expand2square(image)
-        elif image_process_mode in ["Default", "Crop"]:
-            pass
-        elif image_process_mode == "Resize":
-            image = image.resize((336, 336))
-        else:
-            raise ValueError(f"Invalid image_process_mode: {image_process_mode}")
-        if max(image.size) > max_len:
-            max_hw, min_hw = max(image.size), min(image.size)
-            aspect_ratio = max_hw / min_hw
-            shortest_edge = int(min(max_len / aspect_ratio, min_len, min_hw))
-            longest_edge = int(shortest_edge * aspect_ratio)
-            W, H = image.size
-            if H > W:
-                H, W = longest_edge, shortest_edge
-            else:
-                H, W = shortest_edge, longest_edge
-            image = image.resize((W, H))
-        if return_pil:
-            return image
-        else:
-            buffered = BytesIO()
-            image.save(buffered, format=image_format)
-            img_b64_str = base64.b64encode(buffered.getvalue()).decode()
-            return img_b64_str
-
     def get_images(self, return_pil=False):
         images = []
         for i, (role, msg) in enumerate(self.messages[self.offset:]):
             if i % 2 == 0:
                 if type(msg) is tuple:
+                    import base64
+                    from io import BytesIO
+                    from PIL import Image
                     msg, image, image_process_mode = msg
-                    image = self.process_image(image, image_process_mode, return_pil=return_pil)
-                    images.append(image)
+                    if image_process_mode == "Pad":
+                        def expand2square(pil_img, background_color=(122, 116, 104)):
+                            width, height = pil_img.size
+                            if width == height:
+                                return pil_img
+                            elif width > height:
+                                result = Image.new(pil_img.mode, (width, width), background_color)
+                                result.paste(pil_img, (0, (width - height) // 2))
+                                return result
+                            else:
+                                result = Image.new(pil_img.mode, (height, height), background_color)
+                                result.paste(pil_img, ((height - width) // 2, 0))
+                                return result
+                        image = expand2square(image)
+                    elif image_process_mode in ["Default", "Crop"]:
+                        pass
+                    elif image_process_mode == "Resize":
+                        image = image.resize((336, 336))
+                    else:
+                        raise ValueError(f"Invalid image_process_mode: {image_process_mode}")
+                    max_hw, min_hw = max(image.size), min(image.size)
+                    aspect_ratio = max_hw / min_hw
+                    max_len, min_len = 800, 400
+                    shortest_edge = int(min(max_len / aspect_ratio, min_len, min_hw))
+                    longest_edge = int(shortest_edge * aspect_ratio)
+                    W, H = image.size
+                    if longest_edge != max(image.size):
+                        if H > W:
+                            H, W = longest_edge, shortest_edge
+                        else:
+                            H, W = shortest_edge, longest_edge
+                        image = image.resize((W, H))
+                    if return_pil:
+                        images.append(image)
+                    else:
+                        buffered = BytesIO()
+                        image.save(buffered, format="PNG")
+                        img_b64_str = base64.b64encode(buffered.getvalue()).decode()
+                        images.append(img_b64_str)
         return images
 
     def to_gradio_chatbot(self):
@@ -164,11 +161,24 @@ class Conversation:
         for i, (role, msg) in enumerate(self.messages[self.offset:]):
             if i % 2 == 0:
                 if type(msg) is tuple:
+                    import base64
+                    from io import BytesIO
                     msg, image, image_process_mode = msg
-                    img_b64_str = self.process_image(
-                        image, "Default", return_pil=False,
-                        image_format='JPEG')
-                    img_str = f'<img src="data:image/jpeg;base64,{img_b64_str}" alt="user upload image" />'
+                    max_hw, min_hw = max(image.size), min(image.size)
+                    aspect_ratio = max_hw / min_hw
+                    max_len, min_len = 800, 400
+                    shortest_edge = int(min(max_len / aspect_ratio, min_len, min_hw))
+                    longest_edge = int(shortest_edge * aspect_ratio)
+                    W, H = image.size
+                    if H > W:
+                        H, W = longest_edge, shortest_edge
+                    else:
+                        H, W = shortest_edge, longest_edge
+                    image = image.resize((W, H))
+                    buffered = BytesIO()
+                    image.save(buffered, format="JPEG")
+                    img_b64_str = base64.b64encode(buffered.getvalue()).decode()
+                    img_str = f'<img src="data:image/png;base64,{img_b64_str}" alt="user upload image" />'
                     msg = img_str + msg.replace('<image>', '').strip()
                     ret.append([msg, None])
                 else:
@@ -347,38 +357,13 @@ conv_llava_v1_mmtag = Conversation(
     version="v1_mmtag",
 )
 
-conv_mistral_instruct = Conversation(
-    system="",
-    roles=("USER", "ASSISTANT"),
-    version="llama_v2",
-    messages=(),
-    offset=0,
-    sep_style=SeparatorStyle.LLAMA_2,
-    sep="",
-    sep2="</s>",
-)
-
-conv_chatml_direct = Conversation(
-    system="""<|im_start|>system
-Answer the questions.""",
-    roles=("<|im_start|>user\n", "<|im_start|>assistant\n"),
-    version="mpt",
-    messages=(),
-    offset=0,
-    sep_style=SeparatorStyle.MPT,
-    sep="<|im_end|>",
-)
-
-default_conversation = conv_vicuna_v1
+default_conversation = conv_vicuna_v0
 conv_templates = {
     "default": conv_vicuna_v0,
     "v0": conv_vicuna_v0,
     "v1": conv_vicuna_v1,
     "vicuna_v1": conv_vicuna_v1,
     "llama_2": conv_llama_2,
-    "mistral_instruct": conv_mistral_instruct,
-    "chatml_direct": conv_chatml_direct,
-    "mistral_direct": conv_chatml_direct,
 
     "plain": conv_llava_plain,
     "v0_plain": conv_llava_plain,

llava/eval/eval_pope.py

@@ -1,81 +0,0 @@
-import os
-import json
-import argparse
-
-def eval_pope(answers, label_file):
-    label_list = [json.loads(q)['label'] for q in open(label_file, 'r')]
-
-    for answer in answers:
-        text = answer['text']
-
-        # Only keep the first sentence
-        if text.find('.') != -1:
-            text = text.split('.')[0]
-
-        text = text.replace(',', '')
-        words = text.split(' ')
-        if 'No' in words or 'not' in words or 'no' in words:
-            answer['text'] = 'no'
-        else:
-            answer['text'] = 'yes'
-
-    for i in range(len(label_list)):
-        if label_list[i] == 'no':
-            label_list[i] = 0
-        else:
-            label_list[i] = 1
-
-    pred_list = []
-    for answer in answers:
-        if answer['text'] == 'no':
-            pred_list.append(0)
-        else:
-            pred_list.append(1)
-
-    pos = 1
-    neg = 0
-    yes_ratio = pred_list.count(1) / len(pred_list)
-
-    TP, TN, FP, FN = 0, 0, 0, 0
-    for pred, label in zip(pred_list, label_list):
-        if pred == pos and label == pos:
-            TP += 1
-        elif pred == pos and label == neg:
-            FP += 1
-        elif pred == neg and label == neg:
-            TN += 1
-        elif pred == neg and label == pos:
-            FN += 1
-
-    print('TP\tFP\tTN\tFN\t')
-    print('{}\t{}\t{}\t{}'.format(TP, FP, TN, FN))
-
-    precision = float(TP) / float(TP + FP)
-    recall = float(TP) / float(TP + FN)
-    f1 = 2*precision*recall / (precision + recall)
-    acc = (TP + TN) / (TP + TN + FP + FN)
-    print('Accuracy: {}'.format(acc))
-    print('Precision: {}'.format(precision))
-    print('Recall: {}'.format(recall))
-    print('F1 score: {}'.format(f1))
-    print('Yes ratio: {}'.format(yes_ratio))
-    print('%.3f, %.3f, %.3f, %.3f, %.3f' % (f1, acc, precision, recall, yes_ratio) )
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--annotation-dir", type=str)
-    parser.add_argument("--question-file", type=str)
-    parser.add_argument("--result-file", type=str)
-    args = parser.parse_args()
-
-    questions = [json.loads(line) for line in open(args.question_file)]
-    questions = {question['question_id']: question for question in questions}
-    answers = [json.loads(q) for q in open(args.result_file)]
-    for file in os.listdir(args.annotation_dir):
-        assert file.startswith('coco_pope_')
-        assert file.endswith('.json')
-        category = file[10:-5]
-        cur_answers = [x for x in answers if questions[x['question_id']]['category'] == category]
-        print('Category: {}, # samples: {}'.format(category, len(cur_answers)))
-        eval_pope(cur_answers, os.path.join(args.annotation_dir, file))
-        print("====================================")

llava/eval/eval_science_qa.py

@@ -32,7 +32,6 @@ def get_pred_idx(prediction, choices, options):
     if prediction in options[:len(choices)]:
         return options.index(prediction)
     else:
-        return -1
         return random.choice(range(len(choices)))
 
 
@@ -56,23 +55,16 @@ if __name__ == "__main__":
 
     for prob_id, prob in split_problems.items():
         if prob_id not in predictions:
-            pred = {'text': 'FAILED', 'prompt': 'Unknown'}
-            pred_text = 'FAILED'
+            continue
+        pred = predictions[prob_id]
+        pred_text = pred['text']
+
+        pattern = re.compile(r'The answer is ([A-Z]).')
+        res = pattern.findall(pred_text)
+        if len(res) == 1:
+            answer = res[0]  # 'A', 'B', ...
         else:
-            pred = predictions[prob_id]
-            pred_text = pred['text']
-
-        if pred_text in args.options:
-            answer = pred_text
-        elif len(pred_text) >= 3 and pred_text[0] in args.options and pred_text[1:3] == ". ":
-            answer = pred_text[0]
-        else:
-            pattern = re.compile(r'The answer is ([A-Z]).')
-            res = pattern.findall(pred_text)
-            if len(res) == 1:
-                answer = res[0]  # 'A', 'B', ...
-            else:
-                answer = "FAILED"
+            answer = "FAILED"
 
         pred_idx = get_pred_idx(answer, prob['choices'], args.options)
 
@@ -95,14 +87,7 @@ if __name__ == "__main__":
 
     correct = len(results['correct'])
     total = len(results['correct']) + len(results['incorrect'])
-
-    ###### IMG ######
-    multimodal_correct = len([x for x in results['correct'] if x['is_multimodal']])
-    multimodal_incorrect = len([x for x in results['incorrect'] if x['is_multimodal']])
-    multimodal_total = multimodal_correct + multimodal_incorrect
-    ###### IMG ######
-
-    print(f'Total: {total}, Correct: {correct}, Accuracy: {correct / total * 100:.2f}%, IMG-Accuracy: {multimodal_correct / multimodal_total * 100:.2f}%')
+    print(f'Total: {total}, Correct: {correct}, Accuracy: {correct / total * 100:.2f}%')
 
     sqa_results['acc'] = correct / total * 100
     sqa_results['correct'] = correct

llava/eval/eval_textvqa.py

@@ -1,65 +0,0 @@
-import os
-import argparse
-import json
-import re
-
-from llava.eval.m4c_evaluator import TextVQAAccuracyEvaluator
-
-
-def get_args():
-    parser = argparse.ArgumentParser()
-    parser.add_argument('--annotation-file', type=str)
-    parser.add_argument('--result-file', type=str)
-    parser.add_argument('--result-dir', type=str)
-    return parser.parse_args()
-
-
-def prompt_processor(prompt):
-    if prompt.startswith('OCR tokens: '):
-        pattern = r"Question: (.*?) Short answer:"
-        match = re.search(pattern, prompt, re.DOTALL)
-        question = match.group(1)
-    elif 'Reference OCR token: ' in prompt and len(prompt.split('\n')) == 3:
-        if prompt.startswith('Reference OCR token:'):
-            question = prompt.split('\n')[1]
-        else:
-            question = prompt.split('\n')[0]
-    elif len(prompt.split('\n')) == 2:
-        question = prompt.split('\n')[0]
-    else:
-        assert False
-
-    return question.lower()
-
-
-def eval_single(annotation_file, result_file):
-    experiment_name = os.path.splitext(os.path.basename(result_file))[0]
-    print(experiment_name)
-    annotations = json.load(open(annotation_file))['data']
-    annotations = {(annotation['image_id'], annotation['question'].lower()): annotation for annotation in annotations}
-    results = [json.loads(line) for line in open(result_file)]
-
-    pred_list = []
-    for result in results:
-        annotation = annotations[(result['question_id'], prompt_processor(result['prompt']))]
-        pred_list.append({
-            "pred_answer": result['text'],
-            "gt_answers": annotation['answers'],
-        })
-
-    evaluator = TextVQAAccuracyEvaluator()
-    print('Samples: {}\nAccuracy: {:.2f}%\n'.format(len(pred_list), 100. * evaluator.eval_pred_list(pred_list)))
-
-
-if __name__ == "__main__":
-    args = get_args()
-
-    if args.result_file is not None:
-        eval_single(args.annotation_file, args.result_file)
-
-    if args.result_dir is not None:
-        for result_file in sorted(os.listdir(args.result_dir)):
-            if not result_file.endswith('.jsonl'):
-                print(f'Skipping {result_file}')
-                continue
-            eval_single(args.annotation_file, os.path.join(args.result_dir, result_file))

llava/eval/m4c_evaluator.py

@@ -1,334 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates.
-import re
-
-from tqdm import tqdm
-
-
-class EvalAIAnswerProcessor:
-    """
-    Processes an answer similar to Eval AI
-        copied from
-        https://github.com/facebookresearch/mmf/blob/c46b3b3391275b4181567db80943473a89ab98ab/pythia/tasks/processors.py#L897
-    """
-
-    CONTRACTIONS = {
-        "aint": "ain't",
-        "arent": "aren't",
-        "cant": "can't",
-        "couldve": "could've",
-        "couldnt": "couldn't",
-        "couldn'tve": "couldn't've",
-        "couldnt've": "couldn't've",
-        "didnt": "didn't",
-        "doesnt": "doesn't",
-        "dont": "don't",
-        "hadnt": "hadn't",
-        "hadnt've": "hadn't've",
-        "hadn'tve": "hadn't've",
-        "hasnt": "hasn't",
-        "havent": "haven't",
-        "hed": "he'd",
-        "hed've": "he'd've",
-        "he'dve": "he'd've",
-        "hes": "he's",
-        "howd": "how'd",
-        "howll": "how'll",
-        "hows": "how's",
-        "Id've": "I'd've",
-        "I'dve": "I'd've",
-        "Im": "I'm",
-        "Ive": "I've",
-        "isnt": "isn't",
-        "itd": "it'd",
-        "itd've": "it'd've",
-        "it'dve": "it'd've",
-        "itll": "it'll",
-        "let's": "let's",
-        "maam": "ma'am",
-        "mightnt": "mightn't",
-        "mightnt've": "mightn't've",
-        "mightn'tve": "mightn't've",
-        "mightve": "might've",
-        "mustnt": "mustn't",
-        "mustve": "must've",
-        "neednt": "needn't",
-        "notve": "not've",
-        "oclock": "o'clock",
-        "oughtnt": "oughtn't",
-        "ow's'at": "'ow's'at",
-        "'ows'at": "'ow's'at",
-        "'ow'sat": "'ow's'at",
-        "shant": "shan't",
-        "shed've": "she'd've",
-        "she'dve": "she'd've",
-        "she's": "she's",
-        "shouldve": "should've",
-        "shouldnt": "shouldn't",
-        "shouldnt've": "shouldn't've",
-        "shouldn'tve": "shouldn't've",
-        "somebody'd": "somebodyd",
-        "somebodyd've": "somebody'd've",
-        "somebody'dve": "somebody'd've",
-        "somebodyll": "somebody'll",
-        "somebodys": "somebody's",
-        "someoned": "someone'd",
-        "someoned've": "someone'd've",
-        "someone'dve": "someone'd've",
-        "someonell": "someone'll",
-        "someones": "someone's",
-        "somethingd": "something'd",
-        "somethingd've": "something'd've",
-        "something'dve": "something'd've",
-        "somethingll": "something'll",
-        "thats": "that's",
-        "thered": "there'd",
-        "thered've": "there'd've",
-        "there'dve": "there'd've",
-        "therere": "there're",
-        "theres": "there's",
-        "theyd": "they'd",
-        "theyd've": "they'd've",
-        "they'dve": "they'd've",
-        "theyll": "they'll",
-        "theyre": "they're",
-        "theyve": "they've",
-        "twas": "'twas",
-        "wasnt": "wasn't",
-        "wed've": "we'd've",
-        "we'dve": "we'd've",
-        "weve": "we've",
-        "werent": "weren't",
-        "whatll": "what'll",
-        "whatre": "what're",
-        "whats": "what's",
-        "whatve": "what've",
-        "whens": "when's",
-        "whered": "where'd",
-        "wheres": "where's",
-        "whereve": "where've",
-        "whod": "who'd",
-        "whod've": "who'd've",
-        "who'dve": "who'd've",
-        "wholl": "who'll",
-        "whos": "who's",
-        "whove": "who've",
-        "whyll": "why'll",
-        "whyre": "why're",
-        "whys": "why's",
-        "wont": "won't",
-        "wouldve": "would've",
-        "wouldnt": "wouldn't",
-        "wouldnt've": "wouldn't've",
-        "wouldn'tve": "wouldn't've",
-        "yall": "y'all",
-        "yall'll": "y'all'll",
-        "y'allll": "y'all'll",
-        "yall'd've": "y'all'd've",
-        "y'alld've": "y'all'd've",
-        "y'all'dve": "y'all'd've",
-        "youd": "you'd",
-        "youd've": "you'd've",
-        "you'dve": "you'd've",
-        "youll": "you'll",
-        "youre": "you're",
-        "youve": "you've",
-    }
-
-    NUMBER_MAP = {
-        "none": "0",
-        "zero": "0",
-        "one": "1",
-        "two": "2",
-        "three": "3",
-        "four": "4",
-        "five": "5",
-        "six": "6",
-        "seven": "7",
-        "eight": "8",
-        "nine": "9",
-        "ten": "10",
-    }
-    ARTICLES = ["a", "an", "the"]
-    PERIOD_STRIP = re.compile(r"(?!<=\d)(\.)(?!\d)")
-    COMMA_STRIP = re.compile(r"(?<=\d)(\,)+(?=\d)")
-    PUNCTUATIONS = [
-        ";",
-        r"/",
-        "[",
-        "]",
-        '"',
-        "{",
-        "}",
-        "(",
-        ")",
-        "=",
-        "+",
-        "\\",
-        "_",
-        "-",
-        ">",
-        "<",
-        "@",
-        "`",
-        ",",
-        "?",
-        "!",
-    ]
-
-    def __init__(self, *args, **kwargs):
-        pass
-
-    def word_tokenize(self, word):
-        word = word.lower()
-        word = word.replace(",", "").replace("?", "").replace("'s", " 's")
-        return word.strip()
-
-    def process_punctuation(self, in_text):
-        out_text = in_text
-        for p in self.PUNCTUATIONS:
-            if (p + " " in in_text or " " + p in in_text) or (
-                re.search(self.COMMA_STRIP, in_text) is not None
-            ):
-                out_text = out_text.replace(p, "")
-            else:
-                out_text = out_text.replace(p, " ")
-        out_text = self.PERIOD_STRIP.sub("", out_text, re.UNICODE)
-        return out_text
-
-    def process_digit_article(self, in_text):
-        out_text = []
-        temp_text = in_text.lower().split()
-        for word in temp_text:
-            word = self.NUMBER_MAP.setdefault(word, word)
-            if word not in self.ARTICLES:
-                out_text.append(word)
-            else:
-                pass
-        for word_id, word in enumerate(out_text):
-            if word in self.CONTRACTIONS:
-                out_text[word_id] = self.CONTRACTIONS[word]
-        out_text = " ".join(out_text)
-        return out_text
-
-    def __call__(self, item):
-        item = self.word_tokenize(item)
-        item = item.replace("\n", " ").replace("\t", " ").strip()
-        item = self.process_punctuation(item)
-        item = self.process_digit_article(item)
-        return item
-
-
-class TextVQAAccuracyEvaluator:
-    def __init__(self):
-        self.answer_processor = EvalAIAnswerProcessor()
-
-    def _compute_answer_scores(self, raw_answers):
-        """
-        compute the accuracy (soft score) of human answers
-        """
-        answers = [self.answer_processor(a) for a in raw_answers]
-        assert len(answers) == 10
-        gt_answers = list(enumerate(answers))
-        unique_answers = set(answers)
-        unique_answer_scores = {}
-
-        for unique_answer in unique_answers:
-            accs = []
-            for gt_answer in gt_answers:
-                other_answers = [item for item in gt_answers if item != gt_answer]
-                matching_answers = [
-                    item for item in other_answers if item[1] == unique_answer
-                ]
-                acc = min(1, float(len(matching_answers)) / 3)
-                accs.append(acc)
-            unique_answer_scores[unique_answer] = sum(accs) / len(accs)
-
-        return unique_answer_scores
-
-    def eval_pred_list(self, pred_list):
-        pred_scores = []
-        for entry in tqdm(pred_list):
-            pred_answer = self.answer_processor(entry["pred_answer"])
-            unique_answer_scores = self._compute_answer_scores(entry["gt_answers"])
-            score = unique_answer_scores.get(pred_answer, 0.0)
-            pred_scores.append(score)
-
-        accuracy = sum(pred_scores) / len(pred_scores)
-        return accuracy
-
-
-class STVQAAccuracyEvaluator:
-    def __init__(self):
-        self.answer_processor = EvalAIAnswerProcessor()
-
-    def eval_pred_list(self, pred_list):
-        pred_scores = []
-        for entry in pred_list:
-            pred_answer = self.answer_processor(entry["pred_answer"])
-            gts = [self.answer_processor(a) for a in entry["gt_answers"]]
-            score = 1.0 if pred_answer in gts else 0.0
-            pred_scores.append(score)
-
-        accuracy = sum(pred_scores) / len(pred_scores)
-        return accuracy
-
-
-class STVQAANLSEvaluator:
-    def __init__(self):
-        import editdistance  # install with `pip install editdistance`
-
-        self.get_edit_distance = editdistance.eval
-
-    def get_anls(self, s1, s2):
-        s1 = s1.lower().strip()
-        s2 = s2.lower().strip()
-        iou = 1 - self.get_edit_distance(s1, s2) / max(len(s1), len(s2))
-        anls = iou if iou >= 0.5 else 0.0
-        return anls
-
-    def eval_pred_list(self, pred_list):
-        pred_scores = []
-        for entry in pred_list:
-            anls = max(
-                self.get_anls(entry["pred_answer"], gt) for gt in entry["gt_answers"]
-            )
-            pred_scores.append(anls)
-
-        accuracy = sum(pred_scores) / len(pred_scores)
-        return accuracy
-
-
-class TextCapsBleu4Evaluator:
-    def __init__(self):
-        # The following script requires Java 1.8.0 and pycocotools installed.
-        # The pycocoevalcap can be installed with pip as
-        # pip install git+https://github.com/ronghanghu/coco-caption.git@python23
-        # Original pycocoevalcap code is at https://github.com/tylin/coco-caption
-        # but has no python3 support yet.
-        try:
-            from pycocoevalcap.bleu.bleu import Bleu
-            from pycocoevalcap.tokenizer.ptbtokenizer import PTBTokenizer
-        except ModuleNotFoundError:
-            print(
-                "Please install pycocoevalcap module using "
-                "pip install git+https://github.com/ronghanghu/coco-caption.git@python23"  # noqa
-            )
-            raise
-
-        self.tokenizer = PTBTokenizer()
-        self.scorer = Bleu(4)
-
-    def eval_pred_list(self, pred_list):
-        # Create reference and hypotheses captions.
-        gts = {}
-        res = {}
-        for idx, entry in enumerate(pred_list):
-            gts[idx] = [{"caption": a} for a in entry["gt_answers"]]
-            res[idx] = [{"caption": entry["pred_answer"]}]
-
-        gts = self.tokenizer.tokenize(gts)
-        res = self.tokenizer.tokenize(res)
-        score, _ = self.scorer.compute_score(gts, res)
-
-        bleu4 = score[3]  # score is (Bleu-1, Bleu-2, Bleu-3, Bleu-4)
-        return bleu4

llava/eval/model_qa.py

@@ -10,6 +10,25 @@ from llava.conversation import default_conversation
 from llava.utils import disable_torch_init
 
 
+# new stopping implementation
+class KeywordsStoppingCriteria(StoppingCriteria):
+    def __init__(self, keywords, tokenizer, input_ids):
+        self.keywords = keywords
+        self.tokenizer = tokenizer
+        self.start_len = None
+        self.input_ids = input_ids
+
+    def __call__(self, output_ids: torch.LongTensor, scores: torch.FloatTensor, **kwargs) -> bool:
+        if self.start_len is None:
+            self.start_len = self.input_ids.shape[1]
+        else:
+            outputs = self.tokenizer.batch_decode(output_ids[:, self.start_len:], skip_special_tokens=True)[0]
+            for keyword in self.keywords:
+                if keyword in outputs:
+                    return True
+        return False
+
+
 @torch.inference_mode()
 def eval_model(model_name, questions_file, answers_file):
     # Model
@@ -31,12 +50,14 @@ def eval_model(model_name, questions_file, answers_file):
         prompt = conv.get_prompt()
         inputs = tokenizer([prompt])
         input_ids = torch.as_tensor(inputs.input_ids).cuda()
+        stopping_criteria = KeywordsStoppingCriteria([conv.sep], tokenizer, input_ids)
         output_ids = model.generate(
             input_ids,
             do_sample=True,
             use_cache=True,
             temperature=0.7,
-            max_new_tokens=1024,)
+            max_new_tokens=1024,
+            stopping_criteria=[stopping_criteria])
         outputs = tokenizer.batch_decode(output_ids, skip_special_tokens=True)[0]
         try:
             index = outputs.index(conv.sep, len(prompt))

llava/eval/model_vqa.py

@@ -9,7 +9,7 @@ from llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_S
 from llava.conversation import conv_templates, SeparatorStyle
 from llava.model.builder import load_pretrained_model
 from llava.utils import disable_torch_init
-from llava.mm_utils import tokenizer_image_token, process_images, get_model_name_from_path
+from llava.mm_utils import tokenizer_image_token, get_model_name_from_path, KeywordsStoppingCriteria
 
 from PIL import Image
 import math
@@ -55,15 +55,18 @@ def eval_model(args):
 
         input_ids = tokenizer_image_token(prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors='pt').unsqueeze(0).cuda()
 
-        image = Image.open(os.path.join(args.image_folder, image_file)).convert('RGB')
-        image_tensor = process_images([image], image_processor, model.config)[0]
+        image = Image.open(os.path.join(args.image_folder, image_file))
+        image_tensor = image_processor.preprocess(image, return_tensors='pt')['pixel_values'][0]
+
+        stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2
+        keywords = [stop_str]
+        stopping_criteria = KeywordsStoppingCriteria(keywords, tokenizer, input_ids)
 
         with torch.inference_mode():
             output_ids = model.generate(
                 input_ids,
                 images=image_tensor.unsqueeze(0).half().cuda(),
-                image_sizes=[image.size],
-                do_sample=True if args.temperature > 0 else False,
+                do_sample=True,
                 temperature=args.temperature,
                 top_p=args.top_p,
                 num_beams=args.num_beams,
@@ -71,7 +74,15 @@ def eval_model(args):
                 max_new_tokens=1024,
                 use_cache=True)
 
-        outputs = tokenizer.batch_decode(output_ids, skip_special_tokens=True)[0].strip()
+        input_token_len = input_ids.shape[1]
+        n_diff_input_output = (input_ids != output_ids[:, :input_token_len]).sum().item()
+        if n_diff_input_output > 0:
+            print(f'[Warning] {n_diff_input_output} output_ids are not the same as the input_ids')
+        outputs = tokenizer.batch_decode(output_ids[:, input_token_len:], skip_special_tokens=True)[0]
+        outputs = outputs.strip()
+        if outputs.endswith(stop_str):
+            outputs = outputs[:-len(stop_str)]
+        outputs = outputs.strip()
 
         ans_id = shortuuid.uuid()
         ans_file.write(json.dumps({"question_id": idx,

llava/eval/model_vqa_loader.py

@@ -1,144 +0,0 @@
-import argparse
-import torch
-import os
-import json
-from tqdm import tqdm
-import shortuuid
-
-from llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN
-from llava.conversation import conv_templates, SeparatorStyle
-from llava.model.builder import load_pretrained_model
-from llava.utils import disable_torch_init
-from llava.mm_utils import tokenizer_image_token, process_images, get_model_name_from_path
-from torch.utils.data import Dataset, DataLoader
-
-from PIL import Image
-import math
-
-
-def split_list(lst, n):
-    """Split a list into n (roughly) equal-sized chunks"""
-    chunk_size = math.ceil(len(lst) / n)  # integer division
-    return [lst[i:i+chunk_size] for i in range(0, len(lst), chunk_size)]
-
-
-def get_chunk(lst, n, k):
-    chunks = split_list(lst, n)
-    return chunks[k]
-
-
-# Custom dataset class
-class CustomDataset(Dataset):
-    def __init__(self, questions, image_folder, tokenizer, image_processor, model_config):
-        self.questions = questions
-        self.image_folder = image_folder
-        self.tokenizer = tokenizer
-        self.image_processor = image_processor
-        self.model_config = model_config
-
-    def __getitem__(self, index):
-        line = self.questions[index]
-        image_file = line["image"]
-        qs = line["text"]
-        if self.model_config.mm_use_im_start_end:
-            qs = DEFAULT_IM_START_TOKEN + DEFAULT_IMAGE_TOKEN + DEFAULT_IM_END_TOKEN + '\n' + qs
-        else:
-            qs = DEFAULT_IMAGE_TOKEN + '\n' + qs
-
-        conv = conv_templates[args.conv_mode].copy()
-        conv.append_message(conv.roles[0], qs)
-        conv.append_message(conv.roles[1], None)
-        prompt = conv.get_prompt()
-
-        image = Image.open(os.path.join(self.image_folder, image_file)).convert('RGB')
-        image_tensor = process_images([image], self.image_processor, self.model_config)[0]
-
-        input_ids = tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors='pt')
-
-        return input_ids, image_tensor, image.size
-
-    def __len__(self):
-        return len(self.questions)
-
-
-def collate_fn(batch):
-    input_ids, image_tensors, image_sizes = zip(*batch)
-    input_ids = torch.stack(input_ids, dim=0)
-    image_tensors = torch.stack(image_tensors, dim=0)
-    return input_ids, image_tensors, image_sizes
-
-
-# DataLoader
-def create_data_loader(questions, image_folder, tokenizer, image_processor, model_config, batch_size=1, num_workers=4):
-    assert batch_size == 1, "batch_size must be 1"
-    dataset = CustomDataset(questions, image_folder, tokenizer, image_processor, model_config)
-    data_loader = DataLoader(dataset, batch_size=batch_size, num_workers=num_workers, shuffle=False, collate_fn=collate_fn)
-    return data_loader
-
-
-def eval_model(args):
-    # Model
-    disable_torch_init()
-    model_path = os.path.expanduser(args.model_path)
-    model_name = get_model_name_from_path(model_path)
-    tokenizer, model, image_processor, context_len = load_pretrained_model(model_path, args.model_base, model_name)
-
-    questions = [json.loads(q) for q in open(os.path.expanduser(args.question_file), "r")]
-    questions = get_chunk(questions, args.num_chunks, args.chunk_idx)
-    answers_file = os.path.expanduser(args.answers_file)
-    os.makedirs(os.path.dirname(answers_file), exist_ok=True)
-    ans_file = open(answers_file, "w")
-
-    if 'plain' in model_name and 'finetune' not in model_name.lower() and 'mmtag' not in args.conv_mode:
-        args.conv_mode = args.conv_mode + '_mmtag'
-        print(f'It seems that this is a plain model, but it is not using a mmtag prompt, auto switching to {args.conv_mode}.')
-
-    data_loader = create_data_loader(questions, args.image_folder, tokenizer, image_processor, model.config)
-
-    for (input_ids, image_tensor, image_sizes), line in tqdm(zip(data_loader, questions), total=len(questions)):
-        idx = line["question_id"]
-        cur_prompt = line["text"]
-
-        input_ids = input_ids.to(device='cuda', non_blocking=True)
-
-        with torch.inference_mode():
-            output_ids = model.generate(
-                input_ids,
-                images=image_tensor.to(dtype=torch.float16, device='cuda', non_blocking=True),
-                image_sizes=image_sizes,
-                do_sample=True if args.temperature > 0 else False,
-                temperature=args.temperature,
-                top_p=args.top_p,
-                num_beams=args.num_beams,
-                max_new_tokens=args.max_new_tokens,
-                use_cache=True)
-
-        outputs = tokenizer.batch_decode(output_ids, skip_special_tokens=True)[0].strip()
-
-        ans_id = shortuuid.uuid()
-        ans_file.write(json.dumps({"question_id": idx,
-                                   "prompt": cur_prompt,
-                                   "text": outputs,
-                                   "answer_id": ans_id,
-                                   "model_id": model_name,
-                                   "metadata": {}}) + "\n")
-        # ans_file.flush()
-    ans_file.close()
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--model-path", type=str, default="facebook/opt-350m")
-    parser.add_argument("--model-base", type=str, default=None)
-    parser.add_argument("--image-folder", type=str, default="")
-    parser.add_argument("--question-file", type=str, default="tables/question.jsonl")
-    parser.add_argument("--answers-file", type=str, default="answer.jsonl")
-    parser.add_argument("--conv-mode", type=str, default="llava_v1")
-    parser.add_argument("--num-chunks", type=int, default=1)
-    parser.add_argument("--chunk-idx", type=int, default=0)
-    parser.add_argument("--temperature", type=float, default=0.2)
-    parser.add_argument("--top_p", type=float, default=None)
-    parser.add_argument("--num_beams", type=int, default=1)
-    parser.add_argument("--max_new_tokens", type=int, default=128)
-    args = parser.parse_args()
-
-    eval_model(args)

llava/eval/model_vqa_mmbench.py

@@ -1,160 +0,0 @@
-import argparse
-import torch
-import os
-import json
-import pandas as pd
-from tqdm import tqdm
-import shortuuid
-
-from llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN
-from llava.conversation import conv_templates, SeparatorStyle
-from llava.model.builder import load_pretrained_model
-from llava.utils import disable_torch_init
-from llava.mm_utils import tokenizer_image_token, process_images, load_image_from_base64, get_model_name_from_path
-
-from PIL import Image
-import math
-
-
-all_options = ['A', 'B', 'C', 'D']
-
-
-def split_list(lst, n):
-    """Split a list into n (roughly) equal-sized chunks"""
-    chunk_size = math.ceil(len(lst) / n)  # integer division
-    return [lst[i:i+chunk_size] for i in range(0, len(lst), chunk_size)]
-
-
-def get_chunk(lst, n, k):
-    chunks = split_list(lst, n)
-    return chunks[k]
-
-
-def is_none(value):
-    if value is None:
-        return True
-    if type(value) is float and math.isnan(value):
-        return True
-    if type(value) is str and value.lower() == 'nan':
-        return True
-    if type(value) is str and value.lower() == 'none':
-        return True
-    return False
-
-def get_options(row, options):
-    parsed_options = []
-    for option in options:
-        option_value = row[option]
-        if is_none(option_value):
-            break
-        parsed_options.append(option_value)
-    return parsed_options
-
-
-def eval_model(args):
-    # Model
-    disable_torch_init()
-    model_path = os.path.expanduser(args.model_path)
-    model_name = get_model_name_from_path(model_path)
-    tokenizer, model, image_processor, context_len = load_pretrained_model(model_path, args.model_base, model_name)
-
-    questions = pd.read_table(os.path.expanduser(args.question_file))
-    questions = get_chunk(questions, args.num_chunks, args.chunk_idx)
-    answers_file = os.path.expanduser(args.answers_file)
-    os.makedirs(os.path.dirname(answers_file), exist_ok=True)
-    ans_file = open(answers_file, "w")
-
-    if 'plain' in model_name and 'finetune' not in model_name.lower() and 'mmtag' not in args.conv_mode:
-        args.conv_mode = args.conv_mode + '_mmtag'
-        print(f'It seems that this is a plain model, but it is not using a mmtag prompt, auto switching to {args.conv_mode}.')
-
-    for index, row in tqdm(questions.iterrows(), total=len(questions)):
-        options = get_options(row, all_options)
-        cur_option_char = all_options[:len(options)]
-
-        if args.all_rounds:
-            num_rounds = len(options)
-        else:
-            num_rounds = 1
-
-        for round_idx in range(num_rounds):
-            idx = row['index']
-            question = row['question']
-            hint = row['hint']
-            image = load_image_from_base64(row['image'])
-            if not is_none(hint):
-                question = hint + '\n' + question
-            for option_char, option in zip(all_options[:len(options)], options):
-                question = question + '\n' + option_char + '. ' + option
-            qs = cur_prompt = question
-            if model.config.mm_use_im_start_end:
-                qs = DEFAULT_IM_START_TOKEN + DEFAULT_IMAGE_TOKEN + DEFAULT_IM_END_TOKEN + '\n' + qs
-            else:
-                qs = DEFAULT_IMAGE_TOKEN + '\n' + qs
-
-            if args.single_pred_prompt:
-                if args.lang == 'cn':
-                    qs = qs + '\n' + "请直接回答选项字母。"
-                else:
-                    qs = qs + '\n' + "Answer with the option's letter from the given choices directly."
-
-            conv = conv_templates[args.conv_mode].copy()
-            conv.append_message(conv.roles[0], qs)
-            conv.append_message(conv.roles[1], None)
-            prompt = conv.get_prompt()
-
-            input_ids = tokenizer_image_token(prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors='pt').unsqueeze(0).cuda()
-
-            image_tensor = process_images([image], image_processor, model.config)[0]
-
-            with torch.inference_mode():
-                output_ids = model.generate(
-                    input_ids,
-                    images=image_tensor.unsqueeze(0).half().cuda(),
-                    image_sizes=[image.size],
-                    do_sample=True if args.temperature > 0 else False,
-                    temperature=args.temperature,
-                    top_p=args.top_p,
-                    num_beams=args.num_beams,
-                    # no_repeat_ngram_size=3,
-                    max_new_tokens=1024,
-                    use_cache=True)
-
-            outputs = tokenizer.batch_decode(output_ids, skip_special_tokens=True)[0].strip()
-
-            ans_id = shortuuid.uuid()
-            ans_file.write(json.dumps({"question_id": idx,
-                                    "round_id": round_idx,
-                                    "prompt": cur_prompt,
-                                    "text": outputs,
-                                    "options": options,
-                                    "option_char": cur_option_char,
-                                    "answer_id": ans_id,
-                                    "model_id": model_name,
-                                    "metadata": {}}) + "\n")
-            ans_file.flush()
-
-            # rotate options
-            options = options[1:] + options[:1]
-            cur_option_char = cur_option_char[1:] + cur_option_char[:1]
-    ans_file.close()
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--model-path", type=str, default="facebook/opt-350m")
-    parser.add_argument("--model-base", type=str, default=None)
-    parser.add_argument("--image-folder", type=str, default="")
-    parser.add_argument("--question-file", type=str, default="tables/question.jsonl")
-    parser.add_argument("--answers-file", type=str, default="answer.jsonl")
-    parser.add_argument("--conv-mode", type=str, default="llava_v1")
-    parser.add_argument("--num-chunks", type=int, default=1)
-    parser.add_argument("--chunk-idx", type=int, default=0)
-    parser.add_argument("--temperature", type=float, default=0.2)
-    parser.add_argument("--top_p", type=float, default=None)
-    parser.add_argument("--num_beams", type=int, default=1)
-    parser.add_argument("--all-rounds", action="store_true")
-    parser.add_argument("--single-pred-prompt", action="store_true")
-    parser.add_argument("--lang", type=str, default="en")
-    args = parser.parse_args()
-
-    eval_model(args)

llava/eval/model_vqa_science.py

@@ -9,7 +9,7 @@ from llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_S
 from llava.conversation import conv_templates, SeparatorStyle
 from llava.model.builder import load_pretrained_model
 from llava.utils import disable_torch_init
-from llava.mm_utils import tokenizer_image_token, process_images, get_model_name_from_path
+from llava.mm_utils import tokenizer_image_token, get_model_name_from_path, KeywordsStoppingCriteria
 
 from PIL import Image
 import math
@@ -47,9 +47,8 @@ def eval_model(args):
         if 'image' in line:
             image_file = line["image"]
             image = Image.open(os.path.join(args.image_folder, image_file))
-            image_tensor = process_images([image], image_processor, model.config)[0]
+            image_tensor = image_processor.preprocess(image, return_tensors='pt')['pixel_values'][0]
             images = image_tensor.unsqueeze(0).half().cuda()
-            image_sizes = [image.size]
             if getattr(model.config, 'mm_use_im_start_end', False):
                 qs = DEFAULT_IM_START_TOKEN + DEFAULT_IMAGE_TOKEN + DEFAULT_IM_END_TOKEN + '\n' + qs
             else:
@@ -57,11 +56,6 @@ def eval_model(args):
             cur_prompt = '<image>' + '\n' + cur_prompt
         else:
             images = None
-            image_sizes = None
-
-        if args.single_pred_prompt:
-            qs = qs + '\n' + "Answer with the option's letter from the given choices directly."
-            cur_prompt = cur_prompt + '\n' + "Answer with the option's letter from the given choices directly."
 
         conv = conv_templates[args.conv_mode].copy()
         conv.append_message(conv.roles[0], qs)
@@ -70,18 +64,56 @@ def eval_model(args):
 
         input_ids = tokenizer_image_token(prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors='pt').unsqueeze(0).cuda()
 
+        stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2
+        keywords = [stop_str]
+        stopping_criteria = [KeywordsStoppingCriteria(keywords, tokenizer, input_ids)] if conv.version == "v0" else None
+
         with torch.inference_mode():
             output_ids = model.generate(
                 input_ids,
                 images=images,
-                image_sizes=image_sizes,
-                do_sample=True if args.temperature > 0 else False,
-                temperature=args.temperature,
+                do_sample=True,
+                temperature=0.2,
                 max_new_tokens=1024,
                 use_cache=True,
+                stopping_criteria=stopping_criteria,
             )
 
-        outputs = tokenizer.batch_decode(output_ids, skip_special_tokens=True)[0].strip()
+        input_token_len = input_ids.shape[1]
+        n_diff_input_output = (input_ids != output_ids[:, :input_token_len]).sum().item()
+        if n_diff_input_output > 0:
+            print(f'[Warning] {n_diff_input_output} output_ids are not the same as the input_ids')
+        outputs = tokenizer.batch_decode(output_ids[:, input_token_len:], skip_special_tokens=True)[0]
+        outputs = outputs.strip()
+        if outputs.endswith(stop_str):
+            outputs = outputs[:-len(stop_str)]
+        outputs = outputs.strip()
+
+        # prompt for answer
+        if args.answer_prompter:
+            outputs_reasoning = outputs
+            input_ids = tokenizer_image_token(prompt + outputs_reasoning + ' ###\nANSWER:', tokenizer, IMAGE_TOKEN_INDEX, return_tensors='pt').unsqueeze(0).cuda()
+
+            with torch.inference_mode():
+                output_ids = model.generate(
+                    input_ids,
+                    images=images,
+                    do_sample=True,
+                    temperature=0.2,
+                    max_new_tokens=64,
+                    use_cache=True,
+                    stopping_criteria=[stopping_criteria])
+
+            input_token_len = input_ids.shape[1]
+            n_diff_input_output = (input_ids != output_ids[:, :input_token_len]).sum().item()
+            if n_diff_input_output > 0:
+                print(f'[Warning] {n_diff_input_output} output_ids are not the same as the input_ids')
+            outputs = tokenizer.batch_decode(output_ids[:, input_token_len:], skip_special_tokens=True)[0]
+            outputs = outputs.strip()
+            if outputs.endswith(stop_str):
+                outputs = outputs[:-len(stop_str)]
+            outputs = outputs.strip()
+            outputs = outputs_reasoning + '\n The answer is ' + outputs
 
         ans_id = shortuuid.uuid()
         ans_file.write(json.dumps({"question_id": idx,
@@ -103,9 +135,7 @@ if __name__ == "__main__":
     parser.add_argument("--conv-mode", type=str, default="llava_v0")
     parser.add_argument("--num-chunks", type=int, default=1)
     parser.add_argument("--chunk-idx", type=int, default=0)
-    parser.add_argument("--temperature", type=float, default=0.2)
     parser.add_argument("--answer-prompter", action="store_true")
-    parser.add_argument("--single-pred-prompt", action="store_true")
     args = parser.parse_args()
 
     eval_model(args)

llava/eval/run_llava.py

@@ -1,80 +1,43 @@
 import argparse
 import torch
 
-from llava.constants import (
-    IMAGE_TOKEN_INDEX,
-    DEFAULT_IMAGE_TOKEN,
-    DEFAULT_IM_START_TOKEN,
-    DEFAULT_IM_END_TOKEN,
-    IMAGE_PLACEHOLDER,
-)
+from llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN
 from llava.conversation import conv_templates, SeparatorStyle
 from llava.model.builder import load_pretrained_model
 from llava.utils import disable_torch_init
-from llava.mm_utils import (
-    process_images,
-    tokenizer_image_token,
-    get_model_name_from_path,
-)
+from llava.mm_utils import tokenizer_image_token, get_model_name_from_path, KeywordsStoppingCriteria
 
 from PIL import Image
 
 import requests
 from PIL import Image
 from io import BytesIO
-import re
-
-
-def image_parser(args):
-    out = args.image_file.split(args.sep)
-    return out
 
 
 def load_image(image_file):
-    if image_file.startswith("http") or image_file.startswith("https"):
+    if image_file.startswith('http') or image_file.startswith('https'):
         response = requests.get(image_file)
-        image = Image.open(BytesIO(response.content)).convert("RGB")
+        image = Image.open(BytesIO(response.content)).convert('RGB')
     else:
-        image = Image.open(image_file).convert("RGB")
+        image = Image.open(image_file).convert('RGB')
     return image
 
 
-def load_images(image_files):
-    out = []
-    for image_file in image_files:
-        image = load_image(image_file)
-        out.append(image)
-    return out
-
-
 def eval_model(args):
     # Model
     disable_torch_init()
 
     model_name = get_model_name_from_path(args.model_path)
-    tokenizer, model, image_processor, context_len = load_pretrained_model(
-        args.model_path, args.model_base, model_name
-    )
+    tokenizer, model, image_processor, context_len = load_pretrained_model(args.model_path, args.model_base, model_name)
 
     qs = args.query
-    image_token_se = DEFAULT_IM_START_TOKEN + DEFAULT_IMAGE_TOKEN + DEFAULT_IM_END_TOKEN
-    if IMAGE_PLACEHOLDER in qs:
-        if model.config.mm_use_im_start_end:
-            qs = re.sub(IMAGE_PLACEHOLDER, image_token_se, qs)
-        else:
-            qs = re.sub(IMAGE_PLACEHOLDER, DEFAULT_IMAGE_TOKEN, qs)
+    if model.config.mm_use_im_start_end:
+        qs = DEFAULT_IM_START_TOKEN + DEFAULT_IMAGE_TOKEN + DEFAULT_IM_END_TOKEN + '\n' + qs
     else:
-        if model.config.mm_use_im_start_end:
-            qs = image_token_se + "\n" + qs
-        else:
-            qs = DEFAULT_IMAGE_TOKEN + "\n" + qs
+        qs = DEFAULT_IMAGE_TOKEN + '\n' + qs
 
-    if "llama-2" in model_name.lower():
+    if 'llama-2' in model_name.lower():
         conv_mode = "llava_llama_2"
-    elif "mistral" in model_name.lower():
-        conv_mode = "mistral_instruct"
-    elif "v1.6-34b" in model_name.lower():
-        conv_mode = "chatml_direct"
     elif "v1" in model_name.lower():
         conv_mode = "llava_v1"
     elif "mpt" in model_name.lower():
@@ -83,11 +46,7 @@ def eval_model(args):
         conv_mode = "llava_v0"
 
     if args.conv_mode is not None and conv_mode != args.conv_mode:
-        print(
-            "[WARNING] the auto inferred conversation mode is {}, while `--conv-mode` is {}, using {}".format(
-                conv_mode, args.conv_mode, args.conv_mode
-            )
-        )
+        print('[WARNING] the auto inferred conversation mode is {}, while `--conv-mode` is {}, using {}'.format(conv_mode, args.conv_mode, args.conv_mode))
     else:
         args.conv_mode = conv_mode
 
@@ -96,38 +55,36 @@ def eval_model(args):
     conv.append_message(conv.roles[1], None)
     prompt = conv.get_prompt()
 
-    image_files = image_parser(args)
-    images = load_images(image_files)
-    image_sizes = [x.size for x in images]
-    images_tensor = process_images(
-        images,
-        image_processor,
-        model.config
-    ).to(model.device, dtype=torch.float16)
-
-    input_ids = (
-        tokenizer_image_token(prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt")
-        .unsqueeze(0)
-        .cuda()
-    )
+    image = load_image(args.image_file)
+    image_tensor = image_processor.preprocess(image, return_tensors='pt')['pixel_values'].half().cuda()
+
+    input_ids = tokenizer_image_token(prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors='pt').unsqueeze(0).cuda()
+
+    stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2
+    keywords = [stop_str]
+    stopping_criteria = KeywordsStoppingCriteria(keywords, tokenizer, input_ids)
 
     with torch.inference_mode():
         output_ids = model.generate(
             input_ids,
-            images=images_tensor,
-            image_sizes=image_sizes,
-            do_sample=True if args.temperature > 0 else False,
-            temperature=args.temperature,
-            top_p=args.top_p,
-            num_beams=args.num_beams,
-            max_new_tokens=args.max_new_tokens,
+            images=image_tensor,
+            do_sample=True,
+            temperature=0.2,
+            max_new_tokens=1024,
             use_cache=True,
-        )
-
-    outputs = tokenizer.batch_decode(output_ids, skip_special_tokens=True)[0].strip()
+            stopping_criteria=[stopping_criteria])
+
+    input_token_len = input_ids.shape[1]
+    n_diff_input_output = (input_ids != output_ids[:, :input_token_len]).sum().item()
+    if n_diff_input_output > 0:
+        print(f'[Warning] {n_diff_input_output} output_ids are not the same as the input_ids')
+    outputs = tokenizer.batch_decode(output_ids[:, input_token_len:], skip_special_tokens=True)[0]
+    outputs = outputs.strip()
+    if outputs.endswith(stop_str):
+        outputs = outputs[:-len(stop_str)]
+    outputs = outputs.strip()
     print(outputs)
 
-
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     parser.add_argument("--model-path", type=str, default="facebook/opt-350m")
@@ -135,11 +92,6 @@ if __name__ == "__main__":
     parser.add_argument("--image-file", type=str, required=True)
     parser.add_argument("--query", type=str, required=True)
     parser.add_argument("--conv-mode", type=str, default=None)
-    parser.add_argument("--sep", type=str, default=",")
-    parser.add_argument("--temperature", type=float, default=0.2)
-    parser.add_argument("--top_p", type=float, default=None)
-    parser.add_argument("--num_beams", type=int, default=1)
-    parser.add_argument("--max_new_tokens", type=int, default=512)
     args = parser.parse_args()
 
     eval_model(args)

llava/eval/summarize_gpt_review.py

@@ -9,10 +9,8 @@ import argparse
 def parse_args():
     parser = argparse.ArgumentParser(description='ChatGPT-based QA evaluation.')
     parser.add_argument('-d', '--dir', default=None)
-    parser.add_argument('-v', '--version', default=None)
-    parser.add_argument('-s', '--select', nargs='*', default=None)
-    parser.add_argument('-f', '--files', nargs='*', default=[])
-    parser.add_argument('-i', '--ignore', nargs='*', default=[])
+    parser.add_argument('-f', '--files', nargs='*', default=None)
+    parser.add_argument('-i', '--ignore', nargs='*', default=None)
     return parser.parse_args()
 
 
@@ -22,27 +20,19 @@ if __name__ == '__main__':
     if args.ignore is not None:
         args.ignore = [int(x) for x in args.ignore]
 
-    if len(args.files) > 0:
+    if args.files is not None and len(args.files) > 0:
         review_files = args.files
     else:
-        review_files = [x for x in os.listdir(args.dir) if x.endswith('.jsonl') and (x.startswith('gpt4_text') or x.startswith('reviews_') or x.startswith('review_') or 'review' in args.dir)]
+        review_files = [x for x in os.listdir(args.dir) if x.endswith('.jsonl') and (x.startswith('gpt4_text') or x.startswith('reviews_') or x.startswith('review_'))]
 
     for review_file in sorted(review_files):
         config = os.path.basename(review_file).replace('gpt4_text_', '').replace('.jsonl', '')
-        if args.select is not None and any(x not in config for x in args.select):
-            continue
-        if '0613' in config:
-            version = '0613'
-        else:
-            version = '0314'
-        if args.version is not None and args.version != version:
-            continue
         scores = defaultdict(list)
         print(config)
         with open(os.path.join(args.dir, review_file) if args.dir is not None else review_file) as f:
             for review_str in f:
                 review = json.loads(review_str)
-                if review['question_id'] in args.ignore:
+                if args.ignore is not None and review['question_id'] in args.ignore:
                     continue
                 if 'category' in review:
                     scores[review['category']].append(review['tuple'])
@@ -56,5 +46,5 @@ if __name__ == '__main__':
             stats = np.asarray(v).mean(0).tolist()
             stats = [round(x, 3) for x in stats]
             # print(k, stats, round(stats[1]/stats[0]*100, 1))
-            print(k, round(stats[1]/stats[0]*100, 1), round(stats[0] * 10, 1), round(stats[1] * 10, 1))
+            print(k, round(stats[1]/stats[0]*100, 1))
         print('=================================')

llava/mm_utils.py

@@ -1,150 +1,12 @@
 from PIL import Image
 from io import BytesIO
 import base64
-import torch
-import math
-import ast
 
+import torch
 from transformers import StoppingCriteria
 from llava.constants import IMAGE_TOKEN_INDEX
 
 
-def select_best_resolution(original_size, possible_resolutions):
-    """
-    Selects the best resolution from a list of possible resolutions based on the original size.
-
-    Args:
-        original_size (tuple): The original size of the image in the format (width, height).
-        possible_resolutions (list): A list of possible resolutions in the format [(width1, height1), (width2, height2), ...].
-
-    Returns:
-        tuple: The best fit resolution in the format (width, height).
-    """
-    original_width, original_height = original_size
-    best_fit = None
-    max_effective_resolution = 0
-    min_wasted_resolution = float('inf')
-
-    for width, height in possible_resolutions:
-        scale = min(width / original_width, height / original_height)
-        downscaled_width, downscaled_height = int(original_width * scale), int(original_height * scale)
-        effective_resolution = min(downscaled_width * downscaled_height, original_width * original_height)
-        wasted_resolution = (width * height) - effective_resolution
-
-        if effective_resolution > max_effective_resolution or (effective_resolution == max_effective_resolution and wasted_resolution < min_wasted_resolution):
-            max_effective_resolution = effective_resolution
-            min_wasted_resolution = wasted_resolution
-            best_fit = (width, height)
-
-    return best_fit
-
-
-def resize_and_pad_image(image, target_resolution):
-    """
-    Resize and pad an image to a target resolution while maintaining aspect ratio.
-
-    Args:
-        image (PIL.Image.Image): The input image.
-        target_resolution (tuple): The target resolution (width, height) of the image.
-
-    Returns:
-        PIL.Image.Image: The resized and padded image.
-    """
-    original_width, original_height = image.size
-    target_width, target_height = target_resolution
-
-    scale_w = target_width / original_width
-    scale_h = target_height / original_height
-
-    if scale_w < scale_h:
-        new_width = target_width
-        new_height = min(math.ceil(original_height * scale_w), target_height)
-    else:
-        new_height = target_height
-        new_width = min(math.ceil(original_width * scale_h), target_width)
-
-    # Resize the image
-    resized_image = image.resize((new_width, new_height))
-
-    new_image = Image.new('RGB', (target_width, target_height), (0, 0, 0))
-    paste_x = (target_width - new_width) // 2
-    paste_y = (target_height - new_height) // 2
-    new_image.paste(resized_image, (paste_x, paste_y))
-
-    return new_image
-
-
-def divide_to_patches(image, patch_size):
-    """
-    Divides an image into patches of a specified size.
-
-    Args:
-        image (PIL.Image.Image): The input image.
-        patch_size (int): The size of each patch.
-
-    Returns:
-        list: A list of PIL.Image.Image objects representing the patches.
-    """
-    patches = []
-    width, height = image.size
-    for i in range(0, height, patch_size):
-        for j in range(0, width, patch_size):
-            box = (j, i, j + patch_size, i + patch_size)
-            patch = image.crop(box)
-            patches.append(patch)
-
-    return patches
-
-
-def get_anyres_image_grid_shape(image_size, grid_pinpoints, patch_size):
-    """
-    Calculate the shape of the image patch grid after the preprocessing for images of any resolution.
-
-    Args:
-        image_size (tuple): The size of the input image in the format (width, height).
-        grid_pinpoints (str): A string representation of a list of possible resolutions.
-        patch_size (int): The size of each image patch.
-
-    Returns:
-        tuple: The shape of the image patch grid in the format (width, height).
-    """
-    if type(grid_pinpoints) is list:
-        possible_resolutions = grid_pinpoints
-    else:
-        possible_resolutions = ast.literal_eval(grid_pinpoints)
-    width, height = select_best_resolution(image_size, possible_resolutions)
-    return width // patch_size, height // patch_size
-
-
-def process_anyres_image(image, processor, grid_pinpoints):
-    """
-    Process an image with variable resolutions.
-
-    Args:
-        image (PIL.Image.Image): The input image to be processed.
-        processor: The image processor object.
-        grid_pinpoints (str): A string representation of a list of possible resolutions.
-
-    Returns:
-        torch.Tensor: A tensor containing the processed image patches.
-    """
-    if type(grid_pinpoints) is list:
-        possible_resolutions = grid_pinpoints
-    else:
-        possible_resolutions = ast.literal_eval(grid_pinpoints)
-    best_resolution = select_best_resolution(image.size, possible_resolutions)
-    image_padded = resize_and_pad_image(image, best_resolution)
-
-    patches = divide_to_patches(image_padded, processor.crop_size['height'])
-
-    image_original_resize = image.resize((processor.size['shortest_edge'], processor.size['shortest_edge']))
-
-    image_patches = [image_original_resize] + patches
-    image_patches = [processor.preprocess(image_patch, return_tensors='pt')['pixel_values'][0]
-                     for image_patch in image_patches]
-    return torch.stack(image_patches, dim=0)
-
-
 def load_image_from_base64(image):
     return Image.open(BytesIO(base64.b64decode(image)))
 
@@ -171,10 +33,6 @@ def process_images(images, image_processor, model_cfg):
             image = expand2square(image, tuple(int(x*255) for x in image_processor.image_mean))
             image = image_processor.preprocess(image, return_tensors='pt')['pixel_values'][0]
             new_images.append(image)
-    elif image_aspect_ratio == "anyres":
-        for image in images:
-            image = process_anyres_image(image, image_processor, model_cfg.image_grid_pinpoints)
-            new_images.append(image)
     else:
         return image_processor(images, return_tensors='pt')['pixel_values']
     if all(x.shape == new_images[0].shape for x in new_images):
@@ -212,36 +70,30 @@ def get_model_name_from_path(model_path):
     else:
         return model_paths[-1]
 
+
+
+
 class KeywordsStoppingCriteria(StoppingCriteria):
     def __init__(self, keywords, tokenizer, input_ids):
         self.keywords = keywords
         self.keyword_ids = []
-        self.max_keyword_len = 0
         for keyword in keywords:
             cur_keyword_ids = tokenizer(keyword).input_ids
             if len(cur_keyword_ids) > 1 and cur_keyword_ids[0] == tokenizer.bos_token_id:
                 cur_keyword_ids = cur_keyword_ids[1:]
-            if len(cur_keyword_ids) > self.max_keyword_len:
-                self.max_keyword_len = len(cur_keyword_ids)
             self.keyword_ids.append(torch.tensor(cur_keyword_ids))
         self.tokenizer = tokenizer
         self.start_len = input_ids.shape[1]
-    
-    def call_for_batch(self, output_ids: torch.LongTensor, scores: torch.FloatTensor, **kwargs) -> bool:
-        offset = min(output_ids.shape[1] - self.start_len, self.max_keyword_len)
+
+    def __call__(self, output_ids: torch.LongTensor, scores: torch.FloatTensor, **kwargs) -> bool:
+        assert output_ids.shape[0] == 1, "Only support batch size 1 (yet)"  # TODO
+        offset = min(output_ids.shape[1] - self.start_len, 3)
         self.keyword_ids = [keyword_id.to(output_ids.device) for keyword_id in self.keyword_ids]
         for keyword_id in self.keyword_ids:
-            truncated_output_ids = output_ids[0, -keyword_id.shape[0]:]
-            if torch.equal(truncated_output_ids, keyword_id):
+            if output_ids[0, -keyword_id.shape[0]:] == keyword_id:
                 return True
         outputs = self.tokenizer.batch_decode(output_ids[:, -offset:], skip_special_tokens=True)[0]
         for keyword in self.keywords:
             if keyword in outputs:
                 return True
         return False
-    
-    def __call__(self, output_ids: torch.LongTensor, scores: torch.FloatTensor, **kwargs) -> bool:
-        outputs = []
-        for i in range(output_ids.shape[0]):
-            outputs.append(self.call_for_batch(output_ids[i].unsqueeze(0), scores))
-        return all(outputs)

llava/model/__init__.py

@@ -1,6 +1,2 @@
-try:
-    from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
-    from .language_model.llava_mpt import LlavaMptForCausalLM, LlavaMptConfig
-    from .language_model.llava_mistral import LlavaMistralForCausalLM, LlavaMistralConfig
-except:
-    pass
+from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig
+from .language_model.llava_mpt import LlavaMPTForCausalLM, LlavaMPTConfig

llava/model/builder.py

@@ -23,11 +23,9 @@ from llava.model import *
 from llava.constants import DEFAULT_IMAGE_PATCH_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN
 
 
-def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, load_4bit=False, device_map="auto", device="cuda", use_flash_attn=False, **kwargs):
-    kwargs = {"device_map": device_map, **kwargs}
-
-    if device != "cuda":
-        kwargs['device_map'] = {"": device}
+def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, load_4bit=False, device_map="auto"):
+    kwargs = {"device_map": device_map}
+    kwargs["offload_folder"] = "offload"
 
     if load_8bit:
         kwargs['load_in_8bit'] = True
@@ -42,16 +40,12 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l
     else:
         kwargs['torch_dtype'] = torch.float16
 
-    if use_flash_attn:
-        kwargs['attn_implementation'] = 'flash_attention_2'
-
     if 'llava' in model_name.lower():
         # Load LLaVA model
         if 'lora' in model_name.lower() and model_base is None:
             warnings.warn('There is `lora` in model name but no `model_base` is provided. If you are loading a LoRA model, please provide the `model_base` argument. Detailed instruction: https://github.com/haotian-liu/LLaVA#launch-a-model-worker-lora-weights-unmerged.')
         if 'lora' in model_name.lower() and model_base is not None:
-            from llava.model.language_model.llava_llama import LlavaConfig
-            lora_cfg_pretrained = LlavaConfig.from_pretrained(model_path)
+            lora_cfg_pretrained = AutoConfig.from_pretrained(model_path)
             tokenizer = AutoTokenizer.from_pretrained(model_base, use_fast=False)
             print('Loading LLaVA from base model...')
             model = LlavaLlamaForCausalLM.from_pretrained(model_base, low_cpu_mem_usage=True, config=lora_cfg_pretrained, **kwargs)
@@ -92,7 +86,7 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l
                     shutil.copyfile(os.path.join(model_base, 'configuration_mpt.py'), os.path.join(model_path, 'configuration_mpt.py'))
                 tokenizer = AutoTokenizer.from_pretrained(model_base, use_fast=True)
                 cfg_pretrained = AutoConfig.from_pretrained(model_path, trust_remote_code=True)
-                model = LlavaMptForCausalLM.from_pretrained(model_base, low_cpu_mem_usage=True, config=cfg_pretrained, **kwargs)
+                model = LlavaMPTForCausalLM.from_pretrained(model_base, low_cpu_mem_usage=True, config=cfg_pretrained, **kwargs)
             else:
                 tokenizer = AutoTokenizer.from_pretrained(model_base, use_fast=False)
                 cfg_pretrained = AutoConfig.from_pretrained(model_path)
@@ -104,28 +98,17 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l
         else:
             if 'mpt' in model_name.lower():
                 tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True)
-                model = LlavaMptForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, **kwargs)
-            elif 'mistral' in model_name.lower():
-                tokenizer = AutoTokenizer.from_pretrained(model_path)
-                model = LlavaMistralForCausalLM.from_pretrained(
-                    model_path,
-                    low_cpu_mem_usage=True,
-                    **kwargs
-                )
+                model = LlavaMPTForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, **kwargs)
             else:
                 tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
-                model = LlavaLlamaForCausalLM.from_pretrained(
-                    model_path,
-                    low_cpu_mem_usage=True,
-                    **kwargs
-                )
+                model = LlavaLlamaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, **kwargs)
     else:
         # Load language model
         if model_base is not None:
             # PEFT model
             from peft import PeftModel
             tokenizer = AutoTokenizer.from_pretrained(model_base, use_fast=False)
-            model = AutoModelForCausalLM.from_pretrained(model_base, low_cpu_mem_usage=True, **kwargs)
+            model = AutoModelForCausalLM.from_pretrained(model_base, torch_dtype=torch.float16, low_cpu_mem_usage=True, device_map="auto")
             print(f"Loading LoRA weights from {model_path}")
             model = PeftModel.from_pretrained(model, model_path)
             print(f"Merging weights")
@@ -154,9 +137,10 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l
 
         vision_tower = model.get_vision_tower()
         if not vision_tower.is_loaded:
-            vision_tower.load_model(device_map=device_map)
-        if device_map != 'auto':
-            vision_tower.to(device=device_map, dtype=torch.float16)
+            vision_tower.load_model()
+        
+
+        vision_tower.to(device=model.device, dtype=torch.float16)
         image_processor = vision_tower.image_processor
 
     if hasattr(model.config, "max_sequence_length"):

llava/model/language_model/llava_llama.py

@@ -17,18 +17,18 @@ from typing import List, Optional, Tuple, Union
 
 import torch
 import torch.nn as nn
+from torch.nn import CrossEntropyLoss
 
 from transformers import AutoConfig, AutoModelForCausalLM, \
                          LlamaConfig, LlamaModel, LlamaForCausalLM
 
 from transformers.modeling_outputs import CausalLMOutputWithPast
-from transformers.generation.utils import GenerateOutput
 
 from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
 
 
 class LlavaConfig(LlamaConfig):
-    model_type = "llava_llama"
+    model_type = "llava"
 
 
 class LlavaLlamaModel(LlavaMetaModel, LlamaModel):
@@ -44,8 +44,7 @@ class LlavaLlamaForCausalLM(LlamaForCausalLM, LlavaMetaForCausalLM):
     def __init__(self, config):
         super(LlamaForCausalLM, self).__init__(config)
         self.model = LlavaLlamaModel(config)
-        self.pretraining_tp = config.pretraining_tp
-        self.vocab_size = config.vocab_size
+
         self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
 
         # Initialize weights and apply final processing
@@ -58,7 +57,6 @@ class LlavaLlamaForCausalLM(LlamaForCausalLM, LlavaMetaForCausalLM):
         self,
         input_ids: torch.LongTensor = None,
         attention_mask: Optional[torch.Tensor] = None,
-        position_ids: Optional[torch.LongTensor] = None,
         past_key_values: Optional[List[torch.FloatTensor]] = None,
         inputs_embeds: Optional[torch.FloatTensor] = None,
         labels: Optional[torch.LongTensor] = None,
@@ -66,93 +64,77 @@ class LlavaLlamaForCausalLM(LlamaForCausalLM, LlavaMetaForCausalLM):
         output_attentions: Optional[bool] = None,
         output_hidden_states: Optional[bool] = None,
         images: Optional[torch.FloatTensor] = None,
-        image_sizes: Optional[List[List[int]]] = None,
         return_dict: Optional[bool] = None,
     ) -> Union[Tuple, CausalLMOutputWithPast]:
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        input_ids, attention_mask, past_key_values, inputs_embeds, labels = self.prepare_inputs_labels_for_multimodal(input_ids, attention_mask, past_key_values, labels, images)
 
-        if inputs_embeds is None:
-            (
-                input_ids,
-                position_ids,
-                attention_mask,
-                past_key_values,
-                inputs_embeds,
-                labels
-            ) = self.prepare_inputs_labels_for_multimodal(
-                input_ids,
-                position_ids,
-                attention_mask,
-                past_key_values,
-                labels,
-                images,
-                image_sizes
-            )
-
-        return super().forward(
+        # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)
+        outputs = self.model(
             input_ids=input_ids,
             attention_mask=attention_mask,
-            position_ids=position_ids,
             past_key_values=past_key_values,
             inputs_embeds=inputs_embeds,
-            labels=labels,
             use_cache=use_cache,
             output_attentions=output_attentions,
             output_hidden_states=output_hidden_states,
             return_dict=return_dict
         )
 
-    @torch.no_grad()
-    def generate(
-        self,
-        inputs: Optional[torch.Tensor] = None,
-        images: Optional[torch.Tensor] = None,
-        image_sizes: Optional[torch.Tensor] = None,
-        **kwargs,
-    ) -> Union[GenerateOutput, torch.LongTensor]:
-        position_ids = kwargs.pop("position_ids", None)
-        attention_mask = kwargs.pop("attention_mask", None)
-        if "inputs_embeds" in kwargs:
-            raise NotImplementedError("`inputs_embeds` is not supported")
-
-        if images is not None:
-            (
-                inputs,
-                position_ids,
-                attention_mask,
-                _,
-                inputs_embeds,
-                _
-            ) = self.prepare_inputs_labels_for_multimodal(
-                inputs,
-                position_ids,
-                attention_mask,
-                None,
-                None,
-                images,
-                image_sizes=image_sizes
-            )
-        else:
-            inputs_embeds = self.get_model().embed_tokens(inputs)
-
-        return super().generate(
-            position_ids=position_ids,
-            attention_mask=attention_mask,
-            inputs_embeds=inputs_embeds,
-            **kwargs
+        hidden_states = outputs[0]
+        logits = self.lm_head(hidden_states)
+
+        loss = None
+        if labels is not None:
+            # Shift so that tokens < n predict n
+            shift_logits = logits[..., :-1, :].contiguous()
+            shift_labels = labels[..., 1:].contiguous()
+            # Flatten the tokens
+            loss_fct = CrossEntropyLoss()
+            shift_logits = shift_logits.view(-1, self.config.vocab_size)
+            shift_labels = shift_labels.view(-1)
+            # Enable model/pipeline parallelism
+            shift_labels = shift_labels.to(shift_logits.device)
+            loss = loss_fct(shift_logits, shift_labels)
+
+        if not return_dict:
+            output = (logits,) + outputs[1:]
+            return (loss,) + output if loss is not None else output
+
+        return CausalLMOutputWithPast(
+            loss=loss,
+            logits=logits,
+            past_key_values=outputs.past_key_values,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
         )
 
-    def prepare_inputs_for_generation(self, input_ids, past_key_values=None,
-                                      inputs_embeds=None, **kwargs):
-        images = kwargs.pop("images", None)
-        image_sizes = kwargs.pop("image_sizes", None)
-        inputs = super().prepare_inputs_for_generation(
-            input_ids, past_key_values=past_key_values, inputs_embeds=inputs_embeds, **kwargs
+    def prepare_inputs_for_generation(
+        self, input_ids, past_key_values=None, attention_mask=None, inputs_embeds=None, **kwargs
+    ):
+        if past_key_values:
+            input_ids = input_ids[:, -1:]
+
+        # if `inputs_embeds` are passed, we only want to use them in the 1st generation step
+        if inputs_embeds is not None and past_key_values is None:
+            model_inputs = {"inputs_embeds": inputs_embeds}
+        else:
+            model_inputs = {"input_ids": input_ids}
+
+        model_inputs.update(
+            {
+                "past_key_values": past_key_values,
+                "use_cache": kwargs.get("use_cache"),
+                "attention_mask": attention_mask,
+                "images": kwargs.get("images", None),
+            }
         )
-        if images is not None:
-            inputs['images'] = images
-        if image_sizes is not None:
-            inputs['image_sizes'] = image_sizes
-        return inputs
+        return model_inputs
 
-AutoConfig.register("llava_llama", LlavaConfig)
+AutoConfig.register("llava", LlavaConfig)
 AutoModelForCausalLM.register(LlavaConfig, LlavaLlamaForCausalLM)