add README.md

README.md

@@ -1,3 +1,188 @@
 ---
 license: mit
 ---
+
+
+# IflyBotVLM
+
+
+## Introduction
+
+IflyBotVLM is a 8B open-source vision-language model(VLM) designed for embodied brain.
+
+## Model Architecture
+
+
+
+## Model Performance
+
+IflyBotVLM demonstrates superior performance across various challenging benchmarks.
+
+![image/png](https://huggingface.co/datasets/IflyBot/IflyBotVLM-Repo/resolve/main/images/benchmark_performance.png)
+
+![image/png](https://huggingface.co/datasets/IflyBot/IflyBotVLM-Repo/resolve/main/images/table-performances.png)
+
+IflyBotVLM-8B achieves state-of-the-art (SOTA) or near-SOTA performance on ten spatial comprehension, spatial perception, and temporal task planning benchmarks: Where2Place, Refspatial-bench, ShareRobot-affordance, ShareRobot-trajectory, BLINK(spatial), EmbSpatial, ERQA, CVBench, SAT, EgoPlan2.
+
+## Quick Start
+
+### Using 🤗  Transformers to Chat
+
+We provide an example code to run `InternVL3-8B` using `transformers`.
+
+> Please use transformers>=4.37.2 to ensure the model works normally.
+
+<details>
+<summary>Python code</summary>
+
+```python
+import math
+import numpy as np
+import torch
+import torchvision.transforms as T
+from PIL import Image
+from torchvision.transforms.functional import InterpolationMode
+from transformers import AutoModel, AutoTokenizer,AutoConfig
+from tqdm import tqdm
+import json
+
+
+IMAGENET_MEAN = (0.485, 0.456, 0.406)
+IMAGENET_STD = (0.229, 0.224, 0.225)
+
+class IflyRoboInference:
+    def __init__(self, model_path=''):
+        self.model = AutoModel.from_pretrained(
+            model_path,
+            torch_dtype=torch.bfloat16,
+            load_in_8bit=False,
+            low_cpu_mem_usage=True,
+            use_flash_attn=True,
+            trust_remote_code=True,
+            device_map="balanced").eval()  # "auto", "balanced"
+        self.tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True, use_fast=False)
+        self.generation_config = dict(
+            do_sample=True,
+            temperature=0.5,
+            top_p = 0.0,
+            top_k = 1,
+            max_new_tokens=16384
+            )
+
+    def build_transform(self, input_size):
+        MEAN, STD = IMAGENET_MEAN, IMAGENET_STD
+        transform = T.Compose([
+            T.Lambda(lambda img: img.convert('RGB') if img.mode != 'RGB' else img),
+            T.Resize((input_size, input_size), interpolation=InterpolationMode.BICUBIC),
+            T.ToTensor(),
+            T.Normalize(mean=MEAN, std=STD)
+        ])
+        return transform
+
+    def find_closest_aspect_ratio(self, aspect_ratio, target_ratios, width, height, image_size):
+        best_ratio_diff = float('inf')
+        best_ratio = (1, 1)
+        area = width * height
+        for ratio in target_ratios:
+            target_aspect_ratio = ratio[0] / ratio[1]
+            ratio_diff = abs(aspect_ratio - target_aspect_ratio)
+            if ratio_diff < best_ratio_diff:
+                best_ratio_diff = ratio_diff
+                best_ratio = ratio
+            elif ratio_diff == best_ratio_diff:
+                if area > 0.5 * image_size * image_size * ratio[0] * ratio[1]:
+                    best_ratio = ratio
+        return best_ratio
+
+    def dynamic_preprocess(self, image, min_num=1, max_num=12, image_size=896, use_thumbnail=False):
+        orig_width, orig_height = image.size
+        aspect_ratio = orig_width / orig_height
+
+        target_ratios = set(
+            (i, j) for n in range(min_num, max_num + 1) for i in range(1, n + 1) for j in range(1, n + 1) if
+            i * j <= max_num and i * j >= min_num)
+        target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1])
+
+        target_aspect_ratio = self.find_closest_aspect_ratio(
+            aspect_ratio, target_ratios, orig_width, orig_height, image_size)
+
+        target_width = image_size * target_aspect_ratio[0]
+        target_height = image_size * target_aspect_ratio[1]
+        blocks = target_aspect_ratio[0] * target_aspect_ratio[1]
+
+        resized_img = image.resize((target_width, target_height))
+        processed_images = []
+        for i in range(blocks):
+            box = (
+                (i % (target_width // image_size)) * image_size,
+                (i // (target_width // image_size)) * image_size,
+                ((i % (target_width // image_size)) + 1) * image_size,
+                ((i // (target_width // image_size)) + 1) * image_size
+            )
+            split_img = resized_img.crop(box)
+            processed_images.append(split_img)
+        assert len(processed_images) == blocks
+        if use_thumbnail and len(processed_images) != 1:
+            thumbnail_img = image.resize((image_size, image_size))
+            processed_images.append(thumbnail_img)
+        return processed_images
+
+    def load_image(self, image_file, input_size=896, max_num=12):
+        image = Image.open(image_file).convert('RGB')
+        transform = self.build_transform(input_size=input_size)
+        images = self.dynamic_preprocess(image, image_size=input_size, use_thumbnail=True, max_num=max_num)
+        pixel_values = [transform(image) for image in images]
+        pixel_values = torch.stack(pixel_values)
+        return pixel_values
+
+
+    def forward_multi_image(self, image_paths: list,  question: dict):
+        pixel_values = []
+        num_patches_list = []
+        resize_size = 448
+        for  i, image_path in enumerate(image_paths):
+            pixel_value = self.load_image(image_path, input_size=resize_size).to(torch.bfloat16).cuda()
+            pixel_values.append(pixel_value)
+            num_patches_list.append(pixel_value.size(0))
+        pixel_values = torch.cat(tuple(pixel_values), dim=0)
+        print(question)
+        response, history = self.model.chat(self.tokenizer, pixel_values, question["prompt"], self.generation_config, history=None, return_history=True)
+        print(response)
+
+
+def test_spatial_from_blink():
+    hf_path = "IflyBot/IflyBotVLM"
+    ifly_robo_infer = IflyRoboInference(hf_path)
+    question = {
+        "idx": "val_Spatial_Relation_143",
+        "sub_task" : "Spatial Relation",
+        "prompt": "<image> Is the person behind the cup?\nSelect from the following choices.\n(A) yes\n(B) no.\nPlease answer directly with only the letter of the correct option and nothing else."
+    }
+    image_path = [
+        "./examples-images/val_Spatial_Relation_143_1.jpg"
+    ]
+    ifly_robo_infer.forward_multi_image(image_path, question)
+
+
+def test_visual_correspondence_from_blink():
+    hf_path = "IflyBot/IflyBotVLM"
+    ifly_robo_infer = IflyRoboInference(hf_path)
+    question = {
+        "idx": "val_Visual_Correspondence_1",
+        "sub_task" : "Visual Correspondence",
+        "prompt": "<image> <image> A point is circled on the first image, labeled with REF. We change the camera position or lighting and shoot the second image. You are given multiple red-circled points on the second image, choices of \"A, B, C, D\" are drawn beside each circle. Which point on the second image corresponds to the point in the first image? Select from the following options.\n(A) Point A\n(B) Point B\n(C) Point C\n(D) Point D.\nPlease answer directly with only the letter of the correct option and nothing else."
+    }
+    image_path = [
+        "./examples-images/val_Visual_Correspondence_1_1.jpg",
+        "./examples-images/val_Visual_Correspondence_1_2.jpg"
+    ]
+    ifly_robo_infer.forward_multi_image(image_path, question)
+
+
+if __name__ == '__main__':
+    test_spatial_from_blink()
+    test_visual_correspondence_from_blink()
+    test_task_plan_from_egoplan2()
+```
+
+</details>