Added Flask to Readme

.gitattributes

@@ -33,6 +33,3 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
-snowman.png filter=lfs diff=lfs merge=lfs -text
-pikachu.png filter=lfs diff=lfs merge=lfs -text
-pikachu_bbox.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -5,12 +5,6 @@
 ---
 # Kosmos-2: Grounding Multimodal Large Language Models to the World
 
-**This model (remote code on the Hub) is deprecated. Please use https://huggingface.co/microsoft/kosmos-2-patch14-224**
-
-**There are some changes in terms of input formats: see the model card in https://huggingface.co/microsoft/kosmos-2-patch14-224**
-
-~~**(There is an on going effort to port `Kosmos-2` directly into `transformers`. This repository (remote code) might need some more bug fixes later, including breaking changes.)**~~
-
 <a href="https://huggingface.co/ydshieh/kosmos-2-patch14-224/resolve/main/annotated_snowman.jpg" target="_blank"><figure><img src="https://huggingface.co/ydshieh/kosmos-2-patch14-224/resolve/main/annotated_snowman.jpg" width="384"><figcaption><b>[An image of a snowman warming himself by a fire.]</b></figcaption></figure></a>
 
 
@@ -32,7 +26,7 @@ processor = AutoProcessor.from_pretrained("ydshieh/kosmos-2-patch14-224", trust_
 
 prompt = "<grounding>An image of"
 
-url = "https://huggingface.co/ydshieh/kosmos-2-patch14-224/resolve/main/snowman.png"
+url = "https://huggingface.co/ydshieh/kosmos-2-patch14-224/resolve/main/snowman.jpg"
 image = Image.open(requests.get(url, stream=True).raw)
 
 # The original Kosmos-2 demo saves the image first then reload it. For some images, this will give slightly different image input and change the generation outputs.
@@ -211,215 +205,3 @@ draw_entity_boxes_on_image(image, entities, show=True)
 Here is the annotated image:
 
 <a href="https://huggingface.co/ydshieh/kosmos-2-patch14-224/resolve/main/annotated_snowman.jpg" target="_blank"><img src="https://huggingface.co/ydshieh/kosmos-2-patch14-224/resolve/main/annotated_snowman.jpg" width="500"></a>
-
-
-## Tasks
-
-This model is capable of performing different tasks through changing the prompts.
-
-First, let's define a function to run a prompt.
-
-```python
-import requests
-
-from PIL import Image
-from transformers import AutoProcessor, AutoModelForVision2Seq
-
-
-model = AutoModelForVision2Seq.from_pretrained("ydshieh/kosmos-2-patch14-224", trust_remote_code=True)
-processor = AutoProcessor.from_pretrained("ydshieh/kosmos-2-patch14-224", trust_remote_code=True)
-
-url = "https://huggingface.co/ydshieh/kosmos-2-patch14-224/resolve/main/snowman.png"
-image = Image.open(requests.get(url, stream=True).raw)
-
-def run_example(prompt):
-
-    inputs = processor(text=prompt, images=image, return_tensors="pt")
-    generated_ids = model.generate(
-        pixel_values=inputs["pixel_values"],
-        input_ids=inputs["input_ids"][:, :-1],
-        attention_mask=inputs["attention_mask"][:, :-1],
-        img_features=None,
-        img_attn_mask=inputs["img_attn_mask"][:, :-1],
-        use_cache=True,
-        max_new_tokens=64,
-    )
-    generated_text = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
-    _processed_text = processor.post_process_generation(generated_text, cleanup_and_extract=False)
-    processed_text, entities = processor.post_process_generation(generated_text)
-    print(processed_text)
-    print(entities)
-    print(_processed_text)
-```
-
-Here are the tasks `Kosmos-2` could perform:
-
-### Multimodal Grounding
-
-#### • Phrase Grounding
-```python
-prompt = "<grounding><phrase> a snowman</phrase>"
-run_example(prompt)
-
-# a snowman is warming himself by the fire
-# [('a snowman', (0, 9), [(0.390625, 0.046875, 0.984375, 0.828125)]), ('the fire', (32, 40), [(0.203125, 0.015625, 0.453125, 0.859375)])]
-
-# <grounding><phrase> a snowman</phrase><object><patch_index_0044><patch_index_0863></object> is warming himself by<phrase> the fire</phrase><object><patch_index_0006><patch_index_0878></object>
-```
-
-#### • Referring Expression Comprehension
-```python
-prompt = "<grounding><phrase> a snowman next to a fire</phrase>"
-run_example(prompt)
-
-# a snowman next to a fire
-# [('a snowman next to a fire', (0, 24), [(0.390625, 0.046875, 0.984375, 0.828125)])]
-
-# <grounding><phrase> a snowman next to a fire</phrase><object><patch_index_0044><patch_index_0863></object>
-```
-
-### Multimodal Referring
-
-#### • Referring expression generation
-```python
-prompt = "<grounding><phrase> It</phrase><object><patch_index_0044><patch_index_0863></object> is"
-run_example(prompt)
-
-# It is snowman in a hat and scarf
-# [('It', (0, 2), [(0.390625, 0.046875, 0.984375, 0.828125)])]
-
-# <grounding><phrase> It</phrase><object><patch_index_0044><patch_index_0863></object> is snowman in a hat and scarf
-```
-
-### Perception-Language Tasks
-
-#### • Grounded VQA
-```python
-prompt = "<grounding> Question: What is special about this image? Answer:"
-run_example(prompt)
-
-# Question: What is special about this image? Answer: The image features a snowman sitting by a campfire in the snow.
-# [('a snowman', (71, 80), [(0.390625, 0.046875, 0.984375, 0.828125)]), ('a campfire', (92, 102), [(0.109375, 0.640625, 0.546875, 0.984375)])]
-
-# <grounding> Question: What is special about this image? Answer: The image features<phrase> a snowman</phrase><object><patch_index_0044><patch_index_0863></object> sitting by<phrase> a campfire</phrase><object><patch_index_0643><patch_index_1009></object> in the snow.
-```
-
-#### • Grounded VQA with multimodal referring via bounding boxes
-```python
-prompt = "<grounding> Question: Where is<phrase> the fire</phrase><object><patch_index_0005><patch_index_0911></object> next to? Answer:"
-run_example(prompt)
-
-# Question: Where is the fire next to? Answer: Near the snowman.
-# [('the fire', (19, 27), [(0.171875, 0.015625, 0.484375, 0.890625)]), ('the snowman', (50, 61), [(0.390625, 0.046875, 0.984375, 0.828125)])]
-
-# <grounding> Question: Where is<phrase> the fire</phrase><object><patch_index_0005><patch_index_0911></object> next to? Answer: Near<phrase> the snowman</phrase><object><patch_index_0044><patch_index_0863></object>.
-```
-
-### Grounded Image captioning
-
-#### • Brief
-
-```python
-prompt = "<grounding> An image of"
-run_example(prompt)
-
-# An image of a snowman warming himself by a campfire.
-# [('a snowman', (12, 21), [(0.390625, 0.046875, 0.984375, 0.828125)]), ('a campfire', (41, 51), [(0.109375, 0.640625, 0.546875, 0.984375)])]
-
-# <grounding> An image of<phrase> a snowman</phrase><object><patch_index_0044><patch_index_0863></object> warming himself by<phrase> a campfire</phrase><object><patch_index_0643><patch_index_1009></object>.
-```
-
-#### • Detailed
-
-```python
-prompt = "<grounding> Describe this image in detail:"
-run_example(prompt)
-
-# Describe this image in detail: The image features a snowman sitting by a campfire in the snow. He is wearing a hat, scarf, and gloves, with a pot nearby and a cup
-# [('a campfire', (71, 81), [(0.171875, 0.015625, 0.484375, 0.984375)]), ('a hat', (109, 114), [(0.515625, 0.046875, 0.828125, 0.234375)]), ('scarf', (116, 121), [(0.515625, 0.234375, 0.890625, 0.578125)]), ('gloves', (127, 133), [(0.515625, 0.390625, 0.640625, 0.515625)]), ('a pot', (140, 145), [(0.078125, 0.609375, 0.265625, 0.859375)])]
-
-# <grounding> Describe this image in detail: The image features a snowman sitting by<phrase> a campfire</phrase><object><patch_index_0005><patch_index_1007></object> in the snow. He is wearing<phrase> a hat</phrase><object><patch_index_0048><patch_index_0250></object>,<phrase> scarf</phrase><object><patch_index_0240><patch_index_0604></object>, and<phrase> gloves</phrase><object><patch_index_0400><patch_index_0532></object>, with<phrase> a pot</phrase><object><patch_index_0610><patch_index_0872></object> nearby and<phrase> a cup</phrase><object>
-```
-
-
-## Running the Flask Server
-_flask_kosmos2.py_ shows the implementation of a Flask server for the model.
-It allowes the model to be approached as a REST API.
-
-After starting the server. You can send a POST request to `http://localhost:8005/process_prompt` with the following form data:
-- `prompt`: For example `<grounding> an image of`
-- `image`: The image file as binary data
-
-This in turn will produce a reply with the following JSON format:
-- `message`: The Kosmos-2 generated text
-- `entities`: The extracted entities
-
-An easy way to test this is through an application like Postman. Make sure the image field is set to `File`.
-
-```python
-
-from PIL import Image
-from transformers import AutoProcessor, AutoModelForVision2Seq
-from flask import Flask, request, jsonify
-import json
-
-app = Flask(__name__)
-
-model = AutoModelForVision2Seq.from_pretrained("ydshieh/kosmos-2-patch14-224", trust_remote_code=True)
-processor = AutoProcessor.from_pretrained("ydshieh/kosmos-2-patch14-224", trust_remote_code=True)
-
-
-@app.route('/process_prompt', methods=['POST'])
-def process_prompt():
-    try:
-        # Get the uploaded image data from the POST request
-        uploaded_file = request.files['image']
-        prompt = request.form.get('prompt')
-        image = Image.open(uploaded_file.stream)
-
-        print(image.size)
-
-        inputs = processor(text=prompt, images=image, return_tensors="pt")
-
-        generated_ids = model.generate(
-            pixel_values=inputs["pixel_values"],
-            input_ids=inputs["input_ids"][:, :-1],
-            attention_mask=inputs["attention_mask"][:, :-1],
-            img_features=None,
-            img_attn_mask=inputs["img_attn_mask"][:, :-1],
-            use_cache=True,
-            max_new_tokens=64,
-        )
-        generated_text = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
-
-        # By default, the generated  text is cleanup and the entities are extracted.
-        processed_text, entities = processor.post_process_generation(generated_text)
-        parsed_entities = entities_to_json(entities)
-        print(generated_text)
-        print(processed_text)
-        return jsonify({"message": processed_text, 'entities': parsed_entities})
-    except Exception as e:
-        return jsonify({"error": str(e)})
-
-
-def entities_to_json(entities):
-    result = []
-    for e in entities:
-        label = e[0]
-        box_coords = e[1]
-        box_size = e[2][0]
-        entity_result = {
-            "label": label,
-            "boundingBoxPosition": {"x": box_coords[0], "y": box_coords[1]},
-            "boundingBox": {"x_min": box_size[0], "y_min": box_size[1], "x_max": box_size[2], "y_max": box_size[3]}
-        }
-        print(entity_result)
-        result.append(entity_result)
-
-    return result
-
-
-if __name__ == '__main__':
-    app.run(host='localhost', port=8005)
-
-```

modeling_kosmos2.py

@@ -22,7 +22,6 @@ from typing import List, Optional, Tuple, Union
 import torch
 import torch.utils.checkpoint
 from torch import nn
-from torch.nn import CrossEntropyLoss
 
 from transformers.activations import ACT2FN
 from transformers.modeling_outputs import (
@@ -1008,7 +1007,7 @@ class Kosmos2TextTransformer(nn.Module):
             inputs_embeds = self.embed_tokens(input_ids)
 
         if img_features is not None:
-            inputs_embeds[img_input_mask.to(dtype=torch.bool)] = img_features.view(-1, img_features.size(-1))
+            inputs_embeds[img_input_mask.to(dtype=torch.bool)] = img_features
 
         inputs_embeds = inputs_embeds * self.embed_scale
 

processing_kosmos2.py

@@ -529,8 +529,7 @@ def extract_entities_with_patch_indices(text):
         phrase_tag, phrase, match_content = match.groups()
         if not phrase_tag:
             phrase = None
-            # We take the starting position of `<object>`
-            span = (match.span(0)[0], match.span(0)[0])
+            span = (None, None)
 
         # Split the match_content by the delimiter to get individual patch_index pairs
         patch_index_pairs = match_content.split('</delimiter_of_multi_objects/>')

tokenization_kosmos2_fast.py

@@ -137,6 +137,7 @@ class Kosmos2TokenizerFast(PreTrainedTokenizerFast):
         )
 
         self.vocab_file = vocab_file
+        self.can_save_slow_tokenizer = False if not self.vocab_file else True
 
         self.eod_token = "</doc>"
 
@@ -178,10 +179,6 @@ class Kosmos2TokenizerFast(PreTrainedTokenizerFast):
                 # we need to set `special_tokens=False` to be the same as in the slow tokenizer.
                 self.add_tokens(AddedToken(token, lstrip=True, rstrip=False), special_tokens=False)
 
-    @property
-    def can_save_slow_tokenizer(self) -> bool:
-        return os.path.isfile(self.vocab_file) if self.vocab_file else False
-    
     def build_inputs_with_special_tokens(
         self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
     ) -> List[int]: