change to fp32

app.py

@@ -123,10 +123,10 @@ def get_response(params):
 
             if type(images) is list:
                 images = [
-                    image.to(model.device, dtype=torch.float16) for image in images
+                    image.to(model.device, dtype=torch.float32) for image in images
                 ]
             else:
-                images = images.to(model.device, dtype=torch.float16)
+                images = images.to(model.device, dtype=torch.float32)
 
             replace_token = DEFAULT_IMAGE_TOKEN
             if getattr(model.config, "mm_use_im_start_end", False):
@@ -343,44 +343,43 @@ def build_demo():
                     visible=False,
                 )
 
-                # cur_dir = os.path.dirname(os.path.abspath(__file__))
                 cur_dir = Path(__file__).parent
                 gr.Examples(
                     examples=[
                         [
-                            f"{cur_dir}/examples/market.png",
+                            f"{cur_dir}/images/market.png",
                             "What is the highest number of companies in the domestic market? Answer with detailed steps.",
                         ],
                         [
-                            f"{cur_dir}/examples/college.png",
+                            f"{cur_dir}/images/college.png",
                             "What is the difference between Asians and Whites degree distribution? Answer with detailed steps."
                         ],
                         [
-                            f"{cur_dir}/examples/immigrants.png",
+                            f"{cur_dir}/images/immigrants.png",
                             "How many immigrants are there in 1931?",
                         ],
                         [
-                            f"{cur_dir}/examples/sails.png",
+                            f"{cur_dir}/images/sails.png",
                             "By how much percentage wholesale is less than retail? Answer with detailed steps."
                         ],
                         [
-                            f"{cur_dir}/examples/diseases.png",
+                            f"{cur_dir}/images/diseases.png",
                             "Is the median value of all the bars greater than 30? Answer with detailed steps.",
                         ],
                         [
-                            f"{cur_dir}/examples/economy.png",
+                            f"{cur_dir}/images/economy.png",
                             "Which team has higher economy in 28 min?"
                         ],
                         [
-                            f"{cur_dir}/examples/workers.png",
+                            f"{cur_dir}/images/workers.png",
                             "Generate underlying data table for the chart."
                         ],
                         [
-                            f"{cur_dir}/examples/sports.png",
+                            f"{cur_dir}/images/sports.png",
                             "Create a brief summarization or extract key insights based on the chart image."
                         ],
                         [
-                            f"{cur_dir}/examples/albums.png",
+                            f"{cur_dir}/images/albums.png",
                             "Redraw the chart with Python code."
                         ]
                     ],
@@ -489,7 +488,8 @@ if __name__ == "__main__":
         model_name=args.model_name,
         device="cpu",
         load_4bit=args.load_4bit,
-        load_8bit=args.load_8bit
+        load_8bit=args.load_8bit,
+        torch_dtype=torch.float32,
     )
 
     demo = build_demo()

tinychart/model/builder.py

@@ -40,7 +40,7 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l
             bnb_4bit_use_double_quant=True,
             bnb_4bit_quant_type='nf4'
         )
-    else:
+    elif 'torch_dtype' not in kwargs:
         kwargs['torch_dtype'] = torch.float16
 
     # Load LLaVA model
@@ -97,7 +97,7 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l
                                                         **kwargs)
 
         mm_projector_weights = torch.load(os.path.join(model_path, 'mm_projector.bin'), map_location='cpu')
-        mm_projector_weights = {k: v.to(torch.float16) for k, v in mm_projector_weights.items()}
+        mm_projector_weights = {k: v.to(kwargs['torch_dtype']) for k, v in mm_projector_weights.items()}
         model.load_state_dict(mm_projector_weights, strict=False)
     else:
         tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False, padding_side="right")
@@ -115,7 +115,7 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l
         vision_tower.load_model()
 
     if device != "auto":
-        vision_tower.to(device=device, dtype=torch.float16)
+        vision_tower.to(device=device, dtype=kwargs['torch_dtype'])
 
     image_processor = vision_tower.image_processor