Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,35 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+fastvlm_ax650_context_1k_prefill_640/image_encoder_1024x1024.axmodel filter=lfs diff=lfs merge=lfs -text
+fastvlm_ax650_context_1k_prefill_640/llava_qwen2_p128_l0_together.axmodel filter=lfs diff=lfs merge=lfs -text
+fastvlm_ax650_context_1k_prefill_640/llava_qwen2_p128_l10_together.axmodel filter=lfs diff=lfs merge=lfs -text
+fastvlm_ax650_context_1k_prefill_640/llava_qwen2_p128_l11_together.axmodel filter=lfs diff=lfs merge=lfs -text
+fastvlm_ax650_context_1k_prefill_640/llava_qwen2_p128_l12_together.axmodel filter=lfs diff=lfs merge=lfs -text
+fastvlm_ax650_context_1k_prefill_640/llava_qwen2_p128_l13_together.axmodel filter=lfs diff=lfs merge=lfs -text
+fastvlm_ax650_context_1k_prefill_640/llava_qwen2_p128_l14_together.axmodel filter=lfs diff=lfs merge=lfs -text
+fastvlm_ax650_context_1k_prefill_640/llava_qwen2_p128_l15_together.axmodel filter=lfs diff=lfs merge=lfs -text
+fastvlm_ax650_context_1k_prefill_640/llava_qwen2_p128_l16_together.axmodel filter=lfs diff=lfs merge=lfs -text
+fastvlm_ax650_context_1k_prefill_640/llava_qwen2_p128_l17_together.axmodel filter=lfs diff=lfs merge=lfs -text
+fastvlm_ax650_context_1k_prefill_640/llava_qwen2_p128_l18_together.axmodel filter=lfs diff=lfs merge=lfs -text
+fastvlm_ax650_context_1k_prefill_640/llava_qwen2_p128_l19_together.axmodel filter=lfs diff=lfs merge=lfs -text
+fastvlm_ax650_context_1k_prefill_640/llava_qwen2_p128_l1_together.axmodel filter=lfs diff=lfs merge=lfs -text
+fastvlm_ax650_context_1k_prefill_640/llava_qwen2_p128_l20_together.axmodel filter=lfs diff=lfs merge=lfs -text
+fastvlm_ax650_context_1k_prefill_640/llava_qwen2_p128_l21_together.axmodel filter=lfs diff=lfs merge=lfs -text
+fastvlm_ax650_context_1k_prefill_640/llava_qwen2_p128_l22_together.axmodel filter=lfs diff=lfs merge=lfs -text
+fastvlm_ax650_context_1k_prefill_640/llava_qwen2_p128_l23_together.axmodel filter=lfs diff=lfs merge=lfs -text
+fastvlm_ax650_context_1k_prefill_640/llava_qwen2_p128_l24_together.axmodel filter=lfs diff=lfs merge=lfs -text
+fastvlm_ax650_context_1k_prefill_640/llava_qwen2_p128_l25_together.axmodel filter=lfs diff=lfs merge=lfs -text
+fastvlm_ax650_context_1k_prefill_640/llava_qwen2_p128_l26_together.axmodel filter=lfs diff=lfs merge=lfs -text
+fastvlm_ax650_context_1k_prefill_640/llava_qwen2_p128_l27_together.axmodel filter=lfs diff=lfs merge=lfs -text
+fastvlm_ax650_context_1k_prefill_640/llava_qwen2_p128_l2_together.axmodel filter=lfs diff=lfs merge=lfs -text
+fastvlm_ax650_context_1k_prefill_640/llava_qwen2_p128_l3_together.axmodel filter=lfs diff=lfs merge=lfs -text
+fastvlm_ax650_context_1k_prefill_640/llava_qwen2_p128_l4_together.axmodel filter=lfs diff=lfs merge=lfs -text
+fastvlm_ax650_context_1k_prefill_640/llava_qwen2_p128_l5_together.axmodel filter=lfs diff=lfs merge=lfs -text
+fastvlm_ax650_context_1k_prefill_640/llava_qwen2_p128_l6_together.axmodel filter=lfs diff=lfs merge=lfs -text
+fastvlm_ax650_context_1k_prefill_640/llava_qwen2_p128_l7_together.axmodel filter=lfs diff=lfs merge=lfs -text
+fastvlm_ax650_context_1k_prefill_640/llava_qwen2_p128_l8_together.axmodel filter=lfs diff=lfs merge=lfs -text
+fastvlm_ax650_context_1k_prefill_640/llava_qwen2_p128_l9_together.axmodel filter=lfs diff=lfs merge=lfs -text
+fastvlm_ax650_context_1k_prefill_640/llava_qwen2_post.axmodel filter=lfs diff=lfs merge=lfs -text
+images/image_1.jpg filter=lfs diff=lfs merge=lfs -text
+images/ssd_horse.jpg filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,3 +1,218 @@
----
-license: bsd-3-clause
----
+# FastVLM-1.5B
+
+This version of FastVLM-1.5B has been converted to run on the Axera NPU using **w8a16** quantization.
+
+This model has been optimized with the following LoRA: 
+
+Compatible with Pulsar2 version: 5.1-patch1.
+
+Please note that the context of the model is 1k and the maximum prefill length is 640 tokens.
+
+## Convert tools links:
+
+For those who are interested in model conversion, you can try to export axmodel through the original repo:
+
+https://huggingface.co/apple/FastVLM-1.5B
+
+How to Convert LLM from Huggingface to axmodel[TODO]
+
+## Support Platform
+
+- AX650
+  - AX650N DEMO Board
+  - [M4N-Dock(爱芯派Pro)](https://wiki.sipeed.com/hardware/zh/maixIV/m4ndock/m4ndock.html)
+  - [M.2 Accelerator card](https://axcl-docs.readthedocs.io/zh-cn/latest/doc_guide_hardware.html)
+ 
+|Chips|image encoder 1024|ttft(291tokens)|w8a16|
+|--|--|--|--|
+|AX650| 216.257 ms | 861.213 ms | 13.88 tokens/sec|
+
+
+## How to use
+
+Download all files from this repository to the device
+
+```
+$ tree -L 1
+.
+├── fastvlm_ax650_context_1k_prefill_640
+├── fastvlm_tokenizer
+├── images
+├── infer_axmodel.py
+├── README.md
+└── utils
+
+5 directories, 2 files
+
+```
+
+#### Install transformer
+
+```
+pip install -r requirements.txt
+```
+
+#### Inference with AX650 Host, such as M4N-Dock(爱芯派Pro) or AX650 DEMO Board
+
+Run the following command on the Axera board to start a chat conversation:
+
+```sh
+$ python infer_axmodel.py -v ./fastvlm_ax650_context_1k_prefill_640/image_encoder_1024x1024.axmodel -m ./fastvlm_ax650_context_1k_prefill_640 -t ./fastvlm_tokenizer/
+```
+output:
+
+```bash
+[INFO] Available providers:  ['AXCLRTExecutionProvider']
+Loading config, tokenizer and init model.
+Detected prefixes: ['llava_qwen2'], chosen: llava_qwen2, layers: 28
+Init InferenceSession:   0%|                                                                                                                          | 0/28 [00:00<?, ?it/s][INFO] Using provider: AXCLRTExecutionProvider
+[INFO] SOC Name: AX650N
+[INFO] VNPU type: VNPUType.DISABLED
+[INFO] Compiler version: 5.1-patch1-dirty 140e8d4a-dirty
+Init InferenceSession:   4%|████                                                                                                              | 1/28 [00:01<00:28,  1.05s/it][INFO] Using provider: AXCLRTExecutionProvider
+[INFO] SOC Name: AX650N
+[INFO] VNPU type: VNPUType.DISABLED
+[INFO] Compiler version: 5.1-patch1-dirty 140e8d4a-dirty
+Init InferenceSession:   7%|████████▏                                                                                                         | 2/28 [00:01<00:21,  1.20it/s][INFO] Using provider: AXCLRTExecutionProvider
+[INFO] SOC Name: AX650N
+[INFO] VNPU type: VNPUType.DISABLED
+[INFO] Compiler version: 5.1-patch1-dirty 140e8d4a-dirty
+Init InferenceSession:  11%|████████████▏                                                                                                     | 3/28 [00:02<00:19,  1.30it/s][INFO] Using provider: AXCLRTExecutionProvider
+[INFO] SOC Name: AX650N
+[INFO] VNPU type: VNPUType.DISABLED
+[INFO] Compiler version: 5.1-patch1-dirty 140e8d4a-dirty
+Init InferenceSession:  14%|████████████████▎                                                                                                 | 4/28 [00:03<00:17,  1.36it/s][INFO] Using provider: AXCLRTExecutionProvider
+[INFO] SOC Name: AX650N
+[INFO] VNPU type: VNPUType.DISABLED
+[INFO] Compiler version: 5.1-patch1-dirty 140e8d4a-dirty
+Init InferenceSession:  18%|████████████████████▎                                                                                             | 5/28 [00:03<00:16,  1.40it/s][INFO] Using provider: AXCLRTExecutionProvider
+[INFO] SOC Name: AX650N
+[INFO] VNPU type: VNPUType.DISABLED
+[INFO] Compiler version: 5.1-patch1-dirty 140e8d4a-dirty
+Init InferenceSession:  21%|████████████████████████▍                                                                                         | 6/28 [00:04<00:15,  1.42it/s][INFO] Using provider: AXCLRTExecutionProvider
+[INFO] SOC Name: AX650N
+[INFO] VNPU type: VNPUType.DISABLED
+[INFO] Compiler version: 5.1-patch1-dirty 140e8d4a-dirty
+Init InferenceSession:  25%|████████████████████████████▌                                                                                     | 7/28 [00:05<00:14,  1.43it/s][INFO] Using provider: AXCLRTExecutionProvider
+[INFO] SOC Name: AX650N
+[INFO] VNPU type: VNPUType.DISABLED
+[INFO] Compiler version: 5.1-patch1-dirty 140e8d4a-dirty
+Init InferenceSession:  29%|████████████████████████████████▌                                                                                 | 8/28 [00:05<00:13,  1.44it/s][INFO] Using provider: AXCLRTExecutionProvider
+[INFO] SOC Name: AX650N
+[INFO] VNPU type: VNPUType.DISABLED
+[INFO] Compiler version: 5.1-patch1-dirty 140e8d4a-dirty
+Init InferenceSession:  32%|████████████████████████████████████▋                                                                             | 9/28 [00:06<00:13,  1.44it/s][INFO] Using provider: AXCLRTExecutionProvider
+[INFO] SOC Name: AX650N
+[INFO] VNPU type: VNPUType.DISABLED
+[INFO] Compiler version: 5.1-patch1-dirty 140e8d4a-dirty
+Init InferenceSession:  36%|████████████████████████████████████████▎                                                                        | 10/28 [00:07<00:12,  1.45it/s][INFO] Using provider: AXCLRTExecutionProvider
+[INFO] SOC Name: AX650N
+[INFO] VNPU type: VNPUType.DISABLED
+[INFO] Compiler version: 5.1-patch1-dirty 140e8d4a-dirty
+Init InferenceSession:  39%|████████████████████████████████████████████▍                                                                    | 11/28 [00:07<00:11,  1.45it/s][INFO] Using provider: AXCLRTExecutionProvider
+[INFO] SOC Name: AX650N
+[INFO] VNPU type: VNPUType.DISABLED
+[INFO] Compiler version: 5.1-patch1-dirty 140e8d4a-dirty
+Init InferenceSession:  43%|████████████████████████████████████████████████▍                                                                | 12/28 [00:08<00:11,  1.45it/s][INFO] Using provider: AXCLRTExecutionProvider
+[INFO] SOC Name: AX650N
+[INFO] VNPU type: VNPUType.DISABLED
+[INFO] Compiler version: 5.1-patch1-dirty 140e8d4a-dirty
+Init InferenceSession:  46%|████████████████████████████████████████████████████▍                                                            | 13/28 [00:09<00:10,  1.45it/s][INFO] Using provider: AXCLRTExecutionProvider
+[INFO] SOC Name: AX650N
+[INFO] VNPU type: VNPUType.DISABLED
+[INFO] Compiler version: 5.1-patch1-dirty 140e8d4a-dirty
+Init InferenceSession:  50%|████████████████████████████████████████████████████████▌                                                        | 14/28 [00:09<00:09,  1.46it/s][INFO] Using provider: AXCLRTExecutionProvider
+[INFO] SOC Name: AX650N
+[INFO] VNPU type: VNPUType.DISABLED
+[INFO] Compiler version: 5.1-patch1-dirty 140e8d4a-dirty
+Init InferenceSession:  54%|████████████████████████████████████████████████████████████▌                                                    | 15/28 [00:10<00:08,  1.46it/s][INFO] Using provider: AXCLRTExecutionProvider
+[INFO] SOC Name: AX650N
+[INFO] VNPU type: VNPUType.DISABLED
+[INFO] Compiler version: 5.1-patch1-dirty 140e8d4a-dirty
+Init InferenceSession:  57%|████████████████████████████████████████████████████████████████▌                                                | 16/28 [00:11<00:08,  1.45it/s][INFO] Using provider: AXCLRTExecutionProvider
+[INFO] SOC Name: AX650N
+[INFO] VNPU type: VNPUType.DISABLED
+[INFO] Compiler version: 5.1-patch1-dirty 140e8d4a-dirty
+Init InferenceSession:  61%|████████████████████████████████████████████████████████████████████▌                                            | 17/28 [00:12<00:07,  1.45it/s][INFO] Using provider: AXCLRTExecutionProvider
+[INFO] SOC Name: AX650N
+[INFO] VNPU type: VNPUType.DISABLED
+[INFO] Compiler version: 5.1-patch1-dirty 140e8d4a-dirty
+Init InferenceSession:  64%|████████████████████████████████████████████████████████████████████████▋                                        | 18/28 [00:12<00:06,  1.45it/s][INFO] Using provider: AXCLRTExecutionProvider
+[INFO] SOC Name: AX650N
+[INFO] VNPU type: VNPUType.DISABLED
+[INFO] Compiler version: 5.1-patch1-dirty 140e8d4a-dirty
+Init InferenceSession:  68%|████████████████████████████████████████████████████████████████████████████▋                                    | 19/28 [00:13<00:06,  1.45it/s][INFO] Using provider: AXCLRTExecutionProvider
+[INFO] SOC Name: AX650N
+[INFO] VNPU type: VNPUType.DISABLED
+[INFO] Compiler version: 5.1-patch1-dirty 140e8d4a-dirty
+Init InferenceSession:  71%|████████████████████████████████████████████████████████████████████████████████▋                                | 20/28 [00:14<00:05,  1.45it/s][INFO] Using provider: AXCLRTExecutionProvider
+[INFO] SOC Name: AX650N
+[INFO] VNPU type: VNPUType.DISABLED
+[INFO] Compiler version: 5.1-patch1-dirty 140e8d4a-dirty
+Init InferenceSession:  75%|████████████████████████████████████████████████████████████████████████████████████▊                            | 21/28 [00:14<00:04,  1.45it/s][INFO] Using provider: AXCLRTExecutionProvider
+[INFO] SOC Name: AX650N
+[INFO] VNPU type: VNPUType.DISABLED
+[INFO] Compiler version: 5.1-patch1-dirty 140e8d4a-dirty
+Init InferenceSession:  79%|████████████████████████████████████████████████████████████████████████████████████████▊                        | 22/28 [00:15<00:04,  1.46it/s][INFO] Using provider: AXCLRTExecutionProvider
+[INFO] SOC Name: AX650N
+[INFO] VNPU type: VNPUType.DISABLED
+[INFO] Compiler version: 5.1-patch1-dirty 140e8d4a-dirty
+Init InferenceSession:  82%|████████████████████████████████████████████████████████████████████████████████████████████▊                    | 23/28 [00:16<00:03,  1.45it/s][INFO] Using provider: AXCLRTExecutionProvider
+[INFO] SOC Name: AX650N
+[INFO] VNPU type: VNPUType.DISABLED
+[INFO] Compiler version: 5.1-patch1-dirty 140e8d4a-dirty
+Init InferenceSession:  86%|████████████████████████████████████████████████████████████████████████████████████████████████▊                | 24/28 [00:16<00:02,  1.45it/s][INFO] Using provider: AXCLRTExecutionProvider
+[INFO] SOC Name: AX650N
+[INFO] VNPU type: VNPUType.DISABLED
+[INFO] Compiler version: 5.1-patch1-dirty 140e8d4a-dirty
+Init InferenceSession:  89%|████████████████████████████████████████████████████████████████████████████████████████████████████▉            | 25/28 [00:17<00:02,  1.45it/s][INFO] Using provider: AXCLRTExecutionProvider
+[INFO] SOC Name: AX650N
+[INFO] VNPU type: VNPUType.DISABLED
+[INFO] Compiler version: 5.1-patch1-dirty 140e8d4a-dirty
+Init InferenceSession:  93%|████████████████████████████████████████████████████████████████████████████████████████████████████████▉        | 26/28 [00:18<00:01,  1.45it/s][INFO] Using provider: AXCLRTExecutionProvider
+[INFO] SOC Name: AX650N
+[INFO] VNPU type: VNPUType.DISABLED
+[INFO] Compiler version: 5.1-patch1-dirty 140e8d4a-dirty
+Init InferenceSession:  96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▉    | 27/28 [00:18<00:00,  1.45it/s][INFO] Using provider: AXCLRTExecutionProvider
+[INFO] SOC Name: AX650N
+[INFO] VNPU type: VNPUType.DISABLED
+[INFO] Compiler version: 5.1-patch1-dirty 140e8d4a-dirty
+Init InferenceSession: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 28/28 [00:19<00:00,  1.43it/s]
+[INFO] Using provider: AXCLRTExecutionProvider
+[INFO] SOC Name: AX650N
+[INFO] VNPU type: VNPUType.DISABLED
+[INFO] Compiler version: 5.1-patch1-dirty 140e8d4a-dirty
+Model loaded successfully!
+[INFO] Using provider: AXCLRTExecutionProvider
+[INFO] SOC Name: AX650N
+[INFO] VNPU type: VNPUType.DISABLED
+[INFO] Compiler version: 5.1-patch1-dirty 140e8d4a-dirty
+[INFO]: 输入文本进行对话，或者输入图片路径进行图片理解, ���者输入q退出对话。
+prompt<<who are you
+slice_indices: [0]
+Slice prefill done: 0
+answer >> I am an artificial intelligence designed and developed by Apple Inc. I am a natural language processing model that can understand and respond to user input in a conversational manner. I can answer questions, provide information, and engage in discussions on a wide range of topics. I am designed to be helpful, informative, and friendly, and I am constantly learning and improving to provide the best possible experience for users.
+
+prompt<<./images/ssd_horse.jpg
+slice_indices: [0, 1, 2]
+Slice prefill done: 0
+Slice prefill done: 1
+Slice prefill done: 2
+answer >> The image depicts a serene outdoor scene featuring a person riding a brown horse with a white blaze on its face. The rider, who has short brown hair, is wearing a blue hoodie, blue jeans, and black boots. The horse is equipped with a saddle and a bridle, and it stands on a dirt ground.
+
+In the foreground, a brown dog with a pink collar is sitting on the ground, looking up at the rider with its mouth open, possibly in anticipation or excitement.
+
+In the background, there is a silver pickup truck parked near a fence, and beyond the fence, there are trees and a few people sitting on a bench. The sky is overcast, suggesting a cloudy day. The overall atmosphere of the image is calm and peaceful, capturing a moment of connection between the rider, the horse, and the dog.
+
+prompt<<./images/image_1.jpg
+slice_indices: [0, 1, 2]
+Slice prefill done: 0
+Slice prefill done: 1
+Slice prefill done: 2
+answer >> The image depicts a panda bear in a natural setting. The panda is sitting on the ground, surrounded by green bamboo leaves and plants. The panda has a distinctive black and white fur pattern, with black patches around its eyes, ears, and limbs, and a white face and body. The panda appears to be holding a bamboo leaf in its mouth, which is a common food source for pandas. The background includes a wooden structure, possibly a part of a bamboo enclosure, and some rocks. The overall scene suggests that the panda is in a zoo or a wildlife sanctuary.
+
+prompt<<q
+[INFO]: 对话结束，再见。
+```
+![ssd_horse.jpg](./images/ssd_horse.jpg)
+![iamge_1.jpg](./images/image_1.jpg)

fastvlm_ax650_context_1k_prefill_640/image_encoder_1024x1024.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c8cc3fd059d4f3e81b943c729328cd79e8cade9aebfc3bad550a1153e591a062
+size 178882992

fastvlm_ax650_context_1k_prefill_640/llava_qwen2_p128_l0_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:79696a151261bf94bde60edb042f9083c9d975aa48ebba0d616e8362e96e6677
+size 54742139

fastvlm_ax650_context_1k_prefill_640/llava_qwen2_p128_l10_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:218abf9720bbdd58a994ece53839a1afe3328cbbeb526dcc0bff8792cc437fb3
+size 54741955

fastvlm_ax650_context_1k_prefill_640/llava_qwen2_p128_l11_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f56e5a974e959a7a773c5f95cbfb325216028c8d2ffda9e366420f6524e2d109
+size 54741955

fastvlm_ax650_context_1k_prefill_640/llava_qwen2_p128_l12_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:047045d2052d76f156a3992e05aff0d5f06ccedf4b851a447d6c369a97968fe9
+size 54741955

fastvlm_ax650_context_1k_prefill_640/llava_qwen2_p128_l13_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d69e5193c31fc635938166618e0a23e09d518fd6e8b5518e586624a783aedf07
+size 54741955

fastvlm_ax650_context_1k_prefill_640/llava_qwen2_p128_l14_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f339540b2c7bbf7a6657aeb9a7fc5b883497fd166907050daa890788071619e6
+size 54741955

fastvlm_ax650_context_1k_prefill_640/llava_qwen2_p128_l15_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c7b213a799f7b37bc2c3e329e60c981448e0efb2d9391fbf9041471494a9c3d
+size 54741955

fastvlm_ax650_context_1k_prefill_640/llava_qwen2_p128_l16_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4bdca39e1a218791828a45b6c95c42a491bbacec7de5a6d026c25cbf67f46df9
+size 54741955

fastvlm_ax650_context_1k_prefill_640/llava_qwen2_p128_l17_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f64f203200d16325c4d47c09dfa11c1c41e23263b597f7bc463251a8602170bd
+size 54741955

fastvlm_ax650_context_1k_prefill_640/llava_qwen2_p128_l18_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d2c71776cb616b73c03e92dfef1ceb91603ef2791b1706eb3c0cdc268e19fecd
+size 54741955

fastvlm_ax650_context_1k_prefill_640/llava_qwen2_p128_l19_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ce61b20723c850b40b6d40474738d3c57f689dc4688df020d7022c37eca39f8a
+size 54741955

fastvlm_ax650_context_1k_prefill_640/llava_qwen2_p128_l1_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:88cd3291c4ea1acf28edd8dae72f78a668bece58b4567921da1946bb2648b8db
+size 54741947

fastvlm_ax650_context_1k_prefill_640/llava_qwen2_p128_l20_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c1483294d9013d065f24ff958149818cf924d8d644241a220e117ac065d38b4
+size 54741955

fastvlm_ax650_context_1k_prefill_640/llava_qwen2_p128_l21_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c9b8d0fed2dcd88ad150a99c0c487c936d9449878379e8fe8ea6861ea3f63465
+size 54741955

fastvlm_ax650_context_1k_prefill_640/llava_qwen2_p128_l22_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c3647b94d63f9b2792bebfb4c3d952704e40608751c4555c72ea495773e15a86
+size 54741955

fastvlm_ax650_context_1k_prefill_640/llava_qwen2_p128_l23_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be4dbc1650ca7dcd98a9968a56cc246a67246797e13472a54c3e5356ebd8f9f8
+size 54741955

fastvlm_ax650_context_1k_prefill_640/llava_qwen2_p128_l24_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:024a7592c0b6034ab1cca8ec24f57ed789a199de0a0fdf5405a285fa524ab239
+size 54741955

fastvlm_ax650_context_1k_prefill_640/llava_qwen2_p128_l25_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3576ea7ce42236ef8869555fd67dda93a693f853e7dc876fb98ebe75d28b6c11
+size 54741955

fastvlm_ax650_context_1k_prefill_640/llava_qwen2_p128_l26_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4328d9bde2f1306c75d6b8450b544e3a6ae8a6056ff475c6eb7a08f8b8eaedec
+size 54741955

fastvlm_ax650_context_1k_prefill_640/llava_qwen2_p128_l27_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:35e9ecf6f4d4bf88694e55f8cf8811ac7b4a728c91bef123c274d1f96db63c86
+size 54741955

fastvlm_ax650_context_1k_prefill_640/llava_qwen2_p128_l2_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e64be5c1a4a0c23f7c43cbbc6042452901f4f8f373d79fa9b54e92445e54774f
+size 54741947

fastvlm_ax650_context_1k_prefill_640/llava_qwen2_p128_l3_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f09a880164e78740ad72fa083b3caaaa2a905a4f18f695d681e10d3b23c5b235
+size 54741947

fastvlm_ax650_context_1k_prefill_640/llava_qwen2_p128_l4_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3657f3c8591e91a3ddc9b3f3587fa82086c4b30344fee5dc970d184c7ef80369
+size 54741947

fastvlm_ax650_context_1k_prefill_640/llava_qwen2_p128_l5_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ddfd9b0d314c69b544161026bf3486ab83274089b19092bbbf20e5d95b401f07
+size 54741947

fastvlm_ax650_context_1k_prefill_640/llava_qwen2_p128_l6_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c7c471c660e8ff249a8063748a43b6d763c7828d883d903dd1c57f5e32417f7
+size 54741947

fastvlm_ax650_context_1k_prefill_640/llava_qwen2_p128_l7_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d2c164e9c6a01638c376932fcf56e62c61e57c0d5760b09cda3d9c2c45b74a8a
+size 54741947

fastvlm_ax650_context_1k_prefill_640/llava_qwen2_p128_l8_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:60f4f3c8852374019e3e9f7e762bc0da44bfb9f030e75ee3fab005267a2ee786
+size 54741947

fastvlm_ax650_context_1k_prefill_640/llava_qwen2_p128_l9_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:66901cd9f53f6122239a5841c1f282adf25fee8e6cfe8890785e5c267f4b7fd3
+size 54741947

fastvlm_ax650_context_1k_prefill_640/llava_qwen2_post.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6a1e249b71dee1ec8ed185efbf8edb77fea423977927eca4a57801595f28e9dd
+size 254826994

fastvlm_ax650_context_1k_prefill_640/model.embed_tokens.weight.bfloat16.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e01aea64483c11c7de1dbfb8fac8b07860ad669718deb5cf35b8eb71a0dbe593
+size 466747392

fastvlm_ax650_context_1k_prefill_640/model.embed_tokens.weight.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f682683aa248b48679ed39eb816e4066c7c4a9f53f7b94bad2aef079a63a6d3
+size 933494912

fastvlm_tokenizer/added_tokens.json

@@ -0,0 +1,5 @@
+{
+  "<|endoftext|>": 151643,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644
+}

fastvlm_tokenizer/config.json

@@ -0,0 +1,49 @@
+{
+  "_name_or_path": "./llava-v1.5-13b",
+  "architectures": [
+    "LlavaQwen2ForCausalLM"
+  ],
+  "auto_map": {
+    "AutoConfig": "llava_qwen.LlavaConfig",
+    "AutoModelForCausalLM": "llava_qwen.LlavaQwen2ForCausalLM"
+  },  
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "freeze_mm_mlp_adapter": false,
+  "hidden_act": "silu",
+  "hidden_size": 1536,
+  "image_aspect_ratio": "pad",
+  "image_grid_pinpoints": null,
+  "initializer_range": 0.02,
+  "intermediate_size": 8960,
+  "max_position_embeddings": 32768,
+  "max_window_layers": 28,
+  "mm_hidden_size": 3072,
+  "mm_patch_merge_type": "flat",
+  "mm_projector_lr": null,
+  "mm_projector_type": "mlp2x_gelu",
+  "mm_use_im_patch_token": false,
+  "mm_use_im_start_end": false,
+  "mm_vision_select_feature": "patch",
+  "mm_vision_select_layer": -2,
+  "mm_vision_tower": "mobileclip_l_1024",
+  "model_type": "llava_qwen2",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 2,
+  "rms_norm_eps": 1e-06,
+  "rope_theta": 1000000.0,
+  "sliding_window": 32768,
+  "tie_word_embeddings": true,
+  "tokenizer_model_max_length": 8192,
+  "tokenizer_padding_side": "right",
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.39.3",
+  "tune_mm_mlp_adapter": false,
+  "unfreeze_mm_vision_tower": true,
+  "use_cache": true,
+  "use_mm_proj": true,
+  "use_sliding_window": false,
+  "vocab_size": 151936
+}

fastvlm_tokenizer/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "do_sample": true,
+  "temperature": null,
+  "top_p": null,
+  "transformers_version": "4.39.3"
+}

fastvlm_tokenizer/special_tokens_map.json

@@ -0,0 +1,20 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

fastvlm_tokenizer/tokenizer_config.json

@@ -0,0 +1,44 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": null,
+  "chat_template": "{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }}{% endif %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "model_max_length": 8192,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

infer_axmodel.py

@@ -0,0 +1,123 @@
+import os
+import argparse
+
+from PIL import Image
+import numpy as np
+from utils.llava_qwen import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN, DEFAULT_IMAGE_PATCH_TOKEN
+from utils.llava_qwen import tokenizer_image_token, expand2square
+from utils.infer_func import InferManager
+from transformers import AutoTokenizer, AutoConfig, CLIPImageProcessor
+import axengine as ax
+from ml_dtypes import bfloat16
+import argparse
+
+def load_model_and_tokenizer(model_path):
+    tokenizer = AutoTokenizer.from_pretrained(model_path)
+    config = AutoConfig.from_pretrained(model_path)
+
+    mm_use_im_start_end = getattr(config, "mm_use_im_start_end", False)
+    mm_use_im_patch_token = getattr(config, "mm_use_im_patch_token", True)
+    if mm_use_im_patch_token:
+        tokenizer.add_tokens([DEFAULT_IMAGE_PATCH_TOKEN], special_tokens=True)
+    if mm_use_im_start_end:
+        tokenizer.add_tokens([DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN], special_tokens=True)
+
+    return config, tokenizer
+
+def vision_encoder(image_path, ax_session):
+
+    image_processor = CLIPImageProcessor(size={"shortest_edge": 1024},        # CLIP 支持 336x336
+                                         crop_size={"height": 1024, "width": 1024},
+                                         image_mean=[0, 0, 0],
+                                         image_std=[1/255, 1/255, 1/255]
+                                         )
+
+    image = Image.open(image_path).convert('RGB')
+
+    image = expand2square(image, tuple(int(x*255) for x in image_processor.image_mean))
+    input_image = image_processor.preprocess(image, return_tensors='pt')['pixel_values'][0]
+    input_image = input_image.unsqueeze(0) # add batch dimension
+
+    input_image = input_image.numpy().astype(np.uint8).transpose((0, 2, 3, 1))  # NHWC to NCHW
+    vit_output = ax_session.run(None, {"images": input_image})[0]
+
+    return vit_output
+
+def llm_infer(image_features, llm_path, config, tokenizer, imer, get_input):
+
+    embeds = np.load(os.path.join(llm_path, "model.embed_tokens.weight.npy"))
+
+    prompt = "<|im_start|>system\nYou are a helpful assistant, created by apple company.<|im_end|>\n"
+    question = get_input
+    prompt += "<|im_start|>user\n" + question
+
+    if image_features is not None:
+    #     # for idx in range(len(image_features)):
+        prompt += "\n<img>" + "<image>"*256 + "</img>\n"
+    prompt += "<|im_end|>\n<|im_start|>assistant\n"
+
+    token_ids = tokenizer_image_token(prompt, tokenizer, IMAGE_TOKEN_INDEX)
+
+    # 图像理解
+    prefill_data = np.take(embeds, token_ids, axis=0)
+    prefill_data = prefill_data.astype(bfloat16)
+    token_len = len(token_ids)
+
+    if image_features is not None:
+        image_start_index = np.where(np.array(token_ids) == -200)[0][0] # <image> tag 151646
+        image_insert_index = image_start_index + 1
+        prefill_data[image_insert_index : image_insert_index + 256] = image_features[0, :, :]
+
+    eos_token_id = None
+    if isinstance(config.eos_token_id, list) and len(config.eos_token_id) > 1:
+        eos_token_id = config.eos_token_id
+
+    slice_len = 128
+    # prefill_max_len = 640
+    max_seq_len = 1024  # prefill + decode max length
+
+    # imer = InferManager(config, llm_path, max_seq_len=max_seq_len) # prefill + decode max length
+
+    token_ids = imer.prefill(tokenizer, token_ids, prefill_data, slice_len=slice_len)
+    imer.decode(tokenizer, token_ids, embeds, slice_len=slice_len, eos_token_id=eos_token_id)
+    print("\n")
+
+if __name__ == "__main__":
+
+    args = argparse.ArgumentParser()
+    args.add_argument("--vision_model", "-v", type=str, default="./fastvlm_ax650_context_1k_prefill_640/image_encoder_1024x1024.axmodel", help="Path to the vision axmodel.")
+    args.add_argument("--model_path", "-m", type=str, default="./fastvlm_ax650_context_1k_prefill_640", help="Path to the llm axmodel.")
+    args.add_argument("--tokenizer_path", "-t", type=str, default="./fastvlm_tokenizer", help="Path to the tokenizer.")
+    # args.add_argument("--images", type=str, default=None, help="Paths to the input images.")
+    # args.add_argument("--question", type=str, default="介绍一下你自己", help="The question to ask the model.")
+    
+    args = args.parse_args()
+
+    print("Loading config, tokenizer and init model.")
+    config, tokenizer = load_model_and_tokenizer(model_path=args.tokenizer_path)
+
+    slice_len = 128
+    # prefill_max_len = 640
+    max_seq_len = 1024  # prefill + decode max length
+
+    imer = InferManager(config, args.model_path, max_seq_len=max_seq_len) # prefill + decode max length
+    ax_session = ax.InferenceSession(args.vision_model)
+
+    print(f"[INFO]: 输入文本进行对话，或者输入图片路径进行图片理解, 或者输入q退出对话。")
+    while True:
+        prompt = input("prompt<<")
+        if prompt.strip() == "q":
+            print(f"[INFO]: 对话结束，再见。")
+            break
+        else:
+            get_input = prompt.strip()
+            if get_input.lower().endswith(("jpg", "jpeg", "png")):
+                if not os.path.isfile(get_input):
+                    print("[INFO]: 输入错误，请检查图片输入路径。")
+                    continue
+                image_features = vision_encoder(get_input, ax_session)
+                get_input = "Describe the image in detail."
+                llm_infer(image_features, args.model_path, config, tokenizer, imer, get_input)
+            else:
+                image_features = None
+                llm_infer(image_features, args.model_path, config, tokenizer, imer, get_input)

requirements.txt

@@ -0,0 +1,3 @@
+numpy
+tqdm
+transformers