Qwen2-VL-2B-Instruct-ONNX-Q4-F16

.gitattributes

@@ -47,3 +47,5 @@ models/Qwen2-VL-2B-Instruct/onnx/vision_encoder.onnx_data filter=lfs diff=lfs me
 models/Qwen2-VL-2B-Instruct/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 models/Qwen2-VL-2B-Instruct/onnx/vision_encoder_bnb4.onnx_data filter=lfs diff=lfs merge=lfs -text
 models/Qwen2-VL-2B-Instruct/onnx/vision_encoder_q4.onnx_data filter=lfs diff=lfs merge=lfs -text
+models/Qwen2-VL-2B-Instruct-ONNX-Q4-F16/onnx/QwenVL_A.onnx.data filter=lfs diff=lfs merge=lfs -text
+models/Qwen2-VL-2B-Instruct-ONNX-Q4-F16/onnx/QwenVL_E.onnx.data filter=lfs diff=lfs merge=lfs -text

models/Qwen2-VL-2B-Instruct-ONNX-Q4-F16/.gitattributes

@@ -0,0 +1,38 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.onnx.data filter=lfs diff=lfs merge=lfs -text
+onnx/**/* filter=lfs diff=lfs merge=lfs -text
+**/.git* filter= diff= merge= text

models/Qwen2-VL-2B-Instruct-ONNX-Q4-F16/.gitignore

@@ -0,0 +1,2 @@
+.DS_STORE
+/onnx-dest

models/Qwen2-VL-2B-Instruct-ONNX-Q4-F16/EXPORT.md

@@ -0,0 +1,57 @@
+# Export
+
+The original model was exported using the following process:
+
+The following repos were used:
+* https://github.com/pdufour/Native-LLM-for-Android
+* https://github.com/pdufour/transformers.js/tree/add-block-list
+
+If you close this repo and the above 2 to the same directory you can run the following commands:
+
+**From `Qwen2-VL-2B-Instruct-ONNX-Q4-F16`, run:**
+
+`make all-in-one`
+
+This will create an export of the onnx models.
+
+**The following is a list of all commands available:**
+
+**all-in-one**
+
+Runs all steps (exporting, slimming, quantizing, cleaning, fixing GPU buffers) to produce fully prepared ONNX models.
+
+**export**
+
+Combines export-abcd and export-e to generate ONNX models for all parts.
+
+**export-abcd**
+
+Exports model parts A, B, C, and D by running QwenVL_Export_ABCD.py.
+
+**export-e**
+
+Exports model part E by running QwenVL_Export_E.py.
+
+**slim**
+
+Reduces ONNX model size by removing unnecessary elements for optimized deployment.
+
+**quantize**
+
+Quantizes all model parts (A, B, C, D, and E) to optimize size and performance.
+
+**quantize-%**
+
+Quantizes a specific model part (% can be A, B, C, D, or E) with targeted configurations.
+
+**clean-large-files**
+
+Deletes ONNX files larger than 2GB from the destination directory to retain models that will work for onnx environments.
+
+**fix-gpu-buffers**
+
+Applies fixes to GPU buffers in ONNX files for part E to ensure GPU memory compatibility.
+
+**all**
+
+Alias for all-in-one to run the full ONNX model preparation pipeline.

models/Qwen2-VL-2B-Instruct-ONNX-Q4-F16/Makefile

@@ -0,0 +1,164 @@
+SHELL := /bin/bash
+
+# Configuration variables
+NATIVE_ANDROID = $(abspath ../Native-LLM-for-Android)
+QWEN_VL_DIR = $(NATIVE_ANDROID)/Export_ONNX/QwenVL
+ONNX_SRC_DIR = $(QWEN_VL_DIR)/onnx
+ONNX_DEST_DIR = $(QWEN_VL_DIR)/onnx-dist
+STAGING_DIR = /tmp/transformers.js/staging
+TRANSFORMERS_JS_PATH = ../transformers.js
+ONNX_TOOLS_PATH = $(NATIVE_ANDROID)/ONNX_Tools
+
+# Python paths from venvs
+NATIVE_PYTHON = $(NATIVE_ANDROID)/.venv/bin/python3
+TRANSFORMERS_PYTHON = $(TRANSFORMERS_JS_PATH)/.venv/bin/python3
+
+# Model parts
+PARTS = A B C D E
+
+define progress_bar
+	printf "\r   Progress: \033[1;32m["; \
+	_done=$$(($1 * 20 / $2)); \
+	for ((i=0; i<_done; i++)); do printf "="; done; \
+	printf "\033[0m"; \
+	_left=$$((20 - _done)); \
+	for ((i=0; i<_left; i++)); do printf " "; done; \
+	printf "\033[1;32m]\033[0m $1/$2  Processing: \033[1;34m%s\033[K\033[0m\r" "$3"
+endef
+
+# See https://github.com/pytorch/pytorch/issues/94280#issuecomment-2089196400
+# Original export scripts export a bunch of tensor files, so we merge into one / two files instead.
+export-merged-source-models: export-merged-source-models-first-pass export-merged-source-models-second-pass
+	@echo "✅ Exporting merged source models complete"
+
+export-merged-source-models-first-pass:
+	@echo "💾 First pass: Export all models with merged tensors..."
+	@mkdir -p $(ONNX_DEST_DIR)
+	@files=`find $(ONNX_SRC_DIR) -name "*.onnx"`; \
+	total=`echo "$$files" | wc -w | tr -d ' '`; \
+	echo "Files found (first pass): $$total"; \
+	current=0; \
+	for item in $$files; do \
+	current=$$((current + 1)); \
+	$(call progress_bar,$$current,$$total,$$item); \
+	$(NATIVE_PYTHON) -u -c "import onnx, os, sys; src='$$item'; dest_dir='$(ONNX_DEST_DIR)'; \
+		m = onnx.load(src); \
+		d = os.path.join(dest_dir, os.path.basename(src)); \
+		onnx.save_model(m, d, all_tensors_to_one_file=True, save_as_external_data=True, location=os.path.basename(d)+'.data')" || exit 1; \
+	done; \
+	echo "✅ Done first pass"
+
+export-merged-source-models-second-pass:
+	@echo "💾 Second pass: Converting large models to external data format..."
+	@files=`find $(ONNX_DEST_DIR) -name "*.onnx"`; \
+	total=`echo "$$files" | wc -w | tr -d ' '`; \
+	echo "Files found (second pass): $$total"; \
+	current=0; \
+	for item in $$files; do \
+		current=$$((current + 1)); \
+		$(call progress_bar,$$current,$$total,$$item); \
+		$(NATIVE_PYTHON) -c 'import onnx, os, sys; \
+			src = """'"$$item"'"""; \
+			total_size = os.path.getsize(src); \
+			d = os.path.join(dest_dir, os.path.basename(src)); \
+			total_size += os.path.getsize(src + ".data") if os.path.exists(src + ".data") else 0; \
+			needs_external = total_size > 2e9; \
+			onnx.save_model( \
+				onnx.load(src), \
+				d, \
+				save_as_external_data=needs_external, \
+				all_tensors_to_one_file=True, \
+				location=(os.path.basename(src) + ".data") if needs_external else None \
+			); \
+			not needs_external and os.path.exists(src + ".data") and os.remove(src + ".data") \
+			' || exit 1; \
+	done; \
+	echo "✅ Done second models"
+
+
+all-in-one: export quantize clean-large-files fix-gpu-buffers export-merged-source-models
+	@echo "✨ All done! ONNX models exported, slimmed, quantized and fixed"
+
+export: export-abcd export-e
+	@echo "✅ Export complete"
+
+export-abcd:
+	@echo "🚀 Exporting parts A, B, C, D..."
+	cd ../Native-LLM-for-Android/Export_ONNX/QwenVL && \
+	$(NATIVE_PYTHON) QwenVL_Export_ABCD.py "Qwen/Qwen2-VL-2B-Instruct"
+
+export-e:
+	@echo "🚀 Exporting part E..."
+	cd ../Native-LLM-for-Android/Export_ONNX/QwenVL && \
+	$(NATIVE_PYTHON) QwenVL_Export_E.py "Qwen/Qwen2-VL-2B-Instruct"
+
+slim:
+	@echo "🗜️  Slimming ONNX models..."
+	@files=`find $(ONNX_SRC_DIR) -name "*.onnx" -type f ! -name "QwenVL_E.onnx"`; \
+	total=`echo "$$files" | wc -w | tr -d ' '`; \
+	echo "Files found: $$total"; \
+	current=0; \
+	for item in $$files; do \
+		current=$$((current + 1)); \
+		$(call progress_bar,$$current,$$total,$$item); \
+		onnxslim --verbose "$$item" "$$item" || exit 1; \
+	done; \
+	echo "✅ Slimming complete"
+
+quantize:
+	@echo "⚡ Starting quantization..."
+	for part in $(PARTS); do \
+		$(MAKE) quantize-$$part || exit 1; \
+	done
+	@echo "✅ Quantization complete"
+
+quantize-%:
+	@echo "⚡ Quantizing part $*..."
+	mkdir -p $(ONNX_DEST_DIR)
+	cd $(TRANSFORMERS_JS_PATH) && \
+	mkdir -p $(STAGING_DIR) && \
+	rm -f $(STAGING_DIR)/* && \
+	ln -sf $$(realpath $(ONNX_SRC_DIR))/* $(STAGING_DIR)/ && \
+	find $(STAGING_DIR) -name "*_*_*.onnx_data" -delete && \
+	find $(STAGING_DIR) -name "*_*_*.onnx" -delete && \
+	find $(STAGING_DIR) -name "*.onnx" ! -name "QwenVL_$**.onnx" -delete && \
+	EXTRA_FLAGS=""; \
+	if [ "$*" = "A" ]; then EXTRA_FLAGS="--op_block_list Conv DynamicQuantizeLinear DequantizeLinear Resize"; fi; \
+	echo "Extra Flags for part $*: $$EXTRA_FLAGS" && \
+	PYTHONPATH=$(TRANSFORMERS_JS_PATH) .venv/bin/python3 -m scripts.quantize \
+		--input_folder '$(STAGING_DIR)' \
+		--output_folder '$(ONNX_DEST_DIR)' \
+		--mode q4f16 $$EXTRA_FLAGS
+
+clean-large-files:
+	@echo "🧹 Removing ONNX files over 2GB..."
+	cd $(ONNX_DEST_DIR) && \
+	for f in $$(find . -name "*.onnx" -type f); do \
+		total_size=0; \
+		if [ -f "$$f"".data" ]; then \
+			total_size=$$(( $$(stat -f %z "$$f") + $$(stat -f %z "$$f"".data") )); \
+		elif [ -f "$$f""_data" ]; then \
+			total_size=$$(( $$(stat -f %z "$$f") + $$(stat -f %z "$$f""_data") )); \
+		else \
+			total_size=$$(stat -f %z "$$f"); \
+		fi; \
+		size_mb=$$(( total_size / 1048576 )); \
+		if [ $$total_size -ge 2147483648 ]; then \
+			echo "   Removing $$f (size: $$size_mb MB)..."; \
+			rm -f "$$f" "$$f"".data" "$$f""_data"; \
+		fi \
+	done
+	@echo "✅ Large file cleanup complete"
+
+fix-gpu-buffers:
+	@echo "🔧 Fixing GPU buffers for E models..."
+	@files=`find $(ONNX_DEST_DIR) -name "QwenVL_E_*.onnx" -type f`; \
+	total=`echo "$$files" | wc -w | tr -d ' '`; \
+	echo "Files found: $$total"; \
+	current=0; \
+	for item in $$files; do \
+		current=$$((current + 1)); \
+		$(call progress_bar,$$current,$$total,$$item); \
+		cd $(NATIVE_ANDROID) && .venv/bin/python3 ONNX_Tools/clamp_for_gpu_buffers.py --overwrite "$$item" || exit 1; \
+	done; \
+	echo "✅ GPU buffer fixes complete"

models/Qwen2-VL-2B-Instruct-ONNX-Q4-F16/README.md

@@ -0,0 +1,146 @@
+---
+license: apache-2.0
+base_model:
+- Qwen/Qwen2-VL-2B-Instruct
+---
+# Requirements
+This is compatible with any onnx runtime.
+
+# Running this model
+
+**Javascript**
+
+See https://huggingface.co/spaces/pdufour/Qwen2-VL-2B-Instruct-ONNX-Q4-F16 for a demo.
+
+
+**Python**
+
+Download the following script ./infer.py and then run like so:
+python3 infer.py Qwen/Qwen2-VL-2B-Instruct 'path-to/Qwen2-VL-2B-Instruct-onnx/onnx'
+
+```
+import os
+import sys
+import time
+import torch
+import numpy as np
+import requests
+import onnxruntime as ort
+from PIL import Image
+from io import BytesIO
+from transformers import Qwen2VLConfig, AutoTokenizer
+
+# Command line arguments
+model_path = sys.argv[1]
+onnx_path = sys.argv[2]
+
+# Initialize model config and tokenizer
+model_config = Qwen2VLConfig.from_pretrained(model_path)
+tokenizer = AutoTokenizer.from_pretrained(model_path)
+
+# Model configuration
+max_length = 1024
+num_attention_heads = model_config.num_attention_heads
+num_key_value_heads = model_config.num_key_value_heads
+head_dim = model_config.hidden_size // num_attention_heads
+num_layers = model_config.num_hidden_layers
+
+# Setup ONNX sessions
+session_options = ort.SessionOptions()
+session_options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
+
+# Model paths and sessions
+models = ['A', 'B', 'C', 'D', 'E']
+model_paths = {m: os.path.join(onnx_path, f'QwenVL_{m}_q4f16.onnx') for m in models}
+sessions = {m: ort.InferenceSession(path, sess_options=session_options) for m, path in model_paths.items()}
+
+# Input/output names
+inputs = {
+    'A': sessions['A'].get_inputs()[0].name,
+    'B': [sessions['B'].get_inputs()[i].name for i in range(2)],
+    'C': sessions['C'].get_inputs()[0].name,
+    'D': [inp.name for inp in sessions['D'].get_inputs()],
+    'E': [inp.name for inp in sessions['E'].get_inputs()]
+}
+
+outputs = {
+    'A': sessions['A'].get_outputs()[0].name,
+    'B': sessions['B'].get_outputs()[0].name,
+    'C': sessions['C'].get_outputs()[0].name,
+    'D': [out.name for out in sessions['D'].get_outputs()],
+    'E': [out.name for out in sessions['E'].get_outputs()]
+}
+
+# Process image
+image_url = 'https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/car.jpg'
+image = Image.open(BytesIO(requests.get(image_url).content)).resize((960, 960)).convert('RGB')
+image_array = np.expand_dims(np.transpose(np.array(image).astype(np.float32), (2, 0, 1)), axis=0) / 255.
+
+# Prepare inputs
+prompt = "Describe this image."
+formatted_prompt = f"\n<|im_start|>user\n<|vision_start|><|vision_end|>{prompt}<|im_end|>\n<|im_start|>assistant\n"
+input_ids = tokenizer(formatted_prompt, return_tensors='pt')['input_ids']
+input_lengths = np.array([input_ids.shape[1]], dtype=np.int64)
+tokens = np.zeros(max_length, dtype=np.int32)
+tokens[:input_ids.shape[1]] = input_ids[0, :]
+position = np.zeros(1, dtype=np.int64)
+
+# Initialize caches
+key_cache = np.zeros((num_layers, num_key_value_heads, max_length, head_dim), dtype=np.float16)
+value_cache = key_cache.copy()
+
+# Process initial inputs
+hidden_states = sessions['B'].run(
+    [outputs['B']],
+    {inputs['B'][0]: tokens, inputs['B'][1]: input_lengths}
+)[0]
+
+batch_size = np.array(0, dtype=np.int32)
+batch_size, = sessions['C'].run([outputs['C']], {inputs['C']: batch_size})
+
+# Process image features
+image_features = sessions['A'].run([outputs['A']], {inputs['A']: image_array})[0]
+total_ids = 100  # 10 * 10 from original factors
+input_lengths += total_ids
+remaining_tokens = np.array(max_length - input_lengths[0] - total_ids, dtype=np.int32)
+tokens_to_stop = np.array(input_lengths[0] - 5, dtype=np.int32)
+
+hidden_states, batch_size = sessions['D'].run(
+    outputs['D'],
+    dict(zip(inputs['D'],
+        [hidden_states, image_features, input_lengths, tokens_to_stop, remaining_tokens]))
+)
+
+# Generate tokens
+start_time = time.time()
+for i in range(12):  # MAX_ITERATIONS
+    token, key_cache, value_cache = sessions['E'].run(
+        outputs['E'],
+        dict(zip(inputs['E'],
+            [hidden_states, np.array([-65504. if i==0 else 0.], dtype=np.float16),
+             key_cache, value_cache, position, input_lengths, batch_size,
+             np.array([1-total_ids+10 if i==0 else position[0]+1], dtype=np.float16)]))
+    )
+
+    if token in [151643, 151645]:  # End tokens
+        break
+
+    if i < 1:
+        position += input_lengths[0]
+        input_lengths[0] = 1
+    else:
+        position += 1
+
+    tokens[0] = token
+    hidden_states = sessions['B'].run(
+        [outputs['B']],
+        {inputs['B'][0]: tokens, inputs['B'][1]: input_lengths}
+    )[0]
+    print(tokenizer.decode(token), end='', flush=True)
+
+print(f"\nTotal time: {time.time() - start_time:.2f}s")
+
+```
+
+# Technical Information:
+- [EXPORT.md](EXPORT.md)

models/Qwen2-VL-2B-Instruct-ONNX-Q4-F16/config.json

@@ -0,0 +1,52 @@
+{
+  "architectures": [
+    "Qwen2VLForConditionalGeneration"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "vision_start_token_id": 151652,
+  "vision_end_token_id": 151653,
+  "vision_token_id": 151654,
+  "image_token_id": 151655,
+  "video_token_id": 151656,
+  "hidden_act": "silu",
+  "hidden_size": 1536,
+  "initializer_range": 0.02,
+  "intermediate_size": 8960,
+  "max_position_embeddings": 32768,
+  "max_window_layers": 28,
+  "model_type": "qwen2_vl",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 2,
+  "rms_norm_eps": 1e-06,
+  "rope_theta": 1000000.0,
+  "sliding_window": 32768,
+  "tie_word_embeddings": true,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.41.2",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vision_config": {
+    "depth": 32,
+    "embed_dim": 1280,
+    "mlp_ratio": 4,
+    "num_heads": 16,
+    "in_chans": 3,
+    "hidden_size": 1536,
+    "patch_size": 14,
+    "spatial_merge_size": 2,
+    "spatial_patch_size": 14,
+    "temporal_patch_size": 2
+  },
+  "rope_scaling": {
+    "type": "mrope",
+    "mrope_section": [
+      16,
+      24,
+      24
+    ]
+  },
+  "vocab_size": 151936
+}

models/Qwen2-VL-2B-Instruct-ONNX-Q4-F16/embeddings_bf16.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5387220a9d57908c2c6fa69bcfa64fda4234e59103fa74f56d07eaa6f9af2493
+size 466747392

models/Qwen2-VL-2B-Instruct-ONNX-Q4-F16/generation_config.json

@@ -0,0 +1,15 @@
+{
+    "bos_token_id": 151643,
+    "pad_token_id": 151643,
+    "do_sample": true,
+    "eos_token_id": [
+      151645,
+      151643
+    ],
+    "repetition_penalty": 1.0,
+    "temperature": 0.01,
+    "top_p": 0.001,
+    "top_k": 1,
+    "transformers_version": "4.37.0"
+  }
+  

models/Qwen2-VL-2B-Instruct-ONNX-Q4-F16/infer.py

@@ -0,0 +1,120 @@
+import os
+import sys
+import time
+import torch
+import numpy as np
+import requests
+import onnxruntime as ort
+from PIL import Image
+from io import BytesIO
+from transformers import Qwen2VLConfig, AutoTokenizer
+
+# Command line arguments
+model_path = sys.argv[1]
+onnx_path = sys.argv[2]
+
+# Initialize model config and tokenizer
+model_config = Qwen2VLConfig.from_pretrained(model_path)
+tokenizer = AutoTokenizer.from_pretrained(model_path)
+
+# Model configuration
+max_length = 1024
+num_attention_heads = model_config.num_attention_heads
+num_key_value_heads = model_config.num_key_value_heads
+head_dim = model_config.hidden_size // num_attention_heads
+num_layers = model_config.num_hidden_layers
+
+# Setup ONNX sessions
+session_options = ort.SessionOptions()
+session_options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
+
+# Model paths and sessions
+models = ['A', 'B', 'C', 'D', 'E']
+model_paths = {m: os.path.join(onnx_path, f'QwenVL_{m}_q4f16.onnx') for m in models}
+sessions = {m: ort.InferenceSession(path, sess_options=session_options) for m, path in model_paths.items()}
+
+# Input/output names
+inputs = {
+    'A': sessions['A'].get_inputs()[0].name,
+    'B': [sessions['B'].get_inputs()[i].name for i in range(2)],
+    'C': sessions['C'].get_inputs()[0].name,
+    'D': [inp.name for inp in sessions['D'].get_inputs()],
+    'E': [inp.name for inp in sessions['E'].get_inputs()]
+}
+
+outputs = {
+    'A': sessions['A'].get_outputs()[0].name,
+    'B': sessions['B'].get_outputs()[0].name,
+    'C': sessions['C'].get_outputs()[0].name,
+    'D': [out.name for out in sessions['D'].get_outputs()],
+    'E': [out.name for out in sessions['E'].get_outputs()]
+}
+
+# Process image
+image_url = 'https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/car.jpg'
+image = Image.open(BytesIO(requests.get(image_url).content)).resize((960, 960)).convert('RGB')
+image_array = np.expand_dims(np.transpose(np.array(image).astype(np.float32), (2, 0, 1)), axis=0) / 255.
+
+# Prepare inputs
+prompt = "Describe this image."
+formatted_prompt = f"\n<|im_start|>user\n<|vision_start|><|vision_end|>{prompt}<|im_end|>\n<|im_start|>assistant\n"
+input_ids = tokenizer(formatted_prompt, return_tensors='pt')['input_ids']
+input_lengths = np.array([input_ids.shape[1]], dtype=np.int64)
+tokens = np.zeros(max_length, dtype=np.int32)
+tokens[:input_ids.shape[1]] = input_ids[0, :]
+position = np.zeros(1, dtype=np.int64)
+
+# Initialize caches
+key_cache = np.zeros((num_layers, num_key_value_heads, max_length, head_dim), dtype=np.float16)
+value_cache = key_cache.copy()
+
+# Process initial inputs
+hidden_states = sessions['B'].run(
+    [outputs['B']],
+    {inputs['B'][0]: tokens, inputs['B'][1]: input_lengths}
+)[0]
+
+batch_size = np.array(0, dtype=np.int32)
+batch_size, = sessions['C'].run([outputs['C']], {inputs['C']: batch_size})
+
+# Process image features
+image_features = sessions['A'].run([outputs['A']], {inputs['A']: image_array})[0]
+total_ids = 100  # 10 * 10 from original factors
+input_lengths += total_ids
+remaining_tokens = np.array(max_length - input_lengths[0] - total_ids, dtype=np.int32)
+tokens_to_stop = np.array(input_lengths[0] - 5, dtype=np.int32)
+
+hidden_states, batch_size = sessions['D'].run(
+    outputs['D'],
+    dict(zip(inputs['D'],
+        [hidden_states, image_features, input_lengths, tokens_to_stop, remaining_tokens]))
+)
+
+# Generate tokens
+start_time = time.time()
+for i in range(12):  # MAX_ITERATIONS
+    token, key_cache, value_cache = sessions['E'].run(
+        outputs['E'],
+        dict(zip(inputs['E'],
+            [hidden_states, np.array([-65504. if i==0 else 0.], dtype=np.float16),
+             key_cache, value_cache, position, input_lengths, batch_size,
+             np.array([1-total_ids+10 if i==0 else position[0]+1], dtype=np.float16)]))
+    )
+
+    if token in [151643, 151645]:  # End tokens
+        break
+
+    if i < 1:
+        position += input_lengths[0]
+        input_lengths[0] = 1
+    else:
+        position += 1
+
+    tokens[0] = token
+    hidden_states = sessions['B'].run(
+        [outputs['B']],
+        {inputs['B'][0]: tokens, inputs['B'][1]: input_lengths}
+    )[0]
+    print(tokenizer.decode(token), end='', flush=True)
+
+print(f"\nTotal time: {time.time() - start_time:.2f}s")

models/Qwen2-VL-2B-Instruct-ONNX-Q4-F16/llm_config.json

@@ -0,0 +1,28 @@
+{
+    "hidden_size": 1536,
+    "layer_nums": 28,
+    "attention_mask": "float",
+    "key_value_shape": [
+        2,
+        1,
+        0,
+        2,
+        128
+    ],
+    "prompt_template": "<|im_start|>user\n%s<|im_end|>\n<|im_start|>assistant\n",
+    "is_visual": true,
+    "image_mean": [
+        122.7709383,
+        116.7460125,
+        104.09373615000001
+    ],
+    "image_norm": [
+        0.01459842661924292,
+        0.015007768493717056,
+        0.014220065717024088
+    ],
+    "image_size": 420,
+    "vision_start": 151652,
+    "vision_end": 151653,
+    "image_pad": 151655
+}

models/Qwen2-VL-2B-Instruct-ONNX-Q4-F16/onnx/QwenVL_A.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9359181d8a217fd066b6201ca88d39ceef8d84464e886fa3af3634b767807967
+size 22863481

models/Qwen2-VL-2B-Instruct-ONNX-Q4-F16/onnx/QwenVL_A.onnx.data

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:48c2e8d0ebb88762b324860ca74abd35d4848b08f84619e71acc5122a0e46c8f
+size 5322170368

models/Qwen2-VL-2B-Instruct-ONNX-Q4-F16/onnx/QwenVL_A_q4f16.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fdd6e4c85b5a835227106c01b31be8220eb4684026e726372a97c74cfdbcd983
+size 1330987067

models/Qwen2-VL-2B-Instruct-ONNX-Q4-F16/onnx/QwenVL_B.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b752394955396a0684cb491ebf802645ad6e73a29f4f2392c6bfd77759d7d86
+size 234019162

models/Qwen2-VL-2B-Instruct-ONNX-Q4-F16/onnx/QwenVL_B_q4f16.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2c5981ece4d144bc7f5352e56bb19d0d4b3bf22d1f8c472a106fcdcf83a9ebdf
+size 233983290

models/Qwen2-VL-2B-Instruct-ONNX-Q4-F16/onnx/QwenVL_C.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:09090f067d75cbfb62f90fc1f783529ede85e07006da80681fbb6f535baa29d6
+size 10335

models/Qwen2-VL-2B-Instruct-ONNX-Q4-F16/onnx/QwenVL_C_q4f16.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc4f5f601f5ac0b16632e4dc953ce7009f8c2bf0c5e5c1553b5250cda832a68a
+size 6364

models/Qwen2-VL-2B-Instruct-ONNX-Q4-F16/onnx/QwenVL_D.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f4393146a8d328f1eae43e9058f391a1ef07048d6793747dab948838fcdfd1e6
+size 26762

models/Qwen2-VL-2B-Instruct-ONNX-Q4-F16/onnx/QwenVL_D_q4f16.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a5b46671d4d41a864d1390ef87eb5819e8c6fd044cded45e688301ae8eb7ab57
+size 25118

models/Qwen2-VL-2B-Instruct-ONNX-Q4-F16/onnx/QwenVL_E.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b602930000f109874f028142d62fc488908d65e30be235565efa310d3d32c89
+size 1505816

models/Qwen2-VL-2B-Instruct-ONNX-Q4-F16/onnx/QwenVL_E.onnx.data

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:691badb0cb9420b85d6483c5cf529af29cf8e864f8214f27b4412de3b5ab3097
+size 12349714432

models/Qwen2-VL-2B-Instruct-ONNX-Q4-F16/onnx/QwenVL_E_q4f16.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:599c81da0035adf98d77db9b5776e7070017887394d06dd901c4d72125f6fd2b
+size 996827324

models/Qwen2-VL-2B-Instruct-ONNX-Q4-F16/source.txt

@@ -0,0 +1 @@
+https://huggingface.co/pdufour/Qwen2-VL-2B-Instruct-ONNX-Q4-F16

models/Qwen2-VL-2B-Instruct-ONNX-Q4-F16/tokenizer_config.json

@@ -0,0 +1,129 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": ["<|im_start|>", "<|im_end|>", "<|object_ref_start|>","<|object_ref_end|>","<|box_start|>","<|box_end|>","<|quad_start|>","<|quad_end|>","<|vision_start|>","<|vision_end|>","<|vision_pad|>","<|image_pad|>","<|video_pad|>"],
+  "bos_token": null,
+  "chat_template": "{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n{% endif %}<|im_start|>{{ message['role'] }}\n{% if message['content'] is string %}{{ message['content'] }}<|im_end|>\n{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>\n{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "padding_side": "left",
+  "errors": "replace",
+  "model_max_length": 32768,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}