Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,47 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+assert/ocr.jpg filter=lfs diff=lfs merge=lfs -text
+assert/person.jpg filter=lfs diff=lfs merge=lfs -text
+assert/phrase_grounding.jpg filter=lfs diff=lfs merge=lfs -text
+image_encoder_mlp.axmodel filter=lfs diff=lfs merge=lfs -text
+qwen2.5_tokenizer/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+qwen2_p128_l0_together.axmodel filter=lfs diff=lfs merge=lfs -text
+qwen2_p128_l10_together.axmodel filter=lfs diff=lfs merge=lfs -text
+qwen2_p128_l11_together.axmodel filter=lfs diff=lfs merge=lfs -text
+qwen2_p128_l12_together.axmodel filter=lfs diff=lfs merge=lfs -text
+qwen2_p128_l13_together.axmodel filter=lfs diff=lfs merge=lfs -text
+qwen2_p128_l14_together.axmodel filter=lfs diff=lfs merge=lfs -text
+qwen2_p128_l15_together.axmodel filter=lfs diff=lfs merge=lfs -text
+qwen2_p128_l16_together.axmodel filter=lfs diff=lfs merge=lfs -text
+qwen2_p128_l17_together.axmodel filter=lfs diff=lfs merge=lfs -text
+qwen2_p128_l18_together.axmodel filter=lfs diff=lfs merge=lfs -text
+qwen2_p128_l19_together.axmodel filter=lfs diff=lfs merge=lfs -text
+qwen2_p128_l1_together.axmodel filter=lfs diff=lfs merge=lfs -text
+qwen2_p128_l20_together.axmodel filter=lfs diff=lfs merge=lfs -text
+qwen2_p128_l21_together.axmodel filter=lfs diff=lfs merge=lfs -text
+qwen2_p128_l22_together.axmodel filter=lfs diff=lfs merge=lfs -text
+qwen2_p128_l23_together.axmodel filter=lfs diff=lfs merge=lfs -text
+qwen2_p128_l24_together.axmodel filter=lfs diff=lfs merge=lfs -text
+qwen2_p128_l25_together.axmodel filter=lfs diff=lfs merge=lfs -text
+qwen2_p128_l26_together.axmodel filter=lfs diff=lfs merge=lfs -text
+qwen2_p128_l27_together.axmodel filter=lfs diff=lfs merge=lfs -text
+qwen2_p128_l28_together.axmodel filter=lfs diff=lfs merge=lfs -text
+qwen2_p128_l29_together.axmodel filter=lfs diff=lfs merge=lfs -text
+qwen2_p128_l2_together.axmodel filter=lfs diff=lfs merge=lfs -text
+qwen2_p128_l30_together.axmodel filter=lfs diff=lfs merge=lfs -text
+qwen2_p128_l31_together.axmodel filter=lfs diff=lfs merge=lfs -text
+qwen2_p128_l32_together.axmodel filter=lfs diff=lfs merge=lfs -text
+qwen2_p128_l33_together.axmodel filter=lfs diff=lfs merge=lfs -text
+qwen2_p128_l34_together.axmodel filter=lfs diff=lfs merge=lfs -text
+qwen2_p128_l35_together.axmodel filter=lfs diff=lfs merge=lfs -text
+qwen2_p128_l3_together.axmodel filter=lfs diff=lfs merge=lfs -text
+qwen2_p128_l4_together.axmodel filter=lfs diff=lfs merge=lfs -text
+qwen2_p128_l5_together.axmodel filter=lfs diff=lfs merge=lfs -text
+qwen2_p128_l6_together.axmodel filter=lfs diff=lfs merge=lfs -text
+qwen2_p128_l7_together.axmodel filter=lfs diff=lfs merge=lfs -text
+qwen2_p128_l8_together.axmodel filter=lfs diff=lfs merge=lfs -text
+qwen2_p128_l9_together.axmodel filter=lfs diff=lfs merge=lfs -text
+qwen2_post.axmodel filter=lfs diff=lfs merge=lfs -text
+test_data/ocr.jpg filter=lfs diff=lfs merge=lfs -text
+test_data/person.jpg filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,3 +1,167 @@
 ---
-license: bsd-3-clause
+license: mit
+language:
+- en
+- zh
+base_model:
+- nvidia/LocateAnything-3B
+pipeline_tag: detection, VLM
+library_name: transformers
+tags:
+- LocateAnything-3B
+- Int4
+- VLM
+- GPTQ
 ---
+
+# LocateAnything-3B
+
+This version of LocateAnything-3B have been converted to run on the Axera NPU using **w4a16** quantization. 
+
+Compatible with Pulsar2 version: 6.0
+
+## Convert tools links:
+
+For those who are interested in model conversion, you can try to export axmodel through the original repo : 
+
+- https://huggingface.co/nvidia/LocateAnything-3B
+
+[Pulsar2 Link, How to Convert LLM from Huggingface to axmodel](https://pulsar2-docs.readthedocs.io/en/latest/appendix/build_llm.html) 
+
+[AXera NPU HOST LLM Runtime](TODO) 
+
+
+## Support Platform
+
+- AX650
+  - AX650N DEMO Board
+  - [M4N-Dock(爱芯派Pro)](https://wiki.sipeed.com/hardware/zh/maixIV/m4ndock/m4ndock.html)
+  - [M.2 Accelerator card](https://docs.m5stack.com/zh_CN/ai_hardware/LLM-8850_Card)
+
+**Image Process**
+|Chips| input size | image num | image encoder | ttft(493 tokens) | w4a16 | CMM | Flash |
+|--|--|--|--|--|--|--|--|
+|AX650| 560*560 | 1 | 1152.583 ms | 2072.06 ms | 10.61 tokens/sec| 2.9GiB | 3.2GiB |
+
+The DDR capacity refers to the CMM memory that needs to be consumed. Ensure that the CMM memory allocation on the development board is greater than this value.
+
+## How to use
+
+## 安装 axllm
+方式一：克隆仓库后执行安装脚本：
+
+```shell
+git clone -b axllm https://github.com/AXERA-TECH/ax-llm.git
+cd ax-llm
+./install.sh
+```
+
+方式二：一行命令安装（默认分支 `axllm`）：
+
+```shell
+curl -fsSL https://raw.githubusercontent.com/AXERA-TECH/ax-llm/axllm/install.sh | bash
+```
+
+方式三：下载Github Actions CI 导出的可执行程序（适合没有编译环境的用户）：
+
+如果没有编译环境，请到：
+`https://github.com/AXERA-TECH/ax-llm/actions?query=branch%3Aaxllm`
+下载 **最新 CI 导出的可执行程序**（`axllm`），然后：
+
+```shell
+chmod +x axllm
+sudo mv axllm /usr/bin/axllm
+```
+
+## 模型下载（Hugging Face）
+先创建模型目录并进入，然后下载到该目录：
+
+```shell
+mkdir -p AXERA-TECH/LocateAnything-3B
+cd AXERA-TECH/LocateAnything-3B
+hf download AXERA-TECH/LocateAnything-3B --local-dir .
+
+# structure of the downloaded files
+tree -L 3
+.
+└── AXERA-TECH
+    └── LocateAnything-3B
+        |-- assert
+        |-- config.json
+        |-- gradio_locateanything_axengine.py
+        |-- image_encoder_mlp.axmodel
+        |-- infer_locateanything_axengine.py
+        |-- model.embed_tokens.weight.bfloat16.bin
+        |-- post_config.json
+        |-- qwen2.5_tokenizer
+        |-- qwen2_5_tokenizer.txt
+        |-- qwen2_p128_l0_together.axmodel
+        |-- qwen2_p128_l10_together.axmodel
+        |-- qwen2_p128_l11_together.axmodel
+        |-- qwen2_p128_l12_together.axmodel
+        |-- qwen2_p128_l13_together.axmodel
+        |-- qwen2_p128_l14_together.axmodel
+        |-- qwen2_p128_l15_together.axmodel
+        |-- qwen2_p128_l16_together.axmodel
+        |-- qwen2_p128_l17_together.axmodel
+        |-- qwen2_p128_l18_together.axmodel
+        |-- qwen2_p128_l19_together.axmodel
+        |-- qwen2_p128_l1_together.axmodel
+        |-- qwen2_p128_l20_together.axmodel
+        |-- qwen2_p128_l21_together.axmodel
+        |-- qwen2_p128_l22_together.axmodel
+        |-- qwen2_p128_l23_together.axmodel
+        |-- qwen2_p128_l24_together.axmodel
+        |-- qwen2_p128_l25_together.axmodel
+        |-- qwen2_p128_l26_together.axmodel
+        |-- qwen2_p128_l27_together.axmodel
+        |-- qwen2_p128_l28_together.axmodel
+        |-- qwen2_p128_l29_together.axmodel
+        |-- qwen2_p128_l2_together.axmodel
+        |-- qwen2_p128_l30_together.axmodel
+        |-- qwen2_p128_l31_together.axmodel
+        |-- qwen2_p128_l32_together.axmodel
+        |-- qwen2_p128_l33_together.axmodel
+        |-- qwen2_p128_l34_together.axmodel
+        |-- qwen2_p128_l35_together.axmodel
+        |-- qwen2_p128_l3_together.axmodel
+        |-- qwen2_p128_l4_together.axmodel
+        |-- qwen2_p128_l5_together.axmodel
+        |-- qwen2_p128_l6_together.axmodel
+        |-- qwen2_p128_l7_together.axmodel
+        |-- qwen2_p128_l8_together.axmodel
+        |-- qwen2_p128_l9_together.axmodel
+        |-- qwen2_post.axmodel
+        |-- results
+        `-- test_data
+
+4 directories, 44 files
+```
+
+## Inference with AX650 Host, such as M4N-Dock(爱芯派Pro) or AX650N DEMO Board
+
+### Gradio Demo
+
+```shell
+(base) root@ax650:~/LocateAnything# python gradio_locateanything_axengine.py
+[INFO] Available providers:  ['AxEngineExecutionProvider', 'AXCLRTExecutionProvider']
+[Gradio] starting LocateAnything UI
+[Gradio] local: http://127.0.0.1:7860
+[Gradio] LAN:   http://10.126.29.50:7860
+[Gradio] LAN:   http://10.126.29.68:7860
+[Gradio] LAN:   http://172.17.0.1:7860
+[Gradio] Use another computer in the same LAN to open the LAN URL.
+* Running on local URL:  http://0.0.0.0:7860
+* To create a public link, set `share=True` in `launch()`.
+```
+
+Output:
+
+detection:
+![detection](./assert/person.jpg)
+
+ocr:
+![ocr](./assert/ocr.jpg)
+
+phrase grounding:
+![phrase grounding](./assert/phrase_grounding.jpg)

config.json

@@ -0,0 +1,16 @@
+{
+    "system_prompt": "you are a helpful assistant.",
+    "model_name": "AXERA-TECH/Qwen2.5-locateanything",
+    "url_tokenizer_model": "qwen2_5_tokenizer.txt",
+    "tokenizer_type": "Qwen2_5",
+    "post_config_path": "post_config.json",
+    "template_filename_axmodel": "qwen2_p128_l%d_together.axmodel",
+    "axmodel_num": 36,
+    "filename_post_axmodel": "qwen2_post.axmodel",
+    "filename_tokens_embed": "model.embed_tokens.weight.bfloat16.bin",
+    "tokens_embed_num": 152681,
+    "tokens_embed_size": 2048,
+    "use_mmap_load_embed": true,
+    "use_mmap_load_layer": true,
+    "devices": [0]
+}

gradio_locateanything_axengine.py

@@ -0,0 +1,477 @@
+#!/usr/bin/env python3
+"""Gradio UI for LocateAnything axengine inference.
+
+The UI reuses infer_locateanything_axengine.py and keeps the same pure
+axengine path: image encoder + AR LLM decode + strict streaming geometry
+decode. Gradio is intentionally imported lazily so the script can print a
+clear dependency error on boards where it is not preinstalled.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import queue
+import re
+import subprocess
+import socket
+import threading
+import time
+from pathlib import Path
+from types import SimpleNamespace
+
+from PIL import Image, UnidentifiedImageError
+
+from infer_locateanything_axengine import (
+    DEFAULT_IMAGE_ENCODER,
+    DEFAULT_LLM_DIR,
+    DEFAULT_SYSTEM_PROMPT,
+    DEFAULT_TARGET,
+    DEFAULT_TOKENIZER,
+    LocateAnythingAxEngineRunner,
+    GeometryImageDrawer,
+    PROMPT_SPECS,
+    Point,
+    Box,
+    StreamingGeometryDecoder,
+    build_task_prompt,
+    time_geometry_decode,
+)
+
+
+SCRIPT_DIR = Path(__file__).resolve().parent
+UI_DIR = SCRIPT_DIR / "results" / "gradio"
+INPUT_DIR = UI_DIR / "inputs"
+OUTPUT_DIR = UI_DIR / "outputs"
+
+UI_TASKS = {
+    "Object Detection": "object_detection",
+    "Phrase Grounding (Single Box)": "phrase_grounding_single",
+    "Phrase Grounding (Multiple Boxes)": "phrase_grounding_multi",
+    "Text Grounding": "text_grounding",
+    "Scene Text Detection (OCR)": "scene_text_detection",
+    "Document Layout Analysis": "document_layout",
+    "GUI Grounding (Box)": "gui_grounding_box",
+    "GUI Grounding (Point)": "gui_grounding_point",
+    "Pointing": "pointing",
+}
+
+gr = None
+
+
+def ensure_gradio() -> None:
+    global gr
+    if gr is not None:
+        return
+    try:
+        import gradio as gradio_module
+    except ModuleNotFoundError:
+        print("[ERROR] gradio is not installed.")
+        print("Install it first, for example: python3 -m pip install gradio")
+        raise SystemExit(1)
+    gr = gradio_module
+
+
+class RunnerState:
+    def __init__(self, tokenizer: str, llm_dir: str, image_encoder: str):
+        self.tokenizer = tokenizer
+        self.llm_dir = llm_dir
+        self.image_encoder = image_encoder
+        self.runner: LocateAnythingAxEngineRunner | None = None
+        self.lock = threading.Lock()
+
+    def get_runner(self) -> LocateAnythingAxEngineRunner:
+        if self.runner is None:
+            self.runner = LocateAnythingAxEngineRunner(self.tokenizer, self.llm_dir, self.image_encoder)
+        return self.runner
+
+    def close(self) -> None:
+        if self.runner is not None:
+            self.runner.close()
+            self.runner = None
+
+
+def get_lan_ips() -> list[str]:
+    ips = set()
+    try:
+        output = subprocess.check_output(["hostname", "-I"], text=True, timeout=2)
+        for item in output.split():
+            if item and not item.startswith("127.") and ":" not in item:
+                ips.add(item)
+    except (OSError, subprocess.SubprocessError):
+        pass
+
+    try:
+        output = subprocess.check_output(["ip", "-4", "addr", "show"], text=True, timeout=2)
+        for line in output.splitlines():
+            line = line.strip()
+            if not line.startswith("inet "):
+                continue
+            item = line.split()[1].split("/")[0]
+            if item and not item.startswith("127."):
+                ips.add(item)
+    except (OSError, subprocess.SubprocessError):
+        pass
+
+    try:
+        hostname = socket.gethostname()
+        for item in socket.gethostbyname_ex(hostname)[2]:
+            if item and not item.startswith("127."):
+                ips.add(item)
+    except OSError:
+        pass
+
+    try:
+        sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+        sock.connect(("8.8.8.8", 80))
+        ip = sock.getsockname()[0]
+        if ip and not ip.startswith("127."):
+            ips.add(ip)
+        sock.close()
+    except OSError:
+        pass
+
+    return sorted(ips) or ["127.0.0.1"]
+
+
+def find_available_port(start_port: int, retries: int) -> int:
+    for port in range(start_port, start_port + retries + 1):
+        try:
+            with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
+                sock.bind(("127.0.0.1", port))
+            return port
+        except OSError:
+            continue
+    raise OSError(f"Cannot find empty port in range: {start_port}-{start_port + retries}")
+
+
+def save_uploaded_image(image: Image.Image) -> str:
+    INPUT_DIR.mkdir(parents=True, exist_ok=True)
+    ts = time.strftime("%Y%m%d_%H%M%S")
+    path = INPUT_DIR / f"upload_{ts}_{int(time.time() * 1000) % 1000:03d}.jpg"
+    image.convert("RGB").save(path)
+    return str(path)
+
+
+def load_ui_image(path: str) -> Image.Image | None:
+    if not path or not os.path.exists(path):
+        return None
+    for _ in range(5):
+        try:
+            with Image.open(path) as image:
+                return image.convert("RGB").copy()
+        except (OSError, UnidentifiedImageError):
+            time.sleep(0.05)
+    return None
+
+
+def drain_update_queue(update_queue: queue.Queue) -> None:
+    while True:
+        try:
+            update_queue.get_nowait()
+        except queue.Empty:
+            return
+
+
+def build_prompt(ui_task: str, target_text: str, custom_prompt: str) -> tuple[str, str, str, str]:
+    task = UI_TASKS[ui_task]
+    if custom_prompt.strip():
+        ns = SimpleNamespace(prompt=custom_prompt.strip(), task=task, target=target_text.strip(), categories=None, phrase=None)
+        return build_task_prompt(ns)
+
+    target = target_text.strip() or DEFAULT_TARGET
+    spec = PROMPT_SPECS[task]
+    categories = target if spec.target_kind == "categories" else None
+    phrase = target if spec.target_kind == "phrase" else None
+    ns = SimpleNamespace(prompt=None, task=task, target=target, categories=categories, phrase=phrase)
+    return build_task_prompt(ns)
+
+
+def box_to_dict(box: Box) -> dict[str, float]:
+    return {"x1": box.x1, "y1": box.y1, "x2": box.x2, "y2": box.y2}
+
+
+def point_to_dict(point: Point) -> dict[str, float]:
+    return {"x": point.x, "y": point.y}
+
+
+def format_coord(value: float) -> int:
+    return int(round(value))
+
+
+def extract_ref_labels(text: str) -> list[str]:
+    refs = re.findall(r"<ref>(.*?)</ref>\s*<box>", text, flags=re.DOTALL)
+    return [ref.strip() for ref in refs]
+
+
+def format_detection_results(
+    label: str,
+    boxes: list[Box],
+    points: list[Point],
+    ref_labels: list[str] | None = None,
+) -> str:
+    fallback_ref = label or "text"
+    lines = []
+    ref_labels = ref_labels or []
+    for index, box in enumerate(boxes):
+        ref = ref_labels[index] if index < len(ref_labels) and ref_labels[index] else fallback_ref
+        box_value = [format_coord(box.x1), format_coord(box.y1), format_coord(box.x2), format_coord(box.y2)]
+        lines.append(json.dumps([ref, {"box": box_value}], ensure_ascii=False))
+    for point in points:
+        point_value = [format_coord(point.x), format_coord(point.y)]
+        lines.append(json.dumps([fallback_ref, {"point": point_value}], ensure_ascii=False))
+    return "\n".join(lines) if lines else "[]"
+
+
+def make_json_result(
+    *,
+    task: str,
+    output_type: str,
+    target: str,
+    prompt: str,
+    token_ids: list[int],
+    text: str,
+    boxes: list[Box],
+    points: list[Point],
+    stream_boxes: list[dict],
+    stream_points: list[dict],
+    stream_consistent: bool,
+    stream_points_consistent: bool,
+    timings: dict,
+    output_image: str,
+) -> dict:
+    return {
+        "task": task,
+        "task_output_type": output_type,
+        "target": target,
+        "prompt": prompt,
+        "token_ids": token_ids,
+        "text": text,
+        "boxes": [box_to_dict(box) for box in boxes],
+        "points": [point_to_dict(point) for point in points],
+        "stream_boxes": stream_boxes,
+        "stream_points": stream_points,
+        "stream_consistent": stream_consistent,
+        "stream_points_consistent": stream_points_consistent,
+        "timings": timings,
+        "output_image": output_image,
+    }
+
+
+def create_infer_fn(state: RunnerState, args: argparse.Namespace):
+    def infer(
+        image: Image.Image | None,
+        ui_task: str,
+        target_text: str,
+        custom_prompt: str,
+        max_new_tokens: int,
+        temperature: float,
+        top_p: float,
+        repetition_penalty: float,
+        seed: int,
+    ):
+        if image is None:
+            yield "请先上传图片。", None, None
+            return
+
+        OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
+        image_path = save_uploaded_image(image)
+        ts = time.strftime("%Y%m%d_%H%M%S")
+        stem = f"{UI_TASKS[ui_task]}_{ts}_{int(time.time() * 1000) % 1000:03d}"
+        output_image = str(OUTPUT_DIR / f"{stem}.jpg")
+        output_json = str(OUTPUT_DIR / f"{stem}.json")
+        image.convert("RGB").save(output_image)
+
+        prompt, task, output_type, target = build_prompt(ui_task, target_text, custom_prompt)
+        image_size = Image.open(image_path).size
+        stream_decoder = StreamingGeometryDecoder(image_size)
+        stream_drawer = GeometryImageDrawer(image_path, output_image)
+        stream_boxes: list[dict] = []
+        stream_points: list[dict] = []
+        update_queue: queue.Queue = queue.Queue()
+
+        yield "", load_ui_image(output_image), None
+
+        def on_token(token_id: int, step: int, piece: str, elapsed_s: float) -> None:
+            geometry = stream_decoder.push(token_id)
+            if geometry is None:
+                return
+            kind, value = geometry
+            if kind == "box":
+                assert isinstance(value, Box)
+                index = len(stream_boxes) + 1
+                stream_drawer.add_box(value, index)
+                stream_boxes.append(
+                    {
+                        "index": index,
+                        "token_step": step,
+                        "elapsed_s": elapsed_s,
+                        "box": box_to_dict(value),
+                    }
+                )
+                update_queue.put(("update", None, output_image, None))
+            else:
+                assert isinstance(value, Point)
+                index = len(stream_points) + 1
+                stream_drawer.add_point(value, index)
+                stream_points.append(
+                    {
+                        "index": index,
+                        "token_step": step,
+                        "elapsed_s": elapsed_s,
+                        "point": point_to_dict(value),
+                    }
+                )
+                update_queue.put(("update", None, output_image, None))
+
+        def worker() -> None:
+            try:
+                with state.lock:
+                    runner = state.get_runner()
+                    token_ids, text, decoded_image_size, used_seed, timings = runner.generate(
+                        prompt=prompt,
+                        image_path=image_path,
+                        max_new_tokens=int(max_new_tokens),
+                        system_prompt=args.system_prompt,
+                        temperature=float(temperature),
+                        top_p=float(top_p),
+                        repetition_penalty=float(repetition_penalty),
+                        seed=int(seed),
+                        on_token=on_token,
+                    )
+
+                boxes, points, geometry_decode_s = time_geometry_decode(token_ids, decoded_image_size)
+                timings["geometry_decode_s"] = geometry_decode_s
+                stream_consistent = [item["box"] for item in stream_boxes] == [box_to_dict(box) for box in boxes]
+                stream_points_consistent = [item["point"] for item in stream_points] == [
+                    point_to_dict(point) for point in points
+                ]
+
+                if not stream_consistent or not stream_points_consistent:
+                    from infer_locateanything_axengine import draw_geometries
+
+                    draw_geometries(image_path, boxes, points, output_image)
+
+                result = make_json_result(
+                    task=task,
+                    output_type=output_type,
+                    target=target,
+                    prompt=prompt,
+                    token_ids=token_ids,
+                    text=text,
+                    boxes=boxes,
+                    points=points,
+                    stream_boxes=stream_boxes,
+                    stream_points=stream_points,
+                    stream_consistent=stream_consistent,
+                    stream_points_consistent=stream_points_consistent,
+                    timings=timings,
+                    output_image=output_image,
+                )
+                with open(output_json, "w", encoding="utf-8") as f:
+                    json.dump(result, f, ensure_ascii=False, indent=2)
+
+                ref_labels = extract_ref_labels(text) if task == "scene_text_detection" else None
+                result_text = format_detection_results(target, boxes, points, ref_labels)
+                drain_update_queue(update_queue)
+                update_queue.put(("final", result_text, output_image, output_json))
+            except Exception as exc:
+                drain_update_queue(update_queue)
+                update_queue.put(("error", f"[ERROR] {type(exc).__name__}: {exc}", output_image, None))
+            finally:
+                update_queue.put(("done", None, None, None))
+
+        thread = threading.Thread(target=worker, daemon=True)
+        thread.start()
+
+        while True:
+            kind, log_value, image_value, file_value = update_queue.get()
+            if kind == "done":
+                break
+            if kind == "update":
+                yield gr.update(), load_ui_image(image_value), gr.update()
+            else:
+                yield log_value, load_ui_image(image_value), file_value
+
+    return infer
+
+
+def build_ui(state: RunnerState, args: argparse.Namespace):
+    infer_fn = create_infer_fn(state, args)
+    with gr.Blocks(title="LocateAnything") as demo:
+        gr.Markdown("# LocateAnything Gradio UI")
+        gr.Markdown(
+            "上传图片，选择任务类型，输入类别、category set 或 phrase 后开始推理。OCR 任务可留空；box/point 会在生成过程中逐个写入输出图。"
+        )
+        with gr.Row():
+            with gr.Column(scale=1):
+                image = gr.Image(label="图片上传", type="pil")
+                task = gr.Dropdown(
+                    label="任务类别",
+                    choices=list(UI_TASKS),
+                    value="Object Detection",
+                )
+                target = gr.Textbox(label="类别 / category set / phrase（OCR 可留空）", value=DEFAULT_TARGET)
+                custom_prompt = gr.Textbox(label="自定义 prompt（可选，填写后覆盖任务模板）", value="")
+                with gr.Accordion("生成参数", open=False):
+                    max_new_tokens = gr.Slider(32, 512, value=512, step=1, label="max_new_tokens")
+                    temperature = gr.Slider(0.0, 2.0, value=0.7, step=0.05, label="temperature")
+                    top_p = gr.Slider(0.1, 1.0, value=0.9, step=0.01, label="top_p")
+                    repetition_penalty = gr.Slider(1.0, 2.0, value=1.1, step=0.05, label="repetition_penalty")
+                    seed = gr.Number(value=42, precision=0, label="seed")
+                run_btn = gr.Button("开始推理", variant="primary")
+            with gr.Column(scale=1):
+                output_image = gr.Image(label="原图结果流式可视化", type="pil")
+                log = gr.Textbox(label="最终检测结果 [ref, box]", lines=24)
+                json_file = gr.File(label="标准 JSON 结果下载")
+
+        run_btn.click(
+            infer_fn,
+            inputs=[image, task, target, custom_prompt, max_new_tokens, temperature, top_p, repetition_penalty, seed],
+            outputs=[log, output_image, json_file],
+        )
+    return demo
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="LocateAnything Gradio UI")
+    parser.add_argument("--tokenizer", default=DEFAULT_TOKENIZER)
+    parser.add_argument("--llm-dir", default=DEFAULT_LLM_DIR)
+    parser.add_argument("--image-encoder", default=DEFAULT_IMAGE_ENCODER)
+    parser.add_argument("--system-prompt", default=DEFAULT_SYSTEM_PROMPT)
+    parser.add_argument("--host", default="0.0.0.0")
+    parser.add_argument("--port", type=int, default=7860)
+    parser.add_argument("--port-retries", type=int, default=20, help="Try following ports when --port is occupied.")
+    parser.add_argument("--share", action="store_true")
+    return parser.parse_args()
+
+
+def main() -> None:
+    args = parse_args()
+    ensure_gradio()
+    UI_DIR.mkdir(parents=True, exist_ok=True)
+    state = RunnerState(args.tokenizer, args.llm_dir, args.image_encoder)
+    demo = build_ui(state, args)
+    selected_port = find_available_port(args.port, args.port_retries)
+    if selected_port != args.port:
+        print(f"[Gradio] port {args.port} is occupied, fallback to {selected_port}", flush=True)
+
+    print("[Gradio] starting LocateAnything UI", flush=True)
+    print(f"[Gradio] local: http://127.0.0.1:{selected_port}", flush=True)
+    for ip in get_lan_ips():
+        print(f"[Gradio] LAN:   http://{ip}:{selected_port}", flush=True)
+    print("[Gradio] Use another computer in the same LAN to open the LAN URL.", flush=True)
+
+    try:
+        demo.queue(default_concurrency_limit=1).launch(
+            server_name=args.host,
+            server_port=selected_port,
+            share=args.share,
+        )
+    finally:
+        state.close()
+
+
+if __name__ == "__main__":
+    main()

image_encoder_mlp.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6f72727633e605bf7eda5eb0fd478a425297e065b161b9029f027c89cb545f93
+size 592696349

infer_locateanything_axengine.py

@@ -0,0 +1,997 @@
+#!/usr/bin/env python3
+"""Pure Python LocateAnything inference with axengine.
+
+This runner does not use axllm serve or OpenAI-compatible HTTP APIs. It runs:
+  1. image_encoder_mlp.axmodel
+  2. llm-4bit-650/qwen2_p128_l*_together.axmodel
+  3. llm-4bit-650/qwen2_post.axmodel
+
+The implementation mirrors infer_axmodel.py's prefill/decode/post flow and
+injects image encoder embeddings at LocateAnything's <IMG_CONTEXT> positions.
+"""
+
+from __future__ import annotations
+
+import argparse
+import atexit
+import json
+import os
+import time
+from dataclasses import dataclass
+from typing import Sequence
+
+import numpy as np
+from axengine import InferenceSession
+from ml_dtypes import bfloat16
+from PIL import Image, ImageDraw, ImageFont
+from tokenizers import Tokenizer
+
+
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+DEFAULT_TOKENIZER = os.path.join(SCRIPT_DIR, "qwen2.5_tokenizer", "tokenizer.json")
+DEFAULT_LLM_DIR = os.path.join(SCRIPT_DIR)
+DEFAULT_IMAGE_ENCODER = os.path.join(SCRIPT_DIR, "image_encoder_mlp.axmodel")
+DEFAULT_IMAGE = os.path.join(SCRIPT_DIR, "test_data", "person.jpg")
+DEFAULT_SYSTEM_PROMPT = "You are a helpful assistant."
+DEFAULT_TASK = "object_detection"
+DEFAULT_TARGET = "person"
+
+PATCH_SIZE = 14
+VISION_SIZE = 560
+VISION_TOKENS = 400
+HIDDEN_SIZE = 2048
+VOCAB_SIZE = 152681
+KV_CACHE_LEN = 1024
+
+TOK_BOX_START = 151668
+TOK_BOX_END = 151669
+TOK_COORD_START = 151677
+TOK_COORD_END = 152677
+TOK_IMG_CONTEXT = 151665
+TOK_EOS = 151645
+
+
+@dataclass
+class Box:
+    x1: float
+    y1: float
+    x2: float
+    y2: float
+
+
+@dataclass
+class Point:
+    x: float
+    y: float
+
+
+@dataclass
+class StreamBox:
+    index: int
+    token_step: int
+    elapsed_s: float
+    box: Box
+
+
+@dataclass
+class StreamPoint:
+    index: int
+    token_step: int
+    elapsed_s: float
+    point: Point
+
+
+@dataclass
+class LayerFiles:
+    layer_paths: list[str]
+    post_path: str
+    embed_path: str
+
+
+@dataclass(frozen=True)
+class PromptSpec:
+    output_type: str
+    target_kind: str | None
+    template: str
+
+
+PROMPT_SPECS = {
+    "object_detection": PromptSpec(
+        "box",
+        "categories",
+        "Locate all the instances that matches the following description:{target}",
+    ),
+    "phrase_grounding_single": PromptSpec(
+        "box",
+        "phrase",
+        "Locate a single instance that matches the following description: {target}.",
+    ),
+    "phrase_grounding_multi": PromptSpec(
+        "box",
+        "phrase",
+        "Locate all the instances that match the following description: {target}.",
+    ),
+    "text_grounding": PromptSpec("box", "phrase", "Please locate the text referred as {target}."),
+    "scene_text_detection": PromptSpec("box", None, "Detect all the text in box format."),
+    "document_layout": PromptSpec(
+        "box",
+        "categories",
+        "Detect all the objects in the image that belong to the category set: {target}.",
+    ),
+    "gui_grounding_box": PromptSpec(
+        "box",
+        "phrase",
+        "Locate the region that matches the following description: {target}.",
+    ),
+    "gui_grounding_point": PromptSpec("point", "phrase", "Point to: {target}."),
+    "pointing": PromptSpec("point", "phrase", "Point to: {target}."),
+}
+
+
+def release_ax_inference_session(session) -> None:
+    inner = getattr(session, "_sess", None)
+    unload = getattr(inner, "_unload", None)
+    if not callable(unload):
+        return
+    try:
+        unload()
+    except Exception as exc:
+        print(f"[WARN] Failed to unload axengine session cleanly: {exc}")
+    finally:
+        try:
+            inner._unload = lambda: None
+        except Exception:
+            pass
+
+
+def dtype_from_axengine(dtype) -> np.dtype:
+    name = str(dtype).lower()
+    if "bfloat16" in name or "bf16" in name:
+        return bfloat16
+    if "float32" in name or "fp32" in name:
+        return np.float32
+    if "float16" in name or "fp16" in name:
+        return np.float16
+    if "uint32" in name or "u32" in name:
+        return np.uint32
+    raise ValueError(f"Unsupported axengine dtype: {dtype}")
+
+
+def ensure_finite(name: str, arr: np.ndarray) -> None:
+    arr32 = np.asarray(arr, dtype=np.float32)
+    if not np.isfinite(arr32).all():
+        raise RuntimeError(f"{name} contains NaN/Inf")
+
+
+def resolve_model_files(llm_dir: str) -> LayerFiles:
+    cfg_path = os.path.join(llm_dir, "config.json")
+    with open(cfg_path, "r", encoding="utf-8") as f:
+        cfg = json.load(f)
+
+    template = cfg.get("template_filename_axmodel", "qwen2_p128_l%d_together.axmodel")
+    num_layers = int(cfg.get("axmodel_num", 36))
+    layer_paths = [os.path.join(llm_dir, template % i) for i in range(num_layers)]
+    missing = [p for p in layer_paths if not os.path.exists(p)]
+    if missing:
+        raise FileNotFoundError(f"Missing layer axmodel: {missing[0]}")
+
+    post_path = os.path.join(llm_dir, cfg.get("filename_post_axmodel", "qwen2_post.axmodel"))
+    embed_path = os.path.join(llm_dir, cfg.get("filename_tokens_embed", "model.embed_tokens.weight.bfloat16.bin"))
+    if not os.path.exists(post_path):
+        raise FileNotFoundError(post_path)
+    if not os.path.exists(embed_path):
+        raise FileNotFoundError(embed_path)
+    return LayerFiles(layer_paths=layer_paths, post_path=post_path, embed_path=embed_path)
+
+
+def image_to_patches(image_path: str) -> tuple[np.ndarray, tuple[int, int]]:
+    image = Image.open(image_path).convert("RGB")
+    original_size = image.size
+    if image.size != (VISION_SIZE, VISION_SIZE):
+        image = image.resize((VISION_SIZE, VISION_SIZE), Image.Resampling.BICUBIC)
+
+    # Match C++ LocateAnythingImageProcessor: uint8 RGB after Pillow-like
+    # resize, then normalize each pixel as v / 127.5 - 1.0.
+    arr = np.asarray(image, dtype=np.uint8).astype(np.float32)
+    arr = arr / 127.5 - 1.0
+    arr = arr.transpose(2, 0, 1)
+
+    c, h, w = arr.shape
+    h_grid = h // PATCH_SIZE
+    w_grid = w // PATCH_SIZE
+    patches = arr.reshape(c, h_grid, PATCH_SIZE, w_grid, PATCH_SIZE)
+    patches = patches.transpose(1, 3, 0, 2, 4)
+    patches = np.ascontiguousarray(patches.reshape(-1, c, PATCH_SIZE, PATCH_SIZE), dtype=np.float32)
+    return patches, original_size
+
+
+def build_locateanything_prompt(
+    system_prompt: str,
+    prompt: str,
+    image_tokens: int = VISION_TOKENS,
+) -> str:
+    media = "<image 1><img>" + "<IMG_CONTEXT>" * image_tokens + "</img>"
+    return (
+        f"<|im_start|>system\n{system_prompt}\n<|im_end|>\n"
+        "<|im_start|>user\n"
+        + media
+        + prompt
+        + "<|im_end|>\n<|im_start|>assistant\n"
+    )
+
+
+def make_feed(shapes: dict[str, tuple[int, ...]], values: dict[str, np.ndarray]) -> dict[str, np.ndarray]:
+    return {name: value for name, value in values.items() if name in shapes}
+
+
+def build_task_prompt(args: argparse.Namespace) -> tuple[str, str, str, str]:
+    if args.prompt:
+        return args.prompt, "custom", "unknown", "custom"
+
+    spec = PROMPT_SPECS[args.task]
+    if spec.target_kind == "categories":
+        target = args.categories or args.target
+    elif spec.target_kind == "phrase":
+        target = args.phrase or args.target
+    else:
+        target = ""
+
+    if spec.target_kind is not None and not target:
+        raise ValueError(f"--task {args.task} requires --target, --categories, or --phrase")
+
+    prompt = spec.template.format(target=target)
+    return prompt, args.task, spec.output_type, target
+
+
+class LocateAnythingAxEngineRunner:
+    def __init__(self, tokenizer_path: str, llm_dir: str, image_encoder_path: str):
+        self.tokenizer = Tokenizer.from_file(tokenizer_path)
+        self.files = resolve_model_files(llm_dir)
+        self.embed_matrix = np.memmap(self.files.embed_path, mode="r", dtype=np.uint16).view(bfloat16).reshape(
+            VOCAB_SIZE, HIDDEN_SIZE
+        )
+
+        self.image_encoder = InferenceSession(image_encoder_path)
+        self.layer_sessions = [InferenceSession(path) for path in self.files.layer_paths]
+        self.post_session = InferenceSession(self.files.post_path)
+        self._closed = False
+        atexit.register(self.close)
+
+        self.layer_decode_input_shapes: list[dict[str, tuple[int, ...]]] = []
+        self.layer_decode_input_dtypes: list[dict[str, np.dtype]] = []
+        self.layer_decode_output_names: list[list[str]] = []
+        self.layer_prefill_input_shapes: list[list[dict[str, tuple[int, ...]]]] = []
+        self.layer_prefill_input_dtypes: list[list[dict[str, np.dtype]]] = []
+        self.layer_prefill_output_names: list[list[list[str]]] = []
+
+        for session in self.layer_sessions:
+            decode_inputs = session.get_inputs(shape_group=0)
+            self.layer_decode_input_shapes.append({x.name: tuple(x.shape) for x in decode_inputs})
+            self.layer_decode_input_dtypes.append({x.name: dtype_from_axengine(x.dtype) for x in decode_inputs})
+            self.layer_decode_output_names.append([x.name for x in session.get_outputs(shape_group=0)])
+
+            group_shapes = []
+            group_dtypes = []
+            group_outputs = []
+            for shape_group in range(1, 64):
+                try:
+                    inputs = session.get_inputs(shape_group=shape_group)
+                    outputs = session.get_outputs(shape_group=shape_group)
+                except Exception:
+                    break
+                group_shapes.append({x.name: tuple(x.shape) for x in inputs})
+                group_dtypes.append({x.name: dtype_from_axengine(x.dtype) for x in inputs})
+                group_outputs.append([x.name for x in outputs])
+            self.layer_prefill_input_shapes.append(group_shapes)
+            self.layer_prefill_input_dtypes.append(group_dtypes)
+            self.layer_prefill_output_names.append(group_outputs)
+
+        self.prefill_len = int(self.layer_prefill_input_shapes[0][0]["input"][1])
+        self.hidden_dtype = self.layer_decode_input_dtypes[0]["input"]
+
+    def close(self) -> None:
+        if self._closed:
+            return
+        release_ax_inference_session(self.image_encoder)
+        for session in self.layer_sessions:
+            release_ax_inference_session(session)
+        release_ax_inference_session(self.post_session)
+        self._closed = True
+
+    def encode_text(self, text: str) -> list[int]:
+        return self.tokenizer.encode(text, add_special_tokens=False).ids
+
+    def decode_text(self, token_ids: Sequence[int]) -> str:
+        return self.tokenizer.decode(list(token_ids), skip_special_tokens=False)
+
+    def embed_token(self, token_id: int) -> np.ndarray:
+        return np.asarray(self.embed_matrix[int(token_id)], dtype=self.hidden_dtype).reshape(1, 1, HIDDEN_SIZE)
+
+    def alloc_layer_states(self) -> tuple[list[np.ndarray], list[np.ndarray]]:
+        k_states = []
+        v_states = []
+        for shapes, dtypes in zip(self.layer_decode_input_shapes, self.layer_decode_input_dtypes):
+            k_states.append(np.zeros(shapes["K_cache"], dtype=dtypes["K_cache"]))
+            v_states.append(np.zeros(shapes["V_cache"], dtype=dtypes["V_cache"]))
+        return k_states, v_states
+
+    def encode_image(self, image_path: str) -> tuple[np.ndarray, tuple[int, int], dict[str, float]]:
+        t0 = time.perf_counter()
+        patches, original_size = image_to_patches(image_path)
+        preprocess_s = time.perf_counter() - t0
+
+        t1 = time.perf_counter()
+        input_name = self.image_encoder.get_inputs()[0].name
+        output = self.image_encoder.run(None, {input_name: patches})[0]
+        image_encoder_s = time.perf_counter() - t1
+
+        output = np.asarray(output, dtype=np.float32)
+        if output.shape != (VISION_TOKENS, HIDDEN_SIZE):
+            raise RuntimeError(f"Unexpected image encoder output shape: {output.shape}")
+        ensure_finite("image encoder output", output)
+        timings = {
+            "image_preprocess_s": preprocess_s,
+            "image_encoder_s": image_encoder_s,
+            "image_total_s": preprocess_s + image_encoder_s,
+        }
+        return output.astype(self.hidden_dtype), original_size, timings
+
+    def build_prompt_embeddings(
+        self,
+        prompt: str,
+        image_embeds: np.ndarray,
+        system_prompt: str,
+    ) -> tuple[list[int], np.ndarray]:
+        text = build_locateanything_prompt(
+            system_prompt,
+            prompt,
+            image_tokens=image_embeds.shape[0],
+        )
+        token_ids = self.encode_text(text)
+        image_positions = [i for i, token_id in enumerate(token_ids) if token_id == TOK_IMG_CONTEXT]
+        if len(image_positions) != image_embeds.shape[0]:
+            raise RuntimeError(
+                f"image placeholder count mismatch: prompt={len(image_positions)} image_embeds={image_embeds.shape[0]}"
+            )
+
+        embeds = np.asarray(self.embed_matrix[np.asarray(token_ids, dtype=np.int64)], dtype=self.hidden_dtype)
+        embeds[image_positions, :] = image_embeds
+        return token_ids, embeds
+
+    @staticmethod
+    def prefill_history_capacity(shapes: dict[str, tuple[int, ...]]) -> int:
+        input_len = int(shapes.get("input", (1, 128))[1])
+        mask_shape = shapes.get("mask")
+        if mask_shape is not None and len(mask_shape) == 3:
+            return max(0, int(mask_shape[-1]) - input_len)
+        k_shape = shapes.get("K_cache")
+        if k_shape is not None and len(k_shape) >= 2:
+            return int(k_shape[1])
+        return 0
+
+    def select_prefill_shape_group(self, layer_idx: int, history_len: int) -> int:
+        groups = self.layer_prefill_input_shapes[layer_idx]
+        candidates = []
+        for offset, shapes in enumerate(groups):
+            cap = self.prefill_history_capacity(shapes)
+            if cap >= history_len:
+                candidates.append((cap, offset + 1))
+        if candidates:
+            return min(candidates)[1]
+        return max(range(1, len(groups) + 1), key=lambda gid: self.prefill_history_capacity(groups[gid - 1]))
+
+    def run_prefill(self, prompt_embeds: np.ndarray):
+        k_states, v_states = self.alloc_layer_states()
+        last_hidden = None
+        total_len = int(prompt_embeds.shape[0])
+
+        for start in range(0, total_len, self.prefill_len):
+            chunk = prompt_embeds[start : start + self.prefill_len]
+            chunk_len = int(chunk.shape[0])
+            data = np.zeros((1, self.prefill_len, HIDDEN_SIZE), dtype=self.hidden_dtype)
+            data[0, :chunk_len, :] = chunk
+
+            for layer_idx, session in enumerate(self.layer_sessions):
+                shape_group = self.select_prefill_shape_group(layer_idx, start)
+                layer_shapes = self.layer_prefill_input_shapes[layer_idx][shape_group - 1]
+                layer_dtypes = self.layer_prefill_input_dtypes[layer_idx][shape_group - 1]
+                history_cap = self.prefill_history_capacity(layer_shapes)
+                history_len = min(start, history_cap)
+
+                indices = np.zeros(layer_shapes["indices"], dtype=layer_dtypes["indices"])
+                indices.reshape(-1)[:chunk_len] = np.arange(start, start + chunk_len, dtype=np.uint32)
+
+                mask = np.full(layer_shapes["mask"], -65536.0, dtype=np.float32)
+                for q in range(chunk_len):
+                    mask[:, q, : history_len + q + 1] = 0.0
+                mask = mask.astype(layer_dtypes["mask"])
+
+                k_feed = np.zeros(layer_shapes["K_cache"], dtype=layer_dtypes["K_cache"])
+                v_feed = np.zeros(layer_shapes["V_cache"], dtype=layer_dtypes["V_cache"])
+                if history_len > 0:
+                    k_feed[:, :history_len, :] = k_states[layer_idx][:, :history_len, :]
+                    v_feed[:, :history_len, :] = v_states[layer_idx][:, :history_len, :]
+
+                outputs = session.run(
+                    None,
+                    make_feed(
+                        layer_shapes,
+                        {
+                            "K_cache": k_feed,
+                            "V_cache": v_feed,
+                            "indices": indices,
+                            "input": data.astype(layer_dtypes["input"], copy=False),
+                            "mask": mask,
+                        },
+                    ),
+                    shape_group=shape_group,
+                )
+                output_map = dict(zip(self.layer_prefill_output_names[layer_idx][shape_group - 1], outputs))
+                k_out = output_map.get("K_cache_out")
+                if k_out is not None:
+                    k_states[layer_idx][:, start : start + chunk_len, :] = k_out[:, :chunk_len, :]
+                v_out = output_map.get("V_cache_out")
+                if v_out is not None:
+                    v_states[layer_idx][:, start : start + chunk_len, :] = v_out[:, :chunk_len, :]
+                data = output_map["output"]
+                ensure_finite(f"prefill layer {layer_idx}", data)
+
+            last_hidden = data[:, chunk_len - 1 : chunk_len, :]
+        return k_states, v_states, last_hidden
+
+    def run_decode_step(
+        self,
+        hidden: np.ndarray,
+        position: int,
+        k_states: list[np.ndarray],
+        v_states: list[np.ndarray],
+    ) -> np.ndarray:
+        data = hidden
+        for layer_idx, session in enumerate(self.layer_sessions):
+            layer_shapes = self.layer_decode_input_shapes[layer_idx]
+            layer_dtypes = self.layer_decode_input_dtypes[layer_idx]
+
+            indices = np.zeros(layer_shapes["indices"], dtype=layer_dtypes["indices"])
+            indices.reshape(-1)[0] = position
+
+            mask = np.full(layer_shapes["mask"], -65536.0, dtype=np.float32)
+            valid_past = min(position, layer_shapes["mask"][-1] - 1)
+            if valid_past > 0:
+                mask[:, :, :valid_past] = 0.0
+            mask[:, :, -1:] = 0.0
+            mask = mask.astype(layer_dtypes["mask"])
+
+            outputs = session.run(
+                None,
+                make_feed(
+                    layer_shapes,
+                    {
+                        "K_cache": k_states[layer_idx],
+                        "V_cache": v_states[layer_idx],
+                        "indices": indices,
+                        "input": data.astype(layer_dtypes["input"], copy=False),
+                        "mask": mask,
+                    },
+                ),
+                shape_group=0,
+            )
+            output_map = dict(zip(self.layer_decode_output_names[layer_idx], outputs))
+            k_out = output_map.get("K_cache_out")
+            if k_out is not None:
+                pos_end = position + k_out.shape[1]
+                k_states[layer_idx][:, position:pos_end, :] = k_out
+            v_out = output_map.get("V_cache_out")
+            if v_out is not None:
+                pos_end = position + v_out.shape[1]
+                v_states[layer_idx][:, position:pos_end, :] = v_out
+            data = output_map["output"]
+            ensure_finite(f"decode layer {layer_idx}", data)
+        return data
+
+    def run_post(self, hidden: np.ndarray) -> np.ndarray:
+        logits = self.post_session.run(None, {"input": hidden.astype(self.hidden_dtype, copy=False)})[0]
+        ensure_finite("post logits", logits)
+        return np.asarray(logits, dtype=np.float32).reshape(-1)
+
+    @staticmethod
+    def sample_next_token(
+        logits: np.ndarray,
+        rng: np.random.Generator | None,
+        temperature: float,
+        top_p: float,
+        repetition_penalty: float,
+        generated_history: Sequence[int],
+    ) -> int:
+        scores = logits.astype(np.float64)
+        if repetition_penalty != 1.0 and generated_history:
+            for token_id in set(int(x) for x in generated_history if 0 <= int(x) < scores.shape[0]):
+                if scores[token_id] > 0:
+                    scores[token_id] /= repetition_penalty
+                else:
+                    scores[token_id] *= repetition_penalty
+
+        if temperature > 0.0:
+            scores = scores / float(temperature)
+        if top_p is not None and top_p < 1.0:
+            order = np.argsort(scores)[::-1]
+            sorted_scores = scores[order]
+            stable_scores = sorted_scores - np.max(sorted_scores)
+            sorted_probs = np.exp(stable_scores)
+            sorted_probs /= np.sum(sorted_probs)
+            cumulative = np.cumsum(sorted_probs)
+            remove = cumulative > top_p
+            if remove.shape[0] > 1:
+                remove[1:] = remove[:-1]
+            remove[0] = False
+            scores[order[remove]] = -np.inf
+
+        if temperature <= 0.0:
+            return int(np.argmax(scores))
+
+        scores -= np.max(scores)
+        probs = np.exp(scores)
+        probs_sum = float(np.sum(probs))
+        if not np.isfinite(probs_sum) or probs_sum <= 0.0:
+            return int(np.argmax(logits))
+        probs /= probs_sum
+
+        assert rng is not None
+        return int(rng.choice(np.arange(probs.shape[0]), p=probs))
+
+    def generate_from_state(
+        self,
+        prompt_len: int,
+        k_states: list[np.ndarray],
+        v_states: list[np.ndarray],
+        last_hidden: np.ndarray,
+        max_new_tokens: int,
+        temperature: float,
+        top_p: float,
+        repetition_penalty: float,
+        seed: int,
+        prompt_token_ids: Sequence[int],
+        on_token=None,
+    ) -> list[int]:
+        rng = None if temperature <= 0.0 else np.random.default_rng(seed)
+        generated: list[int] = []
+        generated_history = list(prompt_token_ids)
+        decode_start = time.perf_counter()
+        for step in range(max_new_tokens):
+            next_token = self.sample_next_token(
+                self.run_post(last_hidden),
+                rng,
+                temperature,
+                top_p,
+                repetition_penalty,
+                generated_history,
+            )
+            generated.append(next_token)
+            generated_history.append(next_token)
+            if on_token is not None:
+                on_token(
+                    next_token,
+                    step,
+                    self.decode_text([next_token]),
+                    time.perf_counter() - decode_start,
+                )
+            if next_token == TOK_EOS:
+                break
+            last_hidden = self.run_decode_step(
+                self.embed_token(next_token),
+                prompt_len + step,
+                k_states,
+                v_states,
+            )
+        return generated
+
+    def generate(
+        self,
+        prompt: str,
+        image_path: str,
+        max_new_tokens: int,
+        system_prompt: str,
+        temperature: float,
+        top_p: float,
+        repetition_penalty: float,
+        seed: int,
+        on_token=None,
+    ) -> tuple[list[int], str, tuple[int, int], int, dict[str, float]]:
+        image_embeds, image_size, timings = self.encode_image(image_path)
+        print(
+            f"[Image] encoded {image_path}, embeds={image_embeds.shape}, "
+            f"preprocess={timings['image_preprocess_s']:.3f}s "
+            f"encoder={timings['image_encoder_s']:.3f}s "
+            f"total={timings['image_total_s']:.3f}s"
+        )
+
+        t_prompt = time.perf_counter()
+        token_ids, prompt_embeds = self.build_prompt_embeddings(prompt, image_embeds, system_prompt)
+        timings["prompt_build_s"] = time.perf_counter() - t_prompt
+        print(f"[Prompt] tokens={len(token_ids)}, image_tokens={token_ids.count(TOK_IMG_CONTEXT)} style=original")
+        if len(token_ids) + max_new_tokens >= KV_CACHE_LEN:
+            raise RuntimeError(f"prompt + max_new_tokens exceeds KV cache: {len(token_ids)} + {max_new_tokens} >= {KV_CACHE_LEN}")
+
+        t1 = time.perf_counter()
+        k_states, v_states, last_hidden = self.run_prefill(prompt_embeds)
+        timings["llm_prefill_s"] = time.perf_counter() - t1
+        print(f"[Prefill] cost={timings['llm_prefill_s']:.3f}s")
+
+        t2 = time.perf_counter()
+        print(
+            f"[LLM Decode] seed={seed} temperature={temperature} "
+            f"top_p={top_p} repetition_penalty={repetition_penalty}"
+        )
+        generated = self.generate_from_state(
+            len(token_ids),
+            k_states,
+            v_states,
+            last_hidden,
+            max_new_tokens,
+            temperature,
+            top_p,
+            repetition_penalty,
+            seed,
+            token_ids,
+            on_token=on_token,
+        )
+        timings["llm_generate_s"] = time.perf_counter() - t2
+        timings["llm_total_s"] = timings["llm_prefill_s"] + timings["llm_generate_s"]
+        print(f"[LLM Decode] generated={len(generated)} cost={timings['llm_generate_s']:.3f}s")
+        print(f"[Timing] llm_total={timings['llm_total_s']:.3f}s")
+        return generated, self.decode_text(generated), image_size, seed, timings
+
+
+def token_to_coord(token_id: int) -> int | None:
+    if TOK_COORD_START <= token_id <= TOK_COORD_END:
+        return token_id - TOK_COORD_START
+    return None
+
+
+def coord_to_pixel(coord: int, dim: int) -> float:
+    return coord / 1000 * dim
+
+
+def make_box(coords: Sequence[int], image_size: tuple[int, int]) -> Box:
+    iw, ih = image_size
+    x1, y1, x2, y2 = [int(v) for v in coords]
+    return Box(
+        coord_to_pixel(x1, iw),
+        coord_to_pixel(y1, ih),
+        coord_to_pixel(x2, iw),
+        coord_to_pixel(y2, ih),
+    )
+
+
+def make_point(coords: Sequence[int], image_size: tuple[int, int]) -> Point:
+    iw, ih = image_size
+    x, y = [int(v) for v in coords]
+    return Point(coord_to_pixel(x, iw), coord_to_pixel(y, ih))
+
+
+def decode_boxes_strict(token_ids: Sequence[int], image_size: tuple[int, int]) -> list[Box]:
+    """Decode exact <box><x1><y1><x2><y2></box> token sequences."""
+    boxes: list[Box] = []
+    i = 0
+    while i <= len(token_ids) - 6:
+        if token_ids[i] != TOK_BOX_START:
+            i += 1
+            continue
+        coords = [token_to_coord(token_ids[i + j]) for j in range(1, 5)]
+        if None not in coords and token_ids[i + 5] == TOK_BOX_END:
+            boxes.append(make_box([int(v) for v in coords], image_size))
+            i += 6
+        else:
+            i += 1
+    return boxes
+
+
+def decode_points_strict(token_ids: Sequence[int], image_size: tuple[int, int]) -> list[Point]:
+    """Decode exact <box><x><y></box> point token sequences."""
+    points: list[Point] = []
+    i = 0
+    while i <= len(token_ids) - 4:
+        if token_ids[i] != TOK_BOX_START:
+            i += 1
+            continue
+        coords = [token_to_coord(token_ids[i + 1]), token_to_coord(token_ids[i + 2])]
+        if None not in coords and token_ids[i + 3] == TOK_BOX_END:
+            points.append(make_point([int(v) for v in coords], image_size))
+            i += 4
+        else:
+            i += 1
+    return points
+
+
+class StreamingGeometryDecoder:
+    """Incrementally decode strict box and point token sequences."""
+
+    def __init__(self, image_size: tuple[int, int]):
+        self.image_size = image_size
+        self._state = "search"
+        self._coords: list[int] = []
+
+    def push(self, token_id: int) -> tuple[str, Box | Point] | None:
+        if self._state == "search":
+            if token_id == TOK_BOX_START:
+                self._coords = []
+                self._state = "coords"
+            return None
+
+        if self._state == "coords":
+            coord = token_to_coord(token_id)
+            if coord is not None:
+                self._coords.append(coord)
+                if len(self._coords) == 4:
+                    self._state = "end"
+                return None
+            if token_id == TOK_BOX_END and len(self._coords) == 2:
+                point = make_point(self._coords, self.image_size)
+                self._state = "search"
+                self._coords = []
+                return "point", point
+            self._reset(token_id)
+            return None
+
+        if self._state == "end":
+            if token_id == TOK_BOX_END:
+                box = make_box(self._coords, self.image_size)
+                self._state = "search"
+                self._coords = []
+                return "box", box
+            self._reset(token_id)
+        return None
+
+    def _reset(self, token_id: int) -> None:
+        if token_id == TOK_BOX_START:
+            self._coords = []
+            self._state = "coords"
+        else:
+            self._coords = []
+            self._state = "search"
+
+
+def time_geometry_decode(
+    token_ids: Sequence[int],
+    image_size: tuple[int, int],
+) -> tuple[list[Box], list[Point], float]:
+    t0 = time.perf_counter()
+    boxes = decode_boxes_strict(token_ids, image_size)
+    points = decode_points_strict(token_ids, image_size)
+    return boxes, points, time.perf_counter() - t0
+
+
+BOX_COLORS = ["#00FF00", "#FF0000", "#0000FF", "#FFFF00", "#FF00FF", "#00FFFF", "#FFA500"]
+
+
+def load_box_font():
+    try:
+        return ImageFont.truetype("/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf", 16)
+    except OSError:
+        return ImageFont.load_default()
+
+
+def draw_one_box(draw: ImageDraw.ImageDraw, font, box: Box, idx: int) -> None:
+    color = BOX_COLORS[(idx - 1) % len(BOX_COLORS)]
+    rect = [round(box.x1), round(box.y1), round(box.x2), round(box.y2)]
+    draw.rectangle(rect, outline=color, width=3)
+    label = str(idx)
+    label_pos = (rect[0], max(0, rect[1] - 18))
+    tb = draw.textbbox(label_pos, label, font=font)
+    draw.rectangle(tb, fill=color)
+    draw.text(label_pos, label, fill="black", font=font)
+
+
+def draw_one_point(draw: ImageDraw.ImageDraw, font, point: Point, idx: int) -> None:
+    color = BOX_COLORS[(idx - 1) % len(BOX_COLORS)]
+    x = round(point.x)
+    y = round(point.y)
+    radius = 8
+    draw.ellipse([x - radius, y - radius, x + radius, y + radius], outline=color, fill=color, width=3)
+    label = str(idx)
+    label_pos = (x + radius + 2, max(0, y - radius))
+    tb = draw.textbbox(label_pos, label, font=font)
+    draw.rectangle(tb, fill=color)
+    draw.text(label_pos, label, fill="black", font=font)
+
+
+def draw_geometries(image_path: str, boxes: list[Box], points: list[Point], output: str) -> None:
+    image = Image.open(image_path).convert("RGB")
+    draw = ImageDraw.Draw(image)
+    font = load_box_font()
+    for idx, box in enumerate(boxes, start=1):
+        draw_one_box(draw, font, box, idx)
+    for idx, point in enumerate(points, start=1):
+        draw_one_point(draw, font, point, idx)
+    save_image_atomic(image, output)
+
+
+def save_image_atomic(image: Image.Image, output: str) -> None:
+    output_dir = os.path.dirname(os.path.abspath(output))
+    os.makedirs(output_dir, exist_ok=True)
+    root, ext = os.path.splitext(output)
+    tmp_output = f"{root}.tmp.{os.getpid()}.{time.time_ns()}{ext or '.png'}"
+    try:
+        image.save(tmp_output)
+        os.replace(tmp_output, output)
+    finally:
+        if os.path.exists(tmp_output):
+            os.unlink(tmp_output)
+
+
+class GeometryImageDrawer:
+    def __init__(self, image_path: str, output: str):
+        self.output = output
+        output_dir = os.path.dirname(os.path.abspath(output))
+        os.makedirs(output_dir, exist_ok=True)
+        self.image = Image.open(image_path).convert("RGB")
+        self.draw = ImageDraw.Draw(self.image)
+        self.font = load_box_font()
+
+    def add_box(self, box: Box, idx: int) -> None:
+        draw_one_box(self.draw, self.font, box, idx)
+        save_image_atomic(self.image, self.output)
+
+    def add_point(self, point: Point, idx: int) -> None:
+        draw_one_point(self.draw, self.font, point, idx)
+        save_image_atomic(self.image, self.output)
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="LocateAnything pure axengine Python inference")
+    parser.add_argument("--tokenizer", default=DEFAULT_TOKENIZER)
+    parser.add_argument("--llm-dir", default=DEFAULT_LLM_DIR)
+    parser.add_argument("--image-encoder", default=DEFAULT_IMAGE_ENCODER)
+    parser.add_argument("--image", default=DEFAULT_IMAGE)
+    parser.add_argument("--prompt", default=None, help="Custom prompt. Overrides --task prompt templates.")
+    parser.add_argument("--task", choices=sorted(PROMPT_SPECS), default=DEFAULT_TASK)
+    parser.add_argument("--target", default=DEFAULT_TARGET, help="Fallback target text for prompt templates.")
+    parser.add_argument("--categories", default=None, help="Comma-separated category names for category tasks.")
+    parser.add_argument("--phrase", default=None, help="Free-form phrase for grounding/pointing tasks.")
+    parser.add_argument("--system-prompt", default=DEFAULT_SYSTEM_PROMPT)
+    parser.add_argument("--output", default="output_locateanything_axengine.jpg")
+    parser.add_argument("--max-new-tokens", type=int, default=512)
+    parser.add_argument("--temperature", type=float, default=0.7)
+    parser.add_argument("--top-p", type=float, default=0.9)
+    parser.add_argument("--repetition-penalty", type=float, default=1.1)
+    parser.add_argument("--seed", type=int, default=42)
+    parser.add_argument("--save-response", default=None)
+    return parser.parse_args()
+
+
+def main() -> None:
+    args = parse_args()
+    prompt, prompt_task, prompt_output_type, prompt_target = build_task_prompt(args)
+    stream_decoder: StreamingGeometryDecoder | None = None
+    stream_drawer: GeometryImageDrawer | None = None
+    stream_boxes: list[StreamBox] = []
+    stream_points: list[StreamPoint] = []
+
+    def handle_stream_token(token_id: int, step: int, piece: str, elapsed_s: float) -> None:
+        assert stream_decoder is not None
+        assert stream_drawer is not None
+        print(f"[Stream Token] step={step:03d} token={token_id} text={piece!r}")
+        geometry = stream_decoder.push(token_id)
+        if geometry is None:
+            return
+        kind, value = geometry
+        if kind == "point":
+            assert isinstance(value, Point)
+            stream_point = StreamPoint(
+                index=len(stream_points) + 1,
+                token_step=step,
+                elapsed_s=elapsed_s,
+                point=value,
+            )
+            stream_points.append(stream_point)
+            stream_drawer.add_point(value, stream_point.index)
+            print(
+                f"[Stream Point] #{stream_point.index} step={step} "
+                f"elapsed={elapsed_s:.3f}s saved={args.output}"
+            )
+            return
+
+        assert isinstance(value, Box)
+        stream_box = StreamBox(
+            index=len(stream_boxes) + 1,
+            token_step=step,
+            elapsed_s=elapsed_s,
+            box=value,
+        )
+        stream_boxes.append(stream_box)
+        stream_drawer.add_box(value, stream_box.index)
+        print(
+            f"[Stream Box] #{stream_box.index} step={step} "
+            f"elapsed={elapsed_s:.3f}s saved={args.output}"
+        )
+
+    print(
+        f"[Task Prompt] task={prompt_task} output={prompt_output_type} "
+        f"target={prompt_target!r} prompt={prompt!r}"
+    )
+    runner = LocateAnythingAxEngineRunner(args.tokenizer, args.llm_dir, args.image_encoder)
+    try:
+        image_size_hint = Image.open(args.image).size
+        stream_decoder = StreamingGeometryDecoder(image_size_hint)
+        stream_drawer = GeometryImageDrawer(args.image, args.output)
+        token_ids, text, image_size, used_seed, timings = runner.generate(
+            prompt=prompt,
+            image_path=args.image,
+            max_new_tokens=args.max_new_tokens,
+            system_prompt=args.system_prompt,
+            temperature=args.temperature,
+            top_p=args.top_p,
+            repetition_penalty=args.repetition_penalty,
+            seed=args.seed,
+            on_token=handle_stream_token,
+        )
+    finally:
+        runner.close()
+
+    boxes, points, decode_s = time_geometry_decode(token_ids, image_size)
+    timings["geometry_decode_s"] = decode_s
+    stream_consistent = [x.box for x in stream_boxes] == boxes
+    stream_points_consistent = [x.point for x in stream_points] == points
+    if not stream_consistent or not stream_points_consistent:
+        print("[WARN] Stream geometries differ from final strict decode; rewriting final image.")
+        draw_geometries(args.image, boxes, points, args.output)
+
+    print("\n[LLM output]")
+    print(text)
+    print(f"\n[Seed] {used_seed}")
+    print(f"\n[Geometries] boxes={len(boxes)} points={len(points)} decode={decode_s * 1000:.3f}ms")
+    print(
+        f"[Stream] boxes={len(stream_boxes)} consistent={stream_consistent} "
+        f"points={len(stream_points)} points_consistent={stream_points_consistent}"
+    )
+    for i, box in enumerate(boxes, start=1):
+        print(f"  [{i}] ({box.x1:.2f},{box.y1:.2f}) -> ({box.x2:.2f},{box.y2:.2f})")
+    for i, point in enumerate(points, start=1):
+        print(f"  [P{i}] ({point.x:.2f},{point.y:.2f})")
+
+    if args.save_response:
+        with open(args.save_response, "w", encoding="utf-8") as f:
+            json.dump(
+                {
+                    "task": prompt_task,
+                    "task_output_type": prompt_output_type,
+                    "target": prompt_target,
+                    "prompt": prompt,
+                    "seed": used_seed,
+                    "temperature": args.temperature,
+                    "top_p": args.top_p,
+                    "repetition_penalty": args.repetition_penalty,
+                    "token_ids": token_ids,
+                    "text": text,
+                    "boxes": [box.__dict__ for box in boxes],
+                    "points": [point.__dict__ for point in points],
+                    "stream_boxes": [
+                        {
+                            "index": x.index,
+                            "token_step": x.token_step,
+                            "elapsed_s": x.elapsed_s,
+                            "box": x.box.__dict__,
+                        }
+                        for x in stream_boxes
+                    ],
+                    "stream_points": [
+                        {
+                            "index": x.index,
+                            "token_step": x.token_step,
+                            "elapsed_s": x.elapsed_s,
+                            "point": x.point.__dict__,
+                        }
+                        for x in stream_points
+                    ],
+                    "stream_consistent": stream_consistent,
+                    "stream_points_consistent": stream_points_consistent,
+                    "timings": timings,
+                },
+                f,
+                ensure_ascii=False,
+            )
+        print("[Save] response:", args.save_response)
+    print("[Draw] saved:", args.output)
+
+
+if __name__ == "__main__":
+    main()

model.embed_tokens.weight.bfloat16.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7972da8526034a169b9a0f7e5478d3fe2a5ce61e9dd2a74b3a5b9ad1e4759db9
+size 625381376

post_config.json

@@ -0,0 +1,14 @@
+{
+    "enable_temperature" : false,
+    "temperature" : 0.9,
+    
+    "enable_repetition_penalty" : false,
+    "repetition_penalty" : 1.2,
+    "penalty_window" : 20,
+
+    "enable_top_p_sampling" : false,
+    "top_p" : 0.8,
+
+    "enable_top_k_sampling" : false,
+    "top_k" : 10
+}

qwen2.5_tokenizer/README.md

@@ -0,0 +1,3 @@
+# LLM
+
+Standalone Qwen2.5-3B language model split from LocateAnything. Original keys stripped the `language_model.` prefix.

qwen2.5_tokenizer/added_tokens.json

@@ -0,0 +1,1040 @@
+{
+  "</box>": 151669,
+  "</c>": 152680,
+  "</img>": 151667,
+  "</interval>": 151675,
+  "</quad>": 151671,
+  "</ref>": 151673,
+  "</tool_call>": 151658,
+  "<0>": 151677,
+  "<1000>": 152677,
+  "<100>": 151777,
+  "<101>": 151778,
+  "<102>": 151779,
+  "<103>": 151780,
+  "<104>": 151781,
+  "<105>": 151782,
+  "<106>": 151783,
+  "<107>": 151784,
+  "<108>": 151785,
+  "<109>": 151786,
+  "<10>": 151687,
+  "<110>": 151787,
+  "<111>": 151788,
+  "<112>": 151789,
+  "<113>": 151790,
+  "<114>": 151791,
+  "<115>": 151792,
+  "<116>": 151793,
+  "<117>": 151794,
+  "<118>": 151795,
+  "<119>": 151796,
+  "<11>": 151688,
+  "<120>": 151797,
+  "<121>": 151798,
+  "<122>": 151799,
+  "<123>": 151800,
+  "<124>": 151801,
+  "<125>": 151802,
+  "<126>": 151803,
+  "<127>": 151804,
+  "<128>": 151805,
+  "<129>": 151806,
+  "<12>": 151689,
+  "<130>": 151807,
+  "<131>": 151808,
+  "<132>": 151809,
+  "<133>": 151810,
+  "<134>": 151811,
+  "<135>": 151812,
+  "<136>": 151813,
+  "<137>": 151814,
+  "<138>": 151815,
+  "<139>": 151816,
+  "<13>": 151690,
+  "<140>": 151817,
+  "<141>": 151818,
+  "<142>": 151819,
+  "<143>": 151820,
+  "<144>": 151821,
+  "<145>": 151822,
+  "<146>": 151823,
+  "<147>": 151824,
+  "<148>": 151825,
+  "<149>": 151826,
+  "<14>": 151691,
+  "<150>": 151827,
+  "<151>": 151828,
+  "<152>": 151829,
+  "<153>": 151830,
+  "<154>": 151831,
+  "<155>": 151832,
+  "<156>": 151833,
+  "<157>": 151834,
+  "<158>": 151835,
+  "<159>": 151836,
+  "<15>": 151692,
+  "<160>": 151837,
+  "<161>": 151838,
+  "<162>": 151839,
+  "<163>": 151840,
+  "<164>": 151841,
+  "<165>": 151842,
+  "<166>": 151843,
+  "<167>": 151844,
+  "<168>": 151845,
+  "<169>": 151846,
+  "<16>": 151693,
+  "<170>": 151847,
+  "<171>": 151848,
+  "<172>": 151849,
+  "<173>": 151850,
+  "<174>": 151851,
+  "<175>": 151852,
+  "<176>": 151853,
+  "<177>": 151854,
+  "<178>": 151855,
+  "<179>": 151856,
+  "<17>": 151694,
+  "<180>": 151857,
+  "<181>": 151858,
+  "<182>": 151859,
+  "<183>": 151860,
+  "<184>": 151861,
+  "<185>": 151862,
+  "<186>": 151863,
+  "<187>": 151864,
+  "<188>": 151865,
+  "<189>": 151866,
+  "<18>": 151695,
+  "<190>": 151867,
+  "<191>": 151868,
+  "<192>": 151869,
+  "<193>": 151870,
+  "<194>": 151871,
+  "<195>": 151872,
+  "<196>": 151873,
+  "<197>": 151874,
+  "<198>": 151875,
+  "<199>": 151876,
+  "<19>": 151696,
+  "<1>": 151678,
+  "<200>": 151877,
+  "<201>": 151878,
+  "<202>": 151879,
+  "<203>": 151880,
+  "<204>": 151881,
+  "<205>": 151882,
+  "<206>": 151883,
+  "<207>": 151884,
+  "<208>": 151885,
+  "<209>": 151886,
+  "<20>": 151697,
+  "<210>": 151887,
+  "<211>": 151888,
+  "<212>": 151889,
+  "<213>": 151890,
+  "<214>": 151891,
+  "<215>": 151892,
+  "<216>": 151893,
+  "<217>": 151894,
+  "<218>": 151895,
+  "<219>": 151896,
+  "<21>": 151698,
+  "<220>": 151897,
+  "<221>": 151898,
+  "<222>": 151899,
+  "<223>": 151900,
+  "<224>": 151901,
+  "<225>": 151902,
+  "<226>": 151903,
+  "<227>": 151904,
+  "<228>": 151905,
+  "<229>": 151906,
+  "<22>": 151699,
+  "<230>": 151907,
+  "<231>": 151908,
+  "<232>": 151909,
+  "<233>": 151910,
+  "<234>": 151911,
+  "<235>": 151912,
+  "<236>": 151913,
+  "<237>": 151914,
+  "<238>": 151915,
+  "<239>": 151916,
+  "<23>": 151700,
+  "<240>": 151917,
+  "<241>": 151918,
+  "<242>": 151919,
+  "<243>": 151920,
+  "<244>": 151921,
+  "<245>": 151922,
+  "<246>": 151923,
+  "<247>": 151924,
+  "<248>": 151925,
+  "<249>": 151926,
+  "<24>": 151701,
+  "<250>": 151927,
+  "<251>": 151928,
+  "<252>": 151929,
+  "<253>": 151930,
+  "<254>": 151931,
+  "<255>": 151932,
+  "<256>": 151933,
+  "<257>": 151934,
+  "<258>": 151935,
+  "<259>": 151936,
+  "<25>": 151702,
+  "<260>": 151937,
+  "<261>": 151938,
+  "<262>": 151939,
+  "<263>": 151940,
+  "<264>": 151941,
+  "<265>": 151942,
+  "<266>": 151943,
+  "<267>": 151944,
+  "<268>": 151945,
+  "<269>": 151946,
+  "<26>": 151703,
+  "<270>": 151947,
+  "<271>": 151948,
+  "<272>": 151949,
+  "<273>": 151950,
+  "<274>": 151951,
+  "<275>": 151952,
+  "<276>": 151953,
+  "<277>": 151954,
+  "<278>": 151955,
+  "<279>": 151956,
+  "<27>": 151704,
+  "<280>": 151957,
+  "<281>": 151958,
+  "<282>": 151959,
+  "<283>": 151960,
+  "<284>": 151961,
+  "<285>": 151962,
+  "<286>": 151963,
+  "<287>": 151964,
+  "<288>": 151965,
+  "<289>": 151966,
+  "<28>": 151705,
+  "<290>": 151967,
+  "<291>": 151968,
+  "<292>": 151969,
+  "<293>": 151970,
+  "<294>": 151971,
+  "<295>": 151972,
+  "<296>": 151973,
+  "<297>": 151974,
+  "<298>": 151975,
+  "<299>": 151976,
+  "<29>": 151706,
+  "<2>": 151679,
+  "<300>": 151977,
+  "<301>": 151978,
+  "<302>": 151979,
+  "<303>": 151980,
+  "<304>": 151981,
+  "<305>": 151982,
+  "<306>": 151983,
+  "<307>": 151984,
+  "<308>": 151985,
+  "<309>": 151986,
+  "<30>": 151707,
+  "<310>": 151987,
+  "<311>": 151988,
+  "<312>": 151989,
+  "<313>": 151990,
+  "<314>": 151991,
+  "<315>": 151992,
+  "<316>": 151993,
+  "<317>": 151994,
+  "<318>": 151995,
+  "<319>": 151996,
+  "<31>": 151708,
+  "<320>": 151997,
+  "<321>": 151998,
+  "<322>": 151999,
+  "<323>": 152000,
+  "<324>": 152001,
+  "<325>": 152002,
+  "<326>": 152003,
+  "<327>": 152004,
+  "<328>": 152005,
+  "<329>": 152006,
+  "<32>": 151709,
+  "<330>": 152007,
+  "<331>": 152008,
+  "<332>": 152009,
+  "<333>": 152010,
+  "<334>": 152011,
+  "<335>": 152012,
+  "<336>": 152013,
+  "<337>": 152014,
+  "<338>": 152015,
+  "<339>": 152016,
+  "<33>": 151710,
+  "<340>": 152017,
+  "<341>": 152018,
+  "<342>": 152019,
+  "<343>": 152020,
+  "<344>": 152021,
+  "<345>": 152022,
+  "<346>": 152023,
+  "<347>": 152024,
+  "<348>": 152025,
+  "<349>": 152026,
+  "<34>": 151711,
+  "<350>": 152027,
+  "<351>": 152028,
+  "<352>": 152029,
+  "<353>": 152030,
+  "<354>": 152031,
+  "<355>": 152032,
+  "<356>": 152033,
+  "<357>": 152034,
+  "<358>": 152035,
+  "<359>": 152036,
+  "<35>": 151712,
+  "<360>": 152037,
+  "<361>": 152038,
+  "<362>": 152039,
+  "<363>": 152040,
+  "<364>": 152041,
+  "<365>": 152042,
+  "<366>": 152043,
+  "<367>": 152044,
+  "<368>": 152045,
+  "<369>": 152046,
+  "<36>": 151713,
+  "<370>": 152047,
+  "<371>": 152048,
+  "<372>": 152049,
+  "<373>": 152050,
+  "<374>": 152051,
+  "<375>": 152052,
+  "<376>": 152053,
+  "<377>": 152054,
+  "<378>": 152055,
+  "<379>": 152056,
+  "<37>": 151714,
+  "<380>": 152057,
+  "<381>": 152058,
+  "<382>": 152059,
+  "<383>": 152060,
+  "<384>": 152061,
+  "<385>": 152062,
+  "<386>": 152063,
+  "<387>": 152064,
+  "<388>": 152065,
+  "<389>": 152066,
+  "<38>": 151715,
+  "<390>": 152067,
+  "<391>": 152068,
+  "<392>": 152069,
+  "<393>": 152070,
+  "<394>": 152071,
+  "<395>": 152072,
+  "<396>": 152073,
+  "<397>": 152074,
+  "<398>": 152075,
+  "<399>": 152076,
+  "<39>": 151716,
+  "<3>": 151680,
+  "<400>": 152077,
+  "<401>": 152078,
+  "<402>": 152079,
+  "<403>": 152080,
+  "<404>": 152081,
+  "<405>": 152082,
+  "<406>": 152083,
+  "<407>": 152084,
+  "<408>": 152085,
+  "<409>": 152086,
+  "<40>": 151717,
+  "<410>": 152087,
+  "<411>": 152088,
+  "<412>": 152089,
+  "<413>": 152090,
+  "<414>": 152091,
+  "<415>": 152092,
+  "<416>": 152093,
+  "<417>": 152094,
+  "<418>": 152095,
+  "<419>": 152096,
+  "<41>": 151718,
+  "<420>": 152097,
+  "<421>": 152098,
+  "<422>": 152099,
+  "<423>": 152100,
+  "<424>": 152101,
+  "<425>": 152102,
+  "<426>": 152103,
+  "<427>": 152104,
+  "<428>": 152105,
+  "<429>": 152106,
+  "<42>": 151719,
+  "<430>": 152107,
+  "<431>": 152108,
+  "<432>": 152109,
+  "<433>": 152110,
+  "<434>": 152111,
+  "<435>": 152112,
+  "<436>": 152113,
+  "<437>": 152114,
+  "<438>": 152115,
+  "<439>": 152116,
+  "<43>": 151720,
+  "<440>": 152117,
+  "<441>": 152118,
+  "<442>": 152119,
+  "<443>": 152120,
+  "<444>": 152121,
+  "<445>": 152122,
+  "<446>": 152123,
+  "<447>": 152124,
+  "<448>": 152125,
+  "<449>": 152126,
+  "<44>": 151721,
+  "<450>": 152127,
+  "<451>": 152128,
+  "<452>": 152129,
+  "<453>": 152130,
+  "<454>": 152131,
+  "<455>": 152132,
+  "<456>": 152133,
+  "<457>": 152134,
+  "<458>": 152135,
+  "<459>": 152136,
+  "<45>": 151722,
+  "<460>": 152137,
+  "<461>": 152138,
+  "<462>": 152139,
+  "<463>": 152140,
+  "<464>": 152141,
+  "<465>": 152142,
+  "<466>": 152143,
+  "<467>": 152144,
+  "<468>": 152145,
+  "<469>": 152146,
+  "<46>": 151723,
+  "<470>": 152147,
+  "<471>": 152148,
+  "<472>": 152149,
+  "<473>": 152150,
+  "<474>": 152151,
+  "<475>": 152152,
+  "<476>": 152153,
+  "<477>": 152154,
+  "<478>": 152155,
+  "<479>": 152156,
+  "<47>": 151724,
+  "<480>": 152157,
+  "<481>": 152158,
+  "<482>": 152159,
+  "<483>": 152160,
+  "<484>": 152161,
+  "<485>": 152162,
+  "<486>": 152163,
+  "<487>": 152164,
+  "<488>": 152165,
+  "<489>": 152166,
+  "<48>": 151725,
+  "<490>": 152167,
+  "<491>": 152168,
+  "<492>": 152169,
+  "<493>": 152170,
+  "<494>": 152171,
+  "<495>": 152172,
+  "<496>": 152173,
+  "<497>": 152174,
+  "<498>": 152175,
+  "<499>": 152176,
+  "<49>": 151726,
+  "<4>": 151681,
+  "<500>": 152177,
+  "<501>": 152178,
+  "<502>": 152179,
+  "<503>": 152180,
+  "<504>": 152181,
+  "<505>": 152182,
+  "<506>": 152183,
+  "<507>": 152184,
+  "<508>": 152185,
+  "<509>": 152186,
+  "<50>": 151727,
+  "<510>": 152187,
+  "<511>": 152188,
+  "<512>": 152189,
+  "<513>": 152190,
+  "<514>": 152191,
+  "<515>": 152192,
+  "<516>": 152193,
+  "<517>": 152194,
+  "<518>": 152195,
+  "<519>": 152196,
+  "<51>": 151728,
+  "<520>": 152197,
+  "<521>": 152198,
+  "<522>": 152199,
+  "<523>": 152200,
+  "<524>": 152201,
+  "<525>": 152202,
+  "<526>": 152203,
+  "<527>": 152204,
+  "<528>": 152205,
+  "<529>": 152206,
+  "<52>": 151729,
+  "<530>": 152207,
+  "<531>": 152208,
+  "<532>": 152209,
+  "<533>": 152210,
+  "<534>": 152211,
+  "<535>": 152212,
+  "<536>": 152213,
+  "<537>": 152214,
+  "<538>": 152215,
+  "<539>": 152216,
+  "<53>": 151730,
+  "<540>": 152217,
+  "<541>": 152218,
+  "<542>": 152219,
+  "<543>": 152220,
+  "<544>": 152221,
+  "<545>": 152222,
+  "<546>": 152223,
+  "<547>": 152224,
+  "<548>": 152225,
+  "<549>": 152226,
+  "<54>": 151731,
+  "<550>": 152227,
+  "<551>": 152228,
+  "<552>": 152229,
+  "<553>": 152230,
+  "<554>": 152231,
+  "<555>": 152232,
+  "<556>": 152233,
+  "<557>": 152234,
+  "<558>": 152235,
+  "<559>": 152236,
+  "<55>": 151732,
+  "<560>": 152237,
+  "<561>": 152238,
+  "<562>": 152239,
+  "<563>": 152240,
+  "<564>": 152241,
+  "<565>": 152242,
+  "<566>": 152243,
+  "<567>": 152244,
+  "<568>": 152245,
+  "<569>": 152246,
+  "<56>": 151733,
+  "<570>": 152247,
+  "<571>": 152248,
+  "<572>": 152249,
+  "<573>": 152250,
+  "<574>": 152251,
+  "<575>": 152252,
+  "<576>": 152253,
+  "<577>": 152254,
+  "<578>": 152255,
+  "<579>": 152256,
+  "<57>": 151734,
+  "<580>": 152257,
+  "<581>": 152258,
+  "<582>": 152259,
+  "<583>": 152260,
+  "<584>": 152261,
+  "<585>": 152262,
+  "<586>": 152263,
+  "<587>": 152264,
+  "<588>": 152265,
+  "<589>": 152266,
+  "<58>": 151735,
+  "<590>": 152267,
+  "<591>": 152268,
+  "<592>": 152269,
+  "<593>": 152270,
+  "<594>": 152271,
+  "<595>": 152272,
+  "<596>": 152273,
+  "<597>": 152274,
+  "<598>": 152275,
+  "<599>": 152276,
+  "<59>": 151736,
+  "<5>": 151682,
+  "<600>": 152277,
+  "<601>": 152278,
+  "<602>": 152279,
+  "<603>": 152280,
+  "<604>": 152281,
+  "<605>": 152282,
+  "<606>": 152283,
+  "<607>": 152284,
+  "<608>": 152285,
+  "<609>": 152286,
+  "<60>": 151737,
+  "<610>": 152287,
+  "<611>": 152288,
+  "<612>": 152289,
+  "<613>": 152290,
+  "<614>": 152291,
+  "<615>": 152292,
+  "<616>": 152293,
+  "<617>": 152294,
+  "<618>": 152295,
+  "<619>": 152296,
+  "<61>": 151738,
+  "<620>": 152297,
+  "<621>": 152298,
+  "<622>": 152299,
+  "<623>": 152300,
+  "<624>": 152301,
+  "<625>": 152302,
+  "<626>": 152303,
+  "<627>": 152304,
+  "<628>": 152305,
+  "<629>": 152306,
+  "<62>": 151739,
+  "<630>": 152307,
+  "<631>": 152308,
+  "<632>": 152309,
+  "<633>": 152310,
+  "<634>": 152311,
+  "<635>": 152312,
+  "<636>": 152313,
+  "<637>": 152314,
+  "<638>": 152315,
+  "<639>": 152316,
+  "<63>": 151740,
+  "<640>": 152317,
+  "<641>": 152318,
+  "<642>": 152319,
+  "<643>": 152320,
+  "<644>": 152321,
+  "<645>": 152322,
+  "<646>": 152323,
+  "<647>": 152324,
+  "<648>": 152325,
+  "<649>": 152326,
+  "<64>": 151741,
+  "<650>": 152327,
+  "<651>": 152328,
+  "<652>": 152329,
+  "<653>": 152330,
+  "<654>": 152331,
+  "<655>": 152332,
+  "<656>": 152333,
+  "<657>": 152334,
+  "<658>": 152335,
+  "<659>": 152336,
+  "<65>": 151742,
+  "<660>": 152337,
+  "<661>": 152338,
+  "<662>": 152339,
+  "<663>": 152340,
+  "<664>": 152341,
+  "<665>": 152342,
+  "<666>": 152343,
+  "<667>": 152344,
+  "<668>": 152345,
+  "<669>": 152346,
+  "<66>": 151743,
+  "<670>": 152347,
+  "<671>": 152348,
+  "<672>": 152349,
+  "<673>": 152350,
+  "<674>": 152351,
+  "<675>": 152352,
+  "<676>": 152353,
+  "<677>": 152354,
+  "<678>": 152355,
+  "<679>": 152356,
+  "<67>": 151744,
+  "<680>": 152357,
+  "<681>": 152358,
+  "<682>": 152359,
+  "<683>": 152360,
+  "<684>": 152361,
+  "<685>": 152362,
+  "<686>": 152363,
+  "<687>": 152364,
+  "<688>": 152365,
+  "<689>": 152366,
+  "<68>": 151745,
+  "<690>": 152367,
+  "<691>": 152368,
+  "<692>": 152369,
+  "<693>": 152370,
+  "<694>": 152371,
+  "<695>": 152372,
+  "<696>": 152373,
+  "<697>": 152374,
+  "<698>": 152375,
+  "<699>": 152376,
+  "<69>": 151746,
+  "<6>": 151683,
+  "<700>": 152377,
+  "<701>": 152378,
+  "<702>": 152379,
+  "<703>": 152380,
+  "<704>": 152381,
+  "<705>": 152382,
+  "<706>": 152383,
+  "<707>": 152384,
+  "<708>": 152385,
+  "<709>": 152386,
+  "<70>": 151747,
+  "<710>": 152387,
+  "<711>": 152388,
+  "<712>": 152389,
+  "<713>": 152390,
+  "<714>": 152391,
+  "<715>": 152392,
+  "<716>": 152393,
+  "<717>": 152394,
+  "<718>": 152395,
+  "<719>": 152396,
+  "<71>": 151748,
+  "<720>": 152397,
+  "<721>": 152398,
+  "<722>": 152399,
+  "<723>": 152400,
+  "<724>": 152401,
+  "<725>": 152402,
+  "<726>": 152403,
+  "<727>": 152404,
+  "<728>": 152405,
+  "<729>": 152406,
+  "<72>": 151749,
+  "<730>": 152407,
+  "<731>": 152408,
+  "<732>": 152409,
+  "<733>": 152410,
+  "<734>": 152411,
+  "<735>": 152412,
+  "<736>": 152413,
+  "<737>": 152414,
+  "<738>": 152415,
+  "<739>": 152416,
+  "<73>": 151750,
+  "<740>": 152417,
+  "<741>": 152418,
+  "<742>": 152419,
+  "<743>": 152420,
+  "<744>": 152421,
+  "<745>": 152422,
+  "<746>": 152423,
+  "<747>": 152424,
+  "<748>": 152425,
+  "<749>": 152426,
+  "<74>": 151751,
+  "<750>": 152427,
+  "<751>": 152428,
+  "<752>": 152429,
+  "<753>": 152430,
+  "<754>": 152431,
+  "<755>": 152432,
+  "<756>": 152433,
+  "<757>": 152434,
+  "<758>": 152435,
+  "<759>": 152436,
+  "<75>": 151752,
+  "<760>": 152437,
+  "<761>": 152438,
+  "<762>": 152439,
+  "<763>": 152440,
+  "<764>": 152441,
+  "<765>": 152442,
+  "<766>": 152443,
+  "<767>": 152444,
+  "<768>": 152445,
+  "<769>": 152446,
+  "<76>": 151753,
+  "<770>": 152447,
+  "<771>": 152448,
+  "<772>": 152449,
+  "<773>": 152450,
+  "<774>": 152451,
+  "<775>": 152452,
+  "<776>": 152453,
+  "<777>": 152454,
+  "<778>": 152455,
+  "<779>": 152456,
+  "<77>": 151754,
+  "<780>": 152457,
+  "<781>": 152458,
+  "<782>": 152459,
+  "<783>": 152460,
+  "<784>": 152461,
+  "<785>": 152462,
+  "<786>": 152463,
+  "<787>": 152464,
+  "<788>": 152465,
+  "<789>": 152466,
+  "<78>": 151755,
+  "<790>": 152467,
+  "<791>": 152468,
+  "<792>": 152469,
+  "<793>": 152470,
+  "<794>": 152471,
+  "<795>": 152472,
+  "<796>": 152473,
+  "<797>": 152474,
+  "<798>": 152475,
+  "<799>": 152476,
+  "<79>": 151756,
+  "<7>": 151684,
+  "<800>": 152477,
+  "<801>": 152478,
+  "<802>": 152479,
+  "<803>": 152480,
+  "<804>": 152481,
+  "<805>": 152482,
+  "<806>": 152483,
+  "<807>": 152484,
+  "<808>": 152485,
+  "<809>": 152486,
+  "<80>": 151757,
+  "<810>": 152487,
+  "<811>": 152488,
+  "<812>": 152489,
+  "<813>": 152490,
+  "<814>": 152491,
+  "<815>": 152492,
+  "<816>": 152493,
+  "<817>": 152494,
+  "<818>": 152495,
+  "<819>": 152496,
+  "<81>": 151758,
+  "<820>": 152497,
+  "<821>": 152498,
+  "<822>": 152499,
+  "<823>": 152500,
+  "<824>": 152501,
+  "<825>": 152502,
+  "<826>": 152503,
+  "<827>": 152504,
+  "<828>": 152505,
+  "<829>": 152506,
+  "<82>": 151759,
+  "<830>": 152507,
+  "<831>": 152508,
+  "<832>": 152509,
+  "<833>": 152510,
+  "<834>": 152511,
+  "<835>": 152512,
+  "<836>": 152513,
+  "<837>": 152514,
+  "<838>": 152515,
+  "<839>": 152516,
+  "<83>": 151760,
+  "<840>": 152517,
+  "<841>": 152518,
+  "<842>": 152519,
+  "<843>": 152520,
+  "<844>": 152521,
+  "<845>": 152522,
+  "<846>": 152523,
+  "<847>": 152524,
+  "<848>": 152525,
+  "<849>": 152526,
+  "<84>": 151761,
+  "<850>": 152527,
+  "<851>": 152528,
+  "<852>": 152529,
+  "<853>": 152530,
+  "<854>": 152531,
+  "<855>": 152532,
+  "<856>": 152533,
+  "<857>": 152534,
+  "<858>": 152535,
+  "<859>": 152536,
+  "<85>": 151762,
+  "<860>": 152537,
+  "<861>": 152538,
+  "<862>": 152539,
+  "<863>": 152540,
+  "<864>": 152541,
+  "<865>": 152542,
+  "<866>": 152543,
+  "<867>": 152544,
+  "<868>": 152545,
+  "<869>": 152546,
+  "<86>": 151763,
+  "<870>": 152547,
+  "<871>": 152548,
+  "<872>": 152549,
+  "<873>": 152550,
+  "<874>": 152551,
+  "<875>": 152552,
+  "<876>": 152553,
+  "<877>": 152554,
+  "<878>": 152555,
+  "<879>": 152556,
+  "<87>": 151764,
+  "<880>": 152557,
+  "<881>": 152558,
+  "<882>": 152559,
+  "<883>": 152560,
+  "<884>": 152561,
+  "<885>": 152562,
+  "<886>": 152563,
+  "<887>": 152564,
+  "<888>": 152565,
+  "<889>": 152566,
+  "<88>": 151765,
+  "<890>": 152567,
+  "<891>": 152568,
+  "<892>": 152569,
+  "<893>": 152570,
+  "<894>": 152571,
+  "<895>": 152572,
+  "<896>": 152573,
+  "<897>": 152574,
+  "<898>": 152575,
+  "<899>": 152576,
+  "<89>": 151766,
+  "<8>": 151685,
+  "<900>": 152577,
+  "<901>": 152578,
+  "<902>": 152579,
+  "<903>": 152580,
+  "<904>": 152581,
+  "<905>": 152582,
+  "<906>": 152583,
+  "<907>": 152584,
+  "<908>": 152585,
+  "<909>": 152586,
+  "<90>": 151767,
+  "<910>": 152587,
+  "<911>": 152588,
+  "<912>": 152589,
+  "<913>": 152590,
+  "<914>": 152591,
+  "<915>": 152592,
+  "<916>": 152593,
+  "<917>": 152594,
+  "<918>": 152595,
+  "<919>": 152596,
+  "<91>": 151768,
+  "<920>": 152597,
+  "<921>": 152598,
+  "<922>": 152599,
+  "<923>": 152600,
+  "<924>": 152601,
+  "<925>": 152602,
+  "<926>": 152603,
+  "<927>": 152604,
+  "<928>": 152605,
+  "<929>": 152606,
+  "<92>": 151769,
+  "<930>": 152607,
+  "<931>": 152608,
+  "<932>": 152609,
+  "<933>": 152610,
+  "<934>": 152611,
+  "<935>": 152612,
+  "<936>": 152613,
+  "<937>": 152614,
+  "<938>": 152615,
+  "<939>": 152616,
+  "<93>": 151770,
+  "<940>": 152617,
+  "<941>": 152618,
+  "<942>": 152619,
+  "<943>": 152620,
+  "<944>": 152621,
+  "<945>": 152622,
+  "<946>": 152623,
+  "<947>": 152624,
+  "<948>": 152625,
+  "<949>": 152626,
+  "<94>": 151771,
+  "<950>": 152627,
+  "<951>": 152628,
+  "<952>": 152629,
+  "<953>": 152630,
+  "<954>": 152631,
+  "<955>": 152632,
+  "<956>": 152633,
+  "<957>": 152634,
+  "<958>": 152635,
+  "<959>": 152636,
+  "<95>": 151772,
+  "<960>": 152637,
+  "<961>": 152638,
+  "<962>": 152639,
+  "<963>": 152640,
+  "<964>": 152641,
+  "<965>": 152642,
+  "<966>": 152643,
+  "<967>": 152644,
+  "<968>": 152645,
+  "<969>": 152646,
+  "<96>": 151773,
+  "<970>": 152647,
+  "<971>": 152648,
+  "<972>": 152649,
+  "<973>": 152650,
+  "<974>": 152651,
+  "<975>": 152652,
+  "<976>": 152653,
+  "<977>": 152654,
+  "<978>": 152655,
+  "<979>": 152656,
+  "<97>": 151774,
+  "<980>": 152657,
+  "<981>": 152658,
+  "<982>": 152659,
+  "<983>": 152660,
+  "<984>": 152661,
+  "<985>": 152662,
+  "<986>": 152663,
+  "<987>": 152664,
+  "<988>": 152665,
+  "<989>": 152666,
+  "<98>": 151775,
+  "<990>": 152667,
+  "<991>": 152668,
+  "<992>": 152669,
+  "<993>": 152670,
+  "<994>": 152671,
+  "<995>": 152672,
+  "<996>": 152673,
+  "<997>": 152674,
+  "<998>": 152675,
+  "<999>": 152676,
+  "<99>": 151776,
+  "<9>": 151686,
+  "<IMG_CONTEXT>": 151665,
+  "<box>": 151668,
+  "<img>": 151666,
+  "<interval>": 151674,
+  "<null>": 152678,
+  "<quad>": 151670,
+  "<ref>": 151672,
+  "<switch>": 152679,
+  "<text_mask>": 151676,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

qwen2.5_tokenizer/chat_template.jinja

@@ -0,0 +1,54 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- messages[0]['content'] }}
+    {%- else %}
+        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+    {%- endif %}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+        {%- for tool_call in message.tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

qwen2.5_tokenizer/chat_template.json

@@ -0,0 +1,4 @@
+{
+    "chat_template": "{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n{% endif %}<|im_start|>{{ message['role'] }}\n{% if message['content'] is string %}{{ message['content'] }}<|im_end|>\n{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}<image {{ image_count.value }}>{% endif %}<image-{{ image_count.value }}>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}<video {{ video_count.value }}>{% endif %}<video-{{ video_count.value }}>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>\n{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}"
+}
+  

qwen2.5_tokenizer/config.json

@@ -0,0 +1,125 @@
+{
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "auto_map": {
+    "AutoConfig": "configuration_qwen2.Qwen2Config",
+    "AutoModelForCausalLM": "modeling_qwen2.Qwen2ForCausalLM"
+  },
+  "block_size": 6,
+  "bos_token_id": 151643,
+  "causal_attn": false,
+  "dtype": "bfloat16",
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "layer_types": [
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 32768,
+  "max_window_layers": 70,
+  "model_type": "qwen2",
+  "null_token_id": 152678,
+  "num_attention_heads": 16,
+  "num_hidden_layers": 36,
+  "num_key_value_heads": 2,
+  "pad_token_id": 151643,
+  "quantization_config": {
+    "bits": 4,
+    "checkpoint_format": "gptq",
+    "desc_act": false,
+    "format": "gptq",
+    "group_size": 128,
+    "lm_head": false,
+    "meta": {
+      "act_group_aware": true,
+      "auto_forward_data_parallel": true,
+      "damp_auto_increment": 0.01,
+      "damp_percent": 0.05,
+      "dense_vram_strategy": "exclusive",
+      "dense_vram_strategy_devices": null,
+      "fallback": {
+        "smooth": null,
+        "strategy": "rtn",
+        "threshold": "0.5%"
+      },
+      "foem": null,
+      "gc_mode": "interval",
+      "gptaq": null,
+      "hessian": {
+        "chunk_bytes": null,
+        "chunk_size": null,
+        "staging_dtype": "float32"
+      },
+      "mock_quantization": false,
+      "moe_vram_strategy": "exclusive",
+      "moe_vram_strategy_devices": null,
+      "mse": 0.0,
+      "offload_to_disk": true,
+      "offload_to_disk_path": "/tmp/gptqmodel_42jjgpwr",
+      "pack_impl": "cpu",
+      "quantizer": [
+        "gptqmodel:7.1.0"
+      ],
+      "static_groups": false,
+      "true_sequential": true,
+      "uri": "https://github.com/modelcloud/gptqmodel",
+      "wait_for_submodule_finalizers": false
+    },
+    "method": "gptq",
+    "pack_dtype": "int32",
+    "quant_method": "gptq",
+    "sym": true
+  },
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "sliding_window": null,
+  "switch_token_id": 152679,
+  "text_mask_token_id": 151676,
+  "tie_word_embeddings": true,
+  "transformers_version": "5.10.2",
+  "use_cache": false,
+  "use_sliding_window": false,
+  "vocab_size": 152681
+}

qwen2.5_tokenizer/configuration_qwen2.py

@@ -0,0 +1,148 @@
+# coding=utf-8
+# Copyright 2024 The Qwen team, Alibaba Group and the HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" Qwen2 model configuration"""
+
+from transformers.configuration_utils import PretrainedConfig
+from transformers.utils import logging
+
+
+logger = logging.get_logger(__name__)
+
+QWEN2_PRETRAINED_CONFIG_ARCHIVE_MAP = {
+    "Qwen/Qwen2-7B-beta": "https://huggingface.co/Qwen/Qwen2-7B-beta/resolve/main/config.json",
+}
+
+
+class Qwen2Config(PretrainedConfig):
+    r"""
+    This is the configuration class to store the configuration of a [`Qwen2Model`]. It is used to instantiate a
+    Qwen2 model according to the specified arguments, defining the model architecture. Instantiating a configuration
+    with the defaults will yield a similar configuration to that of
+    Qwen2-7B-beta [Qwen/Qwen2-7B-beta](https://huggingface.co/Qwen/Qwen2-7B-beta).
+
+    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+    documentation from [`PretrainedConfig`] for more information.
+
+
+    Args:
+        vocab_size (`int`, *optional*, defaults to 151936):
+            Vocabulary size of the Qwen2 model. Defines the number of different tokens that can be represented by the
+            `inputs_ids` passed when calling [`Qwen2Model`]
+        hidden_size (`int`, *optional*, defaults to 4096):
+            Dimension of the hidden representations.
+        intermediate_size (`int`, *optional*, defaults to 22016):
+            Dimension of the MLP representations.
+        num_hidden_layers (`int`, *optional*, defaults to 32):
+            Number of hidden layers in the Transformer encoder.
+        num_attention_heads (`int`, *optional*, defaults to 32):
+            Number of attention heads for each attention layer in the Transformer encoder.
+        num_key_value_heads (`int`, *optional*, defaults to 32):
+            This is the number of key_value heads that should be used to implement Grouped Query Attention. If
+            `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
+            `num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When
+            converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
+            by meanpooling all the original heads within that group. For more details checkout [this
+            paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to `32`.
+        hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
+            The non-linear activation function (function or string) in the decoder.
+        max_position_embeddings (`int`, *optional*, defaults to 32768):
+            The maximum sequence length that this model might ever be used with.
+        initializer_range (`float`, *optional*, defaults to 0.02):
+            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
+        rms_norm_eps (`float`, *optional*, defaults to 1e-06):
+            The epsilon used by the rms normalization layers.
+        use_cache (`bool`, *optional*, defaults to `True`):
+            Whether or not the model should return the last key/values attentions (not used by all models). Only
+            relevant if `config.is_decoder=True`.
+        tie_word_embeddings (`bool`, *optional*, defaults to `False`):
+            Whether the model's input and output word embeddings should be tied.
+        rope_theta (`float`, *optional*, defaults to 10000.0):
+            The base period of the RoPE embeddings.
+        use_sliding_window (`bool`, *optional*, defaults to `False`):
+            Whether to use sliding window attention.
+        sliding_window (`int`, *optional*, defaults to 4096):
+            Sliding window attention (SWA) window size. If not specified, will default to `4096`.
+        max_window_layers (`int`, *optional*, defaults to 28):
+            The number of layers that use SWA (Sliding Window Attention). The bottom layers use SWA while the top use full attention.
+        attention_dropout (`float`, *optional*, defaults to 0.0):
+            The dropout ratio for the attention probabilities.
+
+    ```python
+    >>> from transformers import Qwen2Model, Qwen2Config
+
+    >>> # Initializing a Qwen2 style configuration
+    >>> configuration = Qwen2Config()
+
+    >>> # Initializing a model from the Qwen2-7B style configuration
+    >>> model = Qwen2Model(configuration)
+
+    >>> # Accessing the model configuration
+    >>> configuration = model.config
+    ```"""
+
+    model_type = "qwen2"
+    keys_to_ignore_at_inference = ["past_key_values"]
+
+    def __init__(
+        self,
+        vocab_size=151936,
+        hidden_size=4096,
+        intermediate_size=22016,
+        num_hidden_layers=32,
+        num_attention_heads=32,
+        num_key_value_heads=32,
+        hidden_act="silu",
+        max_position_embeddings=32768,
+        initializer_range=0.02,
+        rms_norm_eps=1e-6,
+        use_cache=True,
+        tie_word_embeddings=False,
+        rope_theta=10000.0,
+        use_sliding_window=False,
+        sliding_window=4096,
+        max_window_layers=28,
+        attention_dropout=0.0,
+        **kwargs,
+    ):
+        self.vocab_size = vocab_size
+        self.max_position_embeddings = max_position_embeddings
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.use_sliding_window = use_sliding_window
+        self.sliding_window = sliding_window
+        self.max_window_layers = max_window_layers
+
+        # for backward compatibility
+        if num_key_value_heads is None:
+            num_key_value_heads = num_attention_heads
+
+        self.num_key_value_heads = num_key_value_heads
+        self.hidden_act = hidden_act
+        self.initializer_range = initializer_range
+        self.rms_norm_eps = rms_norm_eps
+        self.use_cache = use_cache
+        self.rope_theta = rope_theta
+        self.attention_dropout = attention_dropout
+        if kwargs.get('attn_implementation', None) is None:
+            self.attn_implementation = kwargs['attn_implementation'] = 'flash_attention_2'
+        else:
+            self.attn_implementation = kwargs['attn_implementation']
+
+        super().__init__(
+            tie_word_embeddings=tie_word_embeddings,
+            **kwargs,
+        )

qwen2.5_tokenizer/generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": 151645,
+  "transformers_version": "5.10.2"
+}

qwen2.5_tokenizer/mask_magi_utils.py

@@ -0,0 +1,101 @@
+# Copyright (c) 2026, NVIDIA CORPORATION.  All rights reserved.
+#
+# NVIDIA CORPORATION and its licensors retain all intellectual property
+# and proprietary rights in and to this software, related documentation
+# and any modifications thereto.  Any use, reproduction, disclosure or
+# distribution of this software and related documentation without an express
+# license agreement from NVIDIA CORPORATION is strictly prohibited.
+
+import torch
+
+# MagiAttention attn_type_map convention
+FULL, CAUSAL = 0, 1
+
+def build_magi_ranges(kv_len: int, q_len: int, block_size: int, ar_decode: bool=False, device: str = "cpu"):
+    """
+    Fixed strategy:
+      - use_cache=True: Mask blocked_k = (kv_len - block_size - 1) column
+      - causal_attn=False: Window interior is FULL (bidirectional)
+      - If q_len==kv_len: Use coarse prefix version (fewer ranges)
+      - Otherwise: General decode version (recompute rows expanding visible region row by row)
+
+    Conventions:
+      - K/V global length kv_len: [0, kv_len)
+      - Current Q is "last q_len tokens"
+      - First r=q_len-block_size rows are recomputed; last block_size rows are window
+    """
+    assert 0 < q_len <= kv_len
+
+    if ar_decode:
+        return {
+            "q_ranges": torch.tensor([[0, q_len]], dtype=torch.int32, device=device).contiguous(),
+            "k_ranges": torch.tensor([[0, kv_len]], dtype=torch.int32, device=device).contiguous(),
+            "attn_type_map": torch.tensor([CAUSAL],    dtype=torch.int32, device=device).contiguous(),
+        }
+
+
+    assert 0 < block_size <= q_len <= kv_len
+    B = block_size
+    r = q_len - B
+    q_global_start = kv_len - q_len
+
+    window_start_k = kv_len - B
+    blocked_k = window_start_k - 1  # The column that is blocked
+
+    q_ranges, k_ranges, types = [], [], []
+
+    # -------- prefix (q_len == kv_len) coarse-grained --------
+    if q_len == kv_len:
+        prefix_len = window_start_k  # kv_len - B
+
+        # prefix->prefix: causal
+        if prefix_len > 0:
+            q_ranges += [[0, prefix_len]]
+            k_ranges += [[0, prefix_len]]
+            types   += [CAUSAL]
+
+        # window->prefix: full, but exclude blocked_k => keys [0, blocked_k)
+        if prefix_len > 0 and blocked_k > 0:
+            q_ranges += [[prefix_len, kv_len]]
+            k_ranges += [[0, blocked_k]]
+            types   += [FULL]
+
+        # window->window: full
+        q_ranges += [[prefix_len, kv_len]]
+        k_ranges += [[prefix_len, kv_len]]
+        types += [FULL]
+
+        return {
+            "q_ranges": torch.tensor(q_ranges, dtype=torch.int32, device=device).contiguous(),
+            "k_ranges": torch.tensor(k_ranges, dtype=torch.int32, device=device).contiguous(),
+            "attn_type_map": torch.tensor(types,    dtype=torch.int32, device=device).contiguous(),
+        }
+
+    # -------- decode / general (q_len < kv_len) --------
+
+    # A) Recomputed rows: expand visible key cutoff row by row (use FULL + single-row q_range for precise shape)
+    for i in range(r):
+        g = q_global_start + i
+        q_ranges.append([i, i + 1])
+        k_ranges.append([0, g + 1])   # Allow keys [0, g]
+        types.append(FULL)
+
+    # B) Window rows: allow prefix but block blocked_k; window interior is full
+    q_win = [r, q_len]
+
+    # prefix keys [0, blocked_k)
+    if blocked_k > 0:
+        q_ranges.append(q_win)
+        k_ranges.append([0, blocked_k])
+        types.append(FULL)
+
+    # window keys [window_start_k, kv_len)
+    q_ranges.append(q_win)
+    k_ranges.append([window_start_k, kv_len])
+    types.append(FULL)
+
+    return {
+        "q_ranges": torch.tensor(q_ranges, dtype=torch.int32, device=device).contiguous(),
+        "k_ranges": torch.tensor(k_ranges, dtype=torch.int32, device=device).contiguous(),
+        "attn_type_map": torch.tensor(types, dtype=torch.int32, device=device).contiguous(),
+    }

qwen2.5_tokenizer/mask_sdpa_utils.py

@@ -0,0 +1,232 @@
+# Copyright (c) 2026, NVIDIA CORPORATION.  All rights reserved.
+#
+# NVIDIA CORPORATION and its licensors retain all intellectual property
+# and proprietary rights in and to this software, related documentation
+# and any modifications thereto.  Any use, reproduction, disclosure or
+# distribution of this software and related documentation without an express
+# license agreement from NVIDIA CORPORATION is strictly prohibited.
+
+import torch
+
+
+def find_prefix_seq_length_by_pe(
+    pe: torch.Tensor
+) -> torch.Tensor:
+    """
+    Find the sequence length where position encoding drops (indicating prefix boundary).
+    Args:
+        pe: Position encoding tensor of shape [Batch size, Sequence length ]
+            Contains position indices for each token in the sequence.
+    Returns:
+        torch.Tensor: A tensor of shape [B] containing:
+            - The index where position encoding drops for each sequence
+            - -1 if no drop occurs in the sequence
+    """
+    batch_size, seq_len = pe.shape
+    prev = pe[:, :-1]
+    curr = pe[:, 1:]
+    drop_mask = curr < prev  #  [batch_size, seq_len-1]
+
+    seq_len = torch.full((batch_size,), -1, dtype=torch.long)
+    
+    for b in range(batch_size):
+        drop_pos = torch.nonzero(drop_mask[b], as_tuple=False)
+        if drop_pos.numel() > 0:
+            i = drop_pos[0].item() + 1  # Take first drop position (+1 because we compared shifted sequences)
+            seq_len[b] = i
+
+    return seq_len
+
+
+
+def update_causal_mask_with_pad_non_visible_2d(
+    input_ids: torch.Tensor,
+    attn_mask_2d: torch.Tensor,
+    text_mask_token_id: int,
+    block_size: int = 4,
+    causal_attn: bool = False
+) -> torch.Tensor:
+    """
+    Updates a 2D attention mask for hole sequence through input_ids and text_mask_token_id
+
+    Args:
+        input_ids: Input token IDs (unused in current implementation)
+        attn_mask_2d: 2D attention mask matrix of shape [seq_len, seq_len] where:
+            - 0.0 indicates allowed attention
+            - -inf indicates masked attention
+        text_mask_token_id: ID representing masked tokens
+        block_size: Size of the diffusion window
+        causal_attn: If True, maintains strict causal masking throughout
+    
+    Returns:
+        Modified attention mask with updated visibility patterns
+    """
+    seq_len = input_ids.shape[0]
+    device = input_ids.device
+
+    # Identify masked tokens and their preceding positions
+    input_mask = input_ids.eq(text_mask_token_id)
+    input_before_mask = torch.zeros_like(input_mask)
+    input_before_mask[:-1] = input_mask[1:]
+    mask_cols = (input_mask | input_before_mask)
+    non_mask = ~mask_cols
+
+    rows = torch.arange(seq_len, device=device)[:, None]
+    cols = torch.arange(seq_len, device=device)
+
+    indices = torch.arange(seq_len, device=device)
+    prev_non_mask = (indices * non_mask).cummax(dim=0).values
+
+    max_value = torch.iinfo(indices.dtype).max
+    mask_indices = torch.where(non_mask, indices, torch.full_like(indices, max_value))
+    reversed_mask_indices = torch.flip(mask_indices, dims=[0])
+    reversed_cummin = reversed_mask_indices.cummin(dim=0).values
+    next_non_mask = torch.flip(reversed_cummin, dims=[0])
+
+    infra_mask = (
+        (cols > prev_non_mask) &
+        (rows >= next_non_mask[None, :]) &
+        mask_cols[None, :]
+    )
+    attn_mask_2d.masked_fill_(infra_mask, -float('inf'))
+
+    if not causal_attn:
+        visible_mask = (
+            (rows > prev_non_mask[None, :]) &
+            (rows < cols) &
+            mask_cols[None, :]
+        )
+        attn_mask_2d.masked_fill_(visible_mask, 0.0)
+
+    return attn_mask_2d
+
+
+def update_causal_mask_for_one_gen_window_2d(
+    input_ids: torch.Tensor,
+    attn_mask_2d: torch.Tensor,
+    block_size: int = 4,
+    use_cache: bool = True,
+    causal_attn: bool = False
+) -> torch.Tensor:
+    """
+    Updates a 2D attention mask for a diffusion window in transformer inference.
+
+    Args:
+        input_ids: Input token IDs (unused in current implementation)
+        attn_mask_2d: 2D attention mask matrix of shape [seq_len, seq_len] where:
+            - 0.0 indicates allowed attention
+            - -inf indicates masked attention
+        block_size: Size of the diffusion window
+        use_cache: Whether key-value cache is being used
+        causal_attn: If True, maintains strict causal masking throughout
+    
+    Returns:
+        Modified attention mask with updated visibility patterns
+    """
+
+    if not causal_attn:
+        # Make the diffusion window (last block_size tokens) fully visible to itself
+        # This allows bidirectional attention within the diffusion window
+        attn_mask_2d[-block_size:, -block_size:] = 0.0
+    if use_cache:
+        # Mask the last token from previous round to prevent recomputation and maintain generation consistency.
+        attn_mask_2d[-block_size:, -block_size-1] = -float('inf')
+
+    return attn_mask_2d
+
+
+def create_block_diff_mask_by_pe_4d(
+    block_size: int, 
+    x0_len_list: torch.Tensor, 
+    position_ids: torch.Tensor, 
+    causal_attn: bool = False
+) -> tuple[torch.Tensor, torch.Tensor]:
+    """Generates a 4D attention mask for block-difference attention patterns.
+
+    The mask consists of three regions:
+    1. Causal block (top-left): Standard causal attention for `x0` tokens.
+    2. Mutual block (bottom-right): Non-causal attention within the same block for non-`x0` tokens.
+    3. Prefix block (bottom-left): Non-`x0` tokens can attend to a prefix of `x0` tokens.
+
+    Args:
+        block_size (int): Size of processing blocks for non-`x0` tokens.
+        x0_len_list (torch.Tensor): Tensor of shape [B] containing lengths of `x0` segments per batch.
+        position_ids (torch.Tensor): Tensor of shape [B, seq_len] containing position IDs.
+        causal_attn (bool, optional): If True, enforces causal masking in mutual blocks. Defaults to False.
+
+    Returns:
+        tuple[torch.Tensor, torch.Tensor]:
+            - A float mask of shape [batch_size, 1, seq_len, seq_len] with `-inf` for masked positions (non visiable).
+            - A boolean mask of shape [batch_size, 1, seq_len, seq_len] indicating allowed attention positions.
+    """
+    batch_size, seq_len = position_ids.shape
+    device = position_ids.device
+    
+    # Create position indices [batch_size, seq_len, seq_len]
+    q_idx = torch.arange(seq_len, device=device).view(1, seq_len, 1)  # [1, seq_len, 1]
+    kv_idx = torch.arange(seq_len, device=device).view(1, 1, seq_len)  # [1, 1, seq_len]
+    
+    # Broadcast to [B, seq_len, seq_len]
+    x0_len = x0_len_list.view(batch_size, 1, 1)  # [batch_size, 1, 1]
+    x0_flag_q = q_idx < x0_len           # [batch_size, seq_len, seq_len]
+    x0_flag_kv = kv_idx < x0_len         
+    
+    # Block indices calculation [batch_size, seq_len, seq_len]
+    q_block_idx = (q_idx - x0_len) // block_size
+    kv_block_idx = (kv_idx - x0_len) // block_size
+    
+    # causal block (top-left)
+    block_causal = x0_flag_q & x0_flag_kv & (q_idx >= kv_idx)
+
+    mutual_condition = (q_idx >= kv_idx) if causal_attn else torch.ones_like(q_idx, dtype=torch.bool)
+    block_mutual = (
+        ~x0_flag_q & ~x0_flag_kv &
+        (q_block_idx == kv_block_idx) &
+        mutual_condition
+    )
+
+    q_blk = torch.div(q_idx - x0_len, block_size, rounding_mode='floor')
+    q_blk_start = (x0_len_list.view(batch_size, 1) + q_blk[:, :, 0] * block_size).clamp(min=0, max=seq_len - 1)
+    prefix_len = position_ids.gather(1, q_blk_start)
+    prefix_len = prefix_len.unsqueeze(2)
+    block_prefix = (~x0_flag_q & x0_flag_kv) & (kv_idx < prefix_len)
+
+    final_mask = (block_causal | block_mutual | block_prefix)
+    customized_mask = torch.full_like(final_mask, float('-inf'), dtype=torch.bfloat16)
+    customized_mask.masked_fill_(final_mask, 0.0)
+    
+    return customized_mask.unsqueeze(1).to(device=device), final_mask.unsqueeze(1).to(device=device)
+
+
+def find_pred_pos_from_input_ids(
+    input_ids: torch.LongTensor = None,
+    text_mask_token_id: int = None,
+) -> torch.Tensor:
+    """Compute the relative prediction positions for masked tokens in a sequence.
+
+    For non-masked positions, the output is 0. For masked positions, the value increments
+    by 1 for each consecutive mask token, indicating how many steps ahead the prediction is.
+
+    Args:
+        input_ids (torch.LongTensor): Input token IDs of shape [batch_size, seq_len].
+        text_mask_token_id (int, optional): Token ID representing masked positions. Defaults to 151666.
+
+    Returns:
+        torch.Tensor: A tensor of shape [batch_size, seq_len] where:
+            - 0 indicates a non-masked token.
+            - n > 0 indicates the nth consecutive masked token (e.g., 1 = first mask, 2 = second mask, etc.).
+    """
+    batch_size, seq_len = input_ids.shape
+    device = input_ids.device
+
+    is_mask = (input_ids == text_mask_token_id)
+
+    base_mask = torch.zeros((batch_size, seq_len), dtype=torch.int8, device=device)
+
+    for b in range(batch_size):
+        for ix in range(1, seq_len):
+            if is_mask[b][ix] == True:
+                # Increment counter if current token is masked
+                base_mask[b][ix] = base_mask[b][ix-1] + 1
+
+    return base_mask

qwen2.5_tokenizer/model.safetensors.index.json

@@ -0,0 +1,442 @@
+{
+  "metadata": {
+    "total_size": 6800310272
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00002-of-00002.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.32.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.32.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.32.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.33.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.33.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.33.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.34.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.34.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.34.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.35.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.35.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.35.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.norm.weight": "model-00002-of-00002.safetensors"
+  }
+}

qwen2.5_tokenizer/model.safetensors.index.json.bak

@@ -0,0 +1,442 @@
+{
+  "metadata": {
+    "total_size": 6800310272
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00002-of-00002.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.32.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.32.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.32.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.33.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.33.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.33.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.34.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.34.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.34.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.35.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.35.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.35.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.norm.weight": "model-00002-of-00002.safetensors"
+  }
+}

qwen2.5_tokenizer/modeling_qwen2.py

@@ -0,0 +1,1738 @@
+# coding=utf-8
+# Copyright 2024 The Qwen team, Alibaba Group and the HuggingFace Inc. team. All rights reserved.
+#
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
+# and OPT implementations in this library. It has been modified from its
+# original forms to accommodate minor architectural differences compared
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" PyTorch Qwen2 model."""
+import inspect
+import math
+import copy
+import warnings
+from functools import partial
+from typing import List, Optional, Tuple, Union
+
+import torch
+import torch.nn.functional as F
+import torch.utils.checkpoint
+from torch import nn
+from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss
+
+from transformers.activations import ACT2FN
+from transformers.cache_utils import Cache, DynamicCache
+from transformers.modeling_attn_mask_utils import _prepare_4d_causal_attention_mask, _prepare_4d_causal_attention_mask_for_sdpa
+from transformers.modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast, SequenceClassifierOutputWithPast
+from transformers.modeling_utils import PreTrainedModel
+from transformers.utils import (
+    add_start_docstrings,
+    add_start_docstrings_to_model_forward,
+    is_flash_attn_2_available,
+    is_flash_attn_greater_or_equal_2_10,
+    logging,
+    replace_return_docstrings,
+)
+from .configuration_qwen2 import Qwen2Config
+
+if is_flash_attn_2_available():
+    from flash_attn import flash_attn_func, flash_attn_varlen_func
+    from flash_attn.bert_padding import index_first_axis, pad_input, unpad_input  # noqa
+
+    _flash_supports_window_size = "window_size" in list(inspect.signature(flash_attn_func).parameters)
+
+
+logger = logging.get_logger(__name__)
+
+# Magi Attention Supported
+_MAGI_AVAILABLE = False
+try:
+    from magi_attention.functional.flex_flash_attn import flex_flash_attn_func
+    _MAGI_AVAILABLE = True
+except ImportError:
+    flex_flash_attn_func = None
+
+
+_CHECKPOINT_FOR_DOC = "Qwen/Qwen2-7B-beta"
+_CONFIG_FOR_DOC = "Qwen2Config"
+
+QWEN2_PRETRAINED_MODEL_ARCHIVE_LIST = [
+    "Qwen/Qwen2-7B-beta",
+    # See all Qwen2 models at https://huggingface.co/models?filter=qwen2
+]
+
+from .mask_sdpa_utils import (
+    find_prefix_seq_length_by_pe,
+    update_causal_mask_with_pad_non_visible_2d,
+    update_causal_mask_for_one_gen_window_2d,
+    create_block_diff_mask_by_pe_4d,
+    find_pred_pos_from_input_ids
+)
+
+from .mask_magi_utils import build_magi_ranges
+
+# Copied from transformers.models.llama.modeling_llama._get_unpad_data
+def _get_unpad_data(attention_mask):
+    seqlens_in_batch = attention_mask.sum(dim=-1, dtype=torch.int32)
+    indices = torch.nonzero(attention_mask.flatten(), as_tuple=False).flatten()
+    max_seqlen_in_batch = seqlens_in_batch.max().item()
+    cu_seqlens = F.pad(torch.cumsum(seqlens_in_batch, dim=0, dtype=torch.torch.int32), (1, 0))
+    return (
+        indices,
+        cu_seqlens,
+        max_seqlen_in_batch,
+    )
+
+
+# Copied from transformers.models.llama.modeling_llama.LlamaRMSNorm with Llama->Qwen2
+class Qwen2RMSNorm(nn.Module):
+    def __init__(self, hidden_size, eps=1e-6):
+        """
+        Qwen2RMSNorm is equivalent to T5LayerNorm
+        """
+        super().__init__()
+        self.weight = nn.Parameter(torch.ones(hidden_size))
+        self.variance_epsilon = eps
+
+    def forward(self, hidden_states):
+        input_dtype = hidden_states.dtype
+        hidden_states = hidden_states.to(torch.float32)
+        variance = hidden_states.pow(2).mean(-1, keepdim=True)
+        hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
+        return self.weight * hidden_states.to(input_dtype)
+
+
+# Copied from transformers.models.llama.modeling_llama.LlamaRotaryEmbedding with Llama->Qwen2
+class Qwen2RotaryEmbedding(nn.Module):
+    def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None):
+        super().__init__()
+
+        self.dim = dim
+        self.max_position_embeddings = max_position_embeddings
+        self.base = base
+        inv_freq = 1.0 / (self.base ** (torch.arange(0, self.dim, 2).float().to(device) / self.dim))
+        self.register_buffer("inv_freq", inv_freq, persistent=False)
+
+        # Build here to make `torch.jit.trace` work.
+        self._set_cos_sin_cache(
+            seq_len=max_position_embeddings, device=self.inv_freq.device, dtype=torch.get_default_dtype()
+        )
+
+    def _set_cos_sin_cache(self, seq_len, device, dtype):
+        self.max_seq_len_cached = seq_len
+        t = torch.arange(self.max_seq_len_cached, device=device, dtype=self.inv_freq.dtype)
+
+        freqs = torch.outer(t, self.inv_freq)
+        # Different from paper, but it uses a different permutation in order to obtain the same calculation
+        emb = torch.cat((freqs, freqs), dim=-1)
+        self.register_buffer("cos_cached", emb.cos().to(dtype), persistent=False)
+        self.register_buffer("sin_cached", emb.sin().to(dtype), persistent=False)
+
+    def forward(self, x, seq_len=None):
+        # x: [bs, num_attention_heads, seq_len, head_size]
+        if seq_len > self.max_seq_len_cached:
+            self._set_cos_sin_cache(seq_len=seq_len, device=x.device, dtype=x.dtype)
+
+        return (
+            self.cos_cached[:seq_len].to(dtype=x.dtype),
+            self.sin_cached[:seq_len].to(dtype=x.dtype),
+        )
+
+
+# Copied from transformers.models.llama.modeling_llama.rotate_half
+def rotate_half(x):
+    """Rotates half the hidden dims of the input."""
+    x1 = x[..., : x.shape[-1] // 2]
+    x2 = x[..., x.shape[-1] // 2 :]
+    return torch.cat((-x2, x1), dim=-1)
+
+
+# Copied from transformers.models.llama.modeling_llama.apply_rotary_pos_emb
+def apply_rotary_pos_emb(q, k, cos, sin, position_ids, unsqueeze_dim=1):
+    """Applies Rotary Position Embedding to the query and key tensors.
+
+    Args:
+        q (`torch.Tensor`): The query tensor.
+        k (`torch.Tensor`): The key tensor.
+        cos (`torch.Tensor`): The cosine part of the rotary embedding.
+        sin (`torch.Tensor`): The sine part of the rotary embedding.
+        position_ids (`torch.Tensor`):
+            The position indices of the tokens corresponding to the query and key tensors. For example, this can be
+            used to pass offsetted position ids when working with a KV-cache.
+        unsqueeze_dim (`int`, *optional*, defaults to 1):
+            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
+            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
+            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
+            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
+            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
+            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
+    Returns:
+        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
+    """
+    cos = cos[position_ids].unsqueeze(unsqueeze_dim)
+    sin = sin[position_ids].unsqueeze(unsqueeze_dim)
+    q_embed = (q * cos) + (rotate_half(q) * sin)
+    k_embed = (k * cos) + (rotate_half(k) * sin)
+    return q_embed, k_embed
+
+
+# Copied from transformers.models.mistral.modeling_mistral.MistralMLP with Mistral->Qwen2
+class Qwen2MLP(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.config = config
+        self.hidden_size = config.hidden_size
+        self.intermediate_size = config.intermediate_size
+        self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)
+        self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)
+        self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=False)
+        self.act_fn = ACT2FN[config.hidden_act]
+
+    def forward(self, x):
+        return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))
+
+
+# Copied from transformers.models.llama.modeling_llama.repeat_kv
+def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:
+    """
+    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
+    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
+    """
+    batch, num_key_value_heads, slen, head_dim = hidden_states.shape
+    if n_rep == 1:
+        return hidden_states
+    hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim)
+    return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim)
+
+
+class Qwen2Attention(nn.Module):
+    """
+    Multi-headed attention from 'Attention Is All You Need' paper. Modified to use sliding window attention: Longformer
+    and "Generating Long Sequences with Sparse Transformers".
+    """
+
+    def __init__(self, config: Qwen2Config, layer_idx: Optional[int] = None):
+        super().__init__()
+        self.config = config
+        self.layer_idx = layer_idx
+        if layer_idx is None:
+            logger.warning_once(
+                f"Instantiating {self.__class__.__name__} without passing `layer_idx` is not recommended and will "
+                "to errors during the forward call, if caching is used. Please make sure to provide a `layer_idx` "
+                "when creating this class."
+            )
+
+        self.hidden_size = config.hidden_size
+        self.num_heads = config.num_attention_heads
+        self.head_dim = self.hidden_size // self.num_heads
+        self.num_key_value_heads = config.num_key_value_heads
+        self.num_key_value_groups = self.num_heads // self.num_key_value_heads
+        self.max_position_embeddings = config.max_position_embeddings
+        self.rope_theta = config.rope_theta
+        self.is_causal = True
+        self.attention_dropout = config.attention_dropout
+
+        if (self.head_dim * self.num_heads) != self.hidden_size:
+            raise ValueError(
+                f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}"
+                f" and `num_heads`: {self.num_heads})."
+            )
+        self.q_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=True)
+        self.k_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=True)
+        self.v_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=True)
+        self.o_proj = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=False)
+
+        self.rotary_emb = Qwen2RotaryEmbedding(
+            self.head_dim,
+            max_position_embeddings=self.max_position_embeddings,
+            base=self.rope_theta,
+        )
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_value: Optional[Cache] = None,
+        output_attentions: bool = False,
+        use_cache: bool = False,
+        **kwargs,
+    ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
+        if "padding_mask" in kwargs:
+            warnings.warn(
+                "Passing `padding_mask` is deprecated and will be removed in v4.37. Please make sure use `attention_mask` instead.`"
+            )
+        bsz, q_len, _ = hidden_states.size()
+
+        query_states = self.q_proj(hidden_states)
+        key_states = self.k_proj(hidden_states)
+        value_states = self.v_proj(hidden_states)
+
+        query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
+        key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
+        value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
+
+        kv_seq_len = key_states.shape[-2]
+        if past_key_value is not None:
+            if self.layer_idx is None:
+                raise ValueError(
+                    f"The cache structure has changed since version v4.36. If you are using {self.__class__.__name__} "
+                    "for auto-regressive decoding with k/v caching, please make sure to initialize the attention class "
+                    "with a layer index."
+                )
+            kv_seq_len += past_key_value.get_seq_length(self.layer_idx)
+        cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len)
+        query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids)
+
+        if past_key_value is not None:
+            cache_kwargs = {"sin": sin, "cos": cos}  # Specific to RoPE models
+            key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs)
+
+        # repeat k/v heads if n_kv_heads < n_heads
+        key_states = repeat_kv(key_states, self.num_key_value_groups)
+        value_states = repeat_kv(value_states, self.num_key_value_groups)
+
+        attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim)
+
+        if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len):
+            raise ValueError(
+                f"Attention weights should be of size {(bsz, self.num_heads, q_len, kv_seq_len)}, but is"
+                f" {attn_weights.size()}"
+            )
+
+        if attention_mask is not None:
+            if attention_mask.size() != (bsz, 1, q_len, kv_seq_len):
+                raise ValueError(
+                    f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}"
+                )
+
+            attn_weights = attn_weights + attention_mask
+
+        # upcast attention to fp32
+        attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype)
+        attn_weights = nn.functional.dropout(attn_weights, p=self.attention_dropout, training=self.training)
+        attn_output = torch.matmul(attn_weights, value_states)
+
+        if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim):
+            raise ValueError(
+                f"`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is"
+                f" {attn_output.size()}"
+            )
+
+        attn_output = attn_output.transpose(1, 2).contiguous()
+        attn_output = attn_output.reshape(bsz, q_len, self.hidden_size)
+
+        attn_output = self.o_proj(attn_output)
+
+        if not output_attentions:
+            attn_weights = None
+
+        return attn_output, attn_weights, past_key_value
+
+
+class Qwen2FlashAttention2(Qwen2Attention):
+    """
+    Qwen2 flash attention module, following Qwen2 attention module. This module inherits from `Qwen2Attention`
+    as the weights of the module stays untouched. The only required change would be on the forward pass
+    where it needs to correctly call the public API of flash attention and deal with padding tokens
+    in case the input contains any of them. Additionally, for sliding window attention, we apply SWA only to the bottom
+    config.max_window_layers layers.
+    """
+
+    # Copied from transformers.models.llama.modeling_llama.LlamaFlashAttention2.__init__
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        # TODO: Should be removed once Flash Attention for RoCm is bumped to 2.1.
+        # flash_attn<2.1 generates top-left aligned causal mask, while what is needed here is bottom-right alignement, that was made default for flash_attn>=2.1. This attribute is used to handle this difference. Reference: https://github.com/Dao-AILab/flash-attention/releases/tag/v2.1.0.
+        # Beware that with flash_attn<2.1, using q_seqlen != k_seqlen (except for the case q_seqlen == 1) produces a wrong mask (top-left).
+        self._flash_attn_uses_top_left_mask = not is_flash_attn_greater_or_equal_2_10()
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_value: Optional[Cache] = None,
+        output_attentions: bool = False,
+        use_cache: bool = False,
+        **kwargs,
+    ):
+        if "padding_mask" in kwargs:
+            warnings.warn(
+                "Passing `padding_mask` is deprecated and will be removed in v4.37. Please make sure use `attention_mask` instead.`"
+            )
+
+            # overwrite attention_mask with padding_mask
+            attention_mask = kwargs.pop("padding_mask")
+        bsz, q_len, _ = hidden_states.size()
+
+        query_states = self.q_proj(hidden_states)
+        key_states = self.k_proj(hidden_states)
+        value_states = self.v_proj(hidden_states)
+
+        query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
+        key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
+        value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
+
+        kv_seq_len = key_states.shape[-2]
+        if past_key_value is not None:
+            if self.layer_idx is None:
+                raise ValueError(
+                    f"The cache structure has changed since version v4.36. If you are using {self.__class__.__name__} "
+                    "for auto-regressive decoding with k/v caching, please make sure to initialize the attention class "
+                    "with a layer index."
+                )
+            kv_seq_len += past_key_value.get_seq_length(self.layer_idx)
+
+        # Because the input can be padded, the absolute sequence length depends on the max position id.
+        rotary_seq_len = max(kv_seq_len, position_ids[:, -1].max().item()) + 1
+        cos, sin = self.rotary_emb(value_states, seq_len=rotary_seq_len)
+
+        query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids)
+
+        use_sliding_windows = (
+            _flash_supports_window_size
+            and getattr(self.config, "sliding_window", None) is not None
+            and kv_seq_len > self.config.sliding_window
+            and self.config.use_sliding_window
+        )
+
+        if not _flash_supports_window_size:
+            logger.warning_once(
+                "The current flash attention version does not support sliding window attention, for a more memory efficient implementation"
+                " make sure to upgrade flash-attn library."
+            )
+
+        if past_key_value is not None:
+            # Activate slicing cache only if the config has a value `sliding_windows` attribute
+            cache_has_contents = past_key_value.get_seq_length(self.layer_idx) > 0
+            if (
+                getattr(self.config, "sliding_window", None) is not None
+                and kv_seq_len > self.config.sliding_window
+                and cache_has_contents
+            ):
+                slicing_tokens = 1 - self.config.sliding_window
+
+                past_key = past_key_value[self.layer_idx][0]
+                past_value = past_key_value[self.layer_idx][1]
+
+                past_key = past_key[:, :, slicing_tokens:, :].contiguous()
+                past_value = past_value[:, :, slicing_tokens:, :].contiguous()
+
+                if past_key.shape[-2] != self.config.sliding_window - 1:
+                    raise ValueError(
+                        f"past key must have a shape of (`batch_size, num_heads, self.config.sliding_window-1, head_dim`), got"
+                        f" {past_key.shape}"
+                    )
+
+                if attention_mask is not None:
+                    attention_mask = attention_mask[:, slicing_tokens:]
+                    attention_mask = torch.cat([attention_mask, torch.ones_like(attention_mask[:, -1:])], dim=-1)
+
+            cache_kwargs = {"sin": sin, "cos": cos}  # Specific to RoPE models
+            key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs)
+
+        # repeat k/v heads if n_kv_heads < n_heads
+        key_states = repeat_kv(key_states, self.num_key_value_groups)
+        value_states = repeat_kv(value_states, self.num_key_value_groups)
+        dropout_rate = 0.0 if not self.training else self.attention_dropout
+
+        # In PEFT, usually we cast the layer norms in float32 for training stability reasons
+        # therefore the input hidden states gets silently casted in float32. Hence, we need
+        # cast them back in float16 just to be sure everything works as expected.
+        input_dtype = query_states.dtype
+        if input_dtype == torch.float32:
+            if torch.is_autocast_enabled():
+                target_dtype = torch.get_autocast_gpu_dtype()
+            # Handle the case where the model is quantized
+            elif hasattr(self.config, "_pre_quantization_dtype"):
+                target_dtype = self.config._pre_quantization_dtype
+            else:
+                target_dtype = self.q_proj.weight.dtype
+
+            logger.warning_once(
+                f"The input hidden states seems to be silently casted in float32, this might be related to"
+                f" the fact you have upcasted embedding or layer norm layers in float32. We will cast back the input in"
+                f" {target_dtype}."
+            )
+
+            query_states = query_states.to(target_dtype)
+            key_states = key_states.to(target_dtype)
+            value_states = value_states.to(target_dtype)
+
+        # Reashape to the expected shape for Flash Attention
+        query_states = query_states.transpose(1, 2)
+        key_states = key_states.transpose(1, 2)
+        value_states = value_states.transpose(1, 2)
+
+        attn_output = self._flash_attention_forward(
+            query_states,
+            key_states,
+            value_states,
+            attention_mask,
+            q_len,
+            dropout=dropout_rate,
+            use_sliding_windows=use_sliding_windows,
+        )
+
+        attn_output = attn_output.reshape(bsz, q_len, self.hidden_size).contiguous()
+        attn_output = self.o_proj(attn_output)
+
+        if not output_attentions:
+            attn_weights = None
+
+        return attn_output, attn_weights, past_key_value
+
+    def _flash_attention_forward(
+        self,
+        query_states,
+        key_states,
+        value_states,
+        attention_mask,
+        query_length,
+        dropout=0.0,
+        softmax_scale=None,
+        use_sliding_windows=False,
+    ):
+        """
+        Calls the forward method of Flash Attention - if the input hidden states contain at least one padding token
+        first unpad the input, then computes the attention scores and pad the final attention scores.
+
+        Args:
+            query_states (`torch.Tensor`):
+                Input query states to be passed to Flash Attention API
+            key_states (`torch.Tensor`):
+                Input key states to be passed to Flash Attention API
+            value_states (`torch.Tensor`):
+                Input value states to be passed to Flash Attention API
+            attention_mask (`torch.Tensor`):
+                The padding mask - corresponds to a tensor of size `(batch_size, seq_len)` where 0 stands for the
+                position of padding tokens and 1 for the position of non-padding tokens.
+            dropout (`int`, *optional*):
+                Attention dropout
+            softmax_scale (`float`, *optional*):
+                The scaling of QK^T before applying softmax. Default to 1 / sqrt(head_dim)
+            use_sliding_windows (`bool`, *optional*):
+                Whether to activate sliding window attention.
+        """
+        if not self._flash_attn_uses_top_left_mask:
+            causal = self.is_causal
+        else:
+            # TODO: Remove the `query_length != 1` check once Flash Attention for RoCm is bumped to 2.1. For details, please see the comment in LlamaFlashAttention2 __init__.
+            causal = self.is_causal and query_length != 1
+
+        # Decide whether to use SWA or not by layer index.
+        if use_sliding_windows and self.layer_idx >= self.config.max_window_layers:
+            use_sliding_windows = False
+
+        # Contains at least one padding token in the sequence
+        if attention_mask is not None:
+            batch_size = query_states.shape[0]
+            query_states, key_states, value_states, indices_q, cu_seq_lens, max_seq_lens = self._upad_input(
+                query_states, key_states, value_states, attention_mask, query_length
+            )
+
+            cu_seqlens_q, cu_seqlens_k = cu_seq_lens
+            max_seqlen_in_batch_q, max_seqlen_in_batch_k = max_seq_lens
+
+            if not use_sliding_windows:
+                attn_output_unpad = flash_attn_varlen_func(
+                    query_states,
+                    key_states,
+                    value_states,
+                    cu_seqlens_q=cu_seqlens_q,
+                    cu_seqlens_k=cu_seqlens_k,
+                    max_seqlen_q=max_seqlen_in_batch_q,
+                    max_seqlen_k=max_seqlen_in_batch_k,
+                    dropout_p=dropout,
+                    softmax_scale=softmax_scale,
+                    causal=causal,
+                )
+            else:
+                attn_output_unpad = flash_attn_varlen_func(
+                    query_states,
+                    key_states,
+                    value_states,
+                    cu_seqlens_q=cu_seqlens_q,
+                    cu_seqlens_k=cu_seqlens_k,
+                    max_seqlen_q=max_seqlen_in_batch_q,
+                    max_seqlen_k=max_seqlen_in_batch_k,
+                    dropout_p=dropout,
+                    softmax_scale=softmax_scale,
+                    causal=causal,
+                    window_size=(self.config.sliding_window, self.config.sliding_window),
+                )
+
+            attn_output = pad_input(attn_output_unpad, indices_q, batch_size, query_length)
+        else:
+            if not use_sliding_windows:
+                attn_output = flash_attn_func(
+                    query_states,
+                    key_states,
+                    value_states,
+                    dropout,
+                    softmax_scale=softmax_scale,
+                    causal=causal,
+                )
+            else:
+                attn_output = flash_attn_func(
+                    query_states,
+                    key_states,
+                    value_states,
+                    dropout,
+                    softmax_scale=softmax_scale,
+                    causal=causal,
+                    window_size=(self.config.sliding_window, self.config.sliding_window),
+                )
+
+        return attn_output
+
+    # Copied from transformers.models.mistral.modeling_mistral.MistralFlashAttention2._upad_input
+    def _upad_input(self, query_layer, key_layer, value_layer, attention_mask, query_length):
+        batch_size, kv_seq_len, num_heads, head_dim = key_layer.shape
+
+        # On the first iteration we need to properly re-create the padding mask
+        # by slicing it on the proper place
+        if kv_seq_len != attention_mask.shape[-1]:
+            attention_mask_num_tokens = attention_mask.shape[-1]
+            attention_mask = attention_mask[:, attention_mask_num_tokens - kv_seq_len :]
+
+        indices_k, cu_seqlens_k, max_seqlen_in_batch_k = _get_unpad_data(attention_mask)
+
+        key_layer = index_first_axis(key_layer.reshape(batch_size * kv_seq_len, num_heads, head_dim), indices_k)
+        value_layer = index_first_axis(value_layer.reshape(batch_size * kv_seq_len, num_heads, head_dim), indices_k)
+
+        if query_length == kv_seq_len:
+            query_layer = index_first_axis(
+                query_layer.reshape(batch_size * kv_seq_len, num_heads, head_dim), indices_k
+            )
+            cu_seqlens_q = cu_seqlens_k
+            max_seqlen_in_batch_q = max_seqlen_in_batch_k
+            indices_q = indices_k
+        elif query_length == 1:
+            max_seqlen_in_batch_q = 1
+            cu_seqlens_q = torch.arange(
+                batch_size + 1, dtype=torch.int32, device=query_layer.device
+            )  # There is a memcpy here, that is very bad.
+            indices_q = cu_seqlens_q[:-1]
+            query_layer = query_layer.squeeze(1)
+        else:
+            # The -q_len: slice assumes left padding.
+            attention_mask = attention_mask[:, -query_length:]
+            query_layer, indices_q, cu_seqlens_q, max_seqlen_in_batch_q = unpad_input(query_layer, attention_mask)
+
+        return (
+            query_layer,
+            key_layer,
+            value_layer,
+            indices_q,
+            (cu_seqlens_q, cu_seqlens_k),
+            (max_seqlen_in_batch_q, max_seqlen_in_batch_k),
+        )
+
+
+# Copied from transformers.models.llama.modeling_llama.LlamaSdpaAttention with Llama->Qwen2
+class Qwen2SdpaAttention(Qwen2Attention):
+    """
+    Qwen2 attention module using torch.nn.functional.scaled_dot_product_attention. This module inherits from
+    `Qwen2Attention` as the weights of the module stays untouched. The only changes are on the forward pass to adapt to
+    SDPA API.
+    """
+
+    # Adapted from Qwen2Attention.forward
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_value: Optional[Cache] = None,
+        output_attentions: bool = False,
+        use_cache: bool = False,
+    ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
+        if output_attentions:
+            # TODO: Improve this warning with e.g. `model.config.attn_implementation = "manual"` once this is implemented.
+            logger.warning_once(
+                "Qwen2Model is using Qwen2SdpaAttention, but `torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to the manual attention implementation, "
+                'but specifying the manual implementation will be required from Transformers version v5.0.0 onwards. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.'
+            )
+            return super().forward(
+                hidden_states=hidden_states,
+                attention_mask=attention_mask,
+                position_ids=position_ids,
+                past_key_value=past_key_value,
+                output_attentions=output_attentions,
+                use_cache=use_cache,
+            )
+
+        bsz, q_len, _ = hidden_states.size()
+
+        query_states = self.q_proj(hidden_states)
+        key_states = self.k_proj(hidden_states)
+        value_states = self.v_proj(hidden_states)
+
+        query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
+        key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
+        value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
+
+        kv_seq_len = key_states.shape[-2]
+        if past_key_value is not None:
+            kv_seq_len += past_key_value.get_seq_length(self.layer_idx)
+        cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len)
+
+        query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids)
+
+        if past_key_value is not None:
+            cache_kwargs = {"sin": sin, "cos": cos}  # Specific to RoPE models
+            key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs)
+
+        key_states = repeat_kv(key_states, self.num_key_value_groups)
+        value_states = repeat_kv(value_states, self.num_key_value_groups)
+
+        if attention_mask is not None:
+            if attention_mask.size() != (bsz, 1, q_len, kv_seq_len):
+                raise ValueError(
+                    f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}"
+                )
+
+        # SDPA with memory-efficient backend is currently (torch==2.1.2) bugged with non-contiguous inputs with custom attn_mask,
+        # Reference: https://github.com/pytorch/pytorch/issues/112577.
+        if query_states.device.type == "cuda" and attention_mask is not None:
+            query_states = query_states.contiguous()
+            key_states = key_states.contiguous()
+            value_states = value_states.contiguous()
+
+        attn_output = torch.nn.functional.scaled_dot_product_attention(
+            query_states,
+            key_states,
+            value_states,
+            attn_mask=attention_mask,
+            dropout_p=self.attention_dropout if self.training else 0.0,
+            is_causal=False,
+        )
+
+        attn_output = attn_output.transpose(1, 2).contiguous()
+        attn_output = attn_output.reshape(bsz, q_len, self.hidden_size)
+
+        attn_output = self.o_proj(attn_output)
+
+        return attn_output, None, past_key_value
+
+
+class Qwen2SdpaAttentionGqa(Qwen2Attention):
+    """
+    Qwen2 attention module using torch.nn.functional.scaled_dot_product_attention. This module inherits from
+    `Qwen2Attention` as the weights of the module stays untouched. The only changes are on the forward pass to adapt to
+    SDPA API.
+    """
+
+    # Adapted from Qwen2Attention.forward
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_value: Optional[Cache] = None,
+        output_attentions: bool = False,
+        use_cache: bool = False,
+    ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
+        if output_attentions:
+            # TODO: Improve this warning with e.g. `model.config.attn_implementation = "manual"` once this is implemented.
+            logger.warning_once(
+                "Qwen2Model is using Qwen2SdpaAttention, but `torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to the manual attention implementation, "
+                'but specifying the manual implementation will be required from Transformers version v5.0.0 onwards. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.'
+            )
+            return super().forward(
+                hidden_states=hidden_states,
+                attention_mask=attention_mask,
+                position_ids=position_ids,
+                past_key_value=past_key_value,
+                output_attentions=output_attentions,
+                use_cache=use_cache,
+            )
+
+        bsz, q_len, _ = hidden_states.size()
+
+        query_states = self.q_proj(hidden_states)
+        key_states = self.k_proj(hidden_states)
+        value_states = self.v_proj(hidden_states)
+
+        query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
+        key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
+        value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
+
+        kv_seq_len = key_states.shape[-2]
+        if past_key_value is not None:
+            kv_seq_len += past_key_value.get_seq_length(self.layer_idx)
+        cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len)
+
+        query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids)
+
+        if past_key_value is not None:
+            cache_kwargs = {"sin": sin, "cos": cos}  # Specific to RoPE models
+            key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs)
+
+        # key_states = repeat_kv(key_states, self.num_key_value_groups)
+        # value_states = repeat_kv(value_states, self.num_key_value_groups)
+
+        if attention_mask is not None:
+            if attention_mask.size() != (bsz, 1, q_len, kv_seq_len):
+                raise ValueError(
+                    f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}"
+                )
+
+        # SDPA with memory-efficient backend is currently (torch==2.1.2) bugged with non-contiguous inputs with custom attn_mask,
+        # Reference: https://github.com/pytorch/pytorch/issues/112577.
+        if query_states.device.type == "cuda" and attention_mask is not None:
+            query_states = query_states.contiguous()
+            key_states = key_states.contiguous()
+            value_states = value_states.contiguous()
+
+        with torch.backends.cuda.sdp_kernel(enable_flash=True,   
+                                    enable_math=True, 
+                                    enable_mem_efficient=False):
+
+            attn_output = torch.nn.functional.scaled_dot_product_attention(
+                query_states,
+                key_states,
+                value_states,
+                attn_mask=attention_mask,
+                enable_gqa=True,
+                dropout_p=self.attention_dropout if self.training else 0.0,
+                is_causal=False,
+            )
+
+        attn_output = attn_output.transpose(1, 2).contiguous()
+        attn_output = attn_output.reshape(bsz, q_len, self.hidden_size)
+
+        attn_output = self.o_proj(attn_output)
+
+        return attn_output, None, past_key_value
+
+
+class Qwen2MagiAttention(Qwen2Attention):
+    """
+    Qwen2 attention using MagiAttention for efficient training with MTP packing support.
+    
+    MagiAttention uses range-based sparse attention patterns:
+    - q_ranges/k_ranges define which query/key ranges attend to each other
+    - attn_type_map specifies causal(1) or full(0) attention for each range pair
+    """
+    
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        if not _MAGI_AVAILABLE:
+            raise ImportError(
+                "magi_attention is not installed. Install with: pip install magi-attention"
+            )
+        self.softmax_scale = self.head_dim ** -0.5
+    
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attention_mask: Optional[dict] = None,  # magi_plan dict
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_value: Optional[Cache] = None,
+        output_attentions: bool = False,
+        use_cache: bool = False,
+    ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
+        if output_attentions:
+            raise NotImplementedError('MagiAttention does not support output_attentions=True')
+
+        bsz, q_len, _ = hidden_states.size()
+        assert bsz == 1, "MagiAttention only supports batch_size=1 (use packing instead)"
+
+        query_states = self.q_proj(hidden_states)
+        key_states = self.k_proj(hidden_states)
+        value_states = self.v_proj(hidden_states)
+
+        # Magi expects [T, H, D] format (no batch dimension)
+        query_states = query_states.view(q_len, self.num_heads, self.head_dim)
+        key_states = key_states.view(q_len, self.num_key_value_heads, self.head_dim)
+        value_states = value_states.view(q_len, self.num_key_value_heads, self.head_dim)
+
+        kv_seq_len = q_len
+        if past_key_value is not None:
+            if self.layer_idx is None:
+                raise ValueError(
+                    f"The cache structure has changed since version v4.36. If you are using {self.__class__.__name__} "
+                    "for auto-regressive decoding with k/v caching, please make sure to initialize the attention class "
+                    "with a layer index."
+                )
+            kv_seq_len += past_key_value.get_seq_length(self.layer_idx)
+        
+        cos, sin = self.rotary_emb(value_states.unsqueeze(0).transpose(1, 2), seq_len=kv_seq_len)
+        
+        # Apply RoPE: need [B, H, L, D] format for apply_rotary_pos_emb
+        q_for_rope = query_states.unsqueeze(0).transpose(1, 2)  # [1, H, L, D]
+        k_for_rope = key_states.unsqueeze(0).transpose(1, 2)    # [1, Hkv, L, D]
+        q_for_rope, k_for_rope = apply_rotary_pos_emb(q_for_rope, k_for_rope, cos, sin, position_ids)
+        
+        # Back to [T, H, D]
+        query_states = q_for_rope.squeeze(0).transpose(0, 1).contiguous()  # [L, H, D]
+        key_states = k_for_rope.squeeze(0).transpose(0, 1).contiguous()    # [L, Hkv, D]
+
+        if past_key_value is not None:
+            cache_kwargs = {"sin": sin, "cos": cos}
+            # Note: Magi doesn't support KV cache in training, this is for potential future use
+            key_states_4d = key_states.unsqueeze(0).transpose(1, 2)
+            value_states_4d = value_states.unsqueeze(0).transpose(1, 2)
+            key_states_4d, value_states_4d = past_key_value.update(
+                key_states_4d, value_states_4d, self.layer_idx, cache_kwargs
+            )
+            key_states = key_states_4d.squeeze(0).transpose(0, 1).contiguous()
+            value_states = value_states_4d.squeeze(0).transpose(0, 1).contiguous()
+
+        # Run Magi Attention
+        # attention_mask is a magi_plan dict with q_ranges, k_ranges, attn_type_map, etc.
+
+        attn_output, _ = flex_flash_attn_func(
+            query_states.contiguous(),
+            key_states.contiguous(),
+            value_states.contiguous(),
+            q_ranges=attention_mask["q_ranges"],
+            k_ranges=attention_mask["k_ranges"],
+            attn_type_map=attention_mask["attn_type_map"],
+            softmax_scale=self.softmax_scale,
+            softcap=0.0,
+            deterministic=False,
+        )  # [T, H, D]
+        
+        # Reshape to [B, L, H*D]
+        attn_output = attn_output.view(1, q_len, self.hidden_size)
+        attn_output = self.o_proj(attn_output)
+
+        return attn_output, None, past_key_value
+
+
+QWEN2_ATTENTION_CLASSES = {
+    "eager": Qwen2Attention,
+    "flash_attention_2": Qwen2FlashAttention2,
+    "sdpa": Qwen2SdpaAttention,
+    "magi": Qwen2MagiAttention,
+}
+
+
+class Qwen2DecoderLayer(nn.Module):
+    def __init__(self, config: Qwen2Config, layer_idx: int):
+        super().__init__()
+        self.hidden_size = config.hidden_size
+
+        if config._attn_implementation == 'magi' and not _MAGI_AVAILABLE:
+            if is_flash_attn_2_available():
+                logger.warning_once(
+                    'magi_attention not available, falling back to flash_attention_2'
+                )
+                config._attn_implementation = 'flash_attention_2'
+            else:
+                logger.warning_once(
+                    'magi_attention not available, falling back to sdpa'
+                )
+                config._attn_implementation = 'sdpa'
+        if config._attn_implementation == 'flash_attention_2' and not is_flash_attn_2_available():
+            logger.warning_once(
+                'flash_attn is not available, falling back to sdpa'
+            )
+            config._attn_implementation = 'sdpa'
+
+        self.self_attn = QWEN2_ATTENTION_CLASSES[config._attn_implementation](config, layer_idx)
+
+        self.mlp = Qwen2MLP(config)
+        self.input_layernorm = Qwen2RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+        self.post_attention_layernorm = Qwen2RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_value: Optional[Tuple[torch.Tensor]] = None,
+        output_attentions: Optional[bool] = False,
+        use_cache: Optional[bool] = False,
+        **kwargs,
+    ) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]:
+        if "padding_mask" in kwargs:
+            warnings.warn(
+                "Passing `padding_mask` is deprecated and will be removed in v4.37. "
+                "Please make sure use `attention_mask` instead.`"
+            )
+        """
+        Args:
+            hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
+            attention_mask (`torch.FloatTensor`, *optional*): attention mask of size
+                `(batch, sequence_length)` where padding elements are indicated by 0.
+            output_attentions (`bool`, *optional*):
+                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
+                returned tensors for more detail.
+            use_cache (`bool`, *optional*):
+                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
+                (see `past_key_values`).
+            past_key_value (`Tuple(torch.FloatTensor)`, *optional*): cached past key and value projection states
+        """
+
+        residual = hidden_states
+
+        hidden_states = self.input_layernorm(hidden_states)
+
+        # Self Attention
+        hidden_states, self_attn_weights, present_key_value = self.self_attn(
+            hidden_states=hidden_states,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_value=past_key_value,
+            output_attentions=output_attentions,
+            use_cache=use_cache,
+        )
+        hidden_states = residual + hidden_states
+
+        # Fully Connected
+        residual = hidden_states
+        hidden_states = self.post_attention_layernorm(hidden_states)
+        hidden_states = self.mlp(hidden_states)
+        hidden_states = residual + hidden_states
+
+        outputs = (hidden_states,)
+
+        if output_attentions:
+            outputs += (self_attn_weights,)
+
+        if use_cache:
+            outputs += (present_key_value,)
+
+        return outputs
+
+
+QWEN2_START_DOCSTRING = r"""
+    This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+    library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+    etc.)
+
+    This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+    Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+    and behavior.
+
+    Parameters:
+        config ([`Qwen2Config`]):
+            Model configuration class with all the parameters of the model. Initializing with a config file does not
+            load the weights associated with the model, only the configuration. Check out the
+            [`~PreTrainedModel.from_pretrained`] method to load the model weights.
+"""
+
+
+@add_start_docstrings(
+    "The bare Qwen2 Model outputting raw hidden-states without any specific head on top.",
+    QWEN2_START_DOCSTRING,
+)
+class Qwen2PreTrainedModel(PreTrainedModel):
+    config_class = Qwen2Config
+    base_model_prefix = "model"
+    supports_gradient_checkpointing = True
+    _no_split_modules = ["Qwen2DecoderLayer"]
+    _skip_keys_device_placement = "past_key_values"
+    _supports_flash_attn_2 = True
+    _supports_sdpa = True
+    _supports_cache_class = True
+
+    @classmethod
+    def _autoset_attn_implementation(cls, config, *args, **kwargs):
+        if getattr(config, '_attn_implementation', None) == 'magi':
+            return config
+        return super()._autoset_attn_implementation(config, *args, **kwargs)
+
+    def _check_and_adjust_attn_implementation(self, attn_implementation, is_init_check=False):
+        if attn_implementation == "magi":
+            return "magi"
+        return super()._check_and_adjust_attn_implementation(attn_implementation, is_init_check)
+
+    def _init_weights(self, module):
+        std = self.config.initializer_range
+        if isinstance(module, nn.Linear):
+            module.weight.data.normal_(mean=0.0, std=std)
+            if module.bias is not None:
+                module.bias.data.zero_()
+        elif isinstance(module, nn.Embedding):
+            module.weight.data.normal_(mean=0.0, std=std)
+            if module.padding_idx is not None:
+                module.weight.data[module.padding_idx].zero_()
+
+
+QWEN2_INPUTS_DOCSTRING = r"""
+    Args:
+        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
+            Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
+            it.
+
+            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+            [`PreTrainedTokenizer.__call__`] for details.
+
+            [What are input IDs?](../glossary#input-ids)
+        attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
+            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
+
+            - 1 for tokens that are **not masked**,
+            - 0 for tokens that are **masked**.
+
+            [What are attention masks?](../glossary#attention-mask)
+
+            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+            [`PreTrainedTokenizer.__call__`] for details.
+
+            If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see
+            `past_key_values`).
+
+            If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`]
+            and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more
+            information on the default strategy.
+
+            - 1 indicates the head is **not masked**,
+            - 0 indicates the head is **masked**.
+        position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+            config.n_positions - 1]`.
+
+            [What are position IDs?](../glossary#position-ids)
+        past_key_values (`Cache` or `tuple(tuple(torch.FloatTensor))`, *optional*):
+            Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
+            blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values`
+            returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`.
+
+            Two formats are allowed:
+            - a [`~cache_utils.Cache`] instance;
+            - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of
+            shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`). This is also known as the legacy
+            cache format.
+
+            The model will output the same cache format that is fed as input. If no `past_key_values` are passed, the
+            legacy cache format will be returned.
+
+            If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't
+            have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids`
+            of shape `(batch_size, sequence_length)`.
+        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
+            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+            is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+            model's internal embedding lookup matrix.
+        use_cache (`bool`, *optional*):
+            If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+            `past_key_values`).
+        output_attentions (`bool`, *optional*):
+            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
+            tensors for more detail.
+        output_hidden_states (`bool`, *optional*):
+            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
+            more detail.
+        return_dict (`bool`, *optional*):
+            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
+"""
+
+
+@add_start_docstrings(
+    "The bare Qwen2 Model outputting raw hidden-states without any specific head on top.",
+    QWEN2_START_DOCSTRING,
+)
+class Qwen2Model(Qwen2PreTrainedModel):
+    """
+    Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`Qwen2DecoderLayer`]
+
+    Args:
+        config: Qwen2Config
+    """
+
+    def __init__(self, config: Qwen2Config):
+        super().__init__(config)
+        self.padding_idx = config.pad_token_id
+        self.vocab_size = config.vocab_size
+
+        self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx)
+        self.layers = nn.ModuleList(
+            [Qwen2DecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)]
+        )
+        self._attn_implementation = config._attn_implementation
+        self.norm = Qwen2RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+
+        self.gradient_checkpointing = False
+        # Initialize weights and apply final processing
+        self.post_init()
+        
+        self.block_size = getattr(config, 'block_size', 6)
+        self.causal_attn = getattr(config, 'causal_attn', False)
+        self.text_mask_token_id = getattr(config, 'text_mask_token_id', 151676)
+
+
+    def get_input_embeddings(self):
+        return self.embed_tokens
+
+    def set_input_embeddings(self, value):
+        self.embed_tokens = value
+
+    def image_processing(self, input_ids, visual_features, image_token_index):
+        if visual_features is not None:
+            input_embeds = self.get_input_embeddings()(input_ids)
+            B, N, C = input_embeds.shape
+            input_embeds = input_embeds.reshape(B * N, C)
+
+            input_ids = input_ids.reshape(B * N)
+            selected = (input_ids == image_token_index)
+            assert selected.sum() != 0
+            input_embeds[selected] = visual_features.reshape(-1, C).to(input_embeds.device)
+            input_embeds = input_embeds.reshape(B, N, C)
+        else:
+            input_embeds = self.get_input_embeddings()(input_ids)
+        return input_embeds
+        
+    @add_start_docstrings_to_model_forward(QWEN2_INPUTS_DOCSTRING)
+    def forward(
+        self,
+        input_ids: torch.LongTensor = None,
+        visual_features: Optional[torch.FloatTensor] = None,
+        image_token_index: int = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[List[torch.FloatTensor]] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+    ) -> Union[Tuple, BaseModelOutputWithPast]:
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        use_cache = use_cache if use_cache is not None else self.config.use_cache
+
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        # retrieve input_ids and inputs_embeds
+        if input_ids is not None and inputs_embeds is not None:
+            raise ValueError("You cannot specify both decoder_input_ids and decoder_inputs_embeds at the same time")
+        elif input_ids is not None:
+            batch_size, seq_length = input_ids.shape
+        elif inputs_embeds is not None:
+            batch_size, seq_length, _ = inputs_embeds.shape
+        else:
+            raise ValueError("You have to specify either decoder_input_ids or decoder_inputs_embeds")
+
+        if self.gradient_checkpointing and self.training:
+            if use_cache:
+                logger.warning_once(
+                    "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..."
+                )
+                use_cache = False
+
+        past_key_values_length = 0
+
+        if use_cache:
+            use_legacy_cache = not isinstance(past_key_values, Cache)
+            if use_legacy_cache:
+                if past_key_values is None:
+                    past_key_values = DynamicCache()
+                else:
+                    past_key_values = DynamicCache.from_legacy_cache(past_key_values)
+            past_key_values_length = past_key_values.get_seq_length()
+
+        if position_ids is None:
+            device = input_ids.device if input_ids is not None else inputs_embeds.device
+            position_ids = torch.arange(
+                past_key_values_length, seq_length + past_key_values_length, dtype=torch.long, device=device
+            )
+            position_ids = position_ids.unsqueeze(0).view(-1, seq_length)
+        else:
+            position_ids = position_ids.view(-1, seq_length).long()
+
+        if inputs_embeds is None:
+            inputs_embeds = self.image_processing(input_ids, visual_features, image_token_index)
+
+        if attention_mask is not None and self._attn_implementation == "magi" and use_cache:
+            is_padding_right = attention_mask[:, -1].sum().item() != batch_size
+            if is_padding_right:
+                raise ValueError(
+                    "You are attempting to perform batched generation with padding_side='right'"
+                    " this may lead to unexpected behaviour for Flash Attention version of Qwen2. Make sure to "
+                    " call `tokenizer.padding_side  = 'left'` before tokenizing the input. "
+                )
+
+        device = input_ids.device if input_ids is not None else inputs_embeds.device
+
+        x0_len = find_prefix_seq_length_by_pe(position_ids).to(device=device)
+
+        def _prepare_block_mask_for_inference(attention_mask):
+            attention_mask = _prepare_4d_causal_attention_mask(
+                attention_mask,
+                (batch_size, seq_length),
+                inputs_embeds,
+                past_key_values_length,
+                sliding_window=self.config.sliding_window,
+            )
+            # switch to ar mode
+            if seq_length == 1 or (input_ids is not None and input_ids[0][-1].item() != self.text_mask_token_id):
+                return attention_mask
+
+            
+            if attention_mask is None or len(attention_mask.shape) != 4:
+                return attention_mask
+
+            # For SDLM, the generation window should set to bidirectional attention
+            if use_cache:
+                update_mask_func = partial(
+                    update_causal_mask_for_one_gen_window_2d,
+                    block_size=self.block_size,
+                    use_cache=use_cache,
+                    causal_attn=self.causal_attn,
+                )
+            else:
+                update_mask_func = partial(
+                    update_causal_mask_with_pad_non_visible_2d,
+                    block_size=self.block_size,
+                    text_mask_token_id=self.text_mask_token_id,
+                    causal_attn=self.causal_attn,
+                )
+
+            new_attention_mask = []
+            for b in range(attention_mask.shape[0]):
+                new_attention_mask.append(
+                    update_mask_func(
+                        input_ids[b],
+                        attention_mask[b][0],
+                    ).unsqueeze(0)
+                )
+            return torch.stack(new_attention_mask, dim=0)
+
+        def _prepare_block_mask_for_training():
+            block_mask, _ = create_block_diff_mask_by_pe_4d(
+                block_size=self.block_size,
+                x0_len_list=x0_len,
+                position_ids=position_ids,
+                causal_attn=self.causal_attn,
+            )
+            return block_mask
+
+        if self._attn_implementation == "magi":
+            ar_decode = seq_length == 1 or (input_ids is not None and input_ids[0][-1].item() != self.text_mask_token_id)
+            attention_mask = build_magi_ranges(
+                kv_len=seq_length + past_key_values_length,
+                q_len=seq_length,
+                block_size=self.block_size,
+                ar_decode=ar_decode,
+                device=device
+            )
+
+        elif self._attn_implementation == "sdpa":
+            attention_mask = _prepare_block_mask_for_training() if self.training else _prepare_block_mask_for_inference(attention_mask)
+
+        else:
+            raise NotImplementedError(f'{self._attn_implementation=}')
+
+
+        hidden_states = inputs_embeds
+
+        # decoder layers
+        all_hidden_states = () if output_hidden_states else None
+        all_self_attns = () if output_attentions else None
+        next_decoder_cache = None
+
+        for decoder_layer in self.layers:
+            if output_hidden_states:
+                all_hidden_states += (hidden_states,)
+
+            if self.gradient_checkpointing and self.training:
+                layer_outputs = self._gradient_checkpointing_func(
+                    decoder_layer.__call__,
+                    hidden_states,
+                    attention_mask,
+                    position_ids,
+                    past_key_values,
+                    output_attentions,
+                    use_cache,
+                )
+            else:
+                layer_outputs = decoder_layer(
+                    hidden_states,
+                    attention_mask=attention_mask,
+                    position_ids=position_ids,
+                    past_key_value=past_key_values,
+                    output_attentions=output_attentions,
+                    use_cache=use_cache,
+                )
+
+            hidden_states = layer_outputs[0]
+
+            if use_cache:
+                next_decoder_cache = layer_outputs[2 if output_attentions else 1]
+
+            if output_attentions:
+                all_self_attns += (layer_outputs[1],)
+
+        hidden_states = self.norm(hidden_states)
+
+        # add hidden states from the last decoder layer
+        if output_hidden_states:
+            all_hidden_states += (hidden_states,)
+
+        next_cache = None
+        if use_cache:
+            next_cache = next_decoder_cache.to_legacy_cache() if use_legacy_cache else next_decoder_cache
+
+        if not return_dict:
+            return tuple(v for v in [hidden_states, next_cache, all_hidden_states, all_self_attns] if v is not None)
+        return BaseModelOutputWithPast(
+            last_hidden_state=hidden_states,
+            past_key_values=next_cache,
+            hidden_states=all_hidden_states,
+            attentions=all_self_attns,
+        )
+
+
+class Qwen2ForCausalLM(Qwen2PreTrainedModel):
+    _tied_weights_keys = ["lm_head.weight"]
+
+    def __init__(self, config):
+        super().__init__(config)
+        self.model = Qwen2Model(config)
+        self.vocab_size = config.vocab_size
+        self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
+
+        self.text_mask_token_id = getattr(config, 'text_mask_token_id', 151676)
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+
+    def get_input_embeddings(self):
+        return self.model.embed_tokens
+
+    def set_input_embeddings(self, value):
+        self.model.embed_tokens = value
+
+    def get_output_embeddings(self):
+        return self.lm_head
+
+    def set_output_embeddings(self, new_embeddings):
+        self.lm_head = new_embeddings
+
+    def set_decoder(self, decoder):
+        self.model = decoder
+
+    def get_decoder(self):
+        return self.model
+
+    @add_start_docstrings_to_model_forward(QWEN2_INPUTS_DOCSTRING)
+    @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC)
+    def forward(
+        self,
+        input_ids: torch.LongTensor = None,
+        visual_features: Optional[torch.FloatTensor] = None,
+        image_token_index: int = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[List[torch.FloatTensor]] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        labels: Optional[torch.LongTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+    ) -> Union[Tuple, CausalLMOutputWithPast]:
+        r"""
+        Args:
+            labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+                Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
+                config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
+                (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
+
+        Returns:
+
+        Example:
+
+        ```python
+        >>> from transformers import AutoTokenizer, Qwen2ForCausalLM
+
+        >>> model = Qwen2ForCausalLM.from_pretrained(PATH_TO_CONVERTED_WEIGHTS)
+        >>> tokenizer = AutoTokenizer.from_pretrained(PATH_TO_CONVERTED_TOKENIZER)
+
+        >>> prompt = "Hey, are you conscious? Can you talk to me?"
+        >>> inputs = tokenizer(prompt, return_tensors="pt")
+
+        >>> # Generate
+        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
+        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
+        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
+        ```"""
+
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)
+        outputs = self.model(
+            input_ids=input_ids,
+            visual_features=visual_features,
+            image_token_index=image_token_index,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+
+        hidden_states = outputs[0]
+        logits = self.lm_head(hidden_states)
+        logits = logits.float()
+
+        loss = None
+        if labels is not None:
+
+            # Shift so that tokens < n predict n
+            shift_logits = logits[..., :-1, :].contiguous()
+            shift_labels = labels[..., 1:].contiguous()
+
+            # Flatten the tokens
+            loss_fct = CrossEntropyLoss()
+            shift_logits = shift_logits.view(-1, self.config.vocab_size)
+
+            shift_labels = shift_labels.view(-1)
+            shift_labels = shift_labels.to(shift_logits.device)
+            loss = loss_fct(shift_logits, shift_labels)
+
+            pos_masks = find_pred_pos_from_input_ids(input_ids, text_mask_token_id=self.text_mask_token_id)
+            shift_input_ids = input_ids[..., :-1].contiguous()
+            shift_pos_masks = pos_masks[:, :-1]
+            shift_input_ids = shift_input_ids.view(-1)
+            max_n_future_tokens = min(4, self.model.block_size)
+            pos_loss_list = torch.zeros(max_n_future_tokens, device=shift_logits.device)
+            shift_pos_masks = shift_pos_masks.reshape(-1)
+
+            for ix in range(max_n_future_tokens):
+                seg_loss = F.cross_entropy(
+                    shift_logits[shift_pos_masks == ix],
+                    shift_labels[shift_pos_masks == ix],
+                    reduction='mean'
+                )
+                pos_loss_list[ix] = seg_loss
+
+
+        if not return_dict:
+            output = (logits,) + outputs[1:]
+            return (loss,) + output if loss is not None else output
+        
+        if self.training:
+            return CausalLMOutputWithPast(
+                loss=loss,
+                logits=logits,
+                past_key_values=outputs.past_key_values,
+                hidden_states=outputs.hidden_states,
+                attentions=outputs.attentions,
+            ), pos_loss_list
+
+        return CausalLMOutputWithPast(
+            loss=loss,
+            logits=logits,
+            past_key_values=outputs.past_key_values,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+        )
+
+    def prepare_inputs_for_generation(
+        self, input_ids, past_key_values=None, attention_mask=None, inputs_embeds=None, **kwargs
+    ):
+        # Omit tokens covered by past_key_values
+        if past_key_values is not None:
+            if isinstance(past_key_values, Cache):
+                cache_length = past_key_values.get_seq_length()
+                past_length = past_key_values.seen_tokens
+                max_cache_length = past_key_values.get_max_length()
+            else:
+                cache_length = past_length = past_key_values[0][0].shape[2]
+                max_cache_length = None
+
+            # Keep only the unprocessed tokens:
+            # 1 - If the length of the attention_mask exceeds the length of input_ids, then we are in a setting where
+            # some of the inputs are exclusively passed as part of the cache (e.g. when passing input_embeds as
+            # input)
+            if attention_mask is not None and attention_mask.shape[1] > input_ids.shape[1]:
+                input_ids = input_ids[:, -(attention_mask.shape[1] - past_length) :]
+            # 2 - If the past_length is smaller than input_ids', then input_ids holds all input tokens. We can discard
+            # input_ids based on the past_length.
+            elif past_length < input_ids.shape[1]:
+                input_ids = input_ids[:, past_length:]
+            # 3 - Otherwise (past_length >= input_ids.shape[1]), let's assume input_ids only has unprocessed tokens.
+
+            # If we are about to go beyond the maximum cache length, we need to crop the input attention mask.
+            if (
+                max_cache_length is not None
+                and attention_mask is not None
+                and cache_length + input_ids.shape[1] > max_cache_length
+            ):
+                attention_mask = attention_mask[:, -max_cache_length:]
+
+        position_ids = kwargs.get("position_ids", None)
+        if attention_mask is not None and position_ids is None:
+            # create position_ids on the fly for batch generation
+            position_ids = attention_mask.long().cumsum(-1) - 1
+            position_ids.masked_fill_(attention_mask == 0, 1)
+            if past_key_values:
+                position_ids = position_ids[:, -input_ids.shape[1] :]
+
+        # if `inputs_embeds` are passed, we only want to use them in the 1st generation step
+        if inputs_embeds is not None and past_key_values is None:
+            model_inputs = {"inputs_embeds": inputs_embeds}
+        else:
+            model_inputs = {"input_ids": input_ids}
+
+        model_inputs.update(
+            {
+                "position_ids": position_ids,
+                "past_key_values": past_key_values,
+                "use_cache": kwargs.get("use_cache"),
+                "attention_mask": attention_mask,
+            }
+        )
+        return model_inputs
+
+    @staticmethod
+    def _reorder_cache(past_key_values, beam_idx):
+        reordered_past = ()
+        for layer_past in past_key_values:
+            reordered_past += (
+                tuple(past_state.index_select(0, beam_idx.to(past_state.device)) for past_state in layer_past),
+            )
+        return reordered_past
+
+
+@add_start_docstrings(
+    """
+    The Qwen2 Model transformer with a sequence classification head on top (linear layer).
+
+    [`Qwen2ForSequenceClassification`] uses the last token in order to do the classification, as other causal models
+    (e.g. GPT-2) do.
+
+    Since it does classification on the last token, it requires to know the position of the last token. If a
+    `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If
+    no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the
+    padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in
+    each row of the batch).
+    """,
+    QWEN2_START_DOCSTRING,
+)
+class Qwen2ForSequenceClassification(Qwen2PreTrainedModel):
+    def __init__(self, config):
+        super().__init__(config)
+        self.num_labels = config.num_labels
+        self.model = Qwen2Model(config)
+        self.score = nn.Linear(config.hidden_size, self.num_labels, bias=False)
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def get_input_embeddings(self):
+        return self.model.embed_tokens
+
+    def set_input_embeddings(self, value):
+        self.model.embed_tokens = value
+
+    @add_start_docstrings_to_model_forward(QWEN2_INPUTS_DOCSTRING)
+    def forward(
+        self,
+        input_ids: torch.LongTensor = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[List[torch.FloatTensor]] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        labels: Optional[torch.LongTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+    ) -> Union[Tuple, SequenceClassifierOutputWithPast]:
+        r"""
+        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
+            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+        """
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        transformer_outputs = self.model(
+            input_ids,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+        hidden_states = transformer_outputs[0]
+        logits = self.score(hidden_states)
+
+        if input_ids is not None:
+            batch_size = input_ids.shape[0]
+        else:
+            batch_size = inputs_embeds.shape[0]
+
+        if self.config.pad_token_id is None and batch_size != 1:
+            raise ValueError("Cannot handle batch sizes > 1 if no padding token is defined.")
+        if self.config.pad_token_id is None:
+            sequence_lengths = -1
+        else:
+            if input_ids is not None:
+                # if no pad token found, use modulo instead of reverse indexing for ONNX compatibility
+                sequence_lengths = torch.eq(input_ids, self.config.pad_token_id).int().argmax(-1) - 1
+                sequence_lengths = sequence_lengths % input_ids.shape[-1]
+                sequence_lengths = sequence_lengths.to(logits.device)
+            else:
+                sequence_lengths = -1
+
+        pooled_logits = logits[torch.arange(batch_size, device=logits.device), sequence_lengths]
+
+        loss = None
+        if labels is not None:
+            labels = labels.to(logits.device)
+            if self.config.problem_type is None:
+                if self.num_labels == 1:
+                    self.config.problem_type = "regression"
+                elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int):
+                    self.config.problem_type = "single_label_classification"
+                else:
+                    self.config.problem_type = "multi_label_classification"
+
+            if self.config.problem_type == "regression":
+                loss_fct = MSELoss()
+                if self.num_labels == 1:
+                    loss = loss_fct(pooled_logits.squeeze(), labels.squeeze())
+                else:
+                    loss = loss_fct(pooled_logits, labels)
+            elif self.config.problem_type == "single_label_classification":
+                loss_fct = CrossEntropyLoss()
+                loss = loss_fct(pooled_logits.view(-1, self.num_labels), labels.view(-1))
+            elif self.config.problem_type == "multi_label_classification":
+                loss_fct = BCEWithLogitsLoss()
+                loss = loss_fct(pooled_logits, labels)
+        if not return_dict:
+            output = (pooled_logits,) + transformer_outputs[1:]
+            return ((loss,) + output) if loss is not None else output
+
+        return SequenceClassifierOutputWithPast(
+            loss=loss,
+            logits=pooled_logits,
+            past_key_values=transformer_outputs.past_key_values,
+            hidden_states=transformer_outputs.hidden_states,
+            attentions=transformer_outputs.attentions,
+        )

qwen2.5_tokenizer/quant_log.csv

@@ -0,0 +1,253 @@
+layer,module,loss,samples,damp,time
+0,self_attn.k_proj,0.0000005085,0.05000,3.240
+0,self_attn.v_proj,0.0000000616,0.05000,3.331
+0,self_attn.q_proj,0.0000024387,0.05000,3.344
+0,self_attn.o_proj,0.0000002079,0.05000,0.973
+0,mlp.gate_proj,0.0000028578,0.05000,1.696
+0,mlp.up_proj,0.0000022065,0.05000,1.714
+0,mlp.down_proj,0.0000005557,0.05000,4.253
+1,self_attn.v_proj,0.0000000238,0.05000,3.020
+1,self_attn.k_proj,0.0000001244,0.05000,3.038
+1,self_attn.q_proj,0.0000004455,0.05000,3.103
+1,self_attn.o_proj,0.0000000834,0.05000,0.920
+1,mlp.up_proj,0.0000957824,0.05000,1.378
+1,mlp.gate_proj,0.0001252921,0.05000,1.436
+1,mlp.down_proj,0.0000000248,0.05000,4.110
+2,self_attn.v_proj,0.0000000376,0.05000,3.022
+2,self_attn.q_proj,0.0000006681,0.05000,3.177
+2,self_attn.k_proj,0.0000001465,0.05000,3.203
+2,self_attn.o_proj,0.0000000641,0.05000,1.082
+2,mlp.gate_proj,0.0000304530,0.05000,1.268
+2,mlp.up_proj,0.0000303662,0.05000,1.334
+2,mlp.down_proj,0.0000120081,0.05000,4.371
+3,self_attn.k_proj,0.0000006040,0.05000,3.001
+3,self_attn.q_proj,0.0000028064,0.05000,3.063
+3,self_attn.v_proj,0.0000001111,0.05000,3.032
+3,self_attn.o_proj,0.0000000855,0.05000,0.928
+3,mlp.up_proj,0.0000360368,0.05000,1.576
+3,mlp.gate_proj,0.0000382443,0.05000,1.606
+3,mlp.down_proj,0.0000003860,0.05000,4.249
+4,self_attn.q_proj,0.0000022345,0.05000,3.843
+4,self_attn.k_proj,0.0000004367,0.05000,3.907
+4,self_attn.v_proj,0.0000001491,0.05000,3.939
+4,self_attn.o_proj,0.0000001581,0.05000,0.863
+4,mlp.up_proj,0.0000284978,0.05000,1.413
+4,mlp.gate_proj,0.0000365522,0.05000,1.449
+4,mlp.down_proj,0.0000003562,0.05000,4.254
+5,self_attn.v_proj,0.0000004122,0.05000,2.760
+5,self_attn.k_proj,0.0000008967,0.05000,2.809
+5,self_attn.q_proj,0.0000050007,0.05000,2.831
+5,self_attn.o_proj,0.0000001279,0.05000,0.942
+5,mlp.gate_proj,0.0000520914,0.05000,1.275
+5,mlp.up_proj,0.0000442293,0.05000,1.282
+5,mlp.down_proj,0.0000007485,0.05000,5.679
+6,self_attn.k_proj,0.0000004771,0.05000,3.021
+6,self_attn.v_proj,0.0000003296,0.05000,3.142
+6,self_attn.q_proj,0.0000030048,0.05000,3.150
+6,self_attn.o_proj,0.0000002717,0.05000,0.938
+6,mlp.up_proj,0.0000590103,0.05000,1.374
+6,mlp.gate_proj,0.0000687621,0.05000,1.421
+6,mlp.down_proj,0.0000011120,0.05000,4.512
+7,self_attn.q_proj,0.0000041045,0.05000,2.921
+7,self_attn.k_proj,0.0000006536,0.05000,2.968
+7,self_attn.v_proj,0.0000004090,0.05000,2.974
+7,self_attn.o_proj,0.0000002103,0.05000,0.866
+7,mlp.gate_proj,0.0000712782,0.05000,1.340
+7,mlp.up_proj,0.0000586724,0.05000,1.357
+7,mlp.down_proj,0.0000014121,0.05000,4.585
+8,self_attn.k_proj,0.0000006371,0.05000,3.163
+8,self_attn.v_proj,0.0000004388,0.05000,3.356
+8,self_attn.q_proj,0.0000041995,0.05000,3.369
+8,self_attn.o_proj,0.0000002558,0.05000,0.978
+8,mlp.up_proj,0.0000598540,0.05000,1.155
+8,mlp.gate_proj,0.0000840462,0.05000,1.206
+8,mlp.down_proj,0.0000018146,0.05000,4.143
+9,self_attn.v_proj,0.0000003784,0.05000,2.766
+9,self_attn.q_proj,0.0000037226,0.05000,2.783
+9,self_attn.k_proj,0.0000006510,0.05000,2.768
+9,self_attn.o_proj,0.0000004260,0.05000,0.930
+9,mlp.up_proj,0.0000524589,0.05000,1.571
+9,mlp.gate_proj,0.0000783262,0.05000,1.598
+9,mlp.down_proj,0.0000024518,0.05000,5.082
+10,self_attn.q_proj,0.0000052897,0.05000,2.622
+10,self_attn.k_proj,0.0000007815,0.05000,2.717
+10,self_attn.v_proj,0.0000006292,0.05000,2.743
+10,self_attn.o_proj,0.0000003979,0.05000,1.276
+10,mlp.gate_proj,0.0000586628,0.05000,1.290
+10,mlp.up_proj,0.0000382535,0.05000,1.371
+10,mlp.down_proj,0.0000026338,0.05000,4.219
+11,self_attn.k_proj,0.0000005685,0.05000,2.966
+11,self_attn.v_proj,0.0000005079,0.05000,2.980
+11,self_attn.q_proj,0.0000038576,0.05000,2.991
+11,self_attn.o_proj,0.0000005243,0.05000,1.047
+11,mlp.up_proj,0.0000218057,0.05000,1.301
+11,mlp.gate_proj,0.0000305343,0.05000,1.333
+11,mlp.down_proj,0.0000027184,0.05000,4.218
+12,self_attn.q_proj,0.0000040775,0.05000,2.488
+12,self_attn.k_proj,0.0000006225,0.05000,2.505
+12,self_attn.v_proj,0.0000004578,0.05000,2.521
+12,self_attn.o_proj,0.0000004894,0.05000,0.662
+12,mlp.gate_proj,0.0000336821,0.05000,1.277
+12,mlp.up_proj,0.0000236778,0.05000,1.291
+12,mlp.down_proj,0.0000022740,0.05000,4.415
+13,self_attn.q_proj,0.0000048006,0.05000,3.943
+13,self_attn.v_proj,0.0000003174,0.05000,3.952
+13,self_attn.k_proj,0.0000008770,0.05000,3.995
+13,self_attn.o_proj,0.0000003654,0.05000,0.734
+13,mlp.gate_proj,0.0000177235,0.05000,1.341
+13,mlp.up_proj,0.0000169780,0.05000,1.368
+13,mlp.down_proj,0.0000021166,0.05000,4.304
+14,self_attn.q_proj,0.0000036348,0.05000,2.822
+14,self_attn.k_proj,0.0000005575,0.05000,2.855
+14,self_attn.v_proj,0.0000003621,0.05000,2.864
+14,self_attn.o_proj,0.0000007141,0.05000,1.039
+14,mlp.up_proj,0.0000165291,0.05000,1.228
+14,mlp.gate_proj,0.0000172143,0.05000,1.244
+14,mlp.down_proj,0.0000020680,0.05000,4.865
+15,self_attn.q_proj,0.0000037588,0.05000,2.502
+15,self_attn.k_proj,0.0000005889,0.05000,2.505
+15,self_attn.v_proj,0.0000003618,0.05000,2.549
+15,self_attn.o_proj,0.0000008058,0.05000,0.835
+15,mlp.up_proj,0.0000149748,0.05000,1.265
+15,mlp.gate_proj,0.0000143409,0.05000,1.283
+15,mlp.down_proj,0.0000018108,0.05000,4.185
+16,self_attn.k_proj,0.0000006456,0.05000,2.311
+16,self_attn.q_proj,0.0000038739,0.05000,2.370
+16,self_attn.v_proj,0.0000003734,0.05000,2.386
+16,self_attn.o_proj,0.0000005621,0.05000,0.878
+16,mlp.gate_proj,0.0000152835,0.05000,1.159
+16,mlp.up_proj,0.0000149767,0.05000,1.230
+16,mlp.down_proj,0.0000016259,0.05000,5.022
+17,self_attn.v_proj,0.0000005680,0.05000,2.477
+17,self_attn.k_proj,0.0000010732,0.05000,2.463
+17,self_attn.q_proj,0.0000062434,0.05000,2.502
+17,self_attn.o_proj,0.0000006299,0.05000,0.774
+17,mlp.up_proj,0.0000129082,0.05000,1.237
+17,mlp.gate_proj,0.0000127357,0.05000,1.305
+17,mlp.down_proj,0.0000018334,0.05000,4.142
+18,self_attn.q_proj,0.0000035806,0.05000,2.411
+18,self_attn.k_proj,0.0000005222,0.05000,2.460
+18,self_attn.v_proj,0.0000003745,0.05000,2.489
+18,self_attn.o_proj,0.0000006935,0.05000,1.018
+18,mlp.up_proj,0.0000123926,0.05000,1.250
+18,mlp.gate_proj,0.0000127913,0.05000,1.322
+18,mlp.down_proj,0.0000014052,0.05000,4.173
+19,self_attn.v_proj,0.0000003944,0.05000,3.463
+19,self_attn.k_proj,0.0000006968,0.05000,3.586
+19,self_attn.q_proj,0.0000039632,0.05000,3.608
+19,self_attn.o_proj,0.0000009026,0.05000,0.818
+19,mlp.gate_proj,0.0000114576,0.05000,1.112
+19,mlp.up_proj,0.0000119524,0.05000,1.161
+19,mlp.down_proj,0.0000014570,0.05000,4.321
+20,self_attn.k_proj,0.0000007010,0.05000,2.711
+20,self_attn.v_proj,0.0000007374,0.05000,2.718
+20,self_attn.q_proj,0.0000060873,0.05000,2.745
+20,self_attn.o_proj,0.0000007991,0.05000,0.912
+20,mlp.gate_proj,0.0000118486,0.05000,1.426
+20,mlp.up_proj,0.0000121822,0.05000,1.442
+20,mlp.down_proj,0.0000014912,0.05000,4.249
+21,self_attn.k_proj,0.0000007058,0.05000,2.460
+21,self_attn.q_proj,0.0000043860,0.05000,2.542
+21,self_attn.v_proj,0.0000004948,0.05000,2.576
+21,self_attn.o_proj,0.0000006633,0.05000,0.855
+21,mlp.up_proj,0.0000111701,0.05000,1.315
+21,mlp.gate_proj,0.0000118580,0.05000,1.357
+21,mlp.down_proj,0.0000010636,0.05000,4.237
+22,self_attn.q_proj,0.0000050432,0.05000,2.470
+22,self_attn.k_proj,0.0000007732,0.05000,2.483
+22,self_attn.v_proj,0.0000006412,0.05000,2.494
+22,self_attn.o_proj,0.0000006260,0.05000,0.899
+22,mlp.gate_proj,0.0000112563,0.05000,1.363
+22,mlp.up_proj,0.0000109995,0.05000,1.422
+22,mlp.down_proj,0.0000010489,0.05000,4.304
+23,self_attn.k_proj,0.0000007014,0.05000,3.266
+23,self_attn.q_proj,0.0000048249,0.05000,3.317
+23,self_attn.v_proj,0.0000003740,0.05000,3.373
+23,self_attn.o_proj,0.0000007296,0.05000,0.939
+23,mlp.gate_proj,0.0000105611,0.05000,1.372
+23,mlp.up_proj,0.0000110128,0.05000,1.381
+23,mlp.down_proj,0.0000012117,0.05000,4.254
+24,self_attn.v_proj,0.0000004095,0.05000,2.385
+24,self_attn.k_proj,0.0000004718,0.05000,2.412
+24,self_attn.q_proj,0.0000034988,0.05000,2.413
+24,self_attn.o_proj,0.0000007943,0.05000,0.955
+24,mlp.up_proj,0.0000105276,0.05000,1.198
+24,mlp.gate_proj,0.0000102487,0.05000,1.258
+24,mlp.down_proj,0.0000014155,0.05000,4.128
+25,self_attn.k_proj,0.0000004508,0.05000,2.690
+25,self_attn.v_proj,0.0000006476,0.05000,2.825
+25,self_attn.q_proj,0.0000047023,0.05000,2.832
+25,self_attn.o_proj,0.0000006867,0.05000,0.852
+25,mlp.up_proj,0.0000116300,0.05000,1.634
+25,mlp.gate_proj,0.0000114514,0.05000,1.708
+25,mlp.down_proj,0.0000016359,0.05000,4.386
+26,self_attn.k_proj,0.0000004960,0.05000,3.098
+26,self_attn.q_proj,0.0000038692,0.05000,3.103
+26,self_attn.v_proj,0.0000006341,0.05000,3.127
+26,self_attn.o_proj,0.0000009083,0.05000,0.835
+26,mlp.up_proj,0.0000133298,0.05000,1.431
+26,mlp.gate_proj,0.0000125842,0.05000,1.454
+26,mlp.down_proj,0.0000022201,0.05000,4.144
+27,self_attn.q_proj,0.0000063679,0.05000,3.371
+27,self_attn.k_proj,0.0000005433,0.05000,3.429
+27,self_attn.v_proj,0.0000008838,0.05000,3.431
+27,self_attn.o_proj,0.0000011006,0.05000,1.076
+27,mlp.gate_proj,0.0000139996,0.05000,1.350
+27,mlp.up_proj,0.0000141233,0.05000,1.353
+27,mlp.down_proj,0.0000025846,0.05000,4.317
+28,self_attn.v_proj,0.0000008987,0.05000,4.054
+28,self_attn.k_proj,0.0000005489,0.05000,4.102
+28,self_attn.q_proj,0.0000051061,0.05000,4.118
+28,self_attn.o_proj,0.0000011722,0.05000,1.007
+28,mlp.up_proj,0.0000153893,0.05000,1.213
+28,mlp.gate_proj,0.0000153670,0.05000,1.274
+28,mlp.down_proj,0.0000028280,0.05000,4.456
+29,self_attn.v_proj,0.0000007629,0.05000,3.439
+29,self_attn.k_proj,0.0000005342,0.05000,3.521
+29,self_attn.q_proj,0.0000046298,0.05000,3.534
+29,self_attn.o_proj,0.0000009201,0.05000,0.812
+29,mlp.gate_proj,0.0000180554,0.05000,1.402
+29,mlp.up_proj,0.0000184227,0.05000,1.406
+29,mlp.down_proj,0.0000038424,0.05000,4.300
+30,self_attn.v_proj,0.0000020394,0.05000,3.110
+30,self_attn.k_proj,0.0000006122,0.05000,3.143
+30,self_attn.q_proj,0.0000070738,0.05000,3.167
+30,self_attn.o_proj,0.0000011594,0.05000,1.005
+30,mlp.gate_proj,0.0000228827,0.05000,1.470
+30,mlp.up_proj,0.0000249021,0.05000,1.495
+30,mlp.down_proj,0.0000158357,0.05000,4.416
+31,self_attn.q_proj,0.0000093229,0.05000,2.831
+31,self_attn.k_proj,0.0000009378,0.05000,2.882
+31,self_attn.v_proj,0.0000024443,0.05000,2.891
+31,self_attn.o_proj,0.0000030832,0.05000,0.684
+31,mlp.gate_proj,0.0000264529,0.05000,1.345
+31,mlp.up_proj,0.0000300895,0.05000,1.400
+31,mlp.down_proj,0.0000134932,0.05000,4.460
+32,self_attn.q_proj,0.0000164472,0.05000,3.071
+32,self_attn.k_proj,0.0000014786,0.05000,3.103
+32,self_attn.v_proj,0.0000073760,0.05000,3.108
+32,self_attn.o_proj,0.0000065663,0.05000,0.774
+32,mlp.up_proj,0.0000336128,0.05000,1.444
+32,mlp.gate_proj,0.0000295613,0.05000,1.467
+32,mlp.down_proj,0.0000133721,0.05000,4.501
+33,self_attn.v_proj,0.0000151532,0.05000,2.874
+33,self_attn.q_proj,0.0000151740,0.05000,2.961
+33,self_attn.k_proj,0.0000012399,0.05000,2.964
+33,self_attn.o_proj,0.0000136846,0.05000,0.722
+33,mlp.gate_proj,0.0000268336,0.05000,1.449
+33,mlp.up_proj,0.0000334955,0.05000,1.466
+33,mlp.down_proj,0.0000303056,0.05000,4.839
+34,self_attn.q_proj,0.0000051019,0.05000,3.116
+34,self_attn.k_proj,0.0000005239,0.05000,3.161
+34,self_attn.v_proj,0.0000017698,0.05000,3.193
+34,self_attn.o_proj,0.0000030042,0.05000,0.749
+34,mlp.up_proj,0.0000342593,0.05000,1.270
+34,mlp.gate_proj,0.0000294181,0.05000,1.299
+34,mlp.down_proj,0.0000177885,0.05000,4.654
+35,self_attn.v_proj,0.0000014588,0.05000,3.398
+35,self_attn.q_proj,0.0000045896,0.05000,3.432
+35,self_attn.k_proj,0.0000004904,0.05000,3.471
+35,self_attn.o_proj,0.0000032275,0.05000,0.855
+35,mlp.gate_proj,0.0000420900,0.05000,1.395
+35,mlp.up_proj,0.0000445427,0.05000,1.458
+35,mlp.down_proj,0.0000312161,0.05000,4.283

qwen2.5_tokenizer/quantize_config.json

@@ -0,0 +1,47 @@
+{
+  "bits": 4,
+  "group_size": 128,
+  "desc_act": false,
+  "lm_head": false,
+  "method": "gptq",
+  "quant_method": "gptq",
+  "format": "gptq",
+  "checkpoint_format": "gptq",
+  "pack_dtype": "int32",
+  "meta": {
+    "quantizer": [
+      "gptqmodel:7.1.0"
+    ],
+    "uri": "https://github.com/modelcloud/gptqmodel",
+    "damp_percent": 0.05,
+    "damp_auto_increment": 0.01,
+    "static_groups": false,
+    "true_sequential": true,
+    "mse": 0.0,
+    "gptaq": null,
+    "foem": null,
+    "act_group_aware": true,
+    "fallback": {
+      "strategy": "rtn",
+      "threshold": "0.5%",
+      "smooth": null
+    },
+    "offload_to_disk": true,
+    "offload_to_disk_path": "/tmp/gptqmodel_42jjgpwr",
+    "pack_impl": "cpu",
+    "gc_mode": "interval",
+    "wait_for_submodule_finalizers": false,
+    "auto_forward_data_parallel": true,
+    "dense_vram_strategy": "exclusive",
+    "dense_vram_strategy_devices": null,
+    "moe_vram_strategy": "exclusive",
+    "moe_vram_strategy_devices": null,
+    "mock_quantization": false,
+    "hessian": {
+      "chunk_size": null,
+      "chunk_bytes": null,
+      "staging_dtype": "float32"
+    }
+  },
+  "sym": true
+}

qwen2.5_tokenizer/special_tokens_map.json

@@ -0,0 +1,1053 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>",
+    "<IMG_CONTEXT>",
+    "<img>",
+    "</img>",
+    "<box>",
+    "</box>",
+    "<quad>",
+    "</quad>",
+    "<ref>",
+    "</ref>",
+    "<interval>",
+    "</interval>",
+    "<text_mask>",
+    "<0>",
+    "<1>",
+    "<2>",
+    "<3>",
+    "<4>",
+    "<5>",
+    "<6>",
+    "<7>",
+    "<8>",
+    "<9>",
+    "<10>",
+    "<11>",
+    "<12>",
+    "<13>",
+    "<14>",
+    "<15>",
+    "<16>",
+    "<17>",
+    "<18>",
+    "<19>",
+    "<20>",
+    "<21>",
+    "<22>",
+    "<23>",
+    "<24>",
+    "<25>",
+    "<26>",
+    "<27>",
+    "<28>",
+    "<29>",
+    "<30>",
+    "<31>",
+    "<32>",
+    "<33>",
+    "<34>",
+    "<35>",
+    "<36>",
+    "<37>",
+    "<38>",
+    "<39>",
+    "<40>",
+    "<41>",
+    "<42>",
+    "<43>",
+    "<44>",
+    "<45>",
+    "<46>",
+    "<47>",
+    "<48>",
+    "<49>",
+    "<50>",
+    "<51>",
+    "<52>",
+    "<53>",
+    "<54>",
+    "<55>",
+    "<56>",
+    "<57>",
+    "<58>",
+    "<59>",
+    "<60>",
+    "<61>",
+    "<62>",
+    "<63>",
+    "<64>",
+    "<65>",
+    "<66>",
+    "<67>",
+    "<68>",
+    "<69>",
+    "<70>",
+    "<71>",
+    "<72>",
+    "<73>",
+    "<74>",
+    "<75>",
+    "<76>",
+    "<77>",
+    "<78>",
+    "<79>",
+    "<80>",
+    "<81>",
+    "<82>",
+    "<83>",
+    "<84>",
+    "<85>",
+    "<86>",
+    "<87>",
+    "<88>",
+    "<89>",
+    "<90>",
+    "<91>",
+    "<92>",
+    "<93>",
+    "<94>",
+    "<95>",
+    "<96>",
+    "<97>",
+    "<98>",
+    "<99>",
+    "<100>",
+    "<101>",
+    "<102>",
+    "<103>",
+    "<104>",
+    "<105>",
+    "<106>",
+    "<107>",
+    "<108>",
+    "<109>",
+    "<110>",
+    "<111>",
+    "<112>",
+    "<113>",
+    "<114>",
+    "<115>",
+    "<116>",
+    "<117>",
+    "<118>",
+    "<119>",
+    "<120>",
+    "<121>",
+    "<122>",
+    "<123>",
+    "<124>",
+    "<125>",
+    "<126>",
+    "<127>",
+    "<128>",
+    "<129>",
+    "<130>",
+    "<131>",
+    "<132>",
+    "<133>",
+    "<134>",
+    "<135>",
+    "<136>",
+    "<137>",
+    "<138>",
+    "<139>",
+    "<140>",
+    "<141>",
+    "<142>",
+    "<143>",
+    "<144>",
+    "<145>",
+    "<146>",
+    "<147>",
+    "<148>",
+    "<149>",
+    "<150>",
+    "<151>",
+    "<152>",
+    "<153>",
+    "<154>",
+    "<155>",
+    "<156>",
+    "<157>",
+    "<158>",
+    "<159>",
+    "<160>",
+    "<161>",
+    "<162>",
+    "<163>",
+    "<164>",
+    "<165>",
+    "<166>",
+    "<167>",
+    "<168>",
+    "<169>",
+    "<170>",
+    "<171>",
+    "<172>",
+    "<173>",
+    "<174>",
+    "<175>",
+    "<176>",
+    "<177>",
+    "<178>",
+    "<179>",
+    "<180>",
+    "<181>",
+    "<182>",
+    "<183>",
+    "<184>",
+    "<185>",
+    "<186>",
+    "<187>",
+    "<188>",
+    "<189>",
+    "<190>",
+    "<191>",
+    "<192>",
+    "<193>",
+    "<194>",
+    "<195>",
+    "<196>",
+    "<197>",
+    "<198>",
+    "<199>",
+    "<200>",
+    "<201>",
+    "<202>",
+    "<203>",
+    "<204>",
+    "<205>",
+    "<206>",
+    "<207>",
+    "<208>",
+    "<209>",
+    "<210>",
+    "<211>",
+    "<212>",
+    "<213>",
+    "<214>",
+    "<215>",
+    "<216>",
+    "<217>",
+    "<218>",
+    "<219>",
+    "<220>",
+    "<221>",
+    "<222>",
+    "<223>",
+    "<224>",
+    "<225>",
+    "<226>",
+    "<227>",
+    "<228>",
+    "<229>",
+    "<230>",
+    "<231>",
+    "<232>",
+    "<233>",
+    "<234>",
+    "<235>",
+    "<236>",
+    "<237>",
+    "<238>",
+    "<239>",
+    "<240>",
+    "<241>",
+    "<242>",
+    "<243>",
+    "<244>",
+    "<245>",
+    "<246>",
+    "<247>",
+    "<248>",
+    "<249>",
+    "<250>",
+    "<251>",
+    "<252>",
+    "<253>",
+    "<254>",
+    "<255>",
+    "<256>",
+    "<257>",
+    "<258>",
+    "<259>",
+    "<260>",
+    "<261>",
+    "<262>",
+    "<263>",
+    "<264>",
+    "<265>",
+    "<266>",
+    "<267>",
+    "<268>",
+    "<269>",
+    "<270>",
+    "<271>",
+    "<272>",
+    "<273>",
+    "<274>",
+    "<275>",
+    "<276>",
+    "<277>",
+    "<278>",
+    "<279>",
+    "<280>",
+    "<281>",
+    "<282>",
+    "<283>",
+    "<284>",
+    "<285>",
+    "<286>",
+    "<287>",
+    "<288>",
+    "<289>",
+    "<290>",
+    "<291>",
+    "<292>",
+    "<293>",
+    "<294>",
+    "<295>",
+    "<296>",
+    "<297>",
+    "<298>",
+    "<299>",
+    "<300>",
+    "<301>",
+    "<302>",
+    "<303>",
+    "<304>",
+    "<305>",
+    "<306>",
+    "<307>",
+    "<308>",
+    "<309>",
+    "<310>",
+    "<311>",
+    "<312>",
+    "<313>",
+    "<314>",
+    "<315>",
+    "<316>",
+    "<317>",
+    "<318>",
+    "<319>",
+    "<320>",
+    "<321>",
+    "<322>",
+    "<323>",
+    "<324>",
+    "<325>",
+    "<326>",
+    "<327>",
+    "<328>",
+    "<329>",
+    "<330>",
+    "<331>",
+    "<332>",
+    "<333>",
+    "<334>",
+    "<335>",
+    "<336>",
+    "<337>",
+    "<338>",
+    "<339>",
+    "<340>",
+    "<341>",
+    "<342>",
+    "<343>",
+    "<344>",
+    "<345>",
+    "<346>",
+    "<347>",
+    "<348>",
+    "<349>",
+    "<350>",
+    "<351>",
+    "<352>",
+    "<353>",
+    "<354>",
+    "<355>",
+    "<356>",
+    "<357>",
+    "<358>",
+    "<359>",
+    "<360>",
+    "<361>",
+    "<362>",
+    "<363>",
+    "<364>",
+    "<365>",
+    "<366>",
+    "<367>",
+    "<368>",
+    "<369>",
+    "<370>",
+    "<371>",
+    "<372>",
+    "<373>",
+    "<374>",
+    "<375>",
+    "<376>",
+    "<377>",
+    "<378>",
+    "<379>",
+    "<380>",
+    "<381>",
+    "<382>",
+    "<383>",
+    "<384>",
+    "<385>",
+    "<386>",
+    "<387>",
+    "<388>",
+    "<389>",
+    "<390>",
+    "<391>",
+    "<392>",
+    "<393>",
+    "<394>",
+    "<395>",
+    "<396>",
+    "<397>",
+    "<398>",
+    "<399>",
+    "<400>",
+    "<401>",
+    "<402>",
+    "<403>",
+    "<404>",
+    "<405>",
+    "<406>",
+    "<407>",
+    "<408>",
+    "<409>",
+    "<410>",
+    "<411>",
+    "<412>",
+    "<413>",
+    "<414>",
+    "<415>",
+    "<416>",
+    "<417>",
+    "<418>",
+    "<419>",
+    "<420>",
+    "<421>",
+    "<422>",
+    "<423>",
+    "<424>",
+    "<425>",
+    "<426>",
+    "<427>",
+    "<428>",
+    "<429>",
+    "<430>",
+    "<431>",
+    "<432>",
+    "<433>",
+    "<434>",
+    "<435>",
+    "<436>",
+    "<437>",
+    "<438>",
+    "<439>",
+    "<440>",
+    "<441>",
+    "<442>",
+    "<443>",
+    "<444>",
+    "<445>",
+    "<446>",
+    "<447>",
+    "<448>",
+    "<449>",
+    "<450>",
+    "<451>",
+    "<452>",
+    "<453>",
+    "<454>",
+    "<455>",
+    "<456>",
+    "<457>",
+    "<458>",
+    "<459>",
+    "<460>",
+    "<461>",
+    "<462>",
+    "<463>",
+    "<464>",
+    "<465>",
+    "<466>",
+    "<467>",
+    "<468>",
+    "<469>",
+    "<470>",
+    "<471>",
+    "<472>",
+    "<473>",
+    "<474>",
+    "<475>",
+    "<476>",
+    "<477>",
+    "<478>",
+    "<479>",
+    "<480>",
+    "<481>",
+    "<482>",
+    "<483>",
+    "<484>",
+    "<485>",
+    "<486>",
+    "<487>",
+    "<488>",
+    "<489>",
+    "<490>",
+    "<491>",
+    "<492>",
+    "<493>",
+    "<494>",
+    "<495>",
+    "<496>",
+    "<497>",
+    "<498>",
+    "<499>",
+    "<500>",
+    "<501>",
+    "<502>",
+    "<503>",
+    "<504>",
+    "<505>",
+    "<506>",
+    "<507>",
+    "<508>",
+    "<509>",
+    "<510>",
+    "<511>",
+    "<512>",
+    "<513>",
+    "<514>",
+    "<515>",
+    "<516>",
+    "<517>",
+    "<518>",
+    "<519>",
+    "<520>",
+    "<521>",
+    "<522>",
+    "<523>",
+    "<524>",
+    "<525>",
+    "<526>",
+    "<527>",
+    "<528>",
+    "<529>",
+    "<530>",
+    "<531>",
+    "<532>",
+    "<533>",
+    "<534>",
+    "<535>",
+    "<536>",
+    "<537>",
+    "<538>",
+    "<539>",
+    "<540>",
+    "<541>",
+    "<542>",
+    "<543>",
+    "<544>",
+    "<545>",
+    "<546>",
+    "<547>",
+    "<548>",
+    "<549>",
+    "<550>",
+    "<551>",
+    "<552>",
+    "<553>",
+    "<554>",
+    "<555>",
+    "<556>",
+    "<557>",
+    "<558>",
+    "<559>",
+    "<560>",
+    "<561>",
+    "<562>",
+    "<563>",
+    "<564>",
+    "<565>",
+    "<566>",
+    "<567>",
+    "<568>",
+    "<569>",
+    "<570>",
+    "<571>",
+    "<572>",
+    "<573>",
+    "<574>",
+    "<575>",
+    "<576>",
+    "<577>",
+    "<578>",
+    "<579>",
+    "<580>",
+    "<581>",
+    "<582>",
+    "<583>",
+    "<584>",
+    "<585>",
+    "<586>",
+    "<587>",
+    "<588>",
+    "<589>",
+    "<590>",
+    "<591>",
+    "<592>",
+    "<593>",
+    "<594>",
+    "<595>",
+    "<596>",
+    "<597>",
+    "<598>",
+    "<599>",
+    "<600>",
+    "<601>",
+    "<602>",
+    "<603>",
+    "<604>",
+    "<605>",
+    "<606>",
+    "<607>",
+    "<608>",
+    "<609>",
+    "<610>",
+    "<611>",
+    "<612>",
+    "<613>",
+    "<614>",
+    "<615>",
+    "<616>",
+    "<617>",
+    "<618>",
+    "<619>",
+    "<620>",
+    "<621>",
+    "<622>",
+    "<623>",
+    "<624>",
+    "<625>",
+    "<626>",
+    "<627>",
+    "<628>",
+    "<629>",
+    "<630>",
+    "<631>",
+    "<632>",
+    "<633>",
+    "<634>",
+    "<635>",
+    "<636>",
+    "<637>",
+    "<638>",
+    "<639>",
+    "<640>",
+    "<641>",
+    "<642>",
+    "<643>",
+    "<644>",
+    "<645>",
+    "<646>",
+    "<647>",
+    "<648>",
+    "<649>",
+    "<650>",
+    "<651>",
+    "<652>",
+    "<653>",
+    "<654>",
+    "<655>",
+    "<656>",
+    "<657>",
+    "<658>",
+    "<659>",
+    "<660>",
+    "<661>",
+    "<662>",
+    "<663>",
+    "<664>",
+    "<665>",
+    "<666>",
+    "<667>",
+    "<668>",
+    "<669>",
+    "<670>",
+    "<671>",
+    "<672>",
+    "<673>",
+    "<674>",
+    "<675>",
+    "<676>",
+    "<677>",
+    "<678>",
+    "<679>",
+    "<680>",
+    "<681>",
+    "<682>",
+    "<683>",
+    "<684>",
+    "<685>",
+    "<686>",
+    "<687>",
+    "<688>",
+    "<689>",
+    "<690>",
+    "<691>",
+    "<692>",
+    "<693>",
+    "<694>",
+    "<695>",
+    "<696>",
+    "<697>",
+    "<698>",
+    "<699>",
+    "<700>",
+    "<701>",
+    "<702>",
+    "<703>",
+    "<704>",
+    "<705>",
+    "<706>",
+    "<707>",
+    "<708>",
+    "<709>",
+    "<710>",
+    "<711>",
+    "<712>",
+    "<713>",
+    "<714>",
+    "<715>",
+    "<716>",
+    "<717>",
+    "<718>",
+    "<719>",
+    "<720>",
+    "<721>",
+    "<722>",
+    "<723>",
+    "<724>",
+    "<725>",
+    "<726>",
+    "<727>",
+    "<728>",
+    "<729>",
+    "<730>",
+    "<731>",
+    "<732>",
+    "<733>",
+    "<734>",
+    "<735>",
+    "<736>",
+    "<737>",
+    "<738>",
+    "<739>",
+    "<740>",
+    "<741>",
+    "<742>",
+    "<743>",
+    "<744>",
+    "<745>",
+    "<746>",
+    "<747>",
+    "<748>",
+    "<749>",
+    "<750>",
+    "<751>",
+    "<752>",
+    "<753>",
+    "<754>",
+    "<755>",
+    "<756>",
+    "<757>",
+    "<758>",
+    "<759>",
+    "<760>",
+    "<761>",
+    "<762>",
+    "<763>",
+    "<764>",
+    "<765>",
+    "<766>",
+    "<767>",
+    "<768>",
+    "<769>",
+    "<770>",
+    "<771>",
+    "<772>",
+    "<773>",
+    "<774>",
+    "<775>",
+    "<776>",
+    "<777>",
+    "<778>",
+    "<779>",
+    "<780>",
+    "<781>",
+    "<782>",
+    "<783>",
+    "<784>",
+    "<785>",
+    "<786>",
+    "<787>",
+    "<788>",
+    "<789>",
+    "<790>",
+    "<791>",
+    "<792>",
+    "<793>",
+    "<794>",
+    "<795>",
+    "<796>",
+    "<797>",
+    "<798>",
+    "<799>",
+    "<800>",
+    "<801>",
+    "<802>",
+    "<803>",
+    "<804>",
+    "<805>",
+    "<806>",
+    "<807>",
+    "<808>",
+    "<809>",
+    "<810>",
+    "<811>",
+    "<812>",
+    "<813>",
+    "<814>",
+    "<815>",
+    "<816>",
+    "<817>",
+    "<818>",
+    "<819>",
+    "<820>",
+    "<821>",
+    "<822>",
+    "<823>",
+    "<824>",
+    "<825>",
+    "<826>",
+    "<827>",
+    "<828>",
+    "<829>",
+    "<830>",
+    "<831>",
+    "<832>",
+    "<833>",
+    "<834>",
+    "<835>",
+    "<836>",
+    "<837>",
+    "<838>",
+    "<839>",
+    "<840>",
+    "<841>",
+    "<842>",
+    "<843>",
+    "<844>",
+    "<845>",
+    "<846>",
+    "<847>",
+    "<848>",
+    "<849>",
+    "<850>",
+    "<851>",
+    "<852>",
+    "<853>",
+    "<854>",
+    "<855>",
+    "<856>",
+    "<857>",
+    "<858>",
+    "<859>",
+    "<860>",
+    "<861>",
+    "<862>",
+    "<863>",
+    "<864>",
+    "<865>",
+    "<866>",
+    "<867>",
+    "<868>",
+    "<869>",
+    "<870>",
+    "<871>",
+    "<872>",
+    "<873>",
+    "<874>",
+    "<875>",
+    "<876>",
+    "<877>",
+    "<878>",
+    "<879>",
+    "<880>",
+    "<881>",
+    "<882>",
+    "<883>",
+    "<884>",
+    "<885>",
+    "<886>",
+    "<887>",
+    "<888>",
+    "<889>",
+    "<890>",
+    "<891>",
+    "<892>",
+    "<893>",
+    "<894>",
+    "<895>",
+    "<896>",
+    "<897>",
+    "<898>",
+    "<899>",
+    "<900>",
+    "<901>",
+    "<902>",
+    "<903>",
+    "<904>",
+    "<905>",
+    "<906>",
+    "<907>",
+    "<908>",
+    "<909>",
+    "<910>",
+    "<911>",
+    "<912>",
+    "<913>",
+    "<914>",
+    "<915>",
+    "<916>",
+    "<917>",
+    "<918>",
+    "<919>",
+    "<920>",
+    "<921>",
+    "<922>",
+    "<923>",
+    "<924>",
+    "<925>",
+    "<926>",
+    "<927>",
+    "<928>",
+    "<929>",
+    "<930>",
+    "<931>",
+    "<932>",
+    "<933>",
+    "<934>",
+    "<935>",
+    "<936>",
+    "<937>",
+    "<938>",
+    "<939>",
+    "<940>",
+    "<941>",
+    "<942>",
+    "<943>",
+    "<944>",
+    "<945>",
+    "<946>",
+    "<947>",
+    "<948>",
+    "<949>",
+    "<950>",
+    "<951>",
+    "<952>",
+    "<953>",
+    "<954>",
+    "<955>",
+    "<956>",
+    "<957>",
+    "<958>",
+    "<959>",
+    "<960>",
+    "<961>",
+    "<962>",
+    "<963>",
+    "<964>",
+    "<965>",
+    "<966>",
+    "<967>",
+    "<968>",
+    "<969>",
+    "<970>",
+    "<971>",
+    "<972>",
+    "<973>",
+    "<974>",
+    "<975>",
+    "<976>",
+    "<977>",
+    "<978>",
+    "<979>",
+    "<980>",
+    "<981>",
+    "<982>",
+    "<983>",
+    "<984>",
+    "<985>",
+    "<986>",
+    "<987>",
+    "<988>",
+    "<989>",
+    "<990>",
+    "<991>",
+    "<992>",
+    "<993>",
+    "<994>",
+    "<995>",
+    "<996>",
+    "<997>",
+    "<998>",
+    "<999>",
+    "<1000>",
+    "<null>",
+    "<switch>",
+    {
+      "content": "</c>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    }
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

qwen2.5_tokenizer/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:76fbf312c2a3772b8cbde8b6a6c45607962278c72a2f152a41740e96c906fa52
+size 11607003

qwen2.5_tokenizer/tokenizer_config.json

@@ -0,0 +1,20 @@
+{
+  "add_prefix_space": false,
+  "auto_map": {
+    "AutoProcessor": "processing_locateanything.LocateAnythingProcessor"
+  },
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "is_local": true,
+  "local_files_only": false,
+  "model_max_length": 16384,
+  "pad_token": "<|endoftext|>",
+  "processor_class": "LocateAnythingProcessor",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2TokenizerFast",
+  "unk_token": null,
+  "_commit_hash": null
+}

qwen2_p128_l0_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:14daea99276e604c1b13acf8e83b3aa115ac5b5b187b5a52576f5086a8d4bd41
+size 49490620

qwen2_p128_l10_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a0892b977ace619fbcad0475e0457334e80949f41ea1eddbf51ebf10cbacd56c
+size 49490620

qwen2_p128_l11_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e9d0707267e03c8712445b4a1a477054d78d98f9e73559a1ea7b3b20ae23f3c9
+size 49490620

qwen2_p128_l12_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ffeb9ccd98be5c9bdd0e86db98a5595652b64784f968e4c2b416b4cebb81711
+size 49490620

qwen2_p128_l13_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d6ae5219c342bfdbd410e41d5d9d00b64487844bcc56fdd48dd4d65a107161f4
+size 49490620

qwen2_p128_l14_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:335274effb74fc6f4811316123943c78816fa77a12bbd66654cbdec1c62c2cf2
+size 49490620

qwen2_p128_l15_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c983bd2cf5fd63289de8286a46dbc48be0ab11067f86a3b77acb865f50b67de
+size 49490620

qwen2_p128_l16_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f9624bbccc69b5b232e0bd077eca1f3b0c47f908deb5ff73dd6720a90b5ec60d
+size 49490620

qwen2_p128_l17_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a20ea0552e1ec36b6c1229498285e74348b07e1b88dd45424cd70d89bdcf7a02
+size 49490620

qwen2_p128_l18_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0899b44d67f5aabb53682df738595eb41a8d3ecc094a32d3a80703460267acc8
+size 49490620

qwen2_p128_l19_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5aec5840d9a8b2737ed66da11cc05b2c5696132793269bcfa8487345d03ee1e4
+size 49490620

qwen2_p128_l1_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0828672f1c3eec700db19c4a8a6c0ec26d0d6b5989eeaab4f4a52958d7904234
+size 49490620

qwen2_p128_l20_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c1be677880d9689e3c989fb53feeb193590baee23698b613105132e762178a8
+size 49490620

qwen2_p128_l21_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4cf863c2968d082c3e2a2625a3a3a00b36d296e1cead6cfdb5026e4bfdaa5979
+size 49490620

qwen2_p128_l22_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8650aee44861e7f118fb0710e0603ed2b921fb7fa737cd200cb19bbe9a828fad
+size 49490620

qwen2_p128_l23_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ed3c4440fb8368d7fb8e849986b82f18a80e1e2cbce4bdacbe4dd8c50c149d8
+size 49490620

qwen2_p128_l24_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c3e09f54ac6e779e0f547d9e4d3481bd6a60c9798b167c4aff2dbbf0f78d16df
+size 49490620

qwen2_p128_l25_together.axmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:364ae783456850df31cfc4a4501808fe8616457992cba900454df6ad9b99e2c0
+size 49490620