Duplicate from infly/Infinity-Parser2-Flash

Co-authored-by: Zuming Huang <zuminghuang@users.noreply.huggingface.co>

.eval_results/parsebench.yaml

@@ -0,0 +1,60 @@
+- dataset:
+    id: llamaindex/ParseBench
+    task_id: mean
+  value: 73.25
+  date: '2026-05-15'
+  source:
+    url: https://huggingface.co/datasets/llamaindex/ParseBench
+    name: ParseBench
+    user: boyang-runllama
+  notes: "Pipeline name: infinity_parser2_flash"
+- dataset:
+    id: llamaindex/ParseBench
+    task_id: text_content
+  value: 89.52
+  date: '2026-05-15'
+  source:
+    url: https://huggingface.co/datasets/llamaindex/ParseBench
+    name: ParseBench
+    user: boyang-runllama
+  notes: "Pipeline name: infinity_parser2_flash"
+- dataset:
+    id: llamaindex/ParseBench
+    task_id: text_formatting
+  value: 57.7
+  date: '2026-05-15'
+  source:
+    url: https://huggingface.co/datasets/llamaindex/ParseBench
+    name: ParseBench
+    user: boyang-runllama
+  notes: "Pipeline name: infinity_parser2_flash"
+- dataset:
+    id: llamaindex/ParseBench
+    task_id: layout
+  value: 80.61
+  date: '2026-05-15'
+  source:
+    url: https://huggingface.co/datasets/llamaindex/ParseBench
+    name: ParseBench
+    user: boyang-runllama
+  notes: "Pipeline name: infinity_parser2_flash"
+- dataset:
+    id: llamaindex/ParseBench
+    task_id: chart
+  value: 55.56
+  date: '2026-05-15'
+  source:
+    url: https://huggingface.co/datasets/llamaindex/ParseBench
+    name: ParseBench
+    user: boyang-runllama
+  notes: "Pipeline name: infinity_parser2_flash"
+- dataset:
+    id: llamaindex/ParseBench
+    task_id: table
+  value: 82.88
+  date: '2026-05-15'
+  source:
+    url: https://huggingface.co/datasets/llamaindex/ParseBench
+    name: ParseBench
+    user: boyang-runllama
+  notes: "Pipeline name: infinity_parser2_flash"

.gitattributes

@@ -0,0 +1,37 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text
+assets/example_1.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,596 @@
+---
+license: apache-2.0
+language:
+  - en
+  - zh
+  - multilingual
+pipeline_tag: image-text-to-text
+library_name: transformers
+tags:
+  - ocr
+  - pdf
+  - document-parsing
+  - document-understanding
+  - layout-analysis
+  - table-recognition
+  - chart-parsing
+  - formula-recognition
+  - chemical-formula
+  - markdown
+  - vision-language
+  - infinity-parser
+  - infinity_parser2
+datasets:
+  - infly/Infinity-Doc2-5M
+---
+
+# Infinity-Parser2-Flash
+
+<p align="center">
+    <img src="assets/logo.png" width="400"/>
+<p>
+
+<p align="center">
+💻 <a href="https://github.com/infly-ai/INF-MLLM">Github</a> |
+📊 <a href="https://huggingface.co/datasets/infly/Infinity-Doc2-5M">Dataset</a> |
+📄 <a href="https://arxiv.org/pdf/2607.07836">Paper</a> |
+🚀 <a href="https://huggingface.co/spaces/infly/Infinity-Parser2-Demo">Demo</a>
+</p>
+
+## News
+
+- [2026-05-11] Released flagship document parsing models: [Infinity-Parser2-Pro](https://huggingface.co/infly/Infinity-Parser2-Pro), [Infinity-Parser2-Flash](https://huggingface.co/infly/Infinity-Parser2-Flash), and the dataset [Infinity-Doc2-5M](https://huggingface.co/datasets/infly/Infinity-Doc2-5M). Infinity-Parser2 achieves SOTA results on olmOCR-bench and ParseBench.
+
+<p align="center">
+    <img src="https://raw.githubusercontent.com/infly-ai/INF-MLLM/main/Infinity-Parser2/assets/newspaper_1.png" width="1200"/>
+</p>
+
+## Introduction
+
+We are excited to release Infinity-Parser2, our latest flagship document understanding model. We offer two distinct variants to address diverse deployment constraints: Infinity-Parser2-Pro, optimized for maximum accuracy in precision-critical tasks, achieves state-of-the-art results on olmOCR-Bench (87.6%) and ParseBench (74.3%), surpassing frontier models including DeepSeek-OCR-2, PaddleOCR-VL-1.5, and MinerU2.5. Infinity-Parser2-Flash, engineered for low-latency inference, delivers a 3.68x speedup over our previous Infinity-Parser-7B model. With significant upgrades to both our data engine and multi-task reinforcement learning approach, the model consolidates robust multi-modal parsing capabilities into a unified architecture, unlocking brand-new zero-shot capabilities across a wide range of real-world business scenarios.
+  
+### Key Features
+
+- **Upgraded Data Engine**: We have comprehensively enhanced our synthetic data engine to support both fixed-layout and flexible-layout document formats. By curating nearly 5 million diverse document parsing samples across a wide range of layouts, combined with a dynamic adaptive sampling strategy, we ensure highly balanced and robust multi-task learning across various document types.
+- **Multi-Task Reinforcement Learning**: We designed a novel verifiable reward system to support Joint Reinforcement Learning (RL), enabling seamless and simultaneous co-optimization of multiple complex tasks, including document parsing, element parsing, chart parsing, chemical formula parsing, document vqa, and general multimodal understanding.
+- **Breakthrough Parsing Performance**: Infinity-Parser2-Pro substantially outperforms our previous 7B model, achieving 87.6% on olmOCR-Bench and 74.3% on ParseBench, surpassing frontier models such as DeepSeek-OCR-2, PaddleOCR-VL, and MinerU2.5.
+- **Inference Acceleration**: Infinity-Parser2-Flash delivers significantly higher efficiency than Infinity-Parser-7B, with inference throughput increased by 3.68x (from 441 to 1,624 tokens/sec), reducing both deployment latency and costs.
+
+## Performance
+
+<p align="left">
+    <img src="https://raw.githubusercontent.com/infly-ai/INF-MLLM/main/Infinity-Parser2/assets/olmocr_bench_perf.png" width="1200"/>
+<p>
+
+<p align="left">
+    <img src="https://raw.githubusercontent.com/infly-ai/INF-MLLM/main/Infinity-Parser2/assets/parsebench_perf.png" width="1200"/>
+<p>
+
+<table align="center">
+  <thead>
+    <tr>
+      <th>Task</th>
+      <th>Infinity-Parser2-Pro</th>
+      <th>Infinity-Parser2-Flash</th>
+      <th>PaddleOCR-VL-1.5</th>
+      <th>DeepSeek-OCR-2</th>
+      <th>MinerU2.5</th>
+      <th>Gemini-3-Pro</th>
+    </tr>
+  </thead>
+  <tbody>
+    <tr>
+      <td colspan=7><b>Document Parsing</b></td>
+    </tr>
+    <tr>
+      <td>olmOCR-bench</td>
+      <td><b>87.6</b></td>
+      <td>86.0</td>
+      <td>78.5</td>
+      <td>76.3</td>
+      <td>75.2</td>
+      <td>-</td>
+    </tr>
+    <tr>
+      <td>ParseBench</td>
+      <td><b>74.3</b></td>
+      <td>72.2</td>
+      <td>66.0</td>
+      <td>41.2</td>
+      <td>45.9</td>
+      <td>69.1‡</td>
+    </tr>
+    <tr>
+      <td>OmniDocBench-v1.6</td>
+      <td>93.95</td>
+      <td>91.98</td>
+      <td><b>94.87</b></td>
+      <td>90.17</td>
+      <td>92.98</td>
+      <td>92.85</td>
+    </tr>
+    <tr>
+      <td colspan=7><b>Layout Analysis (mIoU)</b></td>
+    </tr>
+    <tr>
+      <td>DocLayNet</td>
+      <td>64.93*</td>
+      <td>64.97*</td>
+      <td><b>71.05*</b></td>
+      <td>45.62*</td>
+      <td>67.74*</td>
+      <td>-</td>
+    </tr>
+    <tr>
+      <td>D4LA</td>
+      <td><b>52.41*</b></td>
+      <td>46.05*</td>
+      <td>50.21*</td>
+      <td>33.03*</td>
+      <td>51.62*</td>
+      <td>-</td>
+    </tr>
+    <tr>
+      <td>OmniDocBench-v1.5-Layout</td>
+      <td>74.56*</td>
+      <td>73.07*</td>
+      <td>74.80*</td>
+      <td>55.28*</td>
+      <td><b>76.28*</b></td>
+      <td>-</td>
+    </tr>
+    <tr>
+      <td colspan=7><b>Element Parsing</b></td>
+    </tr>
+    <tr>
+      <td>OmniDocBench-v1.5-TextBlock</td>
+      <td><b>95.05</b></td>
+      <td>94.31</td>
+      <td>94.97*</td>
+      <td>84.13*</td>
+      <td>86.00</td>
+      <td>-</td>
+    </tr>
+    <tr>
+      <td>PubTabNet (val)</td>
+      <td><b>94.76</b></td>
+      <td>92.41</td>
+      <td>84.60</td>
+      <td>89.53*</td>
+      <td>89.07</td>
+      <td>91.40</td>
+    </tr>
+    <tr>
+      <td>UniMERNet</td>
+      <td><b>97.7</b></td>
+      <td>96.5</td>
+      <td>95.8*</td>
+      <td>79.8*</td>
+      <td>96.5</td>
+      <td>96.4</td>
+    </tr>
+    <tr>
+      <td colspan=7><b>Chart Parsing</b></td>
+    </tr>
+    <tr>
+      <td>Chart2Table</td>
+      <td><b>86.5</b></td>
+      <td>80.49</td>
+      <td>86.2*</td>
+      <td>49.7*</td>
+      <td>-</td>
+      <td>-</td>
+    </tr>
+    <tr>
+      <td>Chart2Json</td>
+      <td><b>73.69</b></td>
+      <td>67.66</td>
+      <td>-</td>
+      <td>-</td>
+      <td>-</td>
+      <td>-</td>
+    </tr>
+    <tr>
+      <td colspan=7><b>Chemical Formula Parsing</b></td>
+    </tr>
+    <tr>
+      <td>CoSyn_Chemical</td>
+      <td><b>73.19</b></td>
+      <td>63.34</td>
+      <td>-</td>
+      <td>47.02*</td>
+      <td>-</td>
+      <td>-</td>
+    </tr>
+    <tr>
+      <td colspan=7><b>Document VQA</b></td>
+    </tr>
+    <tr>
+      <td>DocVQA (val)</td>
+      <td><b>96.43</b></td>
+      <td>93.16</td>
+      <td>-</td>
+      <td>43.42*</td>
+      <td>-</td>
+      <td>93.68*</td>
+    </tr>
+    <tr>
+      <td>InfoVQA (val)</td>
+      <td><b>86.26</b></td>
+      <td>75.94</td>
+      <td>-</td>
+      <td>22.07*</td>
+      <td>-</td>
+      <td>85.24*</td>
+    </tr>
+    <tr>
+      <td colspan=7><b>General Multimodal Understanding</b></td>
+    </tr>
+    <tr>
+      <td>AI2D</td>
+      <td>88.89</td>
+      <td>79.53</td>
+      <td>-</td>
+      <td>37.66*</td>
+      <td>-</td>
+      <td><b>91.87*</b></td>
+    </tr>
+    <tr>
+      <td>MathVista (testmini)</td>
+      <td>71.4</td>
+      <td>59.5</td>
+      <td>-</td>
+      <td>-</td>
+      <td>-</td>
+      <td><b>81.8*</b></td>
+    </tr>
+    <tr>
+      <td>MMBench-EN (dev)</td>
+      <td>87.54</td>
+      <td>77.92</td>
+      <td>-</td>
+      <td>-</td>
+      <td>-</td>
+      <td><b>90.29*</b></td>
+    </tr>
+    <tr>
+      <td>MMBench-CN (dev)</td>
+      <td>86.43</td>
+      <td>75.77</td>
+      <td>-</td>
+      <td>-</td>
+      <td>-</td>
+      <td><b>90.98*</b></td>
+    </tr>
+    <tr>
+      <td>MMMU (val)</td>
+      <td><b>61.89</b></td>
+      <td>45.89</td>
+      <td>-</td>
+      <td>-</td>
+      <td>-</td>
+      <td>56.00*</td>
+    </tr>
+    <tr>
+      <td>MMStar</td>
+      <td>69.66</td>
+      <td>57.13</td>
+      <td>-</td>
+      <td>-</td>
+      <td>-</td>
+      <td><b>83.78*</b></td>
+    </tr>
+    <tr>
+      <td>OCRBench</td>
+      <td>86.20</td>
+      <td>81.60</td>
+      <td>-</td>
+      <td>47.20*</td>
+      <td>-</td>
+      <td><b>89.30*</b></td>
+    </tr>
+  </tbody>
+</table>
+
+Note: '*' denotes results evaluated using our internal evaluation tools. '‡' denotes results from the Gemini-3.1-Pro.
+
+## Quick Start
+
+### 1. Minimal "Hello World" (Native Transformers)
+
+If you are looking for a minimal script to parse a single image to Markdown using the native `transformers` library, here is a simple snippet:
+
+```python
+from PIL import Image
+import torch
+from transformers import AutoModelForImageTextToText, AutoProcessor
+from qwen_vl_utils import process_vision_info
+
+# Load the model and processor
+model = AutoModelForImageTextToText.from_pretrained(
+    "infly/Infinity-Parser2-Pro",
+    torch_dtype="float16",
+    device_map="auto",
+)
+processor = AutoProcessor.from_pretrained("infly/Infinity-Parser2-Pro")
+
+# Build the messages for the model
+pil_image = Image.open("demo_data/demo.png").convert("RGB")
+min_pixels = 2048  # 32 * 64
+max_pixels = 16777216  # 4096 * 4096
+prompt = """
+- Extract layout information from the provided PDF image.
+- For each layout element, output its bbox, category, and the text content within the bbox.
+- Bbox format: [x1, y1, x2, y2].
+- Allowed layout categories: ['header', 'title', 'text', 'figure', 'table', 'formula', 'figure_caption', 'table_caption', 'formula_caption', 'figure_footnote', 'table_footnote', 'page_footnote', 'footer'].
+- Text extraction and formatting:
+  1) For 'figure', the text field must be an empty string.
+  2) For 'formula', format text as LaTeX.
+  3) For 'table', format text as HTML.
+  4) For all other categories (e.g., text, title), format text as Markdown.
+- The output text must be exactly the original text from the image, with no translation or rewriting.
+- Sort all layout elements in human reading order.
+- Final output must be a single JSON object.
+"""
+
+messages = [
+    {
+        "role": "user",
+        "content": [
+            {
+                "type": "image",
+                "image": pil_image,
+                "min_pixels": min_pixels,
+                "max_pixels": max_pixels,
+            },
+            {"type": "text", "text": prompt},
+        ],
+    }
+]
+
+chat_template_kwargs = {"enable_thinking": False}
+
+text = processor.apply_chat_template(
+    messages, tokenize=False, add_generation_prompt=True, **chat_template_kwargs
+)
+image_inputs, _ = process_vision_info(messages, image_patch_size=16)
+
+inputs = processor(
+    text=text,
+    images=image_inputs,
+    do_resize=False,
+    padding=True,
+    return_tensors="pt",
+)
+
+# Move all tensors to the same device as the model
+inputs = {
+    k: v.to(model.device) if isinstance(v, torch.Tensor) else v
+    for k, v in inputs.items()
+}
+
+# Generate the response
+generated_ids = model.generate(
+    **inputs,
+    max_new_tokens=32768,
+    temperature=0.0,
+    top_p=1.0,
+)
+
+# Strip input tokens, keeping only the newly generated response
+generated_ids_trimmed = [
+    out_ids[len(in_ids) :]
+    for in_ids, out_ids in zip(inputs["input_ids"], generated_ids)
+]
+output_text = processor.batch_decode(
+    generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False
+)
+print(output_text)
+```
+
+### 2. Advanced Pipeline (infinity_parser2)
+
+For bulk processing, advanced features, or an end-to-end PDF parsing pipeline, we recommend using our infinity_parser2 wrapper.
+
+#### Pre-requisites
+
+```bash
+# Create a Conda environment (Optional)
+conda create -n infinity_parser2 python=3.12
+conda activate infinity_parser2
+
+# Install PyTorch (CUDA). Find the proper version at https://pytorch.org/get-started/previous-versions based on your CUDA version.
+pip install torch==2.10.0 torchvision==0.25.0 torchaudio==2.10.0 --index-url https://download.pytorch.org/whl/cu128
+
+# Install FlashAttention (FlashAttention-2 is recommended by default)
+# Standard install (compiles from source, ~10-30 min):
+pip install flash-attn==2.8.3 --no-build-isolation
+# Faster install: download wheel from https://github.com/Dao-AILab/flash-attention/releases. Then run: pip install /path/to/<wheel_filename>.whl
+# For Hopper GPUs (e.g. H100, H800), we recommend FlashAttention-3 instead. See: https://github.com/Dao-AILab/flash-attention
+# NOTE: The code will prioritize detecting FlashAttention-3. If not found, it falls back to FlashAttention-2.
+
+# Install vLLM
+# NOTE: you may need to run the command below to resolve triton and numpy conflicts before installing vllm.
+# pip uninstall -y pytorch-triton opencv-python opencv-python-headless numpy && rm -rf "$(python -c 'import site; print(site.getsitepackages()[0])')/cv2"
+pip install vllm==0.17.1
+```
+
+#### Install infinity_parser2
+
+Install from PyPI
+
+```bash
+pip install infinity_parser2
+```
+
+Install from source code
+
+```bash
+git clone https://github.com/infly-ai/INF-MLLM.git
+cd INF-MLLM/Infinity-Parser2
+pip install -e .
+```
+
+#### Usage
+
+##### Command Line
+
+The `parser` command is the fastest way to get started.
+
+```bash
+# NOTE: The Infinity-Parser2 model will be automatically downloaded on the first run.
+
+# Parse a PDF (outputs Markdown by default)
+parser demo_data/demo.pdf
+
+# Parse an image
+parser demo_data/demo.png
+
+# Batch parse multiple files
+parser demo_data/demo.pdf demo_data/demo.png -o ./output
+
+# Parse an entire directory
+parser demo_data -o ./output
+
+# Output raw JSON with layout bboxes
+parser demo_data/demo.pdf --output-format json
+
+# Convert to Markdown directly
+parser demo_data/demo.png --task doc2md
+```
+
+```bash
+# View all options
+parser --help
+```
+
+##### Python API
+
+```python
+# NOTE: The Infinity-Parser2 model will be automatically downloaded on the first run.
+
+from infinity_parser2 import InfinityParser2
+
+parser = InfinityParser2()
+
+# Parse a single file (returns Markdown)
+result = parser.parse("demo_data/demo.pdf")
+print(result)
+
+# Parse multiple files (returns list)
+results = parser.parse(["demo_data/demo.pdf", "demo_data/demo.png"])
+
+# Parse a directory (returns dict)
+results = parser.parse("demo_data")
+```
+
+**Output formats:**
+
+| task_type   | Description                                          | Default Output |
+|-------------|------------------------------------------------------|----------------|
+| `doc2json`  | Extract layout elements with bboxes (default)        | Markdown       |
+| `doc2md`    | Directly convert to Markdown                         | Markdown       |
+| `custom`    | Use your own prompt                                 | Raw model output |
+
+```python
+# doc2json: get raw JSON with bbox coordinates
+result = parser.parse("demo_data/demo.pdf", output_format="json")
+
+# doc2md: direct Markdown conversion
+result = parser.parse("demo_data/demo.pdf", task_type="doc2md")
+
+# Custom prompt
+result = parser.parse("demo_data/demo.pdf", task_type="custom",
+                      custom_prompt="Please transform the document's contents into Markdown format.")
+
+# Batch processing with custom batch size
+result = parser.parse("demo_data", batch_size=8)
+
+# Save results to directory
+parser.parse("demo_data/demo.pdf", output_dir="./output")
+```
+
+**Backends:**
+
+Infinity-Parser2 supports three inference backends. By default it uses the **vLLM Engine** (offline batch inference).
+
+```python
+# vLLM Engine (default) — offline batch inference
+parser = InfinityParser2(
+    model_name="infly/Infinity-Parser2-Pro",
+    backend="vllm-engine",        # default
+    tensor_parallel_size=2,
+)
+
+# Transformers — local single-GPU inference
+parser = InfinityParser2(
+    model_name="infly/Infinity-Parser2-Pro",
+    backend="transformers",
+    device="cuda",
+    torch_dtype="bfloat16",       # "float16" or "bfloat16"
+)
+
+# vLLM Server — online HTTP API (start server first)
+parser = InfinityParser2(
+    model_name="infly/Infinity-Parser2-Pro",
+    backend="vllm-server",
+    api_url="http://localhost:8000/v1/chat/completions",
+    api_key="EMPTY",
+)
+```
+
+To start a vLLM server:
+
+```bash
+vllm serve infly/Infinity-Parser2-Pro \
+    --trust-remote-code \
+    --reasoning-parser qwen3 \
+    --host 0.0.0.0 \
+    --port 8000 \
+    --tensor-parallel-size 2 \
+    --gpu-memory-utilization 0.85 \
+    --max-model-len 65536 \
+    --mm-encoder-tp-mode data \
+    --mm-processor-cache-type shm \
+    --enable-prefix-caching
+```
+
+For more details, please refer to the [official guide](https://github.com/infly-ai/INF-MLLM/blob/main/Infinity-Parser2).
+
+### Visual Parsing Examples
+
+| Visualization | Note |
+| --- | --- | 
+| [A-Stock](https://raw.githubusercontent.com/infly-ai/INF-MLLM/main/Infinity-Parser2/assets/a_stock.png) | Easy to miscount colspan in tables |
+| [Multi-Column Layout](https://raw.githubusercontent.com/infly-ai/INF-MLLM/main/Infinity-Parser2/assets/muti_column.png) | Complex layout analysis and reading order recovery. |
+| [Historical Newspaper](https://raw.githubusercontent.com/infly-ai/INF-MLLM/main/Infinity-Parser2/assets/newspaper_2.png) | High probability of bounding box omission caused by ultra-dense text distribution, narrow column margins, and microscopic fonts. |
+| [US-Stock](https://raw.githubusercontent.com/infly-ai/INF-MLLM/main/Infinity-Parser2/assets/us_stock.png) | Precise row alignment across wide frameless spaces and capturing the hierarchical semantics of indented headers. |
+| [Academic Paper (arXiv)](https://raw.githubusercontent.com/infly-ai/INF-MLLM/main/Infinity-Parser2/assets/arxiv.png) | Accurate structural preservation of complex multi-line mathematical formulas, dense inline notations, and deeply nested subscripts/superscripts. |
+| [Magazine Page](https://raw.githubusercontent.com/infly-ai/INF-MLLM/main/Infinity-Parser2/assets/magazine.png) | Complex reading order recovery in an asymmetric multi-column layout. |
+| [Scanned Mathematics](https://raw.githubusercontent.com/infly-ai/INF-MLLM/main/Infinity-Parser2/assets/old_scan_math.png) | Degraded and blurred print |
+
+## Limitations
+
+Infinity-Parser2 has several known limitations to consider. It primarily supports English and Chinese documents, and performance degrades when processing multilingual content. Accuracy may also be reduced when parsing charts with complex layouts, as well as documents containing multi-oriented elements such as table rotated at varying angles. Additionally, the model does not capture fine-grained text formatting (e.g., bold, italic, strikethrough) and exhibits suboptimal multimodal instruction-following capability, meaning it may not always reliably follow complex multi-step visual instructions.
+
+## Acknowledgments
+
+We would like to thank [Qwen3.5](https://github.com/QwenLM/Qwen3.5), [ms-swift](https://github.com/modelscope/ms-swift), [VeRL](https://github.com/verl-project/verl), [lmms-eval](https://github.com/EvolvingLMMs-Lab/lmms-eval), [olmocr](https://huggingface.co/datasets/allenai/olmOCR-bench), [PaddleOCR-VL](https://github.com/PaddlePaddle/PaddleOCR), [MinerU](https://github.com/opendatalab/MinerU), [dots.ocr](https://github.com/rednote-hilab/dots.ocr), [Chandra-OCR-2](https://github.com/datalab-to/chandra) for providing dataset, code and models.
+
+## Citation
+
+```
+@misc{huang2026infinityparser2technicalreport,
+      title={Infinity-Parser2 Technical Report},
+      author={Zuming Huang and Jun Huang and Kexuan Ren and Baode Wang and Weizhen Li and Jianming Feng and Yu Wang and Yichen Yao and Shijun Lin and Yige Tang and Cheng Peng and Weidi Xu and Wei Chu and Yinghui Xu and Yuan Qi},
+      year={2026},
+      eprint={2607.07836},
+      archivePrefix={arXiv},
+      primaryClass={cs.AI},
+      url={https://arxiv.org/abs/2607.07836},
+}
+```
+
+# License
+
+This model is licensed under apache-2.0.

chat_template.jinja

@@ -0,0 +1,154 @@
+{%- set image_count = namespace(value=0) %}
+{%- set video_count = namespace(value=0) %}
+{%- macro render_content(content, do_vision_count, is_system_content=false) %}
+    {%- if content is string %}
+        {{- content }}
+    {%- elif content is iterable and content is not mapping %}
+        {%- for item in content %}
+            {%- if 'image' in item or 'image_url' in item or item.type == 'image' %}
+                {%- if is_system_content %}
+                    {{- raise_exception('System message cannot contain images.') }}
+                {%- endif %}
+                {%- if do_vision_count %}
+                    {%- set image_count.value = image_count.value + 1 %}
+                {%- endif %}
+                {%- if add_vision_id %}
+                    {{- 'Picture ' ~ image_count.value ~ ': ' }}
+                {%- endif %}
+                {{- '<|vision_start|><|image_pad|><|vision_end|>' }}
+            {%- elif 'video' in item or item.type == 'video' %}
+                {%- if is_system_content %}
+                    {{- raise_exception('System message cannot contain videos.') }}
+                {%- endif %}
+                {%- if do_vision_count %}
+                    {%- set video_count.value = video_count.value + 1 %}
+                {%- endif %}
+                {%- if add_vision_id %}
+                    {{- 'Video ' ~ video_count.value ~ ': ' }}
+                {%- endif %}
+                {{- '<|vision_start|><|video_pad|><|vision_end|>' }}
+            {%- elif 'text' in item %}
+                {{- item.text }}
+            {%- else %}
+                {{- raise_exception('Unexpected item type in content.') }}
+            {%- endif %}
+        {%- endfor %}
+    {%- elif content is none or content is undefined %}
+        {{- '' }}
+    {%- else %}
+        {{- raise_exception('Unexpected content type.') }}
+    {%- endif %}
+{%- endmacro %}
+{%- if not messages %}
+    {{- raise_exception('No messages provided.') }}
+{%- endif %}
+{%- if tools and tools is iterable and tools is not mapping %}
+    {{- '<|im_start|>system\n' }}
+    {{- "# Tools\n\nYou have access to the following functions:\n\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>" }}
+    {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n</IMPORTANT>' }}
+    {%- if messages[0].role == 'system' %}
+        {%- set content = render_content(messages[0].content, false, true)|trim %}
+        {%- if content %}
+            {{- '\n\n' + content }}
+        {%- endif %}
+    {%- endif %}
+    {{- '<|im_end|>\n' }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {%- set content = render_content(messages[0].content, false, true)|trim %}
+        {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" %}
+        {%- set content = render_content(message.content, false)|trim %}
+        {%- if not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}
+            {%- set ns.multi_step_tool = false %}
+            {%- set ns.last_query_index = index %}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if ns.multi_step_tool %}
+    {{- raise_exception('No user query found in messages.') }}
+{%- endif %}
+{%- for message in messages %}
+    {%- set content = render_content(message.content, true)|trim %}
+    {%- if message.role == "system" %}
+        {%- if not loop.first %}
+            {{- raise_exception('System message must be at the beginning.') }}
+        {%- endif %}
+    {%- elif message.role == "user" %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- set reasoning_content = reasoning_content|trim %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content + '\n</think>\n\n' + content }}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if tool_call.function is defined %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {%- if loop.first %}
+                    {%- if content|trim %}
+                        {{- '\n\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
+                    {%- else %}
+                        {{- '<tool_call>\n<function=' + tool_call.name + '>\n' }}
+                    {%- endif %}
+                {%- else %}
+                    {{- '\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
+                {%- endif %}
+                {%- if tool_call.arguments is defined %}
+                    {%- for args_name, args_value in tool_call.arguments|items %}
+                        {{- '<parameter=' + args_name + '>\n' }}
+                        {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}
+                        {{- args_value }}
+                        {{- '\n</parameter>\n' }}
+                    {%- endfor %}
+                {%- endif %}
+                {{- '</function>\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.previtem and loop.previtem.role != "tool" %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if not loop.last and loop.nextitem.role != "tool" %}
+            {{- '<|im_end|>\n' }}
+        {%- elif loop.last %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- else %}
+        {{- raise_exception('Unexpected message role.') }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is true %}
+        {{- '<think>\n' }}
+    {%- else %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}

config.json

@@ -0,0 +1,106 @@
+{
+  "architectures": [
+    "Qwen3_5ForConditionalGeneration"
+  ],
+  "bos_token_id": null,
+  "dtype": "bfloat16",
+  "eos_token_id": 248046,
+  "hidden_size": 2048,
+  "image_token_id": 248056,
+  "model_type": "qwen3_5",
+  "pad_token_id": 248044,
+  "rope_theta": 10000000,
+  "text_config": {
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "attn_output_gate": true,
+    "bos_token_id": null,
+    "dtype": "bfloat16",
+    "eos_token_id": 248044,
+    "full_attention_interval": 4,
+    "head_dim": 256,
+    "hidden_act": "silu",
+    "hidden_size": 2048,
+    "initializer_range": 0.02,
+    "intermediate_size": 6144,
+    "layer_types": [
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention"
+    ],
+    "linear_conv_kernel_dim": 4,
+    "linear_key_head_dim": 128,
+    "linear_num_key_heads": 16,
+    "linear_num_value_heads": 16,
+    "linear_value_head_dim": 128,
+    "mamba_ssm_dtype": "float32",
+    "max_position_embeddings": 262144,
+    "mlp_only_layers": [],
+    "model_type": "qwen3_5_text",
+    "mtp_num_hidden_layers": 1,
+    "mtp_use_dedicated_embeddings": false,
+    "num_attention_heads": 8,
+    "num_hidden_layers": 24,
+    "num_key_value_heads": 2,
+    "pad_token_id": null,
+    "partial_rotary_factor": 0.25,
+    "rms_norm_eps": 1e-06,
+    "rope_parameters": {
+      "mrope_interleaved": true,
+      "mrope_section": [
+        11,
+        11,
+        10
+      ],
+      "partial_rotary_factor": 0.25,
+      "rope_theta": 10000000,
+      "rope_type": "default"
+    },
+    "tie_word_embeddings": true,
+    "use_cache": true,
+    "vocab_size": 248320
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "5.3.0",
+  "video_token_id": 248057,
+  "vision_config": {
+    "deepstack_visual_indexes": [],
+    "depth": 24,
+    "dtype": "bfloat16",
+    "hidden_act": "gelu_pytorch_tanh",
+    "hidden_size": 1024,
+    "in_channels": 3,
+    "initializer_range": 0.02,
+    "intermediate_size": 4096,
+    "model_type": "qwen3_5",
+    "num_heads": 16,
+    "num_position_embeddings": 2304,
+    "out_hidden_size": 2048,
+    "patch_size": 16,
+    "spatial_merge_size": 2,
+    "temporal_patch_size": 2
+  },
+  "vision_end_token_id": 248054,
+  "vision_start_token_id": 248053
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:761cb9295e05047bc8628effd7381023458f193955b804098dd332c464f26150
+size 4426558832

processor_config.json

@@ -0,0 +1,63 @@
+{
+  "image_processor": {
+    "data_format": "channels_first",
+    "do_convert_rgb": true,
+    "do_normalize": true,
+    "do_rescale": true,
+    "do_resize": true,
+    "image_mean": [
+      0.5,
+      0.5,
+      0.5
+    ],
+    "image_processor_type": "Qwen2VLImageProcessorFast",
+    "image_std": [
+      0.5,
+      0.5,
+      0.5
+    ],
+    "merge_size": 2,
+    "patch_size": 16,
+    "resample": 3,
+    "rescale_factor": 0.00392156862745098,
+    "size": {
+      "longest_edge": 16777216,
+      "shortest_edge": 65536
+    },
+    "temporal_patch_size": 2
+  },
+  "processor_class": "Qwen3VLProcessor",
+  "video_processor": {
+    "data_format": "channels_first",
+    "default_to_square": true,
+    "do_convert_rgb": true,
+    "do_normalize": true,
+    "do_rescale": true,
+    "do_resize": true,
+    "do_sample_frames": true,
+    "fps": 2,
+    "image_mean": [
+      0.5,
+      0.5,
+      0.5
+    ],
+    "image_std": [
+      0.5,
+      0.5,
+      0.5
+    ],
+    "max_frames": 768,
+    "merge_size": 2,
+    "min_frames": 4,
+    "patch_size": 16,
+    "resample": 3,
+    "rescale_factor": 0.00392156862745098,
+    "return_metadata": false,
+    "size": {
+      "longest_edge": 25165824,
+      "shortest_edge": 4096
+    },
+    "temporal_patch_size": 2,
+    "video_processor_type": "Qwen3VLVideoProcessor"
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:87a7830d63fcf43bf241c3c5242e96e62dd3fdc29224ca26fed8ea333db72de4
+size 19989343

tokenizer_config.json

@@ -0,0 +1,32 @@
+{
+  "add_prefix_space": false,
+  "audio_bos_token": "<|audio_start|>",
+  "audio_eos_token": "<|audio_end|>",
+  "audio_token": "<|audio_pad|>",
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "image_token": "<|image_pad|>",
+  "is_local": true,
+  "model_max_length": 262144,
+  "model_specific_special_tokens": {
+    "audio_bos_token": "<|audio_start|>",
+    "audio_eos_token": "<|audio_end|>",
+    "audio_token": "<|audio_pad|>",
+    "image_token": "<|image_pad|>",
+    "video_token": "<|video_pad|>",
+    "vision_bos_token": "<|vision_start|>",
+    "vision_eos_token": "<|vision_end|>"
+  },
+  "pad_token": "<|endoftext|>",
+  "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
+  "processor_class": "Qwen3VLProcessor",
+  "split_special_tokens": false,
+  "tokenizer_class": "TokenizersBackend",
+  "unk_token": null,
+  "video_token": "<|video_pad|>",
+  "vision_bos_token": "<|vision_start|>",
+  "vision_eos_token": "<|vision_end|>"
+}