Update README.md

README.md

@@ -18,8 +18,24 @@ pipeline_tag: text-generation
 
 # NuMarkdown-Qwen2.5-VL 🖋️📄 → 📝
 
-**NuMarkdown-Qwen2.5-VL** is the first reasoning vision-language trained to converts documents into clean GitHub-flavoured Markdown.
-It is a lightweight fine-tune of **Qwen 2.5-VL-7B** using ~10 k synthetic doc-to-Markdown pairs, GRPO RL phase pass with a layout-centric reward.
+**NuMarkdown-Qwen2.5-VL** is the first reasoning vision-language model trained to converts documents into clean GitHub-flavoured Markdown.
+It is a lightweight fine-tune of **Qwen 2.5-VL-7B** using ~10 k synthetic doc-to-Markdown pairs, followed by a RL phase (GRPO) with a layout-centric reward.
+
+By increasing the output length by 10% to 20%, the model outperform model of it's size and is competitive with top close source reasoning model
+
+---
+## Results
+
+(we plan to realease a markdown arena -similar to llmArena- for complex table to markdown format)
+
+Winrate of our model vs open source alternative:
+
+//
+
+
+Winrate vs close source alternative:
+
+//
 
 ---
 
@@ -34,24 +50,41 @@ It is a lightweight fine-tune of **Qwen 2.5-VL-7B** using ~10 k synthetic doc-to
 ## Quick start: 🤗 Transformers
 
 ```python
+from __future__ import annotations
+
+import torch
 from PIL import Image
-from transformers import AutoModelForCausalLM, AutoTokenizer, AutoProcessor
+from transformers import AutoProcessor, Qwen2_5_VLForConditionalGeneration
 
-model_id = "NM-dev/NuMarkdown-Qwen2.5-VL"
-tok  = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
-proc = AutoProcessor.from_pretrained(model_id, trust_remote_code=True)
-model = AutoModelForCausalLM.from_pretrained(
+model_id = "NM-dev/NuMarkdown-Qwen2.5-VL"       
+
+processor = AutoProcessor.from_pretrained(
     model_id,
-    torch_dtype="bfloat16",
+    trust_remote_code=True,       
+)
+
+model = Qwen2_5_VLForConditionalGeneration.from_pretrained(
+    model_id,
+    torch_dtype=torch.bfloat16,
+    attn_implementation="flash_attention_2",
     device_map="auto",
     trust_remote_code=True,
 )
 
-img = Image.open("invoice_scan.png")
-inputs = proc(text="Convert this to Markdown with reasoning.", image=img,
-              return_tensors="pt").to(model.device)
-out = model.generate(**inputs, max_new_tokens=1024)
-print(tok.decode(out[0], skip_special_tokens=True))
+img = Image.open("invoice_scan.png").convert("RGB")
+messages = [{
+    "role": "user",
+    "content": [
+        {"type": "image"},
+    ],
+}]
+prompt = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
+enc = processor(text=prompt, images=[img], return_tensors="pt").to(model.device)
+
+with torch.no_grad():
+    out = model.generate(**enc, max_new_tokens=1024)
+
+print(processor.decode(out[0], skip_special_tokens=True))
 ```
 
 
@@ -69,7 +102,7 @@ img = Image.open("invoice_scan.png")
 prompt = proc(text="Convert this to Markdown with reasoning.", image=img,
               return_tensors="np")  # numpy arrays for vLLM
 
-params = SamplingParameters(max_tokens=1024, temperature=0.2, top_p=0.95)
+params = SamplingParameters(max_tokens=1024, temperature=0.8, top_p=0.95)
 result = llm.generate([{"prompt": prompt}], params)[0].outputs[0].text
 print(result)
 ```