ThakiOCR-Large-27B-R2B / inference_example.py
thaki-AI's picture
ThakiOCR-Large-27B-R2B: Korean word OCR research preview (LoRA adapter)
e024771 verified
Raw History Blame Contribute Delete
5.5 kB
#!/usr/bin/env python3
"""ThakiOCR-Large-27B-R2B: transcribe one Korean word-crop image.
Loads the base model Qwen/Qwen3.6-27B at a pinned revision, attaches the
R2-B LoRA adapter with PEFT, and runs greedy decoding with the exact prompt,
image-size limits and chat-template settings used in training and evaluation.
python inference_example.py --image word.png
python inference_example.py --image word.png --report # JSON with load checks
Requirements: one GPU with >= 64 GB memory (bf16 base ~ 52 GB at batch 1),
torch, transformers >= 5.0 (model class Qwen3_5ForConditionalGeneration),
peft >= 0.21, pillow. Tested stack: torch 2.13 / transformers 5.15.1 / peft 0.21.2.
Scope: single word crops only. This is not a full-page OCR system.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import time
from pathlib import Path
BASE_REPO = "Qwen/Qwen3.6-27B"
BASE_REVISION = "6a9e13bd6fc8f0983b9b99948120bc37f49c13e9" # pinned; the adapter was trained on this snapshot
ADAPTER_REPO = "ThakiCloud/ThakiOCR-Large-27B-R2B"
ADAPTER_SHA256 = "3255f5fac01689a9a7a194f0e5bc5229519b6ab1702923a1839eef024d4b5112" # adapter_model.safetensors
# Fixed protocol (identical to training/evaluation; changing it changes the numbers in the card).
PROMPT = "์ด๋ฏธ์ง€์— ์ ํžŒ ๊ธ€์ž๋ฅผ ๊ทธ๋Œ€๋กœ ์˜ฎ๊ฒจ ์ ์œผ์„ธ์š”. ๊ธ€์ž๋งŒ ์ถœ๋ ฅํ•˜์„ธ์š”."
IMAGE_KWARGS = {"min_pixels": 64 * 64, "max_pixels": 1024 * 1024}
MAX_NEW_TOKENS = 64
EXPECTED_LORA_MODULES = 496 # decoder linear layers only; vision tower and merger are not adapted
def sha256_file(p: Path) -> str:
h = hashlib.sha256()
with open(p, "rb") as f:
for b in iter(lambda: f.read(1 << 20), b""):
h.update(b)
return h.hexdigest()
def load(base: str, base_revision: str | None, adapter: str, adapter_revision: str | None, device: str):
import torch
import transformers
from peft import PeftModel
kw = {"revision": base_revision} if base_revision and not Path(base).exists() else {}
processor = transformers.AutoProcessor.from_pretrained(base, **kw)
model = transformers.AutoModelForImageTextToText.from_pretrained(base, dtype=torch.bfloat16, **kw).to(device)
akw = {"revision": adapter_revision} if adapter_revision and not Path(adapter).exists() else {}
model = PeftModel.from_pretrained(model, adapter, **akw)
model.eval()
return processor, model
def lora_stats(model) -> dict:
names = [n for n, m in model.named_modules() if hasattr(m, "lora_A") and len(getattr(m, "lora_A", {})) > 0]
vision = [n for n in names if "visual" in n or "vision" in n]
return {"lora_modules": len(names), "lora_in_vision": len(vision), "expected": EXPECTED_LORA_MODULES}
def transcribe(processor, model, image_path: str, device: str) -> dict:
import torch
from PIL import Image
tok = processor.tokenizer
msgs = [{"role": "user", "content": [{"type": "image"}, {"type": "text", "text": PROMPT}]}]
prompt = tok.apply_chat_template(msgs, tokenize=False, add_generation_prompt=True, enable_thinking=False)
with Image.open(image_path) as im:
image = im.convert("RGB")
inputs = processor(text=[prompt], images=[image], return_tensors="pt", images_kwargs=IMAGE_KWARGS)
inputs = {k: v.to(device) for k, v in inputs.items()}
t0 = time.perf_counter()
with torch.no_grad():
out = model.generate(**inputs, max_new_tokens=MAX_NEW_TOKENS, do_sample=False) # greedy; overrides sampling defaults in generation_config
new = out[0, inputs["input_ids"].shape[1]:]
return {"text": tok.decode(new, skip_special_tokens=True).strip(),
"new_tokens": int(new.shape[0]),
"hit_max_new_tokens": int(new.shape[0]) >= MAX_NEW_TOKENS,
"latency_s": round(time.perf_counter() - t0, 3)}
def main() -> int:
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--image", required=True, help="path to one word-crop image")
ap.add_argument("--base", default=BASE_REPO, help="HF repo id or local directory of the base model")
ap.add_argument("--base-revision", default=BASE_REVISION)
ap.add_argument("--adapter", default=ADAPTER_REPO, help="HF repo id or local directory of the adapter")
ap.add_argument("--adapter-revision", default=None)
ap.add_argument("--device", default="cuda:0")
ap.add_argument("--report", action="store_true", help="print JSON with load checks instead of plain text")
a = ap.parse_args()
checks = {}
local = Path(a.adapter) / "adapter_model.safetensors"
if local.exists():
sha = sha256_file(local)
checks["adapter_sha256"] = sha
checks["adapter_sha256_ok"] = sha == ADAPTER_SHA256
processor, model = load(a.base, a.base_revision, a.adapter, a.adapter_revision, a.device)
checks.update(lora_stats(model))
res = transcribe(processor, model, a.image, a.device)
if a.report:
print(json.dumps({"result": res, "checks": checks,
"protocol": {"prompt": PROMPT, "image_kwargs": IMAGE_KWARGS, "max_new_tokens": MAX_NEW_TOKENS,
"decoding": "greedy", "enable_thinking": False,
"base": a.base, "base_revision": a.base_revision}},
ensure_ascii=False))
else:
print(res["text"])
return 0
if __name__ == "__main__":
raise SystemExit(main())