#!/usr/bin/env python3 """ThakiOCR-Large-27B-R2B: transcribe one Korean word-crop image. Loads the base model Qwen/Qwen3.6-27B at a pinned revision, attaches the R2-B LoRA adapter with PEFT, and runs greedy decoding with the exact prompt, image-size limits and chat-template settings used in training and evaluation. python inference_example.py --image word.png python inference_example.py --image word.png --report # JSON with load checks Requirements: one GPU with >= 64 GB memory (bf16 base ~ 52 GB at batch 1), torch, transformers >= 5.0 (model class Qwen3_5ForConditionalGeneration), peft >= 0.21, pillow. Tested stack: torch 2.13 / transformers 5.15.1 / peft 0.21.2. Scope: single word crops only. This is not a full-page OCR system. """ from __future__ import annotations import argparse import hashlib import json import time from pathlib import Path BASE_REPO = "Qwen/Qwen3.6-27B" BASE_REVISION = "6a9e13bd6fc8f0983b9b99948120bc37f49c13e9" # pinned; the adapter was trained on this snapshot ADAPTER_REPO = "ThakiCloud/ThakiOCR-Large-27B-R2B" ADAPTER_SHA256 = "3255f5fac01689a9a7a194f0e5bc5229519b6ab1702923a1839eef024d4b5112" # adapter_model.safetensors # Fixed protocol (identical to training/evaluation; changing it changes the numbers in the card). PROMPT = "이미지에 적힌 글자를 그대로 옮겨 적으세요. 글자만 출력하세요." IMAGE_KWARGS = {"min_pixels": 64 * 64, "max_pixels": 1024 * 1024} MAX_NEW_TOKENS = 64 EXPECTED_LORA_MODULES = 496 # decoder linear layers only; vision tower and merger are not adapted def sha256_file(p: Path) -> str: h = hashlib.sha256() with open(p, "rb") as f: for b in iter(lambda: f.read(1 << 20), b""): h.update(b) return h.hexdigest() def load(base: str, base_revision: str | None, adapter: str, adapter_revision: str | None, device: str): import torch import transformers from peft import PeftModel kw = {"revision": base_revision} if base_revision and not Path(base).exists() else {} processor = transformers.AutoProcessor.from_pretrained(base, **kw) model = transformers.AutoModelForImageTextToText.from_pretrained(base, dtype=torch.bfloat16, **kw).to(device) akw = {"revision": adapter_revision} if adapter_revision and not Path(adapter).exists() else {} model = PeftModel.from_pretrained(model, adapter, **akw) model.eval() return processor, model def lora_stats(model) -> dict: names = [n for n, m in model.named_modules() if hasattr(m, "lora_A") and len(getattr(m, "lora_A", {})) > 0] vision = [n for n in names if "visual" in n or "vision" in n] return {"lora_modules": len(names), "lora_in_vision": len(vision), "expected": EXPECTED_LORA_MODULES} def transcribe(processor, model, image_path: str, device: str) -> dict: import torch from PIL import Image tok = processor.tokenizer msgs = [{"role": "user", "content": [{"type": "image"}, {"type": "text", "text": PROMPT}]}] prompt = tok.apply_chat_template(msgs, tokenize=False, add_generation_prompt=True, enable_thinking=False) with Image.open(image_path) as im: image = im.convert("RGB") inputs = processor(text=[prompt], images=[image], return_tensors="pt", images_kwargs=IMAGE_KWARGS) inputs = {k: v.to(device) for k, v in inputs.items()} t0 = time.perf_counter() with torch.no_grad(): out = model.generate(**inputs, max_new_tokens=MAX_NEW_TOKENS, do_sample=False) # greedy; overrides sampling defaults in generation_config new = out[0, inputs["input_ids"].shape[1]:] return {"text": tok.decode(new, skip_special_tokens=True).strip(), "new_tokens": int(new.shape[0]), "hit_max_new_tokens": int(new.shape[0]) >= MAX_NEW_TOKENS, "latency_s": round(time.perf_counter() - t0, 3)} def main() -> int: ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--image", required=True, help="path to one word-crop image") ap.add_argument("--base", default=BASE_REPO, help="HF repo id or local directory of the base model") ap.add_argument("--base-revision", default=BASE_REVISION) ap.add_argument("--adapter", default=ADAPTER_REPO, help="HF repo id or local directory of the adapter") ap.add_argument("--adapter-revision", default=None) ap.add_argument("--device", default="cuda:0") ap.add_argument("--report", action="store_true", help="print JSON with load checks instead of plain text") a = ap.parse_args() checks = {} local = Path(a.adapter) / "adapter_model.safetensors" if local.exists(): sha = sha256_file(local) checks["adapter_sha256"] = sha checks["adapter_sha256_ok"] = sha == ADAPTER_SHA256 processor, model = load(a.base, a.base_revision, a.adapter, a.adapter_revision, a.device) checks.update(lora_stats(model)) res = transcribe(processor, model, a.image, a.device) if a.report: print(json.dumps({"result": res, "checks": checks, "protocol": {"prompt": PROMPT, "image_kwargs": IMAGE_KWARGS, "max_new_tokens": MAX_NEW_TOKENS, "decoding": "greedy", "enable_thinking": False, "base": a.base, "base_revision": a.base_revision}}, ensure_ascii=False)) else: print(res["text"]) return 0 if __name__ == "__main__": raise SystemExit(main())