diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..fcfaf45cf1c62c506193ce354af91225587338d4 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +kn-en/models/layout/selftest.jpg filter=lfs diff=lfs merge=lfs -text +hi-en/models/layout/selftest.jpg filter=lfs diff=lfs merge=lfs -text diff --git a/hi-en/VERSION b/hi-en/VERSION new file mode 100644 index 0000000000000000000000000000000000000000..781dcb07cd802a4e98079e3f56fc38f7b092c7a0 --- /dev/null +++ b/hi-en/VERSION @@ -0,0 +1 @@ +1.1.3 diff --git a/hi-en/arjuna_hi.yaml b/hi-en/arjuna_hi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9b3242d74b455f6754bf406e7430694be52589de --- /dev/null +++ b/hi-en/arjuna_hi.yaml @@ -0,0 +1,95 @@ +# Arjuna-OCR [Hi & En] v1.1.3 — inference configuration. +# Paths are relative to this file. Environment overrides: KANEN_CONFIG, KANEN_TRT=1, KANEN_ORT_CPU=1, +# KANEN_REC_ONNX, KANEN_BACKGROUND_TRT=0 (build TensorRT engines up front instead of in the background). +name: "Arjuna-OCR [Hi & En]" +version: 1.1.3 # package version; stamped into every output document as model.pipeline + +models: + det: models/det/inference.onnx + layout: models/layout/arjuna_layout_v15.onnx # Arjuna layout v15 (15 classes incl. barcode/list_item/section_header) + layout_labels: models/layout/classes_v15.txt + layout_selftest: models/layout/selftest.jpg + # MUST come from the SAME layout model: a v12 reference rejects every v14 engine at 75 % and silently + # drops the layout stage to the CUDA provider (5x slower end to end). + layout_selftest_ref: models/layout/selftest_ref_v15.json # a v14 reference REJECTS a correct v15 engine + rec: models/rec/arjuna_rec_hi.onnx + charset: models/rec/charset.json + +provider: + policy: auto # auto = trt -> cuda -> cpu + trt_fp16: true + engine_cache: trt_cache + trt_workspace_gb: 4 + cuda_mem_limit_gb: 6 + cudnn_conv_algo_search: HEURISTIC + +detector: + # Resolution-aware rescale. A page whose MEDIAN detected line is shorter than min_line_px is + # re-detected after upscaling so its lines land near target_line_px. The recogniser is trained at + # H=32 and the detector never enlarges a small page on its own, so a low-DPI scan was being read + # at 12-25 px. 0 disables. Costs a second detect pass, and ONLY on the pages that need it. + min_line_px: 32 + target_line_px: 45 + max_page_scale: 3.0 + canvases: [1280, 1920, 2560] + width_classes: [0.5, 0.625, 0.75, 0.875, 1.0] + min_scale: 0.9 + default_canvas: 1280 + thresh: 0.2 + box_thresh: 0.45 + unclip: 1.4 + pad_top: 0.03 + pad_bottom: 0.25 + batch: 2 + +layout: + mode: always + batch: 4 + score_thresh: 0.15 # chosen on real-text survival, not mAP + fill: [1.0, 0.8] # two passes: 1.0 for small blocks, 0.8 keeps whole-page tables inside the head's + # regression range. [1.0] is ~7 % faster and returned 2 fewer lines on the blind set. + selftest_min_match: 1.0 + +recognizer: + buckets: [128, 192, 256, 384, 512, 640, 768, 1024, 1280] + batch: 64 + height: 32 + max_ratio: 40 + +second_pass: + conf: 0.85 + cap: 6 + +runtime: + workers: auto + page_batch: 16 # 16 measured fastest on a 4-core host; 32 is past the knee + log_level: warning + polygons: false + +assemble: + low_res_px: 26 # FLAG (never alter) a line read from very few pixels + retype_visual_min_lines: 4 # a figure/seal holding this many readable lines is a mislabelled panel -> re-type it + row_cross_frac: 0.20 # two side-by-side blocks are ONE table when a spanning strip or crossing lines say so AND the gutter clears 2.5x line height + # KNOWN LIMITATION — see README "Known limitations". Text inside these block types is withheld from page + # `text` (the lines stay in `blocks` with their flags). On registration records this has been observed to + # delete a whole correctly-read panel when layout mislabels it `figure` at low confidence. + suppress_text_in: [] # nothing is ever deleted. Measured on 54 held-out pages / 14,184 words: + # [] loses 0 words, ["handwriting","barcode"] loses 26. Lines keep their + # in_seal / in_handwriting / in_figure flags so a consumer can filter. + # Guards. Without them, suppression deleted 119 lines of REAL text out of 160 withheld across 87 pages of + # registration records — certificate and GRN numbers, dates, the registrar's seal text. Cause: an e-stamp + # panel holds emblems, a barcode and a QR code, so the layout model boxes the WHOLE panel as `figure` at + # confidence 0.31-0.38 over a third of the page. With these guards: 119 -> 28, and `figure` no longer + # appears among suppressed blocks at all. Set any to its OFF value (0.0 / 1.0 / 0) to restore the old + # unconditional behaviour. + suppress_min_conf: 0.50 # a marginal layout call keeps the text + suppress_max_area: 0.10 # a block larger than 10 % of the page is not a figure + suppress_max_lines: 3 + # Merge SAME-CLASS layout blocks drawn over the same region. The layout model sometimes cuts one paragraph + # into boxes that interleave vertically; every line is then recognised correctly but emitted in block order, + # which scrambles the page (observed: 14/14 lines right, all in the wrong place). Requires overlap on BOTH + # axes — x by 0.60 of the narrower box, y by 0.25 of the shorter — so two-column pages, stacked paragraphs + # and nested seals never merge. Measured over 87 pages: non-space characters unchanged; line count falls + # only where fragments of one row rejoin ("2." + "Type of Property" -> "2. Type of Property"). 0 disables. + merge_split_blocks_x: 0.60 + merge_split_blocks_y: 0.25 # a "figure" holding more than 3 readable lines is a mislabelled panel diff --git a/hi-en/kanen_infer/__init__.py b/hi-en/kanen_infer/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..67fe5312e59b11ae537091ed672e076e5a5cc4bd --- /dev/null +++ b/hi-en/kanen_infer/__init__.py @@ -0,0 +1,9 @@ +"""Arjuna-OCR [Kn&En] inference package (Kannada + English document OCR): ONNX Runtime + optional TensorRT, no PyTorch.""" +from pathlib import Path as _P +try: __version__ = (_P(__file__).resolve().parent.parent / "VERSION").read_text(encoding="utf-8").strip() +except Exception: __version__ = "1.1.3" +NAME = "Arjuna-OCR [Kn&En]" # product display name — also `name:` in kanen.yaml (the config value wins at run time) +from .config import load_config, PKG_ROOT # noqa: E402 +from .api import KanEnOCR # noqa: E402 +from .formats import to_json, to_markdown, to_text, to_hocr, to_alto, to_tsv, render # noqa: E402 +__all__ = ["KanEnOCR", "load_config", "PKG_ROOT", "NAME", "__version__", "to_json", "to_markdown", "to_text", "to_hocr", "to_alto", "to_tsv", "render"] diff --git a/hi-en/kanen_infer/api.py b/hi-en/kanen_infer/api.py new file mode 100644 index 0000000000000000000000000000000000000000..5adee2b9ffc29cf6ec3ccc05e8e09d05e9e0428b --- /dev/null +++ b/hi-en/kanen_infer/api.py @@ -0,0 +1,119 @@ +"""Arjuna-OCR [Kn&En] Python API. + + from kanen_infer import KanEnOCR + ocr = KanEnOCR() # loads kanen.yaml, builds/validates engines (warm-up), see ocr.ready() + doc = ocr.page("scan.png") # one kanen-ocr/1.0 document (dict) + docs = ocr.pages(["a.png", "b.pdf"]) # batched; ocr.stream(...) yields documents as they finish + boxes = ocr.detect(img) # text-line polygons only + texts = ocr.recognize(crops) # [(text, conf, min_conf)] for line images + blocks = ocr.layout(img) # layout blocks only + ocr.render(doc, "hocr") # json | md | txt | hocr | alto | tsv + +Inputs: file path, bytes, numpy BGR array, PIL image, file object; PDFs through `pages()`/`stream()` (pypdfium2 or pdftoppm, +optional). The output document never depends on the provider or the batch composition beyond fp16 noise on garbage lines; +no confidence filtering, no text rewriting, no re-ordering happens here.""" +import logging, time +import numpy as np, cv2 +from pathlib import Path +from .config import load_config, PKG_ROOT +from .providers import Providers +from .rec_onnx import KanEnRecognizer +from .pipeline_base import Detector, Layout, crop +from .batch_pipeline import BatchOCR +from .io import to_bgr, expand_inputs, iter_page_sources +from . import formats + +LOG = logging.getLogger("kanen") + +class KanEnOCR: + def __init__(self, config=None, providers=None, layout_mode=None, workers=None, warmup=True, **overrides): + """config: path to a kanen.yaml/json, a dict, or None (package default). providers: 'trt' | 'cuda' | 'cpu' | 'auto' + (overrides config provider.policy). Other keyword overrides use dotted section names, e.g. layout={'mode': 'auto'}.""" + ov = dict(overrides) + if providers: ov.setdefault("provider", {})["policy"] = providers + if layout_mode: ov.setdefault("layout", {})["mode"] = layout_mode + if workers: ov.setdefault("runtime", {})["workers"] = workers + self.cfg = load_config(config, ov); self.name = self.cfg.get("name", "Arjuna-OCR [Kn&En]"); self.version = str(self.cfg.get("version", "1.0.0")) + t0 = time.time(); self.prov = Providers(self.cfg) + self.rec = KanEnRecognizer(cfg=self.cfg, providers=self.prov); self.det = Detector(self.cfg, self.prov) + self.lay = Layout(self.cfg, self.prov) if self.cfg["layout"].get("mode", "always") != "never" else None + self.batch = BatchOCR(self.rec, cfg=self.cfg, providers=self.prov, det=self.det, lay=self.lay) + self._load_s = round(time.time() - t0, 1); self._warm = None + if warmup: self.warmup() + + # ---- status -------------------------------------------------------------------------------------------------- + def warmup(self): + """Run the bundled self-test page through the full pipeline once: builds/validates TensorRT engines for the + default shapes (layout self-test, detector canvas, recognizer profile) so the first real page is not slow.""" + t0 = time.time(); p = self.cfg.path("models", "layout_selftest") + if p.exists(): + img = to_bgr(str(p)); self.batch.run([img], page_batch=1, loader=lambda x: x) + self._warm = round(time.time() - t0, 1); return self.ready() + + def optimize(self, wait=False, timeout=None): + """Build the TensorRT engines for this machine in the background and swap them in once each one is validated + against the session it replaces. Safe to call at any time; returns immediately unless `wait` is set. + Long-lived processes (server, endpoint handler, batch jobs) should call this at startup: the pipeline keeps + serving on the CUDA provider meanwhile, and gets 2-3x faster when the engines are ready.""" + n = self.prov.start_background_builds() if hasattr(self.prov, "start_background_builds") else 0 + if wait and n: return self.prov.wait_for_engines(timeout) + return {"builds_started": n} + + def ready(self): + """Provider each model ended on, fallbacks that happened, engine-cache state, model hashes.""" + st = self.prov.summary() + if self.lay is not None: st.setdefault("layout", {})["engine"] = self.lay.stage + return {"name": self.name, "version": self.version, "config": self.cfg.source, "gpu": self.prov.gpu, "tensorrt": self.prov.trt_ver, + "providers": st, "events": list(self.prov.events), "engine_cache": str(self.prov.cache_root), "load_s": self._load_s, "warmup_s": self._warm, + "model": dict(self.batch.model_ids)} + + # ---- full documents ------------------------------------------------------------------------------------------- + def page(self, image, polygons=None, dpi=None): + """Single page (latency path): one kanen-ocr/1.0 document.""" + img = to_bgr(image); docs, _ = self.batch.process([img], dpi=dpi, with_polygons=self._poly(polygons)); d = docs[0] + if isinstance(image, (str, Path)): d["page"]["file"] = Path(image).name + return d + + def pages(self, inputs, page_batch=None, polygons=None, pdf_dpi=200): + """Batched: list/dir/glob of image or PDF paths, or a list of arrays / PIL images / bytes -> list of documents.""" + return [d for d in self.stream(inputs, page_batch, polygons, pdf_dpi)] + + def stream(self, inputs, page_batch=None, polygons=None, pdf_dpi=200): + """Generator variant of pages(): yields documents in input order as each page batch completes.""" + sources = self._sources(inputs, pdf_dpi); labels = [s[0] for s in sources]; items = [s[1] for s in sources] + page_batch = page_batch or self.cfg["runtime"].get("page_batch", 8); k = 0 + for doc, st in self.batch.stream(items, page_batch=page_batch, loader=to_bgr, with_polygons=self._poly(polygons)): + if labels[k]: doc["page"]["file"] = labels[k] + self.last_stats = st; k += 1; yield doc + + def _sources(self, inputs, pdf_dpi): + if isinstance(inputs, (str, Path)) or (isinstance(inputs, (list, tuple)) and inputs and all(isinstance(x, (str, Path)) for x in inputs)): + return list(iter_page_sources(expand_inputs(inputs), dpi=pdf_dpi)) + if isinstance(inputs, (list, tuple)): return [(getattr(x, "name", None) or "", x) for x in inputs] + return [("", inputs)] + + def _poly(self, polygons): return bool(self.cfg["runtime"].get("polygons", False)) if polygons is None else bool(polygons) + + # ---- single stages -------------------------------------------------------------------------------------------- + def detect(self, image): + """Text-line detection only -> list of (polygon 4x2 float32 in page pixels, score).""" + return self.det(to_bgr(image)) + + def recognize(self, crops, batch=None): + """Recognition only on a list of line images (any size, BGR or gray) -> [(text, confidence, min_conf)].""" + res = self.rec([to_bgr(c) for c in crops], bs=batch) + return [(t, float(c), float(getattr(c, "min_char", c))) for t, c in res] + + def layout(self, image): + """Layout blocks only -> [{type, score, bbox, order}] (v15 labels: text, title, section_header, list_item, table, figure, caption, footnote, formula, header, footer, seal, handwriting, id_panel, barcode).""" + if self.lay is None: self.lay = Layout(self.cfg, self.prov); self.batch.lay = self.lay + return self.lay(to_bgr(image))[0] + + def crops(self, image, boxes=None): + """Perspective line crops for detect() boxes (helper for recognize()).""" + img = to_bgr(image); boxes = self.detect(img) if boxes is None else boxes + return [crop(img, b) for b, _ in boxes] + + # ---- serialisation -------------------------------------------------------------------------------------------- + def render(self, doc, fmt="json"): + return formats.render(doc, fmt) diff --git a/hi-en/kanen_infer/arjuna_cli.py b/hi-en/kanen_infer/arjuna_cli.py new file mode 100644 index 0000000000000000000000000000000000000000..82b9822223ee8c6da92afa1b31217d3f5fae66d3 --- /dev/null +++ b/hi-en/kanen_infer/arjuna_cli.py @@ -0,0 +1,111 @@ +"""`arjuna` command line — the deployment front end. + + arjuna doctor what this machine can do, what the pipeline will use, and why + arjuna ocr page.pdf -o out/ OCR files, directories, globs or PDFs (same engine as kanen-ocr) + arjuna serve --port 8080 HTTP server (FastAPI); builds TensorRT engines in the background + arjuna bench measured pages/s on the bundled sample pages, before and after optimisation + arjuna warm build and validate the TensorRT engines for this machine, then exit + +`ocr` stays on the CUDA/CPU provider for small jobs (a TensorRT build costs more than it saves for a page or two) and +optimises automatically from `--optimize-after` pages upwards. `serve`, `bench` and `warm` always optimise. +""" +import argparse, json, os, sys, time + +def _cfg(args): + from .config import load_config + from . import hardware + cfg = load_config(args.config) + if getattr(args, "provider", None): cfg["provider"]["policy"] = args.provider + changes = [] if getattr(args, "no_autoconfig", False) else hardware.autoconfig(cfg, quality=getattr(args, "quality", None)) + return cfg, changes + +def cmd_doctor(a): + from . import hardware + cfg, changes = _cfg(a) + hw = hardware.probe(cfg["provider"].get("policy")); print(hardware.report(cfg, changes, hw)) + if a.run: + from .api import KanEnOCR + import cv2, glob + pages = sorted(glob.glob(os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "samples", "pages", "*")))[:1] + if pages: + t = time.time(); ocr = KanEnOCR(config=cfg, warmup=False); load = time.time() - t + t = time.time(); doc = ocr.page(cv2.imread(pages[0])); dt = time.time() - t + lines = sum(len(b["lines"]) for b in doc["blocks"]) + print(f"\n load {load:.1f}s\n sample page {dt:.1f}s, {lines} lines, {len(doc['blocks'])} blocks ({os.path.basename(pages[0])})") + print(f" providers {json.dumps({k: v['provider'] for k, v in ocr.prov.status.items() if k in ('det', 'rec', 'layout')})}") + for e in ocr.prov.events[:6]: print(f" {e}") + return 0 + +def cmd_warm(a): + from .api import KanEnOCR + cfg, changes = _cfg(a) + print("building TensorRT engines for this machine (this is the one slow step; it is cached afterwards)...", flush=True) + t = time.time(); ocr = KanEnOCR(config=cfg, warmup=True) + res = ocr.optimize(wait=True, timeout=a.timeout) + print(f"done in {time.time() - t:.0f}s: {json.dumps(res)}") + print("providers:", json.dumps({k: v["provider"] for k, v in ocr.prov.status.items() if k in ("det", "rec", "layout")})) + return 0 + +def cmd_bench(a): + import cv2, glob + from .api import KanEnOCR + cfg, changes = _cfg(a) + root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + pages = sorted(glob.glob(os.path.join(root, "samples", "pages", "*"))) + if not pages: print("no sample pages bundled"); return 1 + imgs = [cv2.imread(p) for p in pages] * max(1, a.repeat) + ocr = KanEnOCR(config=cfg, warmup=True) + def run(tag): + t = time.time(); docs = ocr.pages(imgs); dt = time.time() - t + print(f" {tag:<22} {len(imgs)/dt:5.2f} pages/s ({dt:.1f}s for {len(imgs)} pages, {sum(len(b['lines']) for d in docs for b in d['blocks'])} lines)") + return len(imgs) / dt + print(f"pages: {len(imgs)} · providers before: {json.dumps({k: v['provider'] for k, v in ocr.prov.status.items() if k in ('det','rec','layout')})}") + run("as loaded") + ocr.optimize(wait=True, timeout=a.timeout) + print(f"providers after: {json.dumps({k: v['provider'] for k, v in ocr.prov.status.items() if k in ('det','rec','layout')})}") + run("after optimisation"); run("steady state") + return 0 + +def cmd_serve(a): + import importlib.util + root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + spec = importlib.util.spec_from_file_location("kanen_serve", os.path.join(root, "serve.py")) + mod = importlib.util.module_from_spec(spec); sys.modules["kanen_serve"] = mod; spec.loader.exec_module(mod) + return mod.serve(a) + +def cmd_ocr(a): + from .cli import main as kanen_main + argv = ["--input"] + a.inputs + ["--out-dir", a.out_dir, "--format", a.format] + for flag, val in (("--page-batch", a.page_batch), ("--workers", a.workers), ("--provider", a.provider), ("--config", a.config)): + if val: argv += [flag, str(val)] + if a.json_lines: argv.append("--json-lines") + if a.quiet: argv.append("--quiet") + os.environ["KANEN_OPTIMIZE_AFTER"] = str(a.optimize_after) + return kanen_main(argv) + +def build_parser(): + ap = argparse.ArgumentParser(prog="arjuna", description="Arjuna-OCR [Kn&En] — Kannada + English document OCR") + ap.add_argument("--version", action="store_true") + sub = ap.add_subparsers(dest="cmd") + common = lambda p: (p.add_argument("--config", default=None), p.add_argument("--provider", choices=["auto", "trt", "cuda", "cpu"], default=None), + p.add_argument("--quality", choices=["fast", "balanced", "exact"], default=None), p.add_argument("--no-autoconfig", action="store_true")) + d = sub.add_parser("doctor", help="hardware, providers, auto-configuration"); common(d); d.add_argument("--run", action="store_true", help="also OCR one bundled page"); d.set_defaults(fn=cmd_doctor) + w = sub.add_parser("warm", help="build + validate the TensorRT engines, then exit"); common(w); w.add_argument("--timeout", type=int, default=1800); w.set_defaults(fn=cmd_warm) + b = sub.add_parser("bench", help="measured pages/s on the bundled pages"); common(b); b.add_argument("--repeat", type=int, default=4); b.add_argument("--timeout", type=int, default=1800); b.set_defaults(fn=cmd_bench) + s = sub.add_parser("serve", help="HTTP server"); common(s); s.add_argument("--host", default="0.0.0.0"); s.add_argument("--port", type=int, default=8080) + s.add_argument("--workers", type=int, default=None); s.add_argument("--layout-mode", choices=["always", "auto", "never"], default=None); s.set_defaults(fn=cmd_serve) + o = sub.add_parser("ocr", help="OCR files / directories / globs / PDFs"); common(o) + o.add_argument("inputs", nargs="+"); o.add_argument("--out-dir", "-o", default="arjuna_out"); o.add_argument("--format", "-f", default="json") + o.add_argument("--page-batch", type=int, default=None); o.add_argument("--workers", type=int, default=None); o.add_argument("--json-lines", action="store_true") + o.add_argument("--quiet", "-q", action="store_true"); o.add_argument("--optimize-after", type=int, default=8, help="build TensorRT engines when a job has at least this many pages") + o.set_defaults(fn=cmd_ocr) + return ap + +def main(argv=None): + a = build_parser().parse_args(argv) + if a.version: + from . import __version__; print(__version__); return 0 + if not getattr(a, "fn", None): build_parser().print_help(); return 0 + return a.fn(a) + +if __name__ == "__main__": sys.exit(main()) diff --git a/hi-en/kanen_infer/assemble.py b/hi-en/kanen_infer/assemble.py new file mode 100644 index 0000000000000000000000000000000000000000..c6f168a8d7ec120040fbc417a4d3ade16a7844a0 --- /dev/null +++ b/hi-en/kanen_infer/assemble.py @@ -0,0 +1,505 @@ +"""Assembler: layout blocks (Arjuna layout v12) + text-line polygons + recognised text -> kanen-ocr/1.0 JSON. +Pure geometry, no model. Also: Markdown / plain-text renderers.""" +import numpy as np +from .charset import script_of +import re +WEAK_CHAR = 0.50 # min-char confidence below which a line is flagged weak_char (calibrated on 3,478 dev lines: < 0.5 -> 65 % exact) +_MIX_RE = re.compile(r"(?=\S*[A-Za-z])(?=\S*[\u0900-\u097F])\S+") # token with both a Latin letter and a Devanagari letter (Hindi package) + +TYPE_MAP = {"doc_title": "title", "paragraph_title": "section_title", "text": "text", "abstract": "text", "content": "text", "reference": "text", + "reference_content": "text", "aside_text": "text", "vertical_text": "text", "footnote": "footnote", "vision_footnote": "footnote", + "header": "header", "footer": "footer", "number": "page_number", "table": "table", "image": "figure", "chart": "figure", + "figure_title": "figure_caption", "seal": "seal", "display_formula": "formula", "inline_formula": "formula", "formula_number": "formula", + "algorithm": "text", "header_image": "figure", "footer_image": "figure", + "title": "title", "figure": "figure", "id_panel": "id_panel", "handwriting": "handwriting", # v1.2 layout classes (text/table/seal/header/footer map 1:1 above) + # v15 layout adds six classes v1.2 could not express. `barcode` is the important one: without it an + # e-stamp panel can only be called `figure`, and suppressing that figure deleted 119 lines of real + # text across 87 pages. A barcode carries no readable text, so suppressing IT costs nothing. + "section_header": "section_title", "list_item": "text", "caption": "figure_caption", "barcode": "barcode"} + +def _bbox(poly): p = np.asarray(poly, np.float32); return [float(p[:, 0].min()), float(p[:, 1].min()), float(p[:, 0].max()), float(p[:, 1].max())] + +def _overlap_frac(line_bb, block_bb): + ix = max(0, min(line_bb[2], block_bb[2]) - max(line_bb[0], block_bb[0])); iy = max(0, min(line_bb[3], block_bb[3]) - max(line_bb[1], block_bb[1])) + return ix * iy / max((line_bb[2] - line_bb[0]) * (line_bb[3] - line_bb[1]), 1e-6) + +def merge_line_fragments(lines, y_tol=0.5, x_gap=1.5): + """Merge horizontally adjacent boxes on the same baseline (detector sometimes splits a line at wide gaps). + lines: list of dict(bbox, polygon, text, conf). Returns merged list.""" + lines = sorted(lines, key=lambda l: (l["bbox"][1], l["bbox"][0])); out = [] + for l in lines: + merged = False + for m in out: + h = min(m["bbox"][3] - m["bbox"][1], l["bbox"][3] - l["bbox"][1]) + same_row = abs((m["bbox"][1] + m["bbox"][3]) / 2 - (l["bbox"][1] + l["bbox"][3]) / 2) < y_tol * h + gap = l["bbox"][0] - m["bbox"][2] + if same_row and -0.2 * h < gap < x_gap * h: + m["bbox"] = [min(m["bbox"][0], l["bbox"][0]), min(m["bbox"][1], l["bbox"][1]), max(m["bbox"][2], l["bbox"][2]), max(m["bbox"][3], l["bbox"][3])] + m["text"] = (m["text"] + " " + l["text"]).strip(); m["confidence"] = min(m["confidence"], l["confidence"]); m["min_conf"] = min(m.get("min_conf", 1), l.get("min_conf", 1)); m["polygon"] = None; merged = True; break + if not merged: out.append(dict(l)) + return out + +def xy_cut_order(blocks, page_w, full_frac=0.60): + """Reading order: furniture, then columns top-to-bottom, with full-width blocks as band separators. + + The previous version disabled column ordering whenever ANY block was neither wholly left nor wholly + right. On a two-column book page the centred running header ("संक्षिप्त देवीभागवत", 94 px of a 446 px + page) satisfies that, so column ordering switched off and the fallback `sort by (y, x)` interleaved the + two columns line by line -- the page came back unreadable even though every line was recognised. + + A block is only a band separator if it is genuinely wide (>= `full_frac` of the page). A narrow block + that straddles the midline belongs to the column its CENTRE falls in. Headers and footers are page + furniture: they are lifted out and placed first and last rather than competing with the body. + """ + if not blocks: return [] + y = lambda b: b["bbox"][1] + cx = lambda b: (b["bbox"][0] + b["bbox"][2]) / 2 + wd = lambda b: b["bbox"][2] - b["bbox"][0] + hdr = [b for b in blocks if b["type"] == "header"] + ftr = [b for b in blocks if b["type"] in ("footer", "page_number")] + body = [b for b in blocks if b not in hdr and b not in ftr] + full = [b for b in body if wd(b) >= full_frac * page_w] + cols = [b for b in body if b not in full] + mid = page_w / 2 + left = [b for b in cols if cx(b) < mid] + right = [b for b in cols if cx(b) >= mid] + if len(left) >= 2 and len(right) >= 2: + out, prev = [], -1e18 + for fb in sorted(full, key=y) + [None]: # each full-width block closes a band + lim = y(fb) if fb is not None else 1e18 + out += sorted([b for b in left if prev <= y(b) < lim], key=y) + out += sorted([b for b in right if prev <= y(b) < lim], key=y) + if fb is not None: out.append(fb); prev = y(fb) + body_order = out + else: + body_order = sorted(body, key=lambda b: (round(y(b) / 20), b["bbox"][0])) + return sorted(hdr, key=y) + body_order + sorted(ftr, key=y) + +def rows_of(lines, tol=0.5): + """Cluster line boxes into visual rows (y-centre within tol x the smaller height), left-to-right inside a row.""" + ls = sorted(lines, key=lambda l: (l["bbox"][1] + l["bbox"][3]) / 2); rows = [] + for l in ls: + yc = (l["bbox"][1] + l["bbox"][3]) / 2; h = l["bbox"][3] - l["bbox"][1] + if rows: + r = rows[-1]; ryc = np.mean([(m["bbox"][1] + m["bbox"][3]) / 2 for m in r]); rh = np.median([m["bbox"][3] - m["bbox"][1] for m in r]) + if abs(yc - ryc) < tol * min(h, rh): r.append(l); continue + rows.append([l]) + return [sorted(r, key=lambda l: l["bbox"][0]) for r in rows] + +def split_row_by_gap(r, max_gap_h=2.5): + """Split a visual row into runs where the horizontal gap between neighbours exceeds max_gap_h x line height: two + columns (Kannada | English on ID cards, label | value pairs far apart) must stay separate lines; fragments of one + justified/spaced line (gaps of a few glyph widths) still join. Geometry only.""" + r = sorted(r, key=lambda l: l["bbox"][0]); runs = [[r[0]]] + for l in r[1:]: + prev = runs[-1][-1]; h = max(8.0, min(prev["bbox"][3] - prev["bbox"][1], l["bbox"][3] - l["bbox"][1])) + if l["bbox"][0] - prev["bbox"][2] > max_gap_h * h: runs.append([l]) + else: runs[-1].append(l) + return runs + +def join_row(r): + """One text line per visual row: fragments of a justified/spaced line are joined with single spaces.""" + if len(r) == 1: return dict(r[0]) + return {"bbox": [min(l["bbox"][0] for l in r), min(l["bbox"][1] for l in r), max(l["bbox"][2] for l in r), max(l["bbox"][3] for l in r)], + "polygon": None, "text": " ".join(l["text"] for l in r).strip(), "confidence": min(l["confidence"] for l in r), "min_conf": min(l.get("min_conf", 1) for l in r)} + +def _runs(mask1d, min_gap=3): + idx = np.flatnonzero(mask1d) + if idx.size == 0: return [] + out = []; start = idx[0]; prev = idx[0] + for i in idx[1:]: + if i - prev > min_gap: out.append((start + prev) / 2); start = i + prev = i + out.append((start + prev) / 2); return out + +def _rule_positions(mask, axis, min_len): + """Centre coordinate of every connected component of a rule mask longer than min_len along `axis` (robust to residual tilt).""" + import cv2 + n, _, st, cen = cv2.connectedComponentsWithStats(mask, connectivity=8) + pos = sorted(cen[i][1 - axis] for i in range(1, n) if st[i][cv2.CC_STAT_WIDTH if axis == 0 else cv2.CC_STAT_HEIGHT] >= min_len) + out = [] + for p in pos: + if out and p - out[-1] < 6: out[-1] = (out[-1] + p) / 2 + else: out.append(p) + return out + +def _merge_continuations(rows): + """A row whose cells all sit under occupied columns of the previous row and none in its first column is a wrapped cell.""" + out = [] + for r in rows: + ne = [c for c in r["cells"] if c["text"]] + if out and ne: + prev = out[-1]; pne = [c for c in prev["cells"] if c["text"]] + first = min(c["col"] for c in prev["cells"]) if prev["cells"] else 0 + if all(c["col"] > first for c in ne) and all(any(p["col"] == c["col"] for p in pne) for c in ne): + for c in ne: + p = next(p for p in pne if p["col"] == c["col"]); p["text"] = (p["text"] + " " + c["text"]).strip() + p["bbox"] = [min(p["bbox"][0], c["bbox"][0]), min(p["bbox"][1], c["bbox"][1]), max(p["bbox"][2], c["bbox"][2]), max(p["bbox"][3], c["bbox"][3])] + continue + out.append(r) + return out + +def ruled_grid(gray, bbox, lines): + """Table structure from printed ruling lines (morphological opening + connected components). Horizontal rules give row + bands; inside a band, vertical rules give cells - or, when a band has no vertical rules (aligned-column forms), each + visual row of the band becomes a table row whose cells are the separate line boxes. Returns + rows=[{cells:[{col,text,bbox,confidence}], ruled}] or None when the region has no horizontal rules. Pure geometry.""" + import cv2 + x0, y0, x1, y1 = [int(round(v)) for v in bbox]; reg = gray[max(0, y0):y1, max(0, x0):x1] + if reg.size == 0 or reg.shape[0] < 20 or reg.shape[1] < 20: return None + H, W = reg.shape; bw = cv2.adaptiveThreshold(reg, 255, cv2.ADAPTIVE_THRESH_MEAN_C, cv2.THRESH_BINARY_INV, 31, 9) + hor = cv2.morphologyEx(bw, cv2.MORPH_OPEN, cv2.getStructuringElement(cv2.MORPH_RECT, (max(20, W // 30), 1))) + hor = cv2.dilate(hor, cv2.getStructuringElement(cv2.MORPH_RECT, (max(20, W // 8), 1))) # bridge faint / dashed rules (text-derived runs are rejected below) + loc = [(l["bbox"][0] - x0, l["bbox"][1] - y0, l["bbox"][2] - x0, l["bbox"][3] - y0) for l in lines] + def through_text(y): return any(b + 0.1 * (d - b) < y < d - 0.1 * (d - b) for a, b, c, d in loc) # a rule never crosses the middle of a text line + def through_text_x(x, ia, ib): return any(a + 0.1 * (c - a) < x < c - 0.1 * (c - a) and b < ib and d > ia for a, b, c, d in loc) + hr = [y for y in _rule_positions(hor, 0, 0.25 * W) if 0.01 * H + 4 < y < H - 0.01 * H - 4 and not through_text(y)] + if not hr: return None + ys = [0.0] + hr + [float(H)]; bands = [] + for a, b in zip(ys[:-1], ys[1:]): + ia, ib = int(a), int(b) + if ib - ia < 8: continue + ver = cv2.morphologyEx(bw[ia:ib], cv2.MORPH_OPEN, cv2.getStructuringElement(cv2.MORPH_RECT, (1, max(8, int((ib - ia) * 0.4))))) + ver = cv2.dilate(ver, cv2.getStructuringElement(cv2.MORPH_RECT, (1, max(6, int((ib - ia) * 0.25))))) + vr = [x for x in _rule_positions(ver, 1, 0.85 * (ib - ia)) if 0.03 * W < x < 0.97 * W and not through_text_x(x, ia, ib)] # cell borders span the band, never cross text; table border excluded + bands.append((ia, ib, vr)) + allx = sorted(x for _, _, vr in bands for x in vr); G = [] + for x in allx: + if not G or x - G[-1][-1] > 12: G.append([x]) + else: G[-1].append(x) + G = [float(np.mean(g)) for g in G] + def gcol(x): return sum(1 for g in G if g < x + 6) + rows = [] + for ia, ib, vr in bands: + inband = [l for l in lines if ia <= (l["bbox"][1] + l["bbox"][3]) / 2 - y0 < ib] + if not inband: continue + if vr: # ruled cells: everything between two vertical rules is one cell (a wrapped cell keeps its lines together); + # the global rule set is used so a band whose faint rule was missed still gets the table's columns + xs = [0.0] + G + [float(W)] + percol = [[l for l in inband if ca <= (l["bbox"][0] + l["bbox"][2]) / 2 - x0 < cb] for ca, cb in zip(xs[:-1], xs[1:])] + multi = sum(1 for inc in percol if len(rows_of(inc)) > 1) + # several columns stacked -> the band holds several table rows whose separating rule was not printed/found + groups = rows_of(inband) if multi >= 2 else [inband] + for g in groups: + cells = [] + for ca, cb in zip(xs[:-1], xs[1:]): + inc = [l for l in g if ca <= (l["bbox"][0] + l["bbox"][2]) / 2 - x0 < cb] + if not inc: cells.append({"col": gcol(ca), "text": "", "bbox": None, "confidence": None}); continue + cells.append({"col": gcol(ca), "text": " ".join(join_row(r)["text"] for r in rows_of(inc)), "bbox": [min(l["bbox"][0] for l in inc), min(l["bbox"][1] for l in inc), max(l["bbox"][2] for l in inc), max(l["bbox"][3] for l in inc)], "confidence": min(l["confidence"] for l in inc)}) + rows.append({"cells": cells, "ruled": True}) + if len(groups) > 1: rows[-len(groups):] = _merge_continuations(rows[-len(groups):]) + else: # no vertical rules in this band: each visual row is a table row, each line box a cell + for r in rows_of(inband): + rows.append({"cells": [{"col": (gcol(l["bbox"][0] - x0) if G else ci), "text": l["text"], "bbox": l["bbox"], "confidence": l["confidence"]} for ci, l in enumerate(r)], "ruled": False}) + return rows if rows else None + +def group_table(lines): + """Unruled tables: rows by y-clustering, columns by clustering cell x-starts; a row that only continues cells of the + previous row (no cell in the first column, every cell under an occupied column) is a wrapped cell -> merged upward.""" + if not lines: return [] + rows = rows_of(lines) + spans = sorted(((l["bbox"][0], l["bbox"][2]) for r in rows for l in r)); cols = [] + for a, b in spans: # columns = groups of horizontally overlapping cells (works for left/centre/right alignment) + if cols and a < cols[-1][1] - 4: cols[-1][1] = max(cols[-1][1], b) + else: cols.append([a, b]) + def col_of(l): + xc = (l["bbox"][0] + l["bbox"][2]) / 2 + return next((i for i, (a, b) in enumerate(cols) if a - 4 <= xc <= b + 4), int(np.argmin([abs(xc - (a + b) / 2) for a, b in cols])) if cols else 0) + return _merge_continuations([{"cells": [{"bbox": l["bbox"], "text": l["text"], "confidence": l["confidence"], "col": col_of(l)} for l in r], "ruled": False} for r in rows]) + + +def merge_split_blocks(blocks, x_frac=0.6, y_frac=0.25, page_w=None, full_frac=0.60): + """Merge SAME-CLASS layout blocks that cover the same region. + + The layout model sometimes cuts one continuous paragraph into several boxes that INTERLEAVE vertically. + Each box is individually plausible, so nothing downstream notices; but the assembler emits block by block, + so the lines come out grouped by box instead of in reading order. Observed on a plain Hindi prose page: + three boxes over one paragraph, all 14 lines recognised correctly and every one of them in the wrong place. + + Merging is deliberately conservative, because the failure mode of being too eager is far worse -- it would + glue the columns of a two-column page together: + + * same `type` only -- a seal inside a text region must stay its own block + * the boxes must overlap on the X axis by `x_frac` of the NARROWER box. Side-by-side columns barely + overlap horizontally, so they never merge. + * they must also overlap on the Y axis by `y_frac` of the SHORTER box. Stacked paragraphs in one column + do not overlap vertically, so they never merge either. + + Only both conditions together describe "two boxes drawn over the same block of text". Lines are re-sorted + by geometry inside the merged box by the existing `rows_of`, so reading order is restored. + + The "columns never merge" argument above holds for a PAIR of columns, but not for a chain. On a + two-column Hindi book page one spurious low-confidence box straddled both columns at the foot; it + overlapped the left column enough to absorb it, the union then reached the right column, and both columns + ended up in a single 31-line block. So merging may never turn column-width boxes into a full-width one: + if neither input already spans `full_frac` of the page, the result may not either. `page_w=None` keeps the + old behaviour for callers that do not pass it. + """ + def ov(a0, a1, b0, b1): + return max(0.0, min(a1, b1) - max(a0, b0)) + out, used = [], [False] * len(blocks) + for i, b in enumerate(blocks): + if used[i]: continue + cur = dict(b); cur["bbox"] = list(b["bbox"]); merged = 0 + changed = True + while changed: # keep absorbing: three-way splits need more than one pass + changed = False + for j, c in enumerate(blocks): + if used[j] or j == i or c["type"] != cur["type"]: continue + ax0, ay0, ax1, ay1 = cur["bbox"]; bx0, by0, bx1, by1 = c["bbox"] + wx = ov(ax0, ax1, bx0, bx1) / max(1.0, min(ax1 - ax0, bx1 - bx0)) + wy = ov(ay0, ay1, by0, by1) / max(1.0, min(ay1 - ay0, by1 - by0)) + if wx < x_frac or wy < y_frac: continue + nb = [min(ax0, bx0), min(ay0, by0), max(ax1, bx1), max(ay1, by1)] + if page_w: + # A full-width block and a column-width block are never two halves of one paragraph. + # The spurious box that swallowed both columns was ITSELF already full-width (409 px of + # 446), so a rule about the merged width could not catch it -- the mismatch must be + # tested on the INPUTS. + lim = full_frac * page_w + if ((ax1 - ax0) >= lim) != ((bx1 - bx0) >= lim): continue + if True: + cur["bbox"] = nb + cur["confidence"] = max(cur.get("confidence", 0), c.get("confidence", 0)) + cur["order"] = min(x for x in (cur.get("order"), c.get("order")) if x is not None) \ + if (cur.get("order") is not None or c.get("order") is not None) else None + used[j] = True; merged += 1; changed = True + used[i] = True + if merged: cur["merged_from"] = merged + 1 + out.append(cur) + return out + +VISUAL_TYPES = ("figure", "seal", "id_panel", "barcode") + +def _looks_grid(b, min_rows=2, min_cols=2): + """Do this block's lines tile into >= min_rows x min_cols cells? Clusters line x-centres into column + groups and y-centres into row groups; a paragraph gives 1 column, a table gives 2+.""" + ls = b.get("lines") or [] + if len(ls) < min_rows * min_cols: return False + import numpy as _np + cx = sorted(((l["bbox"][0] + l["bbox"][2]) / 2) for l in ls) + w = (b["bbox"][2] - b["bbox"][0]) or 1.0 + cols = 1 + for a, c in zip(cx, cx[1:]): + if c - a > 0.18 * w: cols += 1 + yc = sorted(((l["bbox"][1] + l["bbox"][3]) / 2) for l in ls) + h = _np.median([l["bbox"][3] - l["bbox"][1] for l in ls]) or 1.0 + rows = 1 + for a, c in zip(yc, yc[1:]): + if c - a > 0.6 * h: rows += 1 + return cols >= min_cols and rows >= min_rows + +def retype_mislabelled_visuals(blocks, min_lines): + """A `figure`/`seal`/`id_panel` holding many READABLE lines is a mislabelled panel, not a picture. + + The config already encodes exactly this judgement for suppression (`suppress_max_lines`: "a figure + holding more than N readable lines is a mislabelled panel"). It was never applied to the block's TYPE, + so a ruled table detected as `figure` kept being ordered and exported as a picture: observed on an + English-Kannada word table where the layout model returned ONE `figure` (conf 0.70, 58 % of the page) + containing 28 readable lines, and 28 of 34 lines carried the `in_figure` flag -- the model's own signal + that the type was wrong. Keyed on that signal, not on any one page. + """ + if not min_lines: return blocks + for b in blocks: + if b["type"] in VISUAL_TYPES and len([l for l in (b.get("lines") or []) if (l.get("text") or "").strip()]) >= min_lines: + b["retyped_from"] = b["type"] + b["type"] = "table" if _looks_grid(b) else "text" + return blocks + +def merge_row_structures(blocks, lines, row_cross_frac=0.20, y_frac=0.50, min_rows=3): + """Side-by-side blocks that are really the two halves of ONE table are read ROW-WISE, not column-wise. + + A label/value table ("1. Property No." | ": A-50, Third Floor") is often returned as two blocks, one per + column. Read block by block that emits every label, then every value -- the rows are destroyed even + though every line is correct. + + The discriminator is the GUTTER. Two genuine columns have a clean vertical channel that no line crosses + anywhere down the page. In a table the channel is crossed constantly: the detector merges label and + value into one box whenever the gap is small ("3. Segment/Block Name & Code : Southcity-I"), and a + numbering strip spans the whole width. So: count the lines sharing the two blocks' y-band that cross the + gutter; if more than `row_cross_frac` of them do, the gutter is not real and the blocks are one region. + + ONE crossing line is not enough -- a two-column prose page had exactly one (a detector error gluing a + line across the column rule), 1.4 % of its lines, while the deed table crosses on ~30 %. + + Merging is all that is needed: `rows_of()` already clusters lines into visual rows and orders them + left-to-right inside a row, so the merged block comes out label, value, label, value. No line is + dropped or moved to the end. + """ + if not blocks or row_cross_frac <= 0: return blocks + out = list(blocks); changed = True + while changed: + changed = False + for i in range(len(out)): + for j in range(i + 1, len(out)): + a, b = out[i], out[j] + if a["type"] != b["type"] or a["type"] == "table": continue + if len(a.get("lines") or []) < min_rows or len(b.get("lines") or []) < min_rows: continue + ax0, ay0, ax1, ay1 = a["bbox"]; bx0, by0, bx1, by1 = b["bbox"] + if ax0 > bx0: (ax0, ay0, ax1, ay1), (bx0, by0, bx1, by1) = (bx0, by0, bx1, by1), (ax0, ay0, ax1, ay1) + gap0, gap1 = ax1, bx0 + if gap1 <= gap0: continue # they overlap in x: not a gutter + oy = min(ay1, by1) - max(ay0, by0) + if oy < y_frac * min(ay1 - ay0, by1 - by0): continue # not the same horizontal band + band0, band1 = max(ay0, by0), min(ay1, by1) + inband = [l for l in lines if l["bbox"][3] > band0 and l["bbox"][1] < band1] + if len(inband) < 4: continue + cross = sum(1 for l in inband if l["bbox"][0] < gap0 and l["bbox"][2] > gap1) + # Second trigger: a whole BLOCK that spans the gutter across most of the shared band. + # On a deed the numbering strip ("2." "3." "4.") is returned as its own block covering the + # full table width, which proves the two halves belong to one structure even when the gutter + # itself is perfectly clean -- blind00_p0000 has 0 of 33 lines crossing yet is one table. + # A stray glued line cannot trigger this: on the two-column prose page the spanning orphan + # covers 22 px of a 560 px band. + # A spanning block only counts as a numbering strip if it has SEVERAL lines. A single glued + # line also spans the gutter -- the two-column prose page has exactly one, and with a short + # shared band it satisfied the 50 % rule and interleaved the columns. Requiring >= min_rows + # lines in the strip AND in both halves separates "one table" from "one bad line". + span = any(c is not a and c is not b and c["bbox"][0] < gap0 and c["bbox"][2] > gap1 + and len(c.get("lines") or []) >= min_rows + and (min(c["bbox"][3], band1) - max(c["bbox"][1], band0)) >= 0.5 * (band1 - band0) + for c in out) + if not span and cross / len(inband) < row_cross_frac: continue # real gutter -> genuine columns + # The merge is only safe if the assembler's OWN row splitter will keep the two halves apart. + # `split_row_by_gap` separates a visual row when the horizontal gap exceeds 2.5x the line + # height; below that, `join_row` CONCATENATES the two sides. On a two-column prose page whose + # columns nearly touch that produced lines like + # "रुरु नामक जानवर उस नारकी जीवके जो पुरुष राजा अथवा राजकर्मचारी होकर" + # -- two unrelated sentences in one line, with every word still present, so a "no word lost" + # check cannot see it. Require the gutter to clear the same threshold. + lh = np.median([l["bbox"][3] - l["bbox"][1] for l in inband]) if inband else 0.0 + if (gap1 - gap0) < 2.5 * lh: continue + a["bbox"] = [min(a["bbox"][0], b["bbox"][0]), min(a["bbox"][1], b["bbox"][1]), + max(a["bbox"][2], b["bbox"][2]), max(a["bbox"][3], b["bbox"][3])] + a["lines"] = (a.get("lines") or []) + (b.get("lines") or []) + a["row_merged"] = a.get("row_merged", 1) + 1 + out.pop(j); changed = True; break + if changed: break + return out + +def assemble(page_w, page_h, layout_blocks, det_boxes, texts, model_ids, timing=None, page_index=0, dpi=None, + with_polygons=True, img=None, suppress_text_in=(), suppress_min_conf=0.0, low_res_px=0.0, retype_visual_min_lines=4, row_cross_frac=0.20, + suppress_max_area=1.0, suppress_max_lines=0, merge_split_blocks_x=0.0, merge_split_blocks_y=0.25): + """suppress_text_in: block types whose recognised text is withheld from `doc["text"]` and replaced by a + `*[type]*` marker, the lines being kept in `blocks` with their flags. + + Default is empty, so behaviour is unchanged unless a config asks for it. Why it exists: a print recognizer + fed a signature or a seal does not abstain — it emits confident nonsense. An end-to-end review of the Hindi + pair found fabricated phone numbers on signature blocks, with confidence high enough that the `check` flag + (which fires on 41.5 % of lines) could not be used to triage them. The layout model already identifies those + blocks; this lets a caller act on that instead of reading invented text out of `doc["text"]`. + Set via `assemble.suppress_text_in` in the config; the Hindi overlay uses + ["handwriting", "seal", "figure"]. + + THREE GUARDS, because unconditional suppression deletes far more signal than noise. Measured over 87 + pages of registration records: 160 lines were withheld, of which **119 looked like real text** -- + 81 in `figure`, 20 in `seal`, only 18 in `handwriting`. Examples lost: certificate and GRN numbers, + `दिनांक:23-04-2020`, `उप/सयुंक्त पंजीयन अधिकारी( वजीराबाद )`. Root cause: an e-stamp panel holds two + emblems, a barcode and a QR code, so the layout model boxes the WHOLE panel as `figure` (conf 0.31-0.38, + up to 35 % of the page) and every line inside it is dropped. + + suppress_min_conf do not suppress below this layout confidence -- a marginal call should keep text + suppress_max_area do not suppress a block larger than this fraction of the page + suppress_max_lines do not suppress a block holding more than this many recognised lines; a `figure` + full of readable text is a mislabelled panel, not a picture + + All default to OFF so existing callers are unchanged.""" + lines = [] + for (poly, dscore), (text, conf) in zip(det_boxes, texts): + if not text.strip(): continue + lines.append({"bbox": _bbox(poly), "polygon": [[float(x), float(y)] for x, y in poly] if with_polygons else None, "text": text, "confidence": round(float(conf), 4), + "min_conf": round(float(getattr(conf, "min_char", None) if getattr(conf, "min_char", None) is not None else conf), 4)}) + blocks = [] + for i, b in enumerate(layout_blocks): + blocks.append({"id": f"b{i}", "type": TYPE_MAP.get(b["type"], "other"), "raw_type": b["type"], "confidence": round(float(b["score"]), 4), "bbox": [round(v, 1) for v in b["bbox"]], "order": b.get("order"), "lines": []}) + if merge_split_blocks_x > 0: + blocks = merge_split_blocks(blocks, merge_split_blocks_x, merge_split_blocks_y, page_w=page_w) + for k, b in enumerate(blocks): b["id"] = f"b{k}" + # assign lines to best-overlapping block; unassigned lines become their own text blocks + orphans = [] + area = lambda bb: max(1.0, (bb[2] - bb[0]) * (bb[3] - bb[1])) + for l in lines: + best, bi = 0.0, -1 + for i, b in enumerate(blocks): + f = _overlap_frac(l["bbox"], b["bbox"]) + # nested blocks (v1.2 layout: a handwriting / seal / id_panel box inside a text region): on a near-tie the + # smaller, more specific block wins, so signature fragments are not glued to the surrounding text rows + if f > best + 0.05 or (f >= best - 0.05 and bi >= 0 and f >= 0.5 and area(b["bbox"]) < area(blocks[bi]["bbox"])): best, bi = f, i + if best >= 0.5: blocks[bi]["lines"].append(l) + else: orphans.append(l) + for l in orphans: + blocks.append({"id": f"b{len(blocks)}", "type": "text", "raw_type": "orphan_line", "confidence": l["confidence"], "bbox": [round(v, 1) for v in l["bbox"]], "order": None, "lines": [l]}) + blocks = [b for b in blocks if b["lines"] or b["type"] in ("figure", "seal", "table", "id_panel", "handwriting", "barcode")] + blocks = retype_mislabelled_visuals(blocks, retype_visual_min_lines) + blocks = merge_row_structures(blocks, lines, row_cross_frac) + # order: native rank where available, else xy-cut; orphans placed by position among ordered blocks + ranked = [b for b in blocks if b["order"] is not None]; unranked = [b for b in blocks if b["order"] is None] + ordered = sorted(ranked, key=lambda b: b["order"]) + for b in unranked: + pos = next((k for k, o in enumerate(ordered) if o["bbox"][1] > b["bbox"][1] and o["bbox"][0] <= b["bbox"][2]), len(ordered)); ordered.insert(pos, b) + if not ranked: ordered = xy_cut_order(blocks, page_w) + full = [] + gray = None + if img is not None: + import cv2; gray = img if img.ndim == 2 else cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) + heights = [l["bbox"][3] - l["bbox"][1] for b in ordered for l in b["lines"]]; med_h = float(np.median(heights)) if heights else 0.0 + for k, b in enumerate(ordered): + b["order"] = k + if b["type"] == "table": + # table cells stay separate boxes; structure from printed rules when present, else from alignment + b["lines"] = [l for r in rows_of(b["lines"]) for l in r] + rows = ruled_grid(gray, b["bbox"], b["lines"]) if gray is not None else None + b["rows"] = rows or group_table(b["lines"]); b["ruled"] = bool(rows) + else: + b["lines"] = [join_row(run) for r in rows_of(b["lines"]) for run in split_row_by_gap(r)] # one text line per visual row run (fragments of a spaced line re-joined; wide-gap columns kept apart) + for li, l in enumerate(b["lines"]): + l["id"] = f"{b['id']}.l{li}"; l["script"] = script_of(l["text"]); l["bbox"] = [round(v, 1) for v in l["bbox"]] + # review flags (never deletion): calibration on 5,478 labelled lines -> conf >= 0.99 is 97.7 % line-exact, + # 0.95-0.98 only 83.5 %; lines inside seal/figure/header blocks are mostly stamp/handwriting fragments + fl = [] + if low_res_px and (l["bbox"][3] - l["bbox"][1]) < low_res_px: fl.append("low_res") + if l["confidence"] < 0.90: fl.append("low_conf") + elif l["confidence"] < 0.98: fl.append("check") + if b["type"] in ("seal", "figure", "handwriting", "id_panel", "barcode"): fl.append("in_" + b["type"]) + # review flags (script/character-repertoire checks); flags only, the text is never altered + if l.get("min_conf", 1.0) < WEAK_CHAR: fl.append("weak_char") # one very uncertain character (garbage lines usually have one) + if _MIX_RE.search(l["text"]): fl.append("script_mix") # Latin and Kannada letters inside one token (other-script garbage) + if med_h and (l["bbox"][3] - l["bbox"][1]) > 3.0 * med_h: fl.append("oversize") # stray ornament / drop-cap sized line + if fl: l["flags"] = fl + if b["type"] in suppress_text_in: + _area = ((b["bbox"][2] - b["bbox"][0]) * (b["bbox"][3] - b["bbox"][1])) / max(1.0, page_w * page_h) + _keep = ((b.get("confidence", 1.0) < suppress_min_conf) # layout was unsure + or (_area > suppress_max_area) # too big to be a figure + or (suppress_max_lines and len(b["lines"]) > suppress_max_lines)) # full of readable text + if _keep: + b["text_suppressed"] = False + b["suppression_overridden"] = round(_area, 4) # visible in the output, not silent + else: + # the lines stay in `blocks` with their flags; only the page text withholds them + b["text_suppressed"] = True + if b["lines"] or b["type"] in ("figure", "seal"): full.append(f"*[{b['type']}]*") + continue + if b["type"] == "table" and b["rows"]: block_text = "\n".join(" | ".join(c["text"] for c in r["cells"]) for r in b["rows"]) + else: block_text = "\n".join(l["text"] for l in b["lines"]) + if block_text.strip(): full.append(block_text) + return {"schema": "kanen-ocr/1.0", "model": model_ids, "page": {"index": page_index, "width": page_w, "height": page_h, "dpi": dpi, "rotation": 0}, + "blocks": ordered, "text": "\n\n".join(full), "timing_ms": timing or {}} + +def to_markdown(doc): + out = [] + for b in doc["blocks"]: + t = b["type"]; txt = " ".join(l["text"] for l in b["lines"]).strip() + if t == "title": out.append(f"# {txt}") + elif t == "section_title": out.append(f"## {txt}") + elif t == "table" and b.get("rows"): + ncol = max((c["col"] for r in b["rows"] for c in r["cells"]), default=0) + 1 + rows = [] + for r in b["rows"]: + cells = [""] * ncol + for c in r["cells"]: cells[c["col"]] = (cells[c["col"]] + " " + c["text"]).strip() + rows.append("| " + " | ".join(x.replace("|", "\\|") for x in cells) + " |") + if rows: out.append(rows[0] + "\n|" + "---|" * ncol + "\n" + "\n".join(rows[1:])) + elif b.get("text_suppressed") or t in ("figure", "seal"): out.append(f"*[{t}]*") + elif t in ("header", "footer", "page_number"): out.append(f"{txt}") + elif txt: out.append(txt) + return "\n\n".join(out) + "\n" diff --git a/hi-en/kanen_infer/batch_pipeline.py b/hi-en/kanen_infer/batch_pipeline.py new file mode 100644 index 0000000000000000000000000000000000000000..39143e9aa6305290f7db33057c3d329b6ac48f8f --- /dev/null +++ b/hi-en/kanen_infer/batch_pipeline.py @@ -0,0 +1,322 @@ +"""Arjuna-OCR [Kn&En] batched throughput pipeline. +Organised for throughput instead of single-page latency (the API's `page()` runs the same code with one page): + * decode + canvas choice in a prefetch thread pool (next page batch is decoded while the GPU works on this one) + * detector runs ONCE per page at a canvas chosen from the page size (no 1280 -> re-detect search), batched across pages + * DB post-processing + perspective crops in worker threads + * recognizer sees the crops of the whole page batch in one width-bucketed call; orientation-180 fallback and the + confidence-gated second pass are pooled the same way + * layout model batched across pages +No output rewriting anywhere: every character comes from the recognizer. All tunables come from the package config. +""" +import time, math, os +from concurrent.futures import ThreadPoolExecutor +import numpy as np, cv2 +from .config import load_config, _FALLBACK +from .providers import Providers +from .pipeline_base import Detector, Layout, crop, MEAN, STD +from .assemble import assemble +from .geometry import rotate_page, deskew_angle, rotate_small + +def _asm(cfg): + """assemble section of the config, tolerant of a plain dict or a Config object.""" + return (cfg.get("assemble", {}) if hasattr(cfg, "get") else {}) or {} + + +def n_readable(res, conf=0.90, min_char=0.50, min_len=4): + """readable text mass = characters in lines that read cleanly (line conf and weakest character high, >= 4 chars so + stray '|' / '1' fragments do not vote). Majority vote for mixed-orientation pages (upright form + rotated header).""" + return sum(len(t) for t, c in res if len(t) >= min_len and float(c) >= conf and (getattr(c, "min_char", None) if getattr(c, "min_char", None) is not None else float(c)) >= min_char) + +def orient_score(res): + """orientation evidence: mean line confidence + mean weakest-character confidence (upside-down text = confident garbage with one very weak frame)""" + if not res: return 0.0 + return float(np.mean([float(c) for _, c in res]) + np.mean([getattr(c, "min_char", None) if getattr(c, "min_char", None) is not None else float(c) for _, c in res])) + +CANVASES = tuple(_FALLBACK["detector"]["canvases"]) +WIDTH_CLASSES = tuple(_FALLBACK["detector"]["width_classes"]) # canvas width as a fraction of the long side -> at most 5 ORT plans per canvas (no square padding) +INV_STD = (1.0 / STD).astype(np.float32); MEAN255 = tuple(float(m * 255) for m in MEAN) # same channel order as Detector (PaddleOCR applies the mean to the BGR image as read) +PAD_VAL = ((1.0 - MEAN) / STD).astype(np.float32) # normalised white, like Detector's white canvas + +def choose_canvas(h, w, min_scale=0.9, canvases=CANVASES): + """Smallest fixed canvas that keeps the page at >= min_scale of its native size (150-DPI private-document scans -> 1920, A4@300 -> 2560).""" + m = max(h, w) + for c in canvases: + if c >= min_scale * m: return c + return canvases[-1] + +def canvas_shape(h, w, min_scale=0.9, canvases=CANVASES, width_classes=WIDTH_CLASSES): + """(limit, canvas_h, canvas_w): long side = limit, short side snapped up to a width class (multiple of 32).""" + limit = choose_canvas(h, w, min_scale, canvases); r = min(1.0, limit / max(h, w)); nh, nw = h * r, w * r + if h >= w: cw = next(int(round(limit * f / 32)) * 32 for f in width_classes if limit * f >= nw); return limit, limit, cw + ch = next(int(round(limit * f / 32)) * 32 for f in width_classes if limit * f >= nh); return limit, ch, limit + +def rotate_boxes(boxes, deg, w, h): + """Map detector polygons into the deskewed page frame (same rotation rotate_small applies to the image).""" + if not boxes: return boxes + M = cv2.getRotationMatrix2D((w / 2, h / 2), deg, 1.0) + return [(cv2.transform(b[None].astype(np.float32), M)[0], sc) for b, sc in boxes] + +def _layout_id(cfg): + """Name of the layout model ACTUALLY loaded. This used to be the literal "arjuna-layout-v12", + so every output document was stamped v12 no matter which model the config pointed at.""" + import os + stem = os.path.splitext(os.path.basename(str(cfg.path("models", "layout"))))[0] + return stem.replace("kanen_layout_", "arjuna-layout-").replace("arjuna_layout_", "arjuna-layout-") + + +class BatchOCR: + def __init__(self, rec=None, cfg=None, providers=None, det=None, lay=None, use_layout=None, layout_mode=None, workers=None, tta_conf=None, tta_cap=None, rec_bs=None, det_bs=None, det_size="small"): + """rec/det/lay: model objects to share (built from cfg when None). layout_mode: 'always' | 'auto' (only pages that look + like tables/forms) | 'never'. tta_cap: second-pass lines per page. Every default comes from the config.""" + from .rec_onnx import KanEnRecognizer + self.cfg = cfg or getattr(rec, "cfg", None) or load_config(); self.prov = providers or getattr(rec, "prov", None) or Providers(self.cfg) + dc, lc, rc, sp, rt = self.cfg["detector"], self.cfg["layout"], self.cfg["recognizer"], self.cfg["second_pass"], self.cfg["runtime"] + self.layout_mode = layout_mode or lc.get("mode", "always") + if use_layout is False: self.layout_mode = "never" + self.rec = rec or KanEnRecognizer(cfg=self.cfg, providers=self.prov); self.det = det or Detector(self.cfg, self.prov) + self.lay = lay if lay is not None else (Layout(self.cfg, self.prov) if self.layout_mode != "never" else None) + self.tta_conf = float(sp["conf"] if tta_conf is None else tta_conf); self.tta_cap = int(sp["cap"] if tta_cap is None else tta_cap) + self.rec_bs = int(rc["batch"] if rec_bs is None else rec_bs); self.det_bs = int(dc["batch"] if det_bs is None else det_bs) + self.canvases = tuple(int(c) for c in dc["canvases"]); self.width_classes = tuple(float(f) for f in dc["width_classes"]); self.min_scale = float(dc["min_scale"]) + # resolution-aware rescale: 0 disables. A page whose MEDIAN detected line is shorter than + # min_line_px is re-detected after upscaling so its lines land near target_line_px. + self.min_line_px = float(dc.get("min_line_px", 0)) + self.target_line_px = float(dc.get("target_line_px", 45)) + self.max_page_scale = float(dc.get("max_page_scale", 3.0)) + self.workers = int(workers) if workers else self.cfg.workers(); self.pool = ThreadPoolExecutor(self.workers); self._planned = set() + self._det_buf = {} # (batch, canvas h, canvas w) -> reusable detector input buffer + self.model_ids = {"layout": _layout_id(self.cfg) if self.lay else None, "det": f"ppocrv6-{det_size}-det-kn-v121", "rec": getattr(self.rec, "name", "kanen-rec"), + "rec_md5": getattr(self.rec, "onnx_md5", None), "rec_onnx_md5": getattr(self.rec, "onnx_md5", None), "charset_md5": getattr(self.rec, "charset_md5", None), "pipeline": str(self.cfg.get("version", "1.0.0"))} + + def _sess(self, limit): + """detector session for a canvas limit (one per limit, as in Detector).""" + return self.det._session(limit) + + def _plan(self, limit, ch, cw, n): + key = (limit, ch, cw, n) + if key not in self._planned: # ORT CUDA EP re-plans on a new input shape (~1 s) -> do it once, outside the timed path + s = self._sess(limit) + z = np.zeros((n, ch, cw, 3), np.uint8) if getattr(self.det, "u8", False) else np.zeros((n, 3, ch, cw), np.float32) + s.run(None, {s.get_inputs()[0].name: z}); self._planned.add(key) + + def _prep_det(self, img, ch, cw, limit, dst=None): + """Normalise one page into the detector's canvas. `dst` is the slice of the batch buffer to write into: filling it + in place avoids building a per-page canvas and then stacking them, which copied ~28 MB per page at canvas 1920.""" + h, w = img.shape[:2]; r = min(1.0, limit / max(h, w)); nh, nw = max(32, int(round(h * r / 32)) * 32), max(32, int(round(w * r / 32)) * 32) + nh, nw = min(nh, ch), min(nw, cw) + if getattr(self.det, "u8", False): # graph normalises: hand it the raw canvas, a quarter of the bytes and no float conversion + if dst is None: + canvas = np.full((ch, cw, 3), 255, np.uint8); canvas[:nh, :nw] = cv2.resize(img, (nw, nh)) + return canvas, (nh, nw, w / nw, h / nh) + if nh < ch or nw < cw: dst[:] = 255 + cv2.resize(img, (nw, nh), dst=dst[:nh, :nw]) + return None, (nh, nw, w / nw, h / nh) + x = cv2.dnn.blobFromImage(cv2.resize(img, (nw, nh)), 1 / 255.0, (nw, nh), MEAN255, swapRB=False)[0] # (3,nh,nw), (x/255 - mean) + x *= INV_STD[:, None, None] + if dst is None: + canvas = np.empty((3, ch, cw), np.float32); canvas[:] = PAD_VAL[:, None, None]; canvas[:, :nh, :nw] = x + return canvas, (nh, nw, w / nw, h / nh) + if nh < ch or nw < cw: dst[:] = PAD_VAL[:, None, None] # only pay for the pad where the page does not cover the canvas + dst[:, :nh, :nw] = x + return None, (nh, nw, w / nw, h / nh) + + # ---- detector, batched over pages that share a canvas shape ---- + def _det_batch(self, imgs, limit, ch, cw): + s = self._sess(limit); self._plan(limit, ch, cw, len(imgs)) + key = (len(imgs), ch, cw) + buf = self._det_buf.get(key) + if buf is None: + buf = self._det_buf[key] = (np.empty((len(imgs), ch, cw, 3), np.uint8) if getattr(self.det, "u8", False) + else np.empty((len(imgs), 3, ch, cw), np.float32)) # reused across batches + meta = [m for _, m in self.pool.map(lambda km: self._prep_det(km[1], ch, cw, limit, buf[km[0]]), list(enumerate(imgs)))] + x = buf + prob = s.run(None, {s.get_inputs()[0].name: x})[0][:, 0] + return list(self.pool.map(lambda k: self.det._post(prob[k][:meta[k][0], :meta[k][1]], meta[k][2], meta[k][3]), range(len(imgs)))) + + def _detect(self, imgs): + """imgs -> boxes per page; one detector pass per page, batched per canvas shape, no square padding.""" + groups = {} + for i, im in enumerate(imgs): groups.setdefault(canvas_shape(*im.shape[:2], min_scale=self.min_scale, canvases=self.canvases, width_classes=self.width_classes), []).append(i) + out = [None] * len(imgs) + for (limit, ch, cw), ids in groups.items(): + for j in range(0, len(ids), self.det_bs): + sub = ids[j:j + self.det_bs] + for i, b in zip(sub, self._det_batch([imgs[i] for i in sub], limit, ch, cw)): out[i] = b + return out + + # Layout batch is fixed (config layout.batch): TensorRT builds one engine per input shape (variable sizes = long rebuilds), so + # pages are always run at the configured batch, zero-padded; the engine is self-tested before its first use (Layout.ensure). + def _layout(self, imgs): + if not self.lay or not imgs: return [[] for _ in imgs] + if self.lay.trt and self.lay.bs not in self.lay.ok: self.lay.ensure(self.lay.bs) # validates (or swaps) the engine batch before the first use + return self.lay.batch(imgs) + + def _crops(self, img, boxes, pad=None): + if pad is None: return [crop(img, b) for b, _ in boxes] + out = [] + for b, _ in boxes: + b = b.copy(); hgt = 0.5 * ((b[3, 1] - b[0, 1]) + (b[2, 1] - b[1, 1])); b[0, 1] -= pad[0] * hgt; b[1, 1] -= pad[0] * hgt; b[2, 1] += pad[1] * hgt; b[3, 1] += pad[1] * hgt + out.append(crop(img, b)) + return out + + # ---------------- stage A (detector side) and stage B (recogniser side) so consecutive batches can overlap ---------------- + def _stage_a(self, imgs): + """detect -> orientation 90/270 -> deskew (boxes rotated, no second pass) -> crops. CPU+GPU(det).""" + t0 = time.perf_counter(); n = len(imgs); rot = [0] * n; ang = [0.0] * n + boxes = self._detect(imgs) + boxes = self._rescale_small_pages(imgs, boxes) + vert = [i for i in range(n) if boxes[i] and np.median([np.linalg.norm(b[1] - b[0]) / max(np.linalg.norm(b[3] - b[0]), 1) for b, _ in boxes[i]]) < 1.0] + if vert: # candidate 90/270: keep the orientation with more readable lines (mixed-orientation forms stay upright) + imgs90 = [rotate_page(imgs[i], 1) for i in vert]; boxes90 = self._detect(imgs90) + for i, im90, b90 in zip(vert, imgs90, boxes90): + good0 = n_readable(self.rec(self._crops(imgs[i], boxes[i]), bs=self.rec_bs)) if boxes[i] else 0 + good90 = n_readable(self.rec(self._crops(im90, b90), bs=self.rec_bs)) if b90 else 0 + if good90 > good0 or (good90 == 0 and good0 == 0): imgs[i], boxes[i], rot[i] = im90, b90, 90 # nothing readable either way -> trust the aspect rule; the 180° check finishes 270° pages + for i in range(n): + ang[i] = deskew_angle(boxes[i]) + if abs(ang[i]) > 0.4: + hh, ww = imgs[i].shape[:2]; imgs[i] = rotate_small(imgs[i], ang[i]); boxes[i] = rotate_boxes(boxes[i], ang[i], ww, hh) + # layout only where the page looks structured (many short boxes) or when forced; runs in its own thread + need_lay = [i for i in range(n) if self.lay and (self.layout_mode == "always" or (self.layout_mode == "auto" and self._structured(boxes[i])))] + lay_future = self.pool.submit(self._layout, [imgs[i] for i in need_lay]) if need_lay else None + # crops: one warp task per crop (the 4-core box has little headroom, but warps release the GIL) + jobs = [(i, b) for i in range(n) for b, _ in boxes[i]] + flat = list(self.pool.map(lambda ib: crop(imgs[ib[0]], ib[1]), jobs)) + return {"imgs": imgs, "boxes": boxes, "rot": rot, "ang": ang, "flat": flat, "jobs": jobs, "lay_future": lay_future, "need_lay": need_lay, "t_a": time.perf_counter() - t0} + + + @staticmethod + def _median_line_px(page_boxes): + """Median detected line height in page pixels, or 0 when nothing was detected.""" + if not page_boxes: return 0.0 + hs = [float(np.linalg.norm(b[3] - b[0])) for b, _ in page_boxes] + return float(np.median(hs)) if hs else 0.0 + + def _rescale_small_pages(self, imgs, boxes): + """Re-detect pages whose text is too small, on an UPSCALED COPY, then map the boxes back. + + The recogniser is trained at H=32 and `Detector.__call__` caps its scale at + `min(1.0, limit / max(h, w))`, so it never enlarges a small page: a low-DPI scan is detected at + 12-25 px lines. Detection, not recognition, is what fails there -- at ~13 px a page comes back as + correctly spelled words in the WRONG ORDER. + + The page image itself is NOT replaced. An earlier version swapped in the upscaled copy, which left + every bbox in the output in upscaled coordinates: a 446x687 input returned boxes out to x=976, + y=1560, so anything drawing them on the original page was wrong. Detection runs on a temporary + enlarged copy and the quads are divided back by the scale, so the whole document stays in INPUT + coordinates and the crop path is untouched. + """ + if not self.min_line_px: return boxes + todo = [] + for i, page in enumerate(boxes): + med = self._median_line_px(page) + if 0 < med < self.min_line_px: + # never enlarge past the biggest detector canvas: beyond it the detector just shrinks the + # page back and the extra pixels are discarded after churning the geometry + h, w = imgs[i].shape[:2] + cap = max(1.0, float(max(self.canvases)) / max(h, w)) + s = min(self.max_page_scale, cap, self.target_line_px / med) + if s > 1.05: todo.append((i, s)) + if not todo: return boxes + big = [] + for i, s in todo: + h, w = imgs[i].shape[:2] + big.append(cv2.resize(imgs[i], (int(round(w * s)), int(round(h * s))), interpolation=cv2.INTER_CUBIC)) + redone = self._detect(big) + for (i, s), page in zip(todo, redone): + boxes[i] = [(np.asarray(q, np.float32) / s, sc) for q, sc in page] # back to INPUT coordinates + return boxes + + def _structured(self, boxes, min_boxes=25, short_frac=0.45): + """Heuristic for 'this page has tables/forms': many boxes and a large share of short ones.""" + if len(boxes) < min_boxes: return False + ws = [np.linalg.norm(b[1] - b[0]) for b, _ in boxes]; med = np.median(ws) + return np.mean([w < 0.5 * med for w in ws]) > short_frac or len(boxes) > 80 + + def _stage_b(self, A, page_indices=None, dpi=None, with_polygons=False): + """recognise (pooled) -> 180° fallback -> capped second pass -> layout join -> assemble. GPU(rec)+CPU.""" + t0 = time.perf_counter(); imgs, boxes, rot, ang, jobs = A["imgs"], A["boxes"], A["rot"], A["ang"], A["jobs"]; n = len(imgs) + res = self.rec(A["flat"], bs=self.rec_bs) if A["flat"] else [] + texts = [[] for _ in range(n)] + for (i, _), r in zip(jobs, res): texts[i].append(r) + # 180° candidates: low mean confidence OR weak characters (upside-down pages read as confident garbage, min-char ~0.3) + top5 = lambda r: sorted(r, key=lambda tc: -len(tc[0]))[:5] + flip = [i for i in range(n) if texts[i] and (np.mean([c for _, c in texts[i]]) < 0.55 or orient_score(top5(texts[i])) < 1.35)] + if flip: + fimgs = [rotate_page(imgs[i], 2) for i in flip]; fboxes = self._detect(fimgs) + fcrops = [self._crops(im, b) for im, b in zip(fimgs, fboxes)]; ff = [c for cs in fcrops for c in cs]; fres = self.rec(ff, bs=self.rec_bs) if ff else []; p = 0 + for i, im, b, cs in zip(flip, fimgs, fboxes, fcrops): + r = fres[p:p + len(cs)]; p += len(cs) + if r and orient_score(top5(r)) > orient_score(top5(texts[i])) + 0.10: imgs[i], boxes[i], texts[i], rot[i] = im, b, r, (rot[i] + 180) % 360 + # confidence-gated second pass: only the lowest-confidence lines (cap per page), pooled across the batch + low = [] + for i in range(n): + cand = sorted([(c, j) for j, (_, c) in enumerate(texts[i]) if c < self.tta_conf])[:self.tta_cap]; low += [(i, j) for _, j in cand] + if low: + alt = list(self.pool.map(lambda ij: self._crops(imgs[ij[0]], [boxes[ij[0]][ij[1]]], pad=(0.06, 0.08))[0], low)) + for (i, j), (tx, c) in zip(low, self.rec(alt, bs=self.rec_bs)): + if c > texts[i][j][1] + 0.02: texts[i][j] = (tx, c) + t_rec = time.perf_counter() - t0; t1 = time.perf_counter() + blocks = [[] for _ in range(n)] + if A["lay_future"]: + for i, bl in zip(A["need_lay"], A["lay_future"].result()): blocks[i] = bl + for i in range(n): # layout ran on the un-deskewed page: rotate its blocks into the deskewed frame + if abs(ang[i]) > 0.4 and blocks[i]: + hh, ww = imgs[i].shape[:2]; M = cv2.getRotationMatrix2D((ww / 2, hh / 2), ang[i], 1.0) + for b in blocks[i]: + x0, y0, x1, y1 = b["bbox"]; pts = cv2.transform(np.array([[[x0, y0], [x1, y0], [x1, y1], [x0, y1]]], np.float32), M)[0] + b["bbox"] = [float(pts[:, 0].min()), float(pts[:, 1].min()), float(pts[:, 0].max()), float(pts[:, 1].max())] + t_lay = time.perf_counter() - t1; t2 = time.perf_counter() + def build(i): + d = assemble(imgs[i].shape[1], imgs[i].shape[0], blocks[i], boxes[i], texts[i], self.model_ids, None, (page_indices or list(range(n)))[i], dpi, with_polygons, img=imgs[i], + suppress_text_in=tuple(_asm(self.cfg).get("suppress_text_in", ())), + low_res_px=float(_asm(self.cfg).get("low_res_px", 0)), + retype_visual_min_lines=int(_asm(self.cfg).get("retype_visual_min_lines", 4)), + row_cross_frac=float(_asm(self.cfg).get("row_cross_frac", 0.20)), + suppress_min_conf=float(_asm(self.cfg).get("suppress_min_conf", 0.0)), + suppress_max_area=float(_asm(self.cfg).get("suppress_max_area", 1.0)), + suppress_max_lines=int(_asm(self.cfg).get("suppress_max_lines", 0)), + merge_split_blocks_x=float(_asm(self.cfg).get("merge_split_blocks_x", 0.0)), + merge_split_blocks_y=float(_asm(self.cfg).get("merge_split_blocks_y", 0.25))) + d["page"]["rotation"] = rot[i]; d["page"]["deskew_deg"] = round(ang[i], 2); d["page"]["second_pass_lines"] = sum(1 for a, _ in low if a == i); d["page"]["layout_run"] = i in A["need_lay"]; return d + docs = list(self.pool.map(build, range(n))) + T = {"det+crop": A["t_a"], "rec": t_rec, "layout_wait": t_lay, "assemble": time.perf_counter() - t2} + for d in docs: d["timing_ms"] = {k: round(v * 1000 / n, 1) for k, v in T.items()} + return docs, T + + def process(self, imgs, page_indices=None, dpi=None, with_polygons=False): + """One page batch, stages A then B (no overlap). Use run() for the pipelined version.""" + t0 = time.perf_counter(); A = self._stage_a(imgs); docs, T = self._stage_b(A, page_indices, dpi, with_polygons); T["total"] = time.perf_counter() - t0 + return docs, T + + def run(self, paths, page_batch=None, loader=None, with_polygons=False): + """Pipelined batched run over all pages; returns (docs, stats). See `stream` for the generator variant.""" + docs = []; stats = {} + for d, st in self.stream(paths, page_batch, loader, with_polygons): docs.append(d); stats = st + return docs, stats + + def stream(self, paths, page_batch=None, loader=None, with_polygons=False): + """Pipelined: decode(k+2) | stage A(k+1) | stage B(k) run concurrently. stage A is detector-GPU + CPU crops, + stage B is recogniser-GPU + CPU assembly, so the two GPU models interleave and the CPU work hides behind them. + paths: file paths (decoded with cv2.imread) or anything `loader` turns into a BGR array. + Yields (doc, running_stats) per page as soon as its page batch is finished; the last stats dict is the total.""" + page_batch = int(page_batch or self.cfg["runtime"].get("page_batch", 8)); loader = loader or cv2.imread + batches = [paths[i:i + page_batch] for i in range(0, len(paths), page_batch)] + if not batches: return + dec = lambda ps: list(self.pool.map(loader, ps)) + stageA = ThreadPoolExecutor(1) # one worker: stage A of batch k+1 overlaps stage B of batch k + agg = {}; npages = 0; t0 = time.perf_counter() + try: + imgs0 = dec(batches[0]); a_fut = stageA.submit(self._stage_a, imgs0); nxt = self.pool.submit(dec, batches[1]) if len(batches) > 1 else None + for k in range(len(batches)): + A = a_fut.result() + if k + 1 < len(batches): + imgs1 = nxt.result(); nxt = self.pool.submit(dec, batches[k + 2]) if k + 2 < len(batches) else None + a_fut = stageA.submit(self._stage_a, imgs1) + d, T = self._stage_b(A, page_indices=list(range(npages, npages + len(A["imgs"]))), with_polygons=with_polygons); npages += len(d) + for kk, v in T.items(): agg[kk] = agg.get(kk, 0.0) + v + wall = time.perf_counter() - t0 + st = {"pages": npages, "wall_s": wall, "pages_per_s": npages / max(wall, 1e-9), "ms_per_page": {kk: round(v * 1000 / max(npages, 1), 1) for kk, v in agg.items()}} + for doc in d: yield doc, st + finally: + stageA.shutdown(wait=False) diff --git a/hi-en/kanen_infer/charset.py b/hi-en/kanen_infer/charset.py new file mode 100644 index 0000000000000000000000000000000000000000..f84d086ca38f5c8b8e8efb98b77e0bf0314b5dde --- /dev/null +++ b/hi-en/kanen_infer/charset.py @@ -0,0 +1,72 @@ +"""KanEn dictionary + label normaliser. +Classes: CTC blank (0) + ASCII printable (incl. space) + Kannada block (used code points) + document symbols. +Everything NFC. ZWJ/ZWNJ are stripped (decision: fonts render the same joined form; keep labels visual).""" +import unicodedata, re + +ASCII = [chr(i) for i in range(32, 127)] # 95 incl. space +KANNADA = [chr(c) for c in range(0x0C80, 0x0CF4)] # candidate block; filtered by 'assigned' +KANNADA = [c for c in KANNADA if unicodedata.category(c) not in ("Cn",)] +SYMBOLS = list("₹–—‘’“”•·°×÷½¼¾§¶।॥…‑‐−™©®") +ZW = {"\u200c", "\u200d", "\u200b", "\ufeff", "\u2060", "\xad", "\u200e", "\u200f"} + +def build_dict(): + seen, out = set(), [] + for c in ASCII + KANNADA + SYMBOLS: + if c not in seen: seen.add(c); out.append(c) + return out # index 0 in the model = blank; chars start at 1 + +CHARS = build_dict() +CHAR2ID = {c: i + 1 for i, c in enumerate(CHARS)} +NUM_CLASSES = len(CHARS) + 1 + +_ws = re.compile(r"[ \t ]+") +QUOTE_MAP = {"‘": "'", "’": "'", "“": '"', "”": '"'} # optional folding for eval only + +def normalize(s: str, fold_quotes: bool = False) -> str: + s = unicodedata.normalize("NFC", s) + s = "".join(ch for ch in s if ch not in ZW) + s = s.replace("಼", "") # Kannada nukta: not used in printed text + if fold_quotes: + s = "".join(QUOTE_MAP.get(ch, ch) for ch in s) + s = _ws.sub(" ", s).strip() + return s + +def encode(s: str): + ids, oov = [], [] + for ch in s: + i = CHAR2ID.get(ch) + if i is None: oov.append(ch) + else: ids.append(i) + return ids, oov + +def decode(ids): + return "".join(CHARS[i - 1] for i in ids if 0 < i <= len(CHARS)) + +def script_of(s: str) -> str: + kn = sum(0x0C80 <= ord(c) <= 0x0CFF for c in s); en = sum(c.isascii() and c.isalpha() for c in s) + dig = sum(c.isdigit() for c in s) + if kn and en: return "mixed" + if kn: return "kn" + if en: return "en" + return "num" if dig else "other" + +if __name__ == "__main__": + print("classes incl. blank:", NUM_CLASSES, "| kannada:", len(KANNADA), "| ascii:", len(ASCII), "| symbols:", len(SYMBOLS)) + + +# --- Kannada orthographic sanity (label QA only; never used to alter model output) --- +_VSIGN = set(chr(c) for c in range(0x0CBE, 0x0CCD)) | {"\u0CD5", "\u0CD6"} # dependent vowel signs + length marks +_VIRAMA = "\u0CCD"; _CONS = set(chr(c) for c in range(0x0C95, 0x0CBA)) | {"\u0CDE"} # incl. archaic ೞ +def kannada_orthography_issues(s: str): + """Returns a list of reasons the string cannot be valid printed Kannada (empty list = looks valid).""" + issues = [] + for i, ch in enumerate(s): + prev = s[i - 1] if i else "" + if ch in _VSIGN and (prev in _VSIGN or prev == _VIRAMA or prev == "" or prev == " "): issues.append(f"vowel sign after '{prev}' at {i}") + if ch == _VIRAMA and (prev == _VIRAMA or prev in _VSIGN or prev not in _CONS): issues.append(f"virama after '{prev}' at {i}") + # Latin/ASCII letters glued inside a Kannada word (no space between scripts) + import re + if re.search(r"[\u0C80-\u0CFF][A-Za-z]|[A-Za-z][\u0C80-\u0CFF]", s): issues.append("latin letter glued to kannada") + if re.search(r"[0-9][\u0CBE-\u0CCD]", s): issues.append("digit followed by kannada mark") + if re.search(r"[\u0CDE\u0CF1\u0CF2]", s): issues.append("rare/archaic kannada letter") + return issues diff --git a/hi-en/kanen_infer/cli.py b/hi-en/kanen_infer/cli.py new file mode 100644 index 0000000000000000000000000000000000000000..f7cdc7b9b253fc0d6b07b6d454e88eee7f3c38f4 --- /dev/null +++ b/hi-en/kanen_infer/cli.py @@ -0,0 +1,71 @@ +"""`kanen-ocr` command line. + + kanen-ocr --input scans/ --out-dir out/ --format all + kanen-ocr --input "pages/*.png" --out-dir out/ --format json --page-batch 8 --workers 8 --provider trt + kanen-ocr --input report.pdf --out-dir out/ --format txt,tsv --json-lines + +Exit codes: 0 ok · 1 no input pages · 2 models/engines failed to load · 3 some pages failed (others written). +The summary line reports pages, pages/s, the provider each model ended on and every fallback that happened.""" +import argparse, json, os, sys, time, traceback +from pathlib import Path + +FORMAT_CHOICES = ["json", "md", "txt", "hocr", "alto", "tsv", "all"] + +def build_parser(): + ap = argparse.ArgumentParser(prog="kanen-ocr", description="Arjuna-OCR [Kn&En]: layout-aware Kannada + English document OCR (kanen-ocr/1.0 JSON and other serialisations)") + ap.add_argument("--input", "-i", nargs="+", default=None, help="image file(s), directory, glob, or PDF") + ap.add_argument("--out-dir", "-o", default="kanen_out", help="output directory (one file per page and format)") + ap.add_argument("--format", "-f", default="json", help="comma list of json,md,txt,hocr,alto,tsv or 'all'") + ap.add_argument("--json-lines", action="store_true", help="also write results.jsonl (one document per line) and results.json (list)") + ap.add_argument("--page-batch", type=int, default=None); ap.add_argument("--layout-mode", choices=["always", "auto", "never"], default=None) + ap.add_argument("--workers", type=int, default=None); ap.add_argument("--provider", choices=["auto", "trt", "cuda", "cpu"], default=None) + ap.add_argument("--polygons", action="store_true", help="include 4-point polygons in line objects"); ap.add_argument("--pdf-dpi", type=int, default=200) + ap.add_argument("--config", default=None, help="alternative kanen.yaml / .json"); ap.add_argument("--quiet", "-q", action="store_true"); ap.add_argument("--limit", type=int, default=0) + ap.add_argument("--version", action="store_true", help="print version and exit") + return ap + +def main(argv=None): + a = build_parser().parse_args(argv) + from . import __version__ + if a.version: print(__version__); return 0 + if not a.input: build_parser().error("the following arguments are required: --input/-i") + from .api import KanEnOCR + from .io import expand_inputs + from .formats import FORMATS, render + fmts = FORMAT_CHOICES[:-1] if a.format == "all" else [f.strip() for f in a.format.split(",") if f.strip()] + bad = [f for f in fmts if f not in FORMATS] + if bad: print(f"unknown format(s) {bad}; choose from {FORMAT_CHOICES}", file=sys.stderr); return 1 + paths = expand_inputs(a.input); paths = paths[:a.limit] if a.limit else paths + if not paths: print(f"no input pages match {a.input}", file=sys.stderr); return 1 + say = (lambda *x: None) if a.quiet else (lambda *x: print(*x, file=sys.stderr, flush=True)) + t0 = time.time() + try: + ocr = KanEnOCR(config=a.config, providers=a.provider, layout_mode=a.layout_mode, workers=a.workers) + try: # long jobs earn the TensorRT build; a page or two does not (the build costs more than it saves) + _after = int(os.environ.get("KANEN_OPTIMIZE_AFTER", "8")) + if len(paths) >= _after and hasattr(ocr, "optimize"): ocr.optimize() + except Exception: pass + except Exception as e: + print(f"failed to load models: {type(e).__name__}: {e}", file=sys.stderr); traceback.print_exc(); return 2 + r = ocr.ready(); prov = ", ".join(f"{k}={v.get('provider')}{'' if v.get('provider') != 'trt' else ('-fp16' if v.get('fp16') else '-fp32')}" for k, v in r["providers"].items()) + say(f"{r['name']} {r['version']} | providers: {prov} | load {r['load_s']}s warm-up {r['warmup_s']}s | {len(paths)} input file(s)") + out = Path(a.out_dir); out.mkdir(parents=True, exist_ok=True); docs = []; failed = 0; jl = open(out / "results.jsonl", "w", encoding="utf-8") if a.json_lines else None + try: + for doc in ocr.stream(paths, page_batch=a.page_batch, polygons=a.polygons, pdf_dpi=a.pdf_dpi): + label = doc["page"].get("file", f"page_{doc['page']['index']}"); base, _, pg = label.partition("#") # PDFs: name.pdf#3 -> name_p3 + stem = os.path.splitext(Path(base).name)[0] + (f"_p{pg}" if pg else "") + for f in fmts: + (out / (stem + FORMATS[f])).write_text(render(doc, f), encoding="utf-8") + if jl: jl.write(json.dumps(doc, ensure_ascii=False) + "\n") + docs.append(doc) + except Exception as e: + failed += 1; print(f"page failed: {type(e).__name__}: {e}", file=sys.stderr); traceback.print_exc() + finally: + if jl: jl.close() + if a.json_lines: (out / "results.json").write_text(json.dumps(docs, ensure_ascii=False, indent=1), encoding="utf-8") + st = getattr(ocr, "last_stats", {}) or {}; wall = time.time() - t0; n = len(docs) + fb = [e for v in ocr.ready()["providers"].values() for e in v.get("fallbacks", [])] + say(f"done: {n} pages in {st.get('wall_s', wall):.1f} s = {st.get('pages_per_s', n / max(wall, 1e-9)):.2f} pages/s | ms/page {st.get('ms_per_page', {})} | providers: {prov} | fallbacks: {len(fb)}{(' -> ' + '; '.join(fb)) if fb else ''} | wrote {out}") + return 3 if failed else 0 + +if __name__ == "__main__": sys.exit(main()) diff --git a/hi-en/kanen_infer/config.py b/hi-en/kanen_infer/config.py new file mode 100644 index 0000000000000000000000000000000000000000..9b909ea0dec284ef1d5dc1781d4fd3625ebf1953 --- /dev/null +++ b/hi-en/kanen_infer/config.py @@ -0,0 +1,112 @@ +"""Configuration for the Arjuna-OCR [Kn&En] inference package: `kanen.yaml` next to the package + environment overrides. +Paths are resolved relative to the package root with pathlib (Linux and Windows).""" +import copy, json, logging, os +from pathlib import Path + +PKG_ROOT = Path(__file__).resolve().parent.parent # directory that holds kanen.yaml, models/, trt_cache/ +def _root() -> Path: + """the directory that actually holds kanen.yaml + models/ — the package directory in a clone, the downloaded + Hugging Face snapshot for a pip install (see kanen_infer/models.py)""" + try: + from .models import resolve + return resolve(PKG_ROOT) + except Exception: + return PKG_ROOT +def _cfg_in(root): + """This package ships arjuna_hi.yaml; the Kannada one ships kanen.yaml. Resolve whichever is present + instead of hardcoding kanen.yaml, which made the Hindi package fall back to KANNADA defaults.""" + for n in ("arjuna_hi.yaml", "kanen.yaml"): + if (Path(root) / n).exists(): return Path(root) / n + return Path(root) / "arjuna_hi.yaml" +DEFAULT_CONFIG = _cfg_in(PKG_ROOT) +LOG = logging.getLogger("kanen") + +_FALLBACK = { # used when the config is missing or a key is absent; must mirror the shipped arjuna_hi.yaml. + # It previously mirrored the KANNADA config, so a missing config silently selected the + # Kannada recogniser and the v12 layout inside the Hindi package. + "name": "Arjuna-OCR [Hi & En]", "version": "1.1.3", + "models": {"det": "models/det/inference.onnx", "layout": "models/layout/arjuna_layout_v15.onnx", "layout_labels": "models/layout/classes_v15.txt", + "layout_selftest": "models/layout/selftest.jpg", "layout_selftest_ref": "models/layout/selftest_ref_v15.json", "rec": "models/rec/arjuna_rec_hi.onnx", "charset": "models/rec/charset.json"}, + "provider": {"policy": "auto", "trt_fp16": True, "engine_cache": "trt_cache", "trt_workspace_gb": 4, "cuda_mem_limit_gb": 6, "cudnn_conv_algo_search": "HEURISTIC"}, + "detector": {"canvases": [1280, 1920, 2560], "width_classes": [0.5, 0.625, 0.75, 0.875, 1.0], "min_scale": 0.9, "default_canvas": 1280, + "thresh": 0.2, "box_thresh": 0.45, "unclip": 1.4, "pad_top": 0.03, "pad_bottom": 0.25, "batch": 2}, + "layout": {"mode": "always", "batch": 4, "score_thresh": 0.3, "fill": [1.0, 0.8], "selftest_min_match": 1.0}, + "recognizer": {"buckets": [128, 192, 256, 384, 512, 640, 768, 1024, 1280], "batch": 64, "height": 32, "max_ratio": 40}, + "second_pass": {"conf": 0.85, "cap": 6}, + "runtime": {"workers": "auto", "page_batch": 8, "log_level": "warning", "polygons": False}, +} + +def _merge(base, over): + out = copy.deepcopy(base) + for k, v in (over or {}).items(): + out[k] = _merge(out[k], v) if isinstance(v, dict) and isinstance(out.get(k), dict) else copy.deepcopy(v) + return out + +def _writable(p: Path) -> bool: + try: + target = p if p.exists() else p.parent + target.mkdir(parents=True, exist_ok=True) + probe = target / ".arjuna_write_test"; probe.write_text("ok"); probe.unlink(); return True + except Exception: return False + +def _read(path): + p = Path(path) + text = p.read_text(encoding="utf-8") + if p.suffix.lower() == ".json": return json.loads(text) + import yaml + return yaml.safe_load(text) or {} + +class Config(dict): + """dict with attribute access and resolved paths; `cfg.path('models','rec')` -> absolute Path.""" + root: Path + def __getattr__(self, k): + try: return self[k] + except KeyError as e: raise AttributeError(k) from e + def path(self, *keys) -> Path: + v = self + for k in keys: v = v[k] + p = Path(str(v)).expanduser() + return p if p.is_absolute() else (self.root / p) + def workers(self) -> int: + w = self["runtime"].get("workers", "auto") + return int(w) if str(w).isdigit() else max(4, (os.cpu_count() or 4) // 2) + +def load_config(config=None, overrides=None) -> Config: + """config: None (kanen.yaml next to the package, or $KANEN_CONFIG), a path, or a dict. overrides: nested dict applied last. + Environment: KANEN_TRT=1 -> provider.policy 'trt'; KANEN_ORT_CPU=1 -> 'cpu'; KANEN_REC_ONNX -> models.rec; KANEN_LOG -> runtime.log_level.""" + root = _root() + globals()["DEFAULT_CONFIG"] = _cfg_in(root) + src = None + if isinstance(config, dict): data = _merge(_FALLBACK, config) + else: + src = Path(config) if config else Path(os.environ.get("KANEN_CONFIG") or _cfg_in(root)) + data = _merge(_FALLBACK, _read(src)) if src.exists() else copy.deepcopy(_FALLBACK) + root = Path(os.path.abspath(src)).parent if src.exists() else root + if overrides: data = _merge(data, overrides) + if os.environ.get("KANEN_ORT_CPU"): data["provider"]["policy"] = "cpu" + elif os.environ.get("KANEN_TRT") == "1": data["provider"]["policy"] = "trt" + if os.environ.get("KANEN_REC_ONNX"): data["models"]["rec"] = os.environ["KANEN_REC_ONNX"] + if os.environ.get("KANEN_LOG"): data["runtime"]["log_level"] = os.environ["KANEN_LOG"] + cfg = Config(data); cfg.root = root; cfg.source = str(src) if src else "" + # engine cache: keep it next to the models in a clone, but a pip install resolves the models inside the read-only + # Hugging Face cache, so fall back to the user cache directory (TensorRT engines are machine-specific anyway) + env_cache = os.environ.get("ARJUNA_ENGINE_CACHE") or os.environ.get("KANEN_ENGINE_CACHE") + if env_cache: cfg["provider"]["engine_cache"] = env_cache + else: + cache = cfg.path("provider", "engine_cache") + if not _writable(cache): + home = Path(os.environ.get("XDG_CACHE_HOME", Path.home() / ".cache")) / "arjuna" / "engines" + cfg["provider"]["engine_cache"] = str(home) + LOG.debug(f"engine cache {cache} is not writable, using {home}") + setup_logging(cfg["runtime"].get("log_level", "warning")) + return cfg + +def setup_logging(level="warning"): + lvl = getattr(logging, str(level).upper(), logging.WARNING) + if not LOG.handlers: + h = logging.StreamHandler(); h.setFormatter(logging.Formatter("[%(name)s] %(message)s")); LOG.addHandler(h) + LOG.setLevel(lvl); LOG.propagate = False + try: + import onnxruntime as ort + ort.set_default_logger_severity(3 if lvl >= logging.WARNING else 1) # silence TensorRT/ORT chatter unless debug/info + except Exception: pass diff --git a/hi-en/kanen_infer/formats.py b/hi-en/kanen_infer/formats.py new file mode 100644 index 0000000000000000000000000000000000000000..ae6ab0b8fc11be7d1f5072cbd2b5b747872f48df --- /dev/null +++ b/hi-en/kanen_infer/formats.py @@ -0,0 +1,76 @@ +"""Serialisations of one `kanen-ocr/1.0` document. All of them are views of the same JSON: nothing is filtered, +re-ordered or rewritten — the text, order, confidences and flags are exactly those of the document. + + to_json(doc) the document itself (schema kanen-ocr/1.0) + to_markdown(doc) Markdown: titles, paragraphs, tables as pipe tables, figure/seal markers + to_text(doc) reading-order plain text: one block per paragraph (blank line between), table rows one per line with ' | ' + to_hocr(doc) hOCR 1.2 (ocr_page > ocr_carea > ocr_line > ocrx_word), bbox + x_wconf per line + to_alto(doc) ALTO 4 XML (TextBlock > TextLine > String with WC), one String per line + to_tsv(doc) page, block, line, bbox, text, confidence, min_conf, flags (tab separated, header row) +""" +import json +from xml.sax.saxutils import escape, quoteattr +from .assemble import to_markdown # noqa: F401 (re-exported) + +FORMATS = {"json": ".json", "md": ".md", "txt": ".txt", "hocr": ".hocr", "alto": ".xml", "tsv": ".tsv"} + +def to_json(doc, indent=1): + return json.dumps(doc, ensure_ascii=False, indent=indent) + +def block_text(b): + if b.get("type") == "table" and b.get("rows"): return "\n".join(" | ".join(c["text"] for c in r["cells"]) for r in b["rows"]) + return "\n".join(l["text"] for l in b.get("lines", [])) + +def to_text(doc): + parts = [t for t in (block_text(b) for b in doc["blocks"]) if t.strip()] + return "\n\n".join(parts) + "\n" + +def _ibox(bb): return " ".join(str(int(round(v))) for v in bb) + +def to_hocr(doc, title=None, name="Arjuna-OCR [Kn&En]"): + pg = doc["page"]; ver = doc.get("model", {}).get("pipeline", ""); pidx = pg.get("index", 0) + fn = pg.get("file") or f"page_{pidx}" + out = ['', '', + '', "", f"{escape(title or fn)}", '', + f'', '', "", "", + f'
'] + for b in doc["blocks"]: + bid = b.get("id", "b"); out.append(f'
') + out.append(f'

') + for l in b.get("lines", []): + conf = int(round(100 * float(l["confidence"]))); lid = l.get("id", bid + ".l"); t = "bbox " + _ibox(l["bbox"]) + out.append(f'' f'{escape(l["text"])}') + out.append("

") + out += ["
", "", "", ""] + return "\n".join(out) + +def to_alto(doc, name="Arjuna-OCR [Kn&En]"): + pg = doc["page"]; ver = doc.get("model", {}).get("pipeline", ""); pidx = pg.get("index", 0); fn = pg.get("file") or f"page_{pidx}" + out = ['', '', "", "pixel", + f"{escape(fn)}", + f"{escape(name)}{escape(str(ver))}", + "", "", f'', ""] + def geo(bb): x0, y0, x1, y1 = bb; return f'HPOS="{int(round(x0))}" VPOS="{int(round(y0))}" WIDTH="{int(round(x1 - x0))}" HEIGHT="{int(round(y1 - y0))}"' + for b in doc["blocks"]: + bid = b.get("id", "b"); out.append(f'') + for l in b.get("lines", []): + lid = l.get("id", bid + ".l") + out.append(f'') + out.append("") + out += ["", "", "", "", ""] + return "\n".join(out) + +TSV_HEADER = "page\tblock\tline\tbbox\ttext\tconfidence\tmin_conf\tflags" + +def tsv_rows(doc): + pg = doc["page"]; page = pg.get("file") or str(pg.get("index", 0)) + for b in doc["blocks"]: + for l in b.get("lines", []): + yield [page, b.get("id", ""), l.get("id", ""), ",".join(str(int(round(v))) for v in l["bbox"]), l["text"].replace("\t", " "), f'{float(l["confidence"]):.4f}', f'{float(l.get("min_conf", l["confidence"])):.4f}', " ".join(l.get("flags", []))] + +def to_tsv(doc, header=True): + rows = ["\t".join(r) for r in tsv_rows(doc)] + return ((TSV_HEADER + "\n") if header else "") + "\n".join(rows) + ("\n" if rows else "") + +def render(doc, fmt): + return {"json": to_json, "md": to_markdown, "txt": to_text, "hocr": to_hocr, "alto": to_alto, "tsv": to_tsv}[fmt](doc) diff --git a/hi-en/kanen_infer/geometry.py b/hi-en/kanen_infer/geometry.py new file mode 100644 index 0000000000000000000000000000000000000000..897d747175ceccce8ff365f78bca800c6bc381ba --- /dev/null +++ b/hi-en/kanen_infer/geometry.py @@ -0,0 +1,19 @@ +"""Page geometry helpers (orientation / deskew).""" +import math, numpy as np, cv2 + +def rotate_page(img, k): # k in {0,1,2,3} multiples of 90° counter-clockwise + return np.ascontiguousarray(np.rot90(img, k)) if k else img + +def deskew_angle(boxes): + """Median angle (deg) of wide line boxes; positive = counter-clockwise tilt.""" + angs = [] + for b, _ in boxes: + w = np.linalg.norm(b[1] - b[0]); h = np.linalg.norm(b[3] - b[0]) + if w > 4 * h and w > 60: + angs.append(math.degrees(math.atan2(b[1][1] - b[0][1], b[1][0] - b[0][0]))) + return float(np.median(angs)) if len(angs) >= 5 else 0.0 + +def rotate_small(img, deg): + h, w = img.shape[:2]; M = cv2.getRotationMatrix2D((w / 2, h / 2), deg, 1.0) + return cv2.warpAffine(img, M, (w, h), flags=cv2.INTER_LINEAR, borderValue=(255, 255, 255)) + diff --git a/hi-en/kanen_infer/hardware.py b/hi-en/kanen_infer/hardware.py new file mode 100644 index 0000000000000000000000000000000000000000..32889a87e21fdbf48dd7febc414738ef0d55578b --- /dev/null +++ b/hi-en/kanen_infer/hardware.py @@ -0,0 +1,151 @@ +"""Hardware detection and auto-configuration: what this machine can do, and the settings that make it fast here. + +`probe()` collects GPU / driver / CUDA / TensorRT / onnxruntime / CPU / RAM facts and the provider ONNX Runtime will +actually be able to use (checking that the provider's libraries load, not just that the name is compiled in). +`autoconfig(cfg)` scales the batch sizes and thread counts in the config to the machine. It never touches anything that +changes results — canvases, thresholds, padding and the models stay exactly as shipped; only batching and threads move. +`report()` renders both as text for `arjuna doctor`. +""" +import json, os, platform, shutil, subprocess, sys +import onnxruntime as ort + +def _run(cmd, timeout=10): + try: + r = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout) + return r.stdout.strip() if r.returncode == 0 else "" + except Exception: return "" + +def nvidia_smi(): + """name, driver, total VRAM (MiB), compute capability — per GPU""" + out = _run(["nvidia-smi", "--query-gpu=name,driver_version,memory.total,compute_cap", "--format=csv,noheader,nounits"]) + gpus = [] + for line in out.splitlines(): + parts = [p.strip() for p in line.split(",")] + if len(parts) >= 3: + gpus.append({"name": parts[0], "driver": parts[1], "vram_mb": int(float(parts[2])) if parts[2].replace(".", "").isdigit() else None, + "compute_capability": parts[3] if len(parts) > 3 else None}) + return gpus + +def cuda_runtime_version(): + try: + import ctypes + lib = ctypes.CDLL("libcudart.so.12" if sys.platform != "win32" else "cudart64_12.dll") + v = ctypes.c_int(); lib.cudaRuntimeGetVersion(ctypes.byref(v)); return f"{v.value // 1000}.{(v.value % 1000) // 10}" + except Exception: return None + +def trt_version(): + try: + import tensorrt; return tensorrt.__version__ + except Exception: return None + +def usable_providers(): + """providers whose libraries really load here (ORT lists names its build was compiled with, which is not the same thing)""" + import numpy as np, tempfile + if not hasattr(ort, "get_available_providers"): # both builds were installed and one was removed: they share files + return {"listed": [], "usable": [], "note": "onnxruntime is installed but broken — run: pip install --force-reinstall onnxruntime-gpu (or onnxruntime)"} + names = ort.get_available_providers(); ok = [] + try: + from .providers import preload_cuda_libs; preload_cuda_libs() + except Exception: pass + try: import tensorrt # noqa: F401 (loads libnvinfer so the TensorRT provider library can resolve it) + except Exception: pass + # a 1-node model is enough to see whether a provider can create a session + try: + import onnx + from onnx import helper, TensorProto + g = helper.make_graph([helper.make_node("Relu", ["x"], ["y"])], "t", [helper.make_tensor_value_info("x", TensorProto.FLOAT, [1, 1])], [helper.make_tensor_value_info("y", TensorProto.FLOAT, [1, 1])]) + m = helper.make_model(g, opset_imports=[helper.make_opsetid("", 13)]) + m.ir_version = 10 # a newer onnx writes an IR version this onnxruntime may refuse; the probe only needs to load + path = os.path.join(tempfile.gettempdir(), "kanen_probe.onnx"); open(path, "wb").write(m.SerializeToString()) + except Exception: + return {"listed": names, "usable": names, "note": "onnx not installed: provider names not verified"} + so = ort.SessionOptions(); so.log_severity_level = 4 + import contextlib, io + for p in names: + if p in ("AzureExecutionProvider",): continue + try: # ORT prints its own provider errors to stderr; the probe reports them itself + with contextlib.redirect_stderr(io.StringIO()), contextlib.redirect_stdout(io.StringIO()): + s = ort.InferenceSession(path, so, providers=[p]) + if s.get_providers()[0] == p or p == "CPUExecutionProvider": ok.append(p) + except Exception: pass + if not ok: return {"listed": names, "usable": names, "note": "provider probe could not run; reporting the names onnxruntime lists"} + return {"listed": names, "usable": ok} +def probe(policy=None): + gpus = nvidia_smi(); prov = usable_providers() + if prov.get("note", "").startswith("onnxruntime is installed but broken"): + return {"os": f"{platform.system()} {platform.release()}", "machine": platform.machine(), "python": platform.python_version(), + "cpu_cores": os.cpu_count() or 1, "ram_gb": None, "onnxruntime": getattr(ort, "__version__", "broken"), "gpus": gpus, + "cuda_runtime": None, "tensorrt": trt_version(), "providers_listed": [], "providers_usable": [], "nvidia_smi": bool(shutil.which("nvidia-smi")), + "plan": "cpu", "plan_notes": [prov["note"]]} + hw = {"os": f"{platform.system()} {platform.release()}", "machine": platform.machine(), "python": platform.python_version(), + "cpu_cores": os.cpu_count() or 1, "ram_gb": round(os.sysconf("SC_PAGE_SIZE") * os.sysconf("SC_PHYS_PAGES") / 1e9, 1) if hasattr(os, "sysconf") and "SC_PHYS_PAGES" in os.sysconf_names else None, + "onnxruntime": ort.__version__, "gpus": gpus, "cuda_runtime": cuda_runtime_version(), "tensorrt": trt_version(), + "providers_listed": prov["listed"], "providers_usable": prov["usable"], "nvidia_smi": bool(shutil.which("nvidia-smi"))} + trt_ok = "TensorrtExecutionProvider" in prov["usable"] and hw["tensorrt"] and gpus + cuda_ok = "CUDAExecutionProvider" in prov["usable"] and gpus + ov_ok = "OpenVINOExecutionProvider" in prov["usable"] + hw["plan"] = "trt" if trt_ok else ("cuda" if cuda_ok else ("openvino" if ov_ok else "cpu")) + why = [] + forced = (policy or "").lower() + if forced in ("cpu", "cuda", "trt") and forced != "auto": + if forced != hw["plan"]: + order = {"cpu": 0, "cuda": 1, "trt": 2} + if order.get(forced, 0) <= order.get(hw["plan"], 0): + why.append(f"provider forced to {forced.upper()} by configuration/environment (this machine could run {hw['plan'].upper()})") + hw["plan"] = forced + else: why.append(f"{forced.upper()} requested but not usable here: running {hw['plan'].upper()}") + if not gpus: why.append("no NVIDIA GPU visible (nvidia-smi found nothing)" if hw["nvidia_smi"] else "nvidia-smi not found: no NVIDIA driver on this machine") + elif not cuda_ok: why.append("CUDAExecutionProvider cannot load (install onnxruntime-gpu matching your CUDA, or the CUDA/cuDNN runtime libraries)") + if gpus and cuda_ok and not trt_ok: why.append("TensorRT not usable (no tensorrt wheel or provider libraries): running on the CUDA provider, about 2-3x slower") + if hw["plan"] == "cpu" and not ov_ok: why.append("on Intel CPUs the OpenVINO build is 1.4-1.9x faster: pip uninstall -y onnxruntime && pip install onnxruntime-openvino") + if hw["plan"] == "openvino": why.append("OpenVINO execution provider (CPU); set provider.openvino_device to GPU or NPU for Intel accelerators") + try: # both onnxruntime and onnxruntime-gpu installed: they share the same import name and overwrite each other + from importlib.metadata import distributions + installed = {d.metadata["Name"].lower() for d in distributions() if d.metadata.get("Name")} + if {"onnxruntime", "onnxruntime-gpu"} <= installed: + why.append("both onnxruntime and onnxruntime-gpu are installed; keep one: pip uninstall -y onnxruntime") + except Exception: pass + hw["plan_notes"] = why + return hw + +# VRAM (GB) -> (page_batch, recognizer batch, layout batch, detector batch). Conservative: these only affect speed. +_GPU_TIERS = [(20, (12, 96, 8, 4)), (12, (8, 64, 4, 2)), (7, (6, 48, 4, 2)), (4, (4, 32, 2, 2)), (0, (2, 16, 1, 1))] +def autoconfig(cfg, hw=None, quality=None): + """scale batching/threads to the machine; returns the list of changes (for `doctor` and the logs)""" + hw = hw or probe(cfg["provider"].get("policy")); changes = [] + cores = hw["cpu_cores"] + if hw["plan"] in ("trt", "cuda") and hw["gpus"]: + vram = (hw["gpus"][0].get("vram_mb") or 8192) / 1024.0 + for floor, (pb, rb, lb, db) in _GPU_TIERS: + if vram >= floor: break + mem_limit = max(2.0, min(vram * 0.6, 12.0)) + for section, key, val in (("runtime", "page_batch", pb), ("recognizer", "batch", rb), ("layout", "batch", lb), ("detector", "batch", db), ("provider", "cuda_mem_limit_gb", round(mem_limit, 1))): + if cfg[section].get(key) != val: changes.append(f"{section}.{key}: {cfg[section].get(key)} -> {val}"); cfg[section][key] = val + ws = round(max(1.0, min(vram * 0.25, 4.0)), 1) + if cfg["provider"].get("trt_workspace_gb") != ws: changes.append(f"provider.trt_workspace_gb: {cfg['provider'].get('trt_workspace_gb')} -> {ws}"); cfg["provider"]["trt_workspace_gb"] = ws + else: # CPU only: recognition is the bottleneck, keep batches small and use the cores for the worker pool + for section, key, val in (("runtime", "page_batch", max(1, min(4, cores // 2))), ("recognizer", "batch", 16), ("layout", "batch", 1), ("detector", "batch", 1)): + if cfg[section].get(key) != val: changes.append(f"{section}.{key}: {cfg[section].get(key)} -> {val}"); cfg[section][key] = val + w = max(2, cores - 1 if cores <= 8 else int(cores * 0.75)) + if cfg["runtime"].get("workers") != w: changes.append(f"runtime.workers: {cfg['runtime'].get('workers')} -> {w}"); cfg["runtime"]["workers"] = w + if quality: + q = {"fast": {"canvases": [1280], "second_pass_cap": 0}, "balanced": {}, "exact": {"canvases": [1280, 1920, 2560], "second_pass_cap": 12}}.get(quality, {}) + if "canvases" in q and cfg["detector"].get("canvases") != q["canvases"]: changes.append(f"detector.canvases -> {q['canvases']} (quality={quality})"); cfg["detector"]["canvases"] = q["canvases"] + if "second_pass_cap" in q: changes.append(f"second_pass.cap -> {q['second_pass_cap']} (quality={quality})"); cfg["second_pass"]["cap"] = q["second_pass_cap"] + return changes +def report(cfg=None, changes=None, hw=None): + hw = hw or probe(cfg["provider"].get("policy") if cfg else None); L = [f"Arjuna-OCR — machine report", "", + f" OS {hw['os']} ({hw['machine']}), Python {hw['python']}", + f" CPU / RAM {hw['cpu_cores']} cores" + (f", {hw['ram_gb']} GB RAM" if hw["ram_gb"] else ""), + f" onnxruntime {hw['onnxruntime']}"] + if hw["gpus"]: + for i, g in enumerate(hw["gpus"]): + L.append(f" GPU {i} {g['name']} · driver {g['driver']}" + (f" · {g['vram_mb']} MiB" if g["vram_mb"] else "") + (f" · sm{g['compute_capability']}" if g["compute_capability"] else "")) + else: L.append(" GPU none detected") + L += [f" CUDA runtime {hw['cuda_runtime'] or 'not found'}", f" TensorRT {hw['tensorrt'] or 'not installed'}", + f" providers usable: {', '.join(p.replace('ExecutionProvider','') for p in hw['providers_usable']) or 'none'}", "", + f" plan {hw['plan'].upper()}" + (" (fp16 engines, built on first run in the background)" if hw["plan"] == "trt" else "")] + for n in hw["plan_notes"]: L.append(f" {n}") + if changes: L += ["", " auto-configuration (speed only, results unchanged):"] + [f" {c}" for c in changes] + return "\n".join(L) +if __name__ == "__main__": print(report()) diff --git a/hi-en/kanen_infer/io.py b/hi-en/kanen_infer/io.py new file mode 100644 index 0000000000000000000000000000000000000000..c99046859aa4c96e6157f7a944181d8561e3fc94 --- /dev/null +++ b/hi-en/kanen_infer/io.py @@ -0,0 +1,68 @@ +"""Input handling: file path, bytes, numpy BGR array, PIL image, directory / glob expansion, and PDF pages +(rendered with pypdfium2 if installed, else `pdftoppm` from poppler if on PATH; both optional).""" +import glob, io as _io, os, shutil, subprocess, tempfile +from pathlib import Path +import numpy as np, cv2 + +IMAGE_EXT = {".png", ".jpg", ".jpeg", ".tif", ".tiff", ".bmp", ".webp", ".jp2"} +PDF_EXT = {".pdf"} + +def to_bgr(x): + """Any supported input -> numpy uint8 BGR (H, W, 3). Raises ValueError if it cannot be decoded.""" + if isinstance(x, np.ndarray): + if x.ndim == 2: return cv2.cvtColor(x, cv2.COLOR_GRAY2BGR) + if x.ndim == 3 and x.shape[2] == 4: return cv2.cvtColor(x, cv2.COLOR_BGRA2BGR) + if x.ndim == 3 and x.shape[2] == 3: return np.ascontiguousarray(x) + raise ValueError(f"unsupported array shape {x.shape}") + if isinstance(x, (bytes, bytearray, memoryview)): + im = cv2.imdecode(np.frombuffer(bytes(x), np.uint8), cv2.IMREAD_COLOR) + if im is None: raise ValueError("could not decode image bytes") + return im + if isinstance(x, (str, os.PathLike)): + im = cv2.imdecode(np.fromfile(str(x), np.uint8), cv2.IMREAD_COLOR) # fromfile: unicode paths on Windows + if im is None: raise ValueError(f"could not read image {x}") + return im + if hasattr(x, "convert") and hasattr(x, "size"): # PIL.Image without importing PIL + return cv2.cvtColor(np.asarray(x.convert("RGB")), cv2.COLOR_RGB2BGR) + if hasattr(x, "read"): return to_bgr(x.read()) + raise ValueError(f"unsupported input type {type(x).__name__}") + +def expand_inputs(inputs): + """file / directory / glob / list -> sorted list of image and PDF paths (strings).""" + if isinstance(inputs, (str, os.PathLike)): inputs = [inputs] + out = [] + for it in inputs: + p = Path(it) + if p.is_dir(): out += sorted(str(f) for f in p.iterdir() if f.suffix.lower() in IMAGE_EXT | PDF_EXT) + elif p.exists(): out.append(str(p)) + else: + g = sorted(glob.glob(str(it), recursive=True)) + out += [f for f in g if Path(f).suffix.lower() in IMAGE_EXT | PDF_EXT] + return out + +def pdf_pages(path, dpi=200, pages=None): + """Render a PDF to BGR page images. Uses pypdfium2 when importable, else poppler's pdftoppm; ImportError otherwise. + pages: optional 1-based list of page numbers.""" + try: + import pypdfium2 as pdfium + doc = pdfium.PdfDocument(str(path)); idx = [p - 1 for p in pages] if pages else range(len(doc)); out = [] + for i in idx: + bm = doc[i].render(scale=dpi / 72.0); pil = bm.to_pil(); out.append(cv2.cvtColor(np.asarray(pil.convert("RGB")), cv2.COLOR_RGB2BGR)) + return out + except ImportError: pass + exe = shutil.which("pdftoppm") + if not exe: raise ImportError("PDF input needs `pip install pypdfium2` or poppler's pdftoppm on PATH") + with tempfile.TemporaryDirectory() as td: + cmd = [exe, "-r", str(dpi), "-png"] + if pages: cmd += ["-f", str(min(pages)), "-l", str(max(pages))] + subprocess.run(cmd + [str(path), os.path.join(td, "p")], check=True, capture_output=True) + files = sorted(Path(td).glob("p*.png"), key=lambda f: int("".join(ch for ch in f.stem if ch.isdigit()) or 0)) + if pages: files = [f for f in files if int("".join(ch for ch in f.stem if ch.isdigit())) in set(pages)] + return [to_bgr(str(f)) for f in files] + +def iter_page_sources(paths, dpi=200): + """Yield (label, loader) per page: images load lazily by path; PDFs are rendered page by page.""" + for p in paths: + if Path(p).suffix.lower() in PDF_EXT: + for k, im in enumerate(pdf_pages(p, dpi=dpi)): yield f"{Path(p).name}#{k + 1}", im + else: yield Path(p).name, p diff --git a/hi-en/kanen_infer/models.py b/hi-en/kanen_infer/models.py new file mode 100644 index 0000000000000000000000000000000000000000..f355088a0e1fd916a6add4c95aeaf571c5c720c8 --- /dev/null +++ b/hi-en/kanen_infer/models.py @@ -0,0 +1,65 @@ +"""Where the model files come from. + +Installed from PyPI the wheel carries code only (~200 KB), so the three ONNX models are fetched once from the Hugging Face +repository at a **pinned revision** — a package version always maps to exactly the models it was tested against — and cached +in the usual Hugging Face cache. Inside a cloned repository (models/ next to kanen.yaml) nothing is downloaded. + +Resolution order: + 1. $ARJUNA_MODELS — a directory holding models/ and kanen.yaml (air-gapped installs, custom builds) + 2. models/ next to the package root (running from a clone of the repo) + 3. the Hugging Face repo $ARJUNA_MODEL_REPO (default anandkaman/arjuna-ocr-kn-en-inference) at $ARJUNA_MODEL_REVISION (default the + package version), downloaded on first use +""" +import logging, os +from pathlib import Path + +LOG = logging.getLogger("kanen") +REPO = os.environ.get("ARJUNA_MODEL_REPO", "anandkaman/arjuna-ocr-hi-en") +# The models this code was tested against. Deliberately NOT the Python package version: a packaging fix can ship without +# moving the models, and the pipeline version stamped into every document stays the model's (VERSION / kanen.yaml). +MODEL_REVISION = "v1.1.3" # must track the release: a stale value silently fetches the WRONG models +# This package ships arjuna_hi.yaml, NOT kanen.yaml. Keying the completeness check on "kanen.yaml" made +# _has_models() return False even with the correct Hindi models/ present, so the package downloaded the +# KANNADA repo and ran kanen_rec_e5.onnx (221 Kn/En classes) on Hindi pages — plausible-looking garbage. +CONFIG_NAMES = ("arjuna_hi.yaml", "kanen.yaml") +PATTERNS = ["models/*", "models/**", "arjuna_hi.yaml", "kanen.yaml", "VERSION", "samples/pages/*"] + +def config_path(root: Path): + """the config file this package actually ships, whatever it is called""" + for n in CONFIG_NAMES: + if (root / n).exists(): return root / n + return None +_resolved = None + +def _has_models(root: Path) -> bool: + return config_path(root) is not None and (root / "models").is_dir() and any((root / "models").glob("*/*.onnx")) + +def revision() -> str: + return os.environ.get("ARJUNA_MODEL_REVISION") or MODEL_REVISION + +def resolve(pkg_root: Path, download: bool = True) -> Path: + """return the directory that holds kanen.yaml + models/, downloading it if this is a code-only install""" + global _resolved + if _resolved is not None: return _resolved + env = os.environ.get("ARJUNA_MODELS") + if env: + p = Path(env).expanduser() + if not _has_models(p): raise FileNotFoundError(f"ARJUNA_MODELS={p} does not contain kanen.yaml and models/") + _resolved = p; return p + if _has_models(pkg_root): _resolved = pkg_root; return pkg_root + if not download: return pkg_root + try: + from huggingface_hub import snapshot_download + except ImportError as e: + raise FileNotFoundError( + "model files not found next to the package and huggingface_hub is not installed.\n" + " pip install huggingface_hub (then the models download automatically), or\n" + " set ARJUNA_MODELS=/path/to/arjuna-ocr pointing at a clone of the model repository") from e + rev = revision() + LOG.warning(f"downloading the Arjuna-OCR models from {REPO}@{rev} (about 170 MB, once)") + try: + path = snapshot_download(REPO, revision=rev, allow_patterns=PATTERNS) + except Exception as e: # a package published before its tag exists, or an offline machine + LOG.warning(f"{REPO}@{rev} not available ({type(e).__name__}); falling back to main") + path = snapshot_download(REPO, revision="main", allow_patterns=PATTERNS) + _resolved = Path(path); return _resolved diff --git a/hi-en/kanen_infer/pipeline_base.py b/hi-en/kanen_infer/pipeline_base.py new file mode 100644 index 0000000000000000000000000000000000000000..692b20cc8f8cf11a0cac9c7db175581f78d34dfd --- /dev/null +++ b/hi-en/kanen_infer/pipeline_base.py @@ -0,0 +1,283 @@ +"""Detector (PP-OCRv6 det, DB post-processing) and Layout (Arjuna layout v12, DocLayout-YOLO based) models on ONNX Runtime, plus the crop helpers. +Parameters come from the package config (`kanen.yaml`); sessions come from `providers.Providers` (trt -> cuda -> cpu). +The post-processing is identical to the v1.0 reference: no thresholds, padding or ordering were changed.""" +import logging, os, shutil, time +import numpy as np, cv2, pyclipper, yaml +from shapely.geometry import Polygon +from .config import load_config, PKG_ROOT +from .providers import Providers + +LOG = logging.getLogger("kanen") +ROOT = str(PKG_ROOT) # kept for callers that import ROOT +MEAN = np.array([0.485, 0.456, 0.406], np.float32); STD = np.array([0.229, 0.224, 0.225], np.float32) + +def graph_input(path): + """(input name, is_uint8_NHWC) of an ONNX graph, read without creating a session""" + try: + import onnx + g = onnx.load(path, load_external_data=False).graph + i = g.input[0]; dims = [d.dim_value for d in i.type.tensor_type.shape.dim] + return i.name, (i.type.tensor_type.elem_type == 2 and len(dims) == 4 and dims[-1] == 3) # 2 = UINT8 + except Exception: + return "x", False + +class Detector: + def __init__(self, cfg=None, providers=None, limit=None): + # pad_bottom=0.25: Kannada below-baseline conjuncts were clipped in 88 % of lines with DB's default boxes; + # 25 % bottom extension -> 1.2 % clipped, 0 intrusion into the next line (measured on MILE, 2026-09-13). + self.cfg = cfg or load_config(); self.prov = providers or Providers(self.cfg); d = self.cfg["detector"] + self.path = str(self.cfg.path("models", "det")) + self.limit = int(limit or d.get("default_canvas", 1280)); self.thresh = float(d["thresh"]); self.box_thresh = float(d["box_thresh"]); self.unclip = float(d["unclip"]) + self.pad_top, self.pad_bottom = float(d["pad_top"]), float(d["pad_bottom"]) + # one TensorRT engine covering every canvas shape the batched path can produce (long side in `canvases`, short side + # >= min(width_classes) * smallest canvas, batch <= detector.batch): built once by prepare_engines, never rebuilt mid-run + cv = [int(c) for c in d["canvases"]]; lo = max(32, int(round(cv[0] * min(float(f) for f in d["width_classes"]) / 32)) * 32); hi = max(cv); nb = int(d.get("batch", 2)) + # the shipped graph may take normalised float NCHW ("x") or the raw uint8 canvas NHWC ("x_u8", normalisation folded + # into the graph: no float conversion on the CPU and a quarter of the bytes to upload) + self.inp, self.u8 = graph_input(self.path) + if self.u8: # NHWC uint8 canvas + self.trt_opts = {"trt_profile_min_shapes": f"{self.inp}:1x{lo}x{lo}x3", "trt_profile_opt_shapes": f"{self.inp}:{nb}x1920x1216x3", + "trt_profile_max_shapes": f"{self.inp}:{nb}x{hi}x{hi}x3"} + else: + self.trt_opts = {"trt_profile_min_shapes": f"{self.inp}:1x3x{lo}x{lo}", "trt_profile_opt_shapes": f"{self.inp}:{nb}x3x1920x1216", + "trt_profile_max_shapes": f"{self.inp}:{nb}x3x{hi}x{hi}"} + # one ORT session per canvas size: the CUDA EP re-plans (~1 s) every time the input shape changes, so the + # scale-normalised re-detects must not share a session with the default canvas. + self._sessions = {}; self.s = self._session(self.limit) + def _session(self, limit): + if limit not in self._sessions: + def _same(new, cur, _limit=limit): # probability maps must agree before an engine may replace a working session + m = min(_limit, 960); rs = np.random.RandomState(0) + x = rs.randint(0, 255, (1, m, m, 3), dtype=np.uint8) if self.u8 else rs.rand(1, 3, m, m).astype(np.float32) + try: + a = new.run(None, {new.get_inputs()[0].name: x})[0]; b = cur.run(None, {cur.get_inputs()[0].name: x})[0] + except Exception: return False + return a.shape == b.shape and float(np.abs(a - b).max()) < 0.05 + s = (self.prov.session_async(self.path, "det", validate=_same, trt_opts=self.trt_opts) + if getattr(self.prov, "background_trt", True) else self.prov.session(self.path, "det", trt_opts=self.trt_opts)) + x = (np.zeros((1, limit, limit, 3), np.uint8) if self.u8 else np.zeros((1, 3, limit, limit), np.float32)) + s.run(None, {s.get_inputs()[0].name: x}) # warm-up plan + self._sessions[limit] = s + return self._sessions[limit] + @property + def provider(self): return self.prov.status.get("det", {}).get("provider") + + def __call__(self, img_bgr): + h, w = img_bgr.shape[:2]; self.s = self._session(self.limit) + r = min(1.0, self.limit / max(h, w)) + nh, nw = max(32, int(round(h * r / 32)) * 32), max(32, int(round(w * r / 32)) * 32) + canvas = np.full((self.limit, self.limit, 3), 255, np.uint8) # fixed shape -> no ORT re-planning + canvas[:nh, :nw] = cv2.resize(img_bgr, (nw, nh)) + x = canvas[None] if self.u8 else ((canvas.astype(np.float32) / 255.0 - MEAN) / STD).transpose(2, 0, 1)[None] + prob = self.s.run(None, {self.s.get_inputs()[0].name: x})[0][0, 0][:nh, :nw] + return self._post(prob, w / nw, h / nh) + def _post(self, prob, sx, sy): + """DB post-processing: probability map -> line quads. + + Hot path on a 4-core box (20.6 ms/page before this rewrite), so it avoids Python-level geometry: + * score with cv2.mean over the contour mask instead of boolean fancy indexing (which copies) + * contour area/perimeter from cv2 (shoelace) instead of building a shapely Polygon per box + The pyclipper offset stays: expanding the fitted rectangle analytically instead was 1.8 ms faster again but moved + the boxes (mean IoU 0.96 against the previous output, 304/304 boxes changed), so it was rejected. With the two + changes above the boxes are bit-identical to the previous implementation (verified 304/304 on frozen pages) at + 3.7 ms/page instead of 5.2. + """ + bit = (prob > self.thresh).astype(np.uint8) + cnts, _ = cv2.findContours(bit, cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE) + out = [] + H, W = prob.shape + for c in cnts[:3000]: + if len(c) < 4: continue + pts = c.reshape(-1, 2) + x0, y0 = pts.min(0); x1, y1 = pts.max(0) + 1 + mask = np.zeros((y1 - y0, x1 - x0), np.uint8); cv2.fillPoly(mask, [pts - (x0, y0)], 1) + score = cv2.mean(prob[y0:y1, x0:x1], mask)[0] + if score < self.box_thresh: continue + area = cv2.contourArea(c); per = cv2.arcLength(c, True) + if area < 4: continue + d = area * self.unclip / max(per, 1e-6) + pc = pyclipper.PyclipperOffset(); pc.AddPath(pts.tolist(), pyclipper.JT_ROUND, pyclipper.ET_CLOSEDPOLYGON) + ex = pc.Execute(d) + if not ex: continue + rect = cv2.minAreaRect(np.array(ex[0], np.float32)) + if min(rect[1]) < 3: continue + box = order_pts(cv2.boxPoints(rect)) + box[:, 0] *= sx; box[:, 1] *= sy + hgt = 0.5 * ((box[3, 1] - box[0, 1]) + (box[2, 1] - box[1, 1])) + box[0, 1] -= self.pad_top * hgt; box[1, 1] -= self.pad_top * hgt; box[2, 1] += self.pad_bottom * hgt; box[3, 1] += self.pad_bottom * hgt + box[:, 1] = np.clip(box[:, 1], 0, H * sy) + out.append((box, float(score))) + return out + +def order_pts(b): + b = b[np.argsort(b[:, 0])] + l, r = b[:2][np.argsort(b[:2, 1])], b[2:][np.argsort(b[2:, 1])] + return np.array([l[0], r[0], r[1], l[1]], np.float32) + +def crop(img, box): + w = int(max(np.linalg.norm(box[0] - box[1]), np.linalg.norm(box[2] - box[3]))) + h = int(max(np.linalg.norm(box[0] - box[3]), np.linalg.norm(box[1] - box[2]))) + M = cv2.getPerspectiveTransform(box, np.array([[0, 0], [w, 0], [w, h], [0, h]], np.float32)) + return cv2.warpPerspective(img, M, (w, h), flags=cv2.INTER_CUBIC) + +class Layout: + """v15 layout model: DocLayout-YOLO (YOLOv10-based) trained on DocLayNet + our relabelled Indian pages — 15 classes, + 1024 letterbox input, top-300 selection inside the graph + (NMS-free), output (B, 300, 6) = x0 y0 x1 y1 conf cls in letterbox pixels. No reading-order head: the assembler's + xy-cut supplies the order. + TensorRT's timing-based tactic selection occasionally builds an engine that computes garbage for a correct graph + (seen for this model and for PP-DocLayoutV3 in v1.1: a fresh fp16 engine returned 2 of 7 blocks while PyTorch fp16, + ORT fp16 and rebuilt engines were exact). So every engine is self-tested against the bundled page + reference boxes + before use: fp16 -> fp16 rebuild -> fp32 TensorRT -> CUDA EP. The model itself is fp16-safe.""" + # Fallback only: the real list comes from `models.layout_labels` in the config, in model index order. + # `barcode` is the class v1.2 lacked: without it an e-stamp panel can only be called `figure`, and + # suppressing that figure deleted 119 lines of real text across 87 pages. + NAMES = ["text", "title", "section_header", "list_item", "table", "figure", "caption", "footnote", + "formula", "header", "footer", "seal", "handwriting", "id_panel", "barcode"] + SIZE = 1024 + def __init__(self, cfg=None, providers=None): + self.cfg = cfg or load_config(); self.prov = providers or Providers(self.cfg); lc = self.cfg["layout"] + self.path = str(self.cfg.path("models", "layout")) + use32 = (self.prov.cache_root / "layout_fp32" / "USE_FP32").exists() # a previous run found the fp16 engine wrong on this GPU + if getattr(self.prov, "background_trt", True) and not use32: + # start on CUDA, build fp16 then fp32 TensorRT in the background; an engine is only swapped in when it + # reproduces the reference blocks on the bundled page (the self-test that caught a wrong fp16 engine before) + self.s = self.prov.session_async(self.path, "layout", validate=lambda new, cur: self._validate_session(new), + plan=[(True, "layout"), (False, "layout_fp32")]) + else: + self.s = self.prov.session(self.path, "layout", fp16=False, cache_sub="layout_fp32") if use32 else self.prov.session(self.path, "layout") + self.inp = self.s.get_inputs()[0].name + lp = self.cfg.path("models", "layout_labels") if self.cfg["models"].get("layout_labels") else None + self.labels = [l.strip() for l in open(lp, encoding="utf-8") if l.strip()] if lp and lp.exists() else list(self.NAMES) + self.trt = "Tensorrt" in self.s.get_providers()[0]; self.ok = set(); self.bs = int(lc.get("batch", 4)) # batch the batched pipeline should use + self.thr = float(lc.get("score_thresh", 0.3)); self.selftest_min_match = float(lc.get("selftest_min_match", 0.8)) + # fill: the page's long side occupies this fraction of the canvas. The YOLOv10 head regresses at most 15 x 32 = 480 px per + # side from an anchor, so a block wider/taller than ~900 px of a 1024 canvas (whole-page tables on landscape gazettes / + # assembly records) comes out truncated and under-confident at fill 1.0; at 0.8 every page box fits (table conf 0.18 -> 0.79). + f = lc.get("fill", [1.0, 0.8]); self.fills = [float(v) for v in (f if isinstance(f, (list, tuple)) else [f])]; self.fill = self.fills[0] + st = self.cfg.path("models", "layout_selftest"); self.test_img = cv2.imread(str(st)) if st.exists() else None + rp = self.cfg.path("models", "layout_selftest_ref") if self.cfg["models"].get("layout_selftest_ref") else None + self.test_ref = __import__("json").load(open(rp, encoding="utf-8")) if rp and rp.exists() else None + self.stage = ("trt-fp32" if use32 else "trt-fp16") if self.trt else self.prov.status["layout"]["provider"] + @property + def provider(self): return self.prov.status.get("layout", {}).get("provider") + def _validate_session(self, sess): + """a candidate engine must reproduce the reference blocks on the bundled page before it may serve requests""" + if self.test_img is None or not self.test_ref: return True + try: + x1 = np.zeros((1, 3, self.SIZE, self.SIZE), np.float32); x1[0], _ = self.prep(self.test_img) + sess.run(None, {self.inp: x1}) # extends the optimisation profile to 1..bs before the check + x = np.zeros((self.bs, 3, self.SIZE, self.SIZE), np.float32); x[0] = x1[0] + got = self.parse(sess.run(None, {self.inp: x})[0][0], 1.0, thr=0.3) + except Exception: return False + def iou(a, b): + ix = max(0, min(a[2], b[2]) - max(a[0], b[0])); iy = max(0, min(a[3], b[3]) - max(a[1], b[1])); i = ix * iy + return i / max(1e-6, (a[2] - a[0]) * (a[3] - a[1]) + (b[2] - b[0]) * (b[3] - b[1]) - i) + hit = sum(1 for ref in self.test_ref if any(g["type"] == ref["type"] and iou(g["bbox"], ref["bbox"]) >= 0.7 for g in got)) + return hit / max(1, len(self.test_ref)) >= self.selftest_min_match + def prep(self, img_bgr, fill=None): + """letterbox (top-left, pad 114) to SIZE x SIZE, RGB, 0-1 -> ((3,S,S) float32, scale)""" + S = self.SIZE; h, w = img_bgr.shape[:2]; r = S * (self.fill if fill is None else fill) / max(h, w); nh, nw = max(1, int(round(h * r))), max(1, int(round(w * r))) + can = np.full((S, S, 3), 114, np.uint8); can[:nh, :nw] = cv2.resize(img_bgr, (nw, nh), interpolation=cv2.INTER_AREA) + return can[:, :, ::-1].transpose(2, 0, 1).astype(np.float32) / 255.0, r + def prep_from(self, base, img_bgr, fill): + """(canvas, scale) for `fill`, derived from the canvas already built at the largest fill""" + (arr, r0) = base; S = self.SIZE; h, w = img_bgr.shape[:2] + r = S * fill / max(h, w); nh0, nw0 = max(1, int(round(h * r0))), max(1, int(round(w * r0))) + nh, nw = max(1, int(round(h * r))), max(1, int(round(w * r))) + src = arr[:, :nh0, :nw0].transpose(1, 2, 0) # (nh0, nw0, 3) float32, already /255 + small = cv2.resize(src, (nw, nh), interpolation=cv2.INTER_AREA) + can = np.full((S, S, 3), 114 / 255.0, np.float32); can[:nh, :nw] = small + return can.transpose(2, 0, 1), r + + def run(self, x): + """raw graph call with a validated engine for this batch size; x: (bs,3,S,S) float32 -> (bs,300,6)""" + bs = x.shape[0] + if self.trt and bs not in self.ok: self.ensure(bs) + return self.s.run(None, {self.inp: x})[0] + def parse(self, out, r, thr=None): + thr = self.thr if thr is None else thr; blocks = [] + for x0, y0, x1, y1, sc, cls in out: + if not (sc >= thr) or not np.isfinite([x0, y0, x1, y1]).all() or not 0 <= int(cls) < len(self.labels): continue + blocks.append({"type": self.labels[int(cls)], "score": float(sc), "bbox": [float(x0 / r), float(y0 / r), float(x1 / r), float(y1 / r)], "order": None}) + return blocks + @staticmethod + def merge(primary, extra, page_hw, min_frac=0.7, iou_thr=0.5, covered=0.5): + """primary = boxes from the first fill (kept as they are); extra = boxes from the reduced fill(s), which exist only to + recover blocks too large for the head at full fill: an extra box is added when its long side is >= min_frac of the page's + long side, no same-class primary box overlaps it at IoU >= iou_thr, and same-class primary boxes cover < `covered` of its + area (a form page already split into sub-tables keeps the sub-tables). Geometry only.""" + L = float(max(page_hw)); out = list(primary) + def area(bb): return max(0.0, bb[2] - bb[0]) * max(0.0, bb[3] - bb[1]) + def inter(a, b): return max(0.0, min(a[2], b[2]) - max(a[0], b[0])) * max(0.0, min(a[3], b[3]) - max(a[1], b[1])) + for e in sorted(extra, key=lambda x: -x["score"]): + bb = e["bbox"] + if max(bb[2] - bb[0], bb[3] - bb[1]) < min_frac * L: continue + same = [k for k in out if k["type"] == e["type"]] + if any(inter(bb, k["bbox"]) / max(1e-6, area(bb) + area(k["bbox"]) - inter(bb, k["bbox"])) >= iou_thr for k in same): continue + if sum(inter(bb, k["bbox"]) for k in same) >= covered * area(bb): continue + out.append(e) + return out + def batch(self, imgs): + """blocks for a list of pages: every page is run at each configured fill (one canvas per fill), run at the validated + engine batch (zero-padded), and the per-fill boxes are merged by class-aware NMS.""" + if not imgs: return [] + items = [(i, f) for i in range(len(imgs)) for f in self.fills]; per = [{f: [] for f in self.fills} for _ in imgs]; BS = self.bs + prepped = {} + for j in range(0, len(items), BS): + chunk = items[j:j + BS]; k = len(chunk); x = np.zeros((BS, 3, self.SIZE, self.SIZE), np.float32); rs = [] + for n, (i, f) in enumerate(chunk): + got = prepped.get((i, f)) + if got is None: + # the reduced fills are letterboxes of the same page: resize the largest canvas down instead of + # resampling the full-resolution page again (a 2200 px page costs ~4 ms per extra pass) + base = prepped.get((i, max(self.fills))) + got = prepped[(i, f)] = self.prep_from(base, imgs[i], f) if (base is not None and f < max(self.fills)) else self.prep(imgs[i], f) + x[n], r = got; rs.append(r) + o = self.run(x) + for n, (i, f) in enumerate(chunk): per[i][f] += self.parse(o[n], rs[n]) + prepped.clear() + f0 = self.fills[0] + return [self.merge(d[f0], [b for f in self.fills[1:] for b in d[f]], imgs[i].shape[:2]) if len(self.fills) > 1 else d[f0] for i, d in enumerate(per)] + def _selftest(self, bs): + """fraction of the reference boxes (bundled page) that the engine reproduces (same class, IoU >= 0.7 at conf >= 0.3); + without a reference file: 1.0 if the engine returns >= 3 finite blocks, else 0.""" + if self.test_img is None: return 1.0 + x = np.zeros((bs, 3, self.SIZE, self.SIZE), np.float32); x[0], r = self.prep(self.test_img) + o = self.s.run(None, {self.inp: x})[0][0]; got = self.parse(o, 1.0, thr=0.3) + if not self.test_ref: return 1.0 if len(got) >= 3 else 0.0 + def iou(a, b): + ix = max(0, min(a[2], b[2]) - max(a[0], b[0])); iy = max(0, min(a[3], b[3]) - max(a[1], b[1])); i = ix * iy + return i / max(1e-6, (a[2] - a[0]) * (a[3] - a[1]) + (b[2] - b[0]) * (b[3] - b[1]) - i) + hit = sum(1 for ref in self.test_ref if any(g["type"] == ref["type"] and iou(g["bbox"], ref["bbox"]) >= 0.7 for g in got)) + return hit / max(1, len(self.test_ref)) + def ensure(self, bs): + """validate the TensorRT engine for batch `bs`. Plan: shipped/cached fp16 engine -> fp16 rebuilt with a 1..bs optimisation + profile (RTX 5060 Ti, TensorRT 10.16: the fp16 engine built for the fixed shape [bs=4] drops every wide block, the same + graph built for [1..4] is exact) -> fp32 TensorRT (own cache dir) -> CUDA EP. A fallback below fp16 is remembered in + `/layout_fp32/USE_FP32` so later runs go straight to the working engine instead of rebuilding fp16 each start.""" + st = self.prov.status["layout"]; marker = self.prov.cache_root / "layout_fp32" / "USE_FP32" + plan = [("fp16", True, "layout", False), ("fp16", True, "layout", True), ("fp32", False, "layout_fp32", False)] + start = 2 if (marker.exists() and self.stage == "trt-fp32") else 0 + for attempt in range(start, len(plan)): + tag, fp16, sub, warm = plan[attempt] + t = time.time() + if warm: self.s.run(None, {self.inp: np.zeros((1, 3, self.SIZE, self.SIZE), np.float32)}) # extend the profile to 1..bs before the test + m = self._selftest(bs) + if m >= self.selftest_min_match: + self.ok.add(bs); self.bs = bs; self.stage = f"trt-{tag}"; st["fp16"] = fp16 + if fp16 and marker.exists(): marker.unlink() + return + msg = f"[layout] TensorRT {tag} engine for batch {bs} failed its self-test ({m:.0%} of the reference blocks, attempt {attempt + 1}, {time.time() - t:.0f}s) -> rebuilding" + LOG.warning(msg); st["fallbacks"].append(msg); self.prov.events.append(msg) + self.s = None; shutil.rmtree(str(self.prov.cache_root / sub), ignore_errors=True) + if attempt + 1 < len(plan): + _, nfp16, nsub, _ = plan[attempt + 1]; self.s = self.prov.session(self.path, "layout", fp16=nfp16, cache_sub=nsub) + if not nfp16: marker.parent.mkdir(parents=True, exist_ok=True); marker.write_text("fp16 layout engine failed its self-test on this GPU; using the fp32 TensorRT engine\n") + msg = "[layout] no working TensorRT engine -> CUDA execution provider for the layout model" + LOG.warning(msg); st["fallbacks"].append(msg); self.prov.events.append(msg) + self.s = self.prov.session(self.path, "layout", force="cuda"); self.trt = False; self.bs = bs; self.ok.add(bs); self.stage = st["provider"] + def __call__(self, img_bgr, thr=None): + """single page -> (blocks, None); blocks = [{type, score, bbox, order=None}] in page pixels""" + bl = self.batch([img_bgr])[0] + return ([b for b in bl if b["score"] >= thr] if thr is not None else bl), None diff --git a/hi-en/kanen_infer/providers.py b/hi-en/kanen_infer/providers.py new file mode 100644 index 0000000000000000000000000000000000000000..a358e561cba31af19816b2239e74ae824b905732 --- /dev/null +++ b/hi-en/kanen_infer/providers.py @@ -0,0 +1,248 @@ +"""ONNX Runtime session factory with the provider policy trt -> cuda -> cpu and the engine-cache bookkeeping. + +* `Providers(cfg)` reads the policy (`auto` | `trt` | `cuda` | `cpu`), checks `trt_cache/ENGINES.json` against the local GPU / + TensorRT version (a foreign or stale cache is moved aside so engines rebuild instead of crashing) and records, per model, + which execution provider it finally ended on plus every fallback that happened (`Providers.status`). +* `session(path, key, fp16=True, force=None, trt_opts=None)` creates one `InferenceSession`; a provider that cannot be + loaded (no tensorrt wheel, no CUDA device, corrupt engine) falls through to the next one in the chain. +Nothing here touches model outputs: the same graph runs on whichever provider loads.""" +import json, logging, os, shutil, subprocess, threading, time +from pathlib import Path +import onnxruntime as ort + +LOG = logging.getLogger("kanen") +TRT = "TensorrtExecutionProvider"; CUDA = "CUDAExecutionProvider"; CPU = "CPUExecutionProvider" +OV = "OpenVINOExecutionProvider" # only present in the onnxruntime-openvino build; 1.4-1.9x over the CPU provider +SHORT = {TRT: "trt", CUDA: "cuda", OV: "openvino", CPU: "cpu"} +MANIFEST = "ENGINES.json" +_PRELOADED = False +_BUILD_LOCK = threading.Lock() # one TensorRT build at a time: concurrent builds starve the session that is serving + +def _cuda_libs_loadable(): + import ctypes, sys + names = ["cublasLt64_12.dll", "cudart64_12.dll"] if sys.platform == "win32" else ["libcublasLt.so.12", "libcudart.so.12"] + try: + for n in names: ctypes.CDLL(n) + return True + except OSError: return False + +def preload_cuda_libs(): + """Make the CUDA 12 / cuDNN 9 runtime libraries visible to onnxruntime-gpu. Order: already on the loader path -> + `onnxruntime.preload_dlls()` (nvidia-cublas-cu12 / nvidia-cudnn-cu12 ... pip wheels) -> importing torch (whose wheel + bundles the same libraries). Without any of these onnxruntime silently falls back to the CPU provider.""" + global _PRELOADED + if _PRELOADED: return + _PRELOADED = True + if _cuda_libs_loadable(): return + try: + if hasattr(ort, "preload_dlls"): + import contextlib, io + with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()): ort.preload_dlls() + except Exception as e: LOG.debug(f"preload_dlls: {e}") + if _cuda_libs_loadable(): return + try: + import torch # noqa: F401 + LOG.info("CUDA runtime libraries loaded through torch") + except Exception: + LOG.warning("CUDA 12 runtime libraries not found (install nvidia-cublas-cu12/nvidia-cudnn-cu12 or torch, or put CUDA on PATH): GPU providers will fall back to CPU") + +def gpu_name(): + try: + out = subprocess.run(["nvidia-smi", "--query-gpu=name", "--format=csv,noheader"], capture_output=True, text=True, timeout=10) + names = [l.strip() for l in out.stdout.splitlines() if l.strip()] + return names[0] if names else None + except Exception: return None + +def trt_version(): + try: + import tensorrt; return tensorrt.__version__ + except Exception: return None + +def md5_file(path, n=1 << 20): + import hashlib; h = hashlib.md5() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(n), b""): h.update(chunk) + return h.hexdigest()[:12] + +class SwapSession: + """An InferenceSession that can be replaced at runtime. Used for the background TensorRT build: the pipeline starts on + the CUDA provider immediately (no 1-4 minute wait on an unseen GPU) and this object swaps in the TensorRT session once + it has been built AND validated against the session it replaces. Calls in flight keep using the old session.""" + def __init__(self, session, key, on_swap=None): + self._s = session; self.key = key; self._lock = threading.Lock(); self.swapped = False; self.on_swap = on_swap + @property + def session(self): + with self._lock: return self._s + def swap(self, new): + with self._lock: self._s, self.swapped = new, True + if self.on_swap: self.on_swap(new) + def run(self, *a, **k): return self.session.run(*a, **k) + def get_inputs(self): return self.session.get_inputs() + def get_outputs(self): return self.session.get_outputs() + def get_providers(self): return self.session.get_providers() + def __getattr__(self, name): return getattr(self.__dict__["_s"], name) + +class Providers: + def __init__(self, cfg): + self.cfg = cfg; p = cfg["provider"] + self.policy = str(p.get("policy", "auto")).lower(); self.fp16 = bool(p.get("trt_fp16", True)) + # background_trt: serve on CUDA at once and swap in TensorRT when it is built AND validated (set 0 to build up front) + self.background_trt = os.environ.get("KANEN_BACKGROUND_TRT", str(int(bool(p.get("background_trt", True))))) not in ("0", "false", "False") + self.cache_root = cfg.path("provider", "engine_cache") + if self.policy != "cpu": preload_cuda_libs() + self.available = ort.get_available_providers() + self.status = {} # model key -> dict(requested, provider, fp16, fallbacks[], cache, load_s) + self.events = [] # human-readable fallback log + self.gpu = gpu_name() if CUDA in self.available else None; self.trt_ver = trt_version() + if self.policy in ("auto", "trt"): self._check_manifest() + + # ---- engine cache manifest ------------------------------------------------------------------------------------- + def _check_manifest(self): + """Engines are only valid for the GPU architecture + TensorRT version they were built with. A cache built elsewhere + is moved to trt_cache/stale_/ so the first run rebuilds instead of failing on a foreign engine.""" + mf = self.cache_root / MANIFEST + if not mf.exists() or self.trt_ver is None: return + try: m = json.loads(mf.read_text(encoding="utf-8")) + except Exception: return + same_gpu = (not self.gpu) or (not m.get("gpu")) or (m.get("gpu") == self.gpu) + same_trt = (not m.get("tensorrt")) or (m.get("tensorrt") == self.trt_ver) + same_ort = (not m.get("onnxruntime")) or (m.get("onnxruntime") == ort.__version__) # ORT upgrades change the graph hash -> engines miss + if same_gpu and same_trt and same_ort: return + dst = self.cache_root / f"stale_{time.strftime('%Y%m%d_%H%M%S')}"; dst.mkdir(parents=True, exist_ok=True) + for sub in self.cache_root.iterdir(): + if sub.is_dir() and not sub.name.startswith("stale_"): shutil.move(str(sub), str(dst / sub.name)) + shutil.move(str(mf), str(dst / MANIFEST)) + msg = f"engine cache was built for {m.get('gpu')} / TensorRT {m.get('tensorrt')}, this machine has {self.gpu} / {self.trt_ver}: moved to {dst.name}, engines rebuild on first use (~4 min)" + LOG.warning(msg); self.events.append(msg) + + def write_manifest(self, extra=None): + """Called by prepare_engines after a validated build: per-model status + md5 of every engine file.""" + engines = {} + for sub in sorted(self.cache_root.iterdir()) if self.cache_root.exists() else []: + if sub.is_dir() and not sub.name.startswith("stale_"): + engines[sub.name] = {f.name: {"md5": md5_file(f), "bytes": f.stat().st_size} for f in sorted(sub.iterdir()) if f.suffix in (".engine", ".profile", ".timing")} + models = {k: dict(v, cache=os.path.relpath(v["cache"], self.cache_root) if v.get("cache") else None) for k, v in self.status.items()} # relative: the manifest ships with the package + m = {"package": self.cfg.get("name"), "version": self.cfg.get("version"), "gpu": self.gpu, "tensorrt": self.trt_ver, "onnxruntime": ort.__version__, + "built": time.strftime("%Y-%m-%d %H:%M:%S"), "models": models, "engines": engines} + if extra: m.update(extra) + self.cache_root.mkdir(parents=True, exist_ok=True); (self.cache_root / MANIFEST).write_text(json.dumps(m, indent=1), encoding="utf-8") + return m + + # ---- sessions ---------------------------------------------------------------------------------------------------- + def chain(self, force=None): + pol = (force or self.policy) + ov = ["openvino"] if OV in self.available else [] + order = {"auto": ["trt", "cuda"] + ov + ["cpu"], "trt": ["trt", "cuda"] + ov + ["cpu"], "cuda": ["cuda"] + ov + ["cpu"], + "openvino": ov + ["cpu"], "cpu": ["cpu"]}.get(pol) + if order is None: raise ValueError(f"unknown provider policy {pol!r} (auto|trt|cuda|openvino|cpu)") + return order + + def _opts(self, kind, cache, fp16, trt_opts): + p = self.cfg["provider"] + cuda = (CUDA, {"cudnn_conv_algo_search": p.get("cudnn_conv_algo_search", "HEURISTIC"), "arena_extend_strategy": "kSameAsRequested", + "gpu_mem_limit": int(float(p.get("cuda_mem_limit_gb", 6)) * 1024 ** 3)}) + if kind == "cpu": return [CPU] + if kind == "openvino": + o = {"device_type": str(p.get("openvino_device", "CPU")).upper()} + n = self.cfg["runtime"].get("workers") + if isinstance(n, int) and n > 0: o["num_of_threads"] = int(n) + return [(OV, o), CPU] + if kind == "cuda": return [cuda] + import tensorrt # noqa: F401 (loads libnvinfer / nvinfer_10.dll from the pip wheel so the ORT provider can find it) + cache.mkdir(parents=True, exist_ok=True) + o = {"trt_fp16_enable": bool(fp16), "trt_engine_cache_enable": True, "trt_engine_cache_path": str(cache), "trt_timing_cache_enable": True, + "trt_timing_cache_path": str(cache), "trt_max_workspace_size": int(float(p.get("trt_workspace_gb", 4)) * 1024 ** 3)} + if trt_opts: o.update(trt_opts) + return [(TRT, o), cuda] + + def session(self, path, key, fp16=None, force=None, trt_opts=None, cache_sub=None): + """key: model name for the status table ('det', 'layout', 'rec'); cache_sub: engine sub-directory (default = key).""" + fp16 = self.fp16 if fp16 is None else fp16; cache = self.cache_root / (cache_sub or key) + so = ort.SessionOptions(); so.log_severity_level = 3 + st = self.status.setdefault(key, {"requested": (force or self.policy), "provider": None, "fp16": fp16, "fallbacks": [], "cache": str(cache), "load_s": 0.0}) + t0 = time.time(); last = None + for kind in self.chain(force): + if kind == "cuda" and CUDA not in self.available: st["fallbacks"].append("cuda: CUDAExecutionProvider not available in this onnxruntime build"); continue + if kind == "trt" and TRT not in self.available: st["fallbacks"].append("trt: TensorrtExecutionProvider not available in this onnxruntime build"); continue + if kind == "openvino" and OV not in self.available: continue + for attempt in (1, 2): + try: + s = ort.InferenceSession(str(path), so, providers=self._opts(kind, cache, fp16, trt_opts)) + got = s.get_providers()[0]; want = {"trt": TRT, "cuda": CUDA, "openvino": OV, "cpu": CPU}[kind] + if got != want: # ORT loads silently on the next provider when a library is missing: treat as a failed attempt + msg = f"{key}: {kind} requested but onnxruntime created the session on {SHORT.get(got, got)} (provider library not loadable?)" + LOG.warning(msg); st["fallbacks"].append(msg); self.events.append(msg) + if kind != self.chain(force)[-1]: break + st["provider"] = SHORT.get(got, got); st["fp16"] = fp16 if got == TRT else False; st["load_s"] = round(time.time() - t0, 1) + return s + except Exception as e: # corrupt / foreign engine: clear this model's cache once and retry, then fall through + last = e; text = str(e); msg = f"{key}: {kind} provider failed ({type(e).__name__}: {text.splitlines()[0][:160]})" + LOG.warning(msg); st["fallbacks"].append(msg); self.events.append(msg) + if kind == "trt" and attempt == 1 and cache.exists() and any(w in text.lower() for w in ("engine", "deserializ", "cache")): + stale = cache.parent / f"stale_{time.strftime('%Y%m%d_%H%M%S')}_{cache.name}"; shutil.move(str(cache), str(stale)) + self.events.append(f"{key}: engine cache moved to {stale.name}, rebuilding"); continue + break + raise RuntimeError(f"no execution provider could load {path}: {last}") + + def session_async(self, path, key, validate=None, fp16=None, trt_opts=None, cache_sub=None, plan=None): + """Return a session that works NOW and becomes the TensorRT one later. + If TensorRT is not in the chain (policy cuda/cpu, no wheel, no GPU) this is just `session(...)`. + Otherwise: create the CUDA (or CPU) session, hand it back wrapped in SwapSession, and build TensorRT on a worker + thread. `validate(trt_session, current_session) -> bool` decides whether the new engine may be swapped in; a + failing engine is dropped and the model keeps running on what it had.""" + if "trt" not in self.chain() or TRT not in self.available: + return self.session(path, key, fp16=fp16, trt_opts=trt_opts, cache_sub=cache_sub) + cache = self.cache_root / (cache_sub or key) + if cache.exists() and any(cache.glob("*.engine")): + # engines for this model are already built for this GPU/TensorRT (the manifest check has already moved a + # foreign cache aside): loading them takes a moment, so there is nothing to hide behind a background build + return self.session(path, key, fp16=fp16, trt_opts=trt_opts, cache_sub=cache_sub) + if getattr(self, "_optimize_requested", False): + # optimize() was already called: a session created later (e.g. the detector's second canvas size) should build + # its engine now rather than queue a build nobody will start + return self.session(path, key, fp16=fp16, trt_opts=trt_opts, cache_sub=cache_sub) + base = self.session(path, key, force="cuda", fp16=False, cache_sub=cache_sub) + st = self.status[key]; st["background_trt"] = "building" + holder = SwapSession(base, key) + attempts = plan or [(self.fp16 if fp16 is None else fp16, cache_sub or key)] + def build(): + t0 = time.time() + with _BUILD_LOCK: + for i, (use_fp16, sub) in enumerate(attempts): + tag = f"fp16" if use_fp16 else "fp32" + try: + trt = self.session(path, f"{key}_trt{i}", fp16=use_fp16, trt_opts=trt_opts, cache_sub=sub) + if self.status[f"{key}_trt{i}"]["provider"] != "trt": + raise RuntimeError("build did not end on the TensorRT provider") + if validate is not None and not validate(trt, holder.session): + msg = f"[{key}] TensorRT {tag} engine failed validation after {time.time()-t0:.0f}s" + LOG.warning(msg); self.events.append(msg); st["background_trt"] = f"rejected ({tag})"; continue + holder.swap(trt); st["provider"] = "trt"; st["fp16"] = use_fp16 + st["background_trt"] = f"swapped to {tag} after {time.time()-t0:.0f}s" + LOG.info(f"[{key}] TensorRT {tag} engine ready and validated after {time.time()-t0:.0f}s"); return + except Exception as e: + msg = f"[{key}] TensorRT {tag} build failed ({type(e).__name__}: {str(e).splitlines()[0][:140]})" + LOG.warning(msg); self.events.append(msg); st["background_trt"] = f"failed ({tag})" + self.events.append(f"[{key}] no TensorRT engine passed: staying on {st['provider']}") + self._pending = getattr(self, "_pending", []); self._pending.append((key, build)) + st["background_trt"] = "queued" + return holder + + def start_background_builds(self): + self._optimize_requested = True + """Kick off the queued TensorRT builds. Called by the long-lived entry points (server, endpoint handler, batch + runs) — not by one-shot CLI calls, where a 1-2 minute build would cost more than it saves. Idempotent.""" + self.builds = getattr(self, "builds", []) + for key, fn in getattr(self, "_pending", []): + self.status[key]["background_trt"] = "building" + th = threading.Thread(target=fn, name=f"trt-build-{key}", daemon=True); th.start(); self.builds.append(th) + self._pending = [] + return len(self.builds) + + def wait_for_engines(self, timeout=None): + """block until the background TensorRT builds finish (used by `arjuna bench` and the Docker image build)""" + for th in getattr(self, "builds", []): th.join(timeout) + return {k: v.get("background_trt") for k, v in self.status.items() if "background_trt" in v} + + def summary(self): + return {k: {"provider": v["provider"], "fp16": v["fp16"], "fallbacks": list(v["fallbacks"])} for k, v in self.status.items()} diff --git a/hi-en/kanen_infer/rec_onnx.py b/hi-en/kanen_infer/rec_onnx.py new file mode 100644 index 0000000000000000000000000000000000000000..850e1e82015e06ef0233ba6e7c1ef576f1a9c381 --- /dev/null +++ b/hi-en/kanen_infer/rec_onnx.py @@ -0,0 +1,75 @@ +import os +"""KanEn recognizer, ONNX Runtime only (TensorRT fp16 / CUDA / CPU by the provider policy). Greedy CTC decode with the frozen charset. +Same batching rule as the training-side recognizer: every crop is padded only to its own width bucket (padding a short crop +to a much wider batch width makes the model double the first glyph).""" +import hashlib, json, math +import numpy as np, cv2 +from .config import load_config +from .providers import Providers, md5_file +import logging +LOG = logging.getLogger("kanen") + +class Conf(float): + """line confidence (mean max-prob over emitted characters) carrying `min_char` (weakest emitted character)""" + min_char = None + +class KanEnRecognizer: + def __init__(self, onnx_path=None, buckets=None, cfg=None, providers=None): + self.cfg = cfg or load_config(); self.prov = providers or Providers(self.cfg); rc = self.cfg["recognizer"] + onnx_path = str(onnx_path or self.cfg.path("models", "rec")); self.path = onnx_path + self.buckets = tuple(int(b) for b in (buckets or rc["buckets"])); self.H = int(rc.get("height", 32)); self.MAX_RATIO = float(rc.get("max_ratio", 40)); self.bs = int(rc.get("batch", 64)) + self.chars = json.load(open(self.cfg.path("models", "charset"), encoding="utf-8")) + self.onnx_md5 = md5_file(onnx_path); self.charset_md5 = hashlib.md5("".join(self.chars).encode("utf-8")).hexdigest()[:12] + trt_opts = {"trt_profile_min_shapes": f"image:1x3x{self.H}x{self.buckets[0]}", "trt_profile_max_shapes": f"image:64x3x{self.H}x{self.buckets[-1]}", "trt_profile_opt_shapes": f"image:48x3x{self.H}x256"} + def _same(new, cur): # the swapped engine must decode a probe crop to the same characters as the session it replaces + x = np.full((2, 3, self.H, 256), 0.5, np.float32) + try: + a = new.run(None, {new.get_inputs()[0].name: x})[0]; b = cur.run(None, {cur.get_inputs()[0].name: x})[0] + except Exception: return False + return a.shape == b.shape and bool((a.argmax(-1) == b.argmax(-1)).all()) + self.s = (self.prov.session_async(onnx_path, "rec", validate=_same, trt_opts=trt_opts) + if getattr(self.prov, "background_trt", True) else self.prov.session(onnx_path, "rec", trt_opts=trt_opts)) + self.name = "kanen-rec-" + ("trt" if "Tensorrt" in self.s.get_providers()[0] else "ort") + @property + def provider(self): return self.prov.status.get("rec", {}).get("provider") + + def _provider(self): + try: return self.s.get_providers()[0] + except Exception: return "" + def __call__(self, crops, bs=None): + if not crops: return [] + bs = bs or self.bs; H = self.H + # the CUDA provider runs this graph in fp32 and needs far more memory per item than the fp16 TensorRT engine; + # while we are on CUDA (e.g. during the background engine build) cap the batch so a large configured batch + # cannot exhaust the arena. The cap lifts by itself once the TensorRT engine is swapped in. + if "Tensorrt" not in self._provider(): bs = min(bs, int(os.environ.get("KANEN_CUDA_REC_BATCH", "16"))) + gray = [c if c.ndim == 2 else cv2.cvtColor(c, cv2.COLOR_BGR2GRAY) for c in crops] + ratios = [min(g.shape[1] / max(g.shape[0], 1), self.MAX_RATIO) for g in gray] + bucket_of = lambda i: next((b for b in self.buckets if b >= int(math.ceil(ratios[i] * H / 4) * 4)), self.buckets[-1]) + groups = {} + for i in range(len(gray)): groups.setdefault(bucket_of(i), []).append(i) + out = [None] * len(gray) + for W, members in sorted(groups.items()): + n = max(4, min(bs, int(bs * 320 // max(W, 64)))) + for i in range(0, len(members), n): + idx = members[i:i + n]; x = np.ones((len(idx), 3, H, W), np.float32) + for k, j in enumerate(idx): + g = gray[j]; nw = min(W, max(8, int(round(g.shape[1] * H / max(g.shape[0], 1))))) + r = cv2.resize(g, (nw, H), interpolation=cv2.INTER_AREA if g.shape[0] > H else cv2.INTER_CUBIC).astype(np.float32) / 255.0 + x[k, :, :, :nw] = (r - 0.5) / 0.5 + try: + probs = self.s.run(None, {"image": x})[0] # (B, T, C) softmax + except Exception as e: # out of memory on this device: halve the batch and keep going (never fail a page) + if "memory" not in str(e).lower() and "alloc" not in str(e).lower(): raise + half = max(1, len(idx) // 2); LOG.warning(f"recognizer: {type(e).__name__} at batch {len(idx)} (width {W}), retrying at {half}") + probs = np.concatenate([self.s.run(None, {"image": x[k:k + half]})[0] for k in range(0, len(idx), half)], axis=0) + self.bs = max(4, min(self.bs, half * 2)) # remember the smaller batch for the rest of this run + ids = probs.argmax(-1); pmax = probs.max(-1) + for k, j in enumerate(idx): + row = ids[k]; keep = (row != 0) & (row != np.roll(row, 1)); s, prev = [], 0 + for t in row: + if t != prev and t != 0: s.append(self.chars[t - 1]) + prev = t + c = Conf(float(pmax[k][keep].mean()) if keep.any() else 0.0); c.min_char = float(pmax[k][keep].min()) if keep.any() else 0.0 + out[j] = ("".join(s), c) + return out diff --git a/hi-en/models/det/inference.onnx b/hi-en/models/det/inference.onnx new file mode 100644 index 0000000000000000000000000000000000000000..06dd37303719c268bc3fdf099078ae69109c9941 --- /dev/null +++ b/hi-en/models/det/inference.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e4ef16c346d19d8aa16f3a9f7e42f20124063335c427591ea4518b076ad212a6 +size 9870857 diff --git a/hi-en/models/layout/LICENSE-AGPL-3.0.txt b/hi-en/models/layout/LICENSE-AGPL-3.0.txt new file mode 100644 index 0000000000000000000000000000000000000000..be3f7b28e564e7dd05eaf59d64adba1a4065ac0e --- /dev/null +++ b/hi-en/models/layout/LICENSE-AGPL-3.0.txt @@ -0,0 +1,661 @@ + GNU AFFERO GENERAL PUBLIC LICENSE + Version 3, 19 November 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU Affero General Public License is a free, copyleft license for +software and other kinds of works, specifically designed to ensure +cooperation with the community in the case of network server software. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +our General Public Licenses are intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + Developers that use our General Public Licenses protect your rights +with two steps: (1) assert copyright on the software, and (2) offer +you this License which gives you legal permission to copy, distribute +and/or modify the software. + + A secondary benefit of defending all users' freedom is that +improvements made in alternate versions of the program, if they +receive widespread use, become available for other developers to +incorporate. Many developers of free software are heartened and +encouraged by the resulting cooperation. However, in the case of +software used on network servers, this result may fail to come about. +The GNU General Public License permits making a modified version and +letting the public access it on a server without ever releasing its +source code to the public. + + The GNU Affero General Public License is designed specifically to +ensure that, in such cases, the modified source code becomes available +to the community. It requires the operator of a network server to +provide the source code of the modified version running there to the +users of that server. Therefore, public use of a modified version, on +a publicly accessible server, gives the public access to the source +code of the modified version. + + An older license, called the Affero General Public License and +published by Affero, was designed to accomplish similar goals. This is +a different license, not a version of the Affero GPL, but Affero has +released a new version of the Affero GPL which permits relicensing under +this license. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU Affero General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Remote Network Interaction; Use with the GNU General Public License. + + Notwithstanding any other provision of this License, if you modify the +Program, your modified version must prominently offer all users +interacting with it remotely through a computer network (if your version +supports such interaction) an opportunity to receive the Corresponding +Source of your version by providing access to the Corresponding Source +from a network server at no charge, through some standard or customary +means of facilitating copying of software. This Corresponding Source +shall include the Corresponding Source for any work covered by version 3 +of the GNU General Public License that is incorporated pursuant to the +following paragraph. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the work with which it is combined will remain governed by version +3 of the GNU General Public License. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU Affero General Public License from time to time. Such new versions +will be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU Affero General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU Affero General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU Affero General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If your software can interact with users remotely through a computer +network, you should also make sure that it provides a way for users to +get its source. For example, if your program is a web application, its +interface could display a "Source" link that leads users to an archive +of the code. There are many ways you could offer source, and different +solutions will be better for different programs; see section 13 for the +specific requirements. + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU AGPL, see +. diff --git a/hi-en/models/layout/NOTICE.md b/hi-en/models/layout/NOTICE.md new file mode 100644 index 0000000000000000000000000000000000000000..62343275b80ea066ab928b9783d79e55e48da95c --- /dev/null +++ b/hi-en/models/layout/NOTICE.md @@ -0,0 +1,15 @@ +# Layout model licence — every `*_layout_*.onnx` in this directory + +This package ships `arjuna_layout_v15.onnx` and `kanen_layout_v14.onnx`. **All of them** are DocLayout-YOLO / YOLOv10-based detectors fine-tuned from the +`juliozhao/DocLayout-YOLO-DocStructBench` weights (Apache-2.0) with the `doclayout_yolo` training toolkit, which is +licensed under the **GNU Affero General Public License v3.0** (Ultralytics YOLO lineage). To respect that lineage the +layout model files in this directory are distributed under **AGPL-3.0** (full text: `LICENSE-AGPL-3.0.txt`). + +Everything else in this repository — the inference package `kanen_infer`, the recognizer (SVTRv2 + language pack), +the text detector (PP-OCRv6) and the documentation — is Apache-2.0 as stated in the top-level `LICENSE`. + +Practical consequences: +- Using the package with any of these layout models in a networked service falls under AGPL-3.0 for the layout model. If that + does not suit you, run with `layout.mode: never` in `kanen.yaml` (text lines and tables from ruled grids still work + without the layout model) or plug in another layout model of your choice (`models.layout`, `models.layout_labels`). +- No AGPL toolkit code is included here; inference runs on ONNX Runtime. diff --git a/hi-en/models/layout/arjuna_layout_v15.onnx b/hi-en/models/layout/arjuna_layout_v15.onnx new file mode 100644 index 0000000000000000000000000000000000000000..336428f397f7e399969b0ef95dd56b646c3ab242 --- /dev/null +++ b/hi-en/models/layout/arjuna_layout_v15.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:188ac04201f37cf2a78c47f2793eb2d9ed443379c9b2eb20f26a37ea5a8ecff5 +size 74846629 diff --git a/hi-en/models/layout/classes.txt b/hi-en/models/layout/classes.txt new file mode 100644 index 0000000000000000000000000000000000000000..bc9605265a7477afe2a7b963fe19c6fbde410e61 --- /dev/null +++ b/hi-en/models/layout/classes.txt @@ -0,0 +1,9 @@ +title +text +table +figure +seal +id_panel +header +footer +handwriting diff --git a/hi-en/models/layout/classes_v15.txt b/hi-en/models/layout/classes_v15.txt new file mode 100644 index 0000000000000000000000000000000000000000..f3b79c81a584267eb89562a31ade92b73bf26d03 --- /dev/null +++ b/hi-en/models/layout/classes_v15.txt @@ -0,0 +1,15 @@ +text +title +section_header +list_item +table +figure +caption +footnote +formula +header +footer +seal +handwriting +id_panel +barcode diff --git a/hi-en/models/layout/kanen_layout_v14.onnx b/hi-en/models/layout/kanen_layout_v14.onnx new file mode 100644 index 0000000000000000000000000000000000000000..ca4f7fd922fc0df2b810d002d5863189501982df --- /dev/null +++ b/hi-en/models/layout/kanen_layout_v14.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b1f23ea7ca7f9cc3d237afe0e65d6dd0302e9e9c213a6a1d9fc32ac0691a67a +size 74832733 diff --git a/hi-en/models/layout/selftest.jpg b/hi-en/models/layout/selftest.jpg new file mode 100644 index 0000000000000000000000000000000000000000..01f4fc0c56ea3c12f2d0c25de945389504e8c726 --- /dev/null +++ b/hi-en/models/layout/selftest.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b225479e38d2da866bf7d1fd1519892a132118d65538b39e01ff316d1be4653e +size 114880 diff --git a/hi-en/models/layout/selftest_ref_v14.json b/hi-en/models/layout/selftest_ref_v14.json new file mode 100644 index 0000000000000000000000000000000000000000..8929a2d8bc21bc62a2b19e9f996d6bb025f9d6ee --- /dev/null +++ b/hi-en/models/layout/selftest_ref_v14.json @@ -0,0 +1,72 @@ +[ + { + "type": "title", + "score": 0.968, + "bbox": [ + 486.0, + 213.5, + 537.5, + 229.7 + ] + }, + { + "type": "text", + "score": 0.95, + "bbox": [ + 80.0, + 178.0, + 974.2, + 210.7 + ] + }, + { + "type": "header", + "score": 0.908, + "bbox": [ + 899.8, + 104.4, + 939.9, + 118.1 + ] + }, + { + "type": "header", + "score": 0.901, + "bbox": [ + 83.1, + 103.6, + 123.0, + 116.7 + ] + }, + { + "type": "table", + "score": 0.893, + "bbox": [ + 104.6, + 228.5, + 925.4, + 680.4 + ] + }, + { + "type": "text", + "score": 0.874, + "bbox": [ + 89.2, + 132.9, + 962.2, + 162.4 + ] + }, + { + "type": "header", + "score": 0.857, + "bbox": [ + 378.8, + 103.7, + 620.2, + 121.3 + ] + } +] \ No newline at end of file diff --git a/hi-en/models/layout/selftest_ref_v15.json b/hi-en/models/layout/selftest_ref_v15.json new file mode 100644 index 0000000000000000000000000000000000000000..307d2b4c02085bd6476e58b6675883b09a124324 --- /dev/null +++ b/hi-en/models/layout/selftest_ref_v15.json @@ -0,0 +1,72 @@ +[ + { + "type": "text", + "score": 0.966, + "bbox": [ + 84.7, + 126.9, + 935.3, + 164.9 + ] + }, + { + "type": "header", + "score": 0.965, + "bbox": [ + 382.2, + 103.7, + 620.8, + 120.9 + ] + }, + { + "type": "section_header", + "score": 0.964, + "bbox": [ + 486.0, + 213.8, + 537.0, + 230.0 + ] + }, + { + "type": "text", + "score": 0.964, + "bbox": [ + 83.0, + 169.8, + 937.2, + 206.1 + ] + }, + { + "type": "header", + "score": 0.961, + "bbox": [ + 83.5, + 104.2, + 122.9, + 117.0 + ] + }, + { + "type": "header", + "score": 0.96, + "bbox": [ + 899.2, + 104.6, + 940.5, + 118.3 + ] + }, + { + "type": "table", + "score": 0.958, + "bbox": [ + 75.3, + 229.1, + 946.5, + 683.3 + ] + } +] \ No newline at end of file diff --git a/hi-en/models/rec/arjuna_rec_hi.onnx b/hi-en/models/rec/arjuna_rec_hi.onnx new file mode 100644 index 0000000000000000000000000000000000000000..6fb2131522065970087c6aa4dd4881e71af8f94a --- /dev/null +++ b/hi-en/models/rec/arjuna_rec_hi.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dea1c19158f4931c3f014602e331c92b53138e2a58fd465699e92dcacda6d842 +size 81293562 diff --git a/hi-en/models/rec/charset.json b/hi-en/models/rec/charset.json new file mode 100644 index 0000000000000000000000000000000000000000..504539332812945ddf30c903a75f467519020ae7 --- /dev/null +++ b/hi-en/models/rec/charset.json @@ -0,0 +1 @@ +[" ", "!", "\"", "#", "$", "%", "&", "'", "(", ")", "*", "+", ",", "-", ".", "/", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", ":", ";", "<", "=", ">", "?", "@", "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z", "[", "\\", "]", "^", "_", "`", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z", "{", "|", "}", "~", "ऀ", "ँ", "ं", "ः", "ऄ", "अ", "आ", "इ", "ई", "उ", "ऊ", "ऋ", "ऌ", "ऍ", "ऎ", "ए", "ऐ", "ऑ", "ऒ", "ओ", "औ", "क", "ख", "ग", "घ", "ङ", "च", "छ", "ज", "झ", "ञ", "ट", "ठ", "ड", "ढ", "ण", "त", "थ", "द", "ध", "न", "ऩ", "प", "फ", "ब", "भ", "म", "य", "र", "ऱ", "ल", "ळ", "ऴ", "व", "श", "ष", "स", "ह", "ऺ", "ऻ", "़", "ऽ", "ा", "ि", "ी", "ु", "ू", "ृ", "ॄ", "ॅ", "ॆ", "े", "ै", "ॉ", "ॊ", "ो", "ौ", "्", "ॎ", "ॏ", "ॐ", "॑", "॒", "॓", "॔", "ॕ", "ॖ", "ॗ", "क़", "ख़", "ग़", "ज़", "ड़", "ढ़", "फ़", "य़", "ॠ", "ॡ", "ॢ", "ॣ", "।", "॥", "०", "१", "२", "३", "४", "५", "६", "७", "८", "९", "॰", "ॱ", "ॲ", "ॳ", "ॴ", "ॵ", "ॶ", "ॷ", "ॸ", "ॹ", "ॺ", "ॻ", "ॼ", "ॽ", "ॾ", "ॿ", "–", "—", "‘", "’", "“", "”", "„", "«", "»", "‹", "›", "•", "·", "…", "‑", "‐", "−", "₹", "€", "£", "¥", "¢", "₨", "©", "®", "™", "§", "¶", "†", "‡", "№", "℅", "‰", "µ", "¤", "°", "′", "″", "½", "¼", "¾", "⅓", "⅔", "⅛", "¹", "²", "³", "⁴", "⁵", "⁶", "⁷", "⁸", "⁹", "⁰", "⁻", "∑", "∫", "√", "≤", "≥", "≠", "±", "×", "÷", "∞", "∂", "∆", "∇", "≈", "≡", "∈", "∉", "⊂", "∪", "∩", "π", "α", "β", "γ", "θ", "λ", "μ", "σ", "Ω", "☐", "☑", "☒", "✓", "✗", "✘", "★", "☆", "●", "○", "■", "□", "◆", "▪", "▶", "→", "←", "↑", "↓", "⇒"] \ No newline at end of file diff --git a/kn-en/LICENSE b/kn-en/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..cc6c2a072e61b005fc26b60f4acda3b74bf6e1ee --- /dev/null +++ b/kn-en/LICENSE @@ -0,0 +1,3 @@ +Apache License 2.0 — see https://www.apache.org/licenses/LICENSE-2.0 + +Copyright 2026 Chethan (anandkaman). Base models: PaddleOCR (Apache-2.0), OpenOCR SVTRv2 (Apache-2.0). diff --git a/kn-en/VERSION b/kn-en/VERSION new file mode 100644 index 0000000000000000000000000000000000000000..bf6d67a5dd01e1e7443f7a3225f205fd1b51b26b --- /dev/null +++ b/kn-en/VERSION @@ -0,0 +1 @@ +1.2.0 diff --git a/kn-en/kanen.yaml b/kn-en/kanen.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8626a7ef3d8680f2091da92d0bb469db55abfc53 --- /dev/null +++ b/kn-en/kanen.yaml @@ -0,0 +1,58 @@ +# Arjuna-OCR [Kn&En] v1.2 — inference configuration. +# All paths are relative to the package root (the directory holding this file) unless absolute. +# Environment overrides (kept for compatibility): KANEN_TRT=1 -> provider.policy=trt, KANEN_ORT_CPU=1 -> cpu, +# KANEN_REC_ONNX= -> models.rec, KANEN_CONFIG= -> use another config file, KANEN_LOG=. +name: "Arjuna-OCR [Kn&En]" # product name used in banners / hOCR / ALTO metadata (rename here only) +version: 1.2.0 # package version; stamped into every output document as model.pipeline + +models: + det: models/det/inference.onnx # PP-OCRv6 text detector (small), DB head + layout: models/layout/kanen_layout_v12.onnx # Arjuna layout v12 (DocLayout-YOLO based, 9 classes), 1024 letterbox, (B,300,6) output + layout_labels: models/layout/classes.txt # one class name per line, in model index order + layout_selftest: models/layout/selftest.jpg # public page used to self-test every layout TensorRT engine ... + layout_selftest_ref: models/layout/selftest_ref.json # ... against these reference blocks (CPU ONNX Runtime output, conf >= 0.45) + rec: models/rec/kanen_rec_e5.onnx # SVTRv2-B + Kannada/English pack (E5 = v1.2), CTC, 221 classes + charset: models/rec/charset.json + +provider: + policy: auto # auto = trt -> cuda -> cpu (first one that loads) | trt | cuda | cpu + trt_fp16: true # fp16 TensorRT engines (layout: fp16 -> rebuilt fp16 -> fp32 TensorRT -> CUDA if its self-test fails) + engine_cache: trt_cache # TensorRT engine + timing caches, one sub-directory per model + trt_workspace_gb: 4 + cuda_mem_limit_gb: 6 + cudnn_conv_algo_search: HEURISTIC + +detector: + canvases: [1280, 1920, 2560] # fixed canvas long sides; the batched path picks the smallest that keeps >= min_scale of the page + width_classes: [0.5, 0.625, 0.75, 0.875, 1.0] # short side snapped up to these fractions of the canvas (bounded set of engine shapes) + min_scale: 0.9 + default_canvas: 1280 # canvas of the single-image detect() path + thresh: 0.2 # DB probability threshold + box_thresh: 0.45 # mean box score threshold + unclip: 1.4 + pad_top: 0.03 # box extension (fraction of line height); bottom 0.25 keeps Kannada conjuncts inside the crop + pad_bottom: 0.25 + batch: 2 # pages per detector call + +layout: + mode: always # always | auto (only pages that look like tables/forms) | never + batch: 4 # engine batch; pages are zero-padded to it (one TensorRT engine shape) + score_thresh: 0.3 # block confidence threshold (v12 is calibrated lower than PP-DocLayoutV3 was) + fill: [1.0, 0.8] # the page is run once per value (long side = fill x canvas) and the boxes merged: 1.0 for small blocks, 0.8 keeps whole-page blocks inside the head's regression range + selftest_min_match: 1.0 # every reference block (CPU, conf >= 0.45) must be reproduced at conf >= 0.3 with class + IoU >= 0.7 (a wrong TensorRT engine drops the wide blocks; fp16 noise is ~1e-3) + +recognizer: + buckets: [128, 192, 256, 384, 512, 640, 768, 1024, 1280] # crop width buckets (each crop padded only to its own bucket) + batch: 64 + height: 32 + max_ratio: 40 + +second_pass: # confidence-gated re-read of the weakest lines with a slightly larger crop + conf: 0.85 + cap: 6 # lines per page + +runtime: + workers: auto # CPU worker threads for warps / DB post-processing / assembly (auto = max(4, cores/2)) + page_batch: 8 + log_level: warning # debug | info | warning | error + polygons: false # include 4-point polygons in line objects diff --git a/kn-en/kanen_infer/__init__.py b/kn-en/kanen_infer/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..54e4c56c3483c09686143b788e48cbe3e1f3a73c --- /dev/null +++ b/kn-en/kanen_infer/__init__.py @@ -0,0 +1,9 @@ +"""Arjuna-OCR [Kn&En] inference package (Kannada + English document OCR): ONNX Runtime + optional TensorRT, no PyTorch.""" +from pathlib import Path as _P +try: __version__ = (_P(__file__).resolve().parent.parent / "VERSION").read_text(encoding="utf-8").strip() +except Exception: __version__ = "1.2.0" +NAME = "Arjuna-OCR [Kn&En]" # product display name — also `name:` in kanen.yaml (the config value wins at run time) +from .config import load_config, PKG_ROOT # noqa: E402 +from .api import KanEnOCR # noqa: E402 +from .formats import to_json, to_markdown, to_text, to_hocr, to_alto, to_tsv, render # noqa: E402 +__all__ = ["KanEnOCR", "load_config", "PKG_ROOT", "NAME", "__version__", "to_json", "to_markdown", "to_text", "to_hocr", "to_alto", "to_tsv", "render"] diff --git a/kn-en/kanen_infer/api.py b/kn-en/kanen_infer/api.py new file mode 100644 index 0000000000000000000000000000000000000000..5077d45c29e3ad157c1b0fffa629a801864dc830 --- /dev/null +++ b/kn-en/kanen_infer/api.py @@ -0,0 +1,110 @@ +"""Arjuna-OCR [Kn&En] Python API. + + from kanen_infer import KanEnOCR + ocr = KanEnOCR() # loads kanen.yaml, builds/validates engines (warm-up), see ocr.ready() + doc = ocr.page("scan.png") # one kanen-ocr/1.0 document (dict) + docs = ocr.pages(["a.png", "b.pdf"]) # batched; ocr.stream(...) yields documents as they finish + boxes = ocr.detect(img) # text-line polygons only + texts = ocr.recognize(crops) # [(text, conf, min_conf)] for line images + blocks = ocr.layout(img) # layout blocks only + ocr.render(doc, "hocr") # json | md | txt | hocr | alto | tsv + +Inputs: file path, bytes, numpy BGR array, PIL image, file object; PDFs through `pages()`/`stream()` (pypdfium2 or pdftoppm, +optional). The output document never depends on the provider or the batch composition beyond fp16 noise on garbage lines; +no confidence filtering, no text rewriting, no re-ordering happens here.""" +import logging, time +import numpy as np, cv2 +from pathlib import Path +from .config import load_config, PKG_ROOT +from .providers import Providers +from .rec_onnx import KanEnRecognizer +from .pipeline_base import Detector, Layout, crop +from .batch_pipeline import BatchOCR +from .io import to_bgr, expand_inputs, iter_page_sources +from . import formats + +LOG = logging.getLogger("kanen") + +class KanEnOCR: + def __init__(self, config=None, providers=None, layout_mode=None, workers=None, warmup=True, **overrides): + """config: path to a kanen.yaml/json, a dict, or None (package default). providers: 'trt' | 'cuda' | 'cpu' | 'auto' + (overrides config provider.policy). Other keyword overrides use dotted section names, e.g. layout={'mode': 'auto'}.""" + ov = dict(overrides) + if providers: ov.setdefault("provider", {})["policy"] = providers + if layout_mode: ov.setdefault("layout", {})["mode"] = layout_mode + if workers: ov.setdefault("runtime", {})["workers"] = workers + self.cfg = load_config(config, ov); self.name = self.cfg.get("name", "Arjuna-OCR [Kn&En]"); self.version = str(self.cfg.get("version", "1.0.0")) + t0 = time.time(); self.prov = Providers(self.cfg) + self.rec = KanEnRecognizer(cfg=self.cfg, providers=self.prov); self.det = Detector(self.cfg, self.prov) + self.lay = Layout(self.cfg, self.prov) if self.cfg["layout"].get("mode", "always") != "never" else None + self.batch = BatchOCR(self.rec, cfg=self.cfg, providers=self.prov, det=self.det, lay=self.lay) + self._load_s = round(time.time() - t0, 1); self._warm = None + if warmup: self.warmup() + + # ---- status -------------------------------------------------------------------------------------------------- + def warmup(self): + """Run the bundled self-test page through the full pipeline once: builds/validates TensorRT engines for the + default shapes (layout self-test, detector canvas, recognizer profile) so the first real page is not slow.""" + t0 = time.time(); p = self.cfg.path("models", "layout_selftest") + if p.exists(): + img = to_bgr(str(p)); self.batch.run([img], page_batch=1, loader=lambda x: x) + self._warm = round(time.time() - t0, 1); return self.ready() + + def ready(self): + """Provider each model ended on, fallbacks that happened, engine-cache state, model hashes.""" + st = self.prov.summary() + if self.lay is not None: st.setdefault("layout", {})["engine"] = self.lay.stage + return {"name": self.name, "version": self.version, "config": self.cfg.source, "gpu": self.prov.gpu, "tensorrt": self.prov.trt_ver, + "providers": st, "events": list(self.prov.events), "engine_cache": str(self.prov.cache_root), "load_s": self._load_s, "warmup_s": self._warm, + "model": dict(self.batch.model_ids)} + + # ---- full documents ------------------------------------------------------------------------------------------- + def page(self, image, polygons=None, dpi=None): + """Single page (latency path): one kanen-ocr/1.0 document.""" + img = to_bgr(image); docs, _ = self.batch.process([img], dpi=dpi, with_polygons=self._poly(polygons)); d = docs[0] + if isinstance(image, (str, Path)): d["page"]["file"] = Path(image).name + return d + + def pages(self, inputs, page_batch=None, polygons=None, pdf_dpi=200): + """Batched: list/dir/glob of image or PDF paths, or a list of arrays / PIL images / bytes -> list of documents.""" + return [d for d in self.stream(inputs, page_batch, polygons, pdf_dpi)] + + def stream(self, inputs, page_batch=None, polygons=None, pdf_dpi=200): + """Generator variant of pages(): yields documents in input order as each page batch completes.""" + sources = self._sources(inputs, pdf_dpi); labels = [s[0] for s in sources]; items = [s[1] for s in sources] + page_batch = page_batch or self.cfg["runtime"].get("page_batch", 8); k = 0 + for doc, st in self.batch.stream(items, page_batch=page_batch, loader=to_bgr, with_polygons=self._poly(polygons)): + if labels[k]: doc["page"]["file"] = labels[k] + self.last_stats = st; k += 1; yield doc + + def _sources(self, inputs, pdf_dpi): + if isinstance(inputs, (str, Path)) or (isinstance(inputs, (list, tuple)) and inputs and all(isinstance(x, (str, Path)) for x in inputs)): + return list(iter_page_sources(expand_inputs(inputs), dpi=pdf_dpi)) + if isinstance(inputs, (list, tuple)): return [(getattr(x, "name", None) or "", x) for x in inputs] + return [("", inputs)] + + def _poly(self, polygons): return bool(self.cfg["runtime"].get("polygons", False)) if polygons is None else bool(polygons) + + # ---- single stages -------------------------------------------------------------------------------------------- + def detect(self, image): + """Text-line detection only -> list of (polygon 4x2 float32 in page pixels, score).""" + return self.det(to_bgr(image)) + + def recognize(self, crops, batch=None): + """Recognition only on a list of line images (any size, BGR or gray) -> [(text, confidence, min_conf)].""" + res = self.rec([to_bgr(c) for c in crops], bs=batch) + return [(t, float(c), float(getattr(c, "min_char", c))) for t, c in res] + + def layout(self, image): + """Layout blocks only -> [{type, score, bbox, order}] (labels: title, text, table, figure, seal, id_panel, header, footer, handwriting).""" + if self.lay is None: self.lay = Layout(self.cfg, self.prov); self.batch.lay = self.lay + return self.lay(to_bgr(image))[0] + + def crops(self, image, boxes=None): + """Perspective line crops for detect() boxes (helper for recognize()).""" + img = to_bgr(image); boxes = self.detect(img) if boxes is None else boxes + return [crop(img, b) for b, _ in boxes] + + # ---- serialisation -------------------------------------------------------------------------------------------- + def render(self, doc, fmt="json"): + return formats.render(doc, fmt) diff --git a/kn-en/kanen_infer/assemble.py b/kn-en/kanen_infer/assemble.py new file mode 100644 index 0000000000000000000000000000000000000000..1b6d97a5c8806d292dc87f3732ffa624dbd8d7dd --- /dev/null +++ b/kn-en/kanen_infer/assemble.py @@ -0,0 +1,261 @@ +"""Assembler: layout blocks (Arjuna layout v12) + text-line polygons + recognised text -> kanen-ocr/1.0 JSON. +Pure geometry, no model. Also: Markdown / plain-text renderers.""" +import numpy as np +from .charset import script_of +import re +WEAK_CHAR = 0.50 # min-char confidence below which a line is flagged weak_char (calibrated on 3,478 dev lines: < 0.5 -> 65 % exact) +_MIX_RE = re.compile(r"(?=\S*[A-Za-z])(?=\S*[\u0C85-\u0CB9])\S+") # token with both a Latin letter and a Kannada letter + +TYPE_MAP = {"doc_title": "title", "paragraph_title": "section_title", "text": "text", "abstract": "text", "content": "text", "reference": "text", + "reference_content": "text", "aside_text": "text", "vertical_text": "text", "footnote": "footnote", "vision_footnote": "footnote", + "header": "header", "footer": "footer", "number": "page_number", "table": "table", "image": "figure", "chart": "figure", + "figure_title": "figure_caption", "seal": "seal", "display_formula": "formula", "inline_formula": "formula", "formula_number": "formula", + "algorithm": "text", "header_image": "figure", "footer_image": "figure", + "title": "title", "figure": "figure", "id_panel": "id_panel", "handwriting": "handwriting"} # v1.2 layout classes (text/table/seal/header/footer map 1:1 above) + +def _bbox(poly): p = np.asarray(poly, np.float32); return [float(p[:, 0].min()), float(p[:, 1].min()), float(p[:, 0].max()), float(p[:, 1].max())] + +def _overlap_frac(line_bb, block_bb): + ix = max(0, min(line_bb[2], block_bb[2]) - max(line_bb[0], block_bb[0])); iy = max(0, min(line_bb[3], block_bb[3]) - max(line_bb[1], block_bb[1])) + return ix * iy / max((line_bb[2] - line_bb[0]) * (line_bb[3] - line_bb[1]), 1e-6) + +def merge_line_fragments(lines, y_tol=0.5, x_gap=1.5): + """Merge horizontally adjacent boxes on the same baseline (detector sometimes splits a line at wide gaps). + lines: list of dict(bbox, polygon, text, conf). Returns merged list.""" + lines = sorted(lines, key=lambda l: (l["bbox"][1], l["bbox"][0])); out = [] + for l in lines: + merged = False + for m in out: + h = min(m["bbox"][3] - m["bbox"][1], l["bbox"][3] - l["bbox"][1]) + same_row = abs((m["bbox"][1] + m["bbox"][3]) / 2 - (l["bbox"][1] + l["bbox"][3]) / 2) < y_tol * h + gap = l["bbox"][0] - m["bbox"][2] + if same_row and -0.2 * h < gap < x_gap * h: + m["bbox"] = [min(m["bbox"][0], l["bbox"][0]), min(m["bbox"][1], l["bbox"][1]), max(m["bbox"][2], l["bbox"][2]), max(m["bbox"][3], l["bbox"][3])] + m["text"] = (m["text"] + " " + l["text"]).strip(); m["confidence"] = min(m["confidence"], l["confidence"]); m["min_conf"] = min(m.get("min_conf", 1), l.get("min_conf", 1)); m["polygon"] = None; merged = True; break + if not merged: out.append(dict(l)) + return out + +def xy_cut_order(blocks, page_w): + """Fallback reading order: column split then top-to-bottom.""" + if not blocks: return [] + mid = page_w / 2 + left = [b for b in blocks if b["bbox"][2] < mid * 1.05]; right = [b for b in blocks if b["bbox"][0] > mid * 0.95] + spanning = [b for b in blocks if b not in left and b not in right] + if len(left) >= 2 and len(right) >= 2 and not spanning: + return sorted(left, key=lambda b: b["bbox"][1]) + sorted(right, key=lambda b: b["bbox"][1]) + return sorted(blocks, key=lambda b: (round(b["bbox"][1] / 20), b["bbox"][0])) + +def rows_of(lines, tol=0.5): + """Cluster line boxes into visual rows (y-centre within tol x the smaller height), left-to-right inside a row.""" + ls = sorted(lines, key=lambda l: (l["bbox"][1] + l["bbox"][3]) / 2); rows = [] + for l in ls: + yc = (l["bbox"][1] + l["bbox"][3]) / 2; h = l["bbox"][3] - l["bbox"][1] + if rows: + r = rows[-1]; ryc = np.mean([(m["bbox"][1] + m["bbox"][3]) / 2 for m in r]); rh = np.median([m["bbox"][3] - m["bbox"][1] for m in r]) + if abs(yc - ryc) < tol * min(h, rh): r.append(l); continue + rows.append([l]) + return [sorted(r, key=lambda l: l["bbox"][0]) for r in rows] + +def split_row_by_gap(r, max_gap_h=2.5): + """Split a visual row into runs where the horizontal gap between neighbours exceeds max_gap_h x line height: two + columns (Kannada | English on ID cards, label | value pairs far apart) must stay separate lines; fragments of one + justified/spaced line (gaps of a few glyph widths) still join. Geometry only.""" + r = sorted(r, key=lambda l: l["bbox"][0]); runs = [[r[0]]] + for l in r[1:]: + prev = runs[-1][-1]; h = max(8.0, min(prev["bbox"][3] - prev["bbox"][1], l["bbox"][3] - l["bbox"][1])) + if l["bbox"][0] - prev["bbox"][2] > max_gap_h * h: runs.append([l]) + else: runs[-1].append(l) + return runs + +def join_row(r): + """One text line per visual row: fragments of a justified/spaced line are joined with single spaces.""" + if len(r) == 1: return dict(r[0]) + return {"bbox": [min(l["bbox"][0] for l in r), min(l["bbox"][1] for l in r), max(l["bbox"][2] for l in r), max(l["bbox"][3] for l in r)], + "polygon": None, "text": " ".join(l["text"] for l in r).strip(), "confidence": min(l["confidence"] for l in r), "min_conf": min(l.get("min_conf", 1) for l in r)} + +def _runs(mask1d, min_gap=3): + idx = np.flatnonzero(mask1d) + if idx.size == 0: return [] + out = []; start = idx[0]; prev = idx[0] + for i in idx[1:]: + if i - prev > min_gap: out.append((start + prev) / 2); start = i + prev = i + out.append((start + prev) / 2); return out + +def _rule_positions(mask, axis, min_len): + """Centre coordinate of every connected component of a rule mask longer than min_len along `axis` (robust to residual tilt).""" + import cv2 + n, _, st, cen = cv2.connectedComponentsWithStats(mask, connectivity=8) + pos = sorted(cen[i][1 - axis] for i in range(1, n) if st[i][cv2.CC_STAT_WIDTH if axis == 0 else cv2.CC_STAT_HEIGHT] >= min_len) + out = [] + for p in pos: + if out and p - out[-1] < 6: out[-1] = (out[-1] + p) / 2 + else: out.append(p) + return out + +def _merge_continuations(rows): + """A row whose cells all sit under occupied columns of the previous row and none in its first column is a wrapped cell.""" + out = [] + for r in rows: + ne = [c for c in r["cells"] if c["text"]] + if out and ne: + prev = out[-1]; pne = [c for c in prev["cells"] if c["text"]] + first = min(c["col"] for c in prev["cells"]) if prev["cells"] else 0 + if all(c["col"] > first for c in ne) and all(any(p["col"] == c["col"] for p in pne) for c in ne): + for c in ne: + p = next(p for p in pne if p["col"] == c["col"]); p["text"] = (p["text"] + " " + c["text"]).strip() + p["bbox"] = [min(p["bbox"][0], c["bbox"][0]), min(p["bbox"][1], c["bbox"][1]), max(p["bbox"][2], c["bbox"][2]), max(p["bbox"][3], c["bbox"][3])] + continue + out.append(r) + return out + +def ruled_grid(gray, bbox, lines): + """Table structure from printed ruling lines (morphological opening + connected components). Horizontal rules give row + bands; inside a band, vertical rules give cells - or, when a band has no vertical rules (aligned-column forms), each + visual row of the band becomes a table row whose cells are the separate line boxes. Returns + rows=[{cells:[{col,text,bbox,confidence}], ruled}] or None when the region has no horizontal rules. Pure geometry.""" + import cv2 + x0, y0, x1, y1 = [int(round(v)) for v in bbox]; reg = gray[max(0, y0):y1, max(0, x0):x1] + if reg.size == 0 or reg.shape[0] < 20 or reg.shape[1] < 20: return None + H, W = reg.shape; bw = cv2.adaptiveThreshold(reg, 255, cv2.ADAPTIVE_THRESH_MEAN_C, cv2.THRESH_BINARY_INV, 31, 9) + hor = cv2.morphologyEx(bw, cv2.MORPH_OPEN, cv2.getStructuringElement(cv2.MORPH_RECT, (max(20, W // 30), 1))) + hor = cv2.dilate(hor, cv2.getStructuringElement(cv2.MORPH_RECT, (max(20, W // 8), 1))) # bridge faint / dashed rules (text-derived runs are rejected below) + loc = [(l["bbox"][0] - x0, l["bbox"][1] - y0, l["bbox"][2] - x0, l["bbox"][3] - y0) for l in lines] + def through_text(y): return any(b + 0.1 * (d - b) < y < d - 0.1 * (d - b) for a, b, c, d in loc) # a rule never crosses the middle of a text line + def through_text_x(x, ia, ib): return any(a + 0.1 * (c - a) < x < c - 0.1 * (c - a) and b < ib and d > ia for a, b, c, d in loc) + hr = [y for y in _rule_positions(hor, 0, 0.25 * W) if 0.01 * H + 4 < y < H - 0.01 * H - 4 and not through_text(y)] + if not hr: return None + ys = [0.0] + hr + [float(H)]; bands = [] + for a, b in zip(ys[:-1], ys[1:]): + ia, ib = int(a), int(b) + if ib - ia < 8: continue + ver = cv2.morphologyEx(bw[ia:ib], cv2.MORPH_OPEN, cv2.getStructuringElement(cv2.MORPH_RECT, (1, max(8, int((ib - ia) * 0.4))))) + ver = cv2.dilate(ver, cv2.getStructuringElement(cv2.MORPH_RECT, (1, max(6, int((ib - ia) * 0.25))))) + vr = [x for x in _rule_positions(ver, 1, 0.85 * (ib - ia)) if 0.03 * W < x < 0.97 * W and not through_text_x(x, ia, ib)] # cell borders span the band, never cross text; table border excluded + bands.append((ia, ib, vr)) + allx = sorted(x for _, _, vr in bands for x in vr); G = [] + for x in allx: + if not G or x - G[-1][-1] > 12: G.append([x]) + else: G[-1].append(x) + G = [float(np.mean(g)) for g in G] + def gcol(x): return sum(1 for g in G if g < x + 6) + rows = [] + for ia, ib, vr in bands: + inband = [l for l in lines if ia <= (l["bbox"][1] + l["bbox"][3]) / 2 - y0 < ib] + if not inband: continue + if vr: # ruled cells: everything between two vertical rules is one cell (a wrapped cell keeps its lines together); + # the global rule set is used so a band whose faint rule was missed still gets the table's columns + xs = [0.0] + G + [float(W)] + percol = [[l for l in inband if ca <= (l["bbox"][0] + l["bbox"][2]) / 2 - x0 < cb] for ca, cb in zip(xs[:-1], xs[1:])] + multi = sum(1 for inc in percol if len(rows_of(inc)) > 1) + # several columns stacked -> the band holds several table rows whose separating rule was not printed/found + groups = rows_of(inband) if multi >= 2 else [inband] + for g in groups: + cells = [] + for ca, cb in zip(xs[:-1], xs[1:]): + inc = [l for l in g if ca <= (l["bbox"][0] + l["bbox"][2]) / 2 - x0 < cb] + if not inc: cells.append({"col": gcol(ca), "text": "", "bbox": None, "confidence": None}); continue + cells.append({"col": gcol(ca), "text": " ".join(join_row(r)["text"] for r in rows_of(inc)), "bbox": [min(l["bbox"][0] for l in inc), min(l["bbox"][1] for l in inc), max(l["bbox"][2] for l in inc), max(l["bbox"][3] for l in inc)], "confidence": min(l["confidence"] for l in inc)}) + rows.append({"cells": cells, "ruled": True}) + if len(groups) > 1: rows[-len(groups):] = _merge_continuations(rows[-len(groups):]) + else: # no vertical rules in this band: each visual row is a table row, each line box a cell + for r in rows_of(inband): + rows.append({"cells": [{"col": (gcol(l["bbox"][0] - x0) if G else ci), "text": l["text"], "bbox": l["bbox"], "confidence": l["confidence"]} for ci, l in enumerate(r)], "ruled": False}) + return rows if rows else None + +def group_table(lines): + """Unruled tables: rows by y-clustering, columns by clustering cell x-starts; a row that only continues cells of the + previous row (no cell in the first column, every cell under an occupied column) is a wrapped cell -> merged upward.""" + if not lines: return [] + rows = rows_of(lines) + spans = sorted(((l["bbox"][0], l["bbox"][2]) for r in rows for l in r)); cols = [] + for a, b in spans: # columns = groups of horizontally overlapping cells (works for left/centre/right alignment) + if cols and a < cols[-1][1] - 4: cols[-1][1] = max(cols[-1][1], b) + else: cols.append([a, b]) + def col_of(l): + xc = (l["bbox"][0] + l["bbox"][2]) / 2 + return next((i for i, (a, b) in enumerate(cols) if a - 4 <= xc <= b + 4), int(np.argmin([abs(xc - (a + b) / 2) for a, b in cols])) if cols else 0) + return _merge_continuations([{"cells": [{"bbox": l["bbox"], "text": l["text"], "confidence": l["confidence"], "col": col_of(l)} for l in r], "ruled": False} for r in rows]) + +def assemble(page_w, page_h, layout_blocks, det_boxes, texts, model_ids, timing=None, page_index=0, dpi=None, with_polygons=True, img=None): + """layout_blocks: [{type, score, bbox, order}], det_boxes: [(poly4x2, score)], texts: [(text, conf)] aligned with det_boxes.""" + lines = [] + for (poly, dscore), (text, conf) in zip(det_boxes, texts): + if not text.strip(): continue + lines.append({"bbox": _bbox(poly), "polygon": [[float(x), float(y)] for x, y in poly] if with_polygons else None, "text": text, "confidence": round(float(conf), 4), + "min_conf": round(float(getattr(conf, "min_char", None) if getattr(conf, "min_char", None) is not None else conf), 4)}) + blocks = [] + for i, b in enumerate(layout_blocks): + blocks.append({"id": f"b{i}", "type": TYPE_MAP.get(b["type"], "other"), "raw_type": b["type"], "confidence": round(float(b["score"]), 4), "bbox": [round(v, 1) for v in b["bbox"]], "order": b.get("order", i), "lines": []}) + # assign lines to best-overlapping block; unassigned lines become their own text blocks + orphans = [] + area = lambda bb: max(1.0, (bb[2] - bb[0]) * (bb[3] - bb[1])) + for l in lines: + best, bi = 0.0, -1 + for i, b in enumerate(blocks): + f = _overlap_frac(l["bbox"], b["bbox"]) + # nested blocks (v1.2 layout: a handwriting / seal / id_panel box inside a text region): on a near-tie the + # smaller, more specific block wins, so signature fragments are not glued to the surrounding text rows + if f > best + 0.05 or (f >= best - 0.05 and bi >= 0 and f >= 0.5 and area(b["bbox"]) < area(blocks[bi]["bbox"])): best, bi = f, i + if best >= 0.5: blocks[bi]["lines"].append(l) + else: orphans.append(l) + for l in orphans: + blocks.append({"id": f"b{len(blocks)}", "type": "text", "raw_type": "orphan_line", "confidence": l["confidence"], "bbox": [round(v, 1) for v in l["bbox"]], "order": None, "lines": [l]}) + blocks = [b for b in blocks if b["lines"] or b["type"] in ("figure", "seal", "table", "id_panel", "handwriting")] + # order: native rank where available, else xy-cut; orphans placed by position among ordered blocks + ranked = [b for b in blocks if b["order"] is not None]; unranked = [b for b in blocks if b["order"] is None] + ordered = sorted(ranked, key=lambda b: b["order"]) + for b in unranked: + pos = next((k for k, o in enumerate(ordered) if o["bbox"][1] > b["bbox"][1] and o["bbox"][0] <= b["bbox"][2]), len(ordered)); ordered.insert(pos, b) + if not ranked: ordered = xy_cut_order(blocks, page_w) + full = [] + gray = None + if img is not None: + import cv2; gray = img if img.ndim == 2 else cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) + heights = [l["bbox"][3] - l["bbox"][1] for b in ordered for l in b["lines"]]; med_h = float(np.median(heights)) if heights else 0.0 + for k, b in enumerate(ordered): + b["order"] = k + if b["type"] == "table": + # table cells stay separate boxes; structure from printed rules when present, else from alignment + b["lines"] = [l for r in rows_of(b["lines"]) for l in r] + rows = ruled_grid(gray, b["bbox"], b["lines"]) if gray is not None else None + b["rows"] = rows or group_table(b["lines"]); b["ruled"] = bool(rows) + else: + b["lines"] = [join_row(run) for r in rows_of(b["lines"]) for run in split_row_by_gap(r)] # one text line per visual row run (fragments of a spaced line re-joined; wide-gap columns kept apart) + for li, l in enumerate(b["lines"]): + l["id"] = f"{b['id']}.l{li}"; l["script"] = script_of(l["text"]); l["bbox"] = [round(v, 1) for v in l["bbox"]] + # review flags (never deletion): calibration on 5,478 labelled lines -> conf >= 0.99 is 97.7 % line-exact, + # 0.95-0.98 only 83.5 %; lines inside seal/figure/header blocks are mostly stamp/handwriting fragments + fl = [] + if l["confidence"] < 0.90: fl.append("low_conf") + elif l["confidence"] < 0.98: fl.append("check") + if b["type"] in ("seal", "figure", "handwriting", "id_panel"): fl.append("in_" + b["type"]) + # review flags (script/character-repertoire checks); flags only, the text is never altered + if l.get("min_conf", 1.0) < WEAK_CHAR: fl.append("weak_char") # one very uncertain character (garbage lines usually have one) + if _MIX_RE.search(l["text"]): fl.append("script_mix") # Latin and Kannada letters inside one token (other-script garbage) + if med_h and (l["bbox"][3] - l["bbox"][1]) > 3.0 * med_h: fl.append("oversize") # stray ornament / drop-cap sized line + if fl: l["flags"] = fl + if b["type"] == "table" and b["rows"]: block_text = "\n".join(" | ".join(c["text"] for c in r["cells"]) for r in b["rows"]) + else: block_text = "\n".join(l["text"] for l in b["lines"]) + if block_text.strip(): full.append(block_text) + return {"schema": "kanen-ocr/1.0", "model": model_ids, "page": {"index": page_index, "width": page_w, "height": page_h, "dpi": dpi, "rotation": 0}, + "blocks": ordered, "text": "\n\n".join(full), "timing_ms": timing or {}} + +def to_markdown(doc): + out = [] + for b in doc["blocks"]: + t = b["type"]; txt = " ".join(l["text"] for l in b["lines"]).strip() + if t == "title": out.append(f"# {txt}") + elif t == "section_title": out.append(f"## {txt}") + elif t == "table" and b.get("rows"): + ncol = max((c["col"] for r in b["rows"] for c in r["cells"]), default=0) + 1 + rows = [] + for r in b["rows"]: + cells = [""] * ncol + for c in r["cells"]: cells[c["col"]] = (cells[c["col"]] + " " + c["text"]).strip() + rows.append("| " + " | ".join(x.replace("|", "\\|") for x in cells) + " |") + if rows: out.append(rows[0] + "\n|" + "---|" * ncol + "\n" + "\n".join(rows[1:])) + elif t in ("figure", "seal"): out.append(f"*[{t}]*") + elif t in ("header", "footer", "page_number"): out.append(f"{txt}") + elif txt: out.append(txt) + return "\n\n".join(out) + "\n" diff --git a/kn-en/kanen_infer/batch_pipeline.py b/kn-en/kanen_infer/batch_pipeline.py new file mode 100644 index 0000000000000000000000000000000000000000..9cfa19a1eb74703c44cd6a6eaa633efdb152b3c0 --- /dev/null +++ b/kn-en/kanen_infer/batch_pipeline.py @@ -0,0 +1,230 @@ +"""Arjuna-OCR [Kn&En] batched throughput pipeline. +Organised for throughput instead of single-page latency (the API's `page()` runs the same code with one page): + * decode + canvas choice in a prefetch thread pool (next page batch is decoded while the GPU works on this one) + * detector runs ONCE per page at a canvas chosen from the page size (no 1280 -> re-detect search), batched across pages + * DB post-processing + perspective crops in worker threads + * recognizer sees the crops of the whole page batch in one width-bucketed call; orientation-180 fallback and the + confidence-gated second pass are pooled the same way + * layout model batched across pages +No output rewriting anywhere: every character comes from the recognizer. All tunables come from the package config. +""" +import time, math, os +from concurrent.futures import ThreadPoolExecutor +import numpy as np, cv2 +from .config import load_config, _FALLBACK +from .providers import Providers +from .pipeline_base import Detector, Layout, crop, MEAN, STD +from .assemble import assemble +from .geometry import rotate_page, deskew_angle, rotate_small + +def n_readable(res, conf=0.90, min_char=0.50, min_len=4): + """readable text mass = characters in lines that read cleanly (line conf and weakest character high, >= 4 chars so + stray '|' / '1' fragments do not vote). Majority vote for mixed-orientation pages (upright form + rotated header).""" + return sum(len(t) for t, c in res if len(t) >= min_len and float(c) >= conf and (getattr(c, "min_char", None) if getattr(c, "min_char", None) is not None else float(c)) >= min_char) + +def orient_score(res): + """orientation evidence: mean line confidence + mean weakest-character confidence (upside-down text = confident garbage with one very weak frame)""" + if not res: return 0.0 + return float(np.mean([float(c) for _, c in res]) + np.mean([getattr(c, "min_char", None) if getattr(c, "min_char", None) is not None else float(c) for _, c in res])) + +CANVASES = tuple(_FALLBACK["detector"]["canvases"]) +WIDTH_CLASSES = tuple(_FALLBACK["detector"]["width_classes"]) # canvas width as a fraction of the long side -> at most 5 ORT plans per canvas (no square padding) +INV_STD = (1.0 / STD).astype(np.float32); MEAN255 = tuple(float(m * 255) for m in MEAN) # same channel order as Detector (PaddleOCR applies the mean to the BGR image as read) +PAD_VAL = ((1.0 - MEAN) / STD).astype(np.float32) # normalised white, like Detector's white canvas + +def choose_canvas(h, w, min_scale=0.9, canvases=CANVASES): + """Smallest fixed canvas that keeps the page at >= min_scale of its native size (150-DPI private-document scans -> 1920, A4@300 -> 2560).""" + m = max(h, w) + for c in canvases: + if c >= min_scale * m: return c + return canvases[-1] + +def canvas_shape(h, w, min_scale=0.9, canvases=CANVASES, width_classes=WIDTH_CLASSES): + """(limit, canvas_h, canvas_w): long side = limit, short side snapped up to a width class (multiple of 32).""" + limit = choose_canvas(h, w, min_scale, canvases); r = min(1.0, limit / max(h, w)); nh, nw = h * r, w * r + if h >= w: cw = next(int(round(limit * f / 32)) * 32 for f in width_classes if limit * f >= nw); return limit, limit, cw + ch = next(int(round(limit * f / 32)) * 32 for f in width_classes if limit * f >= nh); return limit, ch, limit + +def rotate_boxes(boxes, deg, w, h): + """Map detector polygons into the deskewed page frame (same rotation rotate_small applies to the image).""" + if not boxes: return boxes + M = cv2.getRotationMatrix2D((w / 2, h / 2), deg, 1.0) + return [(cv2.transform(b[None].astype(np.float32), M)[0], sc) for b, sc in boxes] + +class BatchOCR: + def __init__(self, rec=None, cfg=None, providers=None, det=None, lay=None, use_layout=None, layout_mode=None, workers=None, tta_conf=None, tta_cap=None, rec_bs=None, det_bs=None, det_size="small"): + """rec/det/lay: model objects to share (built from cfg when None). layout_mode: 'always' | 'auto' (only pages that look + like tables/forms) | 'never'. tta_cap: second-pass lines per page. Every default comes from the config.""" + from .rec_onnx import KanEnRecognizer + self.cfg = cfg or getattr(rec, "cfg", None) or load_config(); self.prov = providers or getattr(rec, "prov", None) or Providers(self.cfg) + dc, lc, rc, sp, rt = self.cfg["detector"], self.cfg["layout"], self.cfg["recognizer"], self.cfg["second_pass"], self.cfg["runtime"] + self.layout_mode = layout_mode or lc.get("mode", "always") + if use_layout is False: self.layout_mode = "never" + self.rec = rec or KanEnRecognizer(cfg=self.cfg, providers=self.prov); self.det = det or Detector(self.cfg, self.prov) + self.lay = lay if lay is not None else (Layout(self.cfg, self.prov) if self.layout_mode != "never" else None) + self.tta_conf = float(sp["conf"] if tta_conf is None else tta_conf); self.tta_cap = int(sp["cap"] if tta_cap is None else tta_cap) + self.rec_bs = int(rc["batch"] if rec_bs is None else rec_bs); self.det_bs = int(dc["batch"] if det_bs is None else det_bs) + self.canvases = tuple(int(c) for c in dc["canvases"]); self.width_classes = tuple(float(f) for f in dc["width_classes"]); self.min_scale = float(dc["min_scale"]) + self.workers = int(workers) if workers else self.cfg.workers(); self.pool = ThreadPoolExecutor(self.workers); self._planned = set() + self.model_ids = {"layout": "arjuna-layout-v12" if self.lay else None, "det": f"ppocrv6-{det_size}-det", "rec": getattr(self.rec, "name", "kanen-rec"), + "rec_md5": getattr(self.rec, "onnx_md5", None), "rec_onnx_md5": getattr(self.rec, "onnx_md5", None), "charset_md5": getattr(self.rec, "charset_md5", None), "pipeline": str(self.cfg.get("version", "1.0.0"))} + + def _sess(self, limit): + """detector session for a canvas limit (one per limit, as in Detector).""" + return self.det._session(limit) + + def _plan(self, limit, ch, cw, n): + key = (limit, ch, cw, n) + if key not in self._planned: # ORT CUDA EP re-plans on a new input shape (~1 s) -> do it once, outside the timed path + s = self._sess(limit); s.run(None, {s.get_inputs()[0].name: np.zeros((n, 3, ch, cw), np.float32)}); self._planned.add(key) + + def _prep_det(self, img, ch, cw, limit): + h, w = img.shape[:2]; r = min(1.0, limit / max(h, w)); nh, nw = max(32, int(round(h * r / 32)) * 32), max(32, int(round(w * r / 32)) * 32) + nh, nw = min(nh, ch), min(nw, cw) + x = cv2.dnn.blobFromImage(cv2.resize(img, (nw, nh)), 1 / 255.0, (nw, nh), MEAN255, swapRB=False)[0] # (3,nh,nw), (x/255 - mean) + x *= INV_STD[:, None, None] + canvas = np.empty((3, ch, cw), np.float32); canvas[:] = PAD_VAL[:, None, None]; canvas[:, :nh, :nw] = x + return canvas, (nh, nw, w / nw, h / nh) + + # ---- detector, batched over pages that share a canvas shape ---- + def _det_batch(self, imgs, limit, ch, cw): + s = self._sess(limit); self._plan(limit, ch, cw, len(imgs)) + prepped = list(self.pool.map(lambda im: self._prep_det(im, ch, cw, limit), imgs)) + x = np.stack([p[0] for p in prepped]); meta = [p[1] for p in prepped] + prob = s.run(None, {s.get_inputs()[0].name: x})[0][:, 0] + return list(self.pool.map(lambda k: self.det._post(prob[k][:meta[k][0], :meta[k][1]], meta[k][2], meta[k][3]), range(len(imgs)))) + + def _detect(self, imgs): + """imgs -> boxes per page; one detector pass per page, batched per canvas shape, no square padding.""" + groups = {} + for i, im in enumerate(imgs): groups.setdefault(canvas_shape(*im.shape[:2], min_scale=self.min_scale, canvases=self.canvases, width_classes=self.width_classes), []).append(i) + out = [None] * len(imgs) + for (limit, ch, cw), ids in groups.items(): + for j in range(0, len(ids), self.det_bs): + sub = ids[j:j + self.det_bs] + for i, b in zip(sub, self._det_batch([imgs[i] for i in sub], limit, ch, cw)): out[i] = b + return out + + # Layout batch is fixed (config layout.batch): TensorRT builds one engine per input shape (variable sizes = long rebuilds), so + # pages are always run at the configured batch, zero-padded; the engine is self-tested before its first use (Layout.ensure). + def _layout(self, imgs): + if not self.lay or not imgs: return [[] for _ in imgs] + if self.lay.trt and self.lay.bs not in self.lay.ok: self.lay.ensure(self.lay.bs) # validates (or swaps) the engine batch before the first use + return self.lay.batch(imgs) + + def _crops(self, img, boxes, pad=None): + if pad is None: return [crop(img, b) for b, _ in boxes] + out = [] + for b, _ in boxes: + b = b.copy(); hgt = 0.5 * ((b[3, 1] - b[0, 1]) + (b[2, 1] - b[1, 1])); b[0, 1] -= pad[0] * hgt; b[1, 1] -= pad[0] * hgt; b[2, 1] += pad[1] * hgt; b[3, 1] += pad[1] * hgt + out.append(crop(img, b)) + return out + + # ---------------- stage A (detector side) and stage B (recogniser side) so consecutive batches can overlap ---------------- + def _stage_a(self, imgs): + """detect -> orientation 90/270 -> deskew (boxes rotated, no second pass) -> crops. CPU+GPU(det).""" + t0 = time.perf_counter(); n = len(imgs); rot = [0] * n; ang = [0.0] * n + boxes = self._detect(imgs) + vert = [i for i in range(n) if boxes[i] and np.median([np.linalg.norm(b[1] - b[0]) / max(np.linalg.norm(b[3] - b[0]), 1) for b, _ in boxes[i]]) < 1.0] + if vert: # candidate 90/270: keep the orientation with more readable lines (mixed-orientation forms stay upright) + imgs90 = [rotate_page(imgs[i], 1) for i in vert]; boxes90 = self._detect(imgs90) + for i, im90, b90 in zip(vert, imgs90, boxes90): + good0 = n_readable(self.rec(self._crops(imgs[i], boxes[i]), bs=self.rec_bs)) if boxes[i] else 0 + good90 = n_readable(self.rec(self._crops(im90, b90), bs=self.rec_bs)) if b90 else 0 + if good90 > good0 or (good90 == 0 and good0 == 0): imgs[i], boxes[i], rot[i] = im90, b90, 90 # nothing readable either way -> trust the aspect rule; the 180° check finishes 270° pages + for i in range(n): + ang[i] = deskew_angle(boxes[i]) + if abs(ang[i]) > 0.4: + hh, ww = imgs[i].shape[:2]; imgs[i] = rotate_small(imgs[i], ang[i]); boxes[i] = rotate_boxes(boxes[i], ang[i], ww, hh) + # layout only where the page looks structured (many short boxes) or when forced; runs in its own thread + need_lay = [i for i in range(n) if self.lay and (self.layout_mode == "always" or (self.layout_mode == "auto" and self._structured(boxes[i])))] + lay_future = self.pool.submit(self._layout, [imgs[i] for i in need_lay]) if need_lay else None + # crops: one warp task per crop (the 4-core box has little headroom, but warps release the GIL) + jobs = [(i, b) for i in range(n) for b, _ in boxes[i]] + flat = list(self.pool.map(lambda ib: crop(imgs[ib[0]], ib[1]), jobs)) + return {"imgs": imgs, "boxes": boxes, "rot": rot, "ang": ang, "flat": flat, "jobs": jobs, "lay_future": lay_future, "need_lay": need_lay, "t_a": time.perf_counter() - t0} + + def _structured(self, boxes, min_boxes=25, short_frac=0.45): + """Heuristic for 'this page has tables/forms': many boxes and a large share of short ones.""" + if len(boxes) < min_boxes: return False + ws = [np.linalg.norm(b[1] - b[0]) for b, _ in boxes]; med = np.median(ws) + return np.mean([w < 0.5 * med for w in ws]) > short_frac or len(boxes) > 80 + + def _stage_b(self, A, page_indices=None, dpi=None, with_polygons=False): + """recognise (pooled) -> 180° fallback -> capped second pass -> layout join -> assemble. GPU(rec)+CPU.""" + t0 = time.perf_counter(); imgs, boxes, rot, ang, jobs = A["imgs"], A["boxes"], A["rot"], A["ang"], A["jobs"]; n = len(imgs) + res = self.rec(A["flat"], bs=self.rec_bs) if A["flat"] else [] + texts = [[] for _ in range(n)] + for (i, _), r in zip(jobs, res): texts[i].append(r) + # 180° candidates: low mean confidence OR weak characters (upside-down pages read as confident garbage, min-char ~0.3) + top5 = lambda r: sorted(r, key=lambda tc: -len(tc[0]))[:5] + flip = [i for i in range(n) if texts[i] and (np.mean([c for _, c in texts[i]]) < 0.55 or orient_score(top5(texts[i])) < 1.35)] + if flip: + fimgs = [rotate_page(imgs[i], 2) for i in flip]; fboxes = self._detect(fimgs) + fcrops = [self._crops(im, b) for im, b in zip(fimgs, fboxes)]; ff = [c for cs in fcrops for c in cs]; fres = self.rec(ff, bs=self.rec_bs) if ff else []; p = 0 + for i, im, b, cs in zip(flip, fimgs, fboxes, fcrops): + r = fres[p:p + len(cs)]; p += len(cs) + if r and orient_score(top5(r)) > orient_score(top5(texts[i])) + 0.10: imgs[i], boxes[i], texts[i], rot[i] = im, b, r, (rot[i] + 180) % 360 + # confidence-gated second pass: only the lowest-confidence lines (cap per page), pooled across the batch + low = [] + for i in range(n): + cand = sorted([(c, j) for j, (_, c) in enumerate(texts[i]) if c < self.tta_conf])[:self.tta_cap]; low += [(i, j) for _, j in cand] + if low: + alt = list(self.pool.map(lambda ij: self._crops(imgs[ij[0]], [boxes[ij[0]][ij[1]]], pad=(0.06, 0.08))[0], low)) + for (i, j), (tx, c) in zip(low, self.rec(alt, bs=self.rec_bs)): + if c > texts[i][j][1] + 0.02: texts[i][j] = (tx, c) + t_rec = time.perf_counter() - t0; t1 = time.perf_counter() + blocks = [[] for _ in range(n)] + if A["lay_future"]: + for i, bl in zip(A["need_lay"], A["lay_future"].result()): blocks[i] = bl + for i in range(n): # layout ran on the un-deskewed page: rotate its blocks into the deskewed frame + if abs(ang[i]) > 0.4 and blocks[i]: + hh, ww = imgs[i].shape[:2]; M = cv2.getRotationMatrix2D((ww / 2, hh / 2), ang[i], 1.0) + for b in blocks[i]: + x0, y0, x1, y1 = b["bbox"]; pts = cv2.transform(np.array([[[x0, y0], [x1, y0], [x1, y1], [x0, y1]]], np.float32), M)[0] + b["bbox"] = [float(pts[:, 0].min()), float(pts[:, 1].min()), float(pts[:, 0].max()), float(pts[:, 1].max())] + t_lay = time.perf_counter() - t1; t2 = time.perf_counter() + def build(i): + d = assemble(imgs[i].shape[1], imgs[i].shape[0], blocks[i], boxes[i], texts[i], self.model_ids, None, (page_indices or list(range(n)))[i], dpi, with_polygons, img=imgs[i]) + d["page"]["rotation"] = rot[i]; d["page"]["deskew_deg"] = round(ang[i], 2); d["page"]["second_pass_lines"] = sum(1 for a, _ in low if a == i); d["page"]["layout_run"] = i in A["need_lay"]; return d + docs = list(self.pool.map(build, range(n))) + T = {"det+crop": A["t_a"], "rec": t_rec, "layout_wait": t_lay, "assemble": time.perf_counter() - t2} + for d in docs: d["timing_ms"] = {k: round(v * 1000 / n, 1) for k, v in T.items()} + return docs, T + + def process(self, imgs, page_indices=None, dpi=None, with_polygons=False): + """One page batch, stages A then B (no overlap). Use run() for the pipelined version.""" + t0 = time.perf_counter(); A = self._stage_a(imgs); docs, T = self._stage_b(A, page_indices, dpi, with_polygons); T["total"] = time.perf_counter() - t0 + return docs, T + + def run(self, paths, page_batch=None, loader=None, with_polygons=False): + """Pipelined batched run over all pages; returns (docs, stats). See `stream` for the generator variant.""" + docs = []; stats = {} + for d, st in self.stream(paths, page_batch, loader, with_polygons): docs.append(d); stats = st + return docs, stats + + def stream(self, paths, page_batch=None, loader=None, with_polygons=False): + """Pipelined: decode(k+2) | stage A(k+1) | stage B(k) run concurrently. stage A is detector-GPU + CPU crops, + stage B is recogniser-GPU + CPU assembly, so the two GPU models interleave and the CPU work hides behind them. + paths: file paths (decoded with cv2.imread) or anything `loader` turns into a BGR array. + Yields (doc, running_stats) per page as soon as its page batch is finished; the last stats dict is the total.""" + page_batch = int(page_batch or self.cfg["runtime"].get("page_batch", 8)); loader = loader or cv2.imread + batches = [paths[i:i + page_batch] for i in range(0, len(paths), page_batch)] + if not batches: return + dec = lambda ps: list(self.pool.map(loader, ps)) + stageA = ThreadPoolExecutor(1) # one worker: stage A of batch k+1 overlaps stage B of batch k + agg = {}; npages = 0; t0 = time.perf_counter() + try: + imgs0 = dec(batches[0]); a_fut = stageA.submit(self._stage_a, imgs0); nxt = self.pool.submit(dec, batches[1]) if len(batches) > 1 else None + for k in range(len(batches)): + A = a_fut.result() + if k + 1 < len(batches): + imgs1 = nxt.result(); nxt = self.pool.submit(dec, batches[k + 2]) if k + 2 < len(batches) else None + a_fut = stageA.submit(self._stage_a, imgs1) + d, T = self._stage_b(A, page_indices=list(range(npages, npages + len(A["imgs"]))), with_polygons=with_polygons); npages += len(d) + for kk, v in T.items(): agg[kk] = agg.get(kk, 0.0) + v + wall = time.perf_counter() - t0 + st = {"pages": npages, "wall_s": wall, "pages_per_s": npages / max(wall, 1e-9), "ms_per_page": {kk: round(v * 1000 / max(npages, 1), 1) for kk, v in agg.items()}} + for doc in d: yield doc, st + finally: + stageA.shutdown(wait=False) diff --git a/kn-en/kanen_infer/charset.py b/kn-en/kanen_infer/charset.py new file mode 100644 index 0000000000000000000000000000000000000000..d11d457151055bcef2862db4b04531cdc392247a --- /dev/null +++ b/kn-en/kanen_infer/charset.py @@ -0,0 +1,72 @@ +"""KanEn dictionary + label normaliser. +Classes: CTC blank (0) + ASCII printable (incl. space) + Kannada block (used code points) + document symbols. +Everything NFC. ZWJ/ZWNJ are stripped (decision: fonts render the same joined form; keep labels visual).""" +import unicodedata, re + +ASCII = [chr(i) for i in range(32, 127)] # 95 incl. space +KANNADA = [chr(c) for c in range(0x0C80, 0x0CF4)] # candidate block; filtered by 'assigned' +KANNADA = [c for c in KANNADA if unicodedata.category(c) not in ("Cn",)] +SYMBOLS = list("₹–—‘’“”•·°×÷½¼¾§¶।॥…‑‐−™©®") +ZW = {"\u200c", "\u200d", "\u200b", "\ufeff", "\u2060", "\xad", "\u200e", "\u200f"} + +def build_dict(): + seen, out = set(), [] + for c in ASCII + KANNADA + SYMBOLS: + if c not in seen: seen.add(c); out.append(c) + return out # index 0 in the model = blank; chars start at 1 + +CHARS = build_dict() +CHAR2ID = {c: i + 1 for i, c in enumerate(CHARS)} +NUM_CLASSES = len(CHARS) + 1 + +_ws = re.compile(r"[ \t ]+") +QUOTE_MAP = {"‘": "'", "’": "'", "“": '"', "”": '"'} # optional folding for eval only + +def normalize(s: str, fold_quotes: bool = False) -> str: + s = unicodedata.normalize("NFC", s) + s = "".join(ch for ch in s if ch not in ZW) + s = s.replace("಼", "") # Kannada nukta: not used in printed text + if fold_quotes: + s = "".join(QUOTE_MAP.get(ch, ch) for ch in s) + s = _ws.sub(" ", s).strip() + return s + +def encode(s: str): + ids, oov = [], [] + for ch in s: + i = CHAR2ID.get(ch) + if i is None: oov.append(ch) + else: ids.append(i) + return ids, oov + +def decode(ids): + return "".join(CHARS[i - 1] for i in ids if 0 < i <= len(CHARS)) + +def script_of(s: str) -> str: + kn = sum(0x0C80 <= ord(c) <= 0x0CFF for c in s); en = sum(c.isascii() and c.isalpha() for c in s) + dig = sum(c.isdigit() for c in s) + if kn and en: return "mixed" + if kn: return "kn" + if en: return "en" + return "num" if dig else "other" + +if __name__ == "__main__": + print("classes incl. blank:", NUM_CLASSES, "| kannada:", len(KANNADA), "| ascii:", len(ASCII), "| symbols:", len(SYMBOLS)) + + +# --- Kannada orthographic sanity (label QA only; never used to alter model output) --- +_VSIGN = set(chr(c) for c in range(0x0CBE, 0x0CCD)) | {"\u0CD5", "\u0CD6"} # dependent vowel signs + length marks +_VIRAMA = "\u0CCD"; _CONS = set(chr(c) for c in range(0x0C95, 0x0CBA)) | {"\u0CDE"} # incl. archaic ೞ +def kannada_orthography_issues(s: str): + """Returns a list of reasons the string cannot be valid printed Kannada (empty list = looks valid).""" + issues = [] + for i, ch in enumerate(s): + prev = s[i - 1] if i else "" + if ch in _VSIGN and (prev in _VSIGN or prev == _VIRAMA or prev == "" or prev == " "): issues.append(f"vowel sign after '{prev}' at {i}") + if ch == _VIRAMA and (prev == _VIRAMA or prev in _VSIGN or prev not in _CONS): issues.append(f"virama after '{prev}' at {i}") + # Latin/ASCII letters glued inside a Kannada word (no space between scripts) + import re + if re.search(r"[\u0C80-\u0CFF][A-Za-z]|[A-Za-z][\u0C80-\u0CFF]", s): issues.append("latin letter glued to kannada") + if re.search(r"[0-9][\u0CBE-\u0CCD]", s): issues.append("digit followed by kannada mark") + if re.search(r"[\u0CDE\u0CF1\u0CF2]", s): issues.append("rare/archaic kannada letter") + return issues diff --git a/kn-en/kanen_infer/cli.py b/kn-en/kanen_infer/cli.py new file mode 100644 index 0000000000000000000000000000000000000000..37b2b5fce5a91157bf57636c9e17236035653116 --- /dev/null +++ b/kn-en/kanen_infer/cli.py @@ -0,0 +1,67 @@ +"""`kanen-ocr` command line. + + kanen-ocr --input scans/ --out-dir out/ --format all + kanen-ocr --input "pages/*.png" --out-dir out/ --format json --page-batch 8 --workers 8 --provider trt + kanen-ocr --input report.pdf --out-dir out/ --format txt,tsv --json-lines + +Exit codes: 0 ok · 1 no input pages · 2 models/engines failed to load · 3 some pages failed (others written). +The summary line reports pages, pages/s, the provider each model ended on and every fallback that happened.""" +import argparse, json, os, sys, time, traceback +from pathlib import Path + +FORMAT_CHOICES = ["json", "md", "txt", "hocr", "alto", "tsv", "all"] + +def build_parser(): + ap = argparse.ArgumentParser(prog="kanen-ocr", description="Arjuna-OCR [Kn&En]: layout-aware Kannada + English document OCR (kanen-ocr/1.0 JSON and other serialisations)") + ap.add_argument("--input", "-i", nargs="+", default=None, help="image file(s), directory, glob, or PDF") + ap.add_argument("--out-dir", "-o", default="kanen_out", help="output directory (one file per page and format)") + ap.add_argument("--format", "-f", default="json", help="comma list of json,md,txt,hocr,alto,tsv or 'all'") + ap.add_argument("--json-lines", action="store_true", help="also write results.jsonl (one document per line) and results.json (list)") + ap.add_argument("--page-batch", type=int, default=None); ap.add_argument("--layout-mode", choices=["always", "auto", "never"], default=None) + ap.add_argument("--workers", type=int, default=None); ap.add_argument("--provider", choices=["auto", "trt", "cuda", "cpu"], default=None) + ap.add_argument("--polygons", action="store_true", help="include 4-point polygons in line objects"); ap.add_argument("--pdf-dpi", type=int, default=200) + ap.add_argument("--config", default=None, help="alternative kanen.yaml / .json"); ap.add_argument("--quiet", "-q", action="store_true"); ap.add_argument("--limit", type=int, default=0) + ap.add_argument("--version", action="store_true", help="print version and exit") + return ap + +def main(argv=None): + a = build_parser().parse_args(argv) + from . import __version__ + if a.version: print(__version__); return 0 + if not a.input: build_parser().error("the following arguments are required: --input/-i") + from .api import KanEnOCR + from .io import expand_inputs + from .formats import FORMATS, render + fmts = FORMAT_CHOICES[:-1] if a.format == "all" else [f.strip() for f in a.format.split(",") if f.strip()] + bad = [f for f in fmts if f not in FORMATS] + if bad: print(f"unknown format(s) {bad}; choose from {FORMAT_CHOICES}", file=sys.stderr); return 1 + paths = expand_inputs(a.input); paths = paths[:a.limit] if a.limit else paths + if not paths: print(f"no input pages match {a.input}", file=sys.stderr); return 1 + say = (lambda *x: None) if a.quiet else (lambda *x: print(*x, file=sys.stderr, flush=True)) + t0 = time.time() + try: + ocr = KanEnOCR(config=a.config, providers=a.provider, layout_mode=a.layout_mode, workers=a.workers) + except Exception as e: + print(f"failed to load models: {type(e).__name__}: {e}", file=sys.stderr); traceback.print_exc(); return 2 + r = ocr.ready(); prov = ", ".join(f"{k}={v.get('provider')}{'' if v.get('provider') != 'trt' else ('-fp16' if v.get('fp16') else '-fp32')}" for k, v in r["providers"].items()) + say(f"{r['name']} {r['version']} | providers: {prov} | load {r['load_s']}s warm-up {r['warmup_s']}s | {len(paths)} input file(s)") + out = Path(a.out_dir); out.mkdir(parents=True, exist_ok=True); docs = []; failed = 0; jl = open(out / "results.jsonl", "w", encoding="utf-8") if a.json_lines else None + try: + for doc in ocr.stream(paths, page_batch=a.page_batch, polygons=a.polygons, pdf_dpi=a.pdf_dpi): + label = doc["page"].get("file", f"page_{doc['page']['index']}"); base, _, pg = label.partition("#") # PDFs: name.pdf#3 -> name_p3 + stem = os.path.splitext(Path(base).name)[0] + (f"_p{pg}" if pg else "") + for f in fmts: + (out / (stem + FORMATS[f])).write_text(render(doc, f), encoding="utf-8") + if jl: jl.write(json.dumps(doc, ensure_ascii=False) + "\n") + docs.append(doc) + except Exception as e: + failed += 1; print(f"page failed: {type(e).__name__}: {e}", file=sys.stderr); traceback.print_exc() + finally: + if jl: jl.close() + if a.json_lines: (out / "results.json").write_text(json.dumps(docs, ensure_ascii=False, indent=1), encoding="utf-8") + st = getattr(ocr, "last_stats", {}) or {}; wall = time.time() - t0; n = len(docs) + fb = [e for v in ocr.ready()["providers"].values() for e in v.get("fallbacks", [])] + say(f"done: {n} pages in {st.get('wall_s', wall):.1f} s = {st.get('pages_per_s', n / max(wall, 1e-9)):.2f} pages/s | ms/page {st.get('ms_per_page', {})} | providers: {prov} | fallbacks: {len(fb)}{(' -> ' + '; '.join(fb)) if fb else ''} | wrote {out}") + return 3 if failed else 0 + +if __name__ == "__main__": sys.exit(main()) diff --git a/kn-en/kanen_infer/config.py b/kn-en/kanen_infer/config.py new file mode 100644 index 0000000000000000000000000000000000000000..3345858c8737981a6b52f1248ee4f7e42c5b6e94 --- /dev/null +++ b/kn-en/kanen_infer/config.py @@ -0,0 +1,77 @@ +"""Configuration for the Arjuna-OCR [Kn&En] inference package: `kanen.yaml` next to the package + environment overrides. +Paths are resolved relative to the package root with pathlib (Linux and Windows).""" +import copy, json, logging, os +from pathlib import Path + +PKG_ROOT = Path(__file__).resolve().parent.parent # directory that holds kanen.yaml, models/, trt_cache/ +DEFAULT_CONFIG = PKG_ROOT / "kanen.yaml" +LOG = logging.getLogger("kanen") + +_FALLBACK = { # used when kanen.yaml is missing or a key is absent; identical to the shipped kanen.yaml + "name": "Arjuna-OCR [Kn&En]", "version": "1.2.0", + "models": {"det": "models/det/inference.onnx", "layout": "models/layout/kanen_layout_v12.onnx", "layout_labels": "models/layout/classes.txt", + "layout_selftest": "models/layout/selftest.jpg", "layout_selftest_ref": "models/layout/selftest_ref.json", "rec": "models/rec/kanen_rec_e5.onnx", "charset": "models/rec/charset.json"}, + "provider": {"policy": "auto", "trt_fp16": True, "engine_cache": "trt_cache", "trt_workspace_gb": 4, "cuda_mem_limit_gb": 6, "cudnn_conv_algo_search": "HEURISTIC"}, + "detector": {"canvases": [1280, 1920, 2560], "width_classes": [0.5, 0.625, 0.75, 0.875, 1.0], "min_scale": 0.9, "default_canvas": 1280, + "thresh": 0.2, "box_thresh": 0.45, "unclip": 1.4, "pad_top": 0.03, "pad_bottom": 0.25, "batch": 2}, + "layout": {"mode": "always", "batch": 4, "score_thresh": 0.3, "fill": [1.0, 0.8], "selftest_min_match": 1.0}, + "recognizer": {"buckets": [128, 192, 256, 384, 512, 640, 768, 1024, 1280], "batch": 64, "height": 32, "max_ratio": 40}, + "second_pass": {"conf": 0.85, "cap": 6}, + "runtime": {"workers": "auto", "page_batch": 8, "log_level": "warning", "polygons": False}, +} + +def _merge(base, over): + out = copy.deepcopy(base) + for k, v in (over or {}).items(): + out[k] = _merge(out[k], v) if isinstance(v, dict) and isinstance(out.get(k), dict) else copy.deepcopy(v) + return out + +def _read(path): + p = Path(path) + text = p.read_text(encoding="utf-8") + if p.suffix.lower() == ".json": return json.loads(text) + import yaml + return yaml.safe_load(text) or {} + +class Config(dict): + """dict with attribute access and resolved paths; `cfg.path('models','rec')` -> absolute Path.""" + root: Path + def __getattr__(self, k): + try: return self[k] + except KeyError as e: raise AttributeError(k) from e + def path(self, *keys) -> Path: + v = self + for k in keys: v = v[k] + p = Path(str(v)).expanduser() + return p if p.is_absolute() else (self.root / p) + def workers(self) -> int: + w = self["runtime"].get("workers", "auto") + return int(w) if str(w).isdigit() else max(4, (os.cpu_count() or 4) // 2) + +def load_config(config=None, overrides=None) -> Config: + """config: None (kanen.yaml next to the package, or $KANEN_CONFIG), a path, or a dict. overrides: nested dict applied last. + Environment: KANEN_TRT=1 -> provider.policy 'trt'; KANEN_ORT_CPU=1 -> 'cpu'; KANEN_REC_ONNX -> models.rec; KANEN_LOG -> runtime.log_level.""" + src = None + if isinstance(config, dict): data = _merge(_FALLBACK, config); root = PKG_ROOT + else: + src = Path(config) if config else Path(os.environ.get("KANEN_CONFIG") or DEFAULT_CONFIG) + data = _merge(_FALLBACK, _read(src)) if src.exists() else copy.deepcopy(_FALLBACK) + root = src.resolve().parent if src.exists() else PKG_ROOT + if overrides: data = _merge(data, overrides) + if os.environ.get("KANEN_ORT_CPU"): data["provider"]["policy"] = "cpu" + elif os.environ.get("KANEN_TRT") == "1": data["provider"]["policy"] = "trt" + if os.environ.get("KANEN_REC_ONNX"): data["models"]["rec"] = os.environ["KANEN_REC_ONNX"] + if os.environ.get("KANEN_LOG"): data["runtime"]["log_level"] = os.environ["KANEN_LOG"] + cfg = Config(data); cfg.root = root; cfg.source = str(src) if src else "" + setup_logging(cfg["runtime"].get("log_level", "warning")) + return cfg + +def setup_logging(level="warning"): + lvl = getattr(logging, str(level).upper(), logging.WARNING) + if not LOG.handlers: + h = logging.StreamHandler(); h.setFormatter(logging.Formatter("[%(name)s] %(message)s")); LOG.addHandler(h) + LOG.setLevel(lvl); LOG.propagate = False + try: + import onnxruntime as ort + ort.set_default_logger_severity(3 if lvl >= logging.WARNING else 1) # silence TensorRT/ORT chatter unless debug/info + except Exception: pass diff --git a/kn-en/kanen_infer/formats.py b/kn-en/kanen_infer/formats.py new file mode 100644 index 0000000000000000000000000000000000000000..ee31938c0a1a3b1aa3cf42e6d2c07fc6e486492b --- /dev/null +++ b/kn-en/kanen_infer/formats.py @@ -0,0 +1,76 @@ +"""Serialisations of one `kanen-ocr/1.0` document. All of them are views of the same JSON: nothing is filtered, +re-ordered or rewritten — the text, order, confidences and flags are exactly those of the document. + + to_json(doc) the document itself (schema kanen-ocr/1.0) + to_markdown(doc) Markdown: titles, paragraphs, tables as pipe tables, figure/seal markers + to_text(doc) reading-order plain text: one block per paragraph (blank line between), table rows one per line with ' | ' + to_hocr(doc) hOCR 1.2 (ocr_page > ocr_carea > ocr_line > ocrx_word), bbox + x_wconf per line + to_alto(doc) ALTO 4 XML (TextBlock > TextLine > String with WC), one String per line + to_tsv(doc) page, block, line, bbox, text, confidence, min_conf, flags (tab separated, header row) +""" +import json +from xml.sax.saxutils import escape, quoteattr +from .assemble import to_markdown # noqa: F401 (re-exported) + +FORMATS = {"json": ".json", "md": ".md", "txt": ".txt", "hocr": ".hocr", "alto": ".xml", "tsv": ".tsv"} + +def to_json(doc, indent=1): + return json.dumps(doc, ensure_ascii=False, indent=indent) + +def block_text(b): + if b.get("type") == "table" and b.get("rows"): return "\n".join(" | ".join(c["text"] for c in r["cells"]) for r in b["rows"]) + return "\n".join(l["text"] for l in b.get("lines", [])) + +def to_text(doc): + parts = [t for t in (block_text(b) for b in doc["blocks"]) if t.strip()] + return "\n\n".join(parts) + "\n" + +def _ibox(bb): return " ".join(str(int(round(v))) for v in bb) + +def to_hocr(doc, title=None, name="Arjuna-OCR [Kn&En]"): + pg = doc["page"]; ver = doc.get("model", {}).get("pipeline", ""); pidx = pg.get("index", 0) + fn = pg.get("file") or f"page_{pidx}" + out = ['', '', + '', "", f"{escape(title or fn)}", '', + f'', '', "", "", + f'
'] + for b in doc["blocks"]: + bid = b.get("id", "b"); out.append(f'
') + out.append(f'

') + for l in b.get("lines", []): + conf = int(round(100 * float(l["confidence"]))); lid = l.get("id", bid + ".l"); t = "bbox " + _ibox(l["bbox"]) + out.append(f'' f'{escape(l["text"])}') + out.append("

") + out += ["
", "", "", ""] + return "\n".join(out) + +def to_alto(doc, name="Arjuna-OCR [Kn&En]"): + pg = doc["page"]; ver = doc.get("model", {}).get("pipeline", ""); pidx = pg.get("index", 0); fn = pg.get("file") or f"page_{pidx}" + out = ['', '', "", "pixel", + f"{escape(fn)}", + f"{escape(name)}{escape(str(ver))}", + "", "", f'', ""] + def geo(bb): x0, y0, x1, y1 = bb; return f'HPOS="{int(round(x0))}" VPOS="{int(round(y0))}" WIDTH="{int(round(x1 - x0))}" HEIGHT="{int(round(y1 - y0))}"' + for b in doc["blocks"]: + bid = b.get("id", "b"); out.append(f'') + for l in b.get("lines", []): + lid = l.get("id", bid + ".l") + out.append(f'') + out.append("") + out += ["", "", "", "", ""] + return "\n".join(out) + +TSV_HEADER = "page\tblock\tline\tbbox\ttext\tconfidence\tmin_conf\tflags" + +def tsv_rows(doc): + pg = doc["page"]; page = pg.get("file") or str(pg.get("index", 0)) + for b in doc["blocks"]: + for l in b.get("lines", []): + yield [page, b.get("id", ""), l.get("id", ""), ",".join(str(int(round(v))) for v in l["bbox"]), l["text"].replace("\t", " "), f'{float(l["confidence"]):.4f}', f'{float(l.get("min_conf", l["confidence"])):.4f}', " ".join(l.get("flags", []))] + +def to_tsv(doc, header=True): + rows = ["\t".join(r) for r in tsv_rows(doc)] + return ((TSV_HEADER + "\n") if header else "") + "\n".join(rows) + ("\n" if rows else "") + +def render(doc, fmt): + return {"json": to_json, "md": to_markdown, "txt": to_text, "hocr": to_hocr, "alto": to_alto, "tsv": to_tsv}[fmt](doc) diff --git a/kn-en/kanen_infer/geometry.py b/kn-en/kanen_infer/geometry.py new file mode 100644 index 0000000000000000000000000000000000000000..5c57f572d32f1627cdd46fe6dfaeaabbbd15bb31 --- /dev/null +++ b/kn-en/kanen_infer/geometry.py @@ -0,0 +1,19 @@ +"""Page geometry helpers (orientation / deskew).""" +import math, numpy as np, cv2 + +def rotate_page(img, k): # k in {0,1,2,3} multiples of 90° counter-clockwise + return np.ascontiguousarray(np.rot90(img, k)) if k else img + +def deskew_angle(boxes): + """Median angle (deg) of wide line boxes; positive = counter-clockwise tilt.""" + angs = [] + for b, _ in boxes: + w = np.linalg.norm(b[1] - b[0]); h = np.linalg.norm(b[3] - b[0]) + if w > 4 * h and w > 60: + angs.append(math.degrees(math.atan2(b[1][1] - b[0][1], b[1][0] - b[0][0]))) + return float(np.median(angs)) if len(angs) >= 5 else 0.0 + +def rotate_small(img, deg): + h, w = img.shape[:2]; M = cv2.getRotationMatrix2D((w / 2, h / 2), deg, 1.0) + return cv2.warpAffine(img, M, (w, h), flags=cv2.INTER_LINEAR, borderValue=(255, 255, 255)) + diff --git a/kn-en/kanen_infer/io.py b/kn-en/kanen_infer/io.py new file mode 100644 index 0000000000000000000000000000000000000000..1e12a6931be4da49c5bfcfb7c9e97c91287b298e --- /dev/null +++ b/kn-en/kanen_infer/io.py @@ -0,0 +1,68 @@ +"""Input handling: file path, bytes, numpy BGR array, PIL image, directory / glob expansion, and PDF pages +(rendered with pypdfium2 if installed, else `pdftoppm` from poppler if on PATH; both optional).""" +import glob, io as _io, os, shutil, subprocess, tempfile +from pathlib import Path +import numpy as np, cv2 + +IMAGE_EXT = {".png", ".jpg", ".jpeg", ".tif", ".tiff", ".bmp", ".webp", ".jp2"} +PDF_EXT = {".pdf"} + +def to_bgr(x): + """Any supported input -> numpy uint8 BGR (H, W, 3). Raises ValueError if it cannot be decoded.""" + if isinstance(x, np.ndarray): + if x.ndim == 2: return cv2.cvtColor(x, cv2.COLOR_GRAY2BGR) + if x.ndim == 3 and x.shape[2] == 4: return cv2.cvtColor(x, cv2.COLOR_BGRA2BGR) + if x.ndim == 3 and x.shape[2] == 3: return np.ascontiguousarray(x) + raise ValueError(f"unsupported array shape {x.shape}") + if isinstance(x, (bytes, bytearray, memoryview)): + im = cv2.imdecode(np.frombuffer(bytes(x), np.uint8), cv2.IMREAD_COLOR) + if im is None: raise ValueError("could not decode image bytes") + return im + if isinstance(x, (str, os.PathLike)): + im = cv2.imdecode(np.fromfile(str(x), np.uint8), cv2.IMREAD_COLOR) # fromfile: unicode paths on Windows + if im is None: raise ValueError(f"could not read image {x}") + return im + if hasattr(x, "convert") and hasattr(x, "size"): # PIL.Image without importing PIL + return cv2.cvtColor(np.asarray(x.convert("RGB")), cv2.COLOR_RGB2BGR) + if hasattr(x, "read"): return to_bgr(x.read()) + raise ValueError(f"unsupported input type {type(x).__name__}") + +def expand_inputs(inputs): + """file / directory / glob / list -> sorted list of image and PDF paths (strings).""" + if isinstance(inputs, (str, os.PathLike)): inputs = [inputs] + out = [] + for it in inputs: + p = Path(it) + if p.is_dir(): out += sorted(str(f) for f in p.iterdir() if f.suffix.lower() in IMAGE_EXT | PDF_EXT) + elif p.exists(): out.append(str(p)) + else: + g = sorted(glob.glob(str(it), recursive=True)) + out += [f for f in g if Path(f).suffix.lower() in IMAGE_EXT | PDF_EXT] + return out + +def pdf_pages(path, dpi=200, pages=None): + """Render a PDF to BGR page images. Uses pypdfium2 when importable, else poppler's pdftoppm; ImportError otherwise. + pages: optional 1-based list of page numbers.""" + try: + import pypdfium2 as pdfium + doc = pdfium.PdfDocument(str(path)); idx = [p - 1 for p in pages] if pages else range(len(doc)); out = [] + for i in idx: + bm = doc[i].render(scale=dpi / 72.0); pil = bm.to_pil(); out.append(cv2.cvtColor(np.asarray(pil.convert("RGB")), cv2.COLOR_RGB2BGR)) + return out + except ImportError: pass + exe = shutil.which("pdftoppm") + if not exe: raise ImportError("PDF input needs `pip install pypdfium2` or poppler's pdftoppm on PATH") + with tempfile.TemporaryDirectory() as td: + cmd = [exe, "-r", str(dpi), "-png"] + if pages: cmd += ["-f", str(min(pages)), "-l", str(max(pages))] + subprocess.run(cmd + [str(path), os.path.join(td, "p")], check=True, capture_output=True) + files = sorted(Path(td).glob("p*.png"), key=lambda f: int("".join(ch for ch in f.stem if ch.isdigit()) or 0)) + if pages: files = [f for f in files if int("".join(ch for ch in f.stem if ch.isdigit())) in set(pages)] + return [to_bgr(str(f)) for f in files] + +def iter_page_sources(paths, dpi=200): + """Yield (label, loader) per page: images load lazily by path; PDFs are rendered page by page.""" + for p in paths: + if Path(p).suffix.lower() in PDF_EXT: + for k, im in enumerate(pdf_pages(p, dpi=dpi)): yield f"{Path(p).name}#{k + 1}", im + else: yield Path(p).name, p diff --git a/kn-en/kanen_infer/pipeline_base.py b/kn-en/kanen_infer/pipeline_base.py new file mode 100644 index 0000000000000000000000000000000000000000..2767c97c5b5055853a9242a35132bdd2b393db8e --- /dev/null +++ b/kn-en/kanen_infer/pipeline_base.py @@ -0,0 +1,201 @@ +"""Detector (PP-OCRv6 det, DB post-processing) and Layout (Arjuna layout v12, DocLayout-YOLO based) models on ONNX Runtime, plus the crop helpers. +Parameters come from the package config (`kanen.yaml`); sessions come from `providers.Providers` (trt -> cuda -> cpu). +The post-processing is identical to the v1.0 reference: no thresholds, padding or ordering were changed.""" +import logging, os, shutil, time +import numpy as np, cv2, pyclipper, yaml +from shapely.geometry import Polygon +from .config import load_config, PKG_ROOT +from .providers import Providers + +LOG = logging.getLogger("kanen") +ROOT = str(PKG_ROOT) # kept for callers that import ROOT +MEAN = np.array([0.485, 0.456, 0.406], np.float32); STD = np.array([0.229, 0.224, 0.225], np.float32) + +class Detector: + def __init__(self, cfg=None, providers=None, limit=None): + # pad_bottom=0.25: Kannada below-baseline conjuncts were clipped in 88 % of lines with DB's default boxes; + # 25 % bottom extension -> 1.2 % clipped, 0 intrusion into the next line (measured on MILE, 2026-09-13). + self.cfg = cfg or load_config(); self.prov = providers or Providers(self.cfg); d = self.cfg["detector"] + self.path = str(self.cfg.path("models", "det")) + self.limit = int(limit or d.get("default_canvas", 1280)); self.thresh = float(d["thresh"]); self.box_thresh = float(d["box_thresh"]); self.unclip = float(d["unclip"]) + self.pad_top, self.pad_bottom = float(d["pad_top"]), float(d["pad_bottom"]) + # one TensorRT engine covering every canvas shape the batched path can produce (long side in `canvases`, short side + # >= min(width_classes) * smallest canvas, batch <= detector.batch): built once by prepare_engines, never rebuilt mid-run + cv = [int(c) for c in d["canvases"]]; lo = max(32, int(round(cv[0] * min(float(f) for f in d["width_classes"]) / 32)) * 32); hi = max(cv); nb = int(d.get("batch", 2)) + self.trt_opts = {"trt_profile_min_shapes": f"x:1x3x{lo}x{lo}", "trt_profile_opt_shapes": f"x:{nb}x3x1920x1216", "trt_profile_max_shapes": f"x:{nb}x3x{hi}x{hi}"} + # one ORT session per canvas size: the CUDA EP re-plans (~1 s) every time the input shape changes, so the + # scale-normalised re-detects must not share a session with the default canvas. + self._sessions = {}; self.s = self._session(self.limit) + def _session(self, limit): + if limit not in self._sessions: + s = self.prov.session(self.path, "det", trt_opts=self.trt_opts); x = np.zeros((1, 3, limit, limit), np.float32); s.run(None, {s.get_inputs()[0].name: x}) # warm-up plan + self._sessions[limit] = s + return self._sessions[limit] + @property + def provider(self): return self.prov.status.get("det", {}).get("provider") + + def __call__(self, img_bgr): + h, w = img_bgr.shape[:2]; self.s = self._session(self.limit) + r = min(1.0, self.limit / max(h, w)) + nh, nw = max(32, int(round(h * r / 32)) * 32), max(32, int(round(w * r / 32)) * 32) + canvas = np.full((self.limit, self.limit, 3), 255, np.uint8) # fixed shape -> no ORT re-planning + canvas[:nh, :nw] = cv2.resize(img_bgr, (nw, nh)) + x = ((canvas.astype(np.float32) / 255.0 - MEAN) / STD).transpose(2, 0, 1)[None] + prob = self.s.run(None, {self.s.get_inputs()[0].name: x})[0][0, 0][:nh, :nw] + return self._post(prob, w / nw, h / nh) + def _post(self, prob, sx, sy): + bit = (prob > self.thresh).astype(np.uint8) + cnts, _ = cv2.findContours(bit, cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE) + out = [] + for c in cnts[:3000]: + if len(c) < 4: continue + pts = c.reshape(-1, 2) + x0, y0 = pts.min(0); x1, y1 = pts.max(0) + 1 # score on the bounding rect only (full-map masks cost ~4 ms/box) + mask = np.zeros((y1 - y0, x1 - x0), np.uint8); cv2.fillPoly(mask, [pts - (x0, y0)], 1) + score = prob[y0:y1, x0:x1][mask == 1].mean() if mask.any() else 0 + if score < self.box_thresh: continue + poly = Polygon(pts) + if poly.area < 4: continue + d = poly.area * self.unclip / max(poly.length, 1e-6) + pc = pyclipper.PyclipperOffset(); pc.AddPath(pts.tolist(), pyclipper.JT_ROUND, pyclipper.ET_CLOSEDPOLYGON) + ex = pc.Execute(d) + if not ex: continue + rect = cv2.minAreaRect(np.array(ex[0], np.float32)) + if min(rect[1]) < 3: continue + box = cv2.boxPoints(rect); box = order_pts(box) + box[:, 0] *= sx; box[:, 1] *= sy + hgt = 0.5 * ((box[3, 1] - box[0, 1]) + (box[2, 1] - box[1, 1])) + box[0, 1] -= self.pad_top * hgt; box[1, 1] -= self.pad_top * hgt; box[2, 1] += self.pad_bottom * hgt; box[3, 1] += self.pad_bottom * hgt + box[:, 1] = np.clip(box[:, 1], 0, prob.shape[0] * sy) + out.append((box, float(score))) + return out + +def order_pts(b): + b = b[np.argsort(b[:, 0])] + l, r = b[:2][np.argsort(b[:2, 1])], b[2:][np.argsort(b[2:, 1])] + return np.array([l[0], r[0], r[1], l[1]], np.float32) + +def crop(img, box): + w = int(max(np.linalg.norm(box[0] - box[1]), np.linalg.norm(box[2] - box[3]))) + h = int(max(np.linalg.norm(box[0] - box[3]), np.linalg.norm(box[1] - box[2]))) + M = cv2.getPerspectiveTransform(box, np.array([[0, 0], [w, 0], [w, h], [0, h]], np.float32)) + return cv2.warpPerspective(img, M, (w, h), flags=cv2.INTER_CUBIC) + +class Layout: + """v1.2 layout model: DocLayout-YOLO (YOLOv10-based) fine-tuned for Kannada/English documents — 9 classes (title, text, + table, figure, seal, id_panel, header, footer, handwriting), 1024 letterbox input, top-300 selection inside the graph + (NMS-free), output (B, 300, 6) = x0 y0 x1 y1 conf cls in letterbox pixels. No reading-order head: the assembler's + xy-cut supplies the order. + TensorRT's timing-based tactic selection occasionally builds an engine that computes garbage for a correct graph + (seen for this model and for PP-DocLayoutV3 in v1.1: a fresh fp16 engine returned 2 of 7 blocks while PyTorch fp16, + ORT fp16 and rebuilt engines were exact). So every engine is self-tested against the bundled page + reference boxes + before use: fp16 -> fp16 rebuild -> fp32 TensorRT -> CUDA EP. The model itself is fp16-safe.""" + NAMES = ["title", "text", "table", "figure", "seal", "id_panel", "header", "footer", "handwriting"] + SIZE = 1024 + def __init__(self, cfg=None, providers=None): + self.cfg = cfg or load_config(); self.prov = providers or Providers(self.cfg); lc = self.cfg["layout"] + self.path = str(self.cfg.path("models", "layout")) + use32 = (self.prov.cache_root / "layout_fp32" / "USE_FP32").exists() # a previous run found the fp16 engine wrong on this GPU + self.s = self.prov.session(self.path, "layout", fp16=False, cache_sub="layout_fp32") if use32 else self.prov.session(self.path, "layout") + self.inp = self.s.get_inputs()[0].name + lp = self.cfg.path("models", "layout_labels") if self.cfg["models"].get("layout_labels") else None + self.labels = [l.strip() for l in open(lp, encoding="utf-8") if l.strip()] if lp and lp.exists() else list(self.NAMES) + self.trt = "Tensorrt" in self.s.get_providers()[0]; self.ok = set(); self.bs = int(lc.get("batch", 4)) # batch the batched pipeline should use + self.thr = float(lc.get("score_thresh", 0.3)); self.selftest_min_match = float(lc.get("selftest_min_match", 0.8)) + # fill: the page's long side occupies this fraction of the canvas. The YOLOv10 head regresses at most 15 x 32 = 480 px per + # side from an anchor, so a block wider/taller than ~900 px of a 1024 canvas (whole-page tables on landscape gazettes / + # assembly records) comes out truncated and under-confident at fill 1.0; at 0.8 every page box fits (table conf 0.18 -> 0.79). + f = lc.get("fill", [1.0, 0.8]); self.fills = [float(v) for v in (f if isinstance(f, (list, tuple)) else [f])]; self.fill = self.fills[0] + st = self.cfg.path("models", "layout_selftest"); self.test_img = cv2.imread(str(st)) if st.exists() else None + rp = self.cfg.path("models", "layout_selftest_ref") if self.cfg["models"].get("layout_selftest_ref") else None + self.test_ref = __import__("json").load(open(rp, encoding="utf-8")) if rp and rp.exists() else None + self.stage = ("trt-fp32" if use32 else "trt-fp16") if self.trt else self.prov.status["layout"]["provider"] + @property + def provider(self): return self.prov.status.get("layout", {}).get("provider") + def prep(self, img_bgr, fill=None): + """letterbox (top-left, pad 114) to SIZE x SIZE, RGB, 0-1 -> ((3,S,S) float32, scale)""" + S = self.SIZE; h, w = img_bgr.shape[:2]; r = S * (self.fill if fill is None else fill) / max(h, w); nh, nw = max(1, int(round(h * r))), max(1, int(round(w * r))) + can = np.full((S, S, 3), 114, np.uint8); can[:nh, :nw] = cv2.resize(img_bgr, (nw, nh), interpolation=cv2.INTER_AREA) + return can[:, :, ::-1].transpose(2, 0, 1).astype(np.float32) / 255.0, r + def run(self, x): + """raw graph call with a validated engine for this batch size; x: (bs,3,S,S) float32 -> (bs,300,6)""" + bs = x.shape[0] + if self.trt and bs not in self.ok: self.ensure(bs) + return self.s.run(None, {self.inp: x})[0] + def parse(self, out, r, thr=None): + thr = self.thr if thr is None else thr; blocks = [] + for x0, y0, x1, y1, sc, cls in out: + if not (sc >= thr) or not np.isfinite([x0, y0, x1, y1]).all() or not 0 <= int(cls) < len(self.labels): continue + blocks.append({"type": self.labels[int(cls)], "score": float(sc), "bbox": [float(x0 / r), float(y0 / r), float(x1 / r), float(y1 / r)], "order": None}) + return blocks + @staticmethod + def merge(primary, extra, page_hw, min_frac=0.7, iou_thr=0.5, covered=0.5): + """primary = boxes from the first fill (kept as they are); extra = boxes from the reduced fill(s), which exist only to + recover blocks too large for the head at full fill: an extra box is added when its long side is >= min_frac of the page's + long side, no same-class primary box overlaps it at IoU >= iou_thr, and same-class primary boxes cover < `covered` of its + area (a form page already split into sub-tables keeps the sub-tables). Geometry only.""" + L = float(max(page_hw)); out = list(primary) + def area(bb): return max(0.0, bb[2] - bb[0]) * max(0.0, bb[3] - bb[1]) + def inter(a, b): return max(0.0, min(a[2], b[2]) - max(a[0], b[0])) * max(0.0, min(a[3], b[3]) - max(a[1], b[1])) + for e in sorted(extra, key=lambda x: -x["score"]): + bb = e["bbox"] + if max(bb[2] - bb[0], bb[3] - bb[1]) < min_frac * L: continue + same = [k for k in out if k["type"] == e["type"]] + if any(inter(bb, k["bbox"]) / max(1e-6, area(bb) + area(k["bbox"]) - inter(bb, k["bbox"])) >= iou_thr for k in same): continue + if sum(inter(bb, k["bbox"]) for k in same) >= covered * area(bb): continue + out.append(e) + return out + def batch(self, imgs): + """blocks for a list of pages: every page is run at each configured fill (one canvas per fill), run at the validated + engine batch (zero-padded), and the per-fill boxes are merged by class-aware NMS.""" + if not imgs: return [] + items = [(i, f) for i in range(len(imgs)) for f in self.fills]; per = [{f: [] for f in self.fills} for _ in imgs]; BS = self.bs + for j in range(0, len(items), BS): + chunk = items[j:j + BS]; k = len(chunk); x = np.zeros((BS, 3, self.SIZE, self.SIZE), np.float32); rs = [] + for n, (i, f) in enumerate(chunk): x[n], r = self.prep(imgs[i], f); rs.append(r) + o = self.run(x) + for n, (i, f) in enumerate(chunk): per[i][f] += self.parse(o[n], rs[n]) + f0 = self.fills[0] + return [self.merge(d[f0], [b for f in self.fills[1:] for b in d[f]], imgs[i].shape[:2]) if len(self.fills) > 1 else d[f0] for i, d in enumerate(per)] + def _selftest(self, bs): + """fraction of the reference boxes (bundled page) that the engine reproduces (same class, IoU >= 0.7 at conf >= 0.3); + without a reference file: 1.0 if the engine returns >= 3 finite blocks, else 0.""" + if self.test_img is None: return 1.0 + x = np.zeros((bs, 3, self.SIZE, self.SIZE), np.float32); x[0], r = self.prep(self.test_img) + o = self.s.run(None, {self.inp: x})[0][0]; got = self.parse(o, 1.0, thr=0.3) + if not self.test_ref: return 1.0 if len(got) >= 3 else 0.0 + def iou(a, b): + ix = max(0, min(a[2], b[2]) - max(a[0], b[0])); iy = max(0, min(a[3], b[3]) - max(a[1], b[1])); i = ix * iy + return i / max(1e-6, (a[2] - a[0]) * (a[3] - a[1]) + (b[2] - b[0]) * (b[3] - b[1]) - i) + hit = sum(1 for ref in self.test_ref if any(g["type"] == ref["type"] and iou(g["bbox"], ref["bbox"]) >= 0.7 for g in got)) + return hit / max(1, len(self.test_ref)) + def ensure(self, bs): + """validate the TensorRT engine for batch `bs`. Plan: shipped/cached fp16 engine -> fp16 rebuilt with a 1..bs optimisation + profile (RTX 5060 Ti, TensorRT 10.16: the fp16 engine built for the fixed shape [bs=4] drops every wide block, the same + graph built for [1..4] is exact) -> fp32 TensorRT (own cache dir) -> CUDA EP. A fallback below fp16 is remembered in + `/layout_fp32/USE_FP32` so later runs go straight to the working engine instead of rebuilding fp16 each start.""" + st = self.prov.status["layout"]; marker = self.prov.cache_root / "layout_fp32" / "USE_FP32" + plan = [("fp16", True, "layout", False), ("fp16", True, "layout", True), ("fp32", False, "layout_fp32", False)] + start = 2 if (marker.exists() and self.stage == "trt-fp32") else 0 + for attempt in range(start, len(plan)): + tag, fp16, sub, warm = plan[attempt] + t = time.time() + if warm: self.s.run(None, {self.inp: np.zeros((1, 3, self.SIZE, self.SIZE), np.float32)}) # extend the profile to 1..bs before the test + m = self._selftest(bs) + if m >= self.selftest_min_match: + self.ok.add(bs); self.bs = bs; self.stage = f"trt-{tag}"; st["fp16"] = fp16 + if fp16 and marker.exists(): marker.unlink() + return + msg = f"[layout] TensorRT {tag} engine for batch {bs} failed its self-test ({m:.0%} of the reference blocks, attempt {attempt + 1}, {time.time() - t:.0f}s) -> rebuilding" + LOG.warning(msg); st["fallbacks"].append(msg); self.prov.events.append(msg) + self.s = None; shutil.rmtree(str(self.prov.cache_root / sub), ignore_errors=True) + if attempt + 1 < len(plan): + _, nfp16, nsub, _ = plan[attempt + 1]; self.s = self.prov.session(self.path, "layout", fp16=nfp16, cache_sub=nsub) + if not nfp16: marker.parent.mkdir(parents=True, exist_ok=True); marker.write_text("fp16 layout engine failed its self-test on this GPU; using the fp32 TensorRT engine\n") + msg = "[layout] no working TensorRT engine -> CUDA execution provider for the layout model" + LOG.warning(msg); st["fallbacks"].append(msg); self.prov.events.append(msg) + self.s = self.prov.session(self.path, "layout", force="cuda"); self.trt = False; self.bs = bs; self.ok.add(bs); self.stage = st["provider"] + def __call__(self, img_bgr, thr=None): + """single page -> (blocks, None); blocks = [{type, score, bbox, order=None}] in page pixels""" + bl = self.batch([img_bgr])[0] + return ([b for b in bl if b["score"] >= thr] if thr is not None else bl), None diff --git a/kn-en/kanen_infer/providers.py b/kn-en/kanen_infer/providers.py new file mode 100644 index 0000000000000000000000000000000000000000..c3b6604bdee05a3043e3509fa158b25c3bd9f557 --- /dev/null +++ b/kn-en/kanen_infer/providers.py @@ -0,0 +1,157 @@ +"""ONNX Runtime session factory with the provider policy trt -> cuda -> cpu and the engine-cache bookkeeping. + +* `Providers(cfg)` reads the policy (`auto` | `trt` | `cuda` | `cpu`), checks `trt_cache/ENGINES.json` against the local GPU / + TensorRT version (a foreign or stale cache is moved aside so engines rebuild instead of crashing) and records, per model, + which execution provider it finally ended on plus every fallback that happened (`Providers.status`). +* `session(path, key, fp16=True, force=None, trt_opts=None)` creates one `InferenceSession`; a provider that cannot be + loaded (no tensorrt wheel, no CUDA device, corrupt engine) falls through to the next one in the chain. +Nothing here touches model outputs: the same graph runs on whichever provider loads.""" +import json, logging, os, shutil, subprocess, time +from pathlib import Path +import onnxruntime as ort + +LOG = logging.getLogger("kanen") +TRT = "TensorrtExecutionProvider"; CUDA = "CUDAExecutionProvider"; CPU = "CPUExecutionProvider" +SHORT = {TRT: "trt", CUDA: "cuda", CPU: "cpu"} +MANIFEST = "ENGINES.json" +_PRELOADED = False + +def _cuda_libs_loadable(): + import ctypes, sys + names = ["cublasLt64_12.dll", "cudart64_12.dll"] if sys.platform == "win32" else ["libcublasLt.so.12", "libcudart.so.12"] + try: + for n in names: ctypes.CDLL(n) + return True + except OSError: return False + +def preload_cuda_libs(): + """Make the CUDA 12 / cuDNN 9 runtime libraries visible to onnxruntime-gpu. Order: already on the loader path -> + `onnxruntime.preload_dlls()` (nvidia-cublas-cu12 / nvidia-cudnn-cu12 ... pip wheels) -> importing torch (whose wheel + bundles the same libraries). Without any of these onnxruntime silently falls back to the CPU provider.""" + global _PRELOADED + if _PRELOADED: return + _PRELOADED = True + if _cuda_libs_loadable(): return + try: + if hasattr(ort, "preload_dlls"): + import contextlib, io + with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()): ort.preload_dlls() + except Exception as e: LOG.debug(f"preload_dlls: {e}") + if _cuda_libs_loadable(): return + try: + import torch # noqa: F401 + LOG.info("CUDA runtime libraries loaded through torch") + except Exception: + LOG.warning("CUDA 12 runtime libraries not found (install nvidia-cublas-cu12/nvidia-cudnn-cu12 or torch, or put CUDA on PATH): GPU providers will fall back to CPU") + +def gpu_name(): + try: + out = subprocess.run(["nvidia-smi", "--query-gpu=name", "--format=csv,noheader"], capture_output=True, text=True, timeout=10) + names = [l.strip() for l in out.stdout.splitlines() if l.strip()] + return names[0] if names else None + except Exception: return None + +def trt_version(): + try: + import tensorrt; return tensorrt.__version__ + except Exception: return None + +def md5_file(path, n=1 << 20): + import hashlib; h = hashlib.md5() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(n), b""): h.update(chunk) + return h.hexdigest()[:12] + +class Providers: + def __init__(self, cfg): + self.cfg = cfg; p = cfg["provider"] + self.policy = str(p.get("policy", "auto")).lower(); self.fp16 = bool(p.get("trt_fp16", True)) + self.cache_root = cfg.path("provider", "engine_cache") + if self.policy != "cpu": preload_cuda_libs() + self.available = ort.get_available_providers() + self.status = {} # model key -> dict(requested, provider, fp16, fallbacks[], cache, load_s) + self.events = [] # human-readable fallback log + self.gpu = gpu_name() if CUDA in self.available else None; self.trt_ver = trt_version() + if self.policy in ("auto", "trt"): self._check_manifest() + + # ---- engine cache manifest ------------------------------------------------------------------------------------- + def _check_manifest(self): + """Engines are only valid for the GPU architecture + TensorRT version they were built with. A cache built elsewhere + is moved to trt_cache/stale_/ so the first run rebuilds instead of failing on a foreign engine.""" + mf = self.cache_root / MANIFEST + if not mf.exists() or self.trt_ver is None: return + try: m = json.loads(mf.read_text(encoding="utf-8")) + except Exception: return + same_gpu = (not self.gpu) or (not m.get("gpu")) or (m.get("gpu") == self.gpu) + same_trt = (not m.get("tensorrt")) or (m.get("tensorrt") == self.trt_ver) + if same_gpu and same_trt: return + dst = self.cache_root / f"stale_{time.strftime('%Y%m%d_%H%M%S')}"; dst.mkdir(parents=True, exist_ok=True) + for sub in self.cache_root.iterdir(): + if sub.is_dir() and not sub.name.startswith("stale_"): shutil.move(str(sub), str(dst / sub.name)) + shutil.move(str(mf), str(dst / MANIFEST)) + msg = f"engine cache was built for {m.get('gpu')} / TensorRT {m.get('tensorrt')}, this machine has {self.gpu} / {self.trt_ver}: moved to {dst.name}, engines rebuild on first use (~4 min)" + LOG.warning(msg); self.events.append(msg) + + def write_manifest(self, extra=None): + """Called by prepare_engines after a validated build: per-model status + md5 of every engine file.""" + engines = {} + for sub in sorted(self.cache_root.iterdir()) if self.cache_root.exists() else []: + if sub.is_dir() and not sub.name.startswith("stale_"): + engines[sub.name] = {f.name: {"md5": md5_file(f), "bytes": f.stat().st_size} for f in sorted(sub.iterdir()) if f.suffix in (".engine", ".profile", ".timing")} + models = {k: dict(v, cache=os.path.relpath(v["cache"], self.cache_root) if v.get("cache") else None) for k, v in self.status.items()} # relative: the manifest ships with the package + m = {"package": self.cfg.get("name"), "version": self.cfg.get("version"), "gpu": self.gpu, "tensorrt": self.trt_ver, "onnxruntime": ort.__version__, + "built": time.strftime("%Y-%m-%d %H:%M:%S"), "models": models, "engines": engines} + if extra: m.update(extra) + self.cache_root.mkdir(parents=True, exist_ok=True); (self.cache_root / MANIFEST).write_text(json.dumps(m, indent=1), encoding="utf-8") + return m + + # ---- sessions ---------------------------------------------------------------------------------------------------- + def chain(self, force=None): + pol = (force or self.policy) + order = {"auto": ["trt", "cuda", "cpu"], "trt": ["trt", "cuda", "cpu"], "cuda": ["cuda", "cpu"], "cpu": ["cpu"]}.get(pol) + if order is None: raise ValueError(f"unknown provider policy {pol!r} (auto|trt|cuda|cpu)") + return order + + def _opts(self, kind, cache, fp16, trt_opts): + p = self.cfg["provider"] + cuda = (CUDA, {"cudnn_conv_algo_search": p.get("cudnn_conv_algo_search", "HEURISTIC"), "arena_extend_strategy": "kSameAsRequested", + "gpu_mem_limit": int(float(p.get("cuda_mem_limit_gb", 6)) * 1024 ** 3)}) + if kind == "cpu": return [CPU] + if kind == "cuda": return [cuda] + import tensorrt # noqa: F401 (loads libnvinfer / nvinfer_10.dll from the pip wheel so the ORT provider can find it) + cache.mkdir(parents=True, exist_ok=True) + o = {"trt_fp16_enable": bool(fp16), "trt_engine_cache_enable": True, "trt_engine_cache_path": str(cache), "trt_timing_cache_enable": True, + "trt_timing_cache_path": str(cache), "trt_max_workspace_size": int(float(p.get("trt_workspace_gb", 4)) * 1024 ** 3)} + if trt_opts: o.update(trt_opts) + return [(TRT, o), cuda] + + def session(self, path, key, fp16=None, force=None, trt_opts=None, cache_sub=None): + """key: model name for the status table ('det', 'layout', 'rec'); cache_sub: engine sub-directory (default = key).""" + fp16 = self.fp16 if fp16 is None else fp16; cache = self.cache_root / (cache_sub or key) + so = ort.SessionOptions(); so.log_severity_level = 3 + st = self.status.setdefault(key, {"requested": (force or self.policy), "provider": None, "fp16": fp16, "fallbacks": [], "cache": str(cache), "load_s": 0.0}) + t0 = time.time(); last = None + for kind in self.chain(force): + if kind == "cuda" and CUDA not in self.available: st["fallbacks"].append("cuda: CUDAExecutionProvider not available in this onnxruntime build"); continue + if kind == "trt" and TRT not in self.available: st["fallbacks"].append("trt: TensorrtExecutionProvider not available in this onnxruntime build"); continue + for attempt in (1, 2): + try: + s = ort.InferenceSession(str(path), so, providers=self._opts(kind, cache, fp16, trt_opts)) + got = s.get_providers()[0]; want = {"trt": TRT, "cuda": CUDA, "cpu": CPU}[kind] + if got != want: # ORT loads silently on the next provider when a library is missing: treat as a failed attempt + msg = f"{key}: {kind} requested but onnxruntime created the session on {SHORT.get(got, got)} (provider library not loadable?)" + LOG.warning(msg); st["fallbacks"].append(msg); self.events.append(msg) + if kind != self.chain(force)[-1]: break + st["provider"] = SHORT.get(got, got); st["fp16"] = fp16 if got == TRT else False; st["load_s"] = round(time.time() - t0, 1) + return s + except Exception as e: # corrupt / foreign engine: clear this model's cache once and retry, then fall through + last = e; text = str(e); msg = f"{key}: {kind} provider failed ({type(e).__name__}: {text.splitlines()[0][:160]})" + LOG.warning(msg); st["fallbacks"].append(msg); self.events.append(msg) + if kind == "trt" and attempt == 1 and cache.exists() and any(w in text.lower() for w in ("engine", "deserializ", "cache")): + stale = cache.parent / f"stale_{time.strftime('%Y%m%d_%H%M%S')}_{cache.name}"; shutil.move(str(cache), str(stale)) + self.events.append(f"{key}: engine cache moved to {stale.name}, rebuilding"); continue + break + raise RuntimeError(f"no execution provider could load {path}: {last}") + + def summary(self): + return {k: {"provider": v["provider"], "fp16": v["fp16"], "fallbacks": list(v["fallbacks"])} for k, v in self.status.items()} diff --git a/kn-en/kanen_infer/rec_onnx.py b/kn-en/kanen_infer/rec_onnx.py new file mode 100644 index 0000000000000000000000000000000000000000..d1a9411fd928f5d9a1271da22e5c20907d4ec03b --- /dev/null +++ b/kn-en/kanen_infer/rec_onnx.py @@ -0,0 +1,52 @@ +"""KanEn recognizer, ONNX Runtime only (TensorRT fp16 / CUDA / CPU by the provider policy). Greedy CTC decode with the frozen charset. +Same batching rule as the training-side recognizer: every crop is padded only to its own width bucket (padding a short crop +to a much wider batch width makes the model double the first glyph).""" +import hashlib, json, math +import numpy as np, cv2 +from .config import load_config +from .providers import Providers, md5_file + +class Conf(float): + """line confidence (mean max-prob over emitted characters) carrying `min_char` (weakest emitted character)""" + min_char = None + +class KanEnRecognizer: + def __init__(self, onnx_path=None, buckets=None, cfg=None, providers=None): + self.cfg = cfg or load_config(); self.prov = providers or Providers(self.cfg); rc = self.cfg["recognizer"] + onnx_path = str(onnx_path or self.cfg.path("models", "rec")); self.path = onnx_path + self.buckets = tuple(int(b) for b in (buckets or rc["buckets"])); self.H = int(rc.get("height", 32)); self.MAX_RATIO = float(rc.get("max_ratio", 40)); self.bs = int(rc.get("batch", 64)) + self.chars = json.load(open(self.cfg.path("models", "charset"), encoding="utf-8")) + self.onnx_md5 = md5_file(onnx_path); self.charset_md5 = hashlib.md5("".join(self.chars).encode("utf-8")).hexdigest()[:12] + trt_opts = {"trt_profile_min_shapes": f"image:1x3x{self.H}x{self.buckets[0]}", "trt_profile_max_shapes": f"image:64x3x{self.H}x{self.buckets[-1]}", "trt_profile_opt_shapes": f"image:48x3x{self.H}x256"} + self.s = self.prov.session(onnx_path, "rec", trt_opts=trt_opts) + self.name = "kanen-rec-" + ("trt" if "Tensorrt" in self.s.get_providers()[0] else "ort") + @property + def provider(self): return self.prov.status.get("rec", {}).get("provider") + + def __call__(self, crops, bs=None): + if not crops: return [] + bs = bs or self.bs; H = self.H + gray = [c if c.ndim == 2 else cv2.cvtColor(c, cv2.COLOR_BGR2GRAY) for c in crops] + ratios = [min(g.shape[1] / max(g.shape[0], 1), self.MAX_RATIO) for g in gray] + bucket_of = lambda i: next((b for b in self.buckets if b >= int(math.ceil(ratios[i] * H / 4) * 4)), self.buckets[-1]) + groups = {} + for i in range(len(gray)): groups.setdefault(bucket_of(i), []).append(i) + out = [None] * len(gray) + for W, members in sorted(groups.items()): + n = max(4, min(bs, int(bs * 320 // max(W, 64)))) + for i in range(0, len(members), n): + idx = members[i:i + n]; x = np.ones((len(idx), 3, H, W), np.float32) + for k, j in enumerate(idx): + g = gray[j]; nw = min(W, max(8, int(round(g.shape[1] * H / max(g.shape[0], 1))))) + r = cv2.resize(g, (nw, H), interpolation=cv2.INTER_AREA if g.shape[0] > H else cv2.INTER_LINEAR).astype(np.float32) / 255.0 + x[k, :, :, :nw] = (r - 0.5) / 0.5 + probs = self.s.run(None, {"image": x})[0] # (B, T, C) softmax + ids = probs.argmax(-1); pmax = probs.max(-1) + for k, j in enumerate(idx): + row = ids[k]; keep = (row != 0) & (row != np.roll(row, 1)); s, prev = [], 0 + for t in row: + if t != prev and t != 0: s.append(self.chars[t - 1]) + prev = t + c = Conf(float(pmax[k][keep].mean()) if keep.any() else 0.0); c.min_char = float(pmax[k][keep].min()) if keep.any() else 0.0 + out[j] = ("".join(s), c) + return out diff --git a/kn-en/models/det/inference.onnx b/kn-en/models/det/inference.onnx new file mode 100644 index 0000000000000000000000000000000000000000..d5d3f5b992066b64c7f6bed8ab402860f4325aaa --- /dev/null +++ b/kn-en/models/det/inference.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d73e0058b7a8086bbd57f3d10b8bcd4ff95363f67e06e2762b5e814fe9c9410e +size 9880512 diff --git a/kn-en/models/det/inference.yml b/kn-en/models/det/inference.yml new file mode 100644 index 0000000000000000000000000000000000000000..a2d45b5b83b2f988b6336a17593c2e474e4e6997 --- /dev/null +++ b/kn-en/models/det/inference.yml @@ -0,0 +1,52 @@ +Global: + model_name: PP-OCRv6_small_det +Hpi: + backend_configs: + paddle_infer: + trt_dynamic_shapes: &id001 + x: + - - 1 + - 3 + - 32 + - 32 + - - 1 + - 3 + - 736 + - 736 + - - 1 + - 3 + - 4000 + - 4000 + tensorrt: + dynamic_shapes: *id001 +PostProcess: + box_thresh: 0.45 + max_candidates: 3000 + name: DBPostProcess + thresh: 0.2 + unclip_ratio: 1.4 +PreProcess: + transform_ops: + - DecodeImage: + channel_first: false + img_mode: BGR + - DetLabelEncode: null + - DetResizeForTest: null + - NormalizeImage: + mean: + - 0.485 + - 0.456 + - 0.406 + order: hwc + scale: 1./255. + std: + - 0.229 + - 0.224 + - 0.225 + - ToCHWImage: null + - KeepKeys: + keep_keys: + - image + - shape + - polys + - ignore_tags diff --git a/kn-en/models/layout/LICENSE-AGPL-3.0.txt b/kn-en/models/layout/LICENSE-AGPL-3.0.txt new file mode 100644 index 0000000000000000000000000000000000000000..1468d07c88d6a48dae9360ed0094955b54370224 --- /dev/null +++ b/kn-en/models/layout/LICENSE-AGPL-3.0.txt @@ -0,0 +1,661 @@ + GNU AFFERO GENERAL PUBLIC LICENSE + Version 3, 19 November 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU Affero General Public License is a free, copyleft license for +software and other kinds of works, specifically designed to ensure +cooperation with the community in the case of network server software. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +our General Public Licenses are intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + Developers that use our General Public Licenses protect your rights +with two steps: (1) assert copyright on the software, and (2) offer +you this License which gives you legal permission to copy, distribute +and/or modify the software. + + A secondary benefit of defending all users' freedom is that +improvements made in alternate versions of the program, if they +receive widespread use, become available for other developers to +incorporate. Many developers of free software are heartened and +encouraged by the resulting cooperation. However, in the case of +software used on network servers, this result may fail to come about. +The GNU General Public License permits making a modified version and +letting the public access it on a server without ever releasing its +source code to the public. + + The GNU Affero General Public License is designed specifically to +ensure that, in such cases, the modified source code becomes available +to the community. It requires the operator of a network server to +provide the source code of the modified version running there to the +users of that server. Therefore, public use of a modified version, on +a publicly accessible server, gives the public access to the source +code of the modified version. + + An older license, called the Affero General Public License and +published by Affero, was designed to accomplish similar goals. This is +a different license, not a version of the Affero GPL, but Affero has +released a new version of the Affero GPL which permits relicensing under +this license. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU Affero General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Remote Network Interaction; Use with the GNU General Public License. + + Notwithstanding any other provision of this License, if you modify the +Program, your modified version must prominently offer all users +interacting with it remotely through a computer network (if your version +supports such interaction) an opportunity to receive the Corresponding +Source of your version by providing access to the Corresponding Source +from a network server at no charge, through some standard or customary +means of facilitating copying of software. This Corresponding Source +shall include the Corresponding Source for any work covered by version 3 +of the GNU General Public License that is incorporated pursuant to the +following paragraph. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the work with which it is combined will remain governed by version +3 of the GNU General Public License. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU Affero General Public License from time to time. Such new versions +will be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU Affero General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU Affero General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU Affero General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If your software can interact with users remotely through a computer +network, you should also make sure that it provides a way for users to +get its source. For example, if your program is a web application, its +interface could display a "Source" link that leads users to an archive +of the code. There are many ways you could offer source, and different +solutions will be better for different programs; see section 13 for the +specific requirements. + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU AGPL, see +. diff --git a/kn-en/models/layout/NOTICE.md b/kn-en/models/layout/NOTICE.md new file mode 100644 index 0000000000000000000000000000000000000000..f19487c9185f5b0f562da6f642564da64325a21d --- /dev/null +++ b/kn-en/models/layout/NOTICE.md @@ -0,0 +1,15 @@ +# Layout model licence — `kanen_layout_v12.onnx` + +The v1.2 layout model (Arjuna layout v12) is a DocLayout-YOLO / YOLOv10-based detector fine-tuned from the +`juliozhao/DocLayout-YOLO-DocStructBench` weights (Apache-2.0) with the `doclayout_yolo` training toolkit, which is +licensed under the **GNU Affero General Public License v3.0** (Ultralytics YOLO lineage). To respect that lineage the +layout model file in this directory is distributed under **AGPL-3.0** (full text: `LICENSE-AGPL-3.0.txt`). + +Everything else in this repository — the inference package `kanen_infer`, the recognizer (SVTRv2 + language pack), +the text detector (PP-OCRv6) and the documentation — is Apache-2.0 as stated in the top-level `LICENSE`. + +Practical consequences: +- Using the package with this layout model in a networked service falls under AGPL-3.0 for the layout model. If that + does not suit you, run with `layout.mode: never` in `kanen.yaml` (text lines and tables from ruled grids still work + without the layout model) or plug in another layout model of your choice (`models.layout`, `models.layout_labels`). +- No AGPL toolkit code is included here; inference runs on ONNX Runtime. diff --git a/kn-en/models/layout/classes.txt b/kn-en/models/layout/classes.txt new file mode 100644 index 0000000000000000000000000000000000000000..0acaed433ca4351449e53c9965848c211329dad6 --- /dev/null +++ b/kn-en/models/layout/classes.txt @@ -0,0 +1,9 @@ +title +text +table +figure +seal +id_panel +header +footer +handwriting diff --git a/kn-en/models/layout/kanen_layout_v12.onnx b/kn-en/models/layout/kanen_layout_v12.onnx new file mode 100644 index 0000000000000000000000000000000000000000..4e18257066cd56077e0e607a60a2d6c0b4ad3830 --- /dev/null +++ b/kn-en/models/layout/kanen_layout_v12.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de72e58c373a75fc788ff3f8ef17e2495a609d61cff4c064ccd69e1adaeee1ff +size 74832733 diff --git a/kn-en/models/layout/selftest.jpg b/kn-en/models/layout/selftest.jpg new file mode 100644 index 0000000000000000000000000000000000000000..01f4fc0c56ea3c12f2d0c25de945389504e8c726 --- /dev/null +++ b/kn-en/models/layout/selftest.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b225479e38d2da866bf7d1fd1519892a132118d65538b39e01ff316d1be4653e +size 114880 diff --git a/kn-en/models/layout/selftest_ref.json b/kn-en/models/layout/selftest_ref.json new file mode 100644 index 0000000000000000000000000000000000000000..02fc3eb12375929025a9182e6a26dc2ad5084c5d --- /dev/null +++ b/kn-en/models/layout/selftest_ref.json @@ -0,0 +1,42 @@ +[ + { + "type": "header", + "score": 0.958, + "bbox": [ + 83.5, + 103.0, + 122.9, + 117.5 + ] + }, + { + "type": "header", + "score": 0.951, + "bbox": [ + 900.9, + 103.5, + 940.5, + 118.8 + ] + }, + { + "type": "title", + "score": 0.921, + "bbox": [ + 485.8, + 213.1, + 536.3, + 230.4 + ] + }, + { + "type": "text", + "score": 0.526, + "bbox": [ + 107.8, + 126.6, + 868.8, + 170.1 + ] + } +] \ No newline at end of file diff --git a/kn-en/models/rec/charset.json b/kn-en/models/rec/charset.json new file mode 100644 index 0000000000000000000000000000000000000000..fc08f74be19868b0ec82118b10c010343dc8ae29 --- /dev/null +++ b/kn-en/models/rec/charset.json @@ -0,0 +1 @@ +[" ", "!", "\"", "#", "$", "%", "&", "'", "(", ")", "*", "+", ",", "-", ".", "/", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", ":", ";", "<", "=", ">", "?", "@", "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z", "[", "\\", "]", "^", "_", "`", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z", "{", "|", "}", "~", "ಀ", "ಁ", "ಂ", "ಃ", "಄", "ಅ", "ಆ", "ಇ", "ಈ", "ಉ", "ಊ", "ಋ", "ಌ", "ಎ", "ಏ", "ಐ", "ಒ", "ಓ", "ಔ", "ಕ", "ಖ", "ಗ", "ಘ", "ಙ", "ಚ", "ಛ", "ಜ", "ಝ", "ಞ", "ಟ", "ಠ", "ಡ", "ಢ", "ಣ", "ತ", "ಥ", "ದ", "ಧ", "ನ", "ಪ", "ಫ", "ಬ", "ಭ", "ಮ", "ಯ", "ರ", "ಱ", "ಲ", "ಳ", "ವ", "ಶ", "ಷ", "ಸ", "ಹ", "಼", "ಽ", "ಾ", "ಿ", "ೀ", "ು", "ೂ", "ೃ", "ೄ", "ೆ", "ೇ", "ೈ", "ೊ", "ೋ", "ೌ", "್", "ೕ", "ೖ", "ೝ", "ೞ", "ೠ", "ೡ", "ೢ", "ೣ", "೦", "೧", "೨", "೩", "೪", "೫", "೬", "೭", "೮", "೯", "ೱ", "ೲ", "ೳ", "₹", "–", "—", "‘", "’", "“", "”", "•", "·", "°", "×", "÷", "½", "¼", "¾", "§", "¶", "।", "॥", "…", "‑", "‐", "−", "™", "©", "®", "∩", "∪", "∈", "≤", "≥", "√", "≠", "±"] \ No newline at end of file diff --git a/kn-en/models/rec/kanen_rec_e5.onnx b/kn-en/models/rec/kanen_rec_e5.onnx new file mode 100644 index 0000000000000000000000000000000000000000..29ce4acc68ca6ba9719d5dbf3e4ad56976540608 --- /dev/null +++ b/kn-en/models/rec/kanen_rec_e5.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c9a1b1cc067da879a8e058ac5186b75fee127f64bbf06b4563bfde493f7364b +size 81128782