Download OCR_INDRA/indra/tests/test_textlayer.py from prosperious/ocr-projects-archive: direct link, hf CLI and curl.
- Browser
- Download file 11.6 kB
-
https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/OCR_INDRA/indra/tests/test_textlayer.py
- Command line
-
hf download hf://prosperious/ocr-projects-archive/OCR_INDRA/indra/tests/test_textlayer.py
-
curl -L -o test_textlayer.py https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/OCR_INDRA/indra/tests/test_textlayer.py
11.6 kB
| import numpy as np | |
| from indra_ocr.textlayer import ( | |
| bad_char_count, | |
| order_backtracks, | |
| pdf_order_coherent, | |
| ) | |
| def test_bad_chars_count_replacement_and_private_use(): | |
| assert bad_char_count("ok") == 0 | |
| assert bad_char_count("a�") == 2 | |
| def _box(x0, y0, x1, y1): | |
| return [x0, y0, x1, y1] | |
| def test_two_columns_in_reading_order_are_coherent(): | |
| left = [_box(50, 100 + 15 * i, 290, 110 + 15 * i) for i in range(20)] | |
| right = [_box(310, 100 + 15 * i, 550, 110 + 15 * i) for i in range(20)] | |
| boxes = np.array(left + right, np.float32) | |
| assert order_backtracks(boxes) == 0 and pdf_order_coherent(boxes) | |
| def test_interleaved_columns_are_not_coherent(): | |
| left = [_box(50, 100 + 15 * i, 290, 110 + 15 * i) for i in range(20)] | |
| right = [_box(310, 100 + 15 * i, 550, 110 + 15 * i) for i in range(20)] | |
| # line by line across the gutter: every move back to the left column goes up-left | |
| boxes = np.array([b for pair in zip(left, right) for b in pair], np.float32) | |
| # Geometry alone cannot see it (a table is read the same way) ... | |
| assert order_backtracks(boxes) == 0 | |
| # ... the order model can: it puts the left column first. | |
| col = np.array([0, 1] * 20) | |
| row = np.repeat(np.arange(20), 2) | |
| key = col * 100 + row # true reading position | |
| logits = np.sign(key[None, :] - key[:, None]).astype(np.float32) * 4.0 | |
| assert not pdf_order_coherent(boxes, logits) | |
| good = np.argsort(key) | |
| assert pdf_order_coherent(boxes[good], logits[np.ix_(good, good)]) | |
| def test_lines_written_bottom_up_are_not_coherent(): | |
| boxes = np.array( | |
| [_box(50, 400 - 15 * i, 500, 410 - 15 * i) for i in range(20)], np.float32 | |
| ) | |
| assert not pdf_order_coherent(boxes) | |
| def _chars(text, x0=0.0, w=5.0, gaps=None): | |
| out, x = [], x0 | |
| for i, ch in enumerate(text): | |
| x += (gaps or {}).get(i, 0.0) | |
| out.append({"char": ch, "bbox": [x, 0.0, x + w, 10.0]}) | |
| x += w | |
| return out | |
| def test_cli_source_flags(): | |
| import argparse | |
| from indra_ocr.cli import add_source_args, source_options | |
| ap = argparse.ArgumentParser() | |
| add_source_args(ap) | |
| assert source_options(ap.parse_args([])) == { | |
| "mode": "auto", | |
| "pdf_order": True, | |
| "image_ocr": True, | |
| } | |
| assert source_options(ap.parse_args(["--mode", "ocr"]))["mode"] == "ocr" | |
| o = source_options(ap.parse_args(["--model-order", "--no-image-ocr"])) | |
| assert o["pdf_order"] is False and o["image_ocr"] is False | |
| def test_runs_go_to_the_smallest_containing_line_in_x_order(): | |
| from indra_ocr.document import assign_runs | |
| from indra_ocr.textlayer import TextLine | |
| def run(text, x0, y0, y1, w=4.0): | |
| chars = [ | |
| (c, (x0 + i * w, y0, x0 + (i + 1) * w, y1)) for i, c in enumerate(text) | |
| ] | |
| return TextLine((x0, y0, x0 + w * len(text), y1), text, chars) | |
| boxes = np.array([[0, 0, 100, 20], [70, 2, 95, 18], [0, 30, 100, 50]], np.float32) | |
| runs = [ | |
| run("ab c", 10, 5, 15), | |
| run("\u03b1", 2, 3, 13), | |
| run("xy", 75, 5, 15), | |
| run("far", 500, 5, 15), | |
| ] | |
| toks, lost = assign_runs(runs, boxes, 1.0) | |
| assert "".join(t[0] for t in toks[0]) == "\u03b1 ab c" # runs by x, own order kept | |
| assert [t[2] for t in toks[0]][0] == 1 # content index of the run | |
| assert "".join(t[0] for t in toks[1]) == "xy" # inside both boxes: the smaller wins | |
| assert toks[2] == [] and lost == 3 | |
| def test_a_run_across_two_cells_splits_between_them(): | |
| from indra_ocr.document import assign_runs | |
| from indra_ocr.textlayer import TextLine | |
| text = "12 34" | |
| chars = [(c, (i * 10.0, 0.0, i * 10.0 + 8, 10.0)) for i, c in enumerate(text)] | |
| run = TextLine((0, 0, 48, 10), text, chars) | |
| boxes = np.array([[0, 0, 19, 10], [29, 0, 50, 10]], np.float32) # two cells | |
| toks, lost = assign_runs([run], boxes, 1.0) | |
| assert ["".join(t[0] for t in tk).strip() for tk in toks] == [ | |
| "12", | |
| "34", | |
| ] and lost == 0 | |
| def test_runs_of_a_right_to_left_line_go_right_to_left(): | |
| from indra_ocr.document import assign_runs | |
| from indra_ocr.textlayer import TextLine | |
| def run(text, x0): # logical order, drawn right to left from x0 | |
| chars = [ | |
| (c, (x0 - (i + 1) * 4.0, 5, x0 - i * 4.0, 15)) for i, c in enumerate(text) | |
| ] | |
| return TextLine((x0 - 4.0 * len(text), 5, x0, 15), text, chars) | |
| boxes = np.array([[0, 0, 100, 20]], np.float32) | |
| # two table cells on one row: "שלום" on the right, "עולם" on the left | |
| toks, _ = assign_runs([run("עולם", 30), run("שלום", 90)], boxes, 1.0) | |
| assert "".join(t[0] for t in toks[0]) == "שלום עולם" | |
| def test_ocr_only_reads_scripts_in_its_vocab(): | |
| from types import SimpleNamespace | |
| from indra_ocr.document import readable | |
| ocr = SimpleNamespace( | |
| vocab=SimpleNamespace(index={c: 1 for c in "abcdefghijklmnopqrstuvwxyz"}), | |
| vocab_cjk=None, | |
| ) | |
| line = lambda t: SimpleNamespace(text=t) # noqa: E731 | |
| assert readable(ocr, [line("hello world 123")]) | |
| assert not readable(ocr, [line("مرحبا بالعالم"), line("ok")]) | |
| assert readable(ocr, [line("1234 ...")]) # no letters: nothing to rule out | |
| assert not readable(ocr, [line("中文文本")]) | |
| ocr.vocab_cjk = SimpleNamespace(index={c: 1 for c in "中文文本"}) | |
| assert readable(ocr, [line("中文文本")]) | |
| def test_lines_with_hanzi_go_to_the_cjk_recognizer(): | |
| from indra_ocr.pipeline import OrtOCR | |
| from indra_ocr.vocab import Vocab | |
| class Rec: # emits one fixed token id per line, at every other timestep | |
| def __init__(self, tok): | |
| self.tok = tok | |
| def run(self, _, feed): | |
| b, w = feed["x"].shape[0], feed["x"].shape[-1] // 4 | |
| best = np.zeros((b, w), np.int64) | |
| best[:, ::2] = self.tok | |
| return best, np.ones((b, w), np.float32), np.full(b, w) | |
| class Script: # lines wider than 300 px "hold hanzi" | |
| def run(self, _, feed): | |
| return [(feed["widths"] > 300).astype(np.float32)] | |
| ocr = OrtOCR.__new__(OrtOCR) | |
| ocr.vocab, ocr.vocab_cjk = Vocab(list("ab")), Vocab(list("中b")) | |
| ocr.rec, ocr.rec_cjk, ocr.script, ocr.mask, ocr.height = ( | |
| Rec(1), | |
| Rec(1), | |
| Script(), | |
| None, | |
| 32, | |
| ) | |
| gray = np.full((200, 800), 255, np.uint8) | |
| boxes = np.array([[10, 10, 100, 42, 1], [10, 100, 700, 132, 1]], np.float32) | |
| (t0, _, k0), (t1, _, k1) = ocr.recognize(gray, boxes) | |
| assert set(t0) == {"a"} and isinstance(k0[0][0], int) # main recognizer: ids | |
| assert set(t1) == {"中"} and k1[0][0] == "中" # CJK recognizer: characters | |
| assert ocr._decode(k1) == t1 | |
| def test_a_turned_page_is_turned_upright_before_detection(): | |
| from indra_ocr.pipeline import OrtOCR | |
| class Orient: # says: content turned one quarter turn counter-clockwise | |
| def run(self, _, feed): | |
| assert feed["x"].shape == (1, 1, 768, 768) | |
| return [np.array([[0.01, 0.97, 0.01, 0.01]], np.float32)] | |
| seen = {} | |
| ocr = OrtOCR.__new__(OrtOCR) | |
| ocr.orient = Orient() | |
| ocr.detect = lambda g: ( | |
| seen.setdefault("shape", g.shape) and np.zeros((0, 5), np.float32) | |
| ) | |
| ocr.recognize = lambda g, b: [] | |
| ocr.assemble = lambda wh, b, r, t: __import__("indra_ocr.pipeline").pipeline.Page( | |
| [], t | |
| ) | |
| page = ocr(np.full((300, 500), 255, np.uint8)) # wide: a turned portrait page | |
| assert seen["shape"] == (500, 300) and page.rotation == 90 | |
| ocr.orient = None | |
| assert ocr.orientation(np.full((300, 500), 255, np.uint8)) == 0 | |
| def test_fast_average_linkage_matches_the_brute_force(): | |
| from indra_ocr.order_post import _cluster | |
| def brute(logits): | |
| n = logits.shape[0] | |
| S = logits.astype(np.float64).copy() | |
| np.fill_diagonal(S, 0.0) | |
| size, alive, members = np.ones(n), np.ones(n, bool), [[i] for i in range(n)] | |
| for _ in range(n - 1): | |
| mean = S / np.outer(size, size) | |
| mean[~alive] = -np.inf | |
| mean[:, ~alive] = -np.inf | |
| np.fill_diagonal(mean, -np.inf) | |
| a, b = np.unravel_index(int(np.argmax(mean)), mean.shape) | |
| if mean[a, b] <= 0: | |
| break | |
| S[a] += S[b] | |
| S[:, a] += S[:, b] | |
| S[a, a] = 0.0 | |
| size[a] += size[b] | |
| alive[b] = False | |
| members[a] += members[b] | |
| return sorted(tuple(sorted(members[i])) for i in range(n) if alive[i]) | |
| rng = np.random.default_rng(1) | |
| for _ in range(100): | |
| n = int(rng.integers(1, 40)) | |
| g = rng.integers(0, max(1, n // 4), n) | |
| L = np.where(g[:, None] == g[None], 2.0, -2.0) + rng.normal(0, 1.5, (n, n)) | |
| L = (L + L.T) / 2 | |
| assert sorted(tuple(c) for c in _cluster(L)) == brute(L) | |
| def test_skew_estimate_recovers_a_tilt(): | |
| from indra_ocr.pipeline import estimate_skew, rotate_page | |
| rng = np.random.default_rng(0) | |
| page = np.full((2200, 1700), 255, np.uint8) | |
| for y in range(150, 2050, 45): # text-like rows of ink blobs | |
| x = 150 | |
| while x < 1550: | |
| w = int(rng.integers(20, 90)) | |
| page[y : y + 18, x : x + w] = 30 | |
| x += w + int(rng.integers(10, 25)) | |
| assert abs(estimate_skew(page)) < 0.15 | |
| for tilt in (2.0, -1.3, 0.6): | |
| est = estimate_skew(rotate_page(page, tilt)) | |
| assert abs(est + tilt) <= 0.15, (tilt, est) # the estimate undoes the tilt | |
| assert estimate_skew(np.full((800, 600), 255, np.uint8)) == 0.0 | |
| def test_cjk_lines_join_without_a_space_and_bare_bullets_start_items(): | |
| from indra_ocr.layout_post import _ITEM, join_lines | |
| assert ( | |
| join_lines(["或其中的", "任何部分,在 2020 年"]) | |
| == "或其中的任何部分,在 2020 年" | |
| ) | |
| assert join_lines(["説明します。", "次に"]) == "説明します。次に" | |
| assert ( | |
| join_lines(["한국어 문장", "다음 줄"]) == "한국어 문장 다음 줄" | |
| ) # Korean keeps the space | |
| assert join_lines(["tumori-", "genesis is"]) == "tumorigenesis is" | |
| assert join_lines(["first line", "second"]) == "first line second" | |
| assert _ITEM.match("•") and _ITEM.match("• text") and _ITEM.match("12. text") | |
| assert not _ITEM.match("•text") and not _ITEM.match("12.5 kg") | |
| def test_unrotate_box_maps_a_deskewed_box_back(): | |
| from indra_ocr.pipeline import rotate_page, unrotate_box | |
| page = np.full((400, 600), 255, np.uint8) | |
| page[180:220, 100:500] = 0 # a long dark bar | |
| tilted = rotate_page(page, -3.0) | |
| ys, xs = np.nonzero(tilted < 128) | |
| got = unrotate_box([100, 180, 500, 220], 3.0, (600, 400)) | |
| # the mapped box covers the tilted bar in the input page | |
| assert got[0] <= xs.min() + 2 and got[2] >= xs.max() - 2 | |
| assert got[1] <= ys.min() + 2 and got[3] >= ys.max() - 2 | |
| assert unrotate_box([10, 10, 20, 20], 0.0, (600, 400)) == [10, 10, 20, 20] | |
| def test_math_tokens_back_to_text(): | |
| from indra_ocr.math import detokenize | |
| toks = ( | |
| list("we have ") | |
| + ["$", "x", "_", "{", "i", "}", "\\leq", "1", "$"] | |
| + list(" here") | |
| ) | |
| assert detokenize(toks) == "we have $x _ { i } \\leq 1$ here" | |
| assert detokenize(["$", "\\alpha"]) == "$\\alpha$" # an open span closes | |
| assert detokenize([" the", " bound"]) == "the bound" # word tokens keep their space | |
| def test_math_models_are_optional(tmp_path): | |
| from indra_ocr.math import MathReader | |
| (tmp_path / "math_enc.onnx").write_bytes(b"") # only some of the files | |
| assert MathReader.load(tmp_path, None, None) is None | |