File size: 1,387 Bytes
f340984 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 | from unlimited_ocr_rdna4.postprocess import EOS_STOP, clean_model_output, repetition_warning
def test_clean_model_output_preserves_unicode_and_content() -> None:
raw = (
"<|det|>header [1, 2, 3, 4]<|/det|>Überblick\n"
"<|det|>table [1, 2, 3, 4]<|/det|><table><tr><td>€48.50</td></tr></table>\n"
"[Non-Text]\n"
"\\coloneqq"
f"{EOS_STOP}"
)
assert clean_model_output(raw) == "Überblick\n<table><tr><td>€48.50</td></tr></table>\n\\coloneqq"
def test_clean_model_output_removes_combined_reference_tag() -> None:
raw = "<|ref|>title<|/ref|><|det|>[[1, 2, 3, 4]]<|/det|>Actual title"
assert clean_model_output(raw) == "titleActual title"
def test_clean_model_output_preserves_ordinary_words_and_tex() -> None:
raw = "NonTextile material\nA \\eqqcolon B\nembedded NonText label"
assert clean_model_output(raw) == raw
def test_clean_model_output_preserves_malformed_unclosed_tags() -> None:
raw = "before <|det|>coordinate and recognized content"
assert clean_model_output(raw) == raw
def test_repetition_warning_detects_three_lines() -> None:
text = "start\nrepeating output\nrepeating output\nrepeating output"
assert "repeated line" in (repetition_warning(text) or "")
def test_repetition_warning_accepts_normal_text() -> None:
assert repetition_warning("first\nsecond\nthird") is None
|