File size: 4,080 Bytes
7cb8aac | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 | """Model inference tests. Require a trained checkpoint or the Hub model.
Set HUMANIZE_MODEL (default: checkpoints/humanize-text-model).
"""
import os
import sys
from pathlib import Path
import pytest
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from humanize import Humanizer, MAX_CHARS, protect, restore
MODEL = os.environ.get("HUMANIZE_MODEL", "checkpoints/humanize-text-model")
@pytest.fixture(scope="module")
def humanizer():
return Humanizer(model_id=MODEL)
def test_loads_and_returns_text(humanizer):
out = humanizer.humanize("It is important to note that this is a test.")
assert isinstance(out, str) and out
def test_removes_ai_cliches_en(humanizer):
src = (
"In today's rapidly evolving world, it is important to note that this robust "
"solution serves as a testament to our commitment to innovation. Moreover, "
"we leverage cutting-edge technology to deliver a seamless experience."
)
out = humanizer.humanize(src)
lower = out.lower()
assert "it is important to note that" not in lower
assert "moreover" not in lower
assert "leverage" not in lower
assert "seamless" not in lower
assert "testament" not in lower
assert out != src
def test_removes_ai_cliches_zh(humanizer):
src = "值得注意的是，我们通过赋能团队来助力企业实现降本增效，形成完整的业务闭环。"
out = humanizer.humanize(src)
assert "值得注意的是" not in out
assert "赋能" not in out
assert "降本增效" not in out
assert out != src
def test_protects_urls_numbers_quotes(humanizer):
src = (
"It is important to note that the server at https://api.lynote.ai/v1/detect "
"handles 1,200 requests per second with a 99.5% uptime, as stated in "
'"Project Lynote". Moreover, run `pip install humanize-text==1.4.2`.'
)
out = humanizer.humanize(src)
for token in ("https://api.lynote.ai/v1/detect", "1,200", "99.5%", '"Project Lynote"', "humanize-text==1.4.2"):
assert token in out, f"protected token lost: {token}"
def test_human_prose_stays_similar(humanizer):
src = "I walked the dog this morning and it started raining halfway through the park, so we ran home."
out = humanizer.humanize(src)
# Identity should be near-preserved: allow light edits but no additions.
assert abs(len(out.split()) - len(src.split())) <= 4
def test_short_input_and_limits(humanizer):
with pytest.raises(ValueError):
humanizer.humanize("")
with pytest.raises(ValueError):
humanizer.humanize(" ")
with pytest.raises(ValueError):
humanizer.humanize("x" * (MAX_CHARS + 1))
def test_batch_consistency(humanizer):
texts = [
"It is worth noting that this robust platform is a game-changer. Moreover, it is seamless.",
"值得注意的是，我们通过赋能团队来实现降本增效。",
"I walked the dog this morning and the sky was grey.",
]
batch = humanizer.humanize_batch(texts)
assert len(batch) == 3
for out, src in zip(batch, texts):
assert isinstance(out, str) and out
single = [humanizer.humanize(t) for t in texts]
# batch and single should agree exactly (same decode path)
assert batch == single
def test_protect_restore_roundtrip():
text = "Run `pip install x==1.0` at https://example.com/a?b=1 — cost is $12.50 and \"Q3\" rose 4%."
masked, protected = protect(text)
assert "PROTECTED_0" in masked
assert restore(masked, protected) == text
def test_is_chinese_routing():
from humanize import is_chinese
assert is_chinese("值得注意的是，这是中文测试。")
assert not is_chinese("It is important to note that this is English.")
assert not is_chinese("Mixed text with 中文 characters but mostly English words here.")
def test_polish():
from humanize import Humanizer as H
assert H._polish("hello , world") == "hello, world"
assert H._polish("end., next") == "end. Next"
assert H._polish("a b") == "a b"
|