indic-transliterate / tests /test_validate.py
MeghanaKap's picture
Push transliteration pipeline code + prototype checkpoint + model card
f4386ce verified
Raw
History Blame Contribute Delete
2.31 kB
from transliteration.data.validate import (
ValidationConfig,
is_corrupted_pair,
is_valid_roman,
is_valid_target_script,
run_validation,
)
def test_valid_pair_accepted():
records = [{"source": "t", "language": "hi", "roman": "namaste", "target": "नमस्ते"}]
accepted, stats = run_validation(records)
assert stats.accepted == 1
assert accepted[0]["target"] == "नमस्ते"
def test_empty_rejected():
records = [{"source": "t", "language": "hi", "roman": "", "target": "नमस्ते"}]
accepted, stats = run_validation(records)
assert stats.accepted == 0
assert stats.empty == 1
def test_wrong_script_rejected():
# Tamil target labeled as Hindi -> should be rejected
records = [{"source": "t", "language": "hi", "roman": "vanakkam", "target": "வணக்கம்"}]
accepted, stats = run_validation(records)
assert stats.accepted == 0
assert stats.bad_target_script == 1
def test_non_roman_source_rejected():
records = [{"source": "t", "language": "hi", "roman": "नमस्ते", "target": "नमस्ते"}]
accepted, stats = run_validation(records)
assert stats.accepted == 0
def test_duplicate_rejected():
rec = {"source": "t", "language": "hi", "roman": "namaste", "target": "नमस्ते"}
accepted, stats = run_validation([rec, dict(rec)])
assert stats.accepted == 1
assert stats.duplicate == 1
def test_too_long_rejected():
cfg = ValidationConfig(max_chars=10)
records = [{"source": "t", "language": "hi", "roman": "a" * 20, "target": "न" * 20}]
accepted, stats = run_validation(records, cfg)
assert stats.accepted == 0
assert stats.too_long == 1
def test_corrupted_pair_identical_strings():
assert is_corrupted_pair("namaste", "namaste") is True
def test_is_valid_roman():
cfg = ValidationConfig()
assert is_valid_roman("namaste hai", cfg) is True
assert is_valid_roman("नमस्ते", cfg) is False
def test_is_valid_target_script():
cfg = ValidationConfig()
assert is_valid_target_script("नमस्ते", "hi", cfg) is True
assert is_valid_target_script("வணக்கம்", "hi", cfg) is False
assert is_valid_target_script("வணக்கம்", "ta", cfg) is True