File size: 2,544 Bytes
ad9385f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 | import unittest
from core.annotate import annotate_text, merge_conllu, resolve_model
PART = """# generator = UDPipe 2, https://lindat.mff.cuni.cz/services/udpipe
# udpipe_model = german-hdt-ud-2.17-251125
# newdoc
# newpar
# sent_id = 1
# text = {text}
1\t{text}\t_\tNOUN\t_\t_\t0\troot\t_\t_
"""
class FakeClient:
def __init__(self, error=None):
self.calls = []
self.error = error
def parse_sentence(self, text, model, strip_morphology=True, cache=None):
self.calls.append((text, model, strip_morphology))
if self.error:
return {"conllu": "", "model": model, "error": self.error}
return {"conllu": PART.format(text=text), "model": model, "error": None}
class AnnotateTests(unittest.TestCase):
def test_resolve_model_accepts_label_and_id(self):
self.assertEqual(resolve_model("UDPipe · german-gsd-ud-2.17-251125"), "german-gsd-ud-2.17-251125")
self.assertEqual(resolve_model("german-gsd-ud-2.17-251125"), "german-gsd-ud-2.17-251125")
with self.assertRaises(ValueError):
resolve_model("german")
def test_lines_are_parsed_separately_and_renumbered(self):
client = FakeClient()
result = annotate_text("Eins\n\n Zwei \n", "german-hdt-ud-2.17-251125", client=client)
self.assertEqual([call[0] for call in client.calls], ["Eins", "Zwei"])
self.assertTrue(client.calls[0][2])
conllu = result["conllu"]
self.assertEqual(result["sentences"], 2)
self.assertEqual(conllu.count("# generator"), 1)
self.assertNotIn("# newdoc", conllu)
self.assertIn("# sent_id = 1\n# text = Eins", conllu)
self.assertIn("# sent_id = 2\n# text = Zwei", conllu)
self.assertTrue(conllu.endswith("\n\n"))
def test_morphology_flag_is_passed(self):
client = FakeClient()
annotate_text("Eins", "german-hdt-ud-2.17-251125", include_morphology=True, client=client)
self.assertFalse(client.calls[0][2])
def test_one_line_may_yield_several_sentences(self):
merged = merge_conllu([PART.format(text="A") + PART.format(text="B")], "m")
self.assertIn("# sent_id = 2\n# text = B", merged)
def test_errors(self):
with self.assertRaises(ValueError):
annotate_text(" \n ", "german-hdt-ud-2.17-251125", client=FakeClient())
with self.assertRaises(RuntimeError):
annotate_text("Eins", "german-hdt-ud-2.17-251125", client=FakeClient(error="down"))
if __name__ == "__main__":
unittest.main()
|