File size: 2,492 Bytes
5952424 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 | """Minimal stdlib CoNLL-U reader/writer for the OGA kfold treebank.
Keeps only what the tagger needs (FORM/LEMMA/UPOS/XPOS per syntactic word) but
round-trips every other column and comment untouched, so predictions can be written
into a copy of the gold file and scored with the official conll18 script.
"""
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Iterator
@dataclass
class Token:
tid: str # "1", "2", ... (kept as string; MWT ranges and empty nodes never land here)
form: str
lemma: str
upos: str
xpos: str
feats: str
head: str
deprel: str
deps: str
misc: str
@dataclass
class Sentence:
tokens: list[Token] = field(default_factory=list)
# raw lines in original order, as (kind, payload): kind "comment" | "mwt" | "empty" -> raw
# line, kind "token" -> index into tokens. Preserves exact file structure on write.
lines: list[tuple[str, object]] = field(default_factory=list)
def read_conllu(path) -> Iterator[Sentence]:
sent = Sentence()
with open(path, encoding="utf-8") as f:
for line in f:
line = line.rstrip("\n")
if not line:
if sent.lines:
yield sent
sent = Sentence()
continue
if line.startswith("#"):
sent.lines.append(("comment", line))
continue
cols = line.split("\t")
tid = cols[0]
if "-" in tid:
sent.lines.append(("mwt", line))
elif "." in tid:
sent.lines.append(("empty", line))
else:
sent.lines.append(("token", len(sent.tokens)))
sent.tokens.append(Token(*cols[:10]))
if sent.lines:
yield sent
def write_conllu(sents, preds, path):
"""preds: list (per sentence) of lists of (lemma, upos, xpos) aligned with sent.tokens."""
with open(path, "w", encoding="utf-8") as f:
for sent, ps in zip(sents, preds):
for kind, payload in sent.lines:
if kind == "token":
t = sent.tokens[payload]
lemma, upos, xpos = ps[payload]
f.write("\t".join([t.tid, t.form, lemma, upos, xpos,
t.feats, t.head, t.deprel, t.deps, t.misc]) + "\n")
else:
f.write(payload + "\n")
f.write("\n")
|