"]
for start in range(
0,
len(sources),
batch_size,
):
batch_sources = sources[
start:start + batch_size
]
source_ids_list = []
source_lengths = []
for source_string in batch_sources:
source_ids = (
[sos_id]
+ lemmatizer.vocab.encode(
source_string
)
+ [eos_id]
)
source_ids_list.append(source_ids)
source_lengths.append(
len(source_ids)
)
maximum_length = max(source_lengths)
padded = [
ids
+ [pad_id]
* (maximum_length - len(ids))
for ids in source_ids_list
]
source_tensor = torch.tensor(
padded,
dtype=torch.long,
device=lemmatizer.device,
)
length_tensor = torch.tensor(
source_lengths,
dtype=torch.long,
device=lemmatizer.device,
)
batch_predictions = (
lemmatizer.model.generate(
source_tensor,
length_tensor,
lemmatizer.vocab,
max_len=lemmatizer.max_gen_len,
)
)
predictions.extend(batch_predictions)
return predictions
def write_back_conllu(input_text, predictions):
text = input_text.rstrip("\n")
blocks = re.split(r"\n\n+", text)
output_blocks = []
prediction_index = 0
for block in blocks:
new_lines = []
for line in block.split("\n"):
if not line or line.startswith("#"):
new_lines.append(line)
continue
columns = line.split("\t")
if len(columns) != 10:
new_lines.append(line)
continue
token_id = columns[0]
if "-" in token_id or "." in token_id:
new_lines.append(line)
continue
if prediction_index < len(predictions):
prediction = predictions[
prediction_index
]
else:
prediction = "_"
columns[TARGET_COL_IDX] = (
prediction if prediction else "_"
)
new_lines.append(
"\t".join(columns)
)
prediction_index += 1
output_blocks.append(
"\n".join(new_lines)
)
output_text = (
"\n\n".join(output_blocks).rstrip()
+ "\n\n"
)
return output_text, prediction_index
@spaces.GPU(duration=120)
def lemmatize_conllu_file(
file_obj,
selected_name,
):
if file_obj is None:
return (
gr.update(
value=None,
visible=False,
),
"Please upload a CoNLL-U file.",
)
try:
model_id = selected_model_id(
selected_name
)
input_path = Path(file_obj)
with input_path.open(
encoding="utf8"
) as file:
text = file.read()
lemmatizer = load_lemmatizer(
model_id=model_id,
device="cuda",
)
sources = make_all_sources_from_conllu(
text,
lemmatizer,
)
predictions = predict_sources_batched(
sources,
lemmatizer,
)
output_text, total = write_back_conllu(
text,
predictions,
)
safe_name = re.sub(
r"[^A-Za-z0-9_]+",
"",
selected_name.replace(
" ",
"_",
).replace(
"-",
"_",
),
)
output_path = (
Path(tempfile.gettempdir())
/ (
f"{input_path.stem}."
f"{safe_name}."
"lemmatized.conllu"
)
)
with output_path.open(
"w",
encoding="utf8",
newline="\n",
) as file:
file.write(output_text)
del lemmatizer
torch.cuda.empty_cache()
message = (
f"Done. Wrote {total:,} lemma predictions.\n"
"PUA replacements were applied before inference.\n"
"Input column: FORM.\n"
"Updated column: LEMMA.\n"
"Original FORM values and all other columns were preserved."
)
return (
gr.update(
value=str(output_path),
visible=True,
),
message,
)
except Exception as error:
torch.cuda.empty_cache()
return (
gr.update(
value=None,
visible=False,
),
f"Error: {error}",
)
def reset_download_button(file_obj):
return (
gr.update(
value=None,
visible=False,
),
"",
)
CUSTOM_CSS = """
body {
background: linear-gradient(
135deg,
#eaf3ff 0%,
#ffffff 48%,
#dbeafe 100%
);
}
.gradio-container {
max-width: 980px !important;
margin: auto !important;
font-family: Arial, Helvetica, sans-serif !important;
}
#main-card {
background: #ffffff;
border: 1px solid #bfdbfe;
border-radius: 26px;
padding: 30px;
box-shadow: 0 20px 50px rgba(15, 23, 42, 0.16);
}
#title {
text-align: center;
color: #020617;
font-size: 2.5rem;
font-weight: 900;
}
#subtitle {
text-align: center;
color: #1e40af;
font-size: 1.05rem;
line-height: 1.55;
}
#badge-row {
text-align: center;
margin-bottom: 1.2rem;
}
#badge-row span {
display: inline-block;
background: #eff6ff;
color: #1e3a8a;
border: 1px solid #bfdbfe;
border-radius: 999px;
padding: 7px 13px;
margin: 4px;
font-weight: 700;
}
button {
background: linear-gradient(
90deg,
#020617,
#1d4ed8
) !important;
color: #ffffff !important;
border-radius: 16px !important;
font-weight: 900 !important;
}
#token-card {
background: #f8fbff;
border: 1.5px solid #1d4ed8;
border-radius: 18px;
padding: 18px;
}
.lemma-table {
width: 100%;
border-collapse: collapse;
}
.lemma-table th {
background: #1d4ed8;
color: white;
padding: 12px;
text-align: left;
}
.lemma-table td {
padding: 12px;
border-bottom: 1px solid #bfdbfe;
}
footer {
display: none !important;
}
"""
DEFAULT_LANGUAGE = default_language()
DEFAULT_TREEBANK = default_treebank(
DEFAULT_LANGUAGE
)
APP_THEME = gr.themes.Soft(
primary_hue="blue",
secondary_hue="sky",
neutral_hue="slate",
)
with gr.Blocks(
title="OCS Combined Lemmatizer",
) as demo:
with gr.Column(elem_id="main-card"):
gr.Markdown(
"# Old Church Slavonic Lemmatizer Demo",
elem_id="title",
)
gr.Markdown(
"Paste a tokenized sentence or upload a "
"CoNLL-U file to generate lemma predictions. "
"PUA characters are normalized before tokenization.",
elem_id="subtitle",
)
gr.HTML(
"""
Old Church Slavonic
Combined Model
PUA Normalization
Context-aware
ZeroGPU
"""
)
with gr.Row():
language_input = gr.Dropdown(
label="Language",
choices=LANGUAGES,
value=DEFAULT_LANGUAGE,
)
treebank_input = gr.Dropdown(
label="Model",
choices=treebank_choices(
DEFAULT_LANGUAGE
),
value=DEFAULT_TREEBANK,
)
with gr.Tab("Sentence"):
sentence_input = gr.Textbox(
label="Input sentence",
lines=5,
placeholder=(
"Enter a whitespace-tokenized sentence"
),
)
sentence_button = gr.Button(
"Lemmatize sentence"
)
sentence_output = gr.Textbox(
label="Lemmatized sentence",
lines=5,
)
token_output = gr.HTML()
with gr.Tab("CoNLL-U file"):
conllu_input = gr.File(
label="Upload CoNLL-U file",
file_types=[".conllu", ".txt"],
type="filepath",
)
conllu_button = gr.Button(
"Lemmatize CoNLL-U file"
)
conllu_output = gr.DownloadButton(
label="Download result",
value=None,
visible=False,
)
conllu_message = gr.Textbox(
label="Status",
lines=5,
)
language_input.change(
fn=update_treebanks,
inputs=language_input,
outputs=treebank_input,
)
sentence_button.click(
fn=lemmatize_sentence,
inputs=[
sentence_input,
treebank_input,
],
outputs=[
sentence_output,
token_output,
],
)
conllu_input.change(
fn=reset_download_button,
inputs=conllu_input,
outputs=[
conllu_output,
conllu_message,
],
)
conllu_button.click(
fn=lemmatize_conllu_file,
inputs=[
conllu_input,
treebank_input,
],
outputs=[
conllu_output,
conllu_message,
],
)
demo.queue()
demo.launch(
server_name="0.0.0.0",
server_port=7860,
ssr_mode=False,
theme=APP_THEME,
css=CUSTOM_CSS,
footer_links=[],
)