"""
Universal Dependencies Parser & Visualizer
Gradio Web Application following a calm, pedagogical design.
Tabs:
Tabs are ordered About, Parse, Compare, Annotate & Export, Treebank Guide; About opens by default.
Annotate & Export also defines the public API endpoints (see API_MD).
"""
import html
import inspect
import json
import os
import tempfile
from typing import List, Optional
import gradio as gr
import pandas as pd
ENABLE_SCREENCAST = os.getenv("ENABLE_SCREENCAST", "").strip().lower() in {"1", "true", "yes", "on"}
ScreenRecorder = None
if ENABLE_SCREENCAST:
try:
from gradio_screenrecorder import ScreenRecorder
except ImportError:
ScreenRecorder = None
ENABLE_SCREENCAST = False
from core.udpipe_client import UDPipeClient
from core.conllu_utils import conllu_to_table, compare_trees, extract_tree_signature, BoundedParseCache
from core.treebanks_data import TREEBANK_CATALOG, AVAILABLE_MODELS
from core.annotate import annotate_text, model_ids
from core.tree_export import TreeExportError, export_tree_images
from visualizer.template import render_conllu_visualizer_html, render_side_by_side_visualizers_html
# Initialize client
client = UDPipeClient()
# Load curated examples
EXAMPLES_FILE = os.path.join(os.path.dirname(__file__), "examples.json")
with open(EXAMPLES_FILE, "r", encoding="utf-8") as f:
CURATED_EXAMPLES = json.load(f)
# Stylesheet for calm academic styling with Source Sans Pro
CUSTOM_HEAD = """Please enter a sentence to parse.
",
pd.DataFrame(),
"",
"",
gr.update(visible=False),
pd.DataFrame(),
)
# Restrict strictly to one sentence (first line / sentence split)
if "\n" in sentence:
sentence = sentence.split("\n")[0].strip()
models_dict = AVAILABLE_MODELS.get(lang, AVAILABLE_MODELS["English"])
model_id_a = models_dict.get(primary_model_name, list(models_dict.values())[0])
strip_morphology = not include_morphology
res_a = client.parse_sentence(sentence, model_id_a, strip_morphology=strip_morphology, cache=cache)
error_a = res_a.get("error")
if not error_a and not extract_tree_signature(res_a.get("conllu", "")):
error_a = "The primary model returned no usable parse tokens."
if error_a:
err_msg = f"Error: {html.escape(str(error_a))}
"
return err_msg, pd.DataFrame(), "", "", gr.update(visible=False), pd.DataFrame()
conllu_a = res_a["conllu"]
table_a = conllu_to_table(conllu_a)
if not compare_mode:
# Single Tree Mode
vis_html = render_conllu_visualizer_html(conllu_a, vis_id="vis-primary", title=primary_model_name, show_help=True, model_id=model_id_a)
status_html = ""
return (
vis_html,
table_a,
conllu_a,
status_html,
gr.update(visible=False),
pd.DataFrame(),
)
elif primary_model_name == secondary_model_name:
# Same model selected twice in comparison mode
vis_html = render_conllu_visualizer_html(conllu_a, vis_id="vis-primary", title=primary_model_name, model_id=model_id_a)
m_name_esc = html.escape(primary_model_name)
status_html = (
f"✓ Same UDPipe Parser Selected: "
f"Both parsers are set to {m_name_esc} ({len(table_a)} tokens). "
f"Select a different second parser to inspect differences.
"
)
return (
vis_html,
table_a,
conllu_a,
status_html,
gr.update(visible=False),
pd.DataFrame(),
)
else:
# Comparison Mode: Query Secondary Model
model_id_b = models_dict.get(secondary_model_name, list(models_dict.values())[0])
res_b = client.parse_sentence(sentence, model_id_b, strip_morphology=strip_morphology, cache=cache)
conllu_b = res_b.get("conllu", "")
error_b = res_b.get("error")
if not error_b and not extract_tree_signature(conllu_b):
error_b = "The secondary model returned no usable parse tokens."
if error_b:
vis_html = render_conllu_visualizer_html(
conllu_a, vis_id="vis-primary", title=primary_model_name, model_id=model_id_a
)
status_html = (
"Comparison unavailable: "
f"{html.escape(secondary_model_name)}: {html.escape(str(error_b))}
"
)
return vis_html, table_a, conllu_a, status_html, gr.update(visible=False), pd.DataFrame()
# Diff trees
diff_info = compare_trees(conllu_a, primary_model_name, conllu_b, secondary_model_name)
vis_html = render_side_by_side_visualizers_html(
conllu_a, primary_model_name,
conllu_b, secondary_model_name,
model_id_a=model_id_a, model_id_b=model_id_b
)
raw_conllu_combined = f"# === {primary_model_name} ===\n{conllu_a}\n\n# === {secondary_model_name} ===\n{conllu_b}"
if diff_info["identical"]:
status_html = f"✓ Identical Trees: Both UDPipe parsers produced matching POS tags and syntactic dependency arcs.
"
diff_visible = False
else:
diff_text = html.escape(diff_info["diff_summary"])
model_a_escaped = html.escape(primary_model_name)
model_b_escaped = html.escape(secondary_model_name)
diff_text = diff_text.replace(f"**{model_a_escaped}**", f"{model_a_escaped} ").replace(f"**{model_b_escaped}**", f"{model_b_escaped} ")
status_html = f"⚡ Discrepancies Found: {diff_text}
"
diff_visible = True
return (
vis_html,
table_a,
raw_conllu_combined,
status_html,
gr.update(visible=diff_visible),
diff_info["diff_table"],
)
def _describe_example(example):
description = example["phenomenon"]
if example.get("source_url"):
description += (
f"\n\nSource: [{example['source_corpus']} {example['source_version']}, "
f"sentence {example['sent_id']}]({example['source_url']}) "
f"· [{example['source_license']}](https://creativecommons.org/licenses/by-sa/3.0/)"
)
return description
def _get_example_choices(lang):
return [(ex["text"], i) for i, ex in enumerate(CURATED_EXAMPLES) if ex["language"] == lang]
def _get_first_example_index_for_lang(lang: str) -> int:
for i, ex in enumerate(CURATED_EXAMPLES):
if ex["language"] == lang:
return i
return 0
def resolve_model_label(lang: str, query: str) -> Optional[str]:
"""Resolves a model query (ID, label, or substring) to a display label."""
if not query:
return None
models = list(AVAILABLE_MODELS.get(lang, {}).keys())
for m in models:
if m.lower() == query.lower():
return m
for m in models:
mid = AVAILABLE_MODELS[lang][m]
if mid.lower() == query.lower():
return m
for m in models:
mid = AVAILABLE_MODELS[lang][m]
if query.lower() in mid.lower() or query.lower() in m.lower():
return m
return None
def build_parser(compare=False):
"""Build independent, sentence-first controls for each parsing mode."""
sentence = gr.Textbox(label="Sentence", value="They buy and sell books.",
placeholder="Enter an English or German sentence...",
lines=1, max_lines=1, elem_classes=["sentence-input"])
with gr.Row():
language = gr.Dropdown(label="Language", choices=["English", "German"], value="English", scale=1)
primary = gr.Dropdown(label="UDPipe parser" if not compare else "First UDPipe parser",
choices=list(AVAILABLE_MODELS["English"]),
value=list(AVAILABLE_MODELS["English"])[0], scale=2)
secondary = gr.Dropdown(label="Second UDPipe parser", choices=list(AVAILABLE_MODELS["English"]),
value=list(AVAILABLE_MODELS["English"])[1], visible=compare, scale=2)
with gr.Column(scale=1):
with gr.Row():
submit = gr.Button("Compare" if compare else "Parse", variant="primary", scale=3)
share_btn = gr.Button("🔗 Share", size="sm", scale=2, min_width=75)
include_morphology = gr.Checkbox(label="Include morphology (FEATS)", value=False)
with gr.Accordion("Examples", open=False):
selected_example = gr.State(0)
with gr.Row():
german_examples = gr.Dropdown(label="German Examples",
choices=_get_example_choices("German"),
value=None, scale=1, min_width=240, allow_custom_value=False)
english_examples = gr.Dropdown(label="English Examples",
choices=_get_example_choices("English"),
value=0, scale=1, min_width=240, allow_custom_value=False)
phenomenon = gr.Markdown(_describe_example(CURATED_EXAMPLES[0]))
status = gr.HTML(visible=compare)
visualization = gr.HTML()
with gr.Group(visible=False) as differences:
gr.Markdown("### Differences")
diff_table = gr.DataFrame(interactive=False)
with gr.Accordion("Annotations", open=False):
table = gr.DataFrame(headers=["ID", "Form", "Lemma", "UPOS", "FEATS", "Head", "Deprel"], interactive=False)
with gr.Accordion("CoNLL-U", open=False):
raw = gr.Code(label="CoNLL-U", language="markdown", interactive=False)
def on_language_input(new_lang):
models = list(AVAILABLE_MODELS[new_lang])
return (
gr.update(choices=models, value=models[0]),
gr.update(choices=models, value=models[min(1, len(models)-1)]),
)
def on_sentence_change():
return (
None,
gr.update(value=None),
gr.update(value=None),
"",
)
def on_example_input(new_ex_idx, current_lang, current_primary, current_secondary, morph, cache):
if new_ex_idx is None:
return tuple(gr.update() for _ in range(14))
ex = CURATED_EXAMPLES[int(new_ex_idx)]
ex_lang = ex["language"]
if ex_lang != current_lang:
models = list(AVAILABLE_MODELS[ex_lang])
primary_name = models[0]
secondary_name = models[min(1, len(models)-1)]
model_updates = (
gr.update(choices=models, value=primary_name),
gr.update(choices=models, value=secondary_name),
)
else:
primary_name = current_primary
secondary_name = current_secondary
model_updates = (gr.update(), gr.update())
vis_html, table_df, raw_conllu, status_html, diff_group, diff_df = run_single_or_compare(
lang=ex_lang,
sentence=ex["text"],
primary_model_name=primary_name,
compare_mode=compare,
secondary_model_name=secondary_name,
include_morphology=morph,
cache=cache,
)
return (
ex_lang,
*model_updates,
_describe_example(ex),
ex["text"],
int(new_ex_idx),
gr.update(value=int(new_ex_idx) if ex_lang == "German" else None),
gr.update(value=int(new_ex_idx) if ex_lang == "English" else None),
vis_html,
table_df,
raw_conllu,
status_html,
diff_group,
diff_df,
)
language.input(
on_language_input,
[language],
[primary, secondary],
api_name=False,
)
sentence.input(
on_sentence_change,
None,
[selected_example, german_examples, english_examples, phenomenon],
api_name=False,
)
# An instance, not the class: Gradio deep-copies it per session, and a
# callable default would register a public load endpoint.
# An instance, not the class: Gradio deep-copies State per session, and a
# callable default would publish a load endpoint in the API.
session_cache = gr.State(BoundedParseCache())
for example_dropdown in (german_examples, english_examples):
example_dropdown.input(
on_example_input,
[example_dropdown, language, primary, secondary, include_morphology, session_cache],
[language, primary, secondary, phenomenon, sentence, selected_example,
german_examples, english_examples, visualization, table, raw, status, differences, diff_table],
api_name=False,
)
inputs = [language, sentence, primary, gr.State(compare), secondary, include_morphology, session_cache]
outputs = [visualization, table, raw, status, differences, diff_table]
submit.click(run_single_or_compare, inputs, outputs, api_name=False)
sentence.submit(run_single_or_compare, inputs, outputs, api_name=False)
share_btn.click(
fn=None,
inputs=[],
outputs=[],
js="""() => {
if (window.updateShareUrl) window.updateShareUrl();
const url = window.location.href;
navigator.clipboard.writeText(url).then(() => {
const btn = document.activeElement;
if (btn) {
const oldText = btn.textContent;
btn.textContent = '✓ Copied!';
setTimeout(() => { btn.textContent = oldText; }, 2000);
}
}).catch(() => {
prompt('Copy shareable link:', url);
});
return [];
}""",
api_name=False,
)
return {
"sentence": sentence,
"language": language,
"primary": primary,
"secondary": secondary,
"include_morphology": include_morphology,
"session_cache": session_cache,
"visualization": visualization,
"table": table,
"raw": raw,
"status": status,
"differences": differences,
"diff_table": diff_table,
}
EXPORT_FORMATS = ["CoNLL-U", "SVG", "PNG"]
API_MD = """
Each button below is also an API endpoint. From Python:
```python
from gradio_client import Client
client = Client("simon-clmtd/universal-dependencies-visualizer")
conllu, files = client.predict(
"Bestimmt hat sie die Lösung bestimmt.", "german-hdt-ud-2.17-251125",
False, ["CoNLL-U", "SVG", "PNG"], api_name="/annotate")
files = client.predict(conllu, ["SVG", "PNG"], api_name="/render")
client.predict(api_name="/models")
```
- `/annotate(text, model, include_morphology, formats)`: parses each non-empty line
(at most 50) and returns the CoNLL-U text and the requested files.
- `/render(conllu, formats)`: draws your own or corrected CoNLL-U as SVG and PNG,
exactly like the tree download buttons.
- `/models()`: lists the model IDs.
Images take a few seconds per request because a browser draws each tree.
"""
def _write_exports(conllu: str, formats: List[str], stem: str = "ud-analysis") -> List[str]:
"""Writes the requested CoNLL-U/SVG/PNG files to a fresh temporary directory."""
folder = tempfile.mkdtemp(prefix="ud-export-")
paths = []
if "CoNLL-U" in formats:
path = os.path.join(folder, stem + ".conllu")
with open(path, "w", encoding="utf-8") as handle:
handle.write(conllu)
paths.append(path)
image_formats = [name for name in ("SVG", "PNG") if name in formats]
if image_formats:
try:
images = export_tree_images(conllu)
except TreeExportError as error:
raise gr.Error(str(error))
for name in image_formats:
path = os.path.join(folder, f"{stem}.{name.lower()}")
with open(path, "wb") as handle:
handle.write(images[name.lower()])
paths.append(path)
return paths
def annotate_and_export(text: str, model: str, include_morphology: bool, formats: List[str]):
"""Parses each line of text with one UDPipe model and exports the analysis."""
try:
result = annotate_text(text, model, include_morphology=include_morphology, client=client)
except (ValueError, RuntimeError) as error:
raise gr.Error(str(error))
return result["conllu"], _write_exports(result["conllu"], formats or [])
def render_conllu(conllu: str, formats: List[str]):
"""Draws supplied CoNLL-U as SVG/PNG (and returns it as a file if requested)."""
if not conllu or not conllu.strip():
raise gr.Error("Paste CoNLL-U to render.")
return _write_exports(conllu if conllu.endswith("\n") else conllu + "\n", formats or [])
def list_models() -> List[str]:
"""Returns the selectable UDPipe model IDs."""
return model_ids()
def build_exporter():
"""Batch annotation and file export; its events are the public API."""
gr.Markdown(
"Annotate one sentence per line, or paste CoNLL-U (for example a corrected "
"analysis) and render it. Download the CoNLL-U, SVG, and PNG files below."
)
with gr.Row():
with gr.Column():
text = gr.Textbox(label="Sentences (one per line)", lines=6,
value="Bestimmt hat sie die Lösung bestimmt.")
with gr.Row():
model = gr.Dropdown(label="UDPipe model", choices=model_ids(),
value="german-hdt-ud-2.17-251125", scale=2)
include_morphology = gr.Checkbox(label="Include morphology (FEATS)", value=False, scale=1)
formats = gr.CheckboxGroup(EXPORT_FORMATS, value=EXPORT_FORMATS, label="Files")
annotate = gr.Button("Annotate", variant="primary")
with gr.Column():
conllu = gr.Code(label="CoNLL-U (editable)", language="markdown", interactive=True, lines=12)
render_formats = gr.CheckboxGroup(EXPORT_FORMATS, value=["SVG", "PNG"], label="Files")
render = gr.Button("Render CoNLL-U")
files = gr.File(label="Downloads", file_count="multiple", interactive=False)
with gr.Accordion("Use as an API", open=False):
gr.Markdown(API_MD)
annotate.click(annotate_and_export, [text, model, include_morphology, formats], [conllu, files],
api_name="annotate", concurrency_limit=2)
render.click(render_conllu, [conllu, render_formats], [files],
api_name="render", concurrency_limit=2)
gr.api(list_models, api_name="models")
_blocks_init_params = inspect.signature(gr.Blocks.__init__).parameters
_blocks_kwargs = {"title": "Universal Dependencies Visualizer"}
_launch_kwargs = {}
if "css" in _blocks_init_params:
_blocks_kwargs.update(theme=gr.themes.Base(), css=CUSTOM_CSS, head=CUSTOM_HEAD)
else:
_launch_kwargs.update(theme=gr.themes.Base(), css=CUSTOM_CSS, head=CUSTOM_HEAD)
with gr.Blocks(**_blocks_kwargs) as demo:
if ENABLE_SCREENCAST and ScreenRecorder is not None:
with gr.Accordion("Record screencast (experimental)", open=False, elem_id="screencast-panel"):
gr.Markdown(
"Record your exploration with microphone narration. Choose this browser "
"tab or a window in the screen-sharing dialog, then explore Parse or Compare. "
"Stop recording, then click Download WebM. The video is saved directly "
"from your browser."
)
screen_recorder = ScreenRecorder(
label="Screencast", audio_enabled=True, webcam_overlay=False,
recording_format="webm", max_duration=600,
)
download_recording = gr.Button("Download WebM", size="sm")
# 0.0.1 does not bridge completion events to Gradio or upload FileData.
# Save its browser-local preview instead of relying on that broken path.
download_recording.click(
fn=None, inputs=[], outputs=[], api_name=False, js="""() => {
const video = Array.from(document.querySelectorAll('#screencast-panel .recording-preview video'))
.find(video => video.src.startsWith('blob:'));
if (!video) {
alert('Stop a recording before downloading it.');
return [];
}
const link = document.createElement('a');
link.href = video.src;
link.download = 'ud-exploration.webm';
document.body.appendChild(link);
link.click();
link.remove();
return [];
}"""
)
with gr.Tabs(selected="tab_overview") as tabs:
with gr.Tab("About", id="tab_overview"):
gr.HTML(OVERVIEW_MD)
with gr.Tab("Parse", id="tab_parser"):
parse_ctrls = build_parser()
with gr.Tab("Compare", id="tab_compare"):
compare_ctrls = build_parser(compare=True)
with gr.Tab("Annotate & Export", id="tab_export"):
build_exporter()
with gr.Tab("Treebank Guide", id="tab_guide"):
with gr.Accordion("Training size, quality and conformance", open=False):
# Remove duplicate heading since accordion provides the label
doc_lines = CONFORMANCE_DOC_MD.strip().splitlines()
if doc_lines and doc_lines[0].startswith("### "):
conformance_content = "\n".join(doc_lines[1:]).strip()
else:
conformance_content = CONFORMANCE_DOC_MD
gr.Markdown(conformance_content)
with gr.Accordion("Explore corpora with Grew", open=False):
gr.Markdown(
"Go beyond a single parse: search annotated corpus sentences, "
"look for recurring patterns, and find counterexamples to your expectations. "
"Use Grew's built-in dependency-relation search examples to get started. "
"Inspect several matches and explain what their trees have in common.\n\n"
"| Treebank on Grew | Try exploring |\n"
"| --- | --- |\n"
+ "\n".join(
f"| [{row['Language']}-{row['Treebank']}]({row['GrewURL']}) "
f"| {row['ExplorePrompt']} |"
for row in TREEBANK_CATALOG
)
)
with gr.Accordion("Full treebank catalog", open=False):
catalog_rows = [
[
row["Language"],
row["Treebank"],
row["Words"],
row["TrainingWords"],
row["Sentences"],
row["Genres"],
row["BestFor"],
]
for row in TREEBANK_CATALOG
]
catalog_df = gr.DataFrame(
value=catalog_rows,
headers=["Language", "Treebank", "Total words (UD 2.17)", "Training words", "Sentences", "Text sources", "Role in this app"],
interactive=False,
)
def load_from_request(request: gr.Request):
params = request.query_params if request else {}
if not params:
return tuple(gr.update() for _ in range(22))
tab_param = params.get("tab", "").strip().lower()
sentence_param = params.get("sentence", "").strip()
model_param = params.get("model", "").strip()
model2_param = params.get("model2", "").strip()
lang_param = params.get("lang", "").strip()
morph = params.get("morph", "").strip().lower() in {"1", "true", "yes"}
if tab_param in {"compare", "tab_compare"} or model2_param:
target_tab = "tab_compare"
elif tab_param in {"parse", "tab_parser"} or sentence_param:
target_tab = "tab_parser"
elif tab_param in {"export", "annotate", "tab_export"}:
target_tab = "tab_export"
elif tab_param in {"guide", "treebank", "tab_guide"}:
target_tab = "tab_guide"
elif tab_param in {"about", "overview", "tab_overview"}:
target_tab = "tab_overview"
else:
target_tab = "tab_overview"
if lang_param and lang_param.capitalize() in AVAILABLE_MODELS:
lang = lang_param.capitalize()
elif "german" in model_param.lower() or "german" in model2_param.lower():
lang = "German"
else:
lang = "English"
models_list = list(AVAILABLE_MODELS[lang].keys())
p_model = resolve_model_label(lang, model_param) or models_list[0]
p_model2 = resolve_model_label(lang, model2_param) or (models_list[1] if len(models_list) > 1 else models_list[0])
p_sent_up = sentence_param if sentence_param else gr.update()
p_lang_up = gr.update(choices=["English", "German"], value=lang)
p_model_up = gr.update(choices=models_list, value=p_model)
p_morph_up = gr.update(value=morph)
p_vis = gr.update()
p_table = gr.update()
p_raw = gr.update()
p_status = gr.update()
p_diff_group = gr.update()
p_diff_table = gr.update()
if target_tab == "tab_parser" and sentence_param:
v, t, r, s, dg, dt = run_single_or_compare(
lang=lang,
sentence=sentence_param,
primary_model_name=p_model,
compare_mode=False,
secondary_model_name="",
include_morphology=morph,
cache=parse_ctrls["session_cache"].value,
)
p_vis, p_table, p_raw, p_status, p_diff_group, p_diff_table = v, t, r, s, dg, dt
c_sent_up = sentence_param if sentence_param else gr.update()
c_lang_up = gr.update(choices=["English", "German"], value=lang)
c_model1_up = gr.update(choices=models_list, value=p_model)
c_model2_up = gr.update(choices=models_list, value=p_model2)
c_morph_up = gr.update(value=morph)
c_vis = gr.update()
c_table = gr.update()
c_raw = gr.update()
c_status = gr.update()
c_diff_group = gr.update()
c_diff_table = gr.update()
if target_tab == "tab_compare" and sentence_param:
v, t, r, s, dg, dt = run_single_or_compare(
lang=lang,
sentence=sentence_param,
primary_model_name=p_model,
compare_mode=True,
secondary_model_name=p_model2,
include_morphology=morph,
cache=compare_ctrls["session_cache"].value,
)
c_vis, c_table, c_raw, c_status, c_diff_group, c_diff_table = v, t, r, s, dg, dt
return (
gr.Tabs(selected=target_tab),
p_sent_up, p_lang_up, p_model_up, p_morph_up,
p_vis, p_table, p_raw, p_status, p_diff_group, p_diff_table,
c_sent_up, c_lang_up, c_model1_up, c_model2_up, c_morph_up,
c_vis, c_table, c_raw, c_status, c_diff_group, c_diff_table,
)
load_outputs = [
tabs,
parse_ctrls["sentence"], parse_ctrls["language"], parse_ctrls["primary"], parse_ctrls["include_morphology"],
parse_ctrls["visualization"], parse_ctrls["table"], parse_ctrls["raw"], parse_ctrls["status"],
parse_ctrls["differences"], parse_ctrls["diff_table"],
compare_ctrls["sentence"], compare_ctrls["language"], compare_ctrls["primary"], compare_ctrls["secondary"],
compare_ctrls["include_morphology"],
compare_ctrls["visualization"], compare_ctrls["table"], compare_ctrls["raw"], compare_ctrls["status"],
compare_ctrls["differences"], compare_ctrls["diff_table"],
]
demo.load(load_from_request, inputs=None, outputs=load_outputs, api_name=False)
if __name__ == "__main__":
demo.launch(**_launch_kwargs)