Upload 2 files

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+fonts/Menaion.otf filter=lfs diff=lfs merge=lfs -text

app.py

@@ -0,0 +1,561 @@
+import html
+import os
+from functools import lru_cache
+from pathlib import Path
+
+import gradio as gr
+import stanza
+from huggingface_hub import snapshot_download
+
+
+# ---------------------------------------------------------------------
+# Model configuration
+# ---------------------------------------------------------------------
+
+MODEL_REPO_ID = "usmannawaz/old-church-slavonic-tokenizer-lemmatizer"
+MODEL_VARIANT = "combined"
+
+LANG = "cu"
+POS_PACKAGE = "proiel_nocharlm"
+TOKENIZER_PACKAGE = "proiel"
+LEMMATIZER_PACKAGE = "proiel_nocharlm"
+
+HF_LOCAL_DIR = "hf_models/old-church-slavonic-tokenizer-lemmatizer"
+STANZA_DIR = os.path.join(os.path.expanduser("~"), "stanza_resources")
+
+os.environ["STANZA_RESOURCES_DIR"] = STANZA_DIR
+
+EXAMPLE_TEXT = "тѣма пророчьскꙑи ликъ д͞ховьнꙑѧ благодѣти исплъни сѧ"
+
+
+# ---------------------------------------------------------------------
+# Legacy/private-use characters -> Unicode OCS
+#
+# Duplicate keys from the old app were removed. The values below preserve
+# the replacements that Python actually used previously: the final value
+# of each duplicate key.
+# ---------------------------------------------------------------------
+
+REPLACEMENTS = {
+    "": "и",
+    "": "ч",
+    "": "ѥ",
+    "": "н",
+    "": ":~",
+    "": "~",
+    "": "̅",
+    "": "҆̀",
+    "": "ⷮ",
+    "": "ᲈ",
+    "": "Ч",
+    "": "ꙋ",
+    "": "чᲈ",
+    "": "҅́",
+    "": "꙾",
+    "": "ⷮ",
+    "": "҆́",
+    "ⷭⷭ": "҇",
+    "": "҆́",
+    "": "н҇",
+    "": "ꙶ",
+    "": "оу",
+    "": "ꙁ",
+    "": "ꙑ",
+    "": "ⱑ",
+    "": "с",
+    "": "Ѱ",
+    "": "҇",
+    "": "ⱉ",
+    "": "͡",
+    "": "҃",
+    "ⷣ": "ⷣ҇",
+    "ⷮ": "ⷮ҇",
+    "ⷯ": "ⷯ҇",
+    "": "ꙩ́",
+    "ⷤ҆": "ⷤ҇",
+    "": "",
+    "": "҃",
+    "": "꙯",
+    "": "҃",
+}
+
+
+# ---------------------------------------------------------------------
+# Font setup
+#
+# Add your licensed font file as:
+#     fonts/Menaion.otf
+#
+# The app still starts without it, but the browser uses fallback fonts.
+# ---------------------------------------------------------------------
+
+FONT_DIR = Path("fonts")
+FONT_FILE = FONT_DIR / "Menaion.otf"
+
+if FONT_DIR.exists():
+    gr.set_static_paths(paths=[FONT_DIR])
+
+FONT_FACE_CSS = """
+@font-face {
+    font-family: "Menaion";
+    src: url("/gradio_api/file=fonts/Menaion.otf") format("opentype");
+    font-style: normal;
+    font-weight: normal;
+    font-display: swap;
+}
+"""
+
+
+CUSTOM_CSS = FONT_FACE_CSS + """
+body {
+    background: linear-gradient(
+        135deg,
+        #f8fafc 0%,
+        #eef6ff 50%,
+        #f3f7fb 100%
+    ) !important;
+}
+
+.gradio-container {
+    max-width: 1050px !important;
+    margin: auto !important;
+    font-family: Inter, Arial, sans-serif !important;
+}
+
+#hero {
+    background: linear-gradient(135deg, #0f172a, #0f5f8f);
+    border-radius: 28px;
+    padding: 34px;
+    margin-bottom: 22px;
+    box-shadow: 0 22px 60px rgba(15, 23, 42, 0.22);
+}
+
+#hero h1 {
+    color: #ffffff;
+    font-size: 2.4rem;
+    margin: 0 0 10px 0;
+    font-weight: 900;
+}
+
+#hero p {
+    color: #e0f2fe;
+    font-size: 1.05rem;
+    line-height: 1.55;
+    margin: 0;
+}
+
+#main-card,
+#output-card {
+    background: #ffffff;
+    border: 1px solid #cbd5e1;
+    border-radius: 24px;
+    padding: 24px;
+    box-shadow: 0 18px 45px rgba(15, 23, 42, 0.10);
+}
+
+#output-card {
+    margin-top: 22px;
+}
+
+textarea,
+input {
+    border-radius: 16px !important;
+    border: 1.5px solid #94a3b8 !important;
+    background: #ffffff !important;
+    color: #0f172a !important;
+}
+
+textarea:focus,
+input:focus {
+    border-color: #0f5f8f !important;
+    box-shadow: 0 0 0 3px rgba(14, 116, 144, 0.18) !important;
+}
+
+label {
+    color: #0f172a !important;
+    font-weight: 850 !important;
+}
+
+button {
+    border-radius: 16px !important;
+    font-weight: 900 !important;
+    box-shadow: 0 10px 24px rgba(15, 23, 42, 0.14) !important;
+}
+
+button.primary {
+    background: linear-gradient(90deg, #0f172a, #0f5f8f) !important;
+    color: #ffffff !important;
+    border: none !important;
+}
+
+button.primary:hover {
+    background: linear-gradient(90deg, #020617, #075985) !important;
+    transform: translateY(-1px);
+}
+
+/*
+The input uses Menaion only for visual rendering.
+The underlying string is unchanged until preprocess_text() runs.
+*/
+#ocs-input textarea {
+    font-family: "Menaion", "Noto Sans Cyrillic", serif !important;
+    font-size: 23px !important;
+    font-weight: normal !important;
+    line-height: 1.75 !important;
+}
+
+/* All final Unicode OCS output is rendered with Menaion. */
+.ocs-font {
+    font-family: "Menaion", "Noto Sans Cyrillic", serif !important;
+    font-weight: normal !important;
+}
+
+#sentence-box {
+    background: #f8fafc;
+    border: 1px solid #cbd5e1;
+    border-left: 6px solid #0f5f8f;
+    border-radius: 18px;
+    padding: 16px;
+    color: #0f172a;
+    margin-bottom: 20px;
+}
+
+#sentence-box .label {
+    font-family: Inter, Arial, sans-serif;
+    font-size: 1rem;
+    font-weight: 850;
+}
+
+#sentence-text {
+    color: #0f172a;
+    font-size: 25px;
+    line-height: 1.85;
+    margin-top: 10px;
+    overflow-wrap: anywhere;
+}
+
+#table-title {
+    color: #0f5f8f;
+    font-family: Inter, Arial, sans-serif;
+    font-weight: 900;
+    font-size: 1.05rem;
+    margin: 6px 0 12px 0;
+}
+
+.ocs-table-wrap {
+    border: 1px solid #cbd5e1;
+    border-radius: 16px;
+    overflow: auto;
+}
+
+.ocs-table {
+    width: 100%;
+    border-collapse: collapse;
+    table-layout: fixed;
+}
+
+.ocs-table th {
+    background: #eef6ff;
+    color: #0f172a;
+    font-family: Inter, Arial, sans-serif;
+    font-size: 0.95rem;
+    text-align: left;
+    padding: 12px;
+    border-bottom: 1px solid #cbd5e1;
+}
+
+.ocs-table td {
+    color: #0f172a;
+    font-size: 22px;
+    line-height: 1.6;
+    padding: 11px 12px;
+    border-bottom: 1px solid #e2e8f0;
+    vertical-align: top;
+    overflow-wrap: anywhere;
+}
+
+.ocs-table tr:last-child td {
+    border-bottom: none;
+}
+
+.ocs-table tbody tr:nth-child(even) {
+    background: #f8fafc;
+}
+
+.status-box {
+    border-radius: 16px;
+    padding: 14px 16px;
+    background: #fff7ed;
+    border: 1px solid #fdba74;
+    color: #7c2d12;
+    font-family: Inter, Arial, sans-serif;
+}
+
+footer,
+.api,
+.settings,
+a[href*="gradio.app"],
+button[aria-label="Settings"] {
+    display: none !important;
+}
+"""
+
+
+HERO_HTML = """
+<div id="hero">
+    <h1>OCS Combined Lemmatizer</h1>
+    <p>
+        Pasted text is mapped from legacy/private-use characters to Unicode,
+        processed by Stanza, and rendered with the Menaion font.
+    </p>
+</div>
+"""
+
+
+# ---------------------------------------------------------------------
+# NLP loading
+# ---------------------------------------------------------------------
+
+@lru_cache(maxsize=1)
+def get_model_paths():
+    repo_dir = snapshot_download(
+        repo_id=MODEL_REPO_ID,
+        local_dir=HF_LOCAL_DIR,
+    )
+
+    model_dir = Path(repo_dir)
+    tokenizer_model = (
+        model_dir
+        / "models"
+        / MODEL_VARIANT
+        / "tokenize"
+        / "cu_proiel_tokenizer.pt"
+    )
+    lemma_model = (
+        model_dir
+        / "models"
+        / MODEL_VARIANT
+        / "lemma"
+        / "cu_proiel_nocharlm_lemmatizer.pt"
+    )
+
+    if not tokenizer_model.exists():
+        raise FileNotFoundError(f"Tokenizer model not found: {tokenizer_model}")
+
+    if not lemma_model.exists():
+        raise FileNotFoundError(f"Lemmatizer model not found: {lemma_model}")
+
+    return tokenizer_model, lemma_model
+
+
+@lru_cache(maxsize=1)
+def load_nlp_pipeline():
+    tokenizer_model, lemma_model = get_model_paths()
+
+    # Download only the Stanza resources not supplied by your model repo.
+    stanza.download(
+        lang=LANG,
+        model_dir=STANZA_DIR,
+        processors={"pos": POS_PACKAGE},
+        package=None,
+        verbose=False,
+    )
+
+    return stanza.Pipeline(
+        lang=LANG,
+        dir=STANZA_DIR,
+        package=None,
+        processors={
+            "tokenize": TOKENIZER_PACKAGE,
+            "pos": POS_PACKAGE,
+            "lemma": LEMMATIZER_PACKAGE,
+        },
+        tokenize_model_path=str(tokenizer_model),
+        lemma_model_path=str(lemma_model),
+        tokenize_pretokenized=False,
+        use_gpu=False,
+        verbose=False,
+    )
+
+
+# ---------------------------------------------------------------------
+# Text processing
+# ---------------------------------------------------------------------
+
+def clean_text(text):
+    return str(text or "").strip()
+
+
+def apply_mapping(text, mapping):
+    # Longer sequences must be replaced before their individual characters.
+    ordered = sorted(mapping.items(), key=lambda item: len(item[0]), reverse=True)
+
+    for old_text, new_text in ordered:
+        text = text.replace(old_text, new_text)
+
+    return text
+
+
+def preprocess_text(text):
+    """
+    Pipeline stage:
+      raw pasted text -> Unicode OCS
+
+    Do not apply a font here. Fonts belong only to browser rendering.
+    """
+    return apply_mapping(text, REPLACEMENTS)
+
+
+def make_sentence_html(text):
+    return f"""
+    <div id="sentence-box">
+        <div class="label">Processed Unicode OCS input</div>
+        <div id="sentence-text" class="ocs-font">{html.escape(text)}</div>
+    </div>
+    """
+
+
+def make_results_html(rows):
+    table_rows = "".join(
+        f"""
+        <tr>
+            <td class="ocs-font">{html.escape(token)}</td>
+            <td class="ocs-font">{html.escape(lemma)}</td>
+        </tr>
+        """
+        for token, lemma in rows
+    )
+
+    return f"""
+    <div id="table-title">Token and lemma output</div>
+    <div class="ocs-table-wrap">
+        <table class="ocs-table">
+            <thead>
+                <tr>
+                    <th>Token</th>
+                    <th>Lemma</th>
+                </tr>
+            </thead>
+            <tbody>{table_rows}</tbody>
+        </table>
+    </div>
+    """
+
+
+def analyze_text(text):
+    raw_text = clean_text(text)
+
+    if not raw_text:
+        return (
+            gr.update(visible=True),
+            """
+            <div class="status-box">
+                Please paste a sentence or raw OCS text first.
+            </div>
+            """,
+            "",
+        )
+
+    try:
+        # 1. PUA/legacy mapping -> standard Unicode OCS.
+        processed_text = preprocess_text(raw_text)
+
+        # 2. Stanza receives only the processed Unicode string.
+        nlp = load_nlp_pipeline()
+        doc = nlp(processed_text)
+
+        # 3. Model results remain Unicode strings.
+        rows = [
+            (word.text or "_", word.lemma or "_")
+            for sentence in doc.sentences
+            for word in sentence.words
+        ]
+
+        # 4. Menaion is applied only by CSS during browser visualization.
+        return (
+            gr.update(visible=True),
+            make_sentence_html(processed_text),
+            make_results_html(rows),
+        )
+
+    except Exception as error:
+        return (
+            gr.update(visible=True),
+            f"""
+            <div class="status-box">
+                <strong>Error:</strong> {html.escape(str(error))}
+            </div>
+            """,
+            "",
+        )
+
+
+def clear_all():
+    return "", gr.update(visible=False), "", ""
+
+
+def load_example():
+    return EXAMPLE_TEXT
+
+
+# ---------------------------------------------------------------------
+# Gradio interface
+# ---------------------------------------------------------------------
+
+with gr.Blocks(
+    title="OCS Combined Lemmatizer",
+    css=CUSTOM_CSS,
+    theme=gr.themes.Soft(
+        primary_hue="cyan",
+        secondary_hue="blue",
+        neutral_hue="slate",
+    ),
+) as demo:
+    gr.HTML(HERO_HTML)
+
+    with gr.Column(elem_id="main-card"):
+        text_input = gr.Textbox(
+            label="Input text",
+            lines=8,
+            placeholder="Paste Old Church Slavonic text here...",
+            elem_id="ocs-input",
+        )
+
+        with gr.Row():
+            analyze_button = gr.Button("Lemmatize", variant="primary")
+            example_button = gr.Button("Load example")
+            clear_button = gr.Button("Clear")
+
+    with gr.Column(elem_id="output-card", visible=False) as output_panel:
+        sentence_output = gr.HTML()
+        results_output = gr.HTML()
+
+    analyze_button.click(
+        fn=analyze_text,
+        inputs=text_input,
+        outputs=[output_panel, sentence_output, results_output],
+        show_progress="hidden",
+    )
+
+    example_button.click(
+        fn=load_example,
+        inputs=None,
+        outputs=text_input,
+        show_progress="hidden",
+    )
+
+    clear_button.click(
+        fn=clear_all,
+        inputs=None,
+        outputs=[
+            text_input,
+            output_panel,
+            sentence_output,
+            results_output,
+        ],
+        show_progress="hidden",
+    )
+
+
+if __name__ == "__main__":
+    demo.launch()

fonts/Menaion.otf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:68792e712d2af4bf333a23f24c0b59f3efa3499a58c31f082ecc8a4235cb6fa8
+size 169188